From b25a5f2c32354ce23e23213f61da09a3fb5225a7 Mon Sep 17 00:00:00 2001 From: Brandon Hensley Date: Tue, 19 May 2026 21:47:44 -0500 Subject: [PATCH 1/3] Release v0.8.0 ## What's New - Add label-first graph APIs with automatic label catalog creation, multi-label nodes, and edge label IDs across Rust, Node.js, and Python. - Add direct edge query APIs, edge property indexes, edge query planning, and graph-pattern edge anchors. - Introduce packed core segments with required segment objects in segment.core and optional indexes kept outside the core payload. - Add segment component identity validation and public scrub APIs for detecting stale or mismatched segment sidecars. ## Improvements - Rename public type vocabulary to labels and remove numeric type-ID aliases from connector APIs. - Update WAL replay and recovery around atomic graph batches. - Expand planner stats, endpoint/property intersection planning, and edge-first pattern execution. ## Documentation - Document the pre-1.0 storage-format break for existing database directories. - Refresh API docs, getting started material, architecture overview, examples, and README content for v0.8.0. - Add a community integration link for OvergraphSwiftBridge. --- CHANGELOG.md | 64 + Cargo.lock | 78 +- Cargo.toml | 3 +- README.md | 113 +- benches/async_flush.rs | 56 +- benches/core_ops.rs | 543 +- benches/query_ops.rs | 819 +- benches/vector_ops.rs | 188 +- docs/04-quality/Benchmark-Plan.md | 4 +- docs/04-quality/Benchmark-Runner.md | 7 +- docs/04-quality/reports/README.md | 4 + docs/04-quality/reports/baselines/README.md | 4 + docs/04-quality/workloads/README.md | 19 +- docs/04-quality/workloads/profiles.json | 4 +- .../workloads/scenario-contract.json | 45 + docs/api-reference.md | 2077 +- docs/architecture-overview.md | 66 +- docs/getting-started.md | 177 +- docs/roadmap.md | 18 +- examples/node/knowledge-graph.mjs | 124 +- examples/python/knowledge_graph.py | 199 +- examples/rust/knowledge_graph.rs | 102 +- overgraph-node/Cargo.toml | 2 +- overgraph-node/README.md | 254 +- overgraph-node/__test__/agent-api.mjs | 444 +- overgraph-node/__test__/async-api.mjs | 167 +- overgraph-node/__test__/benchmark-v2.mjs | 427 +- overgraph-node/__test__/benchmark.mjs | 44 +- overgraph-node/__test__/close-stats.mjs | 34 +- .../__test__/connected-components.mjs | 126 +- overgraph-node/__test__/degree.mjs | 150 +- overgraph-node/__test__/edge-cases.mjs | 66 +- overgraph-node/__test__/export.mjs | 66 +- .../__test__/integration-lifecycle.mjs | 18 +- .../__test__/integration-nonblocking.mjs | 18 +- overgraph-node/__test__/neighbors-batch.mjs | 34 +- overgraph-node/__test__/pagination.mjs | 156 +- overgraph-node/__test__/ppr.mjs | 46 +- overgraph-node/__test__/property-indexes.mjs | 243 +- overgraph-node/__test__/queries.mjs | 259 +- overgraph-node/__test__/scrub.mjs | 63 + overgraph-node/__test__/shortest-path.mjs | 184 +- overgraph-node/__test__/sync-api.mjs | 469 +- overgraph-node/__test__/time-range.mjs | 52 +- overgraph-node/__test__/transactions.mjs | 95 +- overgraph-node/__test__/types/declarations.ts | 95 + overgraph-node/__test__/vector-search.mjs | 16 +- overgraph-node/helpers/pack-binary.mjs | 68 +- overgraph-node/index.d.ts | 704 +- overgraph-node/index.js | 117 +- overgraph-node/package-lock.json | 39 +- overgraph-node/package.json | 17 +- overgraph-node/query-types.d.ts | 203 +- overgraph-node/src/lib.rs | 4028 ++-- overgraph-node/tsconfig.types.json | 16 + overgraph-python/Cargo.toml | 2 +- overgraph-python/README.md | 234 +- overgraph-python/pyproject.toml | 2 +- overgraph-python/python/overgraph/__init__.py | 50 +- .../python/overgraph/__init__.pyi | 537 +- .../python/overgraph/async_api.py | 461 +- overgraph-python/src/lib.rs | 2543 ++- overgraph-python/tests/conftest.py | 14 +- overgraph-python/tests/test_analytics.py | 93 +- overgraph-python/tests/test_async.py | 461 +- overgraph-python/tests/test_batch.py | 238 +- .../tests/test_connected_components.py | 250 +- overgraph-python/tests/test_crud.py | 204 +- overgraph-python/tests/test_degree.py | 190 +- overgraph-python/tests/test_edge_cases.py | 156 +- overgraph-python/tests/test_lifecycle.py | 16 +- overgraph-python/tests/test_maintenance.py | 20 +- overgraph-python/tests/test_pagination.py | 102 +- overgraph-python/tests/test_persistence.py | 72 +- .../tests/test_property_indexes.py | 268 +- overgraph-python/tests/test_queries.py | 130 +- overgraph-python/tests/test_query_api.py | 351 +- overgraph-python/tests/test_retention.py | 40 +- overgraph-python/tests/test_scrub.py | 59 + overgraph-python/tests/test_shortest_path.py | 226 +- overgraph-python/tests/test_transactions.py | 95 +- overgraph-python/tests/test_traversal.py | 172 +- overgraph-python/tests/test_vector_search.py | 31 +- src/bin/benchmark_harness.rs | 478 +- src/bin/inspect.rs | 6 +- src/degree_cache.rs | 225 +- src/dense_hnsw.rs | 401 +- src/edge_metadata.rs | 161 + src/encoding.rs | 267 +- src/engine/graph_ops.rs | 1001 +- src/engine/mod.rs | 4139 +++- src/engine/query.rs | 38 + src/engine/query_exec.rs | 3251 ++- src/engine/query_ir.rs | 677 +- src/engine/query_plan.rs | 4332 +++- src/engine/read.rs | 1567 +- src/engine/tests/graph_ops.rs | 3143 +-- src/engine/tests/label_catalog.rs | 2263 +++ src/engine/tests/lifecycle.rs | 4435 +++-- src/engine/tests/query_planner.rs | 16536 +++++++++++----- src/engine/tests/read.rs | 6006 ++++-- src/engine/tests/txn.rs | 712 +- src/engine/tests/wal_atomic.rs | 1281 ++ src/engine/tests/write.rs | 3079 ++- src/engine/txn.rs | 1159 +- src/engine/write.rs | 1649 +- src/lib.rs | 280 +- src/manifest.rs | 251 +- src/memtable.rs | 2213 ++- src/planner_stats.rs | 2141 +- src/scrub.rs | 2070 ++ src/segment_components.rs | 3011 +++ src/segment_reader.rs | 10015 ++++++++-- src/segment_writer.rs | 11238 ++++++++--- src/source_list.rs | 1738 +- src/sparse_postings.rs | 278 +- src/types.rs | 1042 +- src/wal.rs | 550 +- src/wal_sync.rs | 53 +- ...tegration.rs => compaction_integration.rs} | 40 +- ...ion.rs => engine_lifecycle_integration.rs} | 30 +- ...egration.rs => graph_query_integration.rs} | 60 +- tests/identity_no_byte_check.rs | 175 + tests/inspect_integration.rs | 84 +- tests/named_graph_api_integration.rs | 921 + tests/named_query_api_integration.rs | 376 + tests/robustness_integration.rs | 70 +- tests/scrub_integration.rs | 261 + ...tion.rs => secondary_index_integration.rs} | 132 +- ...ation.rs => segment_reopen_integration.rs} | 50 +- tools/bench/python_connector_benchmark.py | 462 +- tools/bench/txn_node_benchmark.mjs | 10 +- tools/bench/txn_python_benchmark.py | 10 +- 133 files changed, 87225 insertions(+), 27593 deletions(-) create mode 100644 overgraph-node/__test__/scrub.mjs create mode 100644 overgraph-node/__test__/types/declarations.ts create mode 100644 overgraph-node/tsconfig.types.json create mode 100644 overgraph-python/tests/test_scrub.py create mode 100644 src/edge_metadata.rs create mode 100644 src/engine/tests/label_catalog.rs create mode 100644 src/engine/tests/wal_atomic.rs create mode 100644 src/scrub.rs create mode 100644 src/segment_components.rs rename tests/{phase4_integration.rs => compaction_integration.rs} (95%) rename tests/{phase1_integration.rs => engine_lifecycle_integration.rs} (84%) rename tests/{phase2_integration.rs => graph_query_integration.rs} (89%) create mode 100644 tests/identity_no_byte_check.rs create mode 100644 tests/named_graph_api_integration.rs create mode 100644 tests/named_query_api_integration.rs create mode 100644 tests/scrub_integration.rs rename tests/{phase5_integration.rs => secondary_index_integration.rs} (60%) rename tests/{phase3_integration.rs => segment_reopen_integration.rs} (90%) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5bf9b9..e9d4dc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,69 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this ## [Unreleased] +## [0.8.0] - 2026-05-20 + +### Breaking Changes + +#### Pre-1.0 Storage Format Reset +- **Existing database directories must be rebuilt for this release.** OverGraph now writes segment format v10 with a new component identity model and packed `segment.core` layout. Databases created by earlier releases are not expected to open on `0.8.0`. +- **Upgrade guidance:** export or re-ingest your data into a fresh database directory when moving to `0.8.0`. This is an intentional pre-1.0 compatibility break so the storage layout, label model, and identity checks can settle before wider production use. +- **Node identity changed from one type token to label sets.** Stored node records now carry node label IDs rather than a single `type_id`. Public node records expose `labels` instead of `type_id`. +- **Edge identity vocabulary changed from type IDs to label IDs.** Edges still have exactly one edge label, but the durable and diagnostic vocabulary is now `label_id` / `labelId`, not `type_id` / `typeId`. +- **No numeric type-ID compatibility aliases.** Public APIs now take node-label and edge-label names such as `"User"` and `"WORKS_AT"`. Ordinary graph APIs auto-create or resolve the internal label IDs for you. Numeric label IDs are exposed only through catalog diagnostics. + +### Added + +#### Public Label Model +- **Named node and edge labels.** Rust, Node.js, and Python APIs now accept label names directly. You no longer pass `type_id` values into normal writes, reads, queries, traversals, vector scopes, prune policies, or exports. +- **Automatic label catalog creation.** Mutating APIs durably create missing node-label and edge-label catalog entries as part of the same logical write plan. Read and query APIs resolve names without creating new catalog entries. +- **Catalog diagnostics.** Added `ensure_node_label`, `ensure_edge_label`, `get_node_label_id`, `get_edge_label_id`, `get_node_label`, `get_edge_label`, `list_node_labels`, and `list_edge_labels` across Rust, Node.js, Python, and async connector surfaces. +- **Multi-label nodes.** Nodes can now carry bounded label sets. Upserts accept one label or multiple labels; the engine maintains deterministic label-membership indexes and enforces conflict rules when the same key maps to different live nodes across supplied labels. +- **Explicit Any/All label filters.** Added `NodeLabelFilter` / `LabelMatchMode` across query, traversal, vector search scope, graph algorithms, export, prune, and pattern APIs so callers can ask for any listed label or every listed label. + +#### Edge Queries And Edge Indexes +- **Direct edge query APIs.** Added `query_edge_ids`, `query_edges`, and `explain_edge_query` across Rust, Node.js, Python, and async connectors. +- **Edge query anchors.** Direct edge queries can combine explicit edge IDs, edge labels, `from` endpoint sets, `to` endpoint sets, either-endpoint sets, pagination, and explicit full-scan opt-in. +- **Canonical edge filters.** Edge queries and graph-pattern edges now support recursive `and` / `or` / `not` filters over weight ranges, validity windows, built-in `updated_at`, property equality, `in`, property ranges, `exists`, and `missing`. +- **Edge property index declarations.** Added `ensure_edge_property_index`, `drop_edge_property_index`, and `list_edge_property_indexes` for optional edge equality and numeric range indexes scoped by edge label. +- **Edge-property-backed planning.** Ready edge property indexes can participate in direct edge query plans and graph-pattern edge-anchor plans while final results are still verified against visible edge records. +- **Graph-pattern edge anchors.** Pattern planning can now start from selective edge labels, endpoint constraints, edge metadata, or indexed edge property predicates instead of always expanding from a node anchor first. + +#### Storage Identity And Scrub +- **Segment component identity.** Added `segment_manifest.dat` component records, source-group dependency digests, build fingerprints, identity headers, required-vs-optional availability rules, and generation-aware optional refresh. +- **Packed core segments.** Added the v10 `segment.core` container for immutable core source truth and required maintained indexes. +- **Public scrub API.** Added database scrub diagnostics for segment identity, packed ranges, external sidecars, missing files, identity header mismatches, dependency mismatches, and semantic index divergence. + +### Changed + +#### API Model +- **Type vocabulary is now label vocabulary.** Public docs, examples, TypeScript declarations, Python stubs, Rust APIs, benchmark metadata, and connector tests now use node labels and edge labels consistently. +- **Node records expose `labels`.** Hydrated node records now return the complete public label set. Node label collection APIs are named `nodes_by_labels`, `get_nodes_by_labels`, `count_nodes_by_labels`, and paged variants. +- **Edge label diagnostics expose label IDs.** Node.js catalog diagnostics now expose `labelId`; Python exposes `label_id`. These are diagnostic token IDs, not ordinary graph API inputs. +- **Graph and vector APIs resolve names internally.** Neighbor, degree, shortest path, traversal, PPR, connected components, vector search, export, prune, query, transaction, graph patch, and batch APIs all accept public names and resolve compact numeric labels inside the engine. +- **Atomic WAL replay batches.** First-use label-token writes and dependent records are grouped with reusable atomic WAL markers, so recovery replays the complete logical mutation or discards an incomplete tail without partial catalog, record, sequence, degree, or ID effects. + +#### Segment Layout +- **Required core objects moved into `segment.core`.** Node records, edge records, tombstones, node metadata, edge metadata, key indexes, node-label indexes, edge-label indexes, timestamp indexes, edge triple indexes, adjacency indexes/postings, vector source-truth blobs, and immutable edge metadata indexes are now packed into one required core container. +- **Optional accelerators stay external.** Declared node and edge property indexes, planner stats, degree deltas, dense HNSW accelerators, and sparse posting-list accelerators remain refreshable optional sidecars. If they are missing, stale, corrupt, or identity-incompatible, reads fall back to the correct non-accelerated path. +- **Flush and compaction share the same index contract.** Required indexes and optional sidecars are built through both flush and compaction paths with matching label semantics, component identity records, and dependency checks. +- **Segment open is identity-aware but still mmap-first.** Required components validate identity at open; hot read paths continue to use raw mmap payload slices without per-query digest checks. + +#### Planner And Execution +- **Edge predicates are planned sources, not just post-filters.** Edge labels, endpoints, edge metadata, temporal windows, weight filters, and ready edge property indexes can all participate in costed plans. +- **Endpoint plus property queries intersect candidate sources.** Queries such as "outgoing WORKS_AT edges from these nodes where role = lead" intersect endpoint/label sources with property-index candidates when that is cheaper than hydrating the endpoint universe. +- **Pattern queries can choose edge-first plans.** High-fanout patterns with selective relationship predicates can anchor on the edge set, then bind endpoint aliases, while preserving deterministic logical result order. +- **Planner stats understand the new model.** Advisory stats now account for node label memberships, edge labels, edge property declarations, sidecar runtime coverage, stale risk, and graph-pattern fanout under the v10 layout. + +### Fixed + +- **Multi-label visibility across every source.** Active memtables, frozen memtables, flushed segments, compaction output, reopened databases, transactions, prune policies, exports, connector hydration, and query plans all suppress stale label memberships and preserve latest-visible node semantics. +- **Edge query correctness hardening.** Edge property filtering, endpoint visibility, tombstones, updated-at windows, valid-at windows, stale index candidates, signed-zero probes, hash collisions, pagination, and graph-pattern edge bindings are verified against visible records. +- **Edge property sidecar lifecycle.** Edge property sidecars are maintained through active writes, frozen memtables, flush, compaction, background builds, drops, reopen, targeted stats refresh, and optional refresh without making bad sidecars authoritative. +- **Packed-core and identity hardening.** Open, compaction, optional refresh, and scrub now reject or quarantine copied, stale, mismatched, missing, malformed, or wrong-container components according to whether the component is required or optional. +- **Crash-recovery atomicity.** Torn first-use label creation, batch writes, graph patches, transaction commits, cascaded deletes, and prune operations no longer leave partial catalog or record state after WAL replay. +- **Connector parity.** Node.js and Python sync/async APIs, TypeScript declarations, Python stubs, docs, examples, and tests now match the Rust core for label inputs, multi-label nodes, edge queries, edge property indexes, scrub, and catalog diagnostics. + ## [0.7.0] - 2026-05-02 ### Added @@ -271,6 +334,7 @@ Initial release. - Cross-platform CI: macOS, Linux, Windows - Benchmark CI with regression detection and cross-language parity validation +[0.8.0]: https://github.com/bhensley5/overgraph/compare/v0.7.0...v0.8.0 [0.7.0]: https://github.com/bhensley5/overgraph/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/bhensley5/overgraph/compare/v0.5.0...v0.6.0 [0.5.0]: https://github.com/Bhensley5/overgraph/compare/v0.4.1...v0.5.0 diff --git a/Cargo.lock b/Cargo.lock index 6fed4a5..b2ce7b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,15 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -129,6 +138,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -205,6 +223,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctor" version = "0.6.3" @@ -221,6 +249,16 @@ version = "0.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dtor" version = "0.1.1" @@ -358,6 +396,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.4.1" @@ -618,7 +666,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "overgraph" -version = "0.7.0" +version = "0.8.0" dependencies = [ "arc-swap", "crc32fast", @@ -628,12 +676,13 @@ dependencies = [ "rmp-serde", "serde", "serde_json", + "sha2", "tempfile", ] [[package]] name = "overgraph-node" -version = "0.7.0" +version = "0.8.0" dependencies = [ "napi", "napi-build", @@ -644,7 +693,7 @@ dependencies = [ [[package]] name = "overgraph-python" -version = "0.7.0" +version = "0.8.0" dependencies = [ "overgraph", "pyo3", @@ -939,6 +988,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "slab" version = "0.4.12" @@ -985,6 +1045,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1009,6 +1075,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 742764f..7fdbadd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = [".", "overgraph-node", "overgraph-python"] [package] name = "overgraph" -version = "0.7.0" +version = "0.8.0" edition = "2021" description = "An absurdly fast embedded graph database. Pure Rust, sub-microsecond reads." license = "MIT OR Apache-2.0" @@ -20,6 +20,7 @@ memmap2 = "0.9" rayon = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" rmp-serde = "1" [features] diff --git a/README.md b/README.md index b8da24c..7bfb40c 100644 --- a/README.md +++ b/README.md @@ -80,29 +80,27 @@ cargo add overgraph ## Quick start +The vector variables in these snippets are placeholders from your embedding model. Replace them with dense arrays that match the configured dimension and sparse `(dimension, weight)` entries from your sparse encoder. + ### Python ```python from overgraph import OverGraph -USER = 1 -PROJECT = 2 -CREATED = 10 - with OverGraph.open("./my-graph", dense_vector_dimension=384) as db: - # Embeddings come from your model (sentence-transformers, OpenAI, etc.) - # dense: model.encode("Alice is an engineer") -> [f32; 384] - # sparse: splade.encode("Alice is an engineer") -> [(token_id, weight), ...] - alice = db.upsert_node(USER, "user:alice", + # Embeddings come from your model. Dense vectors must match the configured dimension. + # Sparse vectors use (dimension, weight) pairs from your sparse encoder. + # Also accepts multiple labels: ["User", "Engineer"] + alice = db.upsert_node("User", "alice", props={"name": "Alice"}, dense_vector=alice_embedding, sparse_vector=alice_sparse) - project = db.upsert_node(PROJECT, "project:overgraph", + project = db.upsert_node("Project", "overgraph", dense_vector=project_embedding, sparse_vector=project_sparse) - db.upsert_edge(alice, project, CREATED) + db.upsert_edge(alice, project, "CREATED") # Hybrid vector search scoped to a graph neighborhood hits = db.vector_search("hybrid", k=10, @@ -120,29 +118,25 @@ with OverGraph.open("./my-graph", dense_vector_dimension=384) as db: ```javascript import { OverGraph } from 'overgraph'; -const USER = 1; -const PROJECT = 2; -const CREATED = 10; - const db = OverGraph.open('./my-graph', { denseVector: { dimension: 384 }, }); -// Embeddings come from your model (sentence-transformers, OpenAI, etc.) -// dense: model.encode("Alice is an engineer") -> Float32Array(384) -// sparse: splade.encode("Alice is an engineer") -> [{ dimension, value }, ...] -const alice = db.upsertNode(USER, 'user:alice', { +// Embeddings come from your model. Dense vectors must match the configured dimension. +// Sparse vectors use { dimension, value } entries from your sparse encoder. +// Also accepts multiple labels: ['User', 'Engineer'] +const alice = db.upsertNode('User', 'alice', { props: { name: 'Alice' }, denseVector: aliceEmbedding, sparseVector: aliceSparse, }); -const project = db.upsertNode(PROJECT, 'project:overgraph', { +const project = db.upsertNode('Project', 'overgraph', { denseVector: projectEmbedding, sparseVector: projectSparse, }); -db.upsertEdge(alice, project, CREATED); +db.upsertEdge(alice, project, 'CREATED'); // Hybrid vector search scoped to a graph neighborhood const hits = db.vectorSearch('hybrid', { @@ -163,10 +157,6 @@ use overgraph::*; use std::collections::BTreeMap; use std::path::Path; -const USER: u32 = 1; -const PROJECT: u32 = 2; -const CREATED: u32 = 10; - fn main() -> Result<(), Box> { let opts = DbOptions { dense_vector: Some(DenseVectorConfig { @@ -178,25 +168,25 @@ fn main() -> Result<(), Box> { }; let mut db = DatabaseEngine::open(Path::new("./my-graph"), &opts)?; - // Embeddings come from your model (sentence-transformers, OpenAI, etc.) - // dense: model.encode("Alice is an engineer") -> Vec with 384 dims - // sparse: splade.encode("Alice is an engineer") -> Vec<(u32, f32)> + // Embeddings come from your model. Dense vectors must match the configured dimension. + // Sparse vectors use (dimension, weight) pairs from your sparse encoder. let mut props = BTreeMap::new(); props.insert("name".into(), PropValue::String("Alice".into())); - let alice = db.upsert_node(USER, "user:alice", UpsertNodeOptions { + // Also accepts multiple labels: &["User", "Engineer"] + let alice = db.upsert_node("User", "alice", UpsertNodeOptions { props, dense_vector: Some(alice_embedding), sparse_vector: Some(alice_sparse), ..Default::default() })?; - let project = db.upsert_node(PROJECT, "project:overgraph", UpsertNodeOptions { + let project = db.upsert_node("Project", "overgraph", UpsertNodeOptions { dense_vector: Some(project_embedding), sparse_vector: Some(project_sparse), ..Default::default() })?; - db.upsert_edge(alice, project, CREATED, UpsertEdgeOptions::default())?; + db.upsert_edge(alice, project, "CREATED", UpsertEdgeOptions::default())?; // Hybrid vector search: dense + sparse with graph scoping let hits = db.vector_search(&VectorSearchRequest { @@ -204,13 +194,16 @@ fn main() -> Result<(), Box> { dense_query: Some(query_embedding), sparse_query: Some(query_sparse), k: 10, // required: 0 returns empty - type_filter: Some(vec![USER, PROJECT]), // default: None (no filtering) + label_filter: Some(NodeLabelFilter { + labels: vec!["User".into(), "Project".into()], + mode: LabelMatchMode::Any, + }), // default: None ef_search: Some(200), // default: 128 scope: Some(VectorSearchScope { // default: None (search all nodes) start_node_id: alice, max_depth: 3, direction: Direction::Outgoing, // default: Outgoing - edge_type_filter: Some(vec![CREATED]), // default: None (all edge types) + edge_label_filter: Some(vec!["CREATED".into()]), // default: None (all edge labels) at_epoch: None, // default: None (current time) }), dense_weight: Some(0.7), // default: 1.0 @@ -228,7 +221,7 @@ fn main() -> Result<(), Box> { ### Async support -Both Python and Node.js connectors include full async variants of every API. Python provides `AsyncOverGraph` with native `asyncio` support. Node.js methods have `Async` suffixed variants (e.g. `upsertNodeAsync`, `vectorSearchAsync`). +Both Python and Node.js connectors include full async variants of every API. Python provides `AsyncOverGraph`, an asyncio wrapper that runs sync operations in a thread pool via `asyncio.to_thread()`. Node.js methods have `Async` suffixed variants (e.g. `upsertNodeAsync`, `vectorSearchAsync`). ## Features @@ -236,11 +229,11 @@ Both Python and Node.js connectors include full async variants of every API. Pyt - **Dense vector search.** Attach `f32` embedding vectors to any node. HNSW indexes are built per segment at flush time for fast approximate nearest neighbor search. Supports cosine, Euclidean, and dot-product distance metrics. One dense vector space per DB with configurable dimension. - **Sparse vector search.** Attach sparse vectors (dimension-value pairs) for keyword-weighted retrieval. Works with pre-computed sparse embeddings from models like SPLADE or BGE-M3. Inverted posting-list indexes for exact dot-product scoring. - **Hybrid search.** Combine dense and sparse results with built-in fusion modes: weighted rank fusion, reciprocal rank fusion, or weighted score fusion. Adjustable `dense_weight` and `sparse_weight` for tuning the blend. -- **Graph-scoped search.** Scope vector search to a graph neighborhood: "find the 10 most similar nodes within 3 hops of node X." Uses traversal-based reachable-node filtering with edge-type and temporal support. Combine graph structure with vector similarity in a single query. +- **Graph-scoped search.** Scope vector search to a graph neighborhood: "find the 10 most similar nodes within 3 hops of node X." Uses traversal-based reachable-node filtering with edge-label and temporal support. Combine graph structure with vector similarity in a single query. - **Zero overhead when unused.** Nodes without vectors pay no storage or runtime cost. Vector index files are only created for segments that contain vectors. ### Core graph operations -- **Upsert semantics.** Nodes are keyed by `(type_id, key)`. Upsert the same key twice and you get an update, not a duplicate. Edges can optionally enforce uniqueness on `(from, to, type_id)`. +- **Upsert semantics.** Nodes carry one or more labels and one key. Each live `(label, key)` membership is unique, so a multi-label node owns the same key in every label it carries. Upserting a key that resolves to the same node through the supplied labels updates it; if supplied label memberships resolve to different nodes, the write is rejected as a conflict. Edges can optionally enforce uniqueness on `(from, to, label)`. - **Batch operations.** `batch_upsert_nodes` and `batch_upsert_edges` amortize WAL and memtable overhead. `get_nodes` and `get_nodes_by_keys` do batched reads with sorted merge-walks instead of per-item lookups. There's also a packed binary format for maximum write throughput. - **Atomic graph patch.** `graph_patch` lets you upsert nodes, upsert edges, delete nodes, delete edges, and invalidate edges in a single atomic operation. - **Explicit transactions.** `begin_write_txn()` / `beginWriteTxn()` gives you ordered staging, rollback, read-own-writes point lookups, local aliases, atomic commit, and clean conflict errors for retry loops. @@ -251,30 +244,30 @@ Both Python and Node.js connectors include full async variants of every API. Pyt - **Decay scoring.** Pass a `decay_lambda` to neighbor queries and edge weights are automatically scaled by `exp(-lambda * age_hours)`. Recent connections matter more. ### Queries and traversal -- **Neighbors and bounded traversal.** `neighbors()` handles 1-hop expansion and returns normal neighbor entry collections in every connector; `traverse()` covers deterministic breadth-first traversal across arbitrary depth windows with optional edge-type filtering, emission-only node-type filtering, and traversal-specific pagination. +- **Neighbors and bounded traversal.** `neighbors()` handles 1-hop expansion and returns normal neighbor entry collections in every connector; `traverse()` covers deterministic breadth-first traversal across arbitrary depth windows with optional edge-label filtering, emission-only node-label filtering, and traversal-specific pagination. - **Depth slices without special-case APIs.** Exact depth-2 traversals are expressed as `traverse(start, 2, min_depth=2)`, so 2-hop use cases stay available without a separate public method family. - **Top-K neighbors.** Get the K highest-scoring neighbors by weight, recency, or decay-adjusted score. - **Personalized PageRank.** Run PPR from seed nodes to find the most relevant nodes in the graph. Rust, Node.js, and Python expose both exact power-iteration PPR and a much faster approximate forward-push mode for seed-centric retrieval workloads. - **Subgraph extraction.** Pull out a connected subgraph up to N hops deep. Good for building local context windows. - **Shortest path.** BFS (unweighted) or bidirectional Dijkstra (weighted). `is_connected` for fast reachability checks. `all_shortest_paths` when there are ties. -- **Connected components.** `connected_components()` returns a global WCC labelling (union-find, near-linear). `component_of(node)` returns the members of a single node's component via BFS. Both support edge-type, node-type, and temporal filters. +- **Connected components.** `connected_components()` returns a global WCC labelling (union-find, near-linear). `component_of(node)` returns the members of a single node's component via BFS. Both support edge-label, node-label, and temporal filters. - **Degree counts.** Count edges, sum weights, and compute averages without materializing neighbor lists. Batch `degrees` for bulk analysis. - **Direct property queries.** `find_nodes` and `find_nodes_paged` do focused equality lookups. `find_nodes_range` and `find_nodes_range_paged` do numeric range scans with exact bound and cursor semantics. -- **Optional property indexes.** Declare equality or numeric range indexes only where they pay off. Use `ensure_node_property_index`, `list_node_property_indexes`, and `drop_node_property_index` to manage them. Public query APIs stay index-transparent: when a matching declaration is `Ready`, OverGraph uses the declaration-backed path; otherwise it falls back to the same public API. -- **Full query APIs.** `query_node_ids`, `query_nodes`, `query_pattern`, and explain APIs combine IDs, keys, types, property equality/IN/range/exists/missing filters, updated-at ranges, and bounded graph patterns without a query string. OverGraph chooses the cheapest legal path with available indexes and planner stats, then verifies results against visible records. +- **Optional property indexes.** Declare node or edge equality/range indexes only where they pay off. Use `ensure_node_property_index` / `ensure_edge_property_index`, list APIs, and drop APIs to manage them. Public query APIs stay index-transparent: when a matching declaration is `Ready`, OverGraph uses the declaration-backed path; otherwise it falls back to the same public API. +- **Full query APIs.** `query_node_ids`, `query_nodes`, `query_edge_ids`, `query_edges`, `query_pattern`, and explain APIs combine IDs, keys, labels, edge labels, endpoint constraints, property equality/IN/range/exists/missing filters, edge metadata filters, updated-at ranges, and bounded graph patterns without a query string. OverGraph chooses the cheapest legal path with available indexes and planner stats, then verifies results against visible records. - **Time-range queries.** Find nodes created or updated within a time window. Sorted timestamp index for efficient range scans. ### Pagination ID-keyed collection APIs use keyset pagination with `limit` and `after`. `traverse()` uses `limit` plus a traversal cursor keyed by `(depth, node_id)`. No offset-based pagination. Traversal cursors assume the same query arguments and a stable logical graph state; strict snapshot isolation across intervening writes is not promised. ### Retention and pruning -- **Manual prune.** Drop nodes older than X, below weight Y, or matching type Z. Incident edges cascade automatically. +- **Manual prune.** Drop nodes older than X, below weight Y, or matching a label. Incident edges cascade automatically. - **Named prune policies.** Register policies like `"short_term_memory"` that run automatically during compaction. Nodes matching any policy are invisible to reads immediately (lazy expiration) and cleaned up during the next compaction pass. ### Storage engine - **Write-ahead log.** Every mutation hits the WAL before the memtable. Crash recovery replays the WAL on startup. - **Configurable durability.** `Immediate` mode fsyncs every write for maximum safety. `GroupCommit` mode (default) batches fsyncs on a 50ms timer for ~20x better write throughput with at most one timer interval of data at risk. -- **Background compaction.** Segments are merged automatically when thresholds are met. Compaction runs on a background thread and never blocks reads or writes. Uses metadata sidecars for fast filtered merging without full record decoding. +- **Background compaction.** Segments are merged automatically when thresholds are met. Compaction runs on a background thread and never blocks reads or writes. Uses packed metadata payloads for fast filtered merging without full record decoding. - **Bulk ingest mode.** Temporarily disable auto-compaction during large write bursts with `ingest_mode()`, then call `end_ingest()` to compact accumulated segments and restore normal behavior. This favors ingest throughput over read performance during the ingest window. - **mmap'd reads.** Immutable segments are memory-mapped. The OS page cache handles caching. Reads never block writes. - **Portable databases.** Each database is a self-contained directory. `cp -r ./my-db /backup/my-db` and you're done. @@ -285,36 +278,36 @@ OverGraph uses a log-structured storage engine purpose-built from scratch in pur **Write path:** Mutations are appended to a write-ahead log and applied to an in-memory memtable. When the memtable reaches its threshold, it's frozen and flushed to disk as an immutable segment in the background. Writes continue unblocked against a fresh memtable. Each segment ships with pre-built adjacency indexes (inbound and outbound), optional declared property-index sidecars, optional advisory planner statistics, optional signed degree-delta sidecars for degree/weight fast paths, and, when the segment contains vectors, HNSW and sparse posting-list indexes. -**Read path:** Queries check the memtable first (freshest data), then merge results across immutable segments using the per-segment indexes. Because every segment carries its own adjacency index, a neighbor query is a handful of index lookups, not a scan across sorted keys. Vector search follows the same model: memtable candidates are found by exact brute-force scan, segment candidates via HNSW or posting-list indexes, then the engine merges and deduplicates across all sources. Property equality and numeric range queries stay index-transparent too: if a matching optional property-index declaration is `Ready`, the engine uses the declaration-backed path, otherwise it falls back to a type-scoped scan through the same public API. Pagination uses early termination to avoid unnecessary work. +**Read path:** Queries check the memtable first (freshest data), then merge results across immutable segments using the per-segment indexes. Because every segment carries its own adjacency index, a neighbor query is a handful of index lookups, not a scan across sorted keys. Vector search follows the same model: memtable candidates are found by exact brute-force scan, segment candidates via HNSW or posting-list indexes, then the engine merges and deduplicates across all sources. Property equality and numeric range queries stay index-transparent too: if a matching optional property-index declaration is `Ready`, the engine uses the declaration-backed path, otherwise it falls back to a label-scoped scan through the same public API. Pagination uses early termination to avoid unnecessary work. -**Compaction:** A background thread merges older segments together, applying tombstones, prune policies, and deduplication. The compaction path uses metadata sidecars to plan merges and raw-copies winning records without full deserialization, then rebuilds unified indexes from metadata. This includes rebuilding HNSW and sparse posting-list indexes for the merged output. Fewer segments after compaction means fewer index lookups per query, but even before compaction, reads are fast because every segment is self-indexed. +**Compaction:** A background thread merges older segments together, applying tombstones, prune policies, and deduplication. The compaction path uses packed metadata payloads to plan merges and raw-copies winning records without full deserialization, then rebuilds unified indexes from metadata. This includes rebuilding HNSW and sparse posting-list indexes for the merged output. Fewer segments after compaction means fewer index lookups per query, but even before compaction, reads are fast because every segment is self-indexed. **On-disk layout:** ``` my-graph/ manifest.current # atomic checkpoint (JSON) - data.wal # append-only write-ahead log + wal_0.wal # append-only write-ahead log generation segments/ seg_0001/ - nodes.dat # node records - edges.dat # edge records - adj_out.idx # outgoing adjacency index - adj_in.idx # incoming adjacency index - key_index.dat # (type_id, key) -> node_id - type_index.dat # type_id -> [id...] - tombstones.dat # deleted IDs - secondary_indexes/ # optional declared property-index sidecars - planner_stats.dat # optional advisory planner statistics + segment_manifest.dat # component table of contents + segment.core # packed immutable core records, metadata, and maintained indexes + secondary_indexes/ # optional declared equality/range property-index sidecars + planner_stats.dat # optional advisory planner statistics, refreshable degree_delta.dat # optional signed degree deltas for fast degree/weight reads - node_dense_vectors.dat # dense vector blob (when present) - node_sparse_vectors.dat # sparse vector blob (when present) - dense_hnsw_graph.dat # HNSW graph index (when present) - sparse_postings.dat # sparse posting lists (when present) - node_vector_meta.dat # vector offsets/lengths per node + dense_hnsw_meta.dat # optional dense-vector HNSW metadata + dense_hnsw_graph.dat # optional dense-vector HNSW graph + sparse_posting_index.dat # optional sparse-vector posting index + sparse_postings.dat # optional sparse-vector posting lists seg_0002/ ... ``` +`segment.core` is addressed through `segment_manifest.dat`. It contains the logical +node/edge record payloads, tombstones, key/label-token/timestamp/triple indexes, adjacency +indexes/postings, node/edge metadata, vector source-truth blobs, and immutable edge +metadata indexes. Refreshable optional accelerators stay outside the packed core so +they can be rebuilt or dropped without rewriting source data. + For a deeper dive, see the [architecture overview](docs/architecture-overview.md). ## Documentation @@ -323,6 +316,10 @@ For a deeper dive, see the [architecture overview](docs/architecture-overview.md - **[API Reference](docs/api-reference.md)** - every method, parameter, type, and return value across Python, Node.js, and Rust. - **[Roadmap](docs/roadmap.md)** - where OverGraph is headed and what's already shipped. +## Community Integrations + +- **[OvergraphSwiftBridge](https://github.com/wildthink/OvergraphSwiftBridge)** - a community-maintained bridge for bringing OverGraph into Swift and Apple-platform projects. + ## Running the benchmarks ```bash diff --git a/benches/async_flush.rs b/benches/async_flush.rs index aff8d8c..673a8bd 100644 --- a/benches/async_flush.rs +++ b/benches/async_flush.rs @@ -35,9 +35,25 @@ const SYNC_FLUSH_INTERVAL: u64 = 3300; fn temp_db_with_opts(opts: DbOptions) -> (tempfile::TempDir, DatabaseEngine) { let dir = tempfile::tempdir().unwrap(); let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + seed_bench_label_tokens(&engine); (dir, engine) } +fn seed_bench_label_tokens(engine: &DatabaseEngine) { + for label_token_id in 1..=8 { + assert_eq!( + engine + .ensure_node_label(&bench_node_label(label_token_id)) + .unwrap(), + label_token_id + ); + } +} + +fn bench_node_label(label_token_id: u32) -> String { + format!("BenchNode{label_token_id}") +} + fn sync_opts() -> DbOptions { DbOptions { create_if_missing: true, @@ -106,7 +122,7 @@ fn simple_sparse_vector(index: usize, nnz: usize) -> Vec<(u32, f32)> { fn pre_populate_plain(engine: &mut DatabaseEngine, count: usize) -> Vec { let inputs: Vec = (0..count) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("seed_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -114,7 +130,7 @@ fn pre_populate_plain(engine: &mut DatabaseEngine, count: usize) -> Vec { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); ids } @@ -122,7 +138,7 @@ fn pre_populate_plain(engine: &mut DatabaseEngine, count: usize) -> Vec { fn pre_populate_dense(engine: &mut DatabaseEngine, count: usize, dim: usize) { let inputs: Vec = (0..count) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("vec_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -130,14 +146,14 @@ fn pre_populate_dense(engine: &mut DatabaseEngine, count: usize, dim: usize) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); } fn pre_populate_sparse(engine: &mut DatabaseEngine, count: usize, nnz: usize) { let inputs: Vec = (0..count) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("svec_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -145,7 +161,7 @@ fn pre_populate_sparse(engine: &mut DatabaseEngine, count: usize, nnz: usize) { sparse_vector: Some(simple_sparse_vector(i, nnz)), }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); } @@ -165,7 +181,7 @@ fn bench_sustained_writes_threshold(c: &mut Criterion) { |(_dir, engine)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("n{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("n{}", i), write_opts(i)) .unwrap(); if (i + 1) % SYNC_FLUSH_INTERVAL == 0 { engine.flush().unwrap(); @@ -182,7 +198,7 @@ fn bench_sustained_writes_threshold(c: &mut Criterion) { |(_dir, engine)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("n{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("n{}", i), write_opts(i)) .unwrap(); } }, @@ -208,7 +224,7 @@ fn bench_writes_with_queued_epochs(c: &mut Criterion) { let (_dir, engine) = temp_db_with_opts(sync_opts()); for j in 0..2000u64 { engine - .upsert_node(1, &format!("pre_{}", j), write_opts(j)) + .upsert_node("BenchNode", &format!("pre_{}", j), write_opts(j)) .unwrap(); if (j + 1) % SYNC_FLUSH_INTERVAL == 0 { engine.flush().unwrap(); @@ -220,7 +236,7 @@ fn bench_writes_with_queued_epochs(c: &mut Criterion) { for i in 0..BURST_SIZE { let k = 2000 + i; engine - .upsert_node(1, &format!("n{}", k), write_opts(k)) + .upsert_node("BenchNode", &format!("n{}", k), write_opts(k)) .unwrap(); if (i + 1) % SYNC_FLUSH_INTERVAL == 0 { engine.flush().unwrap(); @@ -237,7 +253,7 @@ fn bench_writes_with_queued_epochs(c: &mut Criterion) { let (_dir, engine) = temp_db_with_opts(async_opts()); for j in 0..2000u64 { engine - .upsert_node(1, &format!("pre_{}", j), write_opts(j)) + .upsert_node("BenchNode", &format!("pre_{}", j), write_opts(j)) .unwrap(); } (_dir, engine) @@ -246,7 +262,7 @@ fn bench_writes_with_queued_epochs(c: &mut Criterion) { for i in 0..BURST_SIZE { let k = 2000 + i; engine - .upsert_node(1, &format!("n{}", k), write_opts(k)) + .upsert_node("BenchNode", &format!("n{}", k), write_opts(k)) .unwrap(); } }, @@ -276,7 +292,7 @@ fn bench_mixed_writes_reads(c: &mut Criterion) { |(_dir, engine, ids)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("w{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("w{}", i), write_opts(i)) .unwrap(); if (i + 1) % SYNC_FLUSH_INTERVAL == 0 { engine.flush().unwrap(); @@ -298,7 +314,7 @@ fn bench_mixed_writes_reads(c: &mut Criterion) { |(_dir, engine, ids)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("w{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("w{}", i), write_opts(i)) .unwrap(); black_box(engine.get_node(ids[(i as usize) % ids.len()]).unwrap()); } @@ -330,7 +346,7 @@ fn bench_mixed_writes_dense_vector(c: &mut Criterion) { dense_query: Some(simple_dense_vector(dim, 999)), sparse_query: None, k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -350,7 +366,7 @@ fn bench_mixed_writes_dense_vector(c: &mut Criterion) { |(_dir, engine)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("w{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("w{}", i), write_opts(i)) .unwrap(); if (i + 1) % SYNC_FLUSH_INTERVAL == 0 { engine.flush().unwrap(); @@ -374,7 +390,7 @@ fn bench_mixed_writes_dense_vector(c: &mut Criterion) { |(_dir, engine)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("w{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("w{}", i), write_opts(i)) .unwrap(); black_box(engine.vector_search(&request).unwrap()); } @@ -400,7 +416,7 @@ fn bench_mixed_writes_sparse_vector(c: &mut Criterion) { dense_query: None, sparse_query: Some(simple_sparse_vector(999, 8)), k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -418,7 +434,7 @@ fn bench_mixed_writes_sparse_vector(c: &mut Criterion) { |(_dir, engine)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("w{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("w{}", i), write_opts(i)) .unwrap(); if (i + 1) % SYNC_FLUSH_INTERVAL == 0 { engine.flush().unwrap(); @@ -440,7 +456,7 @@ fn bench_mixed_writes_sparse_vector(c: &mut Criterion) { |(_dir, engine)| { for i in 0..BURST_SIZE { engine - .upsert_node(1, &format!("w{}", i), write_opts(i)) + .upsert_node("BenchNode", &format!("w{}", i), write_opts(i)) .unwrap(); black_box(engine.vector_search(&request).unwrap()); } diff --git a/benches/core_ops.rs b/benches/core_ops.rs index ea31939..0ca8799 100644 --- a/benches/core_ops.rs +++ b/benches/core_ops.rs @@ -1,11 +1,11 @@ use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion}; use overgraph::{ AllShortestPathsOptions, DatabaseEngine, DbOptions, DegreeOptions, Direction, EdgeInput, - ExportOptions, GraphPatch, IsConnectedOptions, NeighborOptions, NodeInput, PageRequest, - PprAlgorithm, PprOptions, PropValue, PropertyRangeBound, PrunePolicy, SecondaryIndexKind, - SecondaryIndexRangeDomain, SecondaryIndexState, ShortestPathOptions, TopKOptions, - TraverseOptions, TxnIntent, TxnLocalRef, TxnNodeRef, UpsertEdgeOptions, UpsertNodeOptions, - WalSyncMode, + ExportOptions, GraphPatch, IsConnectedOptions, LabelMatchMode, NeighborOptions, NodeInput, + NodeKeyQuery, NodeLabelFilter, PageRequest, PprAlgorithm, PprOptions, PropValue, + PropertyRangeBound, PrunePolicy, SecondaryIndexKind, SecondaryIndexRangeDomain, + SecondaryIndexState, ShortestPathOptions, TopKOptions, TraverseOptions, TxnIntent, TxnLocalRef, + TxnNodeRef, UpsertEdgeOptions, UpsertNodeOptions, WalSyncMode, }; use std::collections::BTreeMap; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -18,9 +18,31 @@ fn temp_db() -> (tempfile::TempDir, DatabaseEngine) { ..DbOptions::default() }; let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + seed_bench_label_tokens(&engine); (dir, engine) } +fn seed_bench_label_tokens(engine: &DatabaseEngine) { + for label_token_id in 1..=64 { + assert_eq!( + engine + .ensure_node_label(&bench_node_label(label_token_id)) + .unwrap(), + label_token_id + ); + assert_eq!( + engine + .ensure_edge_label(&format!("BenchEdge{label_token_id}")) + .unwrap(), + label_token_id + ); + } +} + +fn bench_node_label(label_token_id: u32) -> String { + format!("BenchNode{label_token_id}") +} + fn bench_upsert_node(c: &mut Criterion) { c.bench_function("upsert_node", |b| { let (_dir, engine) = temp_db(); @@ -28,7 +50,7 @@ fn bench_upsert_node(c: &mut Criterion) { b.iter(|| { let key = format!("node_{}", i); engine - .upsert_node(1, &key, UpsertNodeOptions::default()) + .upsert_node("BenchNode", &key, UpsertNodeOptions::default()) .unwrap(); i += 1; }); @@ -46,7 +68,7 @@ fn bench_upsert_node_with_props(c: &mut Criterion) { props.insert("score".to_string(), PropValue::Float(0.95)); engine .upsert_node( - 1, + "BenchNode", &key, UpsertNodeOptions { props, @@ -65,7 +87,7 @@ fn bench_upsert_edge(c: &mut Criterion) { // Pre-create nodes let inputs: Vec = (0..1000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props: BTreeMap::new(), weight: 1.0, @@ -73,13 +95,13 @@ fn bench_upsert_edge(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); let mut i = 0u64; b.iter(|| { let from = (i % 1000) + 1; let to = ((i + 1) % 1000) + 1; engine - .upsert_edge(from, to, 1, UpsertEdgeOptions::default()) + .upsert_edge(from, to, "BenchEdge", UpsertEdgeOptions::default()) .unwrap(); i += 1; }); @@ -91,7 +113,7 @@ fn bench_get_node(c: &mut Criterion) { let (_dir, engine) = temp_db(); let inputs: Vec = (0..1000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props: BTreeMap::new(), weight: 1.0, @@ -99,7 +121,7 @@ fn bench_get_node(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let mut i = 0usize; b.iter(|| { let id = ids[i % ids.len()]; @@ -112,7 +134,7 @@ fn bench_get_node(c: &mut Criterion) { let (_dir, engine) = temp_db(); let inputs: Vec = (0..1000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props: BTreeMap::new(), weight: 1.0, @@ -120,7 +142,7 @@ fn bench_get_node(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let mut i = 0usize; b.iter(|| { @@ -134,7 +156,7 @@ fn bench_get_node(c: &mut Criterion) { /// Build a hub-and-spokes graph: one hub node with `n` outgoing edges to target nodes. fn build_hub_graph(engine: &mut DatabaseEngine, n: usize) -> u64 { let mut inputs: Vec = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -143,7 +165,7 @@ fn build_hub_graph(engine: &mut DatabaseEngine, n: usize) -> u64 { }]; for i in 0..n { inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("t{}", i), props: BTreeMap::new(), weight: 1.0, @@ -151,34 +173,34 @@ fn build_hub_graph(engine: &mut DatabaseEngine, n: usize) -> u64 { sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let hub = ids[0]; let edges: Vec = ids[1..] .iter() .map(|&target| EdgeInput { from: hub, to: target, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); hub } /// Build a hub graph that stresses Direction::Both self-loop dedup: /// - bidirectional hub <-> spoke edges -/// - many hub self-loops (distinct type IDs to satisfy uniqueness) +/// - many hub self-loops (distinct label IDs to satisfy uniqueness) fn build_hub_both_selfloop_graph( engine: &mut DatabaseEngine, spoke_count: usize, self_loop_count: usize, ) -> u64 { let mut inputs: Vec = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub_both".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -187,7 +209,7 @@ fn build_hub_both_selfloop_graph( }]; for i in 0..spoke_count { inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("both_t{}", i), props: BTreeMap::new(), weight: 1.0, @@ -195,7 +217,7 @@ fn build_hub_both_selfloop_graph( sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let hub = ids[0]; let mut edges: Vec = Vec::with_capacity(spoke_count * 2 + self_loop_count); @@ -203,7 +225,7 @@ fn build_hub_both_selfloop_graph( edges.push(EdgeInput { from: hub, to: target, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -212,7 +234,7 @@ fn build_hub_both_selfloop_graph( edges.push(EdgeInput { from: target, to: hub, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -223,14 +245,14 @@ fn build_hub_both_selfloop_graph( edges.push(EdgeInput { from: hub, to: hub, - type_id: 10_000 + i as u32, + label: format!("BenchEdge{}", 10_000 + i as u32), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); hub } @@ -327,7 +349,7 @@ fn bench_neighbors_with_pit(c: &mut Criterion) { let build_pit_graph = |engine: &mut DatabaseEngine| -> u64 { let mut inputs: Vec = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -336,7 +358,7 @@ fn bench_neighbors_with_pit(c: &mut Criterion) { }]; for i in 0..100 { inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("t{}", i), props: BTreeMap::new(), weight: 1.0, @@ -344,21 +366,21 @@ fn bench_neighbors_with_pit(c: &mut Criterion) { sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let hub = ids[0]; let edges: Vec = ids[1..] .iter() .map(|&target| EdgeInput { from: hub, to: target, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: Some(now - 10000), valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); hub }; @@ -404,7 +426,7 @@ fn bench_find_nodes(c: &mut Criterion) { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String(color.to_string())); NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props, weight: 1.0, @@ -413,28 +435,30 @@ fn bench_find_nodes(c: &mut Criterion) { } }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); }; c.bench_function("find_nodes_1000", |b| { let (_dir, mut engine) = temp_db(); build_find_graph(&mut engine); let val = PropValue::String("red".to_string()); + let label = bench_node_label(1); b.iter(|| { - engine.find_nodes(1, "color", &val).unwrap(); + engine.find_nodes(&label, "color", &val).unwrap(); }); }); c.bench_function("find_nodes_1000_declared", |b| { let (_dir, mut engine) = temp_db(); + let label = bench_node_label(1); let eq = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index(&label, "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, eq.index_id, SecondaryIndexState::Ready); build_find_graph(&mut engine); let val = PropValue::String("red".to_string()); b.iter(|| { - engine.find_nodes(1, "color", &val).unwrap(); + engine.find_nodes(&label, "color", &val).unwrap(); }); }); @@ -443,22 +467,24 @@ fn bench_find_nodes(c: &mut Criterion) { build_find_graph(&mut engine); engine.flush().unwrap(); let val = PropValue::String("red".to_string()); + let label = bench_node_label(1); b.iter(|| { - engine.find_nodes(1, "color", &val).unwrap(); + engine.find_nodes(&label, "color", &val).unwrap(); }); }); c.bench_function("find_nodes_1000_segment_declared", |b| { let (_dir, mut engine) = temp_db(); + let label = bench_node_label(1); let eq = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index(&label, "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, eq.index_id, SecondaryIndexState::Ready); build_find_graph(&mut engine); engine.flush().unwrap(); let val = PropValue::String("red".to_string()); b.iter(|| { - engine.find_nodes(1, "color", &val).unwrap(); + engine.find_nodes(&label, "color", &val).unwrap(); }); }); } @@ -472,7 +498,7 @@ fn bench_flush(c: &mut Criterion) { let (dir, engine) = temp_db(); let node_inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props: BTreeMap::new(), weight: 1.0, @@ -480,19 +506,19 @@ fn bench_flush(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..20) .map(|i| EdgeInput { from: ids[i % 100], to: ids[(i + 1) % 100], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); (dir, engine) }, |(_dir, engine)| { @@ -511,7 +537,7 @@ fn bench_batch_upsert_nodes(c: &mut Criterion) { b.iter(|| { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("batch{}_{}", batch_num, i), props: BTreeMap::new(), weight: 1.0, @@ -519,7 +545,7 @@ fn bench_batch_upsert_nodes(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs).unwrap(); batch_num += 1; }); }); @@ -534,7 +560,7 @@ fn bench_batch_upsert_nodes(c: &mut Criterion) { batch_num * 100, 100, ); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs).unwrap(); batch_num += 1; }); }); @@ -599,14 +625,19 @@ fn wait_for_property_index_state( } fn ensure_property_query_declarations(engine: &mut DatabaseEngine) { + let label = bench_node_label(1); let eq = engine - .ensure_node_property_index(1, PROPERTY_EQ_DECLARED_KEY, SecondaryIndexKind::Equality) + .ensure_node_property_index( + &label, + PROPERTY_EQ_DECLARED_KEY, + SecondaryIndexKind::Equality, + ) .unwrap(); wait_for_property_index_state(engine, eq.index_id, SecondaryIndexState::Ready); let range = engine .ensure_node_property_index( - 1, + &label, PROPERTY_RANGE_DECLARED_KEY, SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -647,7 +678,7 @@ fn make_property_query_nodes(prefix: &str, start: u64, count: usize) -> Vec Vec (tempfile::TempDir, DatabaseEngine) { let (dir, mut engine) = property_bench_db(); ensure_property_query_declarations(&mut engine); let nodes = make_property_query_nodes("query", 0, PROPERTY_QUERY_NODE_COUNT); - engine.batch_upsert_nodes(&nodes).unwrap(); + engine.batch_upsert_nodes(nodes.clone()).unwrap(); engine.flush().unwrap(); (dir, engine) } @@ -708,7 +739,7 @@ fn build_property_flush_engine(with_declarations: bool) -> (tempfile::TempDir, D ensure_property_query_declarations(&mut engine); } let nodes = make_property_query_nodes("flush", 0, PROPERTY_FLUSH_NODE_COUNT); - engine.batch_upsert_nodes(&nodes).unwrap(); + engine.batch_upsert_nodes(nodes.clone()).unwrap(); (dir, engine) } @@ -726,7 +757,7 @@ fn build_property_compaction_engine( start, PROPERTY_COMPACTION_NODES_PER_SEGMENT as usize, ); - engine.batch_upsert_nodes(&nodes).unwrap(); + engine.batch_upsert_nodes(nodes.clone()).unwrap(); engine.flush().unwrap(); } (dir, engine) @@ -742,7 +773,7 @@ fn build_property_compaction_general_budget_engine() -> (tempfile::TempDir, Data start, PROPERTY_COMPACTION_NODES_PER_SEGMENT as usize, ); - engine.batch_upsert_nodes(&nodes).unwrap(); + engine.batch_upsert_nodes(nodes.clone()).unwrap(); engine.flush().unwrap(); } (dir, engine) @@ -754,7 +785,7 @@ fn build_many_stats_sidecars_db() -> tempfile::TempDir { for segment in 0..8u64 { let start = segment * 1_000; let nodes = make_property_query_nodes(&format!("open{}", segment), start, 1_000); - engine.batch_upsert_nodes(&nodes).unwrap(); + engine.batch_upsert_nodes(nodes.clone()).unwrap(); engine.flush().unwrap(); } engine.close().unwrap(); @@ -774,10 +805,11 @@ fn bench_property_indexes(c: &mut Criterion) { query_group.bench_function("equality_declared", |b| { let (_dir, engine) = build_property_query_engine(); + let label = bench_node_label(1); b.iter(|| { black_box( engine - .find_nodes(1, PROPERTY_EQ_DECLARED_KEY, &declared_eq_value) + .find_nodes(&label, PROPERTY_EQ_DECLARED_KEY, &declared_eq_value) .unwrap(), ); }); @@ -785,10 +817,11 @@ fn bench_property_indexes(c: &mut Criterion) { query_group.bench_function("equality_fallback_scan", |b| { let (_dir, engine) = build_property_query_engine(); + let label = bench_node_label(1); b.iter(|| { black_box( engine - .find_nodes(1, PROPERTY_EQ_FALLBACK_KEY, &fallback_eq_value) + .find_nodes(&label, PROPERTY_EQ_FALLBACK_KEY, &fallback_eq_value) .unwrap(), ); }); @@ -796,11 +829,12 @@ fn bench_property_indexes(c: &mut Criterion) { query_group.bench_function("range_declared", |b| { let (_dir, engine) = build_property_query_engine(); + let label = bench_node_label(1); b.iter(|| { black_box( engine .find_nodes_range( - 1, + &label, PROPERTY_RANGE_DECLARED_KEY, Some(&declared_range_lower), Some(&declared_range_upper), @@ -812,11 +846,12 @@ fn bench_property_indexes(c: &mut Criterion) { query_group.bench_function("range_fallback_scan", |b| { let (_dir, engine) = build_property_query_engine(); + let label = bench_node_label(1); b.iter(|| { black_box( engine .find_nodes_range( - 1, + &label, PROPERTY_RANGE_FALLBACK_KEY, Some(&fallback_range_lower), Some(&fallback_range_upper), @@ -828,21 +863,22 @@ fn bench_property_indexes(c: &mut Criterion) { query_group.bench_function("mixed_declared_and_fallback", |b| { let (_dir, engine) = build_property_query_engine(); + let label = bench_node_label(1); b.iter(|| { black_box( engine - .find_nodes(1, PROPERTY_EQ_DECLARED_KEY, &declared_eq_value) + .find_nodes(&label, PROPERTY_EQ_DECLARED_KEY, &declared_eq_value) .unwrap(), ); black_box( engine - .find_nodes(1, PROPERTY_EQ_FALLBACK_KEY, &fallback_eq_value) + .find_nodes(&label, PROPERTY_EQ_FALLBACK_KEY, &fallback_eq_value) .unwrap(), ); black_box( engine .find_nodes_range( - 1, + &label, PROPERTY_RANGE_DECLARED_KEY, Some(&declared_range_lower), Some(&declared_range_upper), @@ -852,7 +888,7 @@ fn bench_property_indexes(c: &mut Criterion) { black_box( engine .find_nodes_range( - 1, + &label, PROPERTY_RANGE_FALLBACK_KEY, Some(&fallback_range_lower), Some(&fallback_range_upper), @@ -982,7 +1018,7 @@ fn bench_compact(c: &mut Criterion) { for seg in 0..5u64 { let node_inputs: Vec = (0..2000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("s{}_n{}", seg, i), props: make_bench_props(seg * 2000 + i), weight: 1.0, @@ -990,19 +1026,19 @@ fn bench_compact(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..400) .map(|i| EdgeInput { from: ids[i % 2000], to: ids[(i + 1) % 2000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); engine.flush().unwrap(); } (dir, engine) @@ -1030,7 +1066,7 @@ fn bench_compact(c: &mut Criterion) { for seg in 0..5u64 { let node_inputs: Vec = (0..2000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("s{}_n{}", seg, i), props: make_bench_props(seg * 2000 + i), weight: 1.0, @@ -1038,19 +1074,19 @@ fn bench_compact(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..400) .map(|i| EdgeInput { from: ids[i % 2000], to: ids[(i + 1) % 2000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); engine.flush().unwrap(); } engine @@ -1059,7 +1095,7 @@ fn bench_compact(c: &mut Criterion) { PrunePolicy { max_age_ms: None, max_weight: Some(0.0), - type_id: Some(u32::MAX), + label: Some("NoopFastMergeBlocker".to_string()), }, ) .unwrap(); @@ -1088,7 +1124,7 @@ fn bench_compact(c: &mut Criterion) { for seg in 0..5u64 { let node_inputs: Vec = (0..2000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props: make_bench_props(seg * 2000 + i), weight: 1.0, @@ -1096,19 +1132,19 @@ fn bench_compact(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..400) .map(|i| EdgeInput { from: ids[i % 2000], to: ids[(i + 1) % 2000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); engine.flush().unwrap(); } (dir, engine) @@ -1135,7 +1171,7 @@ fn bench_compact(c: &mut Criterion) { for seg in 0..5u64 { let node_inputs: Vec = (0..2000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("s{}_n{}", seg, i), props: make_bench_props(seg * 2000 + i), weight: 1.0, @@ -1143,19 +1179,19 @@ fn bench_compact(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..400) .map(|i| EdgeInput { from: ids[i % 2000], to: ids[(i + 1) % 2000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); engine.flush().unwrap(); } // Delete ~20% of nodes → creates tombstones @@ -1193,7 +1229,11 @@ fn bench_group_commit(c: &mut Criterion) { let mut i = 0u64; b.iter(|| { engine - .upsert_node(1, &format!("imm_{}", i), UpsertNodeOptions::default()) + .upsert_node( + "BenchNode", + &format!("imm_{}", i), + UpsertNodeOptions::default(), + ) .unwrap(); i += 1; }); @@ -1214,7 +1254,11 @@ fn bench_group_commit(c: &mut Criterion) { let mut i = 0u64; b.iter(|| { engine - .upsert_node(1, &format!("gc_{}", i), UpsertNodeOptions::default()) + .upsert_node( + "BenchNode", + &format!("gc_{}", i), + UpsertNodeOptions::default(), + ) .unwrap(); i += 1; }); @@ -1235,7 +1279,7 @@ fn bench_group_commit(c: &mut Criterion) { b.iter(|| { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("imm_b{}_{}", batch_num, i), props: BTreeMap::new(), weight: 1.0, @@ -1243,7 +1287,7 @@ fn bench_group_commit(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); batch_num += 1; }); engine.close().unwrap(); @@ -1263,7 +1307,7 @@ fn bench_group_commit(c: &mut Criterion) { b.iter(|| { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("gc_b{}_{}", batch_num, i), props: BTreeMap::new(), weight: 1.0, @@ -1271,7 +1315,7 @@ fn bench_group_commit(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); batch_num += 1; }); engine.close().unwrap(); @@ -1284,7 +1328,7 @@ fn bench_group_commit(c: &mut Criterion) { fn build_multi_hub_graph(engine: &mut DatabaseEngine) -> Vec { let node_inputs: Vec = (0..1100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: if i < 100 { format!("hub{}", i) } else { @@ -1296,7 +1340,7 @@ fn build_multi_hub_graph(engine: &mut DatabaseEngine) -> Vec { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let hub_ids: Vec = ids[..100].to_vec(); let mut edge_inputs = Vec::with_capacity(1000); for h in 0..100 { @@ -1304,7 +1348,7 @@ fn build_multi_hub_graph(engine: &mut DatabaseEngine) -> Vec { edge_inputs.push(EdgeInput { from: ids[h], to: ids[100 + h * 10 + i], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1312,7 +1356,7 @@ fn build_multi_hub_graph(engine: &mut DatabaseEngine) -> Vec { }); } } - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); hub_ids } @@ -1334,7 +1378,7 @@ fn bench_degree(c: &mut Criterion) { }); }); - c.bench_function("degree_fanout_100_type_filtered", |b| { + c.bench_function("degree_fanout_100_edge_label_filtered", |b| { let (_dir, mut engine) = temp_db(); let hub = build_hub_graph(&mut engine, 100); b.iter(|| { @@ -1342,7 +1386,7 @@ fn bench_degree(c: &mut Criterion) { .degree( hub, &DegreeOptions { - type_filter: Some(vec![1]), + edge_label_filter: Some(vec!["BenchEdge1".to_string()]), ..Default::default() }, ) @@ -1350,7 +1394,7 @@ fn bench_degree(c: &mut Criterion) { }); }); - c.bench_function("degree_fanout_100_segment_type_filtered", |b| { + c.bench_function("degree_fanout_100_segment_edge_label_filtered", |b| { let (_dir, mut engine) = temp_db(); let hub = build_hub_graph(&mut engine, 100); engine.flush().unwrap(); @@ -1359,7 +1403,7 @@ fn bench_degree(c: &mut Criterion) { .degree( hub, &DegreeOptions { - type_filter: Some(vec![1]), + edge_label_filter: Some(vec!["BenchEdge1".to_string()]), ..Default::default() }, ) @@ -1435,7 +1479,7 @@ fn bench_advanced_queries(c: &mut Criterion) { let build_2hop_graph = |engine: &mut DatabaseEngine| -> u64 { let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "root".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1444,7 +1488,7 @@ fn bench_advanced_queries(c: &mut Criterion) { }]; for i in 0..100u64 { node_inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("mid_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -1453,7 +1497,7 @@ fn bench_advanced_queries(c: &mut Criterion) { }); for j in 0..10u64 { node_inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("leaf_{}_{}", i, j), props: BTreeMap::new(), weight: 1.0, @@ -1462,7 +1506,7 @@ fn bench_advanced_queries(c: &mut Criterion) { }); } } - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let root = ids[0]; let mut edge_inputs = Vec::new(); for i in 0..100usize { @@ -1470,7 +1514,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edge_inputs.push(EdgeInput { from: root, to: mid, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1481,7 +1525,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edge_inputs.push(EdgeInput { from: mid, to: leaf, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1489,7 +1533,7 @@ fn bench_advanced_queries(c: &mut Criterion) { }); } } - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); root }; @@ -1535,7 +1579,7 @@ fn bench_advanced_queries(c: &mut Criterion) { let level3 = 4usize; let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "root".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1544,7 +1588,7 @@ fn bench_advanced_queries(c: &mut Criterion) { }]; for i in 0..level1 { node_inputs.push(NodeInput { - type_id: 11, + labels: vec![bench_node_label(11)], key: format!("lvl1_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -1555,7 +1599,7 @@ fn bench_advanced_queries(c: &mut Criterion) { for i in 0..level1 { for j in 0..level2 { node_inputs.push(NodeInput { - type_id: if (i + j) % 2 == 0 { 2 } else { 3 }, + labels: vec![bench_node_label(if (i + j) % 2 == 0 { 2 } else { 3 })], key: format!("lvl2_{}_{}", i, j), props: BTreeMap::new(), weight: 1.0, @@ -1568,7 +1612,11 @@ fn bench_advanced_queries(c: &mut Criterion) { for j in 0..level2 { for k in 0..level3 { node_inputs.push(NodeInput { - type_id: if (i + j + k) % 2 == 0 { 2 } else { 3 }, + labels: vec![bench_node_label(if (i + j + k) % 2 == 0 { + 2 + } else { + 3 + })], key: format!("lvl3_{}_{}_{}", i, j, k), props: BTreeMap::new(), weight: 1.0, @@ -1579,7 +1627,7 @@ fn bench_advanced_queries(c: &mut Criterion) { } } - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let root = ids[0]; let level1_offset = 1usize; let level2_offset = level1_offset + level1; @@ -1590,7 +1638,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edge_inputs.push(EdgeInput { from: root, to: lvl1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1602,7 +1650,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edge_inputs.push(EdgeInput { from: lvl1, to: lvl2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1613,7 +1661,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edge_inputs.push(EdgeInput { from: lvl2, to: ids[level3_offset + lvl3_idx], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1622,7 +1670,7 @@ fn bench_advanced_queries(c: &mut Criterion) { } } } - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); (root, level1, level2, level3) }; @@ -1647,9 +1695,17 @@ fn bench_advanced_queries(c: &mut Criterion) { }); }); - let filtered_types = [2u32]; + let filtered_label_ids = [2u32]; + let filtered_node_label_filter = || NodeLabelFilter { + labels: filtered_label_ids + .iter() + .copied() + .map(bench_node_label) + .collect(), + mode: LabelMatchMode::Any, + }; - group.bench_function("traverse_depth_1_to_3_filtered_type2_24x4x4", |b| { + group.bench_function("traverse_depth_1_to_3_filtered_label2_24x4x4", |b| { let (_dir, mut engine) = temp_db(); let (root, _, _, _) = build_layered_traversal_graph(&mut engine); b.iter(|| { @@ -1658,7 +1714,7 @@ fn bench_advanced_queries(c: &mut Criterion) { root, 3, &TraverseOptions { - node_type_filter: Some(filtered_types.to_vec()), + emit_node_label_filter: Some(filtered_node_label_filter()), ..Default::default() }, ) @@ -1666,27 +1722,30 @@ fn bench_advanced_queries(c: &mut Criterion) { }); }); - group.bench_function("traverse_depth_1_to_3_filtered_type2_24x4x4_segment", |b| { - let (_dir, mut engine) = temp_db(); - let (root, _, _, _) = build_layered_traversal_graph(&mut engine); - engine.flush().unwrap(); - b.iter(|| { - engine - .traverse( - root, - 3, - &TraverseOptions { - node_type_filter: Some(filtered_types.to_vec()), - ..Default::default() - }, - ) - .unwrap(); - }); - }); + group.bench_function( + "traverse_depth_1_to_3_filtered_label2_24x4x4_segment", + |b| { + let (_dir, mut engine) = temp_db(); + let (root, _, _, _) = build_layered_traversal_graph(&mut engine); + engine.flush().unwrap(); + b.iter(|| { + engine + .traverse( + root, + 3, + &TraverseOptions { + emit_node_label_filter: Some(filtered_node_label_filter()), + ..Default::default() + }, + ) + .unwrap(); + }); + }, + ); let build_topk_graph = |engine: &mut DatabaseEngine| -> u64 { let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1695,7 +1754,7 @@ fn bench_advanced_queries(c: &mut Criterion) { }]; for i in 0..1000u64 { node_inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("tk_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -1703,20 +1762,20 @@ fn bench_advanced_queries(c: &mut Criterion) { sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let hub = ids[0]; let edge_inputs: Vec = (0..1000) .map(|i| EdgeInput { from: hub, to: ids[1 + i], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0 + (i as u64 % 100) as f32 / 10.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); hub }; @@ -1749,7 +1808,7 @@ fn bench_advanced_queries(c: &mut Criterion) { - 10_000; let inputs: Vec = (0..10_000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("{}_{}", prefix, i), props: BTreeMap::new(), weight: 1.0, @@ -1757,7 +1816,7 @@ fn bench_advanced_queries(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); let to_ms = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -1769,8 +1828,11 @@ fn bench_advanced_queries(c: &mut Criterion) { group.bench_function("find_nodes_by_time_range_10000", |b| { let (_dir, mut engine) = temp_db(); let (from_ms, to_ms) = build_time_range_graph(&mut engine, "ts"); + let label = bench_node_label(1); b.iter(|| { - engine.find_nodes_by_time_range(1, from_ms, to_ms).unwrap(); + engine + .find_nodes_by_time_range(&label, from_ms, to_ms) + .unwrap(); }); }); @@ -1778,8 +1840,11 @@ fn bench_advanced_queries(c: &mut Criterion) { let (_dir, mut engine) = temp_db(); let (from_ms, to_ms) = build_time_range_graph(&mut engine, "ts"); engine.flush().unwrap(); + let label = bench_node_label(1); b.iter(|| { - engine.find_nodes_by_time_range(1, from_ms, to_ms).unwrap(); + engine + .find_nodes_by_time_range(&label, from_ms, to_ms) + .unwrap(); }); }); @@ -1790,9 +1855,10 @@ fn bench_advanced_queries(c: &mut Criterion) { limit: Some(100), after: None, }; + let label = bench_node_label(1); b.iter(|| { engine - .find_nodes_by_time_range_paged(1, from_ms, to_ms, &page) + .find_nodes_by_time_range_paged(&label, from_ms, to_ms, &page) .unwrap(); }); }); @@ -1807,9 +1873,10 @@ fn bench_advanced_queries(c: &mut Criterion) { limit: Some(100), after: None, }; + let label = bench_node_label(1); b.iter(|| { engine - .find_nodes_by_time_range_paged(1, from_ms, to_ms, &page) + .find_nodes_by_time_range_paged(&label, from_ms, to_ms, &page) .unwrap(); }); }, @@ -1818,7 +1885,7 @@ fn bench_advanced_queries(c: &mut Criterion) { let build_ppr_graph = |engine: &mut DatabaseEngine| -> Vec { let node_inputs: Vec = (0..2000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("ppr_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -1826,14 +1893,14 @@ fn bench_advanced_queries(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let node_ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let node_ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..2000usize) .flat_map(|i| { [ EdgeInput { from: node_ids[i], to: node_ids[(i + 1) % 2000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1842,7 +1909,7 @@ fn bench_advanced_queries(c: &mut Criterion) { EdgeInput { from: node_ids[i], to: node_ids[(i + 7) % 2000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.7, valid_from: None, @@ -1851,7 +1918,7 @@ fn bench_advanced_queries(c: &mut Criterion) { ] }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); node_ids }; @@ -1885,7 +1952,7 @@ fn bench_advanced_queries(c: &mut Criterion) { let build_ppr_graph_50k = |engine: &mut DatabaseEngine| -> Vec { let node_inputs: Vec = (0..50_000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("ppr50k_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -1893,14 +1960,14 @@ fn bench_advanced_queries(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let node_ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let node_ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..50_000usize) .flat_map(|i| { [ EdgeInput { from: node_ids[i], to: node_ids[(i + 1) % 50_000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1909,7 +1976,7 @@ fn bench_advanced_queries(c: &mut Criterion) { EdgeInput { from: node_ids[i], to: node_ids[(i + 7) % 50_000], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.7, valid_from: None, @@ -1918,7 +1985,7 @@ fn bench_advanced_queries(c: &mut Criterion) { ] }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); node_ids }; @@ -1956,7 +2023,7 @@ fn bench_advanced_queries(c: &mut Criterion) { let node_inputs: Vec = (0..TOTAL_NODES as u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("pprch20k_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -1964,7 +2031,7 @@ fn bench_advanced_queries(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let node_ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let node_ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let global_hubs: Vec = (0..8usize) .map(|community| community * COMMUNITY_SIZE) @@ -1984,7 +2051,7 @@ fn bench_advanced_queries(c: &mut Criterion) { EdgeInput { from: node_ids[i], to: node_ids[community_base + ((local + 1) % COMMUNITY_SIZE)], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1993,7 +2060,7 @@ fn bench_advanced_queries(c: &mut Criterion) { EdgeInput { from: node_ids[i], to: node_ids[community_base + ((local + 7) % COMMUNITY_SIZE)], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.9, valid_from: None, @@ -2002,7 +2069,7 @@ fn bench_advanced_queries(c: &mut Criterion) { EdgeInput { from: node_ids[i], to: node_ids[community_base + ((local + 31) % COMMUNITY_SIZE)], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.8, valid_from: None, @@ -2011,7 +2078,7 @@ fn bench_advanced_queries(c: &mut Criterion) { EdgeInput { from: node_ids[i], to: node_ids[community_base + ((local * 73 + 19) % COMMUNITY_SIZE)], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.7, valid_from: None, @@ -2023,7 +2090,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edges.push(EdgeInput { from: node_ids[i], to: node_ids[community_hub], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.1, valid_from: None, @@ -2035,7 +2102,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edges.push(EdgeInput { from: node_ids[i], to: node_ids[next_community_base + ((local * 17 + 11) % COMMUNITY_SIZE)], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.35, valid_from: None, @@ -2047,7 +2114,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edges.push(EdgeInput { from: node_ids[i], to: node_ids[prev_community_base + ((local * 29 + 5) % COMMUNITY_SIZE)], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.3, valid_from: None, @@ -2059,7 +2126,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edges.push(EdgeInput { from: node_ids[i], to: node_ids[global_hub], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.25, valid_from: None, @@ -2068,7 +2135,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edges.push(EdgeInput { from: node_ids[i], to: node_ids[next_community_base], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.45, valid_from: None, @@ -2082,7 +2149,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edges.push(EdgeInput { from: node_ids[i], to: node_ids[next_global_hub], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.6, valid_from: None, @@ -2093,7 +2160,7 @@ fn bench_advanced_queries(c: &mut Criterion) { edges }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); node_ids }; @@ -2244,7 +2311,7 @@ fn bench_advanced_queries(c: &mut Criterion) { let build_export_graph = |engine: &mut DatabaseEngine| -> Vec { let node_inputs: Vec = (0..5000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("ex_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -2252,7 +2319,7 @@ fn bench_advanced_queries(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let node_ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let node_ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let edge_inputs: Vec = (0..20_000usize) .filter_map(|i| { let from = node_ids[i % 5000]; @@ -2261,7 +2328,7 @@ fn bench_advanced_queries(c: &mut Criterion) { Some(EdgeInput { from, to, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2272,7 +2339,7 @@ fn bench_advanced_queries(c: &mut Criterion) { } }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); node_ids }; @@ -2310,7 +2377,7 @@ fn bench_recovery(c: &mut Criterion) { for batch in 0..10u64 { let inputs: Vec = (0..500u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("wal_{}", batch * 500 + i), props: BTreeMap::new(), weight: 1.0, @@ -2318,7 +2385,7 @@ fn bench_recovery(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); } engine.close().unwrap(); dir @@ -2339,7 +2406,7 @@ fn bench_recovery(c: &mut Criterion) { for seg in 0..3u64 { let inputs: Vec = (0..2000u64) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("seg{}_{}", seg, i), props: BTreeMap::new(), weight: 1.0, @@ -2347,7 +2414,7 @@ fn bench_recovery(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); } engine.close().unwrap(); @@ -2378,7 +2445,7 @@ fn build_ring_graph(n: usize) -> (tempfile::TempDir, DatabaseEngine, Vec) { let inputs: Vec = (0..n) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("sp_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -2386,7 +2453,7 @@ fn build_ring_graph(n: usize) -> (tempfile::TempDir, DatabaseEngine, Vec) { sparse_vector: None, }) .collect(); - let node_ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let node_ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let edges: Vec = (0..n) .flat_map(|i| { @@ -2396,7 +2463,7 @@ fn build_ring_graph(n: usize) -> (tempfile::TempDir, DatabaseEngine, Vec) { EdgeInput { from, to: node_ids[(i + 1) % n], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight, valid_from: None, @@ -2405,7 +2472,7 @@ fn build_ring_graph(n: usize) -> (tempfile::TempDir, DatabaseEngine, Vec) { EdgeInput { from, to: node_ids[(i + 7) % n], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight, valid_from: None, @@ -2414,7 +2481,7 @@ fn build_ring_graph(n: usize) -> (tempfile::TempDir, DatabaseEngine, Vec) { ] }) .collect(); - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); (dir, engine, node_ids) } @@ -2627,7 +2694,7 @@ fn bench_shortest_path(c: &mut Criterion) { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("d_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -2635,14 +2702,14 @@ fn bench_shortest_path(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let mut diamond_edges = Vec::new(); for &l1 in &ids[1..20] { diamond_edges.push(EdgeInput { from: ids[0], to: l1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2652,7 +2719,7 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l1, to: l2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2664,14 +2731,14 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l2, to: ids[99], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&diamond_edges).unwrap(); + engine.batch_upsert_edges(diamond_edges.clone()).unwrap(); let from = ids[0]; let to = ids[99]; @@ -2700,7 +2767,7 @@ fn bench_shortest_path(c: &mut Criterion) { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("d_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -2708,14 +2775,14 @@ fn bench_shortest_path(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let mut diamond_edges = Vec::new(); for &l1 in &ids[1..20] { diamond_edges.push(EdgeInput { from: ids[0], to: l1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2725,7 +2792,7 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l1, to: l2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2737,14 +2804,14 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l2, to: ids[99], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&diamond_edges).unwrap(); + engine.batch_upsert_edges(diamond_edges.clone()).unwrap(); engine.flush().unwrap(); let from = ids[0]; @@ -2774,7 +2841,7 @@ fn bench_shortest_path(c: &mut Criterion) { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("wd_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -2782,14 +2849,14 @@ fn bench_shortest_path(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let mut diamond_edges = Vec::new(); for &l1 in &ids[1..20] { diamond_edges.push(EdgeInput { from: ids[0], to: l1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2799,7 +2866,7 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l1, to: l2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2811,14 +2878,14 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l2, to: ids[99], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&diamond_edges).unwrap(); + engine.batch_upsert_edges(diamond_edges.clone()).unwrap(); let from = ids[0]; let to = ids[99]; @@ -2848,7 +2915,7 @@ fn bench_shortest_path(c: &mut Criterion) { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("wd_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -2856,14 +2923,14 @@ fn bench_shortest_path(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); let mut diamond_edges = Vec::new(); for &l1 in &ids[1..20] { diamond_edges.push(EdgeInput { from: ids[0], to: l1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2873,7 +2940,7 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l1, to: l2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -2885,14 +2952,14 @@ fn bench_shortest_path(c: &mut Criterion) { diamond_edges.push(EdgeInput { from: l2, to: ids[99], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&diamond_edges).unwrap(); + engine.batch_upsert_edges(diamond_edges.clone()).unwrap(); engine.flush().unwrap(); let from = ids[0]; @@ -2930,15 +2997,33 @@ fn bench_batch_get_by_keys(c: &mut Criterion) { let keys: Vec<(u32, String)> = (0..1000).map(|i| (1u32, format!("key_{:04}", i))).collect(); for (tid, k) in &keys { engine - .upsert_node(*tid, k, UpsertNodeOptions::default()) + .upsert_node(&bench_node_label(*tid), k, UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); // Prepare query slices - let keys_10: Vec<(u32, &str)> = keys[..10].iter().map(|(t, k)| (*t, k.as_str())).collect(); - let keys_100: Vec<(u32, &str)> = keys[..100].iter().map(|(t, k)| (*t, k.as_str())).collect(); - let keys_1000: Vec<(u32, &str)> = keys.iter().map(|(t, k)| (*t, k.as_str())).collect(); + let keys_10: Vec = keys[..10] + .iter() + .map(|(t, k)| NodeKeyQuery { + label: bench_node_label(*t), + key: k.clone(), + }) + .collect(); + let keys_100: Vec = keys[..100] + .iter() + .map(|(t, k)| NodeKeyQuery { + label: bench_node_label(*t), + key: k.clone(), + }) + .collect(); + let keys_1000: Vec = keys + .iter() + .map(|(t, k)| NodeKeyQuery { + label: bench_node_label(*t), + key: k.clone(), + }) + .collect(); group.bench_function("batch_10", |b| { b.iter(|| engine.get_nodes_by_keys(&keys_10).unwrap()); @@ -2951,8 +3036,8 @@ fn bench_batch_get_by_keys(c: &mut Criterion) { }); group.bench_function("loop_100_single", |b| { b.iter(|| { - for &(tid, key) in &keys_100 { - engine.get_node_by_key(tid, key).unwrap(); + for query in &keys_100 { + engine.get_node_by_key(&query.label, &query.key).unwrap(); } }); }); @@ -2966,7 +3051,7 @@ fn txn_intents(batch_num: u64, node_count: usize, edge_count: usize) -> Vec Vec = (0..8) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("txn_existing_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -3017,14 +3102,14 @@ fn bench_write_txn(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let existing_ids = engine.batch_upsert_nodes(&existing_nodes).unwrap(); + let existing_ids = engine.batch_upsert_nodes(existing_nodes.clone()).unwrap(); let mut batch_num = 0u64; b.iter(|| { let mut intents = Vec::with_capacity(16); for i in 0..8 { intents.push(TxnIntent::UpsertNode { alias: None, - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("txn_existing_{}", i), options: UpsertNodeOptions { weight: 1.0 + (batch_num % 100) as f32, @@ -3037,7 +3122,7 @@ fn bench_write_txn(c: &mut Criterion) { alias: None, from: TxnNodeRef::Id(existing_ids[i]), to: TxnNodeRef::Id(existing_ids[(i + 1) % 8]), - type_id: 7, + label: "BenchEdge7".to_string(), options: UpsertEdgeOptions { weight: 1.0 + (batch_num % 100) as f32, ..Default::default() @@ -3055,7 +3140,7 @@ fn bench_write_txn(c: &mut Criterion) { let (_dir, engine) = temp_db(); let existing_nodes: Vec = (0..8) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("patch_existing_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -3063,12 +3148,12 @@ fn bench_write_txn(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let existing_ids = engine.batch_upsert_nodes(&existing_nodes).unwrap(); + let existing_ids = engine.batch_upsert_nodes(existing_nodes.clone()).unwrap(); let mut batch_num = 0u64; b.iter(|| { let nodes: Vec = (0..8) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("patch_existing_{}", i), props: BTreeMap::new(), weight: 1.0 + (batch_num % 100) as f32, @@ -3080,7 +3165,7 @@ fn bench_write_txn(c: &mut Criterion) { .map(|i| EdgeInput { from: existing_ids[i % existing_ids.len()], to: existing_ids[(i + 1) % existing_ids.len()], - type_id: 7, + label: "BenchEdge7".to_string(), props: BTreeMap::new(), weight: 1.0 + (batch_num % 100) as f32, valid_from: None, @@ -3089,7 +3174,7 @@ fn bench_write_txn(c: &mut Criterion) { .collect(); black_box( engine - .graph_patch(&GraphPatch { + .graph_patch(GraphPatch { upsert_nodes: nodes, upsert_edges: edges, invalidate_edges: Vec::new(), @@ -3106,7 +3191,7 @@ fn bench_write_txn(c: &mut Criterion) { let (_dir, engine) = temp_db(); let nodes: Vec = (0..17) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("edge_existing_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -3114,7 +3199,7 @@ fn bench_write_txn(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&nodes).unwrap(); + let ids = engine.batch_upsert_nodes(nodes.clone()).unwrap(); let mut batch_num = 0u64; b.iter(|| { let intents: Vec = (0..16) @@ -3122,7 +3207,7 @@ fn bench_write_txn(c: &mut Criterion) { alias: None, from: TxnNodeRef::Id(ids[i]), to: TxnNodeRef::Id(ids[i + 1]), - type_id: 11, + label: "BenchEdge11".to_string(), options: UpsertEdgeOptions { weight: 1.0 + (batch_num % 100) as f32, ..Default::default() @@ -3140,7 +3225,7 @@ fn bench_write_txn(c: &mut Criterion) { let (_dir, engine) = temp_db(); let nodes: Vec = (0..17) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("patch_edge_existing_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -3148,14 +3233,14 @@ fn bench_write_txn(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&nodes).unwrap(); + let ids = engine.batch_upsert_nodes(nodes.clone()).unwrap(); let mut batch_num = 0u64; b.iter(|| { let edges: Vec = (0..16) .map(|i| EdgeInput { from: ids[i], to: ids[i + 1], - type_id: 11, + label: "BenchEdge11".to_string(), props: BTreeMap::new(), weight: 1.0 + (batch_num % 100) as f32, valid_from: None, @@ -3164,7 +3249,7 @@ fn bench_write_txn(c: &mut Criterion) { .collect(); black_box( engine - .graph_patch(&GraphPatch { + .graph_patch(GraphPatch { upsert_nodes: Vec::new(), upsert_edges: edges, invalidate_edges: Vec::new(), @@ -3180,13 +3265,13 @@ fn bench_write_txn(c: &mut Criterion) { group.bench_function("conflict_update_same_key", |b| { let (_dir, engine) = temp_db(); engine - .upsert_node(1, "conflict", UpsertNodeOptions::default()) + .upsert_node("BenchNode", "conflict", UpsertNodeOptions::default()) .unwrap(); let mut i = 0u64; b.iter(|| { let mut txn = engine.begin_write_txn().unwrap(); txn.upsert_node( - 1, + "BenchNode", "conflict", UpsertNodeOptions { weight: 2.0, @@ -3196,7 +3281,7 @@ fn bench_write_txn(c: &mut Criterion) { .unwrap(); engine .upsert_node( - 1, + "BenchNode", "conflict", UpsertNodeOptions { weight: 3.0 + i as f32, diff --git a/benches/query_ops.rs b/benches/query_ops.rs index d461625..2f31543 100644 --- a/benches/query_ops.rs +++ b/benches/query_ops.rs @@ -1,8 +1,9 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use overgraph::{ - DatabaseEngine, DbOptions, Direction, EdgeInput, EdgePattern, GraphPatternQuery, - NodeFilterExpr, NodeInput, NodePattern, NodeQuery, PageRequest, PatternOrder, PropValue, - PropertyRangeBound, SecondaryIndexKind, SecondaryIndexRangeDomain, SecondaryIndexState, + DatabaseEngine, DbOptions, Direction, EdgeFilterExpr, EdgeInput, EdgePattern, EdgeQuery, + GraphPatternQuery, LabelMatchMode, NodeFilterExpr, NodeInput, NodeLabelFilter, NodePattern, + NodeQuery, PageRequest, PatternOrder, PropValue, PropertyRangeBound, SecondaryIndexKind, + SecondaryIndexRangeDomain, SecondaryIndexState, }; use std::collections::BTreeMap; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -12,7 +13,7 @@ const QUERY_NODES_PER_SEGMENT: usize = 5_000; const QUERY_MEMTABLE_TAIL_COUNT: usize = 5_000; const QUERY_LIMIT: usize = 100; const QUERY_LARGE_UNIVERSE_COUNT: usize = 25_000; -const QUERY_SMALL_TYPE_COUNT: usize = 128; +const QUERY_SMALL_LABEL_COUNT: usize = 128; const QUERY_LARGE_IN_VALUE_COUNT: usize = 512; macro_rules! filter_and { @@ -35,6 +36,7 @@ fn temp_db_with_edge_uniqueness(edge_uniqueness: bool) -> (tempfile::TempDir, Da ..DbOptions::default() }; let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + seed_bench_label_tokens(&engine); (dir, engine) } @@ -42,6 +44,27 @@ fn temp_db() -> (tempfile::TempDir, DatabaseEngine) { temp_db_with_edge_uniqueness(true) } +fn seed_bench_label_tokens(engine: &DatabaseEngine) { + for label_token_id in 1..=256 { + assert_eq!( + engine + .ensure_node_label(&bench_node_label(label_token_id)) + .unwrap(), + label_token_id + ); + assert_eq!( + engine + .ensure_edge_label(&format!("BenchEdge{label_token_id}")) + .unwrap(), + label_token_id + ); + } +} + +fn bench_node_label(label_token_id: u32) -> String { + format!("BenchNode{label_token_id}") +} + fn query_props(i: usize) -> BTreeMap { let mut props = BTreeMap::new(); props.insert( @@ -81,7 +104,7 @@ fn query_props(i: usize) -> BTreeMap { fn query_nodes(prefix: &str, start: usize, count: usize) -> Vec { (start..start + count) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("{prefix}-{i}"), props: query_props(i), weight: 1.0, @@ -91,10 +114,15 @@ fn query_nodes(prefix: &str, start: usize, count: usize) -> Vec { .collect() } -fn query_nodes_with_type(type_id: u32, prefix: &str, start: usize, count: usize) -> Vec { +fn query_nodes_with_label_id( + label_id: u32, + prefix: &str, + start: usize, + count: usize, +) -> Vec { (start..start + count) .map(|i| NodeInput { - type_id, + labels: vec![bench_node_label(label_id)], key: format!("{prefix}-{i}"), props: query_props(i), weight: 1.0, @@ -129,25 +157,51 @@ fn wait_for_property_index_state( } } +fn wait_for_edge_property_index_state( + engine: &DatabaseEngine, + index_id: u64, + expected_state: SecondaryIndexState, +) { + let deadline = Instant::now() + Duration::from_secs(60); + loop { + if engine + .list_edge_property_indexes() + .unwrap() + .into_iter() + .any(|info| info.index_id == index_id && info.state == expected_state) + { + return; + } + assert!( + Instant::now() < deadline, + "timed out waiting for edge property index {} to reach {:?}", + index_id, + expected_state + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + fn ensure_query_indexes(engine: &mut DatabaseEngine) { + let label = bench_node_label(1); let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index(&label, "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(engine, status.index_id, SecondaryIndexState::Ready); let tier = engine - .ensure_node_property_index(1, "tier", SecondaryIndexKind::Equality) + .ensure_node_property_index(&label, "tier", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(engine, tier.index_id, SecondaryIndexState::Ready); let tenant = engine - .ensure_node_property_index(1, "tenant", SecondaryIndexKind::Equality) + .ensure_node_property_index(&label, "tenant", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(engine, tenant.index_id, SecondaryIndexState::Ready); let score = engine .ensure_node_property_index( - 1, + &label, "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -168,13 +222,13 @@ fn load_query_mixed_sources(engine: &DatabaseEngine, prefix: &str) { for segment in 0..QUERY_SEGMENT_COUNT { let start = segment * QUERY_NODES_PER_SEGMENT; let nodes = query_nodes(prefix, start, QUERY_NODES_PER_SEGMENT); - engine.batch_upsert_nodes(&nodes).unwrap(); + engine.batch_upsert_nodes(nodes.clone()).unwrap(); engine.flush().unwrap(); } let tail_start = QUERY_SEGMENT_COUNT * QUERY_NODES_PER_SEGMENT; let tail_nodes = query_nodes(prefix, tail_start, QUERY_MEMTABLE_TAIL_COUNT); - engine.batch_upsert_nodes(&tail_nodes).unwrap(); + engine.batch_upsert_nodes(tail_nodes.clone()).unwrap(); } fn build_fallback_query_engine() -> (tempfile::TempDir, DatabaseEngine) { @@ -183,29 +237,32 @@ fn build_fallback_query_engine() -> (tempfile::TempDir, DatabaseEngine) { (dir, engine) } -fn build_small_type_universe_engine() -> (tempfile::TempDir, DatabaseEngine) { +fn build_small_label_universe_engine() -> (tempfile::TempDir, DatabaseEngine) { let (dir, engine) = temp_db(); - let filler = query_nodes_with_type(2, "large-universe", 0, QUERY_LARGE_UNIVERSE_COUNT); - engine.batch_upsert_nodes(&filler).unwrap(); + let filler = query_nodes_with_label_id(2, "large-universe", 0, QUERY_LARGE_UNIVERSE_COUNT); + engine.batch_upsert_nodes(filler.clone()).unwrap(); engine.flush().unwrap(); - let segment_small = query_nodes_with_type(1, "small-type", 0, QUERY_SMALL_TYPE_COUNT / 2); - engine.batch_upsert_nodes(&segment_small).unwrap(); + let segment_small = query_nodes_with_label_id(1, "small-label", 0, QUERY_SMALL_LABEL_COUNT / 2); + engine.batch_upsert_nodes(segment_small.clone()).unwrap(); engine.flush().unwrap(); - let memtable_small = query_nodes_with_type( + let memtable_small = query_nodes_with_label_id( 1, - "small-type", - QUERY_SMALL_TYPE_COUNT / 2, - QUERY_SMALL_TYPE_COUNT - QUERY_SMALL_TYPE_COUNT / 2, + "small-label", + QUERY_SMALL_LABEL_COUNT / 2, + QUERY_SMALL_LABEL_COUNT - QUERY_SMALL_LABEL_COUNT / 2, ); - engine.batch_upsert_nodes(&memtable_small).unwrap(); + engine.batch_upsert_nodes(memtable_small.clone()).unwrap(); (dir, engine) } fn two_equality_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -223,7 +280,10 @@ fn two_equality_query(limit: Option) -> NodeQuery { fn equality_and_range_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -242,7 +302,10 @@ fn equality_and_range_query(limit: Option) -> NodeQuery { fn broad_equality_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![NodeFilterExpr::PropertyEquals { key: "status".to_string(), value: PropValue::String("inactive".to_string()), @@ -254,7 +317,10 @@ fn broad_equality_query(limit: Option) -> NodeQuery { fn broad_equality_and_selective_equality_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -272,7 +338,10 @@ fn broad_equality_and_selective_equality_query(limit: Option) -> NodeQuer fn broad_equality_and_selective_range_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -291,7 +360,10 @@ fn broad_equality_and_selective_range_query(limit: Option) -> NodeQuery { fn range_stats_selective_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![NodeFilterExpr::PropertyRange { key: "score".to_string(), lower: Some(PropertyRangeBound::Included(PropValue::Int(7))), @@ -304,7 +376,10 @@ fn range_stats_selective_query(limit: Option) -> NodeQuery { fn range_stats_broad_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![NodeFilterExpr::PropertyRange { key: "score".to_string(), lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), @@ -325,7 +400,10 @@ fn now_millis_for_bench() -> i64 { fn timestamp_stats_recent_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![NodeFilterExpr::UpdatedAtRange { lower_ms: Some(now_millis_for_bench().saturating_sub(60_000)), upper_ms: None, @@ -337,7 +415,10 @@ fn timestamp_stats_recent_query(limit: Option) -> NodeQuery { fn timestamp_stats_broad_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![NodeFilterExpr::UpdatedAtRange { lower_ms: Some(0), upper_ms: Some(i64::MAX), @@ -347,9 +428,12 @@ fn timestamp_stats_broad_query(limit: Option) -> NodeQuery { } } -fn type_scan_fallback_query() -> NodeQuery { +fn label_scan_fallback_query() -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![NodeFilterExpr::PropertyEquals { key: "region".to_string(), value: PropValue::String("r03".to_string()), @@ -362,9 +446,12 @@ fn type_scan_fallback_query() -> NodeQuery { } } -fn type_scoped_verify_only_boolean_query() -> NodeQuery { +fn label_scoped_verify_only_boolean_query() -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::And(vec![ NodeFilterExpr::Or(vec![ NodeFilterExpr::PropertyEquals { @@ -411,7 +498,10 @@ fn score_at_least_filter(value: i64) -> NodeFilterExpr { fn boolean_or_union_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::Or(vec![ tenant_eq_filter("t07"), tenant_eq_filter("t11"), @@ -423,7 +513,10 @@ fn boolean_or_union_query(limit: Option) -> NodeQuery { fn boolean_in_union_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::PropertyIn { key: "tenant".to_string(), values: vec![ @@ -440,7 +533,10 @@ fn boolean_in_union_query(limit: Option) -> NodeQuery { fn boolean_and_or_range_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::And(vec![ NodeFilterExpr::Or(vec![ tenant_eq_filter("t91"), @@ -454,9 +550,12 @@ fn boolean_and_or_range_query(limit: Option) -> NodeQuery { } } -fn boolean_verify_only_type_fallback_query(limit: Option) -> NodeQuery { +fn boolean_verify_only_label_fallback_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::Or(vec![ tenant_eq_filter("t07"), NodeFilterExpr::PropertyMissing { @@ -475,7 +574,10 @@ fn boolean_large_in_verify_only_query(limit: Option) -> NodeQuery { values.push(PropValue::String("r03".to_string())); NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::PropertyIn { key: "region".to_string(), values, @@ -485,17 +587,23 @@ fn boolean_large_in_verify_only_query(limit: Option) -> NodeQuery { } } -fn type_only_query(limit: Option) -> NodeQuery { +fn label_only_query(limit: Option) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), page: PageRequest { limit, after: None }, ..Default::default() } } -fn type_with_large_explicit_ids_query(ids: Vec) -> NodeQuery { +fn label_with_large_explicit_ids_query(ids: Vec) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), ids, page: PageRequest { limit: Some(QUERY_LIMIT), @@ -505,9 +613,12 @@ fn type_with_large_explicit_ids_query(ids: Vec) -> NodeQuery { } } -fn type_with_large_keys_query(keys: Vec) -> NodeQuery { +fn label_with_large_keys_query(keys: Vec) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), keys, page: PageRequest { limit: Some(QUERY_LIMIT), @@ -564,7 +675,10 @@ fn explicit_ids_query(ids: &[u64]) -> NodeQuery { fn explicit_ids_and_selective_property_query(ids: &[u64]) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), ids: ids.to_vec(), filter: filter_and![NodeFilterExpr::PropertyEquals { key: "tenant".to_string(), @@ -594,17 +708,17 @@ fn bench_node_queries(c: &mut Criterion) { b.iter(|| black_box(engine.query_nodes(black_box(&query)).unwrap())); }); - group.bench_function("query_node_ids_type_scan_fallback", |b| { + group.bench_function("query_node_ids_label_scan_fallback", |b| { let (_dir, engine) = build_fallback_query_engine(); - let query = type_scan_fallback_query(); + let query = label_scan_fallback_query(); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); group.bench_function( - "query_node_ids_type_scoped_verify_only_boolean_filter", + "query_node_ids_label_scoped_verify_only_boolean_filter", |b| { let (_dir, engine) = build_fallback_query_engine(); - let query = type_scoped_verify_only_boolean_query(); + let query = label_scoped_verify_only_boolean_query(); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }, ); @@ -627,14 +741,14 @@ fn bench_node_queries(c: &mut Criterion) { b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); - group.bench_function("query_node_ids_boolean_verify_only_type_fallback", |b| { + group.bench_function("query_node_ids_boolean_verify_only_label_fallback", |b| { let (_dir, engine) = build_fallback_query_engine(); - let query = boolean_verify_only_type_fallback_query(Some(QUERY_LIMIT)); + let query = boolean_verify_only_label_fallback_query(Some(QUERY_LIMIT)); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); group.bench_function( - "query_node_ids_boolean_large_in_verify_only_type_fallback", + "query_node_ids_boolean_large_in_verify_only_label_fallback", |b| { let (_dir, engine) = build_fallback_query_engine(); let query = boolean_large_in_verify_only_query(Some(QUERY_LIMIT)); @@ -668,26 +782,26 @@ fn bench_node_queries(c: &mut Criterion) { b.iter(|| black_box(engine.query_nodes(black_box(&query)).unwrap())); }); - group.bench_function("query_node_ids_type_only", |b| { + group.bench_function("query_node_ids_label_only", |b| { let (_dir, engine) = build_fallback_query_engine(); - let query = type_only_query(Some(QUERY_LIMIT)); + let query = label_only_query(Some(QUERY_LIMIT)); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); - group.bench_function("query_node_ids_type_vs_large_explicit_ids", |b| { - let (_dir, engine) = build_small_type_universe_engine(); - let ids = (1..=(QUERY_LARGE_UNIVERSE_COUNT + QUERY_SMALL_TYPE_COUNT) as u64).collect(); - let query = type_with_large_explicit_ids_query(ids); + group.bench_function("query_node_ids_label_vs_large_explicit_ids", |b| { + let (_dir, engine) = build_small_label_universe_engine(); + let ids = (1..=(QUERY_LARGE_UNIVERSE_COUNT + QUERY_SMALL_LABEL_COUNT) as u64).collect(); + let query = label_with_large_explicit_ids_query(ids); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); - group.bench_function("query_node_ids_type_vs_large_keys", |b| { - let (_dir, engine) = build_small_type_universe_engine(); + group.bench_function("query_node_ids_label_vs_large_keys", |b| { + let (_dir, engine) = build_small_label_universe_engine(); let mut keys: Vec = (0..QUERY_LARGE_UNIVERSE_COUNT) .map(|i| format!("missing-key-{i}")) .collect(); - keys.extend((0..QUERY_SMALL_TYPE_COUNT).map(|i| format!("small-type-{i}"))); - let query = type_with_large_keys_query(keys); + keys.extend((0..QUERY_SMALL_LABEL_COUNT).map(|i| format!("small-label-{i}"))); + let query = label_with_large_keys_query(keys); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); @@ -700,7 +814,7 @@ fn bench_node_queries(c: &mut Criterion) { group.bench_function("query_node_ids_explicit_ids_verify", |b| { let (_dir, engine) = build_indexed_query_engine(); let nodes = query_nodes("explicit", 0, 512); - let ids = engine.batch_upsert_nodes(&nodes).unwrap(); + let ids = engine.batch_upsert_nodes(nodes.clone()).unwrap(); let query = explicit_ids_query(&ids); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); @@ -797,7 +911,8 @@ fn bench_node_queries(c: &mut Criterion) { group.bench_function("query_node_ids_large_explicit_ids_selective_index", |b| { let (_dir, engine) = build_indexed_query_engine(); - let ids = engine.nodes_by_type(1).unwrap(); + let label = bench_node_label(1); + let ids = engine.nodes_by_labels(&label).unwrap(); let query = explicit_ids_and_selective_property_query(&ids); b.iter(|| black_box(engine.query_node_ids(black_box(&query)).unwrap())); }); @@ -837,15 +952,423 @@ fn bench_node_queries(c: &mut Criterion) { group.finish(); } +fn edge_query_props(i: usize) -> BTreeMap { + let mut props = BTreeMap::new(); + props.insert( + "role".to_string(), + PropValue::String( + if i.is_multiple_of(10) { + "lead" + } else { + "member" + } + .to_string(), + ), + ); + props.insert("score".to_string(), PropValue::Int((i % 100) as i64)); + props +} + +fn build_edge_query_engine() -> (tempfile::TempDir, DatabaseEngine, u64, Vec, i64) { + let (dir, engine) = temp_db(); + let edge_count = QUERY_NODES_PER_SEGMENT + QUERY_MEMTABLE_TAIL_COUNT; + let valid_epoch = 1_700_000_000_100i64; + let mut nodes = Vec::with_capacity(edge_count + 1); + nodes.push(NodeInput { + labels: vec![bench_node_label(1)], + key: "edge-query-source".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }); + nodes.extend((0..edge_count).map(|i| NodeInput { + labels: vec![bench_node_label(2)], + key: format!("edge-query-target-{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + })); + let node_ids = engine.batch_upsert_nodes(nodes.clone()).unwrap(); + let source_id = node_ids[0]; + let target_ids = &node_ids[1..]; + let make_edges = |start: usize, count: usize| -> Vec { + (start..start + count) + .map(|i| EdgeInput { + from: source_id, + to: target_ids[i], + label: "BenchEdge10".to_string(), + props: edge_query_props(i), + weight: if i.is_multiple_of(2) { 2.0 } else { 0.5 }, + valid_from: Some(1_700_000_000_000), + valid_to: Some(1_700_000_010_000), + }) + .collect() + }; + + let mut edge_ids = engine + .batch_upsert_edges(make_edges(0, QUERY_NODES_PER_SEGMENT)) + .unwrap(); + engine.flush().unwrap(); + edge_ids.extend( + engine + .batch_upsert_edges(make_edges( + QUERY_NODES_PER_SEGMENT, + QUERY_MEMTABLE_TAIL_COUNT, + )) + .unwrap(), + ); + (dir, engine, source_id, edge_ids, valid_epoch) +} + +fn build_edge_query_indexed_engine() -> (tempfile::TempDir, DatabaseEngine, u64, Vec, i64) { + let (dir, engine, source_id, edge_ids, valid_epoch) = build_edge_query_engine(); + let label = "BenchEdge10".to_string(); + let role = engine + .ensure_edge_property_index(&label, "role", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, role.index_id, SecondaryIndexState::Ready); + let score = engine + .ensure_edge_property_index( + &label, + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_edge_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + (dir, engine, source_id, edge_ids, valid_epoch) +} + +fn edge_query_with_filter(source_id: u64, filter: Option) -> EdgeQuery { + EdgeQuery { + label: Some("BenchEdge10".to_string()), + from_ids: vec![source_id], + filter, + page: PageRequest { + limit: Some(QUERY_LIMIT), + after: None, + }, + ..Default::default() + } +} + +fn edge_pattern_filter_query(source_id: u64, filter: Option) -> GraphPatternQuery { + GraphPatternQuery { + nodes: vec![ + NodePattern { + alias: "source".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), + ids: vec![source_id], + keys: Vec::new(), + filter: None, + }, + NodePattern { + alias: "target".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::All, + }), + ids: Vec::new(), + keys: Vec::new(), + filter: None, + }, + ], + edges: vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["BenchEdge10".to_string()], + filter, + }], + at_epoch: None, + limit: QUERY_LIMIT, + order: PatternOrder::AnchorThenAliasesAsc, + } +} + +fn edge_property_anchor_pattern_query() -> GraphPatternQuery { + GraphPatternQuery { + nodes: vec![ + NodePattern { + alias: "source".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), + ids: Vec::new(), + keys: Vec::new(), + filter: None, + }, + NodePattern { + alias: "target".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::All, + }), + ids: Vec::new(), + keys: Vec::new(), + filter: None, + }, + ], + edges: vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["BenchEdge10".to_string()], + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("lead".to_string()), + }), + }], + at_epoch: None, + limit: QUERY_LIMIT, + order: PatternOrder::AnchorThenAliasesAsc, + } +} + +fn bench_edge_queries(c: &mut Criterion) { + let mut group = c.benchmark_group("query_edge_planner"); + group.sample_size(20); + + group.bench_function("query_edge_ids_explicit_ids", |b| { + let (_dir, engine, _source_id, edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = EdgeQuery { + ids: edge_ids.iter().take(512).copied().collect(), + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(1.0), + upper: None, + }), + page: PageRequest { + limit: Some(QUERY_LIMIT), + after: None, + }, + ..Default::default() + }; + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_label_only", |b| { + let (_dir, engine, _source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = EdgeQuery { + label: Some("BenchEdge10".to_string()), + page: PageRequest { + limit: Some(QUERY_LIMIT), + after: None, + }, + ..Default::default() + }; + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_from_endpoint_label", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter(source_id, None); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_endpoint_list_label", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = EdgeQuery { + label: Some("BenchEdge10".to_string()), + endpoint_ids: vec![source_id], + page: PageRequest { + limit: Some(QUERY_LIMIT), + after: None, + }, + ..Default::default() + }; + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_weight_range", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::WeightRange { + lower: Some(1.0), + upper: None, + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_updated_at_range", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::UpdatedAtRange { + lower_ms: Some(0), + upper_ms: None, + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_valid_at_endpoint", |b| { + let (_dir, engine, source_id, _edge_ids, valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::ValidAt { + epoch_ms: valid_epoch, + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_property_verifier_bounded", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("lead".to_string()), + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_property_indexed_equality", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_indexed_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("lead".to_string()), + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edge_ids_property_indexed_range", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_indexed_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(90))), + upper: None, + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + group.bench_function("query_edges_metadata_final_page", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::WeightRange { + lower: Some(1.0), + upper: None, + }), + ); + b.iter(|| black_box(engine.query_edges(black_box(&query)).unwrap())); + }); + + group.bench_function("query_pattern_edge_metadata_filter", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_pattern_filter_query( + source_id, + Some(EdgeFilterExpr::WeightRange { + lower: Some(1.0), + upper: None, + }), + ); + b.iter(|| black_box(engine.query_pattern(black_box(&query)).unwrap())); + }); + + group.bench_function("query_pattern_edge_property_filter", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_pattern_filter_query( + source_id, + Some(EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("lead".to_string()), + }), + ); + b.iter(|| black_box(engine.query_pattern(black_box(&query)).unwrap())); + }); + + group.bench_function("query_pattern_edge_property_anchor_indexed", |b| { + let (_dir, engine, _source_id, _edge_ids, _valid_epoch) = build_edge_query_indexed_engine(); + let query = edge_property_anchor_pattern_query(); + b.iter(|| black_box(engine.query_pattern(black_box(&query)).unwrap())); + }); + + group.finish(); + + let mut property_group = c.benchmark_group("edge_property_index_queries"); + property_group.sample_size(20); + + property_group.bench_function("equality_fallback_scan", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("lead".to_string()), + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + property_group.bench_function("equality_declared", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_indexed_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("lead".to_string()), + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + property_group.bench_function("range_fallback_scan", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(90))), + upper: None, + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + property_group.bench_function("range_declared", |b| { + let (_dir, engine, source_id, _edge_ids, _valid_epoch) = build_edge_query_indexed_engine(); + let query = edge_query_with_filter( + source_id, + Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(90))), + upper: None, + }), + ); + b.iter(|| black_box(engine.query_edge_ids(black_box(&query)).unwrap())); + }); + + property_group.finish(); +} + fn build_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { let (dir, mut engine) = temp_db(); ensure_query_indexes(&mut engine); let account_inputs = query_nodes("acct", 0, 1_000); - let account_ids = engine.batch_upsert_nodes(&account_inputs).unwrap(); + let account_ids = engine.batch_upsert_nodes(account_inputs.clone()).unwrap(); let companies: Vec = (0..200) .map(|i| NodeInput { - type_id: 2, + labels: vec![bench_node_label(2)], key: format!("company-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -853,7 +1376,7 @@ fn build_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { sparse_vector: None, }) .collect(); - let company_ids = engine.batch_upsert_nodes(&companies).unwrap(); + let company_ids = engine.batch_upsert_nodes(companies.clone()).unwrap(); let edges: Vec = account_ids .iter() @@ -861,14 +1384,14 @@ fn build_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { .map(|(i, &from)| EdgeInput { from, to: company_ids[i % company_ids.len()], - type_id: 10, + label: "BenchEdge10".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); engine.flush().unwrap(); (dir, engine, company_ids[0]) } @@ -878,7 +1401,10 @@ fn linear_pattern_query() -> GraphPatternQuery { nodes: vec![ NodePattern { alias: "person".to_string(), - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: filter_and![NodeFilterExpr::PropertyEquals { @@ -888,7 +1414,10 @@ fn linear_pattern_query() -> GraphPatternQuery { }, NodePattern { alias: "company".to_string(), - type_id: Some(2), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: None, @@ -899,8 +1428,8 @@ fn linear_pattern_query() -> GraphPatternQuery { from_alias: "person".to_string(), to_alias: "company".to_string(), direction: Direction::Outgoing, - type_filter: Some(vec![10]), - property_predicates: Vec::new(), + label_filter: vec!["BenchEdge10".to_string()], + filter: None, }], at_epoch: None, limit: QUERY_LIMIT, @@ -921,7 +1450,10 @@ fn branching_pattern_query(company_id: u64) -> GraphPatternQuery { let mut query = linear_pattern_query(); query.nodes.push(NodePattern { alias: "peer".to_string(), - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: filter_and![NodeFilterExpr::PropertyEquals { @@ -934,8 +1466,8 @@ fn branching_pattern_query(company_id: u64) -> GraphPatternQuery { from_alias: "peer".to_string(), to_alias: "company".to_string(), direction: Direction::Outgoing, - type_filter: Some(vec![10]), - property_predicates: Vec::new(), + label_filter: vec!["BenchEdge10".to_string()], + filter: None, }); query.nodes[1].ids = vec![company_id]; query @@ -944,8 +1476,8 @@ fn branching_pattern_query(company_id: u64) -> GraphPatternQuery { fn build_high_fanout_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { let (dir, engine) = temp_db(); let source = engine - .batch_upsert_nodes(&[NodeInput { - type_id: 1, + .batch_upsert_nodes(vec![NodeInput { + labels: vec![bench_node_label(1)], key: "fanout-source".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -955,7 +1487,7 @@ fn build_high_fanout_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64 .unwrap()[0]; let targets: Vec = (0..5_000) .map(|i| NodeInput { - type_id: 2, + labels: vec![bench_node_label(2)], key: format!("fanout-target-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -963,20 +1495,20 @@ fn build_high_fanout_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64 sparse_vector: None, }) .collect(); - let target_ids = engine.batch_upsert_nodes(&targets).unwrap(); + let target_ids = engine.batch_upsert_nodes(targets.clone()).unwrap(); let edges: Vec = target_ids .iter() .map(|&target| EdgeInput { from: source, to: target, - type_id: 10, + label: "BenchEdge10".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); engine.flush().unwrap(); (dir, engine, source) } @@ -984,8 +1516,8 @@ fn build_high_fanout_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64 fn build_fanout_anchor_choice_engine() -> (tempfile::TempDir, DatabaseEngine) { let (dir, engine) = temp_db(); let hub = engine - .batch_upsert_nodes(&[NodeInput { - type_id: 1, + .batch_upsert_nodes(vec![NodeInput { + labels: vec![bench_node_label(1)], key: "small-hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -995,7 +1527,7 @@ fn build_fanout_anchor_choice_engine() -> (tempfile::TempDir, DatabaseEngine) { .unwrap()[0]; let mid_inputs: Vec<_> = (0..500) .map(|index| NodeInput { - type_id: 3, + labels: vec![bench_node_label(3)], key: format!("mid-{index:03}"), props: BTreeMap::new(), weight: 1.0, @@ -1003,10 +1535,10 @@ fn build_fanout_anchor_choice_engine() -> (tempfile::TempDir, DatabaseEngine) { sparse_vector: None, }) .collect(); - let mids = engine.batch_upsert_nodes(&mid_inputs).unwrap(); + let mids = engine.batch_upsert_nodes(mid_inputs.clone()).unwrap(); let anchor_inputs: Vec<_> = (0..32) .map(|index| NodeInput { - type_id: 2, + labels: vec![bench_node_label(2)], key: format!("anchor-{index:02}"), props: BTreeMap::new(), weight: 1.0, @@ -1014,13 +1546,13 @@ fn build_fanout_anchor_choice_engine() -> (tempfile::TempDir, DatabaseEngine) { sparse_vector: None, }) .collect(); - let anchors = engine.batch_upsert_nodes(&anchor_inputs).unwrap(); + let anchors = engine.batch_upsert_nodes(anchor_inputs.clone()).unwrap(); let mut edges = Vec::new(); for &mid in &mids { edges.push(EdgeInput { from: hub, to: mid, - type_id: 10, + label: "BenchEdge10".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1031,14 +1563,14 @@ fn build_fanout_anchor_choice_engine() -> (tempfile::TempDir, DatabaseEngine) { edges.push(EdgeInput { from: *mid, to: *anchor, - type_id: 20, + label: "BenchEdge20".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); engine.flush().unwrap(); (dir, engine) } @@ -1048,21 +1580,30 @@ fn fanout_anchor_choice_query() -> GraphPatternQuery { nodes: vec![ NodePattern { alias: "small_hub".to_string(), - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: None, }, NodePattern { alias: "larger_anchor".to_string(), - type_id: Some(2), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: None, }, NodePattern { alias: "middle".to_string(), - type_id: Some(3), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(3)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: None, @@ -1074,16 +1615,16 @@ fn fanout_anchor_choice_query() -> GraphPatternQuery { from_alias: "small_hub".to_string(), to_alias: "middle".to_string(), direction: Direction::Outgoing, - type_filter: Some(vec![10]), - property_predicates: Vec::new(), + label_filter: vec!["BenchEdge10".to_string()], + filter: None, }, EdgePattern { alias: Some("middle_to_anchor".to_string()), from_alias: "middle".to_string(), to_alias: "larger_anchor".to_string(), direction: Direction::Outgoing, - type_filter: Some(vec![20]), - property_predicates: Vec::new(), + label_filter: vec!["BenchEdge20".to_string()], + filter: None, }, ], at_epoch: None, @@ -1095,8 +1636,8 @@ fn fanout_anchor_choice_query() -> GraphPatternQuery { fn build_high_hub_delay_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { let (dir, engine) = temp_db(); let root = engine - .batch_upsert_nodes(&[NodeInput { - type_id: 1, + .batch_upsert_nodes(vec![NodeInput { + labels: vec![bench_node_label(1)], key: "root".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1105,8 +1646,8 @@ fn build_high_hub_delay_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { }]) .unwrap()[0]; let low = engine - .batch_upsert_nodes(&[NodeInput { - type_id: 3, + .batch_upsert_nodes(vec![NodeInput { + labels: vec![bench_node_label(3)], key: "low".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1116,7 +1657,7 @@ fn build_high_hub_delay_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { .unwrap()[0]; let target_inputs: Vec<_> = (0..512) .map(|index| NodeInput { - type_id: 2, + labels: vec![bench_node_label(2)], key: format!("hub-target-{index:03}"), props: BTreeMap::new(), weight: 1.0, @@ -1124,11 +1665,11 @@ fn build_high_hub_delay_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { sparse_vector: None, }) .collect(); - let targets = engine.batch_upsert_nodes(&target_inputs).unwrap(); + let targets = engine.batch_upsert_nodes(target_inputs.clone()).unwrap(); let mut edges = vec![EdgeInput { from: root, to: low, - type_id: 20, + label: "BenchEdge20".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1138,14 +1679,14 @@ fn build_high_hub_delay_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { edges.push(EdgeInput { from: root, to: target, - type_id: 10, + label: "BenchEdge10".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); engine.flush().unwrap(); (dir, engine, root) } @@ -1155,21 +1696,30 @@ fn high_hub_delay_query(root: u64) -> GraphPatternQuery { nodes: vec![ NodePattern { alias: "root".to_string(), - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), ids: vec![root], keys: Vec::new(), filter: None, }, NodePattern { alias: "hub_target".to_string(), - type_id: Some(2), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: None, }, NodePattern { alias: "low_target".to_string(), - type_id: Some(3), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(3)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: None, @@ -1181,16 +1731,16 @@ fn high_hub_delay_query(root: u64) -> GraphPatternQuery { from_alias: "root".to_string(), to_alias: "hub_target".to_string(), direction: Direction::Outgoing, - type_filter: Some(vec![10]), - property_predicates: Vec::new(), + label_filter: vec!["BenchEdge10".to_string()], + filter: None, }, EdgePattern { alias: Some("zzz_low".to_string()), from_alias: "root".to_string(), to_alias: "low_target".to_string(), direction: Direction::Outgoing, - type_filter: Some(vec![20]), - property_predicates: Vec::new(), + label_filter: vec!["BenchEdge20".to_string()], + filter: None, }, ], at_epoch: None, @@ -1202,8 +1752,8 @@ fn high_hub_delay_query(root: u64) -> GraphPatternQuery { fn build_parallel_edge_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u64) { let (dir, engine) = temp_db_with_edge_uniqueness(false); let source = engine - .batch_upsert_nodes(&[NodeInput { - type_id: 1, + .batch_upsert_nodes(vec![NodeInput { + labels: vec![bench_node_label(1)], key: "parallel-source".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1212,8 +1762,8 @@ fn build_parallel_edge_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u }]) .unwrap()[0]; let target = engine - .batch_upsert_nodes(&[NodeInput { - type_id: 2, + .batch_upsert_nodes(vec![NodeInput { + labels: vec![bench_node_label(2)], key: "parallel-target".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1225,14 +1775,14 @@ fn build_parallel_edge_pattern_engine() -> (tempfile::TempDir, DatabaseEngine, u .map(|_| EdgeInput { from: source, to: target, - type_id: 10, + label: "BenchEdge10".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges.clone()).unwrap(); engine.flush().unwrap(); (dir, engine, source) } @@ -1242,14 +1792,20 @@ fn unnamed_edge_constraint_query(source_id: u64) -> GraphPatternQuery { nodes: vec![ NodePattern { alias: "source".to_string(), - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), ids: vec![source_id], keys: Vec::new(), filter: None, }, NodePattern { alias: "target".to_string(), - type_id: Some(2), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::All, + }), ids: Vec::new(), keys: Vec::new(), filter: None, @@ -1260,8 +1816,8 @@ fn unnamed_edge_constraint_query(source_id: u64) -> GraphPatternQuery { from_alias: "source".to_string(), to_alias: "target".to_string(), direction: Direction::Outgoing, - type_filter: Some(vec![10]), - property_predicates: Vec::new(), + label_filter: vec!["BenchEdge10".to_string()], + filter: None, }], at_epoch: None, limit: QUERY_LIMIT, @@ -1318,5 +1874,10 @@ fn bench_pattern_queries(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_node_queries, bench_pattern_queries); +criterion_group!( + benches, + bench_node_queries, + bench_pattern_queries, + bench_edge_queries +); criterion_main!(benches); diff --git a/benches/vector_ops.rs b/benches/vector_ops.rs index 00b3144..6b6dad4 100644 --- a/benches/vector_ops.rs +++ b/benches/vector_ops.rs @@ -1,16 +1,46 @@ use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion}; use overgraph::{ DatabaseEngine, DbOptions, DenseMetric, DenseVectorConfig, Direction, EdgeInput, FusionMode, - HnswConfig, NodeInput, VectorSearchMode, VectorSearchRequest, VectorSearchScope, + HnswConfig, LabelMatchMode, NodeInput, NodeLabelFilter, VectorSearchMode, VectorSearchRequest, + VectorSearchScope, }; use std::collections::BTreeMap; fn temp_db_with_opts(opts: DbOptions) -> (tempfile::TempDir, DatabaseEngine) { let dir = tempfile::tempdir().unwrap(); let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + seed_bench_label_tokens(&engine); (dir, engine) } +fn seed_bench_label_tokens(engine: &DatabaseEngine) { + for label_token_id in 1..=32 { + assert_eq!( + engine + .ensure_node_label(&bench_node_label(label_token_id)) + .unwrap(), + label_token_id + ); + assert_eq!( + engine + .ensure_edge_label(&format!("BenchEdge{label_token_id}")) + .unwrap(), + label_token_id + ); + } +} + +fn bench_node_label(label_token_id: u32) -> String { + format!("BenchNode{label_token_id}") +} + +fn bench_node_label_filter(label_token_id: u32) -> NodeLabelFilter { + NodeLabelFilter { + labels: vec![bench_node_label(label_token_id)], + mode: LabelMatchMode::Any, + } +} + fn dense_bench_config(dimension: u32) -> DenseVectorConfig { DenseVectorConfig { dimension, @@ -25,7 +55,7 @@ fn dense_query_request(query: Vec, k: usize) -> VectorSearchRequest { dense_query: Some(query), sparse_query: None, k, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -40,7 +70,7 @@ fn sparse_query_request(query: Vec<(u32, f32)>, k: usize) -> VectorSearchRequest dense_query: None, sparse_query: Some(query), k, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -49,17 +79,17 @@ fn sparse_query_request(query: Vec<(u32, f32)>, k: usize) -> VectorSearchRequest } } -fn sparse_query_request_with_type_filter( +fn sparse_query_request_with_label_filter( query: Vec<(u32, f32)>, k: usize, - type_filter: Option>, + label_filter: Option, ) -> VectorSearchRequest { VectorSearchRequest { mode: VectorSearchMode::Sparse, dense_query: None, sparse_query: Some(query), k, - type_filter, + label_filter, ef_search: None, scope: None, dense_weight: None, @@ -255,7 +285,7 @@ fn clustered_sparse_inputs( (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("sc{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -275,7 +305,7 @@ fn clustered_sparse_inputs( fn uniform_sparse_inputs(count: usize, dimension_count: u32, nnz: usize) -> Vec { (0..count) .map(|index| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("su{index}"), props: BTreeMap::new(), weight: 1.0, @@ -299,7 +329,7 @@ fn clustered_sparse_multisegment_inputs_a( for i in 0..count { let shared = clustered_sparse_vector(dimension_count, 3, i, cluster_count, nnz); inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("shared_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -307,7 +337,7 @@ fn clustered_sparse_multisegment_inputs_a( sparse_vector: Some(shared), }); inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("stable_a_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -321,8 +351,8 @@ fn clustered_sparse_multisegment_inputs_a( )), }); inputs.push(NodeInput { - type_id: 2, - key: format!("other_type_{i}"), + labels: vec![bench_node_label(2)], + key: format!("other_label_{i}"), props: BTreeMap::new(), weight: 1.0, dense_vector: None, @@ -348,7 +378,7 @@ fn clustered_sparse_multisegment_inputs_b( for i in 0..count { let shared = clustered_sparse_vector(dimension_count, 3, i + 50_000, cluster_count, nnz); inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("shared_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -356,7 +386,7 @@ fn clustered_sparse_multisegment_inputs_b( sparse_vector: Some(scale_sparse_vector(&shared, 1.15)), }); inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("stable_b_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -390,7 +420,7 @@ fn clustered_sparse_overlap_segment_inputs( nnz, ); inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("shared_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -401,7 +431,7 @@ fn clustered_sparse_overlap_segment_inputs( )), }); inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("stable_{segment_index}_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -428,7 +458,7 @@ fn bench_vector_non_vector_parity(c: &mut Criterion) { let (_dir, engine) = temp_db_with_opts(opts); let inputs: Vec = (0..2000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props: BTreeMap::new(), weight: 1.0, @@ -436,7 +466,7 @@ fn bench_vector_non_vector_parity(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let mut i = 0usize; b.iter(|| { @@ -456,7 +486,7 @@ fn bench_vector_non_vector_parity(c: &mut Criterion) { let (_dir, engine) = temp_db_with_opts(opts); let inputs: Vec = (0..2000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n{}", i), props: BTreeMap::new(), weight: 1.0, @@ -464,7 +494,7 @@ fn bench_vector_non_vector_parity(c: &mut Criterion) { sparse_vector: None, }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let mut i = 0usize; b.iter(|| { @@ -483,7 +513,7 @@ fn bench_vector_non_vector_parity(c: &mut Criterion) { let (_dir, engine) = temp_db_with_opts(opts); let plain_inputs: Vec = (0..1000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("plain{}", i), props: BTreeMap::new(), weight: 1.0, @@ -493,7 +523,7 @@ fn bench_vector_non_vector_parity(c: &mut Criterion) { .collect(); let sparse_inputs: Vec = (0..1000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("sparse{}", i), props: BTreeMap::new(), weight: 1.0, @@ -505,8 +535,8 @@ fn bench_vector_non_vector_parity(c: &mut Criterion) { )), }) .collect(); - let plain_ids = engine.batch_upsert_nodes(&plain_inputs).unwrap(); - engine.batch_upsert_nodes(&sparse_inputs).unwrap(); + let plain_ids = engine.batch_upsert_nodes(plain_inputs.clone()).unwrap(); + engine.batch_upsert_nodes(sparse_inputs.clone()).unwrap(); engine.flush().unwrap(); let mut i = 0usize; @@ -535,7 +565,7 @@ fn bench_vector_search_dense(c: &mut Criterion) { let inputs: Vec = (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("c{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -544,7 +574,7 @@ fn bench_vector_search_dense(c: &mut Criterion) { }) }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let request = dense_query_request(clustered_query_vector(64, 3, 7, cluster_count), 10); @@ -567,11 +597,11 @@ fn bench_vector_search_dense(c: &mut Criterion) { let cluster_count = 24usize; let points_per_cluster = 128usize; for segment_index in 0..3usize { - let segment_type = if segment_index == 0 { 1 } else { 2 }; + let segment_label = if segment_index == 0 { 1 } else { 2 }; let inputs: Vec = (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: segment_type, + labels: vec![bench_node_label(segment_label)], key: format!("seg{segment_index}_c{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -585,12 +615,12 @@ fn bench_vector_search_dense(c: &mut Criterion) { }) }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); } let mut request = dense_query_request(clustered_query_vector(64, 3, 9, cluster_count), 10); - request.type_filter = Some(vec![1]); + request.label_filter = Some(bench_node_label_filter(1)); b.iter(|| { black_box(engine.vector_search(black_box(&request)).unwrap()); @@ -610,11 +640,11 @@ fn bench_vector_search_dense(c: &mut Criterion) { let cluster_count = 24usize; let points_per_cluster = 128usize; for batch_index in 0..3usize { - let batch_type = if batch_index == 0 { 1 } else { 2 }; + let batch_label = if batch_index == 0 { 1 } else { 2 }; let inputs: Vec = (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: batch_type, + labels: vec![bench_node_label(batch_label)], key: format!("mt{batch_index}_c{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -628,11 +658,11 @@ fn bench_vector_search_dense(c: &mut Criterion) { }) }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); } let mut request = dense_query_request(clustered_query_vector(64, 3, 9, cluster_count), 10); - request.type_filter = Some(vec![1]); + request.label_filter = Some(bench_node_label_filter(1)); b.iter(|| { black_box(engine.vector_search(black_box(&request)).unwrap()); @@ -649,7 +679,7 @@ fn bench_vector_search_dense(c: &mut Criterion) { let (_dir, engine) = temp_db_with_opts(opts); let inputs: Vec = (0..5000) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("v{}", i), props: BTreeMap::new(), weight: 1.0, @@ -657,7 +687,7 @@ fn bench_vector_search_dense(c: &mut Criterion) { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let mut query = vec![0.0f32; 32]; @@ -687,7 +717,7 @@ fn bench_vector_search_sparse(c: &mut Criterion) { let cluster_count = 24usize; let points_per_cluster = 384usize; let inputs = clustered_sparse_inputs(cluster_count, points_per_cluster, 4096, 12); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let request = @@ -705,7 +735,7 @@ fn bench_vector_search_sparse(c: &mut Criterion) { }; let (_dir, engine) = temp_db_with_opts(opts); let inputs = uniform_sparse_inputs(9_216, 4096, 12); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let request = @@ -728,17 +758,17 @@ fn bench_vector_search_sparse(c: &mut Criterion) { let nnz = 12usize; let inputs_a = clustered_sparse_multisegment_inputs_a(1_536, dimension_count, cluster_count, nnz); - engine.batch_upsert_nodes(&inputs_a).unwrap(); + engine.batch_upsert_nodes(inputs_a.clone()).unwrap(); engine.flush().unwrap(); let inputs_b = clustered_sparse_multisegment_inputs_b(1_536, dimension_count, cluster_count, nnz); - engine.batch_upsert_nodes(&inputs_b).unwrap(); + engine.batch_upsert_nodes(inputs_b.clone()).unwrap(); engine.flush().unwrap(); - let request = sparse_query_request_with_type_filter( + let request = sparse_query_request_with_label_filter( clustered_sparse_query(dimension_count, 3, 11, cluster_count, nnz), 10, - Some(vec![1]), + Some(bench_node_label_filter(1)), ); b.iter(|| { black_box(engine.vector_search(black_box(&request)).unwrap()); @@ -767,13 +797,13 @@ fn bench_vector_search_sparse(c: &mut Criterion) { for input in &mut inputs_b { input.key = format!("mem_b_{}", input.key); } - engine.batch_upsert_nodes(&inputs_a).unwrap(); - engine.batch_upsert_nodes(&inputs_b).unwrap(); + engine.batch_upsert_nodes(inputs_a.clone()).unwrap(); + engine.batch_upsert_nodes(inputs_b.clone()).unwrap(); - let request = sparse_query_request_with_type_filter( + let request = sparse_query_request_with_label_filter( clustered_sparse_query(dimension_count, 3, 11, cluster_count, nnz), 10, - Some(vec![1]), + Some(bench_node_label_filter(1)), ); b.iter(|| { black_box(engine.vector_search(black_box(&request)).unwrap()); @@ -797,7 +827,7 @@ fn bench_sparse_build(c: &mut Criterion) { }; let (_dir, engine) = temp_db_with_opts(opts); let inputs = clustered_sparse_inputs(24, 384, 4096, 12); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); (_dir, engine) }, |(_dir, engine)| { @@ -828,7 +858,7 @@ fn bench_sparse_build(c: &mut Criterion) { cluster_count, nnz, ); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); } (_dir, engine) @@ -854,7 +884,7 @@ fn hybrid_query_request( dense_query: Some(dense), sparse_query: Some(sparse), k, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -873,7 +903,7 @@ fn clustered_hybrid_inputs( (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("h{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -910,7 +940,7 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { let cluster_count = 24usize; let points_per_cluster = 384usize; let inputs = clustered_hybrid_inputs(cluster_count, points_per_cluster, 64, 4096, 12); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let request = hybrid_query_request( @@ -936,7 +966,7 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { let cluster_count = 24usize; let points_per_cluster = 384usize; let inputs = clustered_hybrid_inputs(cluster_count, points_per_cluster, 64, 4096, 12); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); let request = hybrid_query_request( @@ -963,11 +993,11 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { let cluster_count = 24usize; let points_per_cluster = 128usize; for segment_index in 0..3usize { - let segment_type = if segment_index == 0 { 1 } else { 2 }; + let segment_label = if segment_index == 0 { 1 } else { 2 }; let inputs: Vec = (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: segment_type, + labels: vec![bench_node_label(segment_label)], key: format!("seg{segment_index}_h{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -987,7 +1017,7 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { }) }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); } @@ -997,7 +1027,7 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { 10, None, ); - request.type_filter = Some(vec![1]); + request.label_filter = Some(bench_node_label_filter(1)); b.iter(|| { black_box(engine.vector_search(black_box(&request)).unwrap()); @@ -1016,11 +1046,11 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { let cluster_count = 24usize; let points_per_cluster = 128usize; for batch_index in 0..3usize { - let batch_type = if batch_index == 0 { 1 } else { 2 }; + let batch_label = if batch_index == 0 { 1 } else { 2 }; let inputs: Vec = (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: batch_type, + labels: vec![bench_node_label(batch_label)], key: format!("mt{batch_index}_h{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -1040,7 +1070,7 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { }) }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); } let mut request = hybrid_query_request( @@ -1049,7 +1079,7 @@ fn bench_vector_search_hybrid(c: &mut Criterion) { 10, None, ); - request.type_filter = Some(vec![1]); + request.label_filter = Some(bench_node_label_filter(1)); b.iter(|| { black_box(engine.vector_search(black_box(&request)).unwrap()); @@ -1076,7 +1106,7 @@ fn bench_vector_search_scoped(c: &mut Criterion) { let total_spokes = cluster_count * spokes_per_cluster; let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1086,7 +1116,7 @@ fn bench_vector_search_scoped(c: &mut Criterion) { for cluster in 0..cluster_count { for member in 0..spokes_per_cluster { node_inputs.push(NodeInput { - type_id: 2, + labels: vec![bench_node_label(2)], key: format!("spoke_{cluster}_{member}"), props: BTreeMap::new(), weight: 1.0, @@ -1095,21 +1125,21 @@ fn bench_vector_search_scoped(c: &mut Criterion) { }); } } - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let hub_id = ids[0]; let edge_inputs: Vec = (0..total_spokes) .map(|i| EdgeInput { from: hub_id, to: ids[1 + i], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); engine.flush().unwrap(); let request = VectorSearchRequest { @@ -1117,13 +1147,13 @@ fn bench_vector_search_scoped(c: &mut Criterion) { dense_query: Some(clustered_query_vector(64, 3, 7, cluster_count)), sparse_query: None, k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: Some(VectorSearchScope { start_node_id: hub_id, max_depth: 1, direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, at_epoch: None, }), dense_weight: None, @@ -1149,7 +1179,7 @@ fn bench_vector_search_scoped(c: &mut Criterion) { let total_spokes = cluster_count * spokes_per_cluster; let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1159,7 +1189,7 @@ fn bench_vector_search_scoped(c: &mut Criterion) { for cluster in 0..cluster_count { for member in 0..spokes_per_cluster { node_inputs.push(NodeInput { - type_id: 2, + labels: vec![bench_node_label(2)], key: format!("spoke_{cluster}_{member}"), props: BTreeMap::new(), weight: 1.0, @@ -1174,21 +1204,21 @@ fn bench_vector_search_scoped(c: &mut Criterion) { }); } } - let ids = engine.batch_upsert_nodes(&node_inputs).unwrap(); + let ids = engine.batch_upsert_nodes(node_inputs.clone()).unwrap(); let hub_id = ids[0]; let edge_inputs: Vec = (0..total_spokes) .map(|i| EdgeInput { from: hub_id, to: ids[1 + i], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs.clone()).unwrap(); engine.flush().unwrap(); let request = VectorSearchRequest { @@ -1196,13 +1226,13 @@ fn bench_vector_search_scoped(c: &mut Criterion) { dense_query: Some(clustered_query_vector(64, 3, 7, cluster_count)), sparse_query: Some(clustered_sparse_query(4096, 3, 7, cluster_count, 12)), k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: Some(VectorSearchScope { start_node_id: hub_id, max_depth: 1, direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, at_epoch: None, }), dense_weight: None, @@ -1227,7 +1257,7 @@ fn clustered_dense_overlap_segment_inputs( let mut inputs = Vec::with_capacity(count * 2); for i in 0..count { inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("shared_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1240,7 +1270,7 @@ fn clustered_dense_overlap_segment_inputs( sparse_vector: None, }); inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("stable_{segment_index}_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1275,7 +1305,7 @@ fn bench_dense_build(c: &mut Criterion) { let inputs: Vec = (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("d{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -1289,7 +1319,7 @@ fn bench_dense_build(c: &mut Criterion) { }) }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); (_dir, engine) }, |(_dir, engine)| { @@ -1318,7 +1348,7 @@ fn bench_dense_build(c: &mut Criterion) { 64, cluster_count, ); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs.clone()).unwrap(); engine.flush().unwrap(); } (_dir, engine) diff --git a/docs/04-quality/Benchmark-Plan.md b/docs/04-quality/Benchmark-Plan.md index 5b4631e..b9319a4 100644 --- a/docs/04-quality/Benchmark-Plan.md +++ b/docs/04-quality/Benchmark-Plan.md @@ -93,8 +93,8 @@ Graph shapes for each profile: | `S-TRAV-006` | `is_connected` | required | required | required | | `S-TRAV-007` | `traverse` depth 1..3 layered (memtable, no filter) | required | required | required | | `S-TRAV-008` | `traverse` depth 1..3 layered (segmented, no filter) | required | required | required | -| `S-TRAV-009` | `traverse` depth 1..3 layered (memtable, emission-only `node_type_filter`) | required | required | required | -| `S-TRAV-010` | `traverse` depth 1..3 layered (segmented, emission-only `node_type_filter`) | required | required | required | +| `S-TRAV-009` | `traverse` depth 1..3 layered (memtable, emission-only `emit_node_label_filter`) | required | required | required | +| `S-TRAV-010` | `traverse` depth 1..3 layered (segmented, emission-only `emit_node_label_filter`) | required | required | required | | `S-ADV-001` | `top_k_neighbors` | required | required | required | | `S-ADV-003` | `find_nodes_by_time_range(_paged)` | required | required | required | | `S-ADV-004` | `personalized_pagerank` | required | required | required | diff --git a/docs/04-quality/Benchmark-Runner.md b/docs/04-quality/Benchmark-Runner.md index 78a1830..2755183 100644 --- a/docs/04-quality/Benchmark-Runner.md +++ b/docs/04-quality/Benchmark-Runner.md @@ -35,10 +35,13 @@ Files: - Node.js: `connector-benchmark-v3-parity` (shared-profile + shared scenario-contract harness) - Python: `connector-benchmark-v2-parity` (shared-profile + shared scenario-contract harness) - Phase 20b adds Criterion `write_txn/*` microbenches for explicit 4/16/64-intent commits, a 16-intent implicit batch comparator, and a same-key conflict-heavy workload. Connector benchmark harnesses should mirror this with ordered `stage(operations)` arrays for Node.js and Python. -- Phase 23 adds `query_ops` Criterion microbenchmarks and shared query-only parity scenarios: +- Phase 23 adds `query_ops` Criterion microbenchmarks and shared query-only parity scenarios. + Phase 26 extends the same query set with direct edge query parity: - `S-QUERY-001` / `query_node_ids_intersected_predicates` - `S-QUERY-002` / `query_nodes_intersected_predicates_hydrated` - Run only the cross-language Phase 23 matrix with: + - `S-QUERY-003` / `query_edge_ids_endpoint_metadata` + - `S-QUERY-004` / `query_edges_endpoint_property_hydrated` + Run only the cross-language query matrix with: `scripts/bench/run-rust.sh --scenario-set query --profile small --warmup 20 --iters 80` plus the matching Node.js and Python wrapper commands. diff --git a/docs/04-quality/reports/README.md b/docs/04-quality/reports/README.md index af8cfad..e347b9e 100644 --- a/docs/04-quality/reports/README.md +++ b/docs/04-quality/reports/README.md @@ -24,6 +24,10 @@ Additional report assets: Historical naming note: - Pre-Phase 18c benchmark artifacts may label `S-TRAV-002` as `neighbors_2hop`. - That scenario now maps to `traverse_depth_2`, meaning `traverse(start, min_depth=2, max_depth=2, ...)`. +- Pre-CP30.5 archived run payloads may include old numeric-token field names in + `scenario_params`. They are immutable historical benchmark artifacts and are excluded + from current public API vocabulary audits; new runs should use label-ID metadata where + numeric labels are recorded. Regression comparison outputs are typically stored alongside runs or under: diff --git a/docs/04-quality/reports/baselines/README.md b/docs/04-quality/reports/baselines/README.md index 88f179a..6d84fa8 100644 --- a/docs/04-quality/reports/baselines/README.md +++ b/docs/04-quality/reports/baselines/README.md @@ -18,3 +18,7 @@ Notes: - CI workflow uses `docs/04-quality/reports/baselines/gha-ubuntu-22.04//` by default. - CI now treats missing baseline files as an error (instead of silently skipping regression checks). - Small-profile baselines refreshed 2026-03-04 after BigInt→Number Node.js migration (commit 3789864). +- Example baseline snapshots are historical benchmark artifacts. Pre-CP30.5 snapshots may + preserve old numeric-token scenario metadata and are excluded from current public API + vocabulary audits; freshly generated baselines should use label-ID metadata where numeric + labels are recorded. diff --git a/docs/04-quality/workloads/README.md b/docs/04-quality/workloads/README.md index 26beb9a..61c7544 100644 --- a/docs/04-quality/workloads/README.md +++ b/docs/04-quality/workloads/README.md @@ -8,8 +8,8 @@ This directory defines the cross-language benchmark profile contract for Phase 1 ## Determinism Rules - Seed: `1729` -- Node key format: `node:{i}` -- Type IDs cycle through `[1, 2, 3, 4, 5]` +- Node key format: `node-{i}` +- Node labels cycle through `Person`, `Company`, `Document`, `Project`, and `Observation` - Weight formula: `0.5 + ((i mod 500) / 1000.0)` - Temporal edge fields use: - `valid_from = 1700000000000 + i` @@ -48,11 +48,16 @@ All benchmark harnesses (Rust, Node.js, Python) must load these fields consisten - percentile computation method declaration ## Query Scenario Set -Phase 23 adds a query-only scenario set for native planner parity: +Phase 23 adds a query-only scenario set for native planner parity. Phase 26 extends it with +planner-backed direct edge query scenarios: - `S-QUERY-001`: `query_node_ids_intersected_predicates` - `S-QUERY-002`: `query_nodes_intersected_predicates_hydrated` - -These scenarios benchmark planner intersection over existing single-source property indexes. They -must not be described as compound-index benchmarks; maintained compound/composite indexes remain a -separate follow-up. +- `S-QUERY-003`: `query_edge_ids_endpoint_metadata` +- `S-QUERY-004`: `query_edges_endpoint_property_hydrated` + +The node scenarios benchmark planner intersection over existing single-source property indexes. +They must not be described as compound-index benchmarks; maintained compound/composite indexes +remain a separate follow-up. The edge scenarios benchmark endpoint-bounded direct edge queries with +metadata-only verification and bounded property-filter hydration; Phase 26 does not include +maintained edge-property indexes. diff --git a/docs/04-quality/workloads/profiles.json b/docs/04-quality/workloads/profiles.json index 0280cbc..50afce7 100644 --- a/docs/04-quality/workloads/profiles.json +++ b/docs/04-quality/workloads/profiles.json @@ -2,7 +2,7 @@ "schema_version": 1, "determinism": { "seed": 1729, - "node_key_template": "node:{i}", + "node_key_template": "node-{i}", "edge_key_templates": { "chain": "edge:chain:{i}", "hub": "edge:hub:{i}", @@ -10,7 +10,7 @@ "temporal": "edge:temporal:{i}" }, "generator_rules": { - "type_id_cycle": [1, 2, 3, 4, 5], + "label_cycle": ["Person", "Company", "Document", "Project", "Observation"], "weight_base": 0.5, "weight_modulus": 500, "weight_formula": "weight = weight_base + ((i mod weight_modulus) / 1000.0)", diff --git a/docs/04-quality/workloads/scenario-contract.json b/docs/04-quality/workloads/scenario-contract.json index 5ba644e..7cc9f75 100644 --- a/docs/04-quality/workloads/scenario-contract.json +++ b/docs/04-quality/workloads/scenario-contract.json @@ -92,6 +92,36 @@ "iters_divisor": 2, "iters_min": 20 }, + "S-QUERY-003": { + "warmup_divisor": 2, + "warmup_min": 5, + "iters_divisor": 2, + "iters_min": 20 + }, + "S-QUERY-004": { + "warmup_divisor": 2, + "warmup_min": 5, + "iters_divisor": 2, + "iters_min": 20 + }, + "S-QUERY-005": { + "warmup_divisor": 2, + "warmup_min": 5, + "iters_divisor": 2, + "iters_min": 20 + }, + "S-QUERY-006": { + "warmup_divisor": 2, + "warmup_min": 5, + "iters_divisor": 2, + "iters_min": 20 + }, + "S-QUERY-007": { + "warmup_divisor": 2, + "warmup_min": 5, + "iters_divisor": 2, + "iters_min": 20 + }, "S-TRAV-003": { "iters_multiplier": 2 }, @@ -176,6 +206,21 @@ "S-QUERY-002": { "status": "comparable" }, + "S-QUERY-003": { + "status": "comparable" + }, + "S-QUERY-004": { + "status": "comparable" + }, + "S-QUERY-005": { + "status": "comparable" + }, + "S-QUERY-006": { + "status": "comparable" + }, + "S-QUERY-007": { + "status": "comparable" + }, "S-ADV-005": { "status": "comparable" }, diff --git a/docs/api-reference.md b/docs/api-reference.md index bb123c9..ee1ba28 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -18,20 +18,29 @@ Complete reference for OverGraph's public API across **Rust**, **Node.js**, and - [Database Lifecycle](#database-lifecycle) - [open](#open) - [close](#close) + - [close_fast](#close_fast) - [stats](#stats) - [Configuration](#configuration) - [DbOptions](#dboptions) - [WalSyncMode](#walsyncmode) - [DenseVectorConfig](#densevectorconfig) - [Data Model](#data-model) - - [NodeRecord](#noderecord) - - [EdgeRecord](#edgerecord) + - [Node Records](#node-records) + - [Edge Records](#edge-records) - [PropValue](#propvalue) + - [IntoNodeLabels](#intonodelabels-rust-only) - [Direction](#direction) + - [NodeLabelFilter / LabelMatchMode](#nodelabelfilter--labelmatchmode) +- [Catalog APIs](#catalog-apis) + - [ensure_node_label / ensure_edge_label](#ensure_node_label--ensure_edge_label) + - [get_node_label_id / get_edge_label_id](#get_node_label_id--get_edge_label_id) + - [get_node_label / get_edge_label](#get_node_label--get_edge_label) + - [list_node_labels / list_edge_labels](#list_node_labels--list_edge_labels) - [Node Operations](#node-operations) - [upsert_node](#upsert_node) - [get_node](#get_node) - [get_node_by_key](#get_node_by_key) + - [add_node_label / remove_node_label](#add_node_label--remove_node_label) - [delete_node](#delete_node) - [batch_upsert_nodes](#batch_upsert_nodes) - [get_nodes](#get_nodes) @@ -47,20 +56,25 @@ Complete reference for OverGraph's public API across **Rust**, **Node.js**, and - [Atomic Operations](#atomic-operations) - [graph_patch](#graph_patch) - [write transactions](#write-transactions) -- [Type-Based Queries](#type-based-queries) - - [nodes_by_type](#nodes_by_type) - - [edges_by_type](#edges_by_type) - - [get_nodes_by_type](#get_nodes_by_type) - - [get_edges_by_type](#get_edges_by_type) - - [count_nodes_by_type](#count_nodes_by_type) - - [count_edges_by_type](#count_edges_by_type) +- [Label and Edge-Label Queries](#label-and-edge-label-queries) + - [nodes_by_labels](#nodes_by_labels) + - [edges_by_label](#edges_by_label) + - [get_nodes_by_labels](#get_nodes_by_labels) + - [get_edges_by_label](#get_edges_by_label) + - [count_nodes_by_labels](#count_nodes_by_labels) + - [count_edges_by_label](#count_edges_by_label) - [Property Index Management](#property-index-management) - [ensure_node_property_index](#ensure_node_property_index) - [drop_node_property_index](#drop_node_property_index) - [list_node_property_indexes](#list_node_property_indexes) - [NodePropertyIndexInfo](#nodepropertyindexinfo) + - [ensure_edge_property_index](#ensure_edge_property_index) + - [drop_edge_property_index](#drop_edge_property_index) + - [list_edge_property_indexes](#list_edge_property_indexes) + - [EdgePropertyIndexInfo](#edgepropertyindexinfo) - [PropertyRangeBound](#propertyrangebound) - [PropertyRangeCursor](#propertyrangecursor) + - [PropertyRangePageRequest](#propertyrangepagerequest-rust-only) - [PropertyRangePageResult](#propertyrangepageresult) - [Property & Time Queries](#property--time-queries) - [find_nodes](#find_nodes) @@ -71,20 +85,26 @@ Complete reference for OverGraph's public API across **Rust**, **Node.js**, and - [query_node_ids](#query_node_ids) - [query_nodes](#query_nodes) - [explain_node_query](#explain_node_query) + - [Direct Edge Queries](#direct-edge-queries) + - [query_edge_ids](#query_edge_ids) + - [query_edges](#query_edges) + - [explain_edge_query](#explain_edge_query) - [Graph Pattern Queries](#graph-pattern-queries) - [query_pattern](#query_pattern) - [explain_pattern_query](#explain_pattern_query) - [Query Request Types and Plans](#query-request-types-and-plans) - [NodeQuery](#nodequery) - [NodeFilter / QueryNodeFilter](#nodefilter--querynodefilter) + - [EdgeQuery](#edgequery) + - [EdgeFilter / QueryEdgeFilter](#edgefilter--queryedgefilter) - [GraphPatternQuery](#graphpatternquery) - [QueryPlan](#queryplan) - [Validation notes](#validation-notes) - [Pagination](#pagination) - - [nodes_by_type_paged](#nodes_by_type_paged) - - [edges_by_type_paged](#edges_by_type_paged) - - [get_nodes_by_type_paged](#get_nodes_by_type_paged) - - [get_edges_by_type_paged](#get_edges_by_type_paged) + - [nodes_by_labels_paged](#nodes_by_labels_paged) + - [edges_by_label_paged](#edges_by_label_paged) + - [get_nodes_by_labels_paged](#get_nodes_by_labels_paged) + - [get_edges_by_label_paged](#get_edges_by_label_paged) - [find_nodes_paged](#find_nodes_paged) - [find_nodes_range_paged](#find_nodes_range_paged) - [find_nodes_by_time_range_paged](#find_nodes_by_time_range_paged) @@ -122,10 +142,18 @@ Complete reference for OverGraph's public API across **Rust**, **Node.js**, and - [compact_with_progress](#compact_with_progress) - [ingest_mode](#ingest_mode) - [end_ingest](#end_ingest) + - [scrub](#scrub) - [Introspection](#introspection) - [node_count](#node_count) - [edge_count](#edge_count) + - [next_node_id](#next_node_id) + - [next_edge_id](#next_edge_id) - [segment_count](#segment_count) + - [segment_tombstone_node_count](#segment_tombstone_node_count) + - [segment_tombstone_edge_count](#segment_tombstone_edge_count) + - [path](#path) + - [manifest](#manifest) + - [manifest::load_manifest_readonly](#manifestload_manifest_readonly-rust-only) - [Binary Batch Ingestion](#binary-batch-ingestion) - [batch_upsert_nodes_binary](#batch_upsert_nodes_binary) - [batch_upsert_edges_binary](#batch_upsert_edges_binary) @@ -139,7 +167,7 @@ Complete reference for OverGraph's public API across **Rust**, **Node.js**, and **Rust** - add to `Cargo.toml`: ```toml [dependencies] -overgraph = "0.4" +overgraph = "0.7" ``` **Node.js**: @@ -227,7 +255,7 @@ const db = OverGraph.open('./my-graph', { groupCommitIntervalMs: 50, edgeUniqueness: true, denseVector: { dimension: 384, metric: 'cosine' }, - compactAfterNFlushes: 5, + compactAfterNFlushes: 4, }); ``` @@ -240,7 +268,7 @@ db = OverGraph.open( edge_uniqueness=True, dense_vector_dimension=384, dense_vector_metric="cosine", - compact_after_n_flushes=5, + compact_after_n_flushes=4, ) ``` @@ -299,7 +327,8 @@ db.close(force=True) # cancels compaction **Python** supports context manager syntax: ```python with OverGraph.open("./my-graph") as db: - db.upsert_node(1, "alice") + # Also accepts multiple labels: ["User", "Admin"] + db.upsert_node("User", "alice") # db.close() called automatically on exit ``` @@ -307,13 +336,25 @@ with OverGraph.open("./my-graph") as db: --- +### close_fast + +Rust-only fast close. This is the same behavior exposed by `close({ force: true })` in Node.js and `close(force=True)` in Python. + +```rust +db.close_fast()?; +``` + +It cancels any in-progress background compaction, syncs the active WAL, and persists a manifest that retains the WAL generations needed for replay on the next open. + +--- + ### stats Returns a read-only snapshot of current database statistics. **Rust** ```rust -let s = db.stats(); +let s = db.stats()?; println!("segments: {}, WAL bytes: {}", s.segment_count, s.pending_wal_bytes); ``` @@ -366,7 +407,7 @@ Options passed to [`open()`](#open). All fields are optional with sensible defau | memtable_flush_threshold | `usize` | `memtableFlushThreshold` | `memtable_flush_threshold` | `134217728` (128 MB) | When the active memtable exceeds this size in bytes, it is sealed and queued for flush to a segment. | | memtable_hard_cap_bytes | `usize` | `memtableHardCapBytes` | `memtable_hard_cap_bytes` | `536870912` (512 MB) | Writes block when the active memtable exceeds this size and the flush queue is full. Prevents unbounded memory growth under heavy write load. Set to `0` to disable. | | max_immutable_memtables | `usize` | `maxImmutableMemtables` | `max_immutable_memtables` | `4` | Maximum number of sealed memtables allowed before the flush thread must drain one. Controls memory usage under write bursts. | -| edge_uniqueness | `bool` | `edgeUniqueness` | `edge_uniqueness` | `false` | When `true`, `upsert_edge` enforces at most one edge per `(from, to, type_id)` triple. An upsert with the same triple updates the existing edge. When `false`, every `upsert_edge` call creates a new edge. | +| edge_uniqueness | `bool` | `edgeUniqueness` | `edge_uniqueness` | `false` | When `true`, `upsert_edge` enforces at most one edge per `(from, to, label)` triple. An upsert with the same triple updates the existing edge. When `false`, every `upsert_edge` call creates a new edge. | | compact_after_n_flushes | `u32` | `compactAfterNFlushes` | `compact_after_n_flushes` | `4` | Trigger background compaction after this many flushes. Set to `0` to disable auto-compaction. | | dense_vector | `Option` | `denseVector` | See below | `None` | Enable dense vector search. See [DenseVectorConfig](#densevectorconfig). In Python, use separate kwargs: `dense_vector_dimension` and `dense_vector_metric`. | @@ -379,7 +420,9 @@ Controls the trade-off between durability and write throughput. | Immediate | `WalSyncMode::Immediate` | `"immediate"` | `"immediate"` | Every write triggers an `fsync`. Maximum crash safety. Data is durable before the write call returns. Lower throughput (~4ms per write on typical SSDs). | | GroupCommit | `WalSyncMode::GroupCommit { .. }` | `"group-commit"` | `"group_commit"` | Writes are buffered and fsynced on a timer or when the buffer fills. Higher throughput (batched fsync amortizes the cost across many writes). A crash can lose at most one group-commit interval of writes. | -**GroupCommit parameters** (Rust only; Node.js/Python expose these as top-level options): +Current Node.js connector parsing treats unknown `walSyncMode` strings as group commit. Python validates `wal_sync_mode` and rejects unknown strings. + +**GroupCommit parameters** (Node.js/Python expose these as top-level options): | Parameter | Type | Default | Description | |-----------|------|---------|-------------| @@ -394,17 +437,19 @@ Configures the HNSW index for dense vector search. Set once at database creation | Parameter | Rust | Node.js | Python | Default | Description | |-----------|------|---------|--------|---------|-------------| | dimension | `u32` | `dimension: number` | `dense_vector_dimension: int` | — (required if enabling vectors) | Dimensionality of dense vectors. Every node's `dense_vector` must have exactly this many elements. | -| metric | `DenseMetric` | `metric: string` | `dense_vector_metric: str` | `Cosine` | Distance metric for similarity. One of: `Cosine`, `Euclidean`, `DotProduct`. | +| metric | `DenseMetric` | `metric: string` | `dense_vector_metric: str` | `Cosine` | Distance metric for similarity. Rust uses enum variants. Node.js and Python use lower-case strings. | **DenseMetric values:** -| Metric | Description | Score semantics | -|--------|-------------|-----------------| -| `Cosine` | Cosine similarity. Vectors are L2-normalized before comparison. | Higher = more similar (range: -1 to 1). | -| `Euclidean` | L2 (Euclidean) distance. | Lower = more similar (range: 0 to ∞). Results are returned as negative distance so higher scores remain "better." | -| `DotProduct` | Raw inner product. Useful when vectors are already normalized or when magnitude matters. | Higher = more similar. | +| Metric | Rust | Node.js / Python | Score semantics | +|--------|------|------------------|-----------------| +| Cosine | `DenseMetric::Cosine` | `"cosine"` | Higher = more similar (range: -1 to 1). | +| Euclidean | `DenseMetric::Euclidean` | `"euclidean"` | Lower distance is more similar. Results are returned as negative distance so higher scores remain "better." | +| Dot product | `DenseMetric::DotProduct` | `"dot_product"` | Higher = more similar. | + +Current Node.js and Python connector parsers fall back to cosine for unknown metric strings. -**HNSW parameters** (Rust only; Node.js/Python use defaults): +**HNSW parameters** (Node.js/Python use defaults): | Parameter | Type | Default | Description | |-----------|------|---------|-------------| @@ -415,57 +460,67 @@ Configures the HNSW index for dense vector search. Set once at database creation ## Data Model -### NodeRecord +### Node Records -A node stored in the graph. Returned by read operations. +A public, hydrated node record returned by read operations. | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| | id | `u64` | `number` | `int` | Unique, auto-assigned node ID. Monotonically increasing. | -| type_id | `u32` | `number` | `int` | User-defined type identifier. Use constants (e.g., `USER = 1`) for readability. | -| key | `String` | `string` | `str` | Unique key within the type. The `(type_id, key)` pair uniquely identifies a node. | +| labels | `Vec` | `string[]` | `list[str]` | Complete node label set. | +| key | `String` | `string` | `str` | Unique key within the node's label identity. Do not repeat the label in the key unless it is part of an external source ID. | | props | `BTreeMap` | `Record` | `dict[str, Any]` | User-defined properties. See [PropValue](#propvalue) for supported types. Lazily deserialized from MessagePack on first access. | | weight | `f32` | `number` | `float` | Numeric weight. Default `1.0`. Used by pruning policies and scoring algorithms. | | created_at | `i64` | `number` | `int` | Timestamp (ms) when the node was first created. | | updated_at | `i64` | `number` | `int` | Timestamp (ms) of the most recent upsert. | -| dense_vector | `Option` | — | — | Dense vector (Rust only). Node.js and Python access vectors through vector search, not direct record access. | -| sparse_vector | `Option` | — | — | Sparse vector (Rust only). | +| dense_vector / denseVector | `Option` | `number[] \| null` | `list[float] \| None` | Dense vector stored on the node. | +| sparse_vector / sparseVector | `Option` | `SparseEntry[] \| null` | `list[tuple[int, float]] \| None` | Sparse vector stored on the node. | -### EdgeRecord +Rust returns `NodeView`; Node.js returns `NodeView`; Python returns `NodeView`. -An edge (relationship) stored in the graph. +### Edge Records + +A public, hydrated edge record returned by read operations. | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| | id | `u64` | `number` | `int` | Unique, auto-assigned edge ID. | | from / from_id | `u64` | `from: number` | `from_id: int` | Source node ID. | | to / to_id | `u64` | `to: number` | `to_id: int` | Destination node ID. | -| type_id | `u32` | `number` | `int` | User-defined edge type. | +| label | `String` | `label: string` | `label: str` | Public edge label. | | props | `BTreeMap` | `Record` | `dict[str, Any]` | User-defined properties. | | weight | `f32` | `number` | `float` | Edge weight. Default `1.0`. | -| valid_from | `i64` | `number` | `int` | Start of the edge's validity window (ms). `0` means "always valid from the beginning of time." | -| valid_to | `i64` | `number` | `int` | End of the edge's validity window (ms). `i64::MAX` means "no expiration." | +| valid_from | `i64` | `number` | `int` | Start of the edge's validity window (ms). If omitted when writing, OverGraph uses the edge's `created_at` timestamp. | +| valid_to | `i64` | `number` | `int` | End of the edge's validity window (ms). If omitted when writing, OverGraph uses `i64::MAX` / no expiration. | | created_at | `i64` | `number` | `int` | Creation timestamp (ms). | | updated_at | `i64` | `number` | `int` | Last update timestamp (ms). | +Rust returns `EdgeView`; Node.js returns `EdgeView`; Python returns `EdgeView`. + ### PropValue -Property values are strongly typed. The following types are supported across all three languages: +Property values are strongly typed in the Rust core. Connector inputs use their host-language conversion rules and do not expose every Rust variant as a distinct writable type. | Type | Rust | Node.js | Python | Notes | |------|------|---------|--------|-------| | Null | `PropValue::Null` | `null` | `None` | | | Boolean | `PropValue::Bool(bool)` | `boolean` | `bool` | | -| Integer | `PropValue::Int(i64)` | `number` | `int` | 64-bit signed. | -| Unsigned | `PropValue::UInt(u64)` | `number` | `int` | 64-bit unsigned. | +| Integer | `PropValue::Int(i64)` | `number` | `int` | Node.js and normal Python integer inputs write signed integers. | +| Unsigned | `PropValue::UInt(u64)` | Readable as `number` | Readable as `int` | Rust can construct this directly. Connector property inputs do not provide a separate unsigned marker. | | Float | `PropValue::Float(f64)` | `number` | `float` | 64-bit IEEE 754. | | String | `PropValue::String(String)` | `string` | `str` | UTF-8. | -| Bytes | `PropValue::Bytes(Vec)` | `Buffer` | `bytes` | Raw byte data. | +| Bytes | `PropValue::Bytes(Vec)` | Readable as JSON array | `bytes` | Python can write `bytes`. Node.js property input is JSON-like and does not currently convert `Buffer` to `PropValue::Bytes`. | | Array | `PropValue::Array(Vec)` | `any[]` | `list` | Heterogeneous array. | | Map | `PropValue::Map(BTreeMap)` | `object` | `dict` | Nested properties. | Properties are encoded with [MessagePack](https://msgpack.org) internally and converted lazily when accessed from Node.js or Python. +Connector property conversion is intentionally host-language shaped. Node.js writes JSON-like values (`null`, booleans, numbers, strings, arrays, and objects); it does not currently use `Buffer` as a bytes marker or expose a separate unsigned-integer marker. Python writes the same common values plus `bytes`; normal Python `int` inputs write signed integers. Rust callers can construct every `PropValue` variant directly. + +### IntoNodeLabels (Rust only) + +Rust node-label APIs accept `impl IntoNodeLabels` for single-label and multi-label calls. Accepted input forms are `&str`, `String`, `&String`, `&[&str]`, `&[String]`, `Vec`, `&[&str; N]`, and `&[String; N]`. + ### Direction Controls edge traversal direction. Used across traversal and graph analytics APIs. @@ -476,17 +531,144 @@ Controls edge traversal direction. Used across traversal and graph analytics API | Incoming | `Direction::Incoming` | `"incoming"` | `"incoming"` | Follow edges in the `to → from` direction. | | Both | `Direction::Both` | `"both"` | `"both"` | Follow edges in both directions (treat graph as undirected). | +### NodeLabelFilter / LabelMatchMode + +Use `NodeLabelFilter` when callers need explicit `Any` or `All` semantics over node labels. + +```rust +let any_user_or_admin = NodeLabelFilter { + labels: vec!["User".into(), "Admin".into()], + mode: LabelMatchMode::Any, +}; + +let both_user_and_admin = NodeLabelFilter { + labels: vec!["User".into(), "Admin".into()], + mode: LabelMatchMode::All, +}; +``` + +```python +any_user_or_admin = {"labels": ["User", "Admin"], "mode": "any"} +both_user_and_admin = {"labels": ["User", "Admin"], "mode": "all"} +``` + +```javascript +const anyUserOrAdmin = { labels: ['User', 'Admin'], mode: 'any' }; +const bothUserAndAdmin = { labels: ['User', 'Admin'], mode: 'all' }; +``` + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| labels | `Vec` | `labels: string[]` | `"labels": list[str]` | Public node labels to match. Must be non-empty and contain no duplicates. | +| mode | `LabelMatchMode` | `mode: "any" \| "all"` | `"mode": "any" \| "all"` | `Any`/`"any"` matches nodes with at least one listed label. `All`/`"all"` matches nodes with every listed label. | + +--- + +## Catalog APIs + +Catalog APIs explicitly manage or inspect the node-label and edge-label token catalog. Ordinary graph APIs accept and return names; catalog diagnostics are the only public surface that exposes numeric token IDs. + +### ensure_node_label / ensure_edge_label + +Ensure a catalog token exists for a public node label or edge label and return its diagnostic token ID. + +```rust +let user_label_id = db.ensure_node_label("User")?; +let created_label_id = db.ensure_edge_label("CREATED")?; +``` + +```javascript +const userLabelId = db.ensureNodeLabel('User'); +const createdLabelId = db.ensureEdgeLabel('CREATED'); +``` + +```python +user_label_id = db.ensure_node_label("User") +created_label_id = db.ensure_edge_label("CREATED") +``` + +These methods are optional for normal writes: `upsert_node`, `upsert_edge`, batch writes, graph patch, and write transactions auto-create missing names durably. Use explicit ensures when you want catalog IDs for diagnostics or want to prepare names before writes. + +### get_node_label_id / get_edge_label_id + +Read-only lookup from public name to diagnostic token ID. + +```rust +let id = db.get_node_label_id("User")?; +let edge_id = db.get_edge_label_id("CREATED")?; +``` + +```javascript +const id = db.getNodeLabelId('User'); +const edgeId = db.getEdgeLabelId('CREATED'); +``` + +```python +id = db.get_node_label_id("User") +edge_id = db.get_edge_label_id("CREATED") +``` + +Returns `None`/`null` when the name is unknown. + +### get_node_label / get_edge_label + +Diagnostic reverse lookup from token ID to public name. + +```rust +let label = db.get_node_label(label_id)?; +let edge_label = db.get_edge_label(label_id)?; +``` + +```javascript +const label = db.getNodeLabel(labelId); +const edgeLabel = db.getEdgeLabel(labelId); +``` + +```python +label = db.get_node_label(label_id) +edge_label = db.get_edge_label(label_id) +``` + +The node and edge `label_id` / `labelId` arguments are catalog token IDs, not normal graph API inputs. + +### list_node_labels / list_edge_labels + +List published catalog entries. + +```rust +let labels = db.list_node_labels()?; +let edge_labels = db.list_edge_labels()?; +``` + +```javascript +const labels = db.listNodeLabels(); +const edgeLabels = db.listEdgeLabels(); +``` + +```python +labels = db.list_node_labels() +edge_labels = db.list_edge_labels() +``` + +| Entry | Rust fields | Node.js fields | Python fields | +|-------|-------------|----------------|---------------| +| Node label | `label`, `label_id` | `label`, `labelId` | `label`, `label_id` | +| Edge label | `label`, `label_id` | `label`, `labelId` | `label`, `label_id` | + +`label_id` and `labelId` in these entries are diagnostic catalog metadata. Do not use them as input to ordinary node, edge, query, traversal, or vector APIs. + --- ## Node Operations ### upsert_node -Creates a new node or updates an existing one. Nodes are identified by the `(type_id, key)` pair. If a node with the same type and key already exists, it is updated in place (preserving its ID). +Creates a new node or updates an existing one. If the key already resolves to the same node through any supplied label, the node is updated in place; if the same key resolves to different nodes across supplied labels, the write is rejected as a conflict. **Rust** ```rust -let id = db.upsert_node(USER, "alice", UpsertNodeOptions { +// Also accepts multiple labels: &["User", "Admin"] +let id = db.upsert_node("User", "alice", UpsertNodeOptions { props: BTreeMap::from([("role".into(), PropValue::String("admin".into()))]), weight: 1.0, ..Default::default() @@ -495,7 +677,8 @@ let id = db.upsert_node(USER, "alice", UpsertNodeOptions { **Node.js** ```javascript -const id = db.upsertNode(USER, 'alice', { +// Also accepts multiple labels: ['User', 'Admin'] +const id = db.upsertNode('User', 'alice', { props: { role: 'admin' }, weight: 1.0, }); @@ -503,15 +686,16 @@ const id = db.upsertNode(USER, 'alice', { **Python** ```python -id = db.upsert_node(USER, "alice", props={"role": "admin"}, weight=1.0) +# Also accepts multiple labels: ["User", "Admin"] +id = db.upsert_node("User", "alice", props={"role": "admin"}, weight=1.0) ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Default | Description | |-----------|------|---------|--------|----------|---------|-------------| -| type_id | `u32` | `number` | `int` | Yes | — | User-defined type identifier. Arbitrary integer (0–4,294,967,295). Define as constants for readability. | -| key | `&str` | `string` | `str` | Yes | — | Unique key within the type. The `(type_id, key)` pair is the node's identity. If a node with this pair exists, it is updated. | +| labels | `impl IntoNodeLabels` | `string \| string[]` | `str \| list[str]` | Yes | — | One or more public node labels. | +| key | `&str` | `string` | `str` | Yes | — | Unique key scoped by node labels. If the supplied label set and key resolve to an existing node, it is updated. | | props | `BTreeMap` | `Record` | `dict[str, Any]` | No | `{}` | Arbitrary key-value properties. On update, the entire props map is replaced (not merged). | | weight | `f32` | `number` | `float` | No | `1.0` | Numeric weight. Used by pruning policies (`max_weight`) and scoring algorithms. | | dense_vector | `Option>` | `number[]` | `list[float]` | No | `None` | Dense vector for similarity search. Length must match the `dimension` configured at `open()`. Requires `dense_vector` to be enabled in DbOptions. | @@ -532,7 +716,7 @@ The node's ID. If the node was newly created, this is a fresh ID. If the node al #### Behavior -- **Upsert semantics**: On insert, allocates a new ID, sets `created_at` and `updated_at` to the current time. On update, keeps the original `created_at`, refreshes `updated_at`, and replaces all fields (props, weight, vectors). +- **Upsert semantics**: On insert, allocates a new ID, sets `created_at` and `updated_at` to the current time. On update, keeps the original `created_at`, refreshes `updated_at`, and replaces labels, props, weight, and vectors. - **Atomicity**: The write is applied to the WAL and memtable in a single operation. - **Performance**: ~4ms per call in `Immediate` sync mode (dominated by `fsync`). Use [`batch_upsert_nodes`](#batch_upsert_nodes) for bulk operations where a single fsync is shared across the batch. @@ -545,21 +729,21 @@ Retrieves a node by its ID. **Rust** ```rust if let Some(node) = db.get_node(id)? { - println!("key={}, type={}", node.key, node.type_id); + println!("labels={:?}, key={}", node.labels, node.key); } ``` **Node.js** ```javascript const node = db.getNode(id); -if (node) console.log(node.key, node.typeId); +if (node) console.log(node.labels, node.key); ``` **Python** ```python node = db.get_node(id) if node: - print(node.key, node.type_id) + print(node.labels, node.key) ``` #### Parameters @@ -572,7 +756,7 @@ if node: | Rust | Node.js | Python | |------|---------|--------| -| `Result, EngineError>` | `NodeRecord \| null` | `NodeRecord \| None` | +| `Result, EngineError>` | `NodeView \| null` | `NodeView \| None` | Returns `None`/`null` if the node does not exist or has been deleted. @@ -584,37 +768,78 @@ Returns `None`/`null` if the node does not exist or has been deleted. ### get_node_by_key -Looks up a node by its `(type_id, key)` pair. Uses the type index for fast lookup. +Looks up a node by its `(label, key)` pair. Uses the label-scoped key lookup/index for fast lookup. **Rust** ```rust -let node = db.get_node_by_key(USER, "alice")?; +let node = db.get_node_by_key("User", "alice")?; ``` **Node.js** ```javascript -const node = db.getNodeByKey(USER, 'alice'); +const node = db.getNodeByKey('User', 'alice'); ``` **Python** ```python -node = db.get_node_by_key(USER, "alice") +node = db.get_node_by_key("User", "alice") ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| type_id | `u32` | `number` | `int` | Yes | Node type identifier. | -| key | `&str` | `string` | `str` | Yes | Node key within the type. | +| label | `&str` | `string` | `str` | Yes | Node label. | +| key | `&str` | `string` | `str` | Yes | Node key within the label. | #### Returns | Rust | Node.js | Python | |------|---------|--------| -| `Result, EngineError>` | `NodeRecord \| null` | `NodeRecord \| None` | +| `Result, EngineError>` | `NodeView \| null` | `NodeView \| None` | + +Returns `None`/`null` if no node with that `(label, key)` exists. + +--- + +### add_node_label / remove_node_label + +Node label-set mutation helpers. These update a node's label set without changing its ID, key, properties, weight, or vectors. + +```rust +let added = db.add_node_label(node_id, "Admin")?; +let removed = db.remove_node_label(node_id, "Trial")?; +``` + +```javascript +const added = db.addNodeLabel(nodeId, 'Admin'); +const removed = db.removeNodeLabel(nodeId, 'Trial'); +``` + +```python +added = db.add_node_label(node_id, "Admin") +removed = db.remove_node_label(node_id, "Trial") +``` + +#### Parameters -Returns `None`/`null` if no node with that `(type_id, key)` exists. +| Parameter | Rust | Node.js | Python | Required | Description | +|-----------|------|---------|--------|----------|-------------| +| id | `u64` | `number` | `int` | Yes | Node ID to mutate. | +| label | `&str` | `string` | `str` | Yes | Public node label to add or remove. | + +#### Returns + +| Rust | Node.js | Python | Description | +|------|---------|--------|-------------| +| `Result` | `boolean` | `bool` | `true` when the node's label set changed, `false` when the requested label was already present for add or absent for remove. | + +#### Behavior + +- Adding a label auto-creates the label token when needed. +- Adding a label fails if another node already owns the same `(label, key)` identity. +- Removing an unknown or absent label returns `false`. +- Removing the last remaining node label returns an error. --- @@ -665,17 +890,31 @@ Upserts multiple nodes in a single batch with one WAL fsync. Significantly faste **Rust** ```rust let inputs = vec![ - NodeInput { type_id: USER, key: "alice".into(), weight: 1.0, ..Default::default() }, - NodeInput { type_id: USER, key: "bob".into(), weight: 0.8, ..Default::default() }, + NodeInput { + labels: vec!["User".into()], + key: "alice".into(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["User".into(), "Admin".into()], + key: "bob".into(), + props: BTreeMap::from([("role".into(), PropValue::String("viewer".into()))]), + weight: 0.8, + dense_vector: None, + sparse_vector: None, + }, ]; -let ids = db.batch_upsert_nodes(&inputs)?; +let ids = db.batch_upsert_nodes(inputs)?; ``` **Node.js** ```javascript const ids = db.batchUpsertNodes([ - { typeId: USER, key: 'alice', weight: 1.0 }, - { typeId: USER, key: 'bob', weight: 0.8, props: { role: 'viewer' } }, + { labels: ['User'], key: 'alice', weight: 1.0 }, + { labels: ['User', 'Admin'], key: 'bob', weight: 0.8, props: { role: 'viewer' } }, ]); // ids is a Float64Array ``` @@ -683,8 +922,8 @@ const ids = db.batchUpsertNodes([ **Python** ```python ids = db.batch_upsert_nodes([ - {"type_id": USER, "key": "alice", "weight": 1.0}, - {"type_id": USER, "key": "bob", "weight": 0.8, "props": {"role": "viewer"}}, + {"labels": ["User"], "key": "alice", "weight": 1.0}, + {"labels": ["User", "Admin"], "key": "bob", "weight": 0.8, "props": {"role": "viewer"}}, ]) ``` @@ -692,13 +931,13 @@ ids = db.batch_upsert_nodes([ | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| nodes | `&[NodeInput]` | `NodeInput[]` | `list[dict]` | Yes | Array of node inputs. Each element has the same fields as [`upsert_node`](#upsert_node) parameters. | +| nodes | `Vec` | `NodeInput[]` | `list[dict]` | Yes | Array of node inputs. Each element has the same fields as [`upsert_node`](#upsert_node) parameters. | **NodeInput fields:** | Field | Rust | Node.js | Python dict key | Required | Default | Description | |-------|------|---------|-----------------|----------|---------|-------------| -| type_id | `u32` | `typeId: number` | `"type_id"` | Yes | — | Node type. | +| labels | `labels: Vec` | `labels: string \| string[]` | `"labels"` | Yes | — | One or more node labels. Node.js accepts a single string or a non-empty string array for dict-based node inputs. | | key | `String` | `key: string` | `"key"` | Yes | — | Node key. | | props | `BTreeMap` | `props: object` | `"props"` | No | `{}` | Properties. | | weight | `f32` | `weight: number` | `"weight"` | No | `1.0` | Weight. | @@ -726,19 +965,19 @@ Batch-retrieves multiple nodes by ID. Uses a sorted merge-walk across all data s **Rust** ```rust let nodes = db.get_nodes(&[1, 2, 3])?; -// nodes[0] is Option for ID 1, etc. +// nodes[0] is Option for ID 1, etc. ``` **Node.js** ```javascript const nodes = db.getNodes([1, 2, 3]); -// nodes[0] is NodeRecord | null for ID 1, etc. +// nodes[0] is NodeView | null for ID 1, etc. ``` **Python** ```python nodes = db.get_nodes([1, 2, 3]) -# nodes[0] is NodeRecord | None for ID 1, etc. +# nodes[0] is NodeView | None for ID 1, etc. ``` #### Parameters @@ -751,7 +990,7 @@ nodes = db.get_nodes([1, 2, 3]) | Rust | Node.js | Python | |------|---------|--------| -| `Result>, EngineError>` | `(NodeRecord \| null)[]` | `list[NodeRecord \| None]` | +| `Result>, EngineError>` | `(NodeView \| null)[]` | `list[NodeView \| None]` | An array the same length as the input, where each element is the node record or `None`/`null` if that ID doesn't exist. @@ -759,40 +998,46 @@ An array the same length as the input, where each element is the node record or ### get_nodes_by_keys -Batch-retrieves multiple nodes by `(type_id, key)` pairs. +Batch-retrieves multiple nodes by `(label, key)` pairs. **Rust** ```rust -let nodes = db.get_nodes_by_keys(&[(USER, "alice".into()), (USER, "bob".into())])?; +let nodes = db.get_nodes_by_keys(&[ + NodeKeyQuery { label: "User".into(), key: "alice".into() }, + NodeKeyQuery { label: "User".into(), key: "bob".into() }, +])?; ``` **Node.js** ```javascript const nodes = db.getNodesByKeys([ - { typeId: USER, key: 'alice' }, - { typeId: USER, key: 'bob' }, + { label: 'User', key: 'alice' }, + { label: 'User', key: 'bob' }, ]); ``` **Python** ```python -nodes = db.get_nodes_by_keys([(USER, "alice"), (USER, "bob")]) +nodes = db.get_nodes_by_keys([ + {"labels": ["User"], "key": "alice"}, + {"labels": ["User"], "key": "bob"}, +]) ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| keys | `&[(u32, String)]` | `KeyQuery[]` | `list[tuple[int, str]]` | Yes | Array of `(type_id, key)` pairs. | +| keys | `&[NodeKeyQuery]` | `KeyQuery[]` | `list[dict]` | Yes | Array of key lookups. Python uses `{"labels": "User" \| ["User"], "key": ...}` and requires exactly one label because keys are label-scoped. | **KeyQuery** (Node.js): ```typescript -{ typeId: number, key: string } +{ label: string, key: string } ``` #### Returns -Same shape as [`get_nodes`](#get_nodes): an array of `NodeRecord | None` in input order. +Same shape as [`get_nodes`](#get_nodes): an array of node records or `None`/`null` in input order. --- @@ -800,11 +1045,11 @@ Same shape as [`get_nodes`](#get_nodes): an array of `NodeRecord | None` in inpu ### upsert_edge -Creates a new edge or updates an existing one. When `edge_uniqueness` is enabled, edges are identified by the `(from, to, type_id)` triple. +Creates a new edge or updates an existing one. When `edge_uniqueness` is enabled, edges are identified by the `(from, to, label)` triple. **Rust** ```rust -let id = db.upsert_edge(alice_id, project_id, WORKS_ON, UpsertEdgeOptions { +let id = db.upsert_edge(alice_id, project_id, "WORKS_ON", UpsertEdgeOptions { props: BTreeMap::from([("since".into(), PropValue::String("2024".into()))]), weight: 1.0, ..Default::default() @@ -813,7 +1058,7 @@ let id = db.upsert_edge(alice_id, project_id, WORKS_ON, UpsertEdgeOptions { **Node.js** ```javascript -const id = db.upsertEdge(aliceId, projectId, WORKS_ON, { +const id = db.upsertEdge(aliceId, projectId, 'WORKS_ON', { props: { since: '2024' }, weight: 1.0, }); @@ -821,7 +1066,7 @@ const id = db.upsertEdge(aliceId, projectId, WORKS_ON, { **Python** ```python -id = db.upsert_edge(alice_id, project_id, WORKS_ON, +id = db.upsert_edge(alice_id, project_id, "WORKS_ON", props={"since": "2024"}, weight=1.0) ``` @@ -831,10 +1076,10 @@ id = db.upsert_edge(alice_id, project_id, WORKS_ON, |-----------|------|---------|--------|----------|---------|-------------| | from | `u64` | `number` | `int` | Yes | — | Source node ID. | | to | `u64` | `number` | `int` | Yes | — | Destination node ID. | -| type_id | `u32` | `number` | `int` | Yes | — | Edge type identifier. | +| label | `&str` | `string` | `str` | Yes | — | Public edge label such as `"WORKS_ON"` or `"KNOWS"`. | | props | `BTreeMap` | `Record` | `dict[str, Any]` | No | `{}` | Edge properties. Replaced entirely on update. | | weight | `f32` | `number` | `float` | No | `1.0` | Edge weight. Used by shortest path (as cost), top-k scoring, and pruning. | -| valid_from | `Option` | `number` | `int` | No | `0` (always valid) | Start of the edge's temporal validity window (ms). Edges with `valid_from > at_epoch` are excluded from temporal queries. | +| valid_from | `Option` | `number` | `int` | No | edge `created_at` | Start of the edge's temporal validity window (ms). Edges with `valid_from > at_epoch` are excluded from temporal queries. | | valid_to | `Option` | `number` | `int` | No | `i64::MAX` (no expiration) | End of the validity window (ms). Edges with `valid_to <= at_epoch` are excluded from temporal queries. See [`invalidate_edge`](#invalidate_edge). | #### Returns @@ -847,7 +1092,7 @@ The edge ID. #### Behavior -- **With `edge_uniqueness` enabled**: If an edge with the same `(from, to, type_id)` exists, it is updated and the existing ID is returned. Otherwise a new edge is created. +- **With `edge_uniqueness` enabled**: If an edge with the same `(from, to, label)` exists, it is updated and the existing ID is returned. Otherwise a new edge is created. - **With `edge_uniqueness` disabled** (default): Every call creates a new edge (parallel edges are allowed). --- @@ -879,27 +1124,27 @@ edge = db.get_edge(edge_id) #### Returns -`EdgeRecord | None`. Returns `None`/`null` if the edge doesn't exist or has been deleted. +`EdgeView` / `EdgeView` / `EdgeView`, or `None`/`null` if the edge doesn't exist or has been deleted. --- ### get_edge_by_triple -Looks up an edge by its `(from, to, type_id)` triple. Only meaningful when `edge_uniqueness` is enabled. +Looks up an edge by its `(from, to, label)` triple. Only meaningful when `edge_uniqueness` is enabled. **Rust** ```rust -let edge = db.get_edge_by_triple(alice_id, project_id, WORKS_ON)?; +let edge = db.get_edge_by_triple(alice_id, project_id, "WORKS_ON")?; ``` **Node.js** ```javascript -const edge = db.getEdgeByTriple(aliceId, projectId, WORKS_ON); +const edge = db.getEdgeByTriple(aliceId, projectId, 'WORKS_ON'); ``` **Python** ```python -edge = db.get_edge_by_triple(alice_id, project_id, WORKS_ON) +edge = db.get_edge_by_triple(alice_id, project_id, "WORKS_ON") ``` #### Parameters @@ -908,11 +1153,11 @@ edge = db.get_edge_by_triple(alice_id, project_id, WORKS_ON) |-----------|------|---------|--------|----------|-------------| | from | `u64` | `number` | `int` | Yes | Source node ID. | | to | `u64` | `number` | `int` | Yes | Destination node ID. | -| type_id | `u32` | `number` | `int` | Yes | Edge type. | +| label | `&str` | `string` | `str` | Yes | Edge label. | #### Returns -`EdgeRecord | None`. +`EdgeView` / `EdgeView` / `EdgeView`, or `None`/`null`. --- @@ -978,7 +1223,7 @@ updated = db.invalidate_edge(edge_id, int(time.time() * 1000)) | Rust | Node.js | Python | |------|---------|--------| -| `Result, EngineError>` | `EdgeRecord \| null` | `EdgeRecord \| None` | +| `Result, EngineError>` | `EdgeView \| null` | `EdgeView \| None` | The updated edge record, or `None`/`null` if the edge doesn't exist. @@ -1000,25 +1245,41 @@ Upserts multiple edges in a single batch with one WAL fsync. **Rust** ```rust let inputs = vec![ - EdgeInput { from: 1, to: 2, type_id: WORKS_ON, weight: 1.0, ..Default::default() }, - EdgeInput { from: 1, to: 3, type_id: WORKS_ON, weight: 0.5, ..Default::default() }, + EdgeInput { + from: 1, + to: 2, + label: "WORKS_ON".into(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: 1, + to: 3, + label: "WORKS_ON".into(), + props: BTreeMap::new(), + weight: 0.5, + valid_from: None, + valid_to: None, + }, ]; -let ids = db.batch_upsert_edges(&inputs)?; +let ids = db.batch_upsert_edges(inputs)?; ``` **Node.js** ```javascript const ids = db.batchUpsertEdges([ - { from: 1, to: 2, typeId: WORKS_ON, weight: 1.0 }, - { from: 1, to: 3, typeId: WORKS_ON, weight: 0.5 }, + { from: 1, to: 2, label: 'WORKS_ON', weight: 1.0 }, + { from: 1, to: 3, label: 'WORKS_ON', weight: 0.5 }, ]); ``` **Python** ```python ids = db.batch_upsert_edges([ - {"from_id": 1, "to_id": 2, "type_id": WORKS_ON, "weight": 1.0}, - {"from_id": 1, "to_id": 3, "type_id": WORKS_ON, "weight": 0.5}, + {"from_id": 1, "to_id": 2, "label": "WORKS_ON", "weight": 1.0}, + {"from_id": 1, "to_id": 3, "label": "WORKS_ON", "weight": 0.5}, ]) ``` @@ -1026,7 +1287,7 @@ ids = db.batch_upsert_edges([ | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| edges | `&[EdgeInput]` | `EdgeInput[]` | `list[dict]` | Yes | Array of edge inputs. | +| edges | `Vec` | `EdgeInput[]` | `list[dict]` | Yes | Array of edge inputs. | **EdgeInput fields:** @@ -1034,10 +1295,10 @@ ids = db.batch_upsert_edges([ |-------|------|---------|-----------------|----------|---------|-------------| | from | `u64` | `from: number` | `"from_id"` | Yes | — | Source node ID. | | to | `u64` | `to: number` | `"to_id"` | Yes | — | Destination node ID. | -| type_id | `u32` | `typeId: number` | `"type_id"` | Yes | — | Edge type. | +| label | `String` | `label: string` | `"label"` | Yes | — | Edge label. | | props | `BTreeMap` | `props: object` | `"props"` | No | `{}` | Properties. | | weight | `f32` | `weight: number` | `"weight"` | No | `1.0` | Weight. | -| valid_from | `Option` | `validFrom: number` | `"valid_from"` | No | `0` | Validity start. | +| valid_from | `Option` | `validFrom: number` | `"valid_from"` | No | edge `created_at` | Validity start. | | valid_to | `Option` | `validTo: number` | `"valid_to"` | No | `i64::MAX` | Validity end. | #### Returns @@ -1077,7 +1338,7 @@ edges = db.get_edges([10, 20, 30]) #### Returns -Array of `EdgeRecord | None` in input order. +Array of edge records or `None`/`null` in input order. --- @@ -1089,9 +1350,24 @@ Applies multiple operations atomically in a single WAL batch: node upserts, edge **Rust** ```rust -let result = db.graph_patch(&GraphPatch { - upsert_nodes: vec![NodeInput { type_id: USER, key: "carol".into(), ..Default::default() }], - upsert_edges: vec![EdgeInput { from: 1, to: 2, type_id: WORKS_ON, ..Default::default() }], +let result = db.graph_patch(GraphPatch { + upsert_nodes: vec![NodeInput { + labels: vec!["User".into(), "Admin".into()], + key: "carol".into(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }], + upsert_edges: vec![EdgeInput { + from: 1, + to: 2, + label: "WORKS_ON".into(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }], invalidate_edges: vec![(edge_id, now_ms)], delete_node_ids: vec![old_node_id], delete_edge_ids: vec![old_edge_id], @@ -1101,8 +1377,8 @@ let result = db.graph_patch(&GraphPatch { **Node.js** ```javascript const result = db.graphPatch({ - upsertNodes: [{ typeId: USER, key: 'carol' }], - upsertEdges: [{ from: 1, to: 2, typeId: WORKS_ON }], + upsertNodes: [{ labels: ['User'], key: 'carol' }], + upsertEdges: [{ from: 1, to: 2, label: 'WORKS_ON' }], invalidateEdges: [{ edgeId: 5, validTo: Date.now() }], deleteNodeIds: [oldNodeId], deleteEdgeIds: [oldEdgeId], @@ -1112,8 +1388,8 @@ const result = db.graphPatch({ **Python** ```python result = db.graph_patch({ - "upsert_nodes": [{"type_id": USER, "key": "carol"}], - "upsert_edges": [{"from_id": 1, "to_id": 2, "type_id": WORKS_ON}], + "upsert_nodes": [{"labels": ["User"], "key": "carol"}], + "upsert_edges": [{"from_id": 1, "to_id": 2, "label": "WORKS_ON"}], "invalidate_edges": [{"edge_id": 5, "valid_to": int(time.time() * 1000)}], "delete_node_ids": [old_node_id], "delete_edge_ids": [old_edge_id], @@ -1162,10 +1438,15 @@ Transaction reads are intentionally bounded. A transaction can read committed st **Rust** ```rust let mut txn = db.begin_write_txn()?; -let alice = txn.upsert_node_as("alice", USER, "alice", UpsertNodeOptions::default())?; -let bob = txn.upsert_node_as("bob", USER, "bob", UpsertNodeOptions::default())?; -txn.upsert_edge_as("knows", alice, bob, KNOWS, UpsertEdgeOptions::default())?; -let staged = txn.get_node_by_key(USER, "alice")?; +let alice = txn.upsert_node_as("alice", &["User", "Admin"], "alice", UpsertNodeOptions::default())?; +let bob = txn.upsert_node_as("bob", "User", "bob", UpsertNodeOptions::default())?; +txn.upsert_edge_as("knows", alice.clone(), bob.clone(), "KNOWS", UpsertEdgeOptions::default())?; +assert!(txn.add_node_label(alice.clone(), "Manager")?); +assert!(txn.remove_node_label(alice.clone(), "Admin")?); +let staged = txn.get_node_by_key("User", "alice")?; +if let Some(view) = &staged { + println!("staged labels: {:?}", view.labels); +} let result = txn.commit()?; ``` @@ -1173,10 +1454,11 @@ let result = txn.commit()?; ```javascript const txn = db.beginWriteTxn(); txn.stage([ - { op: 'upsertNode', alias: 'alice', typeId: USER, key: 'alice' }, - { op: 'upsertNode', alias: 'bob', typeId: USER, key: 'bob' }, - { op: 'upsertEdge', alias: 'knows', from: { local: 'alice' }, to: { local: 'bob' }, typeId: KNOWS }, + { op: 'upsertNode', alias: 'alice', labels: ['User', 'Admin'], key: 'alice' }, + { op: 'upsertNode', alias: 'bob', labels: ['User'], key: 'bob' }, + { op: 'upsertEdge', alias: 'knows', from: { local: 'alice' }, to: { local: 'bob' }, label: 'KNOWS' }, ]); +txn.addNodeLabel({ local: 'bob' }, 'Trial'); const staged = txn.getNode({ local: 'alice' }); const result = txn.commit(); ``` @@ -1185,10 +1467,11 @@ const result = txn.commit(); ```python txn = db.begin_write_txn() txn.stage([ - {"op": "upsert_node", "alias": "alice", "type_id": USER, "key": "alice"}, - {"op": "upsert_node", "alias": "bob", "type_id": USER, "key": "bob"}, - {"op": "upsert_edge", "alias": "knows", "from": {"local": "alice"}, "to": {"local": "bob"}, "type_id": KNOWS}, + {"op": "upsert_node", "alias": "alice", "labels": ["User", "Admin"], "key": "alice"}, + {"op": "upsert_node", "alias": "bob", "labels": ["User"], "key": "bob"}, + {"op": "upsert_edge", "alias": "knows", "from": {"local": "alice"}, "to": {"local": "bob"}, "label": "KNOWS"}, ]) +txn.add_node_label({"local": "bob"}, "Trial") staged = txn.get_node({"local": "alice"}) result = txn.commit() ``` @@ -1199,19 +1482,35 @@ result = txn.commit() |-----------|------|---------|--------| | Begin | `begin_write_txn()` | `beginWriteTxn()` | `begin_write_txn()` | | Stage node | `upsert_node`, `upsert_node_as` | `upsertNode`, `upsertNodeAs` | `upsert_node`, `upsert_node_as` | +| Mutate node labels | `add_node_label`, `remove_node_label` | `addNodeLabel`, `removeNodeLabel` | `add_node_label`, `remove_node_label` | | Stage edge | `upsert_edge`, `upsert_edge_as` | `upsertEdge`, `upsertEdgeAs` | `upsert_edge`, `upsert_edge_as` | | Bulk ordered stage | `stage_intents(Vec)` | `stage(operations)` | `stage(operations)` | | Reads | `get_node`, `get_edge`, `get_node_by_key`, `get_edge_by_triple` | same camelCase names | same snake_case names | | Finish | `commit`, `rollback` | `commit`, `rollback` | `commit`, `rollback` | +#### Rust Transaction DTOs + +Rust exposes the transaction reference and intent objects directly: + +| Object | Variants / fields | Description | +|--------|-------------------|-------------| +| `TxnNodeRef` | `Id(u64)`, `Key { label, key }`, `Local(TxnLocalRef)` | Node target for transaction writes and bounded transaction reads. `Key` is single-label scoped. | +| `TxnEdgeRef` | `Id(u64)`, `Triple { from, to, label }`, `Local(TxnLocalRef)` | Edge target by ID, by endpoint refs plus edge label, or by local transaction ref. | +| `TxnIntent::UpsertNode` | `alias`, `labels`, `key`, `options` | Ordered staged node upsert. `labels` is the complete node-label set for the write. | +| `TxnIntent::UpsertEdge` | `alias`, `from`, `to`, `label`, `options` | Ordered staged edge upsert using transaction node refs. | +| `TxnIntent::DeleteNode` | `target` | Ordered staged node delete. | +| `TxnIntent::DeleteEdge` | `target` | Ordered staged edge delete. | +| `TxnIntent::InvalidateEdge` | `target`, `valid_to` | Ordered staged temporal edge invalidation. | + #### Builder Methods | Method | Required inputs | Optional inputs | Returns | |--------|-----------------|-----------------|---------| -| `upsert_node` / `upsertNode` | `type_id` / `typeId`, `key` | node upsert options: `props`, `weight`, `dense_vector` / `denseVector`, `sparse_vector` / `sparseVector` | node ref addressable by key | -| `upsert_node_as` / `upsertNodeAs` | `alias`, `type_id` / `typeId`, `key` | node upsert options | local node ref `{ local: alias }` | -| `upsert_edge` / `upsertEdge` | `from`, `to`, `type_id` / `typeId` | edge upsert options: `props`, `weight`, `valid_from` / `validFrom`, `valid_to` / `validTo` | edge ref addressable by triple | -| `upsert_edge_as` / `upsertEdgeAs` | `alias`, `from`, `to`, `type_id` / `typeId` | edge upsert options | local edge ref `{ local: alias }` | +| `upsert_node` / `upsertNode` | `labels`, `key` | node upsert options: `props`, `weight`, `dense_vector` / `denseVector`, `sparse_vector` / `sparseVector` | node ref addressable by key | +| `upsert_node_as` / `upsertNodeAs` | `alias`, `labels`, `key` | node upsert options | local node ref `{ local: alias }` | +| `add_node_label` / `remove_node_label` | node ref, `label` | — | `bool` changed flag | +| `upsert_edge` / `upsertEdge` | `from`, `to`, `label` | edge upsert options: `props`, `weight`, `valid_from` / `validFrom`, `valid_to` / `validTo` | edge ref addressable by triple | +| `upsert_edge_as` / `upsertEdgeAs` | `alias`, `from`, `to`, `label` | edge upsert options | local edge ref `{ local: alias }` | | `delete_node` / `deleteNode` | node ref | — | `void` / `None` | | `delete_edge` / `deleteEdge` | edge ref | — | `void` / `None` | | `invalidate_edge` / `invalidateEdge` | edge ref, `valid_to` / `validTo` | — | `void` / `None` | @@ -1228,28 +1527,31 @@ References are one of: | Ref kind | Node.js | Python | |----------|---------|--------| | Node by ID | `{ id }` | `{"id": id}` | -| Node by key | `{ typeId, key }` | `{"type_id": type_id, "key": key}` | +| Node by key | `{ labels, key }` | `{"labels": label_or_single_label_list, "key": key}` | | Node local alias | `{ local }` | `{"local": local}` | | Edge by ID | `{ id }` | `{"id": id}` | -| Edge by triple | `{ from, to, typeId }` | `{"from": from, "to": to, "type_id": type_id}` | +| Edge by triple | `{ from, to, label }` | `{"from": from, "to": to, "label": label}` | | Edge local alias | `{ local }` | `{"local": local}` | +Node-by-key transaction refs use `labels` and `key` but are still single-label scoped: +`labels` may be a string or a one-item list/array. + Aliases are optional, process-local, and never persisted. When present, aliases must be unique within the transaction across node aliases and unique across edge aliases. #### Transaction Read Views `get_node` and `get_node_by_key` on a transaction return a transaction node view: -| Field | Node.js | Python | Description | -|-------|---------|--------|-------------| -| id | `id?: number` | `"id": int \| None` | Committed node ID when already known; `None`/omitted for staged creates that allocate an ID at commit. | -| local | `local?: string` | `"local": str \| None` | Local alias for aliased staged records. Internal unaliased slots are not exposed as strings. | -| type_id | `typeId` | `"type_id"` | Node type. | -| key | `key` | `"key"` | Node key. | -| props | `props` | `"props"` | Node properties visible inside the transaction. | -| created_at / updated_at | `createdAt?` / `updatedAt?` | `"created_at"` / `"updated_at"` | Present for committed records; absent/`None` for staged creates before commit. | -| weight | `weight` | `"weight"` | Node weight. | -| dense_vector / sparse_vector | `denseVector?` / `sparseVector?` | `"dense_vector"` / `"sparse_vector"` | Staged or committed vectors when present. | +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| id | `Option` | `id?: number` | `"id": int \| None` | Committed node ID when already known; `None`/omitted for staged creates that allocate an ID at commit. | +| local | `Option` | `local?: string` | `"local": str \| None` | Local alias for aliased staged records. Internal unaliased slots are not exposed as strings. | +| labels | `Vec` | `labels: string[]` | `"labels": list[str]` | Complete node label set visible inside the transaction. | +| key | `String` | `key` | `"key"` | Node key. | +| props | `BTreeMap` | `props` | `"props"` | Node properties visible inside the transaction. | +| created_at / updated_at | `Option` | `createdAt?` / `updatedAt?` | `"created_at"` / `"updated_at"` | Present for committed records; absent/`None` for staged creates before commit. | +| weight | `f32` | `weight` | `"weight"` | Node weight. | +| dense_vector / sparse_vector | `Option` / `Option` | `denseVector?` / `sparseVector?` | `"dense_vector"` / `"sparse_vector"` | Staged or committed vectors when present. | `get_edge` and `get_edge_by_triple` return a transaction edge view: @@ -1258,7 +1560,7 @@ Aliases are optional, process-local, and never persisted. When present, aliases | id | `id?: number` | `"id": int \| None` | Committed edge ID when already known; `None`/omitted for staged creates that allocate an ID at commit. | | local | `local?: string` | `"local": str \| None` | Local alias for aliased staged records. | | from / to | `from` / `to` | `"from"` / `"to"` | Endpoint refs visible inside the transaction. | -| type_id | `typeId` | `"type_id"` | Edge type. | +| label | `label` | `"label"` | Edge label. | | props | `props` | `"props"` | Edge properties visible inside the transaction. | | created_at / updated_at | `createdAt?` / `updatedAt?` | `"created_at"` / `"updated_at"` | Present for committed records; absent/`None` for staged creates before commit. | | weight | `weight` | `"weight"` | Edge weight. | @@ -1281,33 +1583,36 @@ After `commit()` or `rollback()`, the transaction handle is closed. Further use --- -## Type-Based Queries +## Label and Edge-Label Queries -### nodes_by_type +### nodes_by_labels -Returns all node IDs of a given type. +Returns all node IDs containing every supplied node label. **Rust** ```rust -let ids: Vec = db.nodes_by_type(USER)?; +let ids: Vec = db.nodes_by_labels("User")?; +let admin_ids: Vec = db.nodes_by_labels(vec!["User".into(), "Admin".into()])?; ``` **Node.js** ```javascript -const ids = db.nodesByType(USER); // Float64Array +const ids = db.nodesByLabels('User'); // Float64Array +const adminIds = db.nodesByLabels(['User', 'Admin']); ``` **Python** ```python -ids = db.nodes_by_type(USER) # IdArray (lazy) +ids = db.nodes_by_labels("User") # IdArray (lazy) ids_list = ids.to_list() # materialize to list[int] +admin_ids = db.nodes_by_labels(["User", "Admin"]) ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| type_id | `u32` | `number` | `int` | Yes | Node type to query. | +| labels | `impl IntoNodeLabels` | `string \| string[]` | `str \| list[str]` | Yes | Label or labels to match. Nodes must contain every supplied node label. | #### Returns @@ -1315,102 +1620,202 @@ ids_list = ids.to_list() # materialize to list[int] |------|---------|--------| | `Result, EngineError>` | `Float64Array` | `IdArray` | -All node IDs of the given type. Filtered (excludes deleted/pruned nodes). +All matching node IDs. Filtered (excludes deleted/pruned nodes). **Python `IdArray`**: A lazy wrapper that avoids copying IDs to Python memory until accessed. Supports `len()`, indexing (`arr[i]`), iteration, `in` operator, and `to_list()`. #### Performance -O(type index size), not O(total nodes). Uses the per-type index. +Single-label input uses the direct per-label fast path. Multi-label input uses `All` semantics, drives from the best label posting, and metadata-verifies current label membership. Use [`query_node_ids`](#query_node_ids) with `NodeLabelFilter` when `Any` semantics are needed. --- -### edges_by_type +### edges_by_label -Returns all edge IDs of a given type. +Returns all edge IDs of a given edge label. +**Rust** ```rust -let ids = db.edges_by_type(WORKS_ON)?; +let ids: Vec = db.edges_by_label("WORKS_ON")?; ``` +**Node.js** ```javascript -const ids = db.edgesByType(WORKS_ON); +const ids = db.edgesByLabel('WORKS_ON'); // Float64Array ``` +**Python** ```python -ids = db.edges_by_type(WORKS_ON) +ids = db.edges_by_label("WORKS_ON") # IdArray (lazy) +ids_list = ids.to_list() # materialize to list[int] ``` -Same signature pattern as [`nodes_by_type`](#nodes_by_type). Filtered (excludes dangling edges from deleted nodes). +#### Parameters + +| Parameter | Rust | Node.js | Python | Required | Description | +|-----------|------|---------|--------|----------|-------------| +| label | `&str` | `string` | `str` | Yes | Public edge label to match, such as `"WORKS_ON"` or `"KNOWS"`. | + +#### Returns + +| Rust | Node.js | Python | +|------|---------|--------| +| `Result, EngineError>` | `Float64Array` | `IdArray` | + +All matching live edge IDs. Tombstoned edges are excluded. Unknown edge labels return an empty result. + +**Python `IdArray`**: A lazy wrapper that avoids copying IDs to Python memory until accessed. Supports `len()`, indexing (`arr[i]`), iteration, `in` operator, and `to_list()`. + +#### Performance + +Uses the edge-label posting index and does not hydrate edge records. Edges have exactly one public label, so this API accepts a single label string. Use [`query_edge_ids`](#query_edge_ids) when you need additional edge predicates. --- -### get_nodes_by_type +### get_nodes_by_labels -Returns full node records for all nodes of a given type. +Returns full node records for nodes containing every supplied node label. ```rust -let nodes: Vec = db.get_nodes_by_type(USER)?; +let nodes: Vec = db.get_nodes_by_labels("User")?; +let admin_nodes: Vec = + db.get_nodes_by_labels(vec!["User".into(), "Admin".into()])?; ``` ```javascript -const nodes = db.getNodesByType(USER); // NodeRecord[] +const nodes = db.getNodesByLabels('User'); // NodeView[] +const admins = db.getNodesByLabels(['User', 'Admin']); ``` ```python -nodes = db.get_nodes_by_type(USER) # list[NodeRecord] +nodes = db.get_nodes_by_labels("User") # list[NodeView] +admins = db.get_nodes_by_labels(["User", "Admin"]) ``` #### Parameters | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| type_id | `u32` / `number` / `int` | Yes | Node type. | +| labels | `impl IntoNodeLabels` / `string \| string[]` / `str \| list[str]` | Yes | Label or labels to match. Nodes must contain every supplied node label. | #### Returns -Array of full `NodeRecord` objects. Includes all fields (id, key, props, weight, timestamps, vectors). +Array of full node records. Includes all public fields (id, labels, key, props, weight, timestamps, vectors). + +Multi-label input always uses `All` semantics. Use [`query_nodes`](#query_nodes) with `NodeLabelFilter` when `Any` semantics are needed. --- -### get_edges_by_type +### get_edges_by_label + +Returns full edge records for all edges of a given edge label. + +**Rust** +```rust +let edges: Vec = db.get_edges_by_label("WORKS_ON")?; +``` -Returns full edge records for all edges of a given type. Same pattern as `get_nodes_by_type`. +**Node.js** +```javascript +const edges = db.getEdgesByLabel('WORKS_ON'); // EdgeView[] +``` + +**Python** +```python +edges = db.get_edges_by_label("WORKS_ON") # list[EdgeView] +``` + +#### Parameters + +| Parameter | Rust | Node.js | Python | Required | Description | +|-----------|------|---------|--------|----------|-------------| +| label | `&str` | `string` | `str` | Yes | Public edge label to match. | + +#### Returns + +Array of full edge records. Includes all public edge fields: id, endpoints (`from`/`to` in Rust and Node.js, `from_id`/`to_id` in Python), label, props, weight, timestamps, and validity window. + +Unknown edge labels return an empty array. Tombstoned edges are excluded. + +#### Performance + +Uses the edge-label posting index to collect matching IDs, then batch-hydrates the matching records. Use [`edges_by_label`](#edges_by_label) when IDs are enough. --- -### count_nodes_by_type +### count_nodes_by_labels -Returns the count of nodes of a given type. More efficient than fetching all IDs and measuring the length. +Returns the count of nodes containing every supplied node label. ```rust -let count: u64 = db.count_nodes_by_type(USER)?; +let count: u64 = db.count_nodes_by_labels("User")?; +let admin_count: u64 = + db.count_nodes_by_labels(vec!["User".into(), "Admin".into()])?; ``` ```javascript -const count = db.countNodesByType(USER); +const count = db.countNodesByLabels('User'); ``` ```python -count = db.count_nodes_by_type(USER) +count = db.count_nodes_by_labels("User") +admin_count = db.count_nodes_by_labels(["User", "Admin"]) ``` +Count uses metadata-only verification and does not hydrate node records or allocate the final ID result vector. Multi-label input always uses `All` semantics. Use [`query_node_ids`](#query_node_ids) with `NodeLabelFilter` when `Any` semantics are needed. + --- -### count_edges_by_type +### count_edges_by_label + +Returns the count of live edges of a given edge label. + +**Rust** +```rust +let count: u64 = db.count_edges_by_label("WORKS_ON")?; +``` + +**Node.js** +```javascript +const count = db.countEdgesByLabel('WORKS_ON'); +``` + +**Python** +```python +count = db.count_edges_by_label("WORKS_ON") +``` + +#### Parameters + +| Parameter | Rust | Node.js | Python | Required | Description | +|-----------|------|---------|--------|----------|-------------| +| label | `&str` | `string` | `str` | Yes | Public edge label to count. | + +#### Returns + +| Rust | Node.js | Python | +|------|---------|--------| +| `Result` | `number` | `int` | + +Unknown edge labels return `0`. Tombstoned edges are excluded. + +#### Performance -Returns the count of edges of a given type. Same pattern as `count_nodes_by_type`. +Counts through the edge-label posting path without hydrating edge records. --- ## Property Index Management -Property indexes are optional declarations on node properties. Public query methods stay the same whether or not you declare an index. +Property indexes are optional declarations on node or edge properties. Public query methods stay the same whether or not you declare an index. Lifecycle rules: - `ensure_node_property_index` registers an equality or numeric range declaration and starts background build work when needed. +- `ensure_edge_property_index` does the same for edge properties, scoped by edge label. - `list_node_property_indexes` exposes declaration kind, range domain, lifecycle state, and any last error from the published read snapshot, so `Ready` means new public reads can use the same ready catalog. +- `list_edge_property_indexes` exposes the same state for edge declarations. - `find_nodes`, `find_nodes_paged`, `find_nodes_range`, and `find_nodes_range_paged` use declaration-backed execution only when a matching declaration is `Ready`. +- `query_edge_ids`, `query_edges`, and `query_pattern` may use ready edge-property declarations as candidate sources while still verifying final edge filters. - If a declaration is absent, `Building`, `Failed`, or cannot be used for a specific lookup, OverGraph falls back to the same public query API for that call. ### ensure_node_property_index @@ -1420,13 +1825,13 @@ Ensures an optional secondary index declaration for a node property. **Rust** ```rust let eq = db.ensure_node_property_index( - USER, + "User", "role", SecondaryIndexKind::Equality, )?; let range = db.ensure_node_property_index( - USER, + "User", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -1436,9 +1841,9 @@ let range = db.ensure_node_property_index( **Node.js** ```javascript -const eq = db.ensureNodePropertyIndex(USER, 'role', { kind: 'equality' }); +const eq = db.ensureNodePropertyIndex('User', 'role', { kind: 'equality' }); -const range = db.ensureNodePropertyIndex(USER, 'score', { +const range = db.ensureNodePropertyIndex('User', 'score', { kind: 'range', domain: 'int', }); @@ -1446,10 +1851,10 @@ const range = db.ensureNodePropertyIndex(USER, 'score', { **Python** ```python -eq = db.ensure_node_property_index(USER, "role", "equality") +eq = db.ensure_node_property_index("User", "role", "equality") range_info = db.ensure_node_property_index( - USER, + "User", "score", "range", domain="int", @@ -1460,7 +1865,7 @@ range_info = db.ensure_node_property_index( | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| type_id | `u32` | `number` | `int` | Yes | Restrict the declaration to this node type. | +| label | `&str` | `string` | `str` | Yes | Restrict the declaration to this node label. | | prop_key | `&str` | `string` | `str` | Yes | Property key to declare. | | kind | `SecondaryIndexKind` | `{ kind: string, domain?: string }` | `str` plus optional `domain=` | Yes | Equality declaration or numeric range declaration. | @@ -1468,7 +1873,7 @@ range_info = db.ensure_node_property_index( | Rust | Node.js | Python | |------|---------|--------| -| `Result` | `JsNodePropertyIndexInfo` | `PyNodePropertyIndexInfo` | +| `Result` | `NodePropertyIndexInfo` | `NodePropertyIndexInfo` | The current declaration info. @@ -1478,7 +1883,7 @@ The current declaration info. - Range declarations use `SecondaryIndexKind::Range { domain: ... }`, `{ kind: 'range', domain: 'int' | 'uint' | 'float' }`, or `"range"` plus `domain="int" | "uint" | "float"`. - Re-ensuring an existing declaration returns the existing declaration info. - Re-ensuring a `Failed` declaration retries it by moving it back to `Building`. -- A `(type_id, prop_key)` pair may have at most one range declaration domain. Trying to ensure the same property with a different range domain returns an error. +- A `(label, prop_key)` pair may have at most one range declaration domain. Trying to ensure the same property with a different range domain returns an error. - A declaration becoming `Ready` is what enables declaration-backed routing. Callers do not switch to a different query method. --- @@ -1490,7 +1895,7 @@ Drops an optional node-property secondary index declaration. **Rust** ```rust let removed = db.drop_node_property_index( - USER, + "User", "role", SecondaryIndexKind::Equality, )?; @@ -1498,12 +1903,12 @@ let removed = db.drop_node_property_index( **Node.js** ```javascript -const removed = db.dropNodePropertyIndex(USER, 'role', { kind: 'equality' }); +const removed = db.dropNodePropertyIndex('User', 'role', { kind: 'equality' }); ``` **Python** ```python -removed = db.drop_node_property_index(USER, "role", "equality") +removed = db.drop_node_property_index("User", "role", "equality") ``` #### Parameters @@ -1520,7 +1925,7 @@ Same parameters and kind values as [`ensure_node_property_index`](#ensure_node_p #### Behavior -- Dropping a declaration removes the optional declaration state and future declaration-backed routing for that property. +- Dropping a declaration removes the optional declaration state and subsequent declaration-backed routing for that property. - Property queries continue to work after a drop. They fall back to scan through the same public query APIs. --- @@ -1531,7 +1936,7 @@ Lists all optional node-property secondary index declarations. **Rust** ```rust -let indexes = db.list_node_property_indexes(); +let indexes = db.list_node_property_indexes()?; ``` **Node.js** @@ -1548,7 +1953,7 @@ indexes = db.list_node_property_indexes() | Rust | Node.js | Python | |------|---------|--------| -| `Vec` | `Array` | `list[PyNodePropertyIndexInfo]` | +| `Vec` | `Array` | `list[NodePropertyIndexInfo]` | One entry per declaration. @@ -1561,7 +1966,7 @@ User-facing declaration information returned by [`ensure_node_property_index`](# | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| | index_id | `u64` | `indexId: number` | `index_id: int` | Stable declaration ID. | -| type_id | `u32` | `typeId: number` | `type_id: int` | Declared node type. | +| label | `String` | `label: string` | `label: str` | Declared node label. | | prop_key | `String` | `propKey: string` | `prop_key: str` | Declared property key. | | kind | `SecondaryIndexKind` | `kind: string` | `kind: str` | `equality` or `range`. | | domain | Encoded in `SecondaryIndexKind::Range` | `domain?: string` | `domain: str \| None` | Range domain for range declarations. Omitted / `None` for equality. | @@ -1575,6 +1980,130 @@ State meanings: --- +### ensure_edge_property_index + +Ensures an optional secondary index declaration for an edge property, scoped to one edge label. + +**Rust** +```rust +let eq = db.ensure_edge_property_index( + "WORKS_AT", + "role", + SecondaryIndexKind::Equality, +)?; + +let range = db.ensure_edge_property_index( + "WORKS_AT", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, +)?; +``` + +**Node.js** +```javascript +const eq = db.ensureEdgePropertyIndex('WORKS_AT', 'role', { kind: 'equality' }); + +const range = db.ensureEdgePropertyIndex('WORKS_AT', 'score', { + kind: 'range', + domain: 'int', +}); +``` + +**Python** +```python +eq = db.ensure_edge_property_index("WORKS_AT", "role", "equality") + +range_info = db.ensure_edge_property_index( + "WORKS_AT", + "score", + "range", + domain="int", +) +``` + +Parameters, kind values, lifecycle states, and domain validation match [`ensure_node_property_index`](#ensure_node_property_index), except `label` is the edge label. + +#### Returns + +| Rust | Node.js | Python | +|------|---------|--------| +| `Result` | `EdgePropertyIndexInfo` | `EdgePropertyIndexInfo` | + +#### Behavior + +- Edge property declarations are edge-label-scoped. A property filter without an edge label cannot use an edge-label-scoped edge-property declaration as a direct-query anchor. +- Ready edge declarations are candidate sources only. `query_edge_ids`, `query_edges`, and graph pattern execution still verify edge metadata and edge property predicates before returning results. +- Direct `EdgeQuery` anchor legality is unchanged: edge property indexes improve planning inside legal direct edge queries, but do not make filter-only direct edge queries legal by themselves. +- Graph patterns may choose a ready edge-property equality or range source as an edge anchor when it is cheaper than node-anchor expansion. + +--- + +### drop_edge_property_index + +Drops an optional edge-property secondary index declaration. + +**Rust** +```rust +let removed = db.drop_edge_property_index( + "WORKS_AT", + "role", + SecondaryIndexKind::Equality, +)?; +``` + +**Node.js** +```javascript +const removed = db.dropEdgePropertyIndex('WORKS_AT', 'role', { kind: 'equality' }); +``` + +**Python** +```python +removed = db.drop_edge_property_index("WORKS_AT", "role", "equality") +``` + +Same parameters and kind values as [`ensure_edge_property_index`](#ensure_edge_property_index). Returns `true` if a declaration existed and was removed, `false` otherwise. + +--- + +### list_edge_property_indexes + +Lists all optional edge-property secondary index declarations. + +**Rust** +```rust +let indexes = db.list_edge_property_indexes()?; +``` + +**Node.js** +```javascript +const indexes = db.listEdgePropertyIndexes(); +``` + +**Python** +```python +indexes = db.list_edge_property_indexes() +``` + +#### Returns + +| Rust | Node.js | Python | +|------|---------|--------| +| `Result, EngineError>` | `Array` | `list[EdgePropertyIndexInfo]` | + +One entry per edge declaration. + +--- + +### EdgePropertyIndexInfo + +User-facing declaration information returned by [`ensure_edge_property_index`](#ensure_edge_property_index) and [`list_edge_property_indexes`](#list_edge_property_indexes). + +Fields match [`NodePropertyIndexInfo`](#nodepropertyindexinfo), with the edge-label scope exposed as `label`: `index_id` / `indexId`, `label`, `prop_key` / `propKey`, `kind`, `domain`, `state`, and `last_error` / `lastError`. + +--- + ### PropertyRangeBound Bound object for [`find_nodes_range`](#find_nodes_range) and [`find_nodes_range_paged`](#find_nodes_range_paged). @@ -1593,8 +2122,8 @@ const upper = { value: 20, inclusive: false, domain: 'int' }; **Python** ```python -lower = PyPropertyRangeBound(10, domain="int") -upper = PyPropertyRangeBound(20, inclusive=False, domain="int") +lower = PropertyRangeBound(10, domain="int") +upper = PropertyRangeBound(20, inclusive=False, domain="int") ``` | Field | Rust | Node.js | Python | Description | @@ -1628,7 +2157,7 @@ PropertyRangeCursor { **Python** ```python -PyPropertyRangeCursor(20, 42, domain="int") +PropertyRangeCursor(20, 42, domain="int") ``` | Field | Rust | Node.js | Python | Description | @@ -1639,6 +2168,24 @@ PyPropertyRangeCursor(20, 42, domain="int") --- +### PropertyRangePageRequest (Rust only) + +Rust request object for [`find_nodes_range_paged`](#find_nodes_range_paged). + +```rust +PropertyRangePageRequest { + limit: Some(100), + after: None, +} +``` + +| Field | Rust | Description | +|-------|------|-------------| +| limit | `Option` | Maximum node IDs to return. `None` means no explicit page size. | +| after | `Option` | Cursor from the previous page. `None` starts at the lower bound. | + +--- + ### PropertyRangePageResult Result object returned by [`find_nodes_range_paged`](#find_nodes_range_paged). @@ -1646,7 +2193,7 @@ Result object returned by [`find_nodes_range_paged`](#find_nodes_range_paged). | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| | items | `Vec` | `Float64Array` | `IdArray` | Node IDs in range order for this page. | -| next_cursor | `Option` | `nextCursor?: JsPropertyRangeCursor` | `next_cursor: PyPropertyRangeCursor \| None` | Cursor for the next page. Omitted / `None` on the last page. | +| next_cursor | `Option` | `nextCursor?: PropertyRangeCursor` | `next_cursor: PropertyRangeCursor \| None` | Cursor for the next page. Omitted / `None` on the last page. | --- @@ -1656,12 +2203,12 @@ Equality and numeric range queries are index-transparent. Callers do not choose ### find_nodes -Finds all nodes of a given type where a specific property matches a given value (exact match). +Finds all nodes with a given label where a specific property matches a given value (exact match). **Rust** ```rust let ids = db.find_nodes( - USER, + "User", "role", &PropValue::String("admin".into()), )?; @@ -1669,19 +2216,19 @@ let ids = db.find_nodes( **Node.js** ```javascript -const ids = db.findNodes(USER, 'role', 'admin'); // Float64Array +const ids = db.findNodes('User', 'role', 'admin'); // Float64Array ``` **Python** ```python -ids = db.find_nodes(USER, "role", "admin") # IdArray +ids = db.find_nodes("User", "role", "admin") # IdArray ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| type_id | `u32` | `number` | `int` | Yes | Restrict search to this node type. | +| label | `&str` | `string` | `str` | Yes | Restrict search to this node label. | | prop_key | `&str` | `string` | `str` | Yes | Property key to match on. | | prop_value | `PropValue` | `any` | `Any` | Yes | Exact value to match. Type must match (string "1" does not match integer 1). | @@ -1691,20 +2238,20 @@ ids = db.find_nodes(USER, "role", "admin") # IdArray |------|---------|--------| | `Result, EngineError>` | `Float64Array` | `IdArray` | -Matching node IDs. If a matching equality declaration is `Ready`, OverGraph uses the declaration-backed index path. Otherwise it scans nodes of the requested type. +Matching node IDs. If a matching equality declaration is `Ready`, OverGraph uses the declaration-backed index path. Otherwise it scans nodes of the requested label. --- ### find_nodes_range -Finds all nodes of a given type where a numeric property falls within a range. +Finds all nodes with a given label where a numeric property falls within a range. Results are ordered by `(property_value asc, node_id asc)`. **Rust** ```rust let ids = db.find_nodes_range( - USER, + "User", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Excluded(PropValue::Int(20))), @@ -1714,7 +2261,7 @@ let ids = db.find_nodes_range( **Node.js** ```javascript const ids = db.findNodesRange( - USER, + 'User', 'score', { value: 10, inclusive: true, domain: 'int' }, { value: 20, inclusive: false, domain: 'int' }, @@ -1724,10 +2271,10 @@ const ids = db.findNodesRange( **Python** ```python ids = db.find_nodes_range( - USER, + "User", "score", - PyPropertyRangeBound(10, domain="int"), - PyPropertyRangeBound(20, inclusive=False, domain="int"), + PropertyRangeBound(10, domain="int"), + PropertyRangeBound(20, inclusive=False, domain="int"), ) ``` @@ -1735,10 +2282,10 @@ ids = db.find_nodes_range( | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| type_id | `u32` | `number` | `int` | Yes | Restrict search to this node type. | +| label | `&str` | `string` | `str` | Yes | Restrict search to this node label. | | prop_key | `&str` | `string` | `str` | Yes | Numeric property key to query. | -| lower | `Option<&PropertyRangeBound>` | `JsPropertyRangeBound \| null \| undefined` | `PyPropertyRangeBound \| None` | No | Lower bound. Omit for an unbounded start. | -| upper | `Option<&PropertyRangeBound>` | `JsPropertyRangeBound \| null \| undefined` | `PyPropertyRangeBound \| None` | No | Upper bound. Omit for an unbounded end. | +| lower | `Option<&PropertyRangeBound>` | `PropertyRangeBound \| null \| undefined` | `PropertyRangeBound \| None` | No | Lower bound. Omit for an unbounded start. | +| upper | `Option<&PropertyRangeBound>` | `PropertyRangeBound \| null \| undefined` | `PropertyRangeBound \| None` | No | Upper bound. Omit for an unbounded end. | #### Returns @@ -1754,32 +2301,32 @@ Matching node IDs in range order. - Numeric domains are exact. `int`, `uint`, and `float` are separate query domains. - If both bounds are present, they must use the same domain. - If a matching range declaration is `Ready`, OverGraph uses the declaration-backed range path. -- If no matching `Ready` declaration exists, OverGraph falls back to a scan of nodes of the requested type. +- If no matching `Ready` declaration exists, OverGraph falls back to a scan of nodes of the requested label. - Invalid bound combinations return an error. --- ### find_nodes_by_time_range -Finds all nodes of a given type with `updated_at` within a time range. +Finds all nodes with a given label and `updated_at` within a time range. ```rust -let ids = db.find_nodes_by_time_range(USER, start_ms, end_ms)?; +let ids = db.find_nodes_by_time_range("User", start_ms, end_ms)?; ``` ```javascript -const ids = db.findNodesByTimeRange(USER, startMs, endMs); +const ids = db.findNodesByTimeRange('User', startMs, endMs); ``` ```python -ids = db.find_nodes_by_time_range(USER, start_ms, end_ms) +ids = db.find_nodes_by_time_range("User", start_ms, end_ms) ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Description | |-----------|------|---------|--------|----------|-------------| -| type_id | `u32` | `number` | `int` | Yes | Node type. | +| label | `&str` | `string` | `str` | Yes | Node label. | | from_ms | `i64` | `number` | `int` | Yes | Start of range (inclusive), ms since epoch. | | to_ms | `i64` | `number` | `int` | Yes | End of range (inclusive), ms since epoch. | @@ -1791,22 +2338,23 @@ Node IDs matching the time range. Uses the timestamp index. ## Queries -Query APIs combine explicit IDs, keys, type constraints, property filters, timestamp filters, and +Query APIs combine explicit IDs, label-scoped keys, label/edge-label constraints, property filters, timestamp filters, and bounded graph patterns through normal function-call and object APIs. OverGraph still has no query string parser. +Query APIs return matching IDs, hydrated records, or bounded pattern ID bindings only. +Projection/query-row result APIs are not part of this surface. + Node queries use a recursive `filter` tree. -Top-level request fields such as `type_id` / `typeId`, `ids`, and `keys` are not part of the +Top-level request fields such as node `label_filter`, edge `label`, `ids`, and `keys` are not part of the filter tree. They are top-level constraints and are ANDed with the filter. Within `ids` and `keys`, values are OR alternatives. -Use `filter` for all node predicates in node queries and node patterns. The old node-level -connector `where` and `predicates` fields are no longer supported for node filters. Edge patterns -still support their existing edge-scoped `where` / `predicates` post-filters. +Use `filter` for all node and edge predicates. Query APIs use the same published read snapshot and visibility rules as direct read APIs. -Internally, OverGraph may use explicit IDs, key lookup, the node type index, ready property +Internally, OverGraph may use explicit IDs, key lookup, the node-label index, ready property equality/range indexes, the timestamp index, sorted intersection, sorted union, fallback scans, or bounded adjacency expansion. Candidate indexes are verified after candidate planning; indexes are never trusted as final truth. @@ -1822,8 +2370,9 @@ Node queries are the API-first query surface for combining top-level constraints node `filter` tree. They are useful when a request needs more than one constraint, when an index may help but should remain optional, or when the same filter should be explained. -Graph pattern queries use the same node filter model on pattern nodes, then expand bounded edge -constraints. Edge property checks remain edge-scoped post-filters in Phase 24. +Direct edge queries and graph pattern edge constraints use the canonical edge `filter` tree for +edge metadata and property predicates. Maintained edge-property indexes are used when available; +otherwise predicates are verified over the planned edge universe. ### Choosing the Right Query API @@ -1836,6 +2385,15 @@ shares the same plan and verifier as `query_node_ids`; only the final payload di Use [`explain_node_query`](#explain_node_query) to inspect the selected physical plan and warnings without executing the page. +Use [`query_edge_ids`](#query_edge_ids) when you need matching edge IDs from explicit edge IDs, edge-label +constraints, endpoint constraints, or an explicit full-scan opt-in. + +Use [`query_edges`](#query_edges) for the same edge query shape when you need hydrated edge records. +Metadata-only filters hydrate only the final page. Property filters hydrate bounded verifier +candidates. + +Use [`explain_edge_query`](#explain_edge_query) to inspect direct edge query planning. + Use direct property and time queries such as [`find_nodes`](#find_nodes), [`find_nodes_range`](#find_nodes_range), and [`find_nodes_by_time_range`](#find_nodes_by_time_range) when you already know you need one direct @@ -1853,7 +2411,10 @@ Runs a node query and returns matching node IDs. **Rust** ```rust let page = db.query_node_ids(&NodeQuery { - type_id: Some(USER), + label_filter: Some(NodeLabelFilter { + labels: vec!["User".into()], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::And(vec![ NodeFilterExpr::PropertyEquals { key: "status".into(), @@ -1873,7 +2434,7 @@ let page = db.query_node_ids(&NodeQuery { **Node.js** ```javascript const page = db.queryNodeIds({ - typeId: USER, + labelFilter: { labels: ['User'], mode: 'all' }, filter: { and: [ { property: 'status', eq: 'active' }, @@ -1887,7 +2448,7 @@ const page = db.queryNodeIds({ **Python** ```python page = db.query_node_ids({ - "type_id": USER, + "label_filter": {"labels": ["User"], "mode": "all"}, "filter": { "and": [ {"property": "status", "eq": "active"}, @@ -1908,7 +2469,7 @@ page = db.query_node_ids({ | Rust | Node.js | Python | |------|---------|--------| -| `Result` | `JsIdPageResult` | `PyIdPageResult` | +| `Result` | `IdPageResult` | `IdPageResult` | Result fields: @@ -1927,7 +2488,10 @@ matching nodes. **Rust** ```rust let page = db.query_nodes(&NodeQuery { - type_id: Some(USER), + label_filter: Some(NodeLabelFilter { + labels: vec!["Document".into(), "Published".into()], + mode: LabelMatchMode::All, + }), filter: Some(NodeFilterExpr::PropertyEquals { key: "status".into(), value: PropValue::String("active".into()), @@ -1940,7 +2504,7 @@ let page = db.query_nodes(&NodeQuery { **Node.js** ```javascript const page = db.queryNodes({ - typeId: MEMORY, + labelFilter: { labels: ['Document', 'Published'], mode: 'all' }, filter: { and: [ { property: 'status', in: ['active', 'trial'] }, @@ -1953,87 +2517,193 @@ const page = db.queryNodes({ }, ], }, - limit: 25, + limit: 25, +}); +``` + +**Python** +```python +page = db.query_nodes({ + "label_filter": {"labels": ["Document", "Published"], "mode": "all"}, + "filter": { + "and": [ + {"property": "status", "in": ["active", "trial"]}, + {"not": {"property": "archived_at", "exists": True}}, + { + "or": [ + {"property": "priority", "gte": 8}, + {"property": "source", "eq": "user"}, + ], + }, + ], + }, + "limit": 25, +}) +``` + +##### Parameters + +| Parameter | Rust | Node.js | Python | Required | Description | +|-----------|------|---------|--------|----------|-------------| +| request | `&NodeQuery` | `QueryNodeRequest` | `dict \| NodeQueryRequest` | Yes | Node query request. See [NodeQuery](#nodequery). | + +##### Returns + +| Rust | Node.js | Python | +|------|---------|--------| +| `Result` | `NodePageResult` | `NodePageResult` | + +Result fields: + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| items | `items: Vec` | `items: NodeView[]` | `items: list[NodeView]` | Hydrated final page of matching nodes. | +| cursor | `next_cursor: Option` | `nextCursor: number \| null` | `next_cursor: int \| None` | Cursor for the next page. Pass it as `after`. | + +Connector node records expose top-level fields eagerly. Property maps are converted only when the +`.props` getter is accessed. + +--- + +#### explain_node_query + +Returns the deterministic planner tree, estimates, and warnings for a node query. It applies the +same validation rules as execution. + +**Rust** +```rust +let plan = db.explain_node_query(&query)?; +``` + +**Node.js** +```javascript +const plan = db.explainNodeQuery({ + labelFilter: { labels: ['User'], mode: 'all' }, + filter: { property: 'status', eq: 'active' }, +}); +``` + +**Python** +```python +plan = db.explain_node_query({ + "label_filter": {"labels": ["User"], "mode": "all"}, + "filter": {"property": "status", "eq": "active"}, +}) +``` + +##### Returns + +| Rust | Node.js | Python | +|------|---------|--------| +| `Result` | `object` | `dict` | + +See [QueryPlan](#queryplan). + +--- + +### Direct Edge Queries + +#### query_edge_ids + +Runs a direct edge query and returns matching edge IDs in ascending edge ID order. + +**Rust** +```rust +let page = db.query_edge_ids(&EdgeQuery { + label: Some("WORKS_AT".into()), + from_ids: vec![person_id], + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::WeightRange { lower: Some(1.0), upper: None }, + EdgeFilterExpr::ValidAt { epoch_ms }, + ])), + page: PageRequest { limit: Some(100), after: None }, + ..Default::default() +})?; +``` + +**Node.js** +```javascript +const page = db.queryEdgeIds({ + label: 'WORKS_AT', + fromIds: [personId], + filter: { + and: [ + { weight: { gte: 1.0 } }, + { validAt: epochMs }, + ], + }, + limit: 100, }); ``` **Python** ```python -page = db.query_nodes({ - "type_id": MEMORY, +page = db.query_edge_ids({ + "label": "WORKS_AT", + "from_ids": [person_id], "filter": { "and": [ - {"property": "status", "in": ["active", "trial"]}, - {"not": {"property": "archived_at", "exists": True}}, - { - "or": [ - {"property": "priority", "gte": 8}, - {"property": "source", "eq": "user"}, - ], - }, + {"weight": {"gte": 1.0}}, + {"valid_at": epoch_ms}, ], }, - "limit": 25, + "limit": 100, }) ``` -##### Parameters - -| Parameter | Rust | Node.js | Python | Required | Description | -|-----------|------|---------|--------|----------|-------------| -| request | `&NodeQuery` | `QueryNodeRequest` | `dict \| NodeQueryRequest` | Yes | Node query request. See [NodeQuery](#nodequery). | - -##### Returns - -| Rust | Node.js | Python | -|------|---------|--------| -| `Result` | `JsNodePageResult` | `PyNodePageResult` | - -Result fields: +#### query_edges -| Field | Rust | Node.js | Python | Description | -|-------|------|---------|--------|-------------| -| items | `items: Vec` | `items: JsNodeRecord[]` | `items: list[PyNodeRecord]` | Hydrated final page of matching nodes. | -| cursor | `next_cursor: Option` | `nextCursor: number \| null` | `next_cursor: int \| None` | Cursor for the next page. Pass it as `after`. | +Runs the same direct edge query as [`query_edge_ids`](#query_edge_ids), then hydrates the final page +of matching edge records. -Connector node records expose top-level fields eagerly. Property maps are converted only when the -`.props` getter is accessed. +**Node.js** +```javascript +const page = db.queryEdges({ + label: 'WORKS_AT', + endpointIds: [personId], + filter: { property: 'role', eq: 'lead' }, + limit: 25, +}); +``` ---- +**Python** +```python +page = db.query_edges({ + "label": "WORKS_AT", + "endpoint_ids": [person_id], + "filter": {"property": "role", "eq": "lead"}, + "limit": 25, +}) +``` -#### explain_node_query +#### explain_edge_query -Returns the deterministic planner tree, estimates, and warnings for a node query. It applies the -same validation rules as execution. +Returns the deterministic planner tree, estimates, and warnings for a direct edge query. **Rust** ```rust -let plan = db.explain_node_query(&query)?; +let plan = db.explain_edge_query(&query)?; ``` **Node.js** ```javascript -const plan = db.explainNodeQuery({ - typeId: USER, - filter: { property: 'status', eq: 'active' }, -}); +const plan = db.explainEdgeQuery({ label: 'WORKS_AT', fromIds: [personId] }); ``` **Python** ```python -plan = db.explain_node_query({ - "type_id": USER, - "filter": {"property": "status", "eq": "active"}, -}) +plan = db.explain_edge_query({"label": "WORKS_AT", "from_ids": [person_id]}) ``` ##### Returns | Rust | Node.js | Python | |------|---------|--------| +| `Result` | `IdPageResult` | `IdPageResult` | +| `Result` | `EdgePageResult` | `EdgePageResult` | | `Result` | `object` | `dict` | -See [QueryPlan](#queryplan). +Rust `QueryEdgeIdsResult` contains `edge_ids: Vec` and `next_cursor: Option`. Rust `QueryEdgesResult` contains `edges: Vec` and `next_cursor: Option`. Node.js and Python page result objects use `items` for the returned IDs or edges. --- @@ -2045,8 +2715,9 @@ Runs a bounded, connected graph pattern query and returns ID bindings for node a aliases. Pattern v1 returns IDs only. Hydrate bound IDs with [`get_nodes`](#get_nodes) and [`get_edges`](#get_edges) when needed. -Node patterns use the same recursive `filter` tree as node queries. Edge pattern `where` / -`predicates` remain Phase 24 edge-scoped post-filters after bounded expansion. +Node patterns use the same recursive `filter` tree as node queries. Edge patterns use canonical +`filter` with the same shape as direct edge queries. Edge pattern `label_filter` is a simple +edge-label list, not a `NodeLabelFilter`. **Rust** ```rust @@ -2054,7 +2725,10 @@ let result = db.query_pattern(&GraphPatternQuery { nodes: vec![ NodePattern { alias: "person".into(), - type_id: Some(USER), + label_filter: Some(NodeLabelFilter { + labels: vec!["User".into(), "Admin".into()], + mode: LabelMatchMode::All, + }), ids: vec![], keys: vec![], filter: Some(NodeFilterExpr::Or(vec![ @@ -2070,7 +2744,10 @@ let result = db.query_pattern(&GraphPatternQuery { }, NodePattern { alias: "company".into(), - type_id: Some(COMPANY), + label_filter: Some(NodeLabelFilter { + labels: vec!["Company".into()], + mode: LabelMatchMode::All, + }), ids: vec![], keys: vec!["acme".into()], filter: None, @@ -2081,11 +2758,11 @@ let result = db.query_pattern(&GraphPatternQuery { from_alias: "person".into(), to_alias: "company".into(), direction: Direction::Outgoing, - type_filter: Some(vec![WORKS_AT]), - property_predicates: vec![EdgePostFilterPredicate::PropertyEquals { + label_filter: vec!["WORKS_AT".into()], + filter: Some(EdgeFilterExpr::PropertyEquals { key: "role".into(), value: PropValue::String("engineer".into()), - }], + }), }], at_epoch: None, limit: 100, @@ -2099,7 +2776,7 @@ const result = db.queryPattern({ nodes: [ { alias: 'person', - typeId: USER, + labelFilter: { labels: ['User'], mode: 'all' }, filter: { or: [ { property: 'status', eq: 'active' }, @@ -2109,7 +2786,7 @@ const result = db.queryPattern({ }, { alias: 'company', - typeId: COMPANY, + labelFilter: { labels: ['Company'], mode: 'all' }, keys: ['acme'], }, ], @@ -2119,10 +2796,8 @@ const result = db.queryPattern({ fromAlias: 'person', toAlias: 'company', direction: 'outgoing', - typeFilter: [WORKS_AT], - - // Edge post-filter remains Phase 23 edge-scoped shape. - where: { role: { eq: 'engineer' } }, + labelFilter: ['WORKS_AT'], + filter: { property: 'role', eq: 'engineer' }, }, ], limit: 100, @@ -2135,7 +2810,7 @@ result = db.query_pattern({ "nodes": [ { "alias": "person", - "type_id": USER, + "label_filter": {"labels": ["User"], "mode": "all"}, "filter": { "or": [ {"property": "status", "eq": "active"}, @@ -2145,7 +2820,7 @@ result = db.query_pattern({ }, { "alias": "company", - "type_id": COMPANY, + "label_filter": {"labels": ["Company"], "mode": "all"}, "keys": ["acme"], }, ], @@ -2155,10 +2830,8 @@ result = db.query_pattern({ "from_alias": "person", "to_alias": "company", "direction": "outgoing", - "type_filter": [WORKS_AT], - - # Edge post-filter remains edge-scoped. - "where": {"role": {"eq": "engineer"}}, + "label_filter": ["WORKS_AT"], + "filter": {"property": "role", "eq": "engineer"}, }, ], "limit": 100, @@ -2228,17 +2901,18 @@ Node query request fields: | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| -| type_id | `Option` | `typeId?: number` | `type_id?: int` | Optional type constraint. Required when using `keys`. | +| label_filter / labelFilter | `Option` | `labelFilter?: { labels: string[], mode: "any" \| "all" }` | `label_filter?: {"labels": list[str], "mode": "any" \| "all"}` | Node-label constraint with explicit `Any` / `All` semantics. A one-label `All` filter uses the single-label fast path. | | ids | `Vec` | `ids?: number[]` | `ids?: list[int]` | Explicit node ID candidates. OR within the list. | -| keys | `Vec` | `keys?: string[]` | `keys?: list[str]` | Type-scoped key candidates. OR within the list. | +| keys | `Vec` | `keys?: string[]` | `keys?: list[str]` | Label-scoped key candidates. OR within the list. | | filter | `Option` | `filter?: QueryNodeFilter \| null` | `filter?: QueryNodeFilter \| None` | Recursive node filter tree. Omit or pass null/None for no filter. | | limit | `page.limit` | `limit?: number` | `limit?: int` | Page size. Omit for unlimited. Connector `0` means unlimited. | | after | `page.after` | `after?: number` | `after?: int` | Cursor from a previous page. Returns items with node IDs strictly greater than `after`. | -| allow_full_scan | `bool` | `allowFullScan?: boolean` | `allow_full_scan?: bool` | Required for type-less full scan fallback. | +| allow_full_scan | `bool` | `allowFullScan?: boolean` | `allow_full_scan?: bool` | Required for unanchored full scan fallback. | -Top-level `type_id` / `typeId`, `ids`, and `keys` are ANDed with `filter`. Type-less verify-only -filters require `allow_full_scan` / `allowFullScan` unless `ids` or `keys` provide a legal bounded -universe. +Top-level `label_filter` / `labelFilter`, `ids`, and `keys` are ANDed with `filter`. +A one-label `All` filter uses the direct node-label fast path. Key lookups require exactly one resolved node label. +Label-less verify-only filters require `allow_full_scan` / `allowFullScan` unless `ids` or `keys` +provide a legal bounded universe. --- @@ -2289,13 +2963,13 @@ The built-in timestamp filter is a structural field. User property names always ```javascript // Built-in node timestamp: db.queryNodeIds({ - typeId: NOTE, + labelFilter: { labels: ['Document'], mode: 'all' }, filter: { updatedAt: { gte: startMs, lt: endMs } }, }); // User property literally named "updatedAt": db.queryNodeIds({ - typeId: NOTE, + labelFilter: { labels: ['Document'], mode: 'all' }, filter: { property: 'updatedAt', eq: 'manual-value' }, }); ``` @@ -2304,19 +2978,68 @@ db.queryNodeIds({ ```python # Built-in node timestamp: db.query_node_ids({ - "type_id": NOTE, + "label_filter": {"labels": ["Document"], "mode": "all"}, "filter": {"updated_at": {"gte": start_ms, "lt": end_ms}}, }) # User property literally named "updated_at": db.query_node_ids({ - "type_id": NOTE, + "label_filter": {"labels": ["Document"], "mode": "all"}, "filter": {"property": "updated_at", "eq": "manual-value"}, }) ``` --- +#### EdgeQuery + +Direct edge query request fields: + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| label | `Option` | `label?: string` | `label?: str` | Optional edge-label constraint. | +| ids | `Vec` | `ids?: number[]` | `ids?: list[int]` | Explicit edge ID candidates. OR within the list. | +| from_ids | `Vec` | `fromIds?: number[]` | `from_ids?: list[int]` | Source endpoint candidates. OR within the list. | +| to_ids | `Vec` | `toIds?: number[]` | `to_ids?: list[int]` | Target endpoint candidates. OR within the list. | +| endpoint_ids | `Vec` | `endpointIds?: number[]` | `endpoint_ids?: list[int]` | Either-endpoint candidates. OR within the list. | +| filter | `Option` | `filter?: QueryEdgeFilter \| null` | `filter?: QueryEdgeFilter \| None` | Recursive edge filter tree. | +| limit | `page.limit` | `limit?: number` | `limit?: int` | Page size. Omit for unlimited. Connector `0` means unlimited. | +| after | `page.after` | `after?: number` | `after?: int` | Cursor from a previous page. Returns edge IDs strictly greater than `after`. | +| allow_full_scan | `bool` | `allowFullScan?: boolean` | `allow_full_scan?: bool` | Required for direct filter-only or unanchored full scans. | + +Top-level edge anchors are ANDed with `filter`; values inside each list are ORed. A filter-only +direct edge query requires explicit full-scan opt-in even when metadata sidecars are available. + +--- + +#### EdgeFilter / QueryEdgeFilter + +Rust uses `EdgeFilterExpr`. Node.js and Python use the canonical recursive `QueryEdgeFilter` +object shape. + +| Filter shape | Node.js | Python | Meaning | +|--------------|---------|--------|---------| +| Equality | `{ property: "role", eq: "lead" }` | `{"property": "role", "eq": "lead"}` | Edge property exactly equals value | +| IN | `{ property: "role", in: ["lead", "owner"] }` | `{"property": "role", "in": ["lead", "owner"]}` | Edge property equals any listed value | +| Range | `{ property: "score", gte: 50 }` | `{"property": "score", "gte": 50}` | Edge property range comparison | +| Exists | `{ property: "role", exists: true }` | `{"property": "role", "exists": True}` | Edge property key is present | +| Missing | `{ property: "role", missing: true }` | `{"property": "role", "missing": True}` | Edge property key is absent | +| Weight range | `{ weight: { gte: 1.0 } }` | `{"weight": {"gte": 1.0}}` | Built-in edge weight range | +| Updated-at range | `{ updatedAt: { gte: ms } }` | `{"updated_at": {"gte": ms}}` | Built-in edge update timestamp range | +| Valid-at | `{ validAt: ms }` | `{"valid_at": ms}` | Half-open validity check: `valid_from <= ms < valid_to` | +| Valid-from range | `{ validFrom: { gte: ms } }` | `{"valid_from": {"gte": ms}}` | Built-in `valid_from` range | +| Valid-to range | `{ validTo: { gt: ms } }` | `{"valid_to": {"gt": ms}}` | Built-in `valid_to` range | +| AND | `{ and: [filter, ...] }` | `{"and": [filter, ...]}` | All children must match | +| OR | `{ or: [filter, ...] }` | `{"or": [filter, ...]}` | Any child may match | +| NOT | `{ not: filter }` | `{"not": filter}` | Child must not match | + +Weight ranges reject NaN. `-0.0` and `+0.0` compare as the same value. Ready edge-property +declarations may provide equality, `IN`, and range candidate sources for edge-label-scoped edge filters; +metadata filters may use private edge metadata sources when available. All edge filters still run +final verification for correctness. + +--- + #### GraphPatternQuery Pattern request fields: @@ -2334,9 +3057,9 @@ Node pattern fields: | Field | Description | |-------|-------------| | alias | Non-empty unique node alias. | -| type_id / typeId | Optional node type constraint. | +| label_filter | `NodeLabelFilter` with explicit `Any` / `All` semantics. A one-label `All` filter uses the single-label fast path. | | ids | Explicit node IDs. | -| keys | Type-scoped keys. Requires `type_id` / `typeId`. | +| keys | Label-scoped keys. Requires exactly one resolved node label from `label_filter`. | | filter | Recursive node filter tree. Omit/null/None means no node filter. | Edge pattern fields: @@ -2347,8 +3070,8 @@ Edge pattern fields: | from_alias / fromAlias | Source alias in the pattern direction. | | to_alias / toAlias | Target alias in the pattern direction. | | direction | `outgoing`, `incoming`, or `both`, relative to `from_alias`. | -| type_filter / typeFilter | Optional edge type list. | -| predicates / where | Bounded edge property post-filters. | +| label_filter / labelFilter | Optional edge-label list. | +| filter | Canonical recursive edge filter tree. | Pattern validation: @@ -2357,10 +3080,9 @@ Pattern validation: - Pattern v1 must be one connected component with at least one edge. - Distinct node aliases bind distinct node IDs. - Reusing the same alias in multiple edges means the same node binding. -- Unbounded initial nodes without a legal bounded universe are rejected. A type-less verify-only +- Unbounded initial nodes without a legal bounded universe are rejected. A label-less verify-only target filter is legal only after bounded edge expansion has produced target IDs. -- Edge property `where` / `predicates` are edge-scoped post-filters. Phase 24 does not support - `filter` on edge patterns. +- Edge `filter` is canonical. --- @@ -2370,10 +3092,12 @@ Explain APIs return: | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| -| kind | `kind` | `kind` | `kind` | `node_query` or `pattern_query` in connectors. | +| kind | `kind` | `kind` | `kind` | `node_query`, `edge_query`, or `pattern_query`. | | root | `root` | `root` | `root` | Recursive plan node. | | estimated candidates | `estimated_candidates` | `estimatedCandidates` | `estimated_candidates` | Optional candidate count estimate. | -| warnings | `warnings` | `warnings` | `warnings` | Stable lower_snake warning strings in connectors. | +| warnings | `warnings` | `warnings` | `warnings` | Stable lower_snake warning strings. | +| notes | `notes` | `notes` | `notes` | Stable lower_snake informational planner notes. | +| public inputs | `public_inputs` | `publicInputs` | `public_inputs` | Normalized public node-label and edge-label names referenced during planning. | Plan node kinds include: @@ -2381,19 +3105,34 @@ Plan node kinds include: |----------------|---------| | `empty_result` | Impossible filter or empty candidate universe. | | `explicit_ids` | Explicit ID candidate universe. | -| `key_lookup` | Type-scoped key lookup. | -| `node_type_index` | Type index candidate source. | +| `key_lookup` | Label-scoped key lookup. | +| `node_label_index` | Label index candidate source. | +| `node_label_any_index` | Node-label `Any` candidate source. | | `property_equality_index` | Ready equality property index candidate source. | | `property_range_index` | Ready range property index candidate source. | | `timestamp_index` | Built-in timestamp index candidate source. | +| `explicit_edge_ids` | Explicit edge ID candidate universe. | +| `edge_label_index` | Edge label index candidate source. | +| `edge_triple_index` | Exact `(from, to, label)` edge lookup source. | +| `edge_endpoint_adjacency` | Endpoint adjacency candidate source. | +| `edge_weight_index` | Optional edge weight sidecar candidate source. | +| `edge_updated_at_index` | Optional edge update-time sidecar candidate source. | +| `edge_validity_index` | Optional edge validity sidecar candidate source. | +| `edge_metadata_scan` | Edge metadata scan candidate source. | +| `edge_property_equality_index` | Ready edge-property equality declaration candidate source. | +| `edge_property_range_index` | Ready edge-property range declaration candidate source. | | `intersect` | Sorted intersection of bounded candidate sources. | | `union` | Sorted union of bounded OR/IN candidate sources. | | `verify_node_filter` | Final visible-record verification of the full node filter. | +| `verify_edge_filter` | Final visible-edge metadata/property verification. | | `adjacency_expansion` | Bounded graph-pattern edge expansion. | | `pattern_expand` | Pattern execution expansion step. | +| `pattern_edge_anchor` | Pattern execution started from a planned edge source. | | `verify_edge_predicates` | Edge post-filter verification. | -| `fallback_type_scan` | Type-scoped scan universe. | +| `fallback_node_label_scan` | Label-scoped scan universe. | | `fallback_full_node_scan` | Explicit full node scan universe. | +| `fallback_edge_label_scan` | Edge-label-scoped edge scan universe. | +| `fallback_full_edge_scan` | Explicit full edge scan universe. | Warning strings include: @@ -2412,6 +3151,17 @@ Warning strings include: | `verify_only_filter` | Some filter subtree ran only through verification. | | `boolean_branch_fallback` | Boolean branch or OR was cheaper or safer as verifier fallback. | | `planning_probe_budget_exceeded` | Planning probe/union budget forced fallback. | +| `unknown_node_label` | A requested node label is not present in the catalog. | +| `unknown_edge_label` | A requested edge label is not present in the catalog. | + +Note strings include: + +| Note | Meaning | +|------|---------| +| `node_label_any_dedupe_before_pagination` | `Any` node-label planning deduplicates candidates before pagination. | +| `node_label_any_final_verification` | `Any` node-label results are verified against final visible node records. | +| `node_label_all_superset_verification` | `All` node-label planning used a superset index source followed by final verification. | +| `stale_node_label_membership_verification` | Node-label index membership may include stale entries and is verified against visible records. | --- @@ -2454,36 +3204,42 @@ The pattern is the same across all paginated methods: - `find_nodes_range_paged` uses a structured range cursor keyed by `(value, node_id)`. - The result includes `items` and `next_cursor` (`None`/`null` when there are no more pages). -### nodes_by_type_paged +### nodes_by_labels_paged -Paginated version of [`nodes_by_type`](#nodes_by_type). Returns IDs only. +Paginated node-label scan. Returns IDs only. ```rust -let page = db.nodes_by_type_paged(USER, &PageRequest { limit: Some(100), after: None })?; +let page = db.nodes_by_labels_paged("User", &PageRequest { limit: Some(100), after: None })?; +let admin_page = db.nodes_by_labels_paged( + vec!["User".into(), "Admin".into()], + &PageRequest { limit: Some(100), after: None }, +)?; // page.items: Vec, page.next_cursor: Option ``` ```javascript -let page = db.nodesByTypePaged(USER, 100); // limit=100, no cursor +let page = db.nodesByLabelsPaged('User', 100); // limit=100, no cursor +let adminPage = db.nodesByLabelsPaged(['User', 'Admin'], 100); // page = { items: Float64Array, nextCursor: number | null } // Next page: -page = db.nodesByTypePaged(USER, 100, page.nextCursor); +page = db.nodesByLabelsPaged('User', 100, page.nextCursor); ``` ```python -page = db.nodes_by_type_paged(USER, limit=100) +page = db.nodes_by_labels_paged("User", limit=100) +admin_page = db.nodes_by_labels_paged(["User", "Admin"], limit=100) # page.items: IdArray, page.next_cursor: int | None # Next page: -page = db.nodes_by_type_paged(USER, limit=100, after=page.next_cursor) +page = db.nodes_by_labels_paged("User", limit=100, after=page.next_cursor) ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Default | Description | |-----------|------|---------|--------|----------|---------|-------------| -| type_id | `u32` | `number` | `int` | Yes | — | Node type. | +| labels | `impl IntoNodeLabels` | `string \| string[]` | `str \| list[str]` | Yes | — | Label or labels to match. Nodes must contain every supplied node label. | | limit | `Option` | `number` | `int` | No | Unlimited | Maximum items per page. | | after | `Option` | `number` | `int` | No | `None` (start from beginning) | Cursor. Returns items with IDs strictly greater than this value. Use `next_cursor` from a previous result. | @@ -2496,36 +3252,118 @@ page = db.nodes_by_type_paged(USER, limit=100, after=page.next_cursor) --- -### edges_by_type_paged +### edges_by_label_paged + +Paginated edge-label scan. Returns edge IDs only. + +```rust +let page = db.edges_by_label_paged( + "WORKS_ON", + &PageRequest { limit: Some(100), after: None }, +)?; +// page.items: Vec, page.next_cursor: Option +``` + +```javascript +let page = db.edgesByLabelPaged('WORKS_ON', 100); // limit=100, no cursor +// page = { items: Float64Array, nextCursor: number | null } + +// Next page: +page = db.edgesByLabelPaged('WORKS_ON', 100, page.nextCursor); +``` + +```python +page = db.edges_by_label_paged("WORKS_ON", limit=100) +# page.items: IdArray, page.next_cursor: int | None + +# Next page: +page = db.edges_by_label_paged("WORKS_ON", limit=100, after=page.next_cursor) +``` + +#### Parameters + +| Parameter | Rust | Node.js | Python | Required | Default | Description | +|-----------|------|---------|--------|----------|---------|-------------| +| label | `&str` | `string` | `str` | Yes | - | Public edge label to match. | +| limit | `Option` | `number` | `int` | No | Unlimited | Maximum edge IDs per page. | +| after | `Option` | `number` | `int` | No | `None` (start from beginning) | Cursor. Returns edge IDs strictly greater than this value. Use `next_cursor` from a previous result. | + +#### Returns: PageResult + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| items | `Vec` | `Float64Array` | `IdArray` | Edge IDs in this page. | +| next_cursor | `Option` | `number \| null` | `int \| None` | Cursor for the next page. `None`/`null` means this is the last page. | -Paginated version of [`edges_by_type`](#edges_by_type). Same pattern as `nodes_by_type_paged`. +Unknown edge labels return an empty page. Tombstoned edges are excluded. Paged edge-label scans are ordered by edge ID. --- -### get_nodes_by_type_paged +### get_nodes_by_labels_paged -Paginated version of [`get_nodes_by_type`](#get_nodes_by_type). Returns full `NodeRecord` objects. +Paginated hydrated node-label scan. Returns full node records. ```rust -let page = db.get_nodes_by_type_paged(USER, &PageRequest { limit: Some(50), after: None })?; -// page.items: Vec +let page = db.get_nodes_by_labels_paged("User", &PageRequest { limit: Some(50), after: None })?; +let admin_page = db.get_nodes_by_labels_paged( + vec!["User".into(), "Admin".into()], + &PageRequest { limit: Some(50), after: None }, +)?; +// page.items: Vec ``` ```javascript -const page = db.getNodesByTypePaged(USER, 50); -// page.items: NodeRecord[] +const page = db.getNodesByLabelsPaged('User', 50); +const adminPage = db.getNodesByLabelsPaged(['User', 'Admin'], 50); +// page.items: NodeView[] ``` ```python -page = db.get_nodes_by_type_paged(USER, limit=50) -# page.items: list[NodeRecord] +page = db.get_nodes_by_labels_paged("User", limit=50) +admin_page = db.get_nodes_by_labels_paged(["User", "Admin"], limit=50) +# page.items: list[NodeView] ``` --- -### get_edges_by_type_paged +### get_edges_by_label_paged + +Paginated hydrated edge-label scan. Returns full edge records. + +```rust +let page = db.get_edges_by_label_paged( + "WORKS_ON", + &PageRequest { limit: Some(50), after: None }, +)?; +// page.items: Vec +``` + +```javascript +const page = db.getEdgesByLabelPaged('WORKS_ON', 50); +// page.items: EdgeView[] +``` + +```python +page = db.get_edges_by_label_paged("WORKS_ON", limit=50) +# page.items: list[EdgeView] +``` + +#### Parameters + +| Parameter | Rust | Node.js | Python | Required | Default | Description | +|-----------|------|---------|--------|----------|---------|-------------| +| label | `&str` | `string` | `str` | Yes | - | Public edge label to match. | +| limit | `Option` | `number` | `int` | No | Unlimited | Maximum edge records per page. | +| after | `Option` | `number` | `int` | No | `None` (start from beginning) | Cursor. Returns edge records with IDs strictly greater than this value. Use `next_cursor` from a previous result. | + +#### Returns: PageResult + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| items | `Vec` | `EdgeView[]` | `list[EdgeView]` | Full edge records in this page. | +| next_cursor | `Option` | `number \| null` | `int \| None` | Cursor for the next page. `None`/`null` means this is the last page. | -Paginated version of [`get_edges_by_type`](#get_edges_by_type). Returns full `EdgeRecord` objects. +Unknown edge labels return an empty page. Tombstoned edges are excluded. The implementation pages IDs first and hydrates only the requested page. --- @@ -2535,7 +3373,7 @@ Paginated version of [`find_nodes`](#find_nodes). ```rust let page = db.find_nodes_paged( - USER, + "User", "role", &PropValue::String("admin".into()), &PageRequest { limit: Some(50), after: None }, @@ -2543,11 +3381,11 @@ let page = db.find_nodes_paged( ``` ```javascript -const page = db.findNodesPaged(USER, 'role', 'admin', { limit: 50 }); +const page = db.findNodesPaged('User', 'role', 'admin', { limit: 50 }); ``` ```python -page = db.find_nodes_paged(USER, "role", "admin", limit=50) +page = db.find_nodes_paged("User", "role", "admin", limit=50) ``` --- @@ -2559,7 +3397,7 @@ Paginated version of [`find_nodes_range`](#find_nodes_range). **Rust** ```rust let page = db.find_nodes_range_paged( - USER, + "User", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Excluded(PropValue::Int(20))), @@ -2573,7 +3411,7 @@ let page = db.find_nodes_range_paged( **Node.js** ```javascript const page = db.findNodesRangePaged( - USER, + 'User', 'score', { value: 10, inclusive: true, domain: 'int' }, { value: 20, inclusive: false, domain: 'int' }, @@ -2584,10 +3422,10 @@ const page = db.findNodesRangePaged( **Python** ```python page = db.find_nodes_range_paged( - USER, + "User", "score", - PyPropertyRangeBound(10, domain="int"), - PyPropertyRangeBound(20, inclusive=False, domain="int"), + PropertyRangeBound(10, domain="int"), + PropertyRangeBound(20, inclusive=False, domain="int"), limit=50, ) ``` @@ -2596,26 +3434,26 @@ page = db.find_nodes_range_paged( | Parameter | Rust | Node.js | Python | Required | Default | Description | |-----------|------|---------|--------|----------|---------|-------------| -| type_id | `u32` | `number` | `int` | Yes | — | Restrict search to this node type. | +| label | `&str` | `string` | `str` | Yes | — | Restrict search to this node label. | | prop_key | `&str` | `string` | `str` | Yes | — | Numeric property key to query. | -| lower | `Option<&PropertyRangeBound>` | `JsPropertyRangeBound \| null \| undefined` | `PyPropertyRangeBound \| None` | No | Unbounded | Lower bound. | -| upper | `Option<&PropertyRangeBound>` | `JsPropertyRangeBound \| null \| undefined` | `PyPropertyRangeBound \| None` | No | Unbounded | Upper bound. | +| lower | `Option<&PropertyRangeBound>` | `PropertyRangeBound \| null \| undefined` | `PropertyRangeBound \| None` | No | Unbounded | Lower bound. | +| upper | `Option<&PropertyRangeBound>` | `PropertyRangeBound \| null \| undefined` | `PropertyRangeBound \| None` | No | Unbounded | Upper bound. | | limit | `Option` | `number` | `int` | No | Unlimited | Maximum items per page. | -| after | `Option` | `JsPropertyRangeCursor` | `PyPropertyRangeCursor` | No | `None` | Cursor from a previous range page. Reuse the same query arguments when resuming. | +| after | `Option` | `PropertyRangeCursor` | `PropertyRangeCursor` | No | `None` | Cursor from a previous range page. Reuse the same query arguments when resuming. | #### Returns: PropertyRangePageResult | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| | items | `Vec` | `Float64Array` | `IdArray` | Node IDs in range order for this page. | -| next_cursor | `Option` | `JsPropertyRangeCursor \| null \| undefined` | `PyPropertyRangeCursor \| None` | Cursor for the next page, or no cursor on the last page. | +| next_cursor | `Option` | `PropertyRangeCursor \| null \| undefined` | `PropertyRangeCursor \| None` | Cursor for the next page, or no cursor on the last page. | #### Behavior - At least one bound is required. - Numeric domains are exact. `int`, `uint`, and `float` are separate query domains. - If both bounds are present, they must use the same domain. -- When resuming with `after`, keep the same `type_id`, `prop_key`, bounds, and domain. +- When resuming with `after`, keep the same `label`, `prop_key`, bounds, and domain. - Invalid bound or cursor combinations return an error. --- @@ -2625,11 +3463,11 @@ page = db.find_nodes_range_paged( Paginated version of [`find_nodes_by_time_range`](#find_nodes_by_time_range). ```javascript -const page = db.findNodesByTimeRangePaged(USER, startMs, endMs, { limit: 50 }); +const page = db.findNodesByTimeRangePaged('User', startMs, endMs, { limit: 50 }); ``` ```python -page = db.find_nodes_by_time_range_paged(USER, start_ms, end_ms, limit=50) +page = db.find_nodes_by_time_range_paged("User", start_ms, end_ms, limit=50) ``` --- @@ -2644,7 +3482,7 @@ Retrieves the immediate neighbors of a node (one hop). The most common graph tra ```rust let entries = db.neighbors(node_id, &NeighborOptions { direction: Direction::Outgoing, - type_filter: Some(vec![WORKS_ON]), + edge_label_filter: Some(vec!["WORKS_ON".into()]), limit: Some(10), at_epoch: None, decay_lambda: None, @@ -2659,7 +3497,7 @@ for entry in &entries { ```javascript const list = db.neighbors(nodeId, { direction: 'outgoing', - typeFilter: [WORKS_ON], + edgeLabelFilter: ['WORKS_ON'], limit: 10, }); @@ -2670,7 +3508,7 @@ for (const n of list) { **Python** ```python -entries = db.neighbors(node_id, direction="outgoing", type_filter=[WORKS_ON], limit=10) +entries = db.neighbors(node_id, direction="outgoing", edge_label_filter=["WORKS_ON"], limit=10) for entry in entries: print(entry.node_id, entry.edge_id, entry.weight) ``` @@ -2681,10 +3519,10 @@ for entry in entries: |-----------|------|---------|--------|----------|---------|-------------| | node_id | `u64` | `number` | `int` | Yes | — | Node to query neighbors for. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Traversal direction. `"outgoing"`, `"incoming"`, or `"both"`. | -| type_filter | `Option>` | `number[]` | `list[int]` | No | `None` (all types) | Only return neighbors connected by edges of these types. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` (all labels) | Only return neighbors connected by edges with these labels. | | limit | `Option` | `number` | `int` | No | `None` (unlimited) | Maximum number of neighbors to return. | | at_epoch | `Option` | `number` | `int` | No | `None` (current time) | Temporal filter. Only edges whose validity window contains this timestamp are included. `None` means the current wall-clock time. | -| decay_lambda | `Option` | `number` | `float` | No | `None` (no decay) | Exponential decay factor. When set, each neighbor's weight is multiplied by `exp(-λ × age_ms)` where `age_ms` is the edge's age. Produces a time-decayed relevance score. | +| decay_lambda | `Option` | `number` | `float` | No | `None` (no decay) | Exponential decay factor. When set, each neighbor's weight is multiplied by `exp(-λ × age_hours)` where `age_hours = max(at_epoch - valid_from, 0) / 3_600_000`. | #### Returns: NeighborEntry @@ -2692,12 +3530,12 @@ for entry in entries: |-------|------|---------|--------|-------------| | node_id | `u64` | `number` | `int` | ID of the neighboring node. | | edge_id | `u64` | `number` | `int` | ID of the connecting edge. | -| edge_type_id | `u32` | `number` | `int` | Type of the connecting edge. | +| label | `String` | `label: string` | `label: str` | Label of the connecting edge. | | weight | `f32` | `number` | `float` | Edge weight (or decay-adjusted score if `decay_lambda` is set). | | valid_from | `i64` | `number` | `int` | Edge validity start (ms). | | valid_to | `i64` | `number` | `int` | Edge validity end (ms). | -**Node.js**: Returns `JsNeighborEntry[]` as plain objects, so you can use normal array access like `list[i].nodeId`. +**Node.js**: Returns `NeighborEntry[]` as plain objects, so you can use normal array access like `list[i].nodeId`. #### Performance @@ -2711,7 +3549,7 @@ Paginated version of [`neighbors`](#neighbors). ```javascript let page = db.neighborsPaged(nodeId, { direction: 'outgoing', limit: 20 }); -// page.items: JsNeighborEntry[], page.nextCursor: number | null +// page.items: NeighborEntry[], page.nextCursor: number | null console.log(page.items[0].nodeId); // Next page: @@ -2747,7 +3585,7 @@ let results = db.neighbors_batch(&[1, 2, 3], &NeighborOptions::default())?; **Node.js** ```javascript const results = db.neighborsBatch([1, 2, 3], { direction: 'outgoing' }); -// results: { queryNodeId: number, neighbors: JsNeighborEntry[] }[] +// results: { queryNodeId: number, neighbors: NeighborEntry[] }[] console.log(results[0].neighbors[0].nodeId); ``` @@ -2763,9 +3601,9 @@ results = db.neighbors_batch([1, 2, 3], direction="outgoing") |-----------|------|---------|--------|----------|-------------| | node_ids | `&[u64]` | `number[]` | `list[int]` | Yes | Node IDs to query neighbors for. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Traversal direction. | -| type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Edge type filter. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Edge label filter. | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter. | -| decay_lambda | `Option` | `number` | `float` | No | `None` | Decay factor. | +| decay_lambda | `Option` | `number` | `float` | No | `None` | Decay factor. Uses hours from `valid_from` when set. | #### Returns @@ -2781,7 +3619,7 @@ Returns the top K neighbors of a node ranked by a scoring criterion. ```rust let top = db.top_k_neighbors(node_id, 5, &TopKOptions { direction: Direction::Outgoing, - scoring: ScoringMode::Weight, + scoring: ScoringMode::DecayAdjusted { lambda: 0.01 }, ..Default::default() })?; ``` @@ -2790,14 +3628,21 @@ let top = db.top_k_neighbors(node_id, 5, &TopKOptions { ```javascript const top = db.topKNeighbors(nodeId, 5, { direction: 'outgoing', - scoring: 'weight', + scoring: 'decay', + decayLambda: 0.01, }); console.log(top[0].nodeId, top[0].weight); ``` **Python** ```python -top = db.top_k_neighbors(node_id, 5, direction="outgoing", scoring="weight") +top = db.top_k_neighbors( + node_id, + 5, + direction="outgoing", + scoring="decay", + decay_lambda=0.01, +) ``` #### Parameters @@ -2807,18 +3652,18 @@ top = db.top_k_neighbors(node_id, 5, direction="outgoing", scoring="weight") | node_id | `u64` | `number` | `int` | Yes | — | Source node. | | k | `usize` | `number` | `int` | Yes | — | Number of top neighbors to return. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Traversal direction. | -| type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Edge type filter. | -| scoring | `ScoringMode` | `string` | `str` | No | `Weight` | Scoring criterion. See below. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Edge label filter. | +| scoring | `ScoringMode` | `string` | `str` | No | `Weight` | Scoring criterion. Rust carries the decay lambda inside `ScoringMode::DecayAdjusted { lambda }`; connectors use `scoring: "decay"` plus `decayLambda` / `decay_lambda`. | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter. | -| decay_lambda | `Option` | `number` | `float` | No | `None` | Required when `scoring = "decay"`. | +| decay_lambda | — | `number` | `float` | No | `None` | Connector-only option required when `scoring = "decay"`. | **Scoring modes:** | Mode | Rust | Node.js / Python | Description | |------|------|------------------|-------------| | Weight | `ScoringMode::Weight` | `"weight"` | Rank by edge weight (descending). | -| RecencyDecay | `ScoringMode::RecencyDecay` | `"recency"` | Rank by recency. More recent edges score higher. | -| — | — | `"decay"` | Exponential decay: `weight × exp(-λ × age_ms)`. Requires `decay_lambda`. | +| Recency | `ScoringMode::Recency` | `"recency"` | Rank by recency. More recent edges score higher. | +| DecayAdjusted | `ScoringMode::DecayAdjusted { lambda }` | `"decay"` | Exponential decay: `weight × exp(-λ × age_hours)`, where `age_hours = max(at_epoch - valid_from, 0) / 3_600_000`. Connectors require `decay_lambda`. | #### Returns @@ -2828,22 +3673,25 @@ Array of `NeighborEntry` sorted by score descending. Length is `min(k, actual_ne ### traverse -Breadth-first traversal from a starting node up to a maximum depth. Supports pagination, type filtering, temporal filtering, and decay scoring. +Breadth-first traversal from a starting node up to a maximum depth. Supports pagination, edge-label filtering, emission-only node-label filtering, temporal filtering, and decay scoring. **Rust** ```rust let result = db.traverse(start_id, &TraverseOptions { min_depth: 1, direction: Direction::Outgoing, - edge_type_filter: Some(vec![WORKS_ON]), - node_type_filter: None, + edge_label_filter: Some(vec!["WORKS_ON".into()]), + emit_node_label_filter: Some(NodeLabelFilter { + labels: vec!["User".into(), "Admin".into()], + mode: LabelMatchMode::Any, + }), at_epoch: None, decay_lambda: None, limit: Some(100), cursor: None, })?; -for hit in &result { +for hit in &result.items { println!("node={}, depth={}", hit.node_id, hit.depth); } ``` @@ -2853,7 +3701,8 @@ for hit in &result { const result = db.traverse(startId, 3, { minDepth: 1, direction: 'outgoing', - edgeTypeFilter: [WORKS_ON], + edgeLabelFilter: ['WORKS_ON'], + emitNodeLabelFilter: { labels: ['User', 'Admin'], mode: 'any' }, limit: 100, }); @@ -2871,7 +3720,9 @@ if (result.nextCursor) { ```python result = db.traverse(start_id, max_depth=3, min_depth=1, direction="outgoing", - edge_type_filter=[WORKS_ON], limit=100) + edge_label_filter=["WORKS_ON"], + emit_node_label_filter={"labels": ["User", "Admin"], "mode": "any"}, + limit=100) for hit in result.items: print(hit.node_id, hit.depth, hit.via_edge_id) @@ -2889,10 +3740,10 @@ if result.next_cursor: | max_depth | (part of TraverseOptions in Rust) | `number` | `int` | Yes | — | Maximum number of hops from the start node. `1` = immediate neighbors, `2` = neighbors of neighbors, etc. | | min_depth | `u32` | `number` | `int` | No | `1` | Minimum depth to include in results. Set to `0` to include the start node itself. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Edge traversal direction. | -| edge_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` (all types) | Only follow edges of these types. | -| node_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` (all types) | Only include nodes of these types in results (edges to other types are still followed). | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` (all labels) | Only follow edges with these labels. | +| emit_node_label_filter | `Option` | `emitNodeLabelFilter: { labels: string[], mode: "any" \| "all" }` | `emit_node_label_filter: dict` | No | `None` (all labels) | Node-label filter for emitted nodes. Traversal may still pass through non-emitted labels. | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter for edge validity. | -| decay_lambda | `Option` | `number` | `float` | No | `None` | Exponential decay scoring. When set, each hit receives a score: `exp(-λ × Σ age_ms)` accumulated along the path. | +| decay_lambda | `Option` | `number` | `float` | No | `None` | Depth-based traversal score. When set, each hit receives `exp(-λ × depth)`. | | limit | `Option` | `number` | `int` | No | `None` (unlimited) | Maximum results per page. Use with `cursor` for pagination. | | cursor | `Option` | `TraversalCursor` | `TraversalCursor` | No | `None` | Resume traversal from a previous page. | @@ -2915,9 +3766,13 @@ Extracts a complete subgraph (all reachable nodes and edges) rooted at a given n **Rust** ```rust -let sg = db.extract_subgraph(root_id, &SubgraphOptions { +let sg = db.extract_subgraph(root_id, 3, &SubgraphOptions { direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, + node_label_filter: Some(NodeLabelFilter { + labels: vec!["User".into()], + mode: LabelMatchMode::Any, + }), at_epoch: None, })?; println!("{} nodes, {} edges", sg.nodes.len(), sg.edges.len()); @@ -2925,13 +3780,21 @@ println!("{} nodes, {} edges", sg.nodes.len(), sg.edges.len()); **Node.js** ```javascript -const sg = db.extractSubgraph(rootId, 3, { direction: 'outgoing' }); +const sg = db.extractSubgraph(rootId, 3, { + direction: 'outgoing', + nodeLabelFilter: { labels: ['User'], mode: 'any' }, +}); console.log(sg.nodes.length, 'nodes,', sg.edges.length, 'edges'); ``` **Python** ```python -sg = db.extract_subgraph(root_id, max_depth=3, direction="outgoing") +sg = db.extract_subgraph( + root_id, + max_depth=3, + direction="outgoing", + node_label_filter={"labels": ["User"], "mode": "any"}, +) print(len(sg.nodes), "nodes,", len(sg.edges), "edges") ``` @@ -2940,17 +3803,18 @@ print(len(sg.nodes), "nodes,", len(sg.edges), "edges") | Parameter | Rust | Node.js | Python | Required | Default | Description | |-----------|------|---------|--------|----------|---------|-------------| | start_node_id | `u64` | `number` | `int` | Yes | — | Root node. | -| max_depth | — (in SubgraphOptions) | `number` | `int` | Yes | — | Maximum hops from root. | +| max_depth | `u32` | `number` | `int` | Yes | — | Maximum hops from root. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Direction. | -| edge_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Edge type filter. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Edge label filter. | +| node_label_filter | `Option` | `nodeLabelFilter: { labels: string[], mode: "any" \| "all" }` | `node_label_filter: dict` | No | `None` | Node-label filter for nodes to include and expand through. | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter. | #### Returns: Subgraph | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| -| nodes | `NodeIdMap` | `NodeRecord[]` | `list[NodeRecord]` | All nodes in the subgraph (full records). | -| edges | `Vec` or tuples | `EdgeRecord[]` | `list[EdgeRecord]` | All edges in the subgraph (full records). | +| nodes | `Vec` | `NodeView[]` | `list[NodeView]` | All nodes in the subgraph (full records). | +| edges | `Vec` | `EdgeView[]` | `list[EdgeView]` | All edges in the subgraph (full records). | ### shortest_path @@ -2967,7 +3831,7 @@ let path = db.shortest_path(from_id, to_id, &ShortestPathOptions { })?; if let Some(p) = path { - println!("path: {:?}, cost: {}", p.nodes, p.cost); + println!("path: {:?}, cost: {}", p.nodes, p.total_cost); } ``` @@ -2998,7 +3862,7 @@ if path: | from | `u64` | `number` | `int` | Yes | — | Source node ID. | | to | `u64` | `number` | `int` | Yes | — | Destination node ID. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Direction to follow edges. | -| type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Only traverse these edge types. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Only traverse these edge labels. | | weight_field | `Option` | `string` | `str` | No | `None` | Property key on edges to use as cost. When `None`, uses `edge.weight`. When set, reads the named property as the edge cost (must be numeric). | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter. | | max_depth | `Option` | `number` | `int` | No | `None` (unlimited) | Stop searching after this many hops. Prevents runaway searches on deep graphs. | @@ -3077,7 +3941,7 @@ connected = db.is_connected(from_id, to_id, direction="both", max_depth=5) | from | `u64` | `number` | `int` | Yes | — | Source node. | | to | `u64` | `number` | `int` | Yes | — | Destination node. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Direction. | -| type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Edge type filter. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Edge label filter. | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter. | | max_depth | `Option` | `number` | `int` | No | `None` | Maximum search depth. | @@ -3111,7 +3975,7 @@ d = db.degree(node_id, direction="both") |-----------|------|---------|--------|----------|---------|-------------| | node_id | `u64` | `number` | `int` | Yes | — | Node to count edges for. | | direction | `Direction` | `string` | `str` | No | `Outgoing` | Which edges to count. | -| type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Only count edges of these types. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Only count edges with these labels. | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter. | #### Returns @@ -3124,7 +3988,7 @@ The edge count. #### Performance -Metadata-only fast path for unfiltered, non-temporal queries when all visible segments have valid degree sidecars. O(edges) walk fallback when filtering by type, using a temporal epoch, running with active prune policies, reading a node with temporal incident edges, or reading through a segment whose degree sidecar is missing/corrupt. +Metadata-only fast path for unfiltered, non-temporal queries when all visible segments have valid degree sidecars. O(edges) walk fallback when filtering by edge label, using a temporal epoch, running with active prune policies, reading a node with temporal incident edges, or reading through a segment whose degree sidecar is missing/corrupt. --- @@ -3197,19 +4061,27 @@ Computes all [weakly connected components](https://en.wikipedia.org/wiki/Connect **Rust** ```rust -let components = db.connected_components(&ComponentOptions::default())?; +let components = db.connected_components(&ComponentOptions { + node_label_filter: Some(NodeLabelFilter { + labels: vec!["User".into()], + mode: LabelMatchMode::Any, + }), + ..Default::default() +})?; // components: NodeIdMap - node_id to component_id ``` **Node.js** ```javascript -const entries = db.connectedComponents(); +const entries = db.connectedComponents({ + nodeLabelFilter: { labels: ['User'], mode: 'any' }, +}); // entries: { nodeId: number, componentId: number }[] ``` **Python** ```python -components = db.connected_components() +components = db.connected_components(node_label_filter={"labels": ["User"], "mode": "any"}) # components: dict[int, int] - node_id to component_id ``` @@ -3217,8 +4089,8 @@ components = db.connected_components() | Parameter | Rust | Node.js | Python | Required | Default | Description | |-----------|------|---------|--------|----------|---------|-------------| -| edge_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Only consider these edge types when determining connectivity. | -| node_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Only include nodes of these types. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Only consider these edge labels when determining connectivity. | +| node_label_filter | `Option` | `nodeLabelFilter: { labels: string[], mode: "any" \| "all" }` | `node_label_filter: dict` | No | `None` | Node-label filter for nodes included in components. | | at_epoch | `Option` | `number` | `int` | No | `None` | Temporal filter. | #### Returns @@ -3232,7 +4104,13 @@ A mapping from every node ID to its component ID. The component ID is the smalle Returns all nodes in the same connected component as a given node. ```rust -let component_id = db.component_of(node_id, &ComponentOptions::default())?; +let node_ids = db.component_of(node_id, &ComponentOptions { + node_label_filter: Some(NodeLabelFilter { + labels: vec!["User".into()], + mode: LabelMatchMode::Any, + }), + ..Default::default() +})?; ``` ```javascript @@ -3240,7 +4118,7 @@ const nodeIds = db.componentOf(nodeId); // Float64Array ``` ```python -node_ids = db.component_of(node_id) # list[int] +node_ids = db.component_of(node_id, node_label_filter={"labels": ["User"], "mode": "any"}) # list[int] ``` #### Parameters @@ -3255,9 +4133,9 @@ Same as [`connected_components`](#connected_components), plus: | Rust | Node.js | Python | |------|---------|--------| -| `Result` (component ID) | `Float64Array` (all node IDs) | `list[int]` (all node IDs) | +| `Result, EngineError>` | `Float64Array` | `list[int]` | -Note: The Rust API returns just the component ID (representative node), while Node.js and Python return all node IDs in the component. +All three surfaces return the sorted node IDs in the same connected component as the requested node. --- @@ -3315,19 +4193,20 @@ for nid, score in zip(result.node_ids, result.scores): | max_iterations | `u32` | `number` | `int` | No | `20` | Maximum power iterations for exact mode. The algorithm stops when it converges or reaches this limit. | | epsilon | `f64` | `number` | `float` | No | `1e-6` | Convergence threshold. Iteration stops when the L1 norm of the score change vector drops below this value. | | approx_residual_tolerance | `f64` | `number` | `float` | No | `1e-5` | Approximate-mode stopping tolerance for forward push. Smaller values improve fidelity and increase work. | -| edge_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Only follow these edge types during the walk. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Only follow these edge labels during the walk. | | max_results | `Option` | `number` | `int` | No | `None` (all) | Return only the top N nodes by score. | #### Returns: PprResult | Field | Rust | Node.js | Python | Description | |-------|------|---------|--------|-------------| -| node_ids | `Vec` | `Float64Array` | `list[int]` | Node IDs sorted by score (descending). | -| scores | `Vec` | `Float64Array` | `list[float]` | Corresponding scores. Exact PPR sums to 1.0 (or very close); approximate PPR is optimized for ranking quality rather than strict normalization. | +| scores | `Vec<(u64, f64)>` | — | — | Rust scored node pairs sorted by score descending. | +| node IDs | — | `nodeIds: Float64Array` | `node_ids: list[int]` | Connector node IDs sorted by score descending. | +| scores | — | `scores: Float64Array` | `scores: list[float]` | Connector scores corresponding to node IDs. Exact PPR sums to 1.0 (or very close); approximate PPR is optimized for ranking quality rather than strict normalization. | | iterations | `u32` | `number` | `int` | Number of exact power iterations performed. Approximate mode returns `0`. | | converged | `bool` | `boolean` | `bool` | Exact mode: whether the algorithm converged within `max_iterations`. Approximate mode: `true` when no node remains above the residual tolerance. | | algorithm | `PprAlgorithm` | `string` | `str` | Which algorithm produced the result. | -| approx | `Option` | `JsPprApproxMeta \| null` | `PyPprApproxMeta \| None` | Approximate-mode metadata. `None`/`null` in exact mode. | +| approx | `Option` | `PprApproxMeta \| null` | `PprApproxMeta \| None` | Approximate-mode metadata. `None`/`null` in exact mode. | --- @@ -3337,34 +4216,86 @@ Exports the graph's adjacency structure as flat arrays. Useful for bulk analysis **Rust** ```rust -let export = db.export_adjacency()?; // uses default options +let export = db.export_adjacency(&ExportOptions { + node_label_filter: Some(NodeLabelFilter { + labels: vec!["User".into(), "Admin".into()], + mode: LabelMatchMode::Any, + }), + include_weights: true, + ..Default::default() +})?; +println!("node label side table: {:?}", export.node_labels); +println!("per-node label indexes: {:?}", export.node_label_indexes); ``` **Node.js** ```javascript -const adj = db.exportAdjacency({ includeWeights: true }); +const adj = db.exportAdjacency({ + nodeLabelFilter: { labels: ['User', 'Admin'], mode: 'any' }, + includeWeights: true, +}); // adj.nodeIds: Float64Array +// adj.edgeLabels: string[] // adj.edgeFrom: Float64Array // adj.edgeTo: Float64Array -// adj.edgeTypeIds: Uint32Array -// adj.edgeWeights: Float64Array | null +// adj.edgeLabelIndexes: Uint32Array +// adj.edgeWeights: Float64Array | undefined ``` **Python** ```python -adj = db.export_adjacency(include_weights=True) +adj = db.export_adjacency( + node_label_filter={"labels": ["User", "Admin"], "mode": "any"}, + include_weights=True, +) # adj.node_ids: list[int] -# adj.edges: list[ExportEdge] - each has from_id, to_id, type_id, weight +# adj.node_labels: list[str] +# adj.node_label_indexes: list[list[int]] +# adj.edge_labels: list[str] +# adj.edges: list[ExportEdge] - each has from_id, to_id, edge_label_index, weight ``` #### Parameters | Parameter | Rust | Node.js | Python | Required | Default | Description | |-----------|------|---------|--------|----------|---------|-------------| -| node_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Only export nodes of these types. | -| edge_type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Only export edges of these types. | +| node_label_filter | `Option` | `nodeLabelFilter: { labels: string[], mode: "any" \| "all" }` | `node_label_filter: dict` | No | `None` | Node-label filter for exported nodes. | +| edge_label_filter | `Option>` | `edgeLabelFilter: string[]` | `edge_label_filter: list[str]` | No | `None` | Only export edges with these labels. | | include_weights | `bool` | `boolean` | `bool` | No | `true` | Include edge weights in the export. Set to `false` to save memory/bandwidth when weights aren't needed. | +#### Returns: AdjacencyExport + +Rust: + +| Field | Type | Description | +|-------|------|-------------| +| node_ids | `Vec` | Live node IDs in the exported graph. | +| node_labels | `Vec` | Export-local node-label side table. | +| node_label_indexes | `Vec>` | Per-node label side-table indexes, aligned with `node_ids`. | +| edge_labels | `Vec` | Export-local edge-label side table. | +| edges | `Vec` | Exported edges. Each `ExportEdge.edge_label_index` references `edge_labels`. | + +Node.js: + +| Field | Type | Description | +|-------|------|-------------| +| nodeIds | `Float64Array` | Live node IDs in the exported graph. | +| edgeLabels | `string[]` | Export-local edge-label side table. | +| edgeFrom | `Float64Array` | Source node IDs, aligned with `edgeTo` and `edgeLabelIndexes`. | +| edgeTo | `Float64Array` | Destination node IDs. | +| edgeLabelIndexes | `Uint32Array` | Edge-label side-table indexes, aligned with `edgeFrom` / `edgeTo`. | +| edgeWeights | `Float64Array \| undefined` | Edge weights when `includeWeights` is true. | + +Python: + +| Field | Type | Description | +|-------|------|-------------| +| node_ids | `list[int]` | Live node IDs in the exported graph. | +| node_labels | `list[str]` | Export-local node-label side table. | +| node_label_indexes | `list[list[int]]` | Per-node label side-table indexes, aligned with `node_ids`. | +| edge_labels | `list[str]` | Export-local edge-label side table. | +| edges | `list[ExportEdge]` | Exported edges. Each edge has `from_id`, `to_id`, `edge_label_index`, and optional `weight`. | + --- ## Vector Search @@ -3379,17 +4310,28 @@ Performs similarity search using dense vectors (HNSW approximate nearest neighbo let hits = db.vector_search(&VectorSearchRequest { mode: VectorSearchMode::Dense, dense_query: Some(vec![0.1, 0.2, 0.3, /* ... 384 dims */]), + sparse_query: None, k: 10, + label_filter: None, ef_search: Some(100), - ..Default::default() + scope: None, + dense_weight: None, + sparse_weight: None, + fusion_mode: None, })?; // Sparse search let hits = db.vector_search(&VectorSearchRequest { mode: VectorSearchMode::Sparse, + dense_query: None, sparse_query: Some(vec![(42, 0.9), (128, 0.5)]), k: 10, - ..Default::default() + label_filter: None, + ef_search: None, + scope: None, + dense_weight: None, + sparse_weight: None, + fusion_mode: None, })?; // Hybrid search @@ -3398,10 +4340,15 @@ let hits = db.vector_search(&VectorSearchRequest { dense_query: Some(embedding), sparse_query: Some(sparse_terms), k: 10, + label_filter: Some(NodeLabelFilter { + labels: vec!["Document".into(), "Published".into()], + mode: LabelMatchMode::All, + }), + ef_search: None, + scope: None, dense_weight: Some(0.7), sparse_weight: Some(0.3), fusion_mode: Some(FusionMode::WeightedScoreFusion), - ..Default::default() })?; ``` @@ -3423,6 +4370,7 @@ const hits = db.vectorSearch('sparse', { // Hybrid search with graph scope const hits = db.vectorSearch('hybrid', { k: 10, + labelFilter: { labels: ['User', 'Project'], mode: 'any' }, denseQuery: embedding, sparseQuery: sparseTerms, denseWeight: 0.7, @@ -3448,6 +4396,7 @@ hits = db.vector_search("sparse", k=10, sparse_query=[(42, 0.9), (128, 0.5)]) hits = db.vector_search("hybrid", k=10, dense_query=embedding, sparse_query=sparse_terms, + label_filter={"labels": ["Document", "Published"], "mode": "all"}, dense_weight=0.7, sparse_weight=0.3, fusion_mode="weighted_score", scope_start_node_id=root_id, @@ -3463,8 +4412,8 @@ hits = db.vector_search("hybrid", k=10, | k | `usize` | `number` | `int` | Yes | — | Number of top results to return. | | dense_query | `Option>` | `number[]` | `list[float]` | Required for `dense`/`hybrid` | `None` | Query vector for dense search. Must have the same dimension as configured at `open()`. | | sparse_query | `Option>` | `SparseEntry[]` | `list[tuple[int, float]]` | Required for `sparse`/`hybrid` | `None` | Query vector for sparse search. List of `(dimension_index, value)` pairs. | -| type_filter | `Option>` | `number[]` | `list[int]` | No | `None` | Restrict results to nodes of these types. | -| ef_search | `Option` | `number` | `int` | No | `2 × k` | HNSW search expansion factor. Higher values improve recall at the cost of latency. Only applies to dense/hybrid modes. | +| label_filter | `Option` | `labelFilter: { labels: string[], mode: "any" \| "all" }` | `label_filter: dict` | No | `None` | Node-label filter. | +| ef_search | `Option` | `number` | `int` | No | `128` | HNSW search expansion factor. The effective dense fetch limit is at least `k` and at least `8`. Higher values improve recall at the cost of latency. Only applies to dense/hybrid modes. | **Hybrid fusion parameters** (only used when `mode = "hybrid"`): @@ -3489,7 +4438,7 @@ hits = db.vector_search("hybrid", k=10, | scope.start_node_id | `u64` | `scope.startNodeId: number` | `scope_start_node_id: int` | No | `None` | Root node for scope traversal. When set, only nodes reachable from this node within `max_depth` are candidates. | | scope.max_depth | `u32` | `scope.maxDepth: number` | `scope_max_depth: int` | Required if scope set | — | Maximum hops from the scope root. | | scope.direction | `Direction` | `scope.direction: string` | `scope_direction: str` | No | `Outgoing` | Direction for scope traversal. | -| scope.edge_type_filter | `Option>` | `scope.edgeTypeFilter: number[]` | `scope_edge_type_filter: list[int]` | No | `None` | Edge types for scope traversal. | +| scope.edge_label_filter | `Option>` | `scope.edgeLabelFilter: string[]` | `scope_edge_label_filter: list[str]` | No | `None` | Edge labels for scope traversal. | | scope.at_epoch | `Option` | `scope.atEpoch: number` | `scope_at_epoch: int` | No | `None` | Temporal filter for scope. | #### Returns: VectorHit @@ -3513,7 +4462,7 @@ Immediately deletes nodes matching the specified criteria. Cascade-deletes all i let result = db.prune(&PrunePolicy { max_age_ms: Some(7 * 24 * 60 * 60 * 1000), // 7 days max_weight: Some(0.1), // weight <= 0.1 - type_id: Some(CONVERSATION), // only conversations + label: Some("Conversation".into()), // only conversations })?; println!("pruned {} nodes, {} edges", result.nodes_pruned, result.edges_pruned); ``` @@ -3522,7 +4471,7 @@ println!("pruned {} nodes, {} edges", result.nodes_pruned, result.edges_pruned); const result = db.prune({ maxAgeMs: 7 * 24 * 60 * 60 * 1000, maxWeight: 0.1, - typeId: CONVERSATION, + label: 'Conversation', }); ``` @@ -3530,7 +4479,7 @@ const result = db.prune({ result = db.prune( max_age_ms=7 * 24 * 60 * 60 * 1000, max_weight=0.1, - type_id=CONVERSATION, + label="Conversation", ) ``` @@ -3540,7 +4489,7 @@ result = db.prune( |-----------|------|---------|--------|----------|---------|-------------| | max_age_ms | `Option` | `number` | `int` | No* | `None` | Delete nodes older than `now - max_age_ms` milliseconds. Age is computed from `updated_at`. | | max_weight | `Option` | `number` | `float` | No* | `None` | Delete nodes with `weight <= max_weight`. | -| type_id | `Option` | `number` | `int` | No | `None` (all types) | Restrict pruning to a single node type. | +| label | `Option` | `string` | `str` | No | `None` (all labels) | Restrict pruning to a single node label. | \* At least one of `max_age_ms` or `max_weight` must be provided. This guards against accidental mass deletion (calling `prune({})` with no criteria is an error). @@ -3562,22 +4511,22 @@ Registers a named prune policy that is automatically applied during [compaction] ```rust db.set_prune_policy("stale-conversations", PrunePolicy { max_age_ms: Some(30 * 24 * 60 * 60 * 1000), // 30 days - type_id: Some(CONVERSATION), - ..Default::default() + max_weight: None, + label: Some("Conversation".into()), })?; ``` ```javascript db.setPrunePolicy('stale-conversations', { maxAgeMs: 30 * 24 * 60 * 60 * 1000, - typeId: CONVERSATION, + label: 'Conversation', }); ``` ```python db.set_prune_policy("stale-conversations", max_age_ms=30 * 24 * 60 * 60 * 1000, - type_id=CONVERSATION) + label="Conversation") ``` #### Parameters @@ -3629,33 +4578,35 @@ existed = db.remove_prune_policy("stale-conversations") Lists all registered prune policies. ```rust -let policies = db.list_prune_policies(); -for (name, policy) in &policies { - println!("{}: max_age_ms={:?}", name, policy.max_age_ms); +let policies = db.list_prune_policies()?; +for info in &policies { + println!("{}: max_age_ms={:?}", info.name, info.policy.max_age_ms); } ``` ```javascript const policies = db.listPrunePolicies(); -// [{ name: string, policy: { maxAgeMs?, maxWeight?, typeId? } }] +// [{ name: string, policy: { maxAgeMs?, maxWeight?, label? } }] ``` ```python policies = db.list_prune_policies() for p in policies: - print(p.name, p.max_age_ms, p.max_weight, p.type_id) + print(p.name, p.max_age_ms, p.max_weight, p.label) ``` #### Returns -Array of named policies. Each entry includes: +Array of named policies. Rust and Node.js entries contain a nested policy object; Python flattens +policy fields onto each entry. -| Field | Rust | Node.js | Python | Description | -|-------|------|---------|--------|-------------| -| name | `String` | `string` | `str` | Policy name. | -| max_age_ms | `Option` | `number \| undefined` | `int \| None` | Age threshold. | -| max_weight | `Option` | `number \| undefined` | `float \| None` | Weight threshold. | -| type_id | `Option` | `number \| undefined` | `int \| None` | Type scope. | +| Field path | Rust | Node.js | Python | Description | +|------------|------|---------|--------|-------------| +| name | `info.name: String` | `entry.name: string` | `p.name: str` | Policy name. | +| policy | `info.policy: PrunePolicy` | `entry.policy: PrunePolicy` | — | Nested policy object in Rust and Node.js. | +| max age | `info.policy.max_age_ms: Option` | `entry.policy.maxAgeMs?: number` | `p.max_age_ms: int \| None` | Age threshold. | +| max weight | `info.policy.max_weight: Option` | `entry.policy.maxWeight?: number` | `p.max_weight: float \| None` | Weight threshold. | +| label | `info.policy.label: Option` | `entry.policy.label?: string` | `p.label: str \| None` | Node-label scope. | --- @@ -3697,7 +4648,7 @@ db.flush(); ``` ```python -info = db.flush() # PySegmentInfo | None +info = db.flush() # SegmentInfo | None ``` #### Returns @@ -3706,13 +4657,15 @@ info = db.flush() # PySegmentInfo | None |------|---------|--------|-------------| | `Result, EngineError>` | `void` | `SegmentInfo \| None` | Info about the written segment, or `None` if the memtable was empty. | -**SegmentInfo** (Python only): +**SegmentInfo** (Rust/Python): -| Field | Type | Description | -|-------|------|-------------| -| id | `int` | Segment ID on disk. | -| node_count | `int` | Nodes in the segment. | -| edge_count | `int` | Edges in the segment. | +| Field | Rust | Python | Description | +|-------|------|--------|-------------| +| id | `u64` | `int` | Segment ID on disk. | +| node_count | `u64` | `int` | Nodes in the segment. | +| edge_count | `u64` | `int` | Edges in the segment. | +| segment_format_version | `u32` | — | Rust segment format version. | +| segment_data_id | `[u8; 32]` | — | Rust segment data identifier. | --- @@ -3850,6 +4803,81 @@ stats = db.end_ingest() # CompactionStats | None --- +### scrub + +Runs an offline integrity check across all segments. Recomputes SHA-256 payload digests for every component and compares them to the digests recorded at write time. Reports mismatches without modifying any data. + +**Rust** +```rust +let report = db.scrub()?; +println!("checked: {}, failed: {}", report.total_components_checked, report.total_components_failed); +for seg in &report.segments { + for f in &seg.findings { + eprintln!("segment {}: {} — {}", seg.segment_id, f.finding_type, f.detail); + } +} +``` + +**Node.js** +```javascript +const report = db.scrub(); +console.log(`checked: ${report.totalComponentsChecked}, failed: ${report.totalComponentsFailed}`); + +// async +const report = await db.scrubAsync(); +``` + +**Python** +```python +report = db.scrub() +print(f"checked: {report.total_components_checked}, failed: {report.total_components_failed}") + +# async +report = await db.scrub() +``` + +#### Parameters + +None. + +#### Returns: ScrubReport + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| segments | `Vec` | `Array` | `list[SegmentScrubResult]` | Per-segment results. | +| total_components_checked | `u64` | `number` | `int` | Total components examined. | +| total_components_ok | `u64` | `number` | `int` | Components that passed all checks. | +| total_components_failed | `u64` | `number` | `int` | Components with at least one finding. | +| total_bytes_digested | `u64` | `number` | `int` | Total payload bytes hashed during the scrub. | +| duration_ms | `u64` | `number` | `int` | Wall-clock time of the scrub in milliseconds. | + +#### SegmentScrubResult + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| segment_id | `u64` | `number` | `int` | Segment that was checked. | +| findings | `Vec` | `Array` | `list[ComponentScrubFinding]` | Problems found (empty if healthy). | +| components_ok | `u64` | `number` | `int` | Components that passed in this segment. | +| bytes_digested | `u64` | `number` | `int` | Payload bytes hashed in this segment. | + +#### ComponentScrubFinding + +| Field | Rust | Node.js | Python | Description | +|-------|------|---------|--------|-------------| +| component_kind | `String` | `string` | `str` | Which component type had the problem (e.g. `"NodeRecords"`, `"PlannerStats"`). | +| finding_type | `ScrubFindingType` | `string` | `str` | Classification: `PayloadDigestMismatch`, `ComponentIdMismatch`, `DependencyDigestMismatch`, `IdentityHeaderMismatch`, `ContainerIdMismatch`, `SegmentIdentityMismatch`, `RangeOverflow`, `RangeOverlap`, `FileMissing`, or `IoError`. | +| detail | `String` | `string` | `str` | Human-readable description of the finding. | + +#### Behavior + +- Scrub is **read-only** — it never modifies data on disk. +- Scrub is **offline** — it is never called automatically during open, query, flush, or compaction. You must call it explicitly. +- Segments are checked in parallel using a shared thread pool. The work is I/O-bound (streaming 64KB-buffered reads + SHA-256), not CPU-bound. +- A healthy database returns `total_components_failed == 0` and an empty `findings` array for every segment. +- If a segment directory is missing (e.g. deleted concurrently), the scrub reports a `FileMissing` finding rather than panicking. + +--- + ## Introspection These methods provide quick diagnostic information. They are **approximate** and counts may slightly overcount when the same ID appears in multiple memtables or segments. @@ -3857,7 +4885,7 @@ These methods provide quick diagnostic information. They are **approximate** and ### node_count ```rust -let count = db.node_count(); // usize +let count = db.node_count()?; // usize ``` Approximate count of live nodes across all data sources. @@ -3865,30 +4893,101 @@ Approximate count of live nodes across all data sources. ### edge_count ```rust -let count = db.edge_count(); // usize +let count = db.edge_count()?; // usize ``` Approximate count of live edges. +### next_node_id + +```rust +let next = db.next_node_id()?; // u64 +``` + +Rust-only diagnostic: the next auto-assigned node ID that would be used by a new node write. + +### next_edge_id + +```rust +let next = db.next_edge_id()?; // u64 +``` + +Rust-only diagnostic: the next auto-assigned edge ID that would be used by a new edge write. + ### segment_count ```rust -let count = db.segment_count(); // usize +let count = db.segment_count()?; // usize ``` Number of on-disk segments. After compaction, this is typically 0 or 1. +### segment_tombstone_node_count + +```rust +let count = db.segment_tombstone_node_count()?; // usize +``` + +Rust-only diagnostic: deleted node records currently retained in immutable segments. + +### segment_tombstone_edge_count + +```rust +let count = db.segment_tombstone_edge_count()?; // usize +``` + +Rust-only diagnostic: deleted edge records currently retained in immutable segments. + +### path + +```rust +let path = db.path(); +``` + +Returns the database directory path as `&Path`. + +### manifest + +Rust diagnostic API for reading the current raw manifest state. + +```rust +let manifest = db.manifest()?; +println!("label token schema: {}", manifest.label_token_schema_version); +``` + +#### Returns: ManifestState + +`ManifestState` is a raw diagnostic object. Ordinary graph APIs accept public label names, not these internal numeric IDs. + +| Field | Rust | Description | +|-------|------|-------------| +| label_token_schema_version | `u32` | Node-label / edge-label catalog schema marker. | +| node_label_tokens | `BTreeMap` | Public node label to internal `label_id`. | +| edge_label_tokens | `BTreeMap` | Public edge label to internal `label_id`. | +| secondary_indexes | `Vec` | Raw optional secondary-index declarations. Node targets use `SecondaryIndexTarget::NodeProperty { label_id, prop_key }`; edge targets use `SecondaryIndexTarget::EdgeProperty { label_id, prop_key }`. | +| segments | `Vec` | Published segment metadata. | + +### manifest::load_manifest_readonly (Rust only) + +Diagnostic read-only manifest loader that inspects the manifest priority chain without writing to disk. + +```rust +let manifest = overgraph::manifest::load_manifest_readonly(Path::new("./my-graph"))?; +``` + +Returns `Result, EngineError>`. The returned manifest is a raw diagnostic view and may contain internal numeric token IDs. + --- ## Binary Batch Ingestion -High-performance binary format for batch upserts. Avoids JSON parsing overhead. Useful when ingesting data from a custom pipeline. +High-performance connector-only binary format for batch upserts. Avoids JSON parsing overhead. Useful when ingesting data from a custom pipeline. Rust callers use the structured `batch_upsert_nodes` and `batch_upsert_edges` APIs directly. ### batch_upsert_nodes_binary ```javascript const buf = Buffer.alloc(/* ... */); -// Format: [count: u32_LE][per node: type_id: u32_LE, weight: f32_LE, key_len: u16_LE, key: utf8, props_len: u32_LE, props: json_utf8] +// Format: "OGNB", version 2, count, then per-node labels/key/props payloads. const ids = db.batchUpsertNodesBinary(buf); ``` @@ -3901,10 +5000,15 @@ ids = db.batch_upsert_nodes_binary(buf) ``` ┌──────────────────────────────────────┐ +│ magic: "OGNB" │ +│ version: u16 = 2 │ │ count: u32 │ ← number of nodes in this batch ├──────────────────────────────────────┤ │ For each node: │ -│ type_id: u32 │ +│ label_count: u8 │ +│ repeated label_count times: │ +│ label_len: u16 │ +│ label: [u8; label_len] (UTF-8) │ │ weight: f32 │ │ key_len: u16 │ │ key: [u8; key_len] (UTF-8) │ @@ -3917,6 +5021,8 @@ ids = db.batch_upsert_nodes_binary(buf) Array of node IDs (same order as packed nodes). +Version 1 node buffers are rejected. Use version 2 for every connector so each packed node carries its full label set. + --- ### batch_upsert_edges_binary @@ -3933,12 +5039,15 @@ ids = db.batch_upsert_edges_binary(buf) ``` ┌──────────────────────────────────────────┐ +│ magic: "OGEB" │ +│ version: u16 = 1 │ │ count: u32 │ ├──────────────────────────────────────────┤ │ For each edge: │ │ from: u64 │ │ to: u64 │ -│ type_id: u32 │ +│ label_len: u16 │ +│ label: [u8; label_len] (UTF-8) │ │ weight: f32 │ │ valid_from: i64 │ │ valid_to: i64 │ @@ -3947,6 +5056,8 @@ ids = db.batch_upsert_edges_binary(buf) └──────────────────────────────────────────┘ ``` +In the packed binary edge format only, `valid_from = 0` and `valid_to = 0` are sentinels for the engine defaults. `valid_from = 0` means "use the edge's `created_at` timestamp"; `valid_to = 0` means "use `i64::MAX` / no expiration." Because of these sentinels, epoch `0` cannot be represented as an explicit edge validity bound in this packed format. + --- ## Error Handling @@ -3969,6 +5080,7 @@ All methods can fail. Errors are returned differently across languages: | `SerializationError(String)` | Property encoding/decoding failed. | | `ManifestError(String)` | Manifest file is corrupt or incompatible. | | `DatabaseNotFound(String)` | Directory doesn't exist and `create_if_missing` is false. | +| `DatabaseClosed` | Operation attempted after the engine was closed. | | `InvalidOperation(String)` | Invalid API usage (e.g., writing to a closed database). | | `TxnConflict(String)` | Explicit write transaction conflict. No WAL entry was appended and the transaction did not commit. | | `TxnClosed` | Explicit write transaction was already committed or rolled back. | @@ -4025,20 +5137,21 @@ const node = await db.getNodeAsync(42); Async methods run on the libuv thread pool. Write operations acquire an exclusive lock; read operations acquire a shared lock (allowing concurrent reads). -**Available async methods:** `closeAsync`, `upsertNodeAsync`, `upsertEdgeAsync`, `batchUpsertNodesAsync`, `batchUpsertEdgesAsync`, `getNodeAsync`, `getEdgeAsync`, `getNodeByKeyAsync`, `getEdgeByTripleAsync`, `getNodesAsync`, `getNodesByKeysAsync`, `getEdgesAsync`, `deleteNodeAsync`, `deleteEdgeAsync`, `invalidateEdgeAsync`, `graphPatchAsync`, `beginWriteTxnAsync`, `neighborsAsync`, `neighborsPagedAsync`, `neighborsBatchAsync`, `traverseAsync`, `topKNeighborsAsync`, `extractSubgraphAsync`, `shortestPathAsync`, `allShortestPathsAsync`, `isConnectedAsync`, `degreeAsync`, `degreesAsync`, `sumEdgeWeightsAsync`, `avgEdgeWeightAsync`, `findNodesAsync`, `findNodesPagedAsync`, `ensureNodePropertyIndexAsync`, `dropNodePropertyIndexAsync`, `listNodePropertyIndexesAsync`, `findNodesRangeAsync`, `findNodesRangePagedAsync`, `findNodesByTimeRangeAsync`, `findNodesByTimeRangePagedAsync`, `nodesByTypeAsync`, `edgesByTypeAsync`, `getNodesByTypeAsync`, `getEdgesByTypeAsync`, `countNodesByTypeAsync`, `countEdgesByTypeAsync`, `nodesByTypePagedAsync`, `edgesByTypePagedAsync`, `getNodesByTypePagedAsync`, `getEdgesByTypePagedAsync`, `personalizedPagerankAsync`, `connectedComponentsAsync`, `componentOfAsync`, `vectorSearchAsync`, `exportAdjacencyAsync`, `pruneAsync`, `setPrunePolicyAsync`, `removePrunePolicyAsync`, `listPrunePoliciesAsync`, `syncAsync`, `flushAsync`, `compactAsync`, `compactWithProgressAsync`, `ingestModeAsync`, `endIngestAsync`. +**Available async methods:** `closeAsync`, `ensureNodeLabelAsync`, `ensureEdgeLabelAsync`, `getNodeLabelIdAsync`, `getEdgeLabelIdAsync`, `getNodeLabelAsync`, `getEdgeLabelAsync`, `listNodeLabelsAsync`, `listEdgeLabelsAsync`, `upsertNodeAsync`, `upsertEdgeAsync`, `addNodeLabelAsync`, `removeNodeLabelAsync`, `batchUpsertNodesAsync`, `batchUpsertEdgesAsync`, `batchUpsertNodesBinaryAsync`, `batchUpsertEdgesBinaryAsync`, `getNodeAsync`, `getEdgeAsync`, `getNodeByKeyAsync`, `getEdgeByTripleAsync`, `getNodesAsync`, `getNodesByKeysAsync`, `getEdgesAsync`, `deleteNodeAsync`, `deleteEdgeAsync`, `invalidateEdgeAsync`, `graphPatchAsync`, `beginWriteTxnAsync`, `neighborsAsync`, `neighborsPagedAsync`, `neighborsBatchAsync`, `traverseAsync`, `topKNeighborsAsync`, `extractSubgraphAsync`, `shortestPathAsync`, `allShortestPathsAsync`, `isConnectedAsync`, `degreeAsync`, `degreesAsync`, `sumEdgeWeightsAsync`, `avgEdgeWeightAsync`, `findNodesAsync`, `findNodesPagedAsync`, `ensureNodePropertyIndexAsync`, `dropNodePropertyIndexAsync`, `listNodePropertyIndexesAsync`, `ensureEdgePropertyIndexAsync`, `dropEdgePropertyIndexAsync`, `listEdgePropertyIndexesAsync`, `findNodesRangeAsync`, `findNodesRangePagedAsync`, `findNodesByTimeRangeAsync`, `findNodesByTimeRangePagedAsync`, `nodesByLabelsAsync`, `edgesByLabelAsync`, `getNodesByLabelsAsync`, `getEdgesByLabelAsync`, `countNodesByLabelsAsync`, `countEdgesByLabelAsync`, `nodesByLabelsPagedAsync`, `edgesByLabelPagedAsync`, `getNodesByLabelsPagedAsync`, `getEdgesByLabelPagedAsync`, `queryNodeIdsAsync`, `queryNodesAsync`, `queryEdgeIdsAsync`, `queryEdgesAsync`, `queryPatternAsync`, `explainNodeQueryAsync`, `explainEdgeQueryAsync`, `explainPatternQueryAsync`, `personalizedPagerankAsync`, `connectedComponentsAsync`, `componentOfAsync`, `vectorSearchAsync`, `exportAdjacencyAsync`, `pruneAsync`, `setPrunePolicyAsync`, `removePrunePolicyAsync`, `listPrunePoliciesAsync`, `syncAsync`, `flushAsync`, `compactAsync`, `compactWithProgressAsync`, `ingestModeAsync`, `endIngestAsync`. `WriteTxn` handles expose async counterparts for the full transaction surface: `upsertNodeAsync`, `upsertNodeAsAsync`, `upsertEdgeAsync`, `upsertEdgeAsAsync`, `deleteNodeAsync`, `deleteEdgeAsync`, `invalidateEdgeAsync`, `stageAsync`, `getNodeAsync`, `getEdgeAsync`, `getNodeByKeyAsync`, `getEdgeByTripleAsync`, `commitAsync`, and `rollbackAsync`. Async transaction operations on one handle execute in call order. ### Python -The `AsyncOverGraph` class wraps every `OverGraph` method with `asyncio.to_thread()`. `begin_write_txn()` returns an `AsyncWriteTxn` whose methods mirror `PyWriteTxn`: +The `AsyncOverGraph` class wraps every `OverGraph` method with `asyncio.to_thread()`. `begin_write_txn()` returns an `AsyncWriteTxn` whose methods mirror `WriteTxn`: ```python from overgraph import AsyncOverGraph async def main(): async with await AsyncOverGraph.open("./my-graph") as db: - node_id = await db.upsert_node(1, "alice") + # Also accepts multiple labels: ["User", "Admin"] + node_id = await db.upsert_node("User", "alice") node = await db.get_node(node_id) neighbors = await db.neighbors(node_id) @@ -4063,35 +5176,51 @@ asyncio.run(main()) | | `stats` | Runtime statistics | | **Nodes** | `upsert_node` | Create or update node | | | `get_node` | Get node by ID | -| | `get_node_by_key` | Get node by type + key | +| | `get_node_by_key` | Get node by label + key | +| | `add_node_label` | Add a node label to an existing node | +| | `remove_node_label` | Remove a node label from an existing node | | | `delete_node` | Delete node (cascade edges) | | | `batch_upsert_nodes` | Batch create/update nodes | | | `get_nodes` | Batch get nodes by ID | -| | `get_nodes_by_keys` | Batch get nodes by type + key | +| | `get_nodes_by_keys` | Batch get nodes by label + key | | **Edges** | `upsert_edge` | Create or update edge | | | `get_edge` | Get edge by ID | -| | `get_edge_by_triple` | Get edge by from + to + type | +| | `get_edge_by_triple` | Get edge by from + to + edge label | | | `delete_edge` | Delete edge | | | `invalidate_edge` | Close validity window | | | `batch_upsert_edges` | Batch create/update edges | | | `get_edges` | Batch get edges by ID | | **Atomic** | `graph_patch` | Multi-op atomic batch | | | `begin_write_txn` / `beginWriteTxn` | Explicit ordered write transaction | -| **Type-Based Queries** | `nodes_by_type` | All node IDs of a type | -| | `edges_by_type` | All edge IDs of a type | -| | `get_nodes_by_type` | All node records of a type | -| | `get_edges_by_type` | All edge records of a type | -| | `count_nodes_by_type` | Count nodes of a type | -| | `count_edges_by_type` | Count edges of a type | -| **Property Indexes** | `ensure_node_property_index` | Declare optional equality or range index | -| | `drop_node_property_index` | Remove optional property index declaration | -| | `list_node_property_indexes` | Inspect declaration state | +| **Catalog** | `ensure_node_label` / `ensureNodeLabel` | Ensure node label token | +| | `ensure_edge_label` / `ensureEdgeLabel` | Ensure edge label token | +| | `get_node_label_id` / `getNodeLabelId` | Diagnostic name-to-ID lookup | +| | `get_edge_label_id` / `getEdgeLabelId` | Diagnostic name-to-ID lookup | +| | `get_node_label` / `getNodeLabel` | Diagnostic ID-to-name lookup | +| | `get_edge_label` / `getEdgeLabel` | Diagnostic ID-to-name lookup | +| | `list_node_labels` / `listNodeLabels` | List node-label catalog entries | +| | `list_edge_labels` / `listEdgeLabels` | List edge-label catalog entries | +| **Label and Edge-Label Queries** | `nodes_by_labels` | Node ID convenience query | +| | `edges_by_label` | All edge IDs of an edge label | +| | `get_nodes_by_labels` | Hydrated node convenience query | +| | `get_edges_by_label` | All edge records of an edge label | +| | `count_nodes_by_labels` | Node count convenience query | +| | `count_edges_by_label` | Count edges of an edge label | +| **Property Indexes** | `ensure_node_property_index` | Declare optional node equality or range index | +| | `drop_node_property_index` | Remove optional node property index declaration | +| | `list_node_property_indexes` | Inspect node declaration state | +| | `ensure_edge_property_index` | Declare optional edge equality or range index | +| | `drop_edge_property_index` | Remove optional edge property index declaration | +| | `list_edge_property_indexes` | Inspect edge declaration state | | **Property & Time Queries** | `find_nodes` | Property search | | | `find_nodes_range` | Numeric property range search | | | `find_nodes_by_time_range` | Time range search | | **Queries** | `query_node_ids` | Node query returning IDs | | | `query_nodes` | Node query returning hydrated nodes | | | `explain_node_query` | Explain a node query plan | +| | `query_edge_ids` | Edge query returning IDs | +| | `query_edges` | Edge query returning hydrated edges | +| | `explain_edge_query` | Explain an edge query plan | | | `query_pattern` | Bounded graph pattern query | | | `explain_pattern_query` | Explain a graph pattern plan | | **Pagination** | `*_paged` | Paginated variants | @@ -4123,5 +5252,13 @@ asyncio.run(main()) | | `compact_with_progress` | Merge with progress | | | `ingest_mode` | Enter bulk mode | | | `end_ingest` | Exit bulk mode + compact | -| **Binary** | `batch_upsert_nodes_binary` | Binary batch nodes | -| | `batch_upsert_edges_binary` | Binary batch edges | +| | `scrub` | Validate database integrity | +| **Introspection** | `path` | Database directory path | +| | `manifest` | Rust raw manifest diagnostics | +| | `next_node_id` | Rust next node ID diagnostic | +| | `next_edge_id` | Rust next edge ID diagnostic | +| | `segment_tombstone_node_count` | Rust segment node tombstone diagnostic | +| | `segment_tombstone_edge_count` | Rust segment edge tombstone diagnostic | +| | `manifest::load_manifest_readonly` | Rust read-only manifest diagnostic | +| **Binary** | `batch_upsert_nodes_binary` | Connector-only binary batch nodes | +| | `batch_upsert_edges_binary` | Connector-only binary batch edges | diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index 12fd687..bb7c701 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -1,10 +1,10 @@ # How OverGraph Works -This is a technical overview of the OverGraph storage engine for contributors and curious engineers. It covers the core architecture without going into every implementation detail. If you want the full internal spec, see [Architecture.md](internal/architecture/Architecture.md). +This is a technical overview of the OverGraph storage engine for contributors and curious engineers. It covers the core architecture without going into every implementation detail. ## The big picture -OverGraph is a log-structured merge tree (LSM) graph database with built-in vector search. If you've worked with LevelDB, RocksDB, or Cassandra's storage engine, the core ideas will feel familiar. The key difference is that OverGraph is purpose-built for graph data: adjacency indexes, typed nodes and edges, temporal validity, decay scoring, and vector indexes (dense HNSW + sparse inverted posting lists) are first-class concepts in the storage format. +OverGraph is a log-structured merge tree (LSM) graph database with built-in vector search. If you've worked with LevelDB, RocksDB, or Cassandra's storage engine, the core ideas will feel familiar. The key difference is that OverGraph is purpose-built for graph data: adjacency indexes, labeled nodes and labeled edges, temporal validity, decay scoring, and vector indexes (dense HNSW + sparse inverted posting lists) are first-class concepts in the storage format. ``` ┌─────────────────────┐ @@ -37,9 +37,9 @@ OverGraph is a log-structured merge tree (LSM) graph database with built-in vect Every mutation follows the same path: -1. **WAL append.** The operation is serialized and appended to the write-ahead log (`data.wal`). This is the durability guarantee: if we crash after this point, we can replay the WAL on restart. +1. **WAL append.** The operation is serialized and appended to the active write-ahead log generation (`wal_.wal`, starting with `wal_0.wal`). This is the durability guarantee: if we crash after this point, we can replay retained WAL generations on restart. -2. **Memtable apply.** The operation is applied to the in-memory memtable. The memtable uses HashMaps for nodes, edges, adjacency lists, key lookups, and type indexes. Reads served from the memtable are always the freshest. +2. **Memtable apply.** The operation is applied to the in-memory memtable. The memtable uses HashMaps for nodes, edges, adjacency lists, key lookups, and internal label-token indexes. Reads served from the memtable are always the freshest. 3. **Ack.** The caller gets back the node or edge ID. The write is durable (depending on sync mode) and immediately visible to subsequent reads. @@ -57,7 +57,7 @@ Reads check multiple sources and merge them: 1. **Memtable** (freshest data, always checked first) 2. **Immutable segments** (scanned newest to oldest) -For point lookups (`get_node`, `get_edge`, `get_node_by_key`), we stop at the first source that has the record. For collection queries (`neighbors`, `find_nodes`, `nodes_by_type`), we merge results from all sources using a K-way merge with a min-heap. Planner-backed queries can also use optional per-segment `planner_stats.dat` sidecars for private cost estimates, adaptive candidate caps, and graph-pattern fanout ordering; the stats are advisory only, and final visible-record verification still decides results. For eligible aggregation queries (`degree`, `degrees`, `sum_edge_weights`, `avg_edge_weight` with no type filter, no explicit epoch, no active prune policy, and valid degree sidecars on all visible segments), reads sum published degree overlays plus per-segment `degree_delta.dat` sidecars without walking adjacency. Filtered, temporal, prune-policy, temporal-edge, or sidecar-unavailable cases fall back to the adjacency walk path. +For point lookups (`get_node`, `get_edge`, `get_node_by_key`), we stop at the first source that has the record. For collection queries (`neighbors`, `find_nodes`, `nodes_by_labels`), we merge results from all sources using a K-way merge with a min-heap. Multi-label convenience scans use `All` semantics over label memberships; `Any` semantics are expressed through explicit node label filters. Planner-backed queries can also use optional per-segment `planner_stats.dat` sidecars for private cost estimates, adaptive candidate caps, and graph-pattern fanout ordering; the stats are advisory only, and final visible-record verification still decides results. For eligible aggregation queries (`degree`, `degrees`, `sum_edge_weights`, `avg_edge_weight` with no edge-label filter, no explicit epoch, no active prune policy, and valid degree sidecars on all visible segments), reads sum published degree overlays plus per-segment `degree_delta.dat` sidecars without walking adjacency. Filtered, temporal, prune-policy, temporal-edge, or sidecar-unavailable cases fall back to the adjacency walk path. Tombstones (from `delete_node` / `delete_edge`) are applied during the merge. Prune policies are also evaluated at read time, so a registered policy takes effect immediately without waiting for compaction. @@ -69,44 +69,46 @@ All collection queries support keyset pagination via `limit` + `after`. The `aft When the memtable exceeds a size threshold (default 128MB), it gets frozen and flushed to disk as a new segment. A fresh memtable is allocated for incoming writes, so the flush never blocks the write path. -Each segment is a directory containing: +Each segment is a directory containing a small manifest plus a packed immutable core: | File | Purpose | |---|---| -| `nodes.dat` | Node records (binary, packed) | -| `edges.dat` | Edge records (binary, packed) | -| `adj_out.idx` / `adj_out.dat` | Outgoing adjacency index + postings | -| `adj_in.idx` / `adj_in.dat` | Incoming adjacency index + postings | -| `key_index.dat` | `(type_id, key)` to node_id mapping | -| `type_index.dat` | `type_id` to sorted list of IDs | +| `segment_manifest.dat` | Component table of contents with identity/dependency records | +| `segment.core` | Packed immutable core payloads and maintained indexes | | `secondary_indexes/` | Optional declared equality/range property-index sidecars | -| `timestamp_index.dat` | Sorted `(updated_at, node_id)` pairs | -| `tombstones.dat` | Set of deleted IDs | -| `metadata.dat` | Sidecar with per-record metadata for fast compaction | | `degree_delta.dat` | Optional signed degree delta sidecar for degree/weight fast paths | | `planner_stats.dat` | Optional advisory planner statistics for private query costing | -| `node_dense_vectors.dat` | Dense vector blob (present only when segment has vectors) | -| `node_sparse_vectors.dat` | Sparse vector blob (present only when segment has vectors) | -| `dense_hnsw_graph.dat` | HNSW graph index for dense ANN search | -| `sparse_postings.dat` | Inverted posting lists for sparse dot-product search | -| `node_vector_meta.dat` | Per-node vector presence, offsets, and lengths | +| `dense_hnsw_meta.dat` / `dense_hnsw_graph.dat` | Optional dense-vector HNSW accelerator | +| `sparse_posting_index.dat` / `sparse_postings.dat` | Optional sparse-vector inverted index accelerator | -Vector files are optional per segment. Segments containing no vectors skip vector file generation entirely: no storage overhead, no index building, no mmap. The manifest tracks which segments have vector data so the engine can skip unnecessary I/O on open. +`segment.core` holds the logical payloads that older documentation described as separate +core files: node and edge records, tombstones, node/edge metadata, key/internal-node-label-token/internal-edge-label-token/timestamp/triple +indexes, adjacency indexes/postings, vector source-truth blobs, and immutable edge metadata +indexes. The segment manifest records each logical component as a range inside `segment.core`, +so readers still expose payload-local byte slices to the same parsers while only mapping the +core container once. -All segment files are immutable after creation. Reads use memory-mapped I/O (`mmap`), so the OS page cache handles caching without any application-level buffer management. This means reads never block writes and there's no cache invalidation to worry about. +Refreshable optional sidecars remain separate files. Declared property indexes, planner stats, +degree deltas, dense HNSW, and sparse postings can be missing, rebuilt, or refreshed without +rewriting `segment.core`; query correctness falls back to scans or exact vector search when an +optional accelerator is unavailable. + +The packed core is immutable after creation. Reads use memory-mapped I/O (`mmap`), so the OS page cache handles caching without any application-level buffer management. This means reads never block writes and there's no cache invalidation to worry about. ### Adjacency index -The adjacency index is the core structure that makes graph traversal fast. For each `(node_id, edge_type_id)` pair, it stores the offset and count of neighbor entries in a postings file. +The adjacency index is the core structure that makes graph traversal fast. Public APIs pass edge-label names; the read boundary resolves those names to internal numeric edge-label tokens before the storage engine walks adjacency. For each `(node_id, edge_label_token)` pair, the index stores the offset and count of neighbor entries in a postings payload. -**Index file** (sorted array, binary searchable): +**Index payload** (count header plus sorted, binary-searchable entries): ``` -(node_id: u64, type_id: u32, offset: u64, count: u32) +[count: u64] +(node_id: u64, edge_label_token: u32, offset: u64, count: u32) ``` -**Postings file** (packed array at each offset): +**Postings payload** (variable-length delta/varint encoded group at each offset): ``` -(edge_id: u64, neighbor_id: u64, weight: f32, valid_from: i64, valid_to: i64) +first: varint(edge_id) + varint(neighbor_id) + f32(weight) + varint(valid_from) + varint(valid_to) +subsequent: varint(edge_id_delta) + varint(neighbor_id) + f32(weight) + varint(valid_from) + varint(valid_to) ``` Looking up neighbors is a binary search in the index followed by a sequential scan of the postings. This is why neighbor lookups are ~2μs even with thousands of edges per node. @@ -128,11 +130,11 @@ Filtered reads, explicit `at_epoch` reads, active prune policies, temporal-edge ### Connected components -`connected_components()` computes a global weakly-connected-component (WCC) labelling using union-find with path compression and union by rank for near-linear O(N·α(N)) time. The algorithm collects all visible nodes via `nodes_by_type()`, then performs a single outgoing `neighbors_batch()` scan to union endpoints. A final pass normalizes each component ID to the minimum node ID in the component for deterministic output. +`connected_components()` computes a global weakly-connected-component (WCC) labelling using union-find with path compression and union by rank for near-linear O(N·α(N)) time. The algorithm collects visible nodes by resolved internal node-label token, then performs a single outgoing `neighbors_batch()` scan to union endpoints. A final pass normalizes each component ID to the minimum node ID in the component for deterministic output. `component_of(node_id, &ComponentOptions)` answers the targeted question "which nodes are in this node's component?" via BFS using `neighbors_batch()` with both-direction traversal per frontier layer. This avoids scanning the entire graph when only one component is needed. -Both methods support edge-type, node-type, and temporal filtering, and respect active prune policies (pruned nodes are invisible to the algorithm). +Both methods support edge-label, node-label, and temporal filtering, and respect active prune policies (pruned nodes are invisible to the algorithm). ### Batch adjacency @@ -158,11 +160,11 @@ Over time, segments accumulate. Old segments may contain outdated versions of re OverGraph's compaction is designed to be fast: -1. **Plan from metadata.** Each segment has a metadata sidecar with per-record summary info (ID, timestamps, weight, tombstone status). The compaction planner reads only sidecars to decide which records survive, without touching the actual record data. +1. **Plan from metadata.** Each segment has packed metadata payloads with per-record summary info (ID, timestamps, weight, tombstone status). The compaction planner reads only metadata payloads to decide which records survive, without touching the actual record data. -2. **Binary copy.** Winning records are copied as raw byte spans from input segments to the output segment. No deserialization, no re-serialization. Just memcpy. +2. **Binary copy.** Winning records are copied as raw byte spans from input logical payloads to the output `segment.core`. No deserialization, no re-serialization. Just memcpy. -3. **Metadata-driven index building.** All output indexes (adjacency, key, type, property, timestamp) are built from the metadata of winning records, not from the records themselves. This avoids a second pass over the data. Vector indexes (HNSW and sparse posting lists) are rebuilt from surviving vector blob payloads and metadata. +3. **Metadata-driven index building.** Maintained core output indexes (adjacency, key, internal label-token, timestamp, triple, and edge metadata indexes) are built from the metadata of winning records, not from the records themselves. This avoids a second pass over the data. Optional declared property indexes, HNSW, and sparse posting lists remain external accelerators. 4. **Cascade deletes.** If a node is tombstoned or pruned, all its incident edges are automatically dropped during the edge merge pass. diff --git a/docs/getting-started.md b/docs/getting-started.md index 8238e1e..1115827 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -29,7 +29,7 @@ A database is a directory on disk. Pass a vector dimension if you want to use de ```python from overgraph import OverGraph -db = OverGraph.open("./my-graph", dense_vector_dimension=384) +db = OverGraph.open("./my-graph", dense_vector_dimension=3) ``` **Node.js** @@ -37,18 +37,18 @@ db = OverGraph.open("./my-graph", dense_vector_dimension=384) import { OverGraph } from 'overgraph'; const db = OverGraph.open('./my-graph', { - denseVector: { dimension: 384 }, + denseVector: { dimension: 3 }, }); ``` **Rust** ```rust use overgraph::*; -use std::path::Path; +use std::{collections::BTreeMap, path::Path}; let opts = DbOptions { dense_vector: Some(DenseVectorConfig { - dimension: 384, + dimension: 3, metric: DenseMetric::Cosine, hnsw: HnswConfig::default(), }), @@ -57,97 +57,91 @@ let opts = DbOptions { let mut db = DatabaseEngine::open(Path::new("./my-graph"), &opts)?; ``` -## Define type IDs +## Choose labels and edge labels -OverGraph uses integers to classify nodes and edges. Define them as constants: - -**Python** -```python -USER = 1 -PROJECT = 2 -WORKS_ON = 10 -``` - -**Node.js** -```javascript -const USER = 1; -const PROJECT = 2; -const WORKS_ON = 10; -``` - -**Rust** -```rust -const USER: u32 = 1; -const PROJECT: u32 = 2; -const WORKS_ON: u32 = 10; -``` +OverGraph uses labels to classify nodes and edge labels to classify edges. They are +ordinary strings at the public API boundary. ## Create nodes and edges **Python** ```python -alice = db.upsert_node(USER, "alice", props={"role": "engineer"}) -bob = db.upsert_node(USER, "bob") -project = db.upsert_node(PROJECT, "atlas", - dense_vector=project_embedding, # from your embedding model - sparse_vector=project_sparse) # from SPLADE, BGE-M3, etc. - -db.upsert_edge(alice, project, WORKS_ON) -db.upsert_edge(bob, project, WORKS_ON, weight=0.5) +project_dense = [0.18, 0.71, 0.39] +project_sparse = [(101, 0.6), (407, 0.8)] + +# Also accepts multiple labels: ["User", "Engineer"] +alice = db.upsert_node("User", "alice", props={"role": "engineer"}) +bob = db.upsert_node("User", "bob") +project = db.upsert_node("Project", "atlas", + dense_vector=project_dense, + sparse_vector=project_sparse) + +db.upsert_edge(alice, project, "WORKS_ON") +db.upsert_edge(bob, project, "WORKS_ON", weight=0.5) ``` **Node.js** ```javascript -const alice = db.upsertNode(USER, 'alice', { props: { role: 'engineer' } }); -const bob = db.upsertNode(USER, 'bob'); -const project = db.upsertNode(PROJECT, 'atlas', { - denseVector: projectEmbedding, // from your embedding model - sparseVector: projectSparse, // from SPLADE, BGE-M3, etc. +const projectDense = [0.18, 0.71, 0.39]; +const projectSparse = [{ dimension: 101, value: 0.6 }, { dimension: 407, value: 0.8 }]; + +// Also accepts multiple labels: ['User', 'Engineer'] +const alice = db.upsertNode('User', 'alice', { props: { role: 'engineer' } }); +const bob = db.upsertNode('User', 'bob'); +const project = db.upsertNode('Project', 'atlas', { + denseVector: projectDense, + sparseVector: projectSparse, }); -db.upsertEdge(alice, project, WORKS_ON); -db.upsertEdge(bob, project, WORKS_ON, { weight: 0.5 }); +db.upsertEdge(alice, project, 'WORKS_ON'); +db.upsertEdge(bob, project, 'WORKS_ON', { weight: 0.5 }); ``` **Rust** ```rust -let alice = db.upsert_node(USER, "alice", UpsertNodeOptions { +let project_dense = vec![0.18_f32, 0.71, 0.39]; +let project_sparse = vec![(101, 0.6_f32), (407, 0.8)]; + +// Also accepts multiple labels: &["User", "Engineer"] +let alice = db.upsert_node("User", "alice", UpsertNodeOptions { props: BTreeMap::from([("role".into(), PropValue::String("engineer".into()))]), ..Default::default() })?; -let bob = db.upsert_node(USER, "bob", UpsertNodeOptions::default())?; -let project = db.upsert_node(PROJECT, "atlas", UpsertNodeOptions { - dense_vector: Some(project_embedding), // from your embedding model - sparse_vector: Some(project_sparse), // from SPLADE, BGE-M3, etc. +let bob = db.upsert_node("User", "bob", UpsertNodeOptions::default())?; +let project = db.upsert_node("Project", "atlas", UpsertNodeOptions { + dense_vector: Some(project_dense), + sparse_vector: Some(project_sparse), ..Default::default() })?; -db.upsert_edge(alice, project, WORKS_ON, UpsertEdgeOptions::default())?; -db.upsert_edge(bob, project, WORKS_ON, UpsertEdgeOptions { weight: 0.5, ..Default::default() })?; +db.upsert_edge(alice, project, "WORKS_ON", UpsertEdgeOptions::default())?; +db.upsert_edge(bob, project, "WORKS_ON", UpsertEdgeOptions { weight: 0.5, ..Default::default() })?; ``` -Upserting the same `(type_id, key)` pair updates the existing node instead of creating a duplicate. +Upsert APIs accept either a single label string or a label collection. Each live +`(label, key)` membership points at one node; if every supplied label/key membership +resolves to the same node, the upsert updates that node instead of creating a duplicate. ## Read data back **Python** ```python node = db.get_node(alice) -node = db.get_node_by_key(USER, "alice") +node = db.get_node_by_key("User", "alice") nodes = db.get_nodes([alice, bob]) # batch read ``` **Node.js** ```javascript const node = db.getNode(alice); -const node2 = db.getNodeByKey(USER, 'alice'); +const node2 = db.getNodeByKey('User', 'alice'); const nodes = db.getNodes([alice, bob]); ``` **Rust** ```rust let node = db.get_node(alice)?; -let node = db.get_node_by_key(USER, "alice")?; +let node = db.get_node_by_key("User", "alice")?; let nodes = db.get_nodes(&[alice, bob])?; ``` @@ -180,8 +174,11 @@ for n in &neighbors { **Python** ```python +query_dense = [0.14, 0.74, 0.36] +query_sparse = [(101, 1.0)] + hits = db.vector_search("hybrid", k=10, - dense_query=query_embedding, + dense_query=query_dense, sparse_query=query_sparse, scope_start_node_id=alice, scope_max_depth=3) @@ -192,9 +189,12 @@ for hit in hits: **Node.js** ```javascript +const queryDense = [0.14, 0.74, 0.36]; +const querySparse = [{ dimension: 101, value: 1.0 }]; + const hits = db.vectorSearch('hybrid', { k: 10, - denseQuery: queryEmbedding, + denseQuery: queryDense, sparseQuery: querySparse, scope: { startNodeId: alice, maxDepth: 3 }, }); @@ -204,19 +204,26 @@ hits.forEach(h => console.log(h.nodeId, h.score)); **Rust** ```rust +let query_dense = vec![0.14_f32, 0.74, 0.36]; +let query_sparse = vec![(101, 1.0_f32)]; + let hits = db.vector_search(&VectorSearchRequest { mode: VectorSearchMode::Hybrid, - dense_query: Some(query_embedding), + dense_query: Some(query_dense), sparse_query: Some(query_sparse), k: 10, + label_filter: None, + ef_search: None, scope: Some(VectorSearchScope { start_node_id: alice, max_depth: 3, direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, at_epoch: None, }), - ..Default::default() + dense_weight: None, + sparse_weight: None, + fusion_mode: None, })?; for hit in &hits { @@ -230,39 +237,53 @@ Property queries work without any extra setup. If a property is hot in your work **Python** ```python -db.ensure_node_property_index(USER, "role", "equality") -db.ensure_node_property_index(PROJECT, "priority", "range", domain="int") +from overgraph import PropertyRangeBound + +db.ensure_node_property_index("User", "role", "equality") +db.ensure_node_property_index("Project", "priority", "range", domain="int") -user_ids = db.find_nodes(USER, "role", "engineer") -priority_ids = db.find_nodes_range(PROJECT, "priority", lower=1, upper=5) +user_ids = db.find_nodes("User", "role", "engineer") +priority_ids = db.find_nodes_range( + "Project", + "priority", + PropertyRangeBound(1, domain="int"), + PropertyRangeBound(5, domain="int"), +) ``` **Node.js** ```javascript -db.ensureNodePropertyIndex(USER, 'role', 'equality'); -db.ensureNodePropertyIndex(PROJECT, 'priority', 'range', { domain: 'int' }); - -const userIds = db.findNodes(USER, 'role', 'engineer'); -const priorityIds = db.findNodesRange(PROJECT, 'priority', { lower: 1, upper: 5 }); +db.ensureNodePropertyIndex('User', 'role', { kind: 'equality' }); +db.ensureNodePropertyIndex('Project', 'priority', { kind: 'range', domain: 'int' }); + +const userIds = db.findNodes('User', 'role', 'engineer'); +const priorityIds = db.findNodesRange( + 'Project', + 'priority', + { value: 1, inclusive: true, domain: 'int' }, + { value: 5, inclusive: true, domain: 'int' }, +); ``` **Rust** ```rust -db.ensure_node_property_index(USER, "role", SecondaryIndexKind::Equality)?; +db.ensure_node_property_index("User", "role", SecondaryIndexKind::Equality)?; db.ensure_node_property_index( - PROJECT, + "Project", "priority", - SecondaryIndexKind::Range(SecondaryIndexRangeDomain::Int), + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, )?; -let user_ids = db.find_nodes(USER, "role", &PropValue::String("engineer".into()))?; +let user_ids = db.find_nodes("User", "role", &PropValue::String("engineer".into()))?; +let lower = PropertyRangeBound::Included(PropValue::Int(1)); +let upper = PropertyRangeBound::Included(PropValue::Int(5)); let priority_ids = db.find_nodes_range( - PROJECT, + "Project", "priority", - Some(PropValue::Int(1)), - Some(PropValue::Int(5)), - true, - true, + Some(&lower), + Some(&upper), )?; ``` @@ -274,7 +295,7 @@ db.close() # Or use a context manager: with OverGraph.open("./my-graph") as db: - db.upsert_node(USER, "alice") + db.upsert_node("User", "alice") ``` **Node.js** @@ -294,7 +315,7 @@ db.close()?; from overgraph import AsyncOverGraph async with await AsyncOverGraph.open("./my-graph") as db: - alice = await db.upsert_node(USER, "alice") + alice = await db.upsert_node("User", "alice") neighbors = await db.neighbors(alice) ``` diff --git a/docs/roadmap.md b/docs/roadmap.md index 1701312..6507567 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -15,11 +15,18 @@ Active development. These are the things we're building right now. OverGraph has always been API-first: you call functions, not write query strings. The query planner keeps that philosophy but adds a planning layer underneath. You describe what you want with a builder pattern, and the planner figures out the fastest way to get it. - **Query IR and builder API.** A composable query model for node predicates and small graph patterns. Aliases, compound predicates, hydration modes, deterministic ordering, limits. Works across Rust, Node.js, and Python. -- **Index-backed candidate sources.** The planner automatically selects from node ID/key lookups, type indexes, equality and range property indexes, timestamp filters, and adjacency sources. It picks the most selective anchor, intersects multiple sources, and falls back gracefully when no index covers the query. +- **Index-backed candidate sources.** The planner automatically selects from node ID/key lookups, node-label indexes, equality and range property indexes, timestamp filters, and adjacency sources. It picks the most selective anchor, intersects multiple sources, and falls back gracefully when no index covers the query. - **Compound predicate planning.** Multi-predicate queries that intersect index results instead of scanning. The planner chooses the cheapest path and verifies remaining predicates after the index narrows the candidates. Full scans require explicit opt-in. - **Graph pattern matching.** Describe a small subgraph pattern (linear chains, branching shapes, repeated-variable equality) and the planner finds matches using the same index infrastructure. Built on the traversal substrate, not a separate engine. - **Explain output.** Every query produces an inspectable plan so you can see exactly which indexes were used, what was filtered, and where time went. +### Multi-label nodes + +Nodes can carry one to ten public labels. Public records expose `labels`, node-label +convenience APIs use plural names, multi-label convenience scans use `All` semantics, +and explicit label filters expose `Any`/`All` modes for query, traversal, vector, and +export APIs. Edges remain single-label. + --- ## Next @@ -27,7 +34,7 @@ OverGraph has always been API-first: you call functions, not write query strings Queued up behind the current work. These build directly on the query planner. ### Compound and composite indexes -Declare a single index over ordered property tuples like `(type_id, status, score)` or `(type_id, tenant_id, updated_at)` with prefix semantics. The planner picks them up automatically. +Declare a single index over ordered property tuples like `(label, status, score)` or `(label, tenant_id, updated_at)` with prefix semantics. The planner picks them up automatically. ### Boolean predicate planning Extend queries beyond AND-only to support `OR`, `NOT`, `IN`, `exists`, and `missing` with index-union/intersection support. @@ -36,7 +43,7 @@ Extend queries beyond AND-only to support `OR`, `NOT`, `IN`, `exists`, and `miss Return exactly what you need: selected properties, selected metadata fields, or compact payloads. Avoid full record hydration when you only need IDs or a few fields. ### Edge queries and edge indexes -Promote edge predicates from post-filters to planned query sources. Query by edge type + property, endpoint + edge property, temporal windows, or weight ranges directly. +Promote edge predicates from post-filters to planned query sources. Query by edge label + property, endpoint + edge property, temporal windows, or weight ranges directly. ### Graph pattern queries v2 Pagination and cursors for pattern results. Variable-length path patterns. Optional pattern pieces. Better branch-order costing. @@ -45,7 +52,7 @@ Pagination and cursors for pattern results. Variable-length path patterns. Optio A read-only query-string surface that compiles into the native query IR and planner. You get the convenience of a familiar syntax without giving up the API-first foundation underneath. This is an adapter, not a replacement. ### Planner statistics and selectivity model -Durable stats for per-type counts, property cardinality, value distribution, and adjacency fanout so the planner can make evidence-based decisions instead of heuristic guesses. +Durable stats for per-label and per-edge-label counts, property cardinality, value distribution, and adjacency fanout so the planner can make evidence-based decisions instead of heuristic guesses. ### Graph-algorithm-scoped vector search Generalize vector search scoping beyond traversal start points. Run dense, sparse, or hybrid search over query results, pattern matches, PPR neighborhoods, community memberships, or explicit node sets. @@ -86,9 +93,6 @@ Betweenness centrality and harmonic closeness over graph projections. Reuses pro ### Triangle counting and clustering coefficients Sorted adjacency intersection for per-node and global triangle counts. Clustering coefficients for identifying tightly connected neighborhoods. -### Multi-label nodes -Upgrade from a single `type_id` to `type_ids` with any/all matching. Format version bump with migration. - ### Schema and constraints Optional property validation, required fields, edge endpoint constraints, and uniqueness enforcement. Manifest-stored, fully opt-in. Schemas are never required. diff --git a/examples/node/knowledge-graph.mjs b/examples/node/knowledge-graph.mjs index 85f2f25..911070a 100644 --- a/examples/node/knowledge-graph.mjs +++ b/examples/node/knowledge-graph.mjs @@ -2,58 +2,60 @@ * OverGraph Example: Knowledge Graph * * This example shows how to use OverGraph to build and query a - * knowledge graph of entities, facts, and relationships. + * knowledge graph of people, projects, facts, and relationships. * * Run: node examples/node/knowledge-graph.mjs * (Requires: npm run build in overgraph-node/ first) */ -import { OverGraph } from '../../overgraph-node/index.js'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; -// --- Type IDs (you define these for your application) --- -const ENTITY = 1; -const FACT = 2; -const CONVERSATION = 3; +import { OverGraph } from '../../overgraph-node/index.js'; -const RELATED_TO = 10; -const MENTIONED_IN = 11; -const SUPPORTS = 12; +// --- Edge labels for this application --- +const RELATED_TO = 'RELATED_TO'; +const MENTIONED_IN = 'MENTIONED_IN'; +const SUPPORTS = 'SUPPORTS'; -async function main() { - // Open (or create) a database in a local directory - const db = OverGraph.open('./example-graph'); +function main() { + const dbPath = mkdtempSync(join(tmpdir(), 'overgraph-knowledge-node-')); + let db; try { + db = OverGraph.open(dbPath); + // --- Build a knowledge graph --- - // Create some entities - const [alice, bob, project] = await db.batchUpsertNodesAsync([ - { typeId: ENTITY, key: 'person:alice', props: { name: 'Alice', role: 'engineer' }, weight: 1.0 }, - { typeId: ENTITY, key: 'person:bob', props: { name: 'Bob', role: 'designer' }, weight: 0.9 }, - { typeId: ENTITY, key: 'project:atlas', props: { name: 'Atlas', status: 'active' }, weight: 0.95 }, + // Create people and a project + const [alice, bob, project] = db.batchUpsertNodes([ + { labels: ['Person', 'Contributor'], key: 'alice', props: { name: 'Alice', role: 'engineer' }, weight: 1.0 }, + { labels: ['Person', 'Contributor'], key: 'bob', props: { name: 'Bob', role: 'designer' }, weight: 0.9 }, + { labels: ['Project'], key: 'atlas', props: { name: 'Atlas', status: 'active' }, weight: 0.95 }, ]); // Create some facts - const [fact1, fact2] = await db.batchUpsertNodesAsync([ - { typeId: FACT, key: 'fact:alice-leads-atlas', props: { text: 'Alice leads the Atlas project' }, weight: 0.9 }, - { typeId: FACT, key: 'fact:bob-designs-atlas', props: { text: 'Bob is the lead designer on Atlas' }, weight: 0.85 }, + const [fact1, fact2] = db.batchUpsertNodes([ + { labels: ['Fact'], key: 'alice-leads-atlas', props: { text: 'Alice leads the Atlas project' }, weight: 0.9 }, + { labels: ['Fact'], key: 'bob-designs-atlas', props: { text: 'Bob is the lead designer on Atlas' }, weight: 0.85 }, ]); // Create a conversation node - const convo = await db.upsertNodeAsync(CONVERSATION, 'convo:2024-01-15', { + const convo = db.upsertNode(['Conversation'], '2024-01-15', { props: { summary: 'Discussed Atlas project timeline' }, weight: 0.7, }); - // Connect everything with typed edges - await db.batchUpsertEdgesAsync([ - { from: alice, to: project, typeId: RELATED_TO, props: { role: 'lead' }, weight: 1.0 }, - { from: bob, to: project, typeId: RELATED_TO, props: { role: 'designer' }, weight: 0.9 }, - { from: alice, to: bob, typeId: RELATED_TO, props: { context: 'teammates' }, weight: 0.8 }, - { from: fact1, to: convo, typeId: MENTIONED_IN, weight: 0.9 }, - { from: fact2, to: convo, typeId: MENTIONED_IN, weight: 0.85 }, - { from: fact1, to: alice, typeId: SUPPORTS, weight: 0.9 }, - { from: fact1, to: project, typeId: SUPPORTS, weight: 0.9 }, + // Connect everything with labeled edges + db.batchUpsertEdges([ + { from: alice, to: project, label: RELATED_TO, props: { role: 'lead' }, weight: 1.0 }, + { from: bob, to: project, label: RELATED_TO, props: { role: 'designer' }, weight: 0.9 }, + { from: alice, to: bob, label: RELATED_TO, props: { context: 'teammates' }, weight: 0.8 }, + { from: fact1, to: convo, label: MENTIONED_IN, weight: 0.9 }, + { from: fact2, to: convo, label: MENTIONED_IN, weight: 0.85 }, + { from: fact1, to: alice, label: SUPPORTS, weight: 0.9 }, + { from: fact1, to: project, label: SUPPORTS, weight: 0.9 }, ]); console.log('Knowledge graph built!\n'); @@ -61,44 +63,44 @@ async function main() { // --- Query the graph --- // 1. Who is Alice connected to? - const aliceNeighbors = await db.neighborsAsync(alice, { + const aliceNeighbors = db.neighbors(alice, { direction: 'outgoing', - typeFilter: [RELATED_TO], + edgeLabelFilter: [RELATED_TO], limit: 10, }); console.log(`Alice's connections (${aliceNeighbors.length}):`); for (let i = 0; i < aliceNeighbors.length; i++) { - const node = await db.getNodeAsync(aliceNeighbors.nodeId(i)); - console.log(` -> ${node.props.name} (weight: ${aliceNeighbors.weight(i)})`); + const node = db.getNode(aliceNeighbors[i].nodeId); + console.log(` -> ${node.props.name} (weight: ${aliceNeighbors[i].weight})`); } - // 2. Find all entities - const entities = await db.getNodesByTypeAsync(ENTITY); - console.log(`\nAll entities (${entities.length}):`); - for (const entity of entities) { - console.log(` ${entity.props.name} (${entity.props.role || entity.props.status})`); + // 2. Find all people + const people = db.getNodesByLabels('Person'); + console.log(`\nAll people (${people.length}):`); + for (const person of people) { + console.log(` ${person.props.name} (${person.props.role})`); } // 3. Top-K: most important connections from the Atlas project - const topK = await db.topKNeighborsAsync(project, 5, { + const topK = db.topKNeighbors(project, 5, { direction: 'incoming', - typeFilter: [RELATED_TO], + edgeLabelFilter: [RELATED_TO], scoring: 'weight', }); console.log(`\nTop connections to Atlas (${topK.length}):`); for (let i = 0; i < topK.length; i++) { - const node = await db.getNodeAsync(topK.nodeId(i)); - console.log(` ${node.props.name} (score: ${topK.weight(i)})`); + const node = db.getNode(topK[i].nodeId); + console.log(` ${node.props.name} (score: ${topK[i].weight})`); } // 4. Personalized PageRank: what's most relevant to Alice? - const ppr = await db.personalizedPagerankAsync([alice], { + const ppr = db.personalizedPagerank([alice], { maxResults: 5, maxIterations: 50, }); console.log(`\nMost relevant to Alice (PPR, ${ppr.scores.length} results):`); for (let i = 0; i < ppr.scores.length; i++) { - const node = await db.getNodeAsync(ppr.nodeIds[i]); + const node = db.getNode(ppr.nodeIds[i]); if (node) { const name = node.props.name || node.props.text || node.props.summary || node.key; console.log(` ${name}: ${ppr.scores[i].toFixed(4)}`); @@ -106,31 +108,47 @@ async function main() { } // 5. Paginated listing - let page = await db.nodesByTypePagedAsync(ENTITY, 2); - console.log(`\nPaginated entities (page 1, ${page.items.length} items):`); + let page = db.nodesByLabelsPaged('Person', 1); + console.log(`\nPaginated people (page 1, ${page.items.length} items):`); for (const id of page.items) { - const node = await db.getNodeAsync(id); + const node = db.getNode(id); console.log(` ${node.props.name}`); } if (page.nextCursor !== null) { - const page2 = await db.nodesByTypePagedAsync(ENTITY, 2, page.nextCursor); + const page2 = db.nodesByLabelsPaged('Person', 1, page.nextCursor); console.log(`Page 2 (${page2.items.length} items):`); for (const id of page2.items) { - const node = await db.getNodeAsync(id); + const node = db.getNode(id); console.log(` ${node.props.name}`); } } // 6. Database stats - const stats = await db.statsAsync(); + const stats = db.stats(); console.log(`\nDatabase stats:`); console.log(` Segments: ${stats.segmentCount}`); console.log(` WAL sync mode: ${stats.walSyncMode}`); } finally { - await db.closeAsync(); + let closeError = null; + if (db) { + try { + db.close(); + } catch (err) { + closeError = err; + } + } + rmSync(dbPath, { recursive: true, force: true }); console.log('\nDatabase closed.'); + if (closeError) { + throw closeError; + } } } -main().catch(console.error); +try { + main(); +} catch (err) { + console.error(err); + process.exitCode = 1; +} diff --git a/examples/python/knowledge_graph.py b/examples/python/knowledge_graph.py index 9ba7418..b238fdb 100644 --- a/examples/python/knowledge_graph.py +++ b/examples/python/knowledge_graph.py @@ -2,121 +2,118 @@ OverGraph Example: Knowledge Graph This example shows how to use OverGraph to build and query a -knowledge graph of entities, facts, and relationships. +knowledge graph of people, projects, facts, and relationships. Run: python examples/python/knowledge_graph.py (Requires: maturin develop in overgraph-python/ first) """ +import tempfile + from overgraph import OverGraph -# Type IDs (you define these for your application) -ENTITY = 1 -FACT = 2 -CONVERSATION = 3 -RELATED_TO = 10 -MENTIONED_IN = 11 -SUPPORTS = 12 +RELATED_TO = "RELATED_TO" +MENTIONED_IN = "MENTIONED_IN" +SUPPORTS = "SUPPORTS" def main(): - # Open (or create) a database in a local directory - with OverGraph.open("./example-graph") as db: - # --- Build a knowledge graph --- - - # Create some entities - alice, bob, project = db.batch_upsert_nodes([ - {"type_id": ENTITY, "key": "person:alice", "props": {"name": "Alice", "role": "engineer"}, "weight": 1.0}, - {"type_id": ENTITY, "key": "person:bob", "props": {"name": "Bob", "role": "designer"}, "weight": 0.9}, - {"type_id": ENTITY, "key": "project:atlas", "props": {"name": "Atlas", "status": "active"}, "weight": 0.95}, - ]) - - # Create some facts - fact1, fact2 = db.batch_upsert_nodes([ - {"type_id": FACT, "key": "fact:alice-leads-atlas", "props": {"text": "Alice leads the Atlas project"}, "weight": 0.9}, - {"type_id": FACT, "key": "fact:bob-designs-atlas", "props": {"text": "Bob is the lead designer on Atlas"}, "weight": 0.85}, - ]) - - # Create a conversation node - convo = db.upsert_node(CONVERSATION, "convo:2024-01-15", - props={"summary": "Discussed Atlas project timeline"}, weight=0.7) - - # Connect everything with typed edges - db.batch_upsert_edges([ - {"from_id": alice, "to_id": project, "type_id": RELATED_TO, "props": {"role": "lead"}, "weight": 1.0}, - {"from_id": bob, "to_id": project, "type_id": RELATED_TO, "props": {"role": "designer"}, "weight": 0.9}, - {"from_id": alice, "to_id": bob, "type_id": RELATED_TO, "props": {"context": "teammates"}, "weight": 0.8}, - {"from_id": fact1, "to_id": convo, "type_id": MENTIONED_IN, "weight": 0.9}, - {"from_id": fact2, "to_id": convo, "type_id": MENTIONED_IN, "weight": 0.85}, - {"from_id": fact1, "to_id": alice, "type_id": SUPPORTS, "weight": 0.9}, - {"from_id": fact1, "to_id": project, "type_id": SUPPORTS, "weight": 0.9}, - ]) - - print("Knowledge graph built!\n") - - # --- Query the graph --- - - # 1. Who is Alice connected to? - neighbors = db.neighbors(alice, "outgoing", type_filter=[RELATED_TO], limit=10) - print(f"Alice's connections ({len(neighbors)}):") - for n in neighbors: - node = db.get_node(n.node_id) - print(f" -> {node.props['name']} (weight: {n.weight})") - - # 2. Find all entities - entities = db.get_nodes_by_type(ENTITY) - print(f"\nAll entities ({len(entities)}):") - for entity in entities: - detail = entity.props.get("role") or entity.props.get("status") - print(f" {entity.props['name']} ({detail})") - - # 3. Top-K: most important connections to the Atlas project - top_k = db.top_k_neighbors(project, k=5, direction="incoming", - type_filter=[RELATED_TO], scoring="weight") - print(f"\nTop connections to Atlas ({len(top_k)}):") - for n in top_k: - node = db.get_node(n.node_id) - print(f" {node.props['name']} (score: {n.weight})") - - # 4. Personalized PageRank: what's most relevant to Alice? - ppr = db.personalized_pagerank([alice], max_results=5, max_iterations=50) - print(f"\nMost relevant to Alice (PPR, {len(ppr.node_ids)} results):") - for node_id, score in zip(ppr.node_ids, ppr.scores): - node = db.get_node(node_id) - if node: - name = node.props.get("name") or node.props.get("text") or node.props.get("summary") or node.key - print(f" {name}: {score:.4f}") - - # 5. Paginated listing - page = db.nodes_by_type_paged(ENTITY, limit=2) - print(f"\nPaginated entities (page 1, {len(page.items)} items):") - for node_id in page.items: - node = db.get_node(node_id) - print(f" {node.props['name']}") - - if page.next_cursor is not None: - page2 = db.nodes_by_type_paged(ENTITY, limit=2, after=page.next_cursor) - print(f"Page 2 ({len(page2.items)} items):") - for node_id in page2.items: + with tempfile.TemporaryDirectory(prefix="overgraph-knowledge-python-") as db_dir: + with OverGraph.open(db_dir) as db: + # --- Build a knowledge graph --- + + # Create people and a project + alice, bob, project = db.batch_upsert_nodes([ + {"labels": ["Person", "Contributor"], "key": "alice", "props": {"name": "Alice", "role": "engineer"}, "weight": 1.0}, + {"labels": ["Person", "Contributor"], "key": "bob", "props": {"name": "Bob", "role": "designer"}, "weight": 0.9}, + {"labels": ["Project"], "key": "atlas", "props": {"name": "Atlas", "status": "active"}, "weight": 0.95}, + ]) + + # Create some facts + fact1, fact2 = db.batch_upsert_nodes([ + {"labels": ["Fact"], "key": "alice-leads-atlas", "props": {"text": "Alice leads the Atlas project"}, "weight": 0.9}, + {"labels": ["Fact"], "key": "bob-designs-atlas", "props": {"text": "Bob is the lead designer on Atlas"}, "weight": 0.85}, + ]) + + # Create a conversation node + convo = db.upsert_node(["Conversation"], "2024-01-15", + props={"summary": "Discussed Atlas project timeline"}, weight=0.7) + + # Connect everything with labeled edges + db.batch_upsert_edges([ + {"from_id": alice, "to_id": project, "label": RELATED_TO, "props": {"role": "lead"}, "weight": 1.0}, + {"from_id": bob, "to_id": project, "label": RELATED_TO, "props": {"role": "designer"}, "weight": 0.9}, + {"from_id": alice, "to_id": bob, "label": RELATED_TO, "props": {"context": "teammates"}, "weight": 0.8}, + {"from_id": fact1, "to_id": convo, "label": MENTIONED_IN, "weight": 0.9}, + {"from_id": fact2, "to_id": convo, "label": MENTIONED_IN, "weight": 0.85}, + {"from_id": fact1, "to_id": alice, "label": SUPPORTS, "weight": 0.9}, + {"from_id": fact1, "to_id": project, "label": SUPPORTS, "weight": 0.9}, + ]) + + print("Knowledge graph built!\n") + + # --- Query the graph --- + + # 1. Who is Alice connected to? + neighbors = db.neighbors(alice, direction="outgoing", edge_label_filter=[RELATED_TO], limit=10) + print(f"Alice's connections ({len(neighbors)}):") + for n in neighbors: + node = db.get_node(n.node_id) + print(f" -> {node.props['name']} (weight: {n.weight})") + + # 2. Find all people + people = db.get_nodes_by_labels("Person") + print(f"\nAll people ({len(people)}):") + for person in people: + print(f" {person.props['name']} ({person.props['role']})") + + # 3. Top-K: most important connections to the Atlas project + top_k = db.top_k_neighbors(project, k=5, direction="incoming", + edge_label_filter=[RELATED_TO], scoring="weight") + print(f"\nTop connections to Atlas ({len(top_k)}):") + for n in top_k: + node = db.get_node(n.node_id) + print(f" {node.props['name']} (score: {n.weight})") + + # 4. Personalized PageRank: what's most relevant to Alice? + ppr = db.personalized_pagerank([alice], max_results=5, max_iterations=50) + print(f"\nMost relevant to Alice (PPR, {len(ppr.node_ids)} results):") + for node_id, score in zip(ppr.node_ids, ppr.scores): + node = db.get_node(node_id) + if node: + name = node.props.get("name") or node.props.get("text") or node.props.get("summary") or node.key + print(f" {name}: {score:.4f}") + + # 5. Paginated listing + page = db.nodes_by_labels_paged("Person", limit=1) + print(f"\nPaginated people (page 1, {len(page.items)} items):") + for node_id in page.items: node = db.get_node(node_id) print(f" {node.props['name']}") - # 6. Retention policies - db.set_prune_policy("short_term", max_age_ms=86_400_000, type_id=CONVERSATION) - policies = db.list_prune_policies() - print(f"\nActive prune policies ({len(policies)}):") - for p in policies: - print(f" {p.name}: max_age_ms={p.max_age_ms}, type_id={p.type_id}") - - # Clean up the policy for this example - db.remove_prune_policy("short_term") - - # 7. Database stats - stats = db.stats() - print(f"\nDatabase stats:") - print(f" Segments: {stats.segment_count}") - print(f" WAL sync mode: {stats.wal_sync_mode}") + if page.next_cursor is not None: + page2 = db.nodes_by_labels_paged("Person", limit=1, after=page.next_cursor) + print(f"Page 2 ({len(page2.items)} items):") + for node_id in page2.items: + node = db.get_node(node_id) + print(f" {node.props['name']}") + + # 6. Retention policies + db.set_prune_policy("short_term", max_age_ms=86_400_000, label="Conversation") + policies = db.list_prune_policies() + print(f"\nActive prune policies ({len(policies)}):") + for p in policies: + print(f" {p.name}: max_age_ms={p.max_age_ms}, label={p.label}") + + # Clean up the policy for this example + db.remove_prune_policy("short_term") + + # 7. Database stats + stats = db.stats() + print(f"\nDatabase stats:") + print(f" Segments: {stats.segment_count}") + print(f" WAL sync mode: {stats.wal_sync_mode}") print("\nDatabase closed.") diff --git a/examples/rust/knowledge_graph.rs b/examples/rust/knowledge_graph.rs index 81752e5..2adcc0c 100644 --- a/examples/rust/knowledge_graph.rs +++ b/examples/rust/knowledge_graph.rs @@ -1,7 +1,7 @@ //! OverGraph Example: Knowledge Graph //! //! This example shows how to use OverGraph to build and query a -//! knowledge graph of entities, facts, and relationships. +//! knowledge graph of people, projects, facts, and relationships. //! //! Run: cargo run --example knowledge_graph @@ -10,16 +10,12 @@ use overgraph::{ UpsertNodeOptions, }; use std::collections::BTreeMap; +use std::env; +use std::fs; use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; -// Type IDs (you define these for your application) -const ENTITY: u32 = 1; -const FACT: u32 = 2; -const CONVERSATION: u32 = 3; - -const RELATED_TO: u32 = 10; -const MENTIONED_IN: u32 = 11; -const SUPPORTS: u32 = 12; +const RELATED_TO: &str = "RELATED_TO"; fn props(pairs: &[(&str, &str)]) -> BTreeMap { pairs @@ -29,52 +25,69 @@ fn props(pairs: &[(&str, &str)]) -> BTreeMap { } fn main() -> Result<(), Box> { - let db = DatabaseEngine::open(Path::new("./example-graph"), &DbOptions::default())?; + let run_id = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(); + let db_path = env::temp_dir().join(format!("overgraph-knowledge-rust-{run_id}")); + + let result = run_example(&db_path); + let _ = fs::remove_dir_all(&db_path); + result +} + +fn run_example(db_path: &Path) -> Result<(), Box> { + let db = DatabaseEngine::open(db_path, &DbOptions::default())?; + db.ensure_node_label("Person")?; + db.ensure_node_label("Contributor")?; + db.ensure_node_label("Project")?; + db.ensure_node_label("Fact")?; + db.ensure_node_label("Conversation")?; + db.ensure_edge_label("RELATED_TO")?; + db.ensure_edge_label("MENTIONED_IN")?; + db.ensure_edge_label("SUPPORTS")?; // --- Build a knowledge graph --- - // Create some entities - let entity_ids = db.batch_upsert_nodes(&[ + // Create people and a project + let node_ids = db.batch_upsert_nodes(vec![ NodeInput { - type_id: ENTITY, - key: "person:alice".into(), + labels: vec!["Person".into(), "Contributor".into()], + key: "alice".into(), props: props(&[("name", "Alice"), ("role", "engineer")]), weight: 1.0, dense_vector: None, sparse_vector: None, }, NodeInput { - type_id: ENTITY, - key: "person:bob".into(), + labels: vec!["Person".into(), "Contributor".into()], + key: "bob".into(), props: props(&[("name", "Bob"), ("role", "designer")]), weight: 0.9, dense_vector: None, sparse_vector: None, }, NodeInput { - type_id: ENTITY, - key: "project:atlas".into(), + labels: vec!["Project".into()], + key: "atlas".into(), props: props(&[("name", "Atlas"), ("status", "active")]), weight: 0.95, dense_vector: None, sparse_vector: None, }, ])?; - let (alice, bob, project) = (entity_ids[0], entity_ids[1], entity_ids[2]); + let (alice, bob, project) = (node_ids[0], node_ids[1], node_ids[2]); // Create some facts - let fact_ids = db.batch_upsert_nodes(&[ + let fact_ids = db.batch_upsert_nodes(vec![ NodeInput { - type_id: FACT, - key: "fact:alice-leads-atlas".into(), + labels: vec!["Fact".into()], + key: "alice-leads-atlas".into(), props: props(&[("text", "Alice leads the Atlas project")]), weight: 0.9, dense_vector: None, sparse_vector: None, }, NodeInput { - type_id: FACT, - key: "fact:bob-designs-atlas".into(), + labels: vec!["Fact".into()], + key: "bob-designs-atlas".into(), props: props(&[("text", "Bob is the lead designer on Atlas")]), weight: 0.85, dense_vector: None, @@ -85,8 +98,8 @@ fn main() -> Result<(), Box> { // Create a conversation node let convo = db.upsert_node( - CONVERSATION, - "convo:2024-01-15", + "Conversation", + "2024-01-15", UpsertNodeOptions { props: props(&[("summary", "Discussed Atlas project timeline")]), weight: 0.7, @@ -94,12 +107,12 @@ fn main() -> Result<(), Box> { }, )?; - // Connect everything with typed edges - db.batch_upsert_edges(&[ + // Connect everything with labeled edges + db.batch_upsert_edges(vec![ EdgeInput { from: alice, to: project, - type_id: RELATED_TO, + label: RELATED_TO.into(), props: props(&[("role", "lead")]), weight: 1.0, valid_from: None, @@ -108,7 +121,7 @@ fn main() -> Result<(), Box> { EdgeInput { from: bob, to: project, - type_id: RELATED_TO, + label: RELATED_TO.into(), props: props(&[("role", "designer")]), weight: 0.9, valid_from: None, @@ -117,7 +130,7 @@ fn main() -> Result<(), Box> { EdgeInput { from: alice, to: bob, - type_id: RELATED_TO, + label: RELATED_TO.into(), props: props(&[("context", "teammates")]), weight: 0.8, valid_from: None, @@ -126,7 +139,7 @@ fn main() -> Result<(), Box> { EdgeInput { from: fact1, to: convo, - type_id: MENTIONED_IN, + label: "MENTIONED_IN".into(), props: Default::default(), weight: 0.9, valid_from: None, @@ -135,7 +148,7 @@ fn main() -> Result<(), Box> { EdgeInput { from: fact2, to: convo, - type_id: MENTIONED_IN, + label: "MENTIONED_IN".into(), props: Default::default(), weight: 0.85, valid_from: None, @@ -144,7 +157,7 @@ fn main() -> Result<(), Box> { EdgeInput { from: fact1, to: alice, - type_id: SUPPORTS, + label: "SUPPORTS".into(), props: Default::default(), weight: 0.9, valid_from: None, @@ -153,7 +166,7 @@ fn main() -> Result<(), Box> { EdgeInput { from: fact1, to: project, - type_id: SUPPORTS, + label: "SUPPORTS".into(), props: Default::default(), weight: 0.9, valid_from: None, @@ -169,7 +182,7 @@ fn main() -> Result<(), Box> { let neighbors = db.neighbors( alice, &NeighborOptions { - type_filter: Some(vec![RELATED_TO]), + edge_label_filter: Some(vec![RELATED_TO.to_string()]), limit: Some(10), ..Default::default() }, @@ -183,20 +196,19 @@ fn main() -> Result<(), Box> { } } - // 2. Find all entities - let entities = db.get_nodes_by_type(ENTITY)?; - println!("\nAll entities ({}):", entities.len()); - for entity in &entities { - let name = match entity.props.get("name") { + // 2. Find all people + let people = db.get_nodes_by_labels("Person")?; + println!("\nAll people ({}):", people.len()); + for person in &people { + let name = match person.props.get("name") { Some(PropValue::String(s)) => s.as_str(), - _ => &entity.key, + _ => &person.key, }; - let detail = entity.props.get("role").or(entity.props.get("status")); - let detail_str = match detail { + let role = match person.props.get("role") { Some(PropValue::String(s)) => s.as_str(), _ => "unknown", }; - println!(" {} ({})", name, detail_str); + println!(" {} ({})", name, role); } // 3. Personalized PageRank: what's most relevant to Alice? diff --git a/overgraph-node/Cargo.toml b/overgraph-node/Cargo.toml index 3a8b7eb..e3da527 100644 --- a/overgraph-node/Cargo.toml +++ b/overgraph-node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "overgraph-node" -version = "0.7.0" +version = "0.8.0" edition = "2021" description = "Node.js native addon for OverGraph" diff --git a/overgraph-node/README.md b/overgraph-node/README.md index 952cc79..e5217f0 100644 --- a/overgraph-node/README.md +++ b/overgraph-node/README.md @@ -2,7 +2,7 @@

OverGraph

An absurdly fast embedded graph database with built-in vector search.
- Pure Rust. Sub-microsecond reads. Native connectors for Node.js and Python.
+ Pure Rust. Sub-microsecond reads. Native Node.js connector.
Built for AI agent memory, knowledge graphs, RAG pipelines, and semantic search.

@@ -22,7 +22,7 @@ OverGraph is a graph database that runs inside your process. No server, no netwo I built it because I wanted a graph database that was genuinely fast. Not "fast for a database," but fast enough that you forget it's there. Node lookups in 34 nanoseconds. Neighbor traversals in 2 microseconds. Batch writes at 1.29M+ nodes per second. And I wanted graph structure and vector similarity to live together in one engine. No separate vector database, no external index, no synchronization headaches. -It's written entirely in Rust and it ships native connectors for Node.js (napi-rs) and Python (PyO3) so you can use it from whatever you're building in. +It's written entirely in Rust and ships a native Node.js connector built with napi-rs, so JavaScript and TypeScript applications call the engine in-process without a server or REST layer. ## Built for @@ -40,109 +40,57 @@ It's written entirely in Rust and it ships native connectors for Node.js (napi-r - **Graph + vectors in one engine.** Dense HNSW and sparse inverted indexes live alongside graph adjacency indexes in the same storage engine. Vector search can be scoped to graph neighborhoods ("find similar nodes within 2 hops of X") without a second database or a synchronization layer. - **Rich graph primitives.** Weighted nodes and edges, temporal validity windows, exponential decay scoring, automatic retention policies. Model relationships that evolve over time, and let the graph clean up what's no longer relevant. - **Fast where it matters.** Node lookups in ~34ns. Neighbor traversal in ~2μs. Batch writes at 1.29M+ nodes/sec. The storage engine is a log-structured merge tree with mmap'd immutable segments, so reads never block writes. -- **Explicit write transactions.** Stage ordered node and edge mutations locally, read your own staged writes, then commit atomically with optimistic conflict detection. Available in Rust, Node.js, and Python. -- **Three languages, one engine.** Rust core with native bindings for Node.js (napi-rs) and Python (PyO3). Not a wrapper around a REST API. Actual FFI into the same Rust engine with minimal overhead. +- **Explicit write transactions.** Stage ordered node and edge mutations locally, read your own staged writes, then commit atomically with optimistic conflict detection through the Node.js API. +- **Native Node.js, one engine.** Rust core with napi-rs bindings. Not a wrapper around a REST API. Actual FFI into the same Rust engine with minimal overhead. - **Full queries as functions.** Use regular APIs for everything: `findNodes` for direct property lookups, `queryNodeIds` / `queryNodes` for full boolean node queries, and `queryPattern` for bounded graph pattern matching. No query strings to parse, escape, or generate. ## Performance -All numbers from a real benchmark suite running on the Rust core (group-commit durability mode, small profile: 10K nodes / 50K edges). Full methodology and reproducibility guide in [`docs/04-quality/Benchmark-Methodology.md`](docs/04-quality/Benchmark-Methodology.md). +All numbers from a real benchmark suite running on the Rust core (group-commit durability mode, small profile: 10K nodes / 50K edges). Full methodology and reproducibility guide in [`docs/04-quality/Benchmark-Methodology.md`](../docs/04-quality/Benchmark-Methodology.md). | Operation | Latency | Throughput | |---|---|---| -| `get_node` | 34 ns | 29M ops/s | -| `upsert_node` | 2.2 μs | 807K ops/s | +| `getNode` | 34 ns | 29M ops/s | +| `upsertNode` | 2.2 μs | 807K ops/s | | `neighbors` (1-hop) | 2.1 μs | 541K ops/s | -| `batch_upsert_nodes` (100) | 77.261 µs | 1.29M nodes/s | -| `top_k_neighbors` | 17.5 μs | 81K ops/s | -| `personalized_pagerank` | 254 μs | 4.3K ops/s | +| `batchUpsertNodes` (100) | 77.261 µs | 1.29M nodes/s | +| `topKNeighbors` | 17.5 μs | 81K ops/s | +| `personalizedPagerank` | 254 μs | 4.3K ops/s | -Node.js and Python connectors add minimal overhead. Batch operations are especially efficient because they amortize the FFI boundary cost. Full cross-language comparison in the [launch benchmark pack](docs/04-quality/reports/2026-03-04-launch-pack-parity/). +The Node.js connector adds minimal overhead. Batch operations are especially efficient because they amortize the FFI boundary cost. Full methodology is covered in the [launch benchmark pack](../docs/04-quality/reports/2026-03-04-launch-pack-parity/). ## Install Prebuilt binaries are available for macOS (ARM + Intel), Linux (x64), and Windows (x64). No Rust toolchain required. -**Python** -```bash -pip install overgraph -``` - -**Node.js** ```bash npm install overgraph ``` -**Rust** -```bash -cargo add overgraph -``` - ## Quick start -### Python - -```python -from overgraph import OverGraph - -USER = 1 -PROJECT = 2 -CREATED = 10 - -with OverGraph.open("./my-graph", dense_vector_dimension=384) as db: - # Embeddings come from your model (sentence-transformers, OpenAI, etc.) - # dense: model.encode("Alice is an engineer") -> [f32; 384] - # sparse: splade.encode("Alice is an engineer") -> [(token_id, weight), ...] - alice = db.upsert_node(USER, "user:alice", - props={"name": "Alice"}, - dense_vector=alice_embedding, - sparse_vector=alice_sparse) - - project = db.upsert_node(PROJECT, "project:overgraph", - dense_vector=project_embedding, - sparse_vector=project_sparse) - - db.upsert_edge(alice, project, CREATED) - - # Hybrid vector search scoped to a graph neighborhood - hits = db.vector_search("hybrid", k=10, - dense_query=query_embedding, - sparse_query=query_sparse, - scope_start_node_id=alice, - scope_max_depth=3) - - for hit in hits: - print(f"node {hit.node_id} score {hit.score:.4f}") -``` - -### Node.js - ```javascript import { OverGraph } from 'overgraph'; -const USER = 1; -const PROJECT = 2; -const CREATED = 10; - const db = OverGraph.open('./my-graph', { denseVector: { dimension: 384 }, }); -// Embeddings come from your model (sentence-transformers, OpenAI, etc.) -// dense: model.encode("Alice is an engineer") -> Float32Array(384) -// sparse: splade.encode("Alice is an engineer") -> [{ dimension, value }, ...] -const alice = db.upsertNode(USER, 'user:alice', { +// Embeddings come from your model. Dense vectors must match the configured dimension. +// Sparse vectors use { dimension, value } entries from your sparse encoder. +// Also accepts multiple labels: ['User', 'Engineer'] +const alice = db.upsertNode('User', 'alice', { props: { name: 'Alice' }, denseVector: aliceEmbedding, sparseVector: aliceSparse, }); -const project = db.upsertNode(PROJECT, 'project:overgraph', { +const project = db.upsertNode('Project', 'overgraph', { denseVector: projectEmbedding, sparseVector: projectSparse, }); -db.upsertEdge(alice, project, CREATED); +db.upsertEdge(alice, project, 'CREATED'); // Hybrid vector search scoped to a graph neighborhood const hits = db.vectorSearch('hybrid', { @@ -156,126 +104,56 @@ hits.forEach(h => console.log(`node ${h.nodeId} score ${h.score.toFixed(4)}`)); db.close(); ``` -### Rust - -```rust -use overgraph::*; -use std::collections::BTreeMap; -use std::path::Path; - -const USER: u32 = 1; -const PROJECT: u32 = 2; -const CREATED: u32 = 10; - -fn main() -> Result<(), Box> { - let opts = DbOptions { - dense_vector: Some(DenseVectorConfig { - dimension: 384, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }), - ..Default::default() - }; - let mut db = DatabaseEngine::open(Path::new("./my-graph"), &opts)?; - - // Embeddings come from your model (sentence-transformers, OpenAI, etc.) - // dense: model.encode("Alice is an engineer") -> Vec with 384 dims - // sparse: splade.encode("Alice is an engineer") -> Vec<(u32, f32)> - let mut props = BTreeMap::new(); - props.insert("name".into(), PropValue::String("Alice".into())); - let alice = db.upsert_node(USER, "user:alice", UpsertNodeOptions { - props, - dense_vector: Some(alice_embedding), - sparse_vector: Some(alice_sparse), - ..Default::default() - })?; - - let project = db.upsert_node(PROJECT, "project:overgraph", UpsertNodeOptions { - dense_vector: Some(project_embedding), - sparse_vector: Some(project_sparse), - ..Default::default() - })?; - - db.upsert_edge(alice, project, CREATED, UpsertEdgeOptions::default())?; - - // Hybrid vector search: dense + sparse with graph scoping - let hits = db.vector_search(&VectorSearchRequest { - mode: VectorSearchMode::Hybrid, - dense_query: Some(query_embedding), - sparse_query: Some(query_sparse), - k: 10, // required: 0 returns empty - type_filter: Some(vec![USER, PROJECT]), // default: None (no filtering) - ef_search: Some(200), // default: 128 - scope: Some(VectorSearchScope { // default: None (search all nodes) - start_node_id: alice, - max_depth: 3, - direction: Direction::Outgoing, // default: Outgoing - edge_type_filter: Some(vec![CREATED]), // default: None (all edge types) - at_epoch: None, // default: None (current time) - }), - dense_weight: Some(0.7), // default: 1.0 - sparse_weight: Some(0.3), // default: 1.0 - fusion_mode: Some(FusionMode::ReciprocalRankFusion), // default: WeightedRankFusion - })?; - for hit in &hits { - println!("node {} score {:.4}", hit.node_id, hit.score); - } - - db.close()?; - Ok(()) -} -``` - ### Async support -Both Python and Node.js connectors include full async variants of every API. Python provides `AsyncOverGraph` with native `asyncio` support. Node.js methods have `Async` suffixed variants (e.g. `upsertNodeAsync`, `vectorSearchAsync`). +The Node.js connector includes `Async` suffixed variants for every API, such as `upsertNodeAsync`, `queryNodesAsync`, and `vectorSearchAsync`. ## Features ### Vector search - **Dense vector search.** Attach `f32` embedding vectors to any node. HNSW indexes are built per segment at flush time for fast approximate nearest neighbor search. Supports cosine, Euclidean, and dot-product distance metrics. One dense vector space per DB with configurable dimension. - **Sparse vector search.** Attach sparse vectors (dimension-value pairs) for keyword-weighted retrieval. Works with pre-computed sparse embeddings from models like SPLADE or BGE-M3. Inverted posting-list indexes for exact dot-product scoring. -- **Hybrid search.** Combine dense and sparse results with built-in fusion modes: weighted rank fusion, reciprocal rank fusion, or weighted score fusion. Adjustable `dense_weight` and `sparse_weight` for tuning the blend. -- **Graph-scoped search.** Scope vector search to a graph neighborhood: "find the 10 most similar nodes within 3 hops of node X." Uses traversal-based reachable-node filtering with edge-type and temporal support. Combine graph structure with vector similarity in a single query. -- **Zero overhead when unused.** Nodes without vectors pay no storage or runtime cost. Vector index files are only created for segments that contain vectors. +- **Hybrid search.** Combine dense and sparse results with built-in fusion modes: weighted rank fusion, reciprocal rank fusion, or weighted score fusion. Adjustable `denseWeight` and `sparseWeight` for tuning the blend. +- **Graph-scoped search.** Scope vector search to a graph neighborhood: "find the 10 most similar nodes within 3 hops of node X." Uses traversal-based reachable-node filtering with edge-label and temporal support. Combine graph structure with vector similarity in a single query. +- **Zero overhead when unused.** Nodes without vectors pay no storage or runtime cost. Vector source payloads and accelerators are only created for segments that contain vectors. ### Core graph operations -- **Upsert semantics.** Nodes are keyed by `(type_id, key)`. Upsert the same key twice and you get an update, not a duplicate. Edges can optionally enforce uniqueness on `(from, to, type_id)`. -- **Batch operations.** `batch_upsert_nodes` and `batch_upsert_edges` amortize WAL and memtable overhead. `get_nodes` and `get_nodes_by_keys` do batched reads with sorted merge-walks instead of per-item lookups. There's also a packed binary format for maximum write throughput. -- **Atomic graph patch.** `graph_patch` lets you upsert nodes, upsert edges, delete nodes, delete edges, and invalidate edges in a single atomic operation. -- **Explicit transactions.** `begin_write_txn()` / `beginWriteTxn()` gives you ordered staging, rollback, read-own-writes point lookups, local aliases, atomic commit, and clean conflict errors for retry loops. +- **Upsert semantics.** Nodes carry one or more labels and one key. Each live `(label, key)` membership is unique, so a multi-label node owns the same key in every label it carries. Node upserts accept a single label string or an array of labels, and node records return `labels`. Edges can optionally enforce uniqueness on `(from, to, label)`. +- **Batch operations.** `batchUpsertNodes` and `batchUpsertEdges` amortize WAL and memtable overhead. `getNodes` and `getNodesByKeys` do batched reads with sorted merge-walks instead of per-item lookups. There's also a packed binary format for maximum write throughput. +- **Atomic graph patch.** `graphPatch` lets you upsert nodes, upsert edges, delete nodes, delete edges, and invalidate edges in a single atomic operation. +- **Explicit transactions.** `beginWriteTxn()` gives you ordered staging, rollback, read-own-writes point lookups, local aliases, atomic commit, and clean conflict errors for retry loops. ### Temporal edges -- **Validity windows.** Edges have optional `valid_from` and `valid_to` timestamps. Query at any point in time with the `at_epoch` parameter and only see edges that were valid at that moment. +- **Validity windows.** Edges have optional `validFrom` and `validTo` timestamps. Query at any point in time with the `atEpoch` parameter and only see edges that were valid at that moment. - **Edge invalidation.** Mark an edge as no longer valid without deleting it. The history is preserved. -- **Decay scoring.** Pass a `decay_lambda` to neighbor queries and edge weights are automatically scaled by `exp(-lambda * age_hours)`. Recent connections matter more. +- **Decay scoring.** Pass `decayLambda` to neighbor queries and edge weights are automatically scaled by `exp(-lambda * age_hours)`. Recent connections matter more. ### Queries and traversal -- **Neighbors and bounded traversal.** `neighbors()` handles 1-hop expansion and returns normal neighbor entry collections in every connector; `traverse()` covers deterministic breadth-first traversal across arbitrary depth windows with optional edge-type filtering, emission-only node-type filtering, and traversal-specific pagination. -- **Depth slices without special-case APIs.** Exact depth-2 traversals are expressed as `traverse(start, 2, min_depth=2)`, so 2-hop use cases stay available without a separate public method family. +- **Neighbors and bounded traversal.** `neighbors()` handles 1-hop expansion and returns normal neighbor entry collections in the Node.js API; `traverse()` covers deterministic breadth-first traversal across arbitrary depth windows with optional edge-label filtering, emission-only node-label filtering, and traversal-specific pagination. +- **Depth slices without special-case APIs.** Exact depth-2 traversals are expressed as `traverse(start, 2, { minDepth: 2 })`, so 2-hop use cases stay available without a separate public method family. - **Top-K neighbors.** Get the K highest-scoring neighbors by weight, recency, or decay-adjusted score. -- **Personalized PageRank.** Run PPR from seed nodes to find the most relevant nodes in the graph. Rust, Node.js, and Python expose both exact power-iteration PPR and a much faster approximate forward-push mode for seed-centric retrieval workloads. +- **Personalized PageRank.** Run PPR from seed nodes with `personalizedPagerank()`, using either exact power iteration or the faster approximate forward-push mode for seed-centric retrieval workloads. - **Subgraph extraction.** Pull out a connected subgraph up to N hops deep. Good for building local context windows. -- **Shortest path.** BFS (unweighted) or bidirectional Dijkstra (weighted). `is_connected` for fast reachability checks. `all_shortest_paths` when there are ties. -- **Connected components.** `connected_components()` returns a global WCC labelling (union-find, near-linear). `component_of(node)` returns the members of a single node's component via BFS. Both support edge-type, node-type, and temporal filters. +- **Shortest path.** BFS (unweighted) or bidirectional Dijkstra (weighted). `isConnected` for fast reachability checks. `allShortestPaths` when there are ties. +- **Connected components.** `connectedComponents()` returns a global WCC labelling (union-find, near-linear). `componentOf(node)` returns the members of a single node's component via BFS. Both support edge-label, node-label, and temporal filters. - **Degree counts.** Count edges, sum weights, and compute averages without materializing neighbor lists. Batch `degrees` for bulk analysis. - **Direct property queries.** `findNodes` and `findNodesPaged` do focused equality lookups. `findNodesRange` and `findNodesRangePaged` do numeric range scans with exact bound and cursor semantics. -- **Optional property indexes.** Declare equality or numeric range indexes only where they pay off. Use `ensureNodePropertyIndex`, `listNodePropertyIndexes`, and `dropNodePropertyIndex` to manage them. Public query APIs stay index-transparent: when a matching declaration is `Ready`, OverGraph uses the declaration-backed path; otherwise it falls back to the same public API. -- **Full query APIs.** `queryNodeIds`, `queryNodes`, `queryPattern`, and explain APIs combine IDs, keys, types, property equality/IN/range/exists/missing filters, updated-at ranges, and bounded graph patterns without a query string. OverGraph chooses the cheapest legal path with available indexes and planner stats, then verifies results against visible records. +- **Optional property indexes.** Declare node or edge equality/range indexes only where they pay off. Use `ensureNodePropertyIndex` / `ensureEdgePropertyIndex`, list APIs, and drop APIs to manage them. Public query APIs stay index-transparent: when a matching declaration is `Ready`, OverGraph uses the declaration-backed path; otherwise it falls back to the same public API. +- **Full query APIs.** `queryNodeIds`, `queryNodes`, `queryEdgeIds`, `queryEdges`, `queryPattern`, and explain APIs combine IDs, keys, node label filters (`{ labels, mode: 'any' | 'all' }`), edge labels, endpoint constraints, property equality/IN/range/exists/missing filters, edge metadata filters, updated-at ranges, and bounded graph patterns without a query string. OverGraph chooses the cheapest legal path with available indexes and planner stats, then verifies results against visible records. - **Time-range queries.** Find nodes created or updated within a time window. Sorted timestamp index for efficient range scans. ### Pagination ID-keyed collection APIs use keyset pagination with `limit` and `after`. `traverse()` uses `limit` plus a traversal cursor keyed by `(depth, node_id)`. No offset-based pagination. Traversal cursors assume the same query arguments and a stable logical graph state; strict snapshot isolation across intervening writes is not promised. ### Retention and pruning -- **Manual prune.** Drop nodes older than X, below weight Y, or matching type Z. Incident edges cascade automatically. +- **Manual prune.** Drop nodes older than X, below weight Y, or matching a label. Incident edges cascade automatically. - **Named prune policies.** Register policies like `"short_term_memory"` that run automatically during compaction. Nodes matching any policy are invisible to reads immediately (lazy expiration) and cleaned up during the next compaction pass. ### Storage engine - **Write-ahead log.** Every mutation hits the WAL before the memtable. Crash recovery replays the WAL on startup. - **Configurable durability.** `Immediate` mode fsyncs every write for maximum safety. `GroupCommit` mode (default) batches fsyncs on a 50ms timer for ~20x better write throughput with at most one timer interval of data at risk. -- **Background compaction.** Segments are merged automatically when thresholds are met. Compaction runs on a background thread and never blocks reads or writes. Uses metadata sidecars for fast filtered merging without full record decoding. -- **Bulk ingest mode.** Temporarily disable auto-compaction during large write bursts with `ingest_mode()`, then call `end_ingest()` to compact accumulated segments and restore normal behavior. This favors ingest throughput over read performance during the ingest window. +- **Background compaction.** Segments are merged automatically when thresholds are met. Compaction runs on a background thread and never blocks reads or writes. Uses packed metadata payloads for fast filtered merging without full record decoding. +- **Bulk ingest mode.** Temporarily disable auto-compaction during large write bursts with `ingestMode()`, then call `endIngest()` to compact accumulated segments and restore normal behavior. This favors ingest throughput over read performance during the ingest window. - **mmap'd reads.** Immutable segments are memory-mapped. The OS page cache handles caching. Reads never block writes. - **Portable databases.** Each database is a self-contained directory. `cp -r ./my-db /backup/my-db` and you're done. @@ -285,77 +163,59 @@ OverGraph uses a log-structured storage engine purpose-built from scratch in pur **Write path:** Mutations are appended to a write-ahead log and applied to an in-memory memtable. When the memtable reaches its threshold, it's frozen and flushed to disk as an immutable segment in the background. Writes continue unblocked against a fresh memtable. Each segment ships with pre-built adjacency indexes (inbound and outbound), optional declared property-index sidecars, optional advisory planner statistics, optional signed degree-delta sidecars for degree/weight fast paths, and, when the segment contains vectors, HNSW and sparse posting-list indexes. -**Read path:** Queries check the memtable first (freshest data), then merge results across immutable segments using the per-segment indexes. Because every segment carries its own adjacency index, a neighbor query is a handful of index lookups, not a scan across sorted keys. Vector search follows the same model: memtable candidates are found by exact brute-force scan, segment candidates via HNSW or posting-list indexes, then the engine merges and deduplicates across all sources. Property equality and numeric range queries stay index-transparent too: if a matching optional property-index declaration is `Ready`, the engine uses the declaration-backed path, otherwise it falls back to a type-scoped scan through the same public API. Pagination uses early termination to avoid unnecessary work. +**Read path:** Queries check the memtable first (freshest data), then merge results across immutable segments using the per-segment indexes. Because every segment carries its own adjacency index, a neighbor query is a handful of index lookups, not a scan across sorted keys. Vector search follows the same model: memtable candidates are found by exact brute-force scan, segment candidates via HNSW or posting-list indexes, then the engine merges and deduplicates across all sources. Property equality and numeric range queries stay index-transparent too: if a matching optional property-index declaration is `Ready`, the engine uses the declaration-backed path, otherwise it falls back to a label-scoped scan through the same public API. Pagination uses early termination to avoid unnecessary work. -**Compaction:** A background thread merges older segments together, applying tombstones, prune policies, and deduplication. The compaction path uses metadata sidecars to plan merges and raw-copies winning records without full deserialization, then rebuilds unified indexes from metadata. This includes rebuilding HNSW and sparse posting-list indexes for the merged output. Fewer segments after compaction means fewer index lookups per query, but even before compaction, reads are fast because every segment is self-indexed. +**Compaction:** A background thread merges older segments together, applying tombstones, prune policies, and deduplication. The compaction path uses packed metadata payloads to plan merges and raw-copies winning records without full deserialization, then rebuilds unified indexes from metadata. This includes rebuilding HNSW and sparse posting-list indexes for the merged output. Fewer segments after compaction means fewer index lookups per query, but even before compaction, reads are fast because every segment is self-indexed. **On-disk layout:** ``` my-graph/ manifest.current # atomic checkpoint (JSON) - data.wal # append-only write-ahead log + wal_0.wal # append-only write-ahead log generation segments/ seg_0001/ - nodes.dat # node records - edges.dat # edge records - adj_out.idx # outgoing adjacency index - adj_in.idx # incoming adjacency index - key_index.dat # (type_id, key) -> node_id - type_index.dat # type_id -> [id...] - tombstones.dat # deleted IDs - secondary_indexes/ # optional declared property-index sidecars - planner_stats.dat # optional advisory planner statistics + segment_manifest.dat # component table of contents + segment.core # packed immutable core records, metadata, and maintained indexes + secondary_indexes/ # optional declared equality/range property-index sidecars + planner_stats.dat # optional advisory planner statistics, refreshable degree_delta.dat # optional signed degree deltas for fast degree/weight reads - node_dense_vectors.dat # dense vector blob (when present) - node_sparse_vectors.dat # sparse vector blob (when present) - dense_hnsw_graph.dat # HNSW graph index (when present) - sparse_postings.dat # sparse posting lists (when present) - node_vector_meta.dat # vector offsets/lengths per node + dense_hnsw_meta.dat # optional dense-vector HNSW metadata + dense_hnsw_graph.dat # optional dense-vector HNSW graph + sparse_posting_index.dat # optional sparse-vector posting index + sparse_postings.dat # optional sparse-vector posting lists seg_0002/ ... ``` -For a deeper dive, see the [architecture overview](docs/architecture-overview.md). +`segment.core` is addressed through `segment_manifest.dat`. It contains the logical +node/edge record payloads, tombstones, key/label-token/timestamp/triple indexes, adjacency +indexes/postings, node/edge metadata, vector source-truth blobs, and immutable edge +metadata indexes. Refreshable optional accelerators stay outside the packed core so +they can be rebuilt or dropped without rewriting source data. + +For a deeper dive, see the [architecture overview](../docs/architecture-overview.md). ## Documentation - **[overgraph.io/docs](https://overgraph.io/docs)** - full documentation, getting started guide, and API reference. -- **[API Reference](../docs/api-reference.md)** - every method, parameter, type, and return value across Python, Node.js, and Rust. +- **[API Reference](../docs/api-reference.md)** - every Node.js method, parameter, type, and return value. - **[Roadmap](../docs/roadmap.md)** - where OverGraph is headed and what's already shipped. ## Running the benchmarks ```bash -# Rust -scripts/bench/run-rust.sh --profile small --warmup 20 --iters 80 - -# Node.js scripts/bench/run-node.sh --profile small --warmup 20 --iters 80 - -# Python -scripts/bench/run-python.sh --profile small --warmup 20 --iters 80 ``` -Benchmark methodology, FAQ, and reproducibility instructions are in [`docs/04-quality/`](docs/04-quality/). +Benchmark methodology, FAQ, and reproducibility instructions are in [`docs/04-quality/`](../docs/04-quality/). ## Building from source ```bash -# Rust core -cargo build --release -cargo test - -# Node.js connector cd overgraph-node npm install npm run build npm test - -# Python connector -cd overgraph-python -pip install maturin -maturin develop -pytest ``` ## Contributing diff --git a/overgraph-node/__test__/agent-api.mjs b/overgraph-node/__test__/agent-api.mjs index 8cc813e..fb54c29 100644 --- a/overgraph-node/__test__/agent-api.mjs +++ b/overgraph-node/__test__/agent-api.mjs @@ -16,34 +16,34 @@ describe('getNodeByKey', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns node by type+key', () => { - const id = db.upsertNode(1, 'alice', { props: { name: 'Alice' }, weight: 1.0 }); - const node = db.getNodeByKey(1, 'alice'); + const id = db.upsertNode('Person', 'alice', { props: { name: 'Alice' }, weight: 1.0 }); + const node = db.getNodeByKey('Person', 'alice'); assert.ok(node); assert.equal(node.id, id); - assert.equal(node.typeId, 1); + assert.deepEqual(node.labels, ['Person']); assert.equal(node.key, 'alice'); assert.equal(node.props.name, 'Alice'); }); it('returns null for missing key', () => { - assert.equal(db.getNodeByKey(1, 'nonexistent'), null); + assert.equal(db.getNodeByKey('Person', 'nonexistent'), null); }); - it('returns null for wrong type_id', () => { - db.upsertNode(5, 'typed', { props: {} }); - assert.equal(db.getNodeByKey(99, 'typed'), null); + it('returns null for wrong label', () => { + db.upsertNode('User', 'typed', { props: {} }); + assert.equal(db.getNodeByKey('MissingLabel', 'typed'), null); }); it('returns null for deleted node', () => { - const id = db.upsertNode(1, 'to-delete', { props: {} }); + const id = db.upsertNode('Person', 'to-delete', { props: {} }); db.deleteNode(id); - assert.equal(db.getNodeByKey(1, 'to-delete'), null); + assert.equal(db.getNodeByKey('Person', 'to-delete'), null); }); it('works after flush (segment source)', () => { - const id = db.upsertNode(1, 'flushed', { props: { v: 1 } }); + const id = db.upsertNode('Person', 'flushed', { props: { v: 1 } }); db.flush(); - const node = db.getNodeByKey(1, 'flushed'); + const node = db.getNodeByKey('Person', 'flushed'); assert.ok(node); assert.equal(node.id, id); }); @@ -58,46 +58,46 @@ describe('getEdgeByTriple', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns edge by from+to+type', () => { - const a = db.upsertNode(1, 'a', { props: {} }); - const b = db.upsertNode(1, 'b', { props: {} }); - const eid = db.upsertEdge(a, b, 10, { props: { rel: 'knows' }, weight: 0.5 }); - const edge = db.getEdgeByTriple(a, b, 10); + const a = db.upsertNode('Person', 'a', { props: {} }); + const b = db.upsertNode('Person', 'b', { props: {} }); + const eid = db.upsertEdge(a, b, 'WORKS_AT', { props: { rel: 'knows' }, weight: 0.5 }); + const edge = db.getEdgeByTriple(a, b, 'WORKS_AT'); assert.ok(edge); assert.equal(edge.id, eid); assert.equal(edge.from, a); assert.equal(edge.to, b); - assert.equal(edge.typeId, 10); + assert.equal(edge.label, 'WORKS_AT'); assert.equal(edge.props.rel, 'knows'); }); - it('returns null for wrong type_id', () => { - const a = db.upsertNode(1, 'c', { props: {} }); - const b = db.upsertNode(1, 'd', { props: {} }); - db.upsertEdge(a, b, 10, { props: {} }); - assert.equal(db.getEdgeByTriple(a, b, 99), null); + it('returns null for wrong edge label', () => { + const a = db.upsertNode('Person', 'c', { props: {} }); + const b = db.upsertNode('Person', 'd', { props: {} }); + db.upsertEdge(a, b, 'WORKS_AT', { props: {} }); + assert.equal(db.getEdgeByTriple(a, b, 'MISSING_EDGE_TYPE'), null); }); it('returns null for reversed direction', () => { - const a = db.upsertNode(1, 'e', { props: {} }); - const b = db.upsertNode(1, 'f', { props: {} }); - db.upsertEdge(a, b, 10, { props: {} }); - assert.equal(db.getEdgeByTriple(b, a, 10), null); + const a = db.upsertNode('Person', 'e', { props: {} }); + const b = db.upsertNode('Person', 'f', { props: {} }); + db.upsertEdge(a, b, 'WORKS_AT', { props: {} }); + assert.equal(db.getEdgeByTriple(b, a, 'WORKS_AT'), null); }); it('returns null after delete', () => { - const a = db.upsertNode(1, 'g', { props: {} }); - const b = db.upsertNode(1, 'h', { props: {} }); - const eid = db.upsertEdge(a, b, 10, { props: {} }); + const a = db.upsertNode('Person', 'g', { props: {} }); + const b = db.upsertNode('Person', 'h', { props: {} }); + const eid = db.upsertEdge(a, b, 'WORKS_AT', { props: {} }); db.deleteEdge(eid); - assert.equal(db.getEdgeByTriple(a, b, 10), null); + assert.equal(db.getEdgeByTriple(a, b, 'WORKS_AT'), null); }); it('works after flush', () => { - const a = db.upsertNode(1, 'i', { props: {} }); - const b = db.upsertNode(1, 'j', { props: {} }); - const eid = db.upsertEdge(a, b, 20, { props: {} }); + const a = db.upsertNode('Person', 'i', { props: {} }); + const b = db.upsertNode('Person', 'j', { props: {} }); + const eid = db.upsertEdge(a, b, 'MENTIONS', { props: {} }); db.flush(); - const edge = db.getEdgeByTriple(a, b, 20); + const edge = db.getEdgeByTriple(a, b, 'MENTIONS'); assert.ok(edge); assert.equal(edge.id, eid); }); @@ -112,11 +112,11 @@ describe('extractSubgraph', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('extracts a simple chain', () => { - const a = db.upsertNode(1, 'sg-a', { props: { name: 'A' } }); - const b = db.upsertNode(1, 'sg-b', { props: { name: 'B' } }); - const c = db.upsertNode(1, 'sg-c', { props: { name: 'C' } }); - db.upsertEdge(a, b, 1); - db.upsertEdge(b, c, 1); + const a = db.upsertNode('Person', 'sg-a', { props: { name: 'A' } }); + const b = db.upsertNode('Person', 'sg-b', { props: { name: 'B' } }); + const c = db.upsertNode('Person', 'sg-c', { props: { name: 'C' } }); + db.upsertEdge(a, b, 'LINKS_TO'); + db.upsertEdge(b, c, 'LINKS_TO'); const sg = db.extractSubgraph(a, 2); assert.ok(sg); @@ -125,7 +125,7 @@ describe('extractSubgraph', () => { }); it('respects maxDepth', () => { - const sg = db.extractSubgraph(db.upsertNode(1, 'sg-a', { props: {} }), 0); + const sg = db.extractSubgraph(db.upsertNode('Person', 'sg-a', { props: {} }), 0); assert.equal(sg.nodes.length, 1); assert.equal(sg.edges.length, 0); }); @@ -140,9 +140,9 @@ describe('getNodes / getEdges (bulk)', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('getNodes returns array matching input order', () => { - const a = db.upsertNode(1, 'bulk-a', { props: { n: 'A' } }); - const b = db.upsertNode(1, 'bulk-b', { props: { n: 'B' } }); - const c = db.upsertNode(1, 'bulk-c', { props: { n: 'C' } }); + const a = db.upsertNode('Person', 'bulk-a', { props: { n: 'A' } }); + const b = db.upsertNode('Person', 'bulk-b', { props: { n: 'B' } }); + const c = db.upsertNode('Person', 'bulk-c', { props: { n: 'C' } }); const results = db.getNodes([a, b, c]); assert.equal(results.length, 3); assert.equal(results[0].key, 'bulk-a'); @@ -151,8 +151,8 @@ describe('getNodes / getEdges (bulk)', () => { }); it('getNodes returns null for missing/deleted', () => { - const a = db.upsertNode(1, 'bulk-d', { props: {} }); - const b = db.upsertNode(1, 'bulk-e', { props: {} }); + const a = db.upsertNode('Person', 'bulk-d', { props: {} }); + const b = db.upsertNode('Person', 'bulk-e', { props: {} }); db.deleteNode(b); const results = db.getNodes([a, b, 99999]); assert.equal(results.length, 3); @@ -167,11 +167,11 @@ describe('getNodes / getEdges (bulk)', () => { }); it('getEdges returns array matching input order', () => { - const a = db.upsertNode(1, 'bulk-ea', { props: {} }); - const b = db.upsertNode(1, 'bulk-eb', { props: {} }); - const c = db.upsertNode(1, 'bulk-ec', { props: {} }); - const e1 = db.upsertEdge(a, b, 1); - const e2 = db.upsertEdge(b, c, 1); + const a = db.upsertNode('Person', 'bulk-ea', { props: {} }); + const b = db.upsertNode('Person', 'bulk-eb', { props: {} }); + const c = db.upsertNode('Person', 'bulk-ec', { props: {} }); + const e1 = db.upsertEdge(a, b, 'LINKS_TO'); + const e2 = db.upsertEdge(b, c, 'LINKS_TO'); const results = db.getEdges([e1, e2, 99999]); assert.equal(results.length, 3); assert.equal(results[0].from, a); @@ -180,9 +180,9 @@ describe('getNodes / getEdges (bulk)', () => { }); it('getNodes works cross-source (memtable + segment)', () => { - const a = db.upsertNode(1, 'bulk-xa', { props: {} }); + const a = db.upsertNode('Person', 'bulk-xa', { props: {} }); db.flush(); - const b = db.upsertNode(1, 'bulk-xb', { props: {} }); + const b = db.upsertNode('Person', 'bulk-xb', { props: {} }); const results = db.getNodes([a, b]); assert.equal(results[0].key, 'bulk-xa'); assert.equal(results[1].key, 'bulk-xb'); @@ -198,13 +198,13 @@ describe('getNodesByKeys (bulk key lookup)', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns matching nodes in input order', () => { - db.upsertNode(1, 'ka', { props: { n: 'A' } }); - db.upsertNode(1, 'kb', { props: { n: 'B' } }); - db.upsertNode(2, 'kc', { props: { n: 'C' } }); + db.upsertNode('Person', 'ka', { props: { n: 'A' } }); + db.upsertNode('Person', 'kb', { props: { n: 'B' } }); + db.upsertNode('Company', 'kc', { props: { n: 'C' } }); const results = db.getNodesByKeys([ - { typeId: 1, key: 'ka' }, - { typeId: 1, key: 'kb' }, - { typeId: 2, key: 'kc' }, + { label: 'Person', key: 'ka' }, + { label: 'Person', key: 'kb' }, + { label: 'Company', key: 'kc' }, ]); assert.equal(results.length, 3); assert.equal(results[0].key, 'ka'); @@ -213,13 +213,13 @@ describe('getNodesByKeys (bulk key lookup)', () => { }); it('returns null for missing and deleted keys', () => { - db.upsertNode(1, 'kd'); - const bid = db.upsertNode(1, 'ke'); + db.upsertNode('Person', 'kd'); + const bid = db.upsertNode('Person', 'ke'); db.deleteNode(bid); const results = db.getNodesByKeys([ - { typeId: 1, key: 'kd' }, - { typeId: 1, key: 'ke' }, - { typeId: 1, key: 'nonexistent' }, + { label: 'Person', key: 'kd' }, + { label: 'Person', key: 'ke' }, + { label: 'Person', key: 'nonexistent' }, ]); assert.equal(results.length, 3); assert.ok(results[0]); @@ -233,8 +233,8 @@ describe('getNodesByKeys (bulk key lookup)', () => { }); it('async variant works', async () => { - db.upsertNode(1, 'kf'); - const results = await db.getNodesByKeysAsync([{ typeId: 1, key: 'kf' }]); + db.upsertNode('Person', 'kf'); + const results = await db.getNodesByKeysAsync([{ label: 'Person', key: 'kf' }]); assert.equal(results.length, 1); assert.equal(results[0].key, 'kf'); }); @@ -249,42 +249,42 @@ describe('async variants of new APIs', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('getNodeByKeyAsync works', async () => { - const id = db.upsertNode(1, 'async-key', { props: { x: 1 } }); - const node = await db.getNodeByKeyAsync(1, 'async-key'); + const id = db.upsertNode('Person', 'async-key', { props: { x: 1 } }); + const node = await db.getNodeByKeyAsync('Person', 'async-key'); assert.ok(node); assert.equal(node.id, id); }); it('getEdgeByTripleAsync works', async () => { - const a = db.upsertNode(1, 'at-a', { props: {} }); - const b = db.upsertNode(1, 'at-b', { props: {} }); - const eid = db.upsertEdge(a, b, 5); - const edge = await db.getEdgeByTripleAsync(a, b, 5); + const a = db.upsertNode('Person', 'at-a', { props: {} }); + const b = db.upsertNode('Person', 'at-b', { props: {} }); + const eid = db.upsertEdge(a, b, 'DEPENDS_ON'); + const edge = await db.getEdgeByTripleAsync(a, b, 'DEPENDS_ON'); assert.ok(edge); assert.equal(edge.id, eid); }); it('getNodesAsync works', async () => { - const a = db.upsertNode(1, 'an-a', { props: {} }); - const b = db.upsertNode(1, 'an-b', { props: {} }); + const a = db.upsertNode('Person', 'an-a', { props: {} }); + const b = db.upsertNode('Person', 'an-b', { props: {} }); const results = await db.getNodesAsync([a, b]); assert.equal(results.length, 2); assert.equal(results[0].key, 'an-a'); }); it('getEdgesAsync works', async () => { - const a = db.upsertNode(1, 'ae-a', { props: {} }); - const b = db.upsertNode(1, 'ae-b', { props: {} }); - const e = db.upsertEdge(a, b, 1); + const a = db.upsertNode('Person', 'ae-a', { props: {} }); + const b = db.upsertNode('Person', 'ae-b', { props: {} }); + const e = db.upsertEdge(a, b, 'LINKS_TO'); const results = await db.getEdgesAsync([e]); assert.equal(results.length, 1); assert.equal(results[0].from, a); }); it('extractSubgraphAsync works', async () => { - const a = db.upsertNode(1, 'esg-a', { props: {} }); - const b = db.upsertNode(1, 'esg-b', { props: {} }); - db.upsertEdge(a, b, 1); + const a = db.upsertNode('Person', 'esg-a', { props: {} }); + const b = db.upsertNode('Person', 'esg-b', { props: {} }); + db.upsertEdge(a, b, 'LINKS_TO'); const sg = await db.extractSubgraphAsync(a, 1); assert.ok(sg); assert.ok(sg.nodes.length >= 2); @@ -303,13 +303,13 @@ describe('graphPatch', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('mixed ops in a single call', () => { - const a = db.upsertNode(1, 'pa', { props: {} }); - const b = db.upsertNode(1, 'pb', { props: {} }); - const e1 = db.upsertEdge(a, b, 1, { props: { v: 'old' } }); + const a = db.upsertNode('Person', 'pa', { props: {} }); + const b = db.upsertNode('Person', 'pb', { props: {} }); + const e1 = db.upsertEdge(a, b, 'LINKS_TO', { props: { v: 'old' } }); const result = db.graphPatch({ - upsertNodes: [{ typeId: 1, key: 'pc', props: { role: 'new' } }], - upsertEdges: [{ from: a, to: b, typeId: 2 }], + upsertNodes: [{ labels: ['Person'], key: 'pc', props: { role: 'new' } }], + upsertEdges: [{ from: a, to: b, label: 'REFERENCES'}], invalidateEdges: [{ edgeId: e1, validTo: 1000 }], deleteEdgeIds: [], deleteNodeIds: [], @@ -326,7 +326,7 @@ describe('graphPatch', () => { // New edge created const e2 = db.getEdge(result.edgeIds[0]); assert.ok(e2); - assert.equal(e2.typeId, 2); + assert.equal(e2.label, 'REFERENCES'); // e1 invalidated const inv = db.getEdge(e1); @@ -340,10 +340,26 @@ describe('graphPatch', () => { assert.equal(result.edgeIds.length, 0); }); + it('uses NodeInput labels shape in patch node upserts', () => { + const result = db.graphPatch({ + upsertNodes: [ + { labels: 'Person', key: 'patch-string-label' }, + { labels: ['Person', 'Admin'], key: 'patch-multi-label' }, + ], + }); + + assert.deepEqual(db.getNode(result.nodeIds[0]).labels, ['Person']); + assert.deepEqual([...db.getNode(result.nodeIds[1]).labels].sort(), ['Admin', 'Person']); + assert.throws( + () => db.graphPatch({ upsertNodes: [{ labels: [], key: 'patch-empty-labels' }] }), + /node label set must contain at least one label/, + ); + }); + it('delete cascades edges', () => { - const x = db.upsertNode(1, 'dx', { props: {} }); - const y = db.upsertNode(1, 'dy', { props: {} }); - const e = db.upsertEdge(x, y, 1); + const x = db.upsertNode('Person', 'dx', { props: {} }); + const y = db.upsertNode('Person', 'dy', { props: {} }); + const e = db.upsertEdge(x, y, 'LINKS_TO'); db.graphPatch({ deleteNodeIds: [x] }); @@ -353,12 +369,12 @@ describe('graphPatch', () => { }); it('deduplicates node upserts within patch', () => { - const existing = db.upsertNode(1, 'dup-node', { props: { v: '1' } }); + const existing = db.upsertNode('Person', 'dup-node', { props: { v: '1' } }); const result = db.graphPatch({ upsertNodes: [ - { typeId: 1, key: 'dup-node', props: { v: '2' } }, - { typeId: 1, key: 'dup-node', props: { v: '3' } }, + { labels: ['Person'], key: 'dup-node', props: { v: '2' } }, + { labels: ['Person'], key: 'dup-node', props: { v: '3' } }, ], }); @@ -372,10 +388,10 @@ describe('graphPatch', () => { }); it('upsert then delete in same patch (delete wins)', () => { - const n = db.upsertNode(1, 'ud', { props: {} }); + const n = db.upsertNode('Person', 'ud', { props: {} }); db.graphPatch({ - upsertNodes: [{ typeId: 1, key: 'ud', props: { v: 'updated' } }], + upsertNodes: [{ labels: ['Person'], key: 'ud', props: { v: 'updated' } }], deleteNodeIds: [n], }); @@ -389,12 +405,12 @@ describe('graphPatch', () => { const dbPath = join(tmpDir2, 'db'); let db2 = OverGraph.open(dbPath, { walSyncMode: 'immediate' }); - const a = db2.upsertNode(1, 'wa', { props: {} }); - const b = db2.upsertNode(1, 'wb', { props: {} }); + const a = db2.upsertNode('Person', 'wa', { props: {} }); + const b = db2.upsertNode('Person', 'wb', { props: {} }); const result = db2.graphPatch({ - upsertNodes: [{ typeId: 1, key: 'wc', props: { role: 'new' } }], - upsertEdges: [{ from: a, to: b, typeId: 5 }], + upsertNodes: [{ labels: ['Person'], key: 'wc', props: { role: 'new' } }], + upsertEdges: [{ from: a, to: b, label: 'DEPENDS_ON'}], }); const nodeId = result.nodeIds[0]; const edgeId = result.edgeIds[0]; @@ -408,7 +424,7 @@ describe('graphPatch', () => { const edge = db2.getEdge(edgeId); assert.ok(edge); - assert.equal(edge.typeId, 5); + assert.equal(edge.label, 'DEPENDS_ON'); db2.close(); rmSync(tmpDir2, { recursive: true, force: true }); @@ -424,12 +440,12 @@ describe('graphPatchAsync', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('async patch works', async () => { - const a = db.upsertNode(1, 'ap-a', { props: {} }); - const b = db.upsertNode(1, 'ap-b', { props: {} }); + const a = db.upsertNode('Person', 'ap-a', { props: {} }); + const b = db.upsertNode('Person', 'ap-b', { props: {} }); const result = await db.graphPatchAsync({ - upsertNodes: [{ typeId: 1, key: 'ap-c' }], - upsertEdges: [{ from: a, to: b, typeId: 3 }], + upsertNodes: [{ labels: ['Person'], key: 'ap-c' }], + upsertEdges: [{ from: a, to: b, label: 'PATCH_EDGE'}], }); assert.equal(result.nodeIds.length, 1); @@ -450,13 +466,13 @@ describe('prune', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('empty policy is rejected', () => { - db.upsertNode(1, 'safe', { props: {}, weight: 1.0 }); + db.upsertNode('Person', 'safe', { props: {}, weight: 1.0 }); assert.throws(() => db.prune({}), /at least max_age_ms or max_weight/); }); it('prunes by weight', () => { - const low = db.upsertNode(1, 'pr-low', { props: {}, weight: 0.1 }); - const high = db.upsertNode(1, 'pr-high', { props: {}, weight: 0.9 }); + const low = db.upsertNode('Person', 'pr-low', { props: {}, weight: 0.1 }); + const high = db.upsertNode('Person', 'pr-high', { props: {}, weight: 0.9 }); const result = db.prune({ maxWeight: 0.5 }); assert.ok(result.nodesPruned >= 1); @@ -465,8 +481,8 @@ describe('prune', () => { }); it('prunes by weight, boundary (<=)', () => { - const exact = db.upsertNode(1, 'pr-exact', { props: {}, weight: 0.5 }); - const above = db.upsertNode(1, 'pr-above', { props: {}, weight: 0.500001 }); + const exact = db.upsertNode('Person', 'pr-exact', { props: {}, weight: 0.5 }); + const above = db.upsertNode('Person', 'pr-above', { props: {}, weight: 0.500001 }); db.prune({ maxWeight: 0.5 }); assert.equal(db.getNode(exact), null); @@ -474,9 +490,9 @@ describe('prune', () => { }); it('cascade deletes edges of pruned nodes', () => { - const a = db.upsertNode(1, 'pr-ca', { props: {}, weight: 0.1 }); - const b = db.upsertNode(1, 'pr-cb', { props: {}, weight: 0.9 }); - const e = db.upsertEdge(a, b, 1); + const a = db.upsertNode('Person', 'pr-ca', { props: {}, weight: 0.1 }); + const b = db.upsertNode('Person', 'pr-cb', { props: {}, weight: 0.9 }); + const e = db.upsertEdge(a, b, 'LINKS_TO'); const result = db.prune({ maxWeight: 0.5 }); assert.equal(db.getNode(a), null); @@ -486,25 +502,25 @@ describe('prune', () => { }); it('type-scoped prune', () => { - const t1 = db.upsertNode(10, 'pr-t1', { props: {}, weight: 0.1 }); - const t2 = db.upsertNode(20, 'pr-t2', { props: {}, weight: 0.1 }); + const t1 = db.upsertNode('PruneTarget', 'pr-t1', { props: {}, weight: 0.1 }); + const t2 = db.upsertNode('PruneOther', 'pr-t2', { props: {}, weight: 0.1 }); - db.prune({ maxWeight: 0.5, typeId: 10 }); + db.prune({ maxWeight: 0.5, label: 'PruneTarget'}); assert.equal(db.getNode(t1), null); assert.ok(db.getNode(t2)); // different type, survives }); it('no matches returns zero counts', () => { - db.upsertNode(1, 'pr-nomatch', { props: {}, weight: 0.9 }); + db.upsertNode('Person', 'pr-nomatch', { props: {}, weight: 0.9 }); const result = db.prune({ maxWeight: 0.01 }); assert.equal(result.nodesPruned, 0); assert.equal(result.edgesPruned, 0); }); it('works after flush (segment source)', () => { - const a = db.upsertNode(1, 'pr-seg', { props: {}, weight: 0.1 }); - const b = db.upsertNode(1, 'pr-seg-keep', { props: {}, weight: 0.9 }); - const e = db.upsertEdge(a, b, 1); + const a = db.upsertNode('Person', 'pr-seg', { props: {}, weight: 0.1 }); + const b = db.upsertNode('Person', 'pr-seg-keep', { props: {}, weight: 0.9 }); + const e = db.upsertEdge(a, b, 'LINKS_TO'); db.flush(); const result = db.prune({ maxWeight: 0.5 }); @@ -519,9 +535,9 @@ describe('prune', () => { const dbPath = join(tmpDir2, 'db'); let db2 = OverGraph.open(dbPath, { walSyncMode: 'immediate' }); - const a = db2.upsertNode(1, 'pr-wal-a', { props: {}, weight: 0.1 }); - const b = db2.upsertNode(1, 'pr-wal-b', { props: {}, weight: 0.9 }); - const e = db2.upsertEdge(a, b, 1); + const a = db2.upsertNode('Person', 'pr-wal-a', { props: {}, weight: 0.1 }); + const b = db2.upsertNode('Person', 'pr-wal-b', { props: {}, weight: 0.9 }); + const e = db2.upsertEdge(a, b, 'LINKS_TO'); db2.prune({ maxWeight: 0.5 }); db2.close(); @@ -546,8 +562,8 @@ describe('pruneAsync', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('async prune works', async () => { - const a = db.upsertNode(1, 'ap-low', { props: {}, weight: 0.1 }); - const b = db.upsertNode(1, 'ap-high', { props: {}, weight: 0.9 }); + const a = db.upsertNode('Person', 'ap-low', { props: {}, weight: 0.1 }); + const b = db.upsertNode('Person', 'ap-high', { props: {}, weight: 0.9 }); const result = await db.pruneAsync({ maxWeight: 0.5 }); assert.ok(result.nodesPruned >= 1); @@ -605,7 +621,7 @@ describe('prune policy survives close/reopen', () => { let db = OverGraph.open(dbPath, { walSyncMode: 'immediate' }); db.setPrunePolicy('age', { maxAgeMs: 30000 }); - db.setPrunePolicy('weight', { maxWeight: 0.1, typeId: 5 }); + db.setPrunePolicy('weight', { maxWeight: 0.1, label: 'User'}); db.close(); db = OverGraph.open(dbPath, { walSyncMode: 'immediate' }); @@ -616,7 +632,7 @@ describe('prune policy survives close/reopen', () => { assert.equal(list[0].policy.maxAgeMs, 30000); assert.equal(list[1].name, 'weight'); assert.ok(Math.abs(list[1].policy.maxWeight - 0.1) < 1e-6); - assert.equal(list[1].policy.typeId, 5); + assert.equal(list[1].policy.label, 'User'); db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -631,14 +647,14 @@ describe('compaction auto-prune', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('prunes matching nodes and cascade-drops edges during compaction', () => { - const a = db.upsertNode(1, 'a', { props: {}, weight: 0.1 }); // will be pruned - const b = db.upsertNode(1, 'b', { props: {}, weight: 0.9 }); - const c = db.upsertNode(1, 'c', { props: {}, weight: 0.9 }); - const e1 = db.upsertEdge(a, b, 1); - const e2 = db.upsertEdge(b, c, 1); + const a = db.upsertNode('Person', 'a', { props: {}, weight: 0.1 }); // will be pruned + const b = db.upsertNode('Person', 'b', { props: {}, weight: 0.9 }); + const c = db.upsertNode('Person', 'c', { props: {}, weight: 0.9 }); + const e1 = db.upsertEdge(a, b, 'LINKS_TO'); + const e2 = db.upsertEdge(b, c, 'LINKS_TO'); db.flush(); - db.upsertNode(1, 'b', { props: {}, weight: 0.9 }); // overlap → forces standard compaction path + db.upsertNode('Person', 'b', { props: {}, weight: 0.9 }); // overlap → forces standard compaction path db.flush(); db.setPrunePolicy('low', { maxWeight: 0.5 }); @@ -663,13 +679,13 @@ describe('compaction auto-prune type-scoped', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('only prunes the targeted type', () => { - const t1 = db.upsertNode(1, 't1-low', { props: {}, weight: 0.1 }); - const t2 = db.upsertNode(2, 't2-low', { props: {}, weight: 0.1 }); + const t1 = db.upsertNode('Person', 't1-low', { props: {}, weight: 0.1 }); + const t2 = db.upsertNode('Company', 't2-low', { props: {}, weight: 0.1 }); db.flush(); - db.upsertNode(1, 't1-low', { props: {}, weight: 0.1 }); // overlap → forces standard compaction path + db.upsertNode('Person', 't1-low', { props: {}, weight: 0.1 }); // overlap → forces standard compaction path db.flush(); - db.setPrunePolicy('type1', { maxWeight: 0.5, typeId: 1 }); + db.setPrunePolicy('type1', { maxWeight: 0.5, label: 'Person'}); const stats = db.compact(); assert.equal(stats.nodesAutoPruned, 1); @@ -711,8 +727,8 @@ describe('read-time policy filtering, getNode', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('policy-excluded node returns null from getNode', () => { - const lo = db.upsertNode(1, 'lo', { props: {}, weight: 0.2 }); - const hi = db.upsertNode(1, 'hi', { props: {}, weight: 0.9 }); + const lo = db.upsertNode('Person', 'lo', { props: {}, weight: 0.2 }); + const hi = db.upsertNode('Person', 'hi', { props: {}, weight: 0.9 }); // Before policy: both visible assert.ok(db.getNode(lo)); @@ -725,8 +741,8 @@ describe('read-time policy filtering, getNode', () => { assert.ok(db.getNode(hi)); // getNodeByKey also filtered - assert.equal(db.getNodeByKey(1, 'lo'), null); - assert.ok(db.getNodeByKey(1, 'hi')); + assert.equal(db.getNodeByKey('Person', 'lo'), null); + assert.ok(db.getNodeByKey('Person', 'hi')); // Remove policy → visible again db.removePrunePolicy('hide-low'); @@ -743,12 +759,12 @@ describe('read-time policy filtering, neighbors', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('policy-excluded neighbor omitted from results', () => { - const a = db.upsertNode(1, 'a', { props: {}, weight: 0.9 }); - const b = db.upsertNode(1, 'b', { props: {}, weight: 0.2 }); // will be excluded - const c = db.upsertNode(1, 'c', { props: {}, weight: 0.8 }); + const a = db.upsertNode('Person', 'a', { props: {}, weight: 0.9 }); + const b = db.upsertNode('Person', 'b', { props: {}, weight: 0.2 }); // will be excluded + const c = db.upsertNode('Person', 'c', { props: {}, weight: 0.8 }); - db.upsertEdge(a, b, 1, { props: {}, weight: 1.0 }); - db.upsertEdge(a, c, 1, { props: {}, weight: 1.0 }); + db.upsertEdge(a, b, 'LINKS_TO', { props: {}, weight: 1.0 }); + db.upsertEdge(a, c, 'LINKS_TO', { props: {}, weight: 1.0 }); // Before policy: 2 neighbors let result = db.neighbors(a, { direction: 'outgoing' }); @@ -772,7 +788,7 @@ describe('read-time policy filtering, toggle visibility', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('add policy hides node, remove policy reveals it', () => { - const id = db.upsertNode(1, 'target', { props: {}, weight: 0.3 }); + const id = db.upsertNode('Person', 'target', { props: {}, weight: 0.3 }); assert.ok(db.getNode(id)); @@ -784,14 +800,14 @@ describe('read-time policy filtering, toggle visibility', () => { }); it('upsert dedup works through policy (no duplicate IDs)', () => { - const id1 = db.upsertNode(1, 'dedup-test', { props: {}, weight: 0.2 }); + const id1 = db.upsertNode('Person', 'dedup-test', { props: {}, weight: 0.2 }); db.setPrunePolicy('p', { maxWeight: 0.5 }); // Hidden from reads assert.equal(db.getNode(id1), null); // Upsert same key; must reuse ID - const id2 = db.upsertNode(1, 'dedup-test', { props: {}, weight: 0.8 }); + const id2 = db.upsertNode('Person', 'dedup-test', { props: {}, weight: 0.8 }); assert.equal(id1, id2); // Now weight 0.8 > 0.5, visible again @@ -803,7 +819,7 @@ describe('read-time policy filtering, toggle visibility', () => { // Phase 12: API Completeness tests -describe('getNodesByType', () => { +describe('getNodesByLabels', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-nbt-')); @@ -815,13 +831,13 @@ describe('getNodesByType', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns hydrated node records by type', () => { - db.upsertNode(1, 'alice', { props: { name: 'Alice' }, weight: 0.9 }); - db.upsertNode(1, 'bob', { props: { name: 'Bob' }, weight: 0.8 }); - db.upsertNode(2, 'charlie', { props: {}, weight: 0.7 }); + db.upsertNode('Person', 'alice', { props: { name: 'Alice' }, weight: 0.9 }); + db.upsertNode('Person', 'bob', { props: { name: 'Bob' }, weight: 0.8 }); + db.upsertNode('Company', 'charlie', { props: {}, weight: 0.7 }); - const type1 = db.getNodesByType(1); + const type1 = db.getNodesByLabels('Person'); assert.equal(type1.length, 2); - assert.ok(type1.every(n => n.typeId === 1)); + assert.ok(type1.every(n => n.labels.includes('Person'))); const keys = type1.map(n => n.key); assert.ok(keys.includes('alice')); assert.ok(keys.includes('bob')); @@ -834,30 +850,30 @@ describe('getNodesByType', () => { }); it('returns empty array for non-existent type', () => { - assert.deepEqual(db.getNodesByType(99), []); + assert.deepEqual(db.getNodesByLabels('MissingLabel'), []); }); it('excludes deleted nodes', () => { - const id = db.upsertNode(3, 'to-delete', { props: {} }); - db.upsertNode(3, 'keeper', { props: {} }); + const id = db.upsertNode('Document', 'to-delete', { props: {} }); + db.upsertNode('Document', 'keeper', { props: {} }); db.deleteNode(id); - const type3 = db.getNodesByType(3); + const type3 = db.getNodesByLabels('Document'); assert.equal(type3.length, 1); assert.equal(type3[0].key, 'keeper'); }); it('works across memtable and segments', () => { - db.upsertNode(4, 'seg1', { props: {} }); + db.upsertNode('Post', 'seg1', { props: {} }); db.flush(); - db.upsertNode(4, 'mem1', { props: {} }); + db.upsertNode('Post', 'mem1', { props: {} }); - const type4 = db.getNodesByType(4); + const type4 = db.getNodesByLabels('Post'); assert.equal(type4.length, 2); }); }); -describe('getEdgesByType', () => { +describe('getEdgesByLabel', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-ebt-')); @@ -869,38 +885,38 @@ describe('getEdgesByType', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns hydrated edge records by type', () => { - const a = db.upsertNode(1, 'a', { props: {} }); - const b = db.upsertNode(1, 'b', { props: {} }); - const c = db.upsertNode(1, 'c', { props: {} }); + const a = db.upsertNode('Person', 'a', { props: {} }); + const b = db.upsertNode('Person', 'b', { props: {} }); + const c = db.upsertNode('Person', 'c', { props: {} }); - db.upsertEdge(a, b, 10, { props: { rel: 'knows' }, weight: 0.9 }); - db.upsertEdge(b, c, 10, { props: { rel: 'likes' }, weight: 0.8 }); - db.upsertEdge(a, c, 20, { props: {}, weight: 0.5 }); + db.upsertEdge(a, b, 'WORKS_AT', { props: { rel: 'knows' }, weight: 0.9 }); + db.upsertEdge(b, c, 'WORKS_AT', { props: { rel: 'likes' }, weight: 0.8 }); + db.upsertEdge(a, c, 'MENTIONS', { props: {}, weight: 0.5 }); - const type10 = db.getEdgesByType(10); + const type10 = db.getEdgesByLabel('WORKS_AT'); assert.equal(type10.length, 2); - assert.ok(type10.every(e => e.typeId === 10)); + assert.ok(type10.every(e => e.label === 'WORKS_AT')); assert.ok(type10.every(e => e.weight > 0)); }); it('returns empty array for non-existent type', () => { - assert.deepEqual(db.getEdgesByType(99), []); + assert.deepEqual(db.getEdgesByLabel('MISSING_EDGE_TYPE'), []); }); it('works across memtable and segments', () => { - const x = db.upsertNode(1, 'x', { props: {} }); - const y = db.upsertNode(1, 'y', { props: {} }); - db.upsertEdge(x, y, 30, { props: {}, weight: 1.0 }); + const x = db.upsertNode('Person', 'x', { props: {} }); + const y = db.upsertNode('Person', 'y', { props: {} }); + db.upsertEdge(x, y, 'OWNS', { props: {}, weight: 1.0 }); db.flush(); - const z = db.upsertNode(1, 'z', { props: {} }); - db.upsertEdge(y, z, 30, { props: {}, weight: 1.0 }); + const z = db.upsertNode('Person', 'z', { props: {} }); + db.upsertEdge(y, z, 'OWNS', { props: {}, weight: 1.0 }); - const type30 = db.getEdgesByType(30); + const type30 = db.getEdgesByLabel('OWNS'); assert.equal(type30.length, 2); }); }); -describe('countNodesByType / countEdgesByType', () => { +describe('countNodesByLabels / countEdgesByLabel', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-cnt-')); @@ -912,41 +928,41 @@ describe('countNodesByType / countEdgesByType', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('counts nodes by type without hydrating records', () => { - db.upsertNode(1, 'n1', { props: {} }); - db.upsertNode(1, 'n2', { props: {} }); - db.upsertNode(2, 'n3', { props: {} }); + db.upsertNode('Person', 'n1', { props: {} }); + db.upsertNode('Person', 'n2', { props: {} }); + db.upsertNode('Company', 'n3', { props: {} }); - assert.equal(db.countNodesByType(1), 2); - assert.equal(db.countNodesByType(2), 1); - assert.equal(db.countNodesByType(99), 0); + assert.equal(db.countNodesByLabels('Person'), 2); + assert.equal(db.countNodesByLabels('Company'), 1); + assert.equal(db.countNodesByLabels('MissingLabel'), 0); }); it('counts edges by type', () => { - const a = db.upsertNode(1, 'ca', { props: {} }); - const b = db.upsertNode(1, 'cb', { props: {} }); - const c = db.upsertNode(1, 'cc', { props: {} }); - db.upsertEdge(a, b, 10, { props: {}, weight: 1.0 }); - db.upsertEdge(b, c, 10, { props: {}, weight: 1.0 }); - db.upsertEdge(a, c, 20, { props: {}, weight: 1.0 }); + const a = db.upsertNode('Person', 'ca', { props: {} }); + const b = db.upsertNode('Person', 'cb', { props: {} }); + const c = db.upsertNode('Person', 'cc', { props: {} }); + db.upsertEdge(a, b, 'WORKS_AT', { props: {}, weight: 1.0 }); + db.upsertEdge(b, c, 'WORKS_AT', { props: {}, weight: 1.0 }); + db.upsertEdge(a, c, 'MENTIONS', { props: {}, weight: 1.0 }); - assert.equal(db.countEdgesByType(10), 2); - assert.equal(db.countEdgesByType(20), 1); - assert.equal(db.countEdgesByType(99), 0); + assert.equal(db.countEdgesByLabel('WORKS_AT'), 2); + assert.equal(db.countEdgesByLabel('MENTIONS'), 1); + assert.equal(db.countEdgesByLabel('MISSING_EDGE_TYPE'), 0); }); it('counts respect tombstones', () => { - const id = db.upsertNode(5, 'temp', { props: {} }); - assert.equal(db.countNodesByType(5), 1); + const id = db.upsertNode('User', 'temp', { props: {} }); + assert.equal(db.countNodesByLabels('User'), 1); db.deleteNode(id); - assert.equal(db.countNodesByType(5), 0); + assert.equal(db.countNodesByLabels('User'), 0); }); it('counts work across memtable and segments', () => { - db.upsertNode(6, 's1', { props: {} }); + db.upsertNode('Location', 's1', { props: {} }); db.flush(); - db.upsertNode(6, 'm1', { props: {} }); + db.upsertNode('Location', 'm1', { props: {} }); - assert.equal(db.countNodesByType(6), 2); + assert.equal(db.countNodesByLabels('Location'), 2); }); }); @@ -958,38 +974,38 @@ describe('async variants of type query APIs', () => { walSyncMode: 'immediate', compactAfterNFlushes: 0, }); - db.upsertNode(1, 'a', { props: { v: 1 }, weight: 0.9 }); - db.upsertNode(1, 'b', { props: { v: 2 }, weight: 0.8 }); - db.upsertNode(2, 'c', { props: {}, weight: 0.7 }); - const na = db.upsertNode(1, 'a', { props: { v: 1 }, weight: 0.9 }); // already exists, same id - const nb = db.upsertNode(1, 'b', { props: { v: 2 }, weight: 0.8 }); - db.upsertEdge(na, nb, 10, { props: {}, weight: 1.0 }); + db.upsertNode('Person', 'a', { props: { v: 1 }, weight: 0.9 }); + db.upsertNode('Person', 'b', { props: { v: 2 }, weight: 0.8 }); + db.upsertNode('Company', 'c', { props: {}, weight: 0.7 }); + const na = db.upsertNode('Person', 'a', { props: { v: 1 }, weight: 0.9 }); // already exists, same id + const nb = db.upsertNode('Person', 'b', { props: { v: 2 }, weight: 0.8 }); + db.upsertEdge(na, nb, 'WORKS_AT', { props: {}, weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('getNodesByTypeAsync returns hydrated records', async () => { - const records = await db.getNodesByTypeAsync(1); + it('getNodesByLabelsAsync returns hydrated records', async () => { + const records = await db.getNodesByLabelsAsync('Person'); assert.equal(records.length, 2); - assert.ok(records.every(n => n.typeId === 1)); + assert.ok(records.every(n => n.labels.includes('Person'))); }); - it('getEdgesByTypeAsync returns hydrated records', async () => { - const records = await db.getEdgesByTypeAsync(10); + it('getEdgesByLabelAsync returns hydrated records', async () => { + const records = await db.getEdgesByLabelAsync('WORKS_AT'); assert.equal(records.length, 1); - assert.equal(records[0].typeId, 10); + assert.equal(records[0].label, 'WORKS_AT'); }); - it('countNodesByTypeAsync returns correct count', async () => { - const count = await db.countNodesByTypeAsync(1); + it('countNodesByLabelsAsync returns correct count', async () => { + const count = await db.countNodesByLabelsAsync('Person'); assert.equal(count, 2); - const zero = await db.countNodesByTypeAsync(99); + const zero = await db.countNodesByLabelsAsync('MissingLabel'); assert.equal(zero, 0); }); - it('countEdgesByTypeAsync returns correct count', async () => { - const count = await db.countEdgesByTypeAsync(10); + it('countEdgesByLabelAsync returns correct count', async () => { + const count = await db.countEdgesByLabelAsync('WORKS_AT'); assert.equal(count, 1); - const zero = await db.countEdgesByTypeAsync(99); + const zero = await db.countEdgesByLabelAsync('MISSING_EDGE_TYPE'); assert.equal(zero, 0); }); }); diff --git a/overgraph-node/__test__/async-api.mjs b/overgraph-node/__test__/async-api.mjs index 3606c6c..7c2cf46 100644 --- a/overgraph-node/__test__/async-api.mjs +++ b/overgraph-node/__test__/async-api.mjs @@ -9,6 +9,51 @@ function freshDb(tmpDir, name) { return OverGraph.open(join(tmpDir, name)); } +describe('async catalog diagnostics', () => { + let tmpDir, db; + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-async-catalog-')); + db = freshDb(tmpDir, 'catalog'); + }); + after(async () => { await db.closeAsync(); rmSync(tmpDir, { recursive: true, force: true }); }); + + it('ensures, gets, and lists node labels and edge labels', async () => { + assert.deepEqual(await db.listNodeLabelsAsync(), []); + assert.deepEqual(await db.listEdgeLabelsAsync(), []); + + const personId = await db.ensureNodeLabelAsync('Person'); + const companyId = await db.ensureNodeLabelAsync('Company'); + const worksAtId = await db.ensureEdgeLabelAsync('WORKS_AT'); + const knowsId = await db.ensureEdgeLabelAsync('KNOWS'); + + assert.equal(await db.ensureNodeLabelAsync('Person'), personId); + assert.equal(await db.ensureEdgeLabelAsync('WORKS_AT'), worksAtId); + assert.equal(await db.getNodeLabelIdAsync('Person'), personId); + assert.equal(await db.getNodeLabelIdAsync('MissingLabel'), null); + assert.equal(await db.getEdgeLabelIdAsync('WORKS_AT'), worksAtId); + assert.equal(await db.getEdgeLabelIdAsync('MISSING_EDGE'), null); + assert.equal(await db.getNodeLabelAsync(personId), 'Person'); + assert.equal(await db.getNodeLabelAsync(999_999), null); + assert.equal(await db.getEdgeLabelAsync(worksAtId), 'WORKS_AT'); + assert.equal(await db.getEdgeLabelAsync(999_999), null); + + assert.deepEqual( + (await db.listNodeLabelsAsync()).sort((a, b) => a.labelId - b.labelId), + [ + { label: 'Person', labelId: personId }, + { label: 'Company', labelId: companyId }, + ], + ); + assert.deepEqual( + (await db.listEdgeLabelsAsync()).sort((a, b) => a.labelId - b.labelId), + [ + { label: 'WORKS_AT', labelId: worksAtId }, + { label: 'KNOWS', labelId: knowsId }, + ], + ); + }); +}); + describe('async upsert + get', () => { let tmpDir, db; before(() => { @@ -18,25 +63,25 @@ describe('async upsert + get', () => { after(async () => { await db.closeAsync(); rmSync(tmpDir, { recursive: true, force: true }); }); it('upsertNodeAsync returns a number', async () => { - const id = await db.upsertNodeAsync(1, 'alice', { props: { age: 30 }, weight: 0.9 }); + const id = await db.upsertNodeAsync('Person', 'alice', { props: { age: 30 }, weight: 0.9 }); assert.equal(typeof id, 'number'); assert.ok(id > 0); }); it('upsertEdgeAsync returns a number', async () => { - const a = await db.upsertNodeAsync(1, 'src'); - const b = await db.upsertNodeAsync(1, 'dst'); - const eid = await db.upsertEdgeAsync(a, b, 5, { props: { rel: 'knows' }, weight: 1.5 }); + const a = await db.upsertNodeAsync('Person', 'src'); + const b = await db.upsertNodeAsync('Person', 'dst'); + const eid = await db.upsertEdgeAsync(a, b, 'DEPENDS_ON', { props: { rel: 'knows' }, weight: 1.5 }); assert.equal(typeof eid, 'number'); assert.ok(eid > 0); }); it('getNodeAsync returns full record', async () => { - const id = await db.upsertNodeAsync(2, 'bob', { props: { color: 'red' } }); + const id = await db.upsertNodeAsync('Company', 'bob', { props: { color: 'red' } }); const n = await db.getNodeAsync(id); assert.ok(n); assert.equal(n.id, id); - assert.equal(n.typeId, 2); + assert.deepEqual(n.labels, ['Company']); assert.equal(n.key, 'bob'); assert.equal(n.props.color, 'red'); }); @@ -47,15 +92,26 @@ describe('async upsert + get', () => { }); it('getEdgeAsync returns full record', async () => { - const a = await db.upsertNodeAsync(1, 'ea'); - const b = await db.upsertNodeAsync(1, 'eb'); - const eid = await db.upsertEdgeAsync(a, b, 10, { props: { kind: 'test' } }); + const a = await db.upsertNodeAsync('Person', 'ea'); + const b = await db.upsertNodeAsync('Person', 'eb'); + const eid = await db.upsertEdgeAsync(a, b, 'WORKS_AT', { props: { kind: 'test' } }); const e = await db.getEdgeAsync(eid); assert.ok(e); assert.equal(e.id, eid); - assert.equal(e.typeId, 10); + assert.equal(e.label, 'WORKS_AT'); assert.equal(e.props.kind, 'test'); }); + + it('addNodeLabelAsync and removeNodeLabelAsync return changed flags', async () => { + const id = await db.upsertNodeAsync(['Person'], 'label-mutation'); + assert.equal(await db.addNodeLabelAsync(id, 'Admin'), true); + assert.equal(await db.addNodeLabelAsync(id, 'Admin'), false); + const withLabel = await db.getNodeAsync(id); + assert.deepEqual([...withLabel.labels].sort(), ['Admin', 'Person']); + assert.equal(await db.removeNodeLabelAsync(id, 'Admin'), true); + assert.equal(await db.removeNodeLabelAsync(id, 'Admin'), false); + assert.deepEqual((await db.getNodeAsync(id)).labels, ['Person']); + }); }); describe('async batch upserts', () => { @@ -68,8 +124,8 @@ describe('async batch upserts', () => { it('batchUpsertNodesAsync returns Float64Array', async () => { const ids = await db.batchUpsertNodesAsync([ - { typeId: 1, key: 'n1' }, - { typeId: 1, key: 'n2', props: { x: 1 } }, + { labels: ['Person'], key: 'n1' }, + { labels: ['Person'], key: 'n2', props: { x: 1 } }, ]); assert.ok(ids instanceof Float64Array); assert.equal(ids.length, 2); @@ -78,11 +134,11 @@ describe('async batch upserts', () => { it('batchUpsertEdgesAsync returns Float64Array', async () => { const [a, b] = await db.batchUpsertNodesAsync([ - { typeId: 1, key: 'ba' }, - { typeId: 1, key: 'bb' }, + { labels: ['Person'], key: 'ba' }, + { labels: ['Person'], key: 'bb' }, ]); const eids = await db.batchUpsertEdgesAsync([ - { from: a, to: b, typeId: 5 }, + { from: a, to: b, label: 'DEPENDS_ON'}, ]); assert.ok(eids instanceof Float64Array); assert.equal(eids.length, 1); @@ -100,14 +156,15 @@ describe('async write transactions', () => { it('stages, reads, and commits asynchronously', async () => { const txn = await db.beginWriteTxnAsync(); await txn.stageAsync([ - { op: 'upsertNode', alias: 'alice', typeId: 1, key: 'alice', props: { name: 'Alice' } }, - { op: 'upsertNode', alias: 'bob', typeId: 1, key: 'bob' }, - { op: 'upsertEdge', alias: 'knows', from: { local: 'alice' }, to: { local: 'bob' }, typeId: 7 }, + { op: 'upsertNode', alias: 'alice', labels: ['Person'], key: 'alice', props: { name: 'Alice' } }, + { op: 'upsertNode', alias: 'bob', labels: ['Person'], key: 'bob' }, + { op: 'upsertEdge', alias: 'knows', from: { local: 'alice' }, to: { local: 'bob' }, label: 'KNOWS'}, ]); const staged = await txn.getNodeAsync({ local: 'alice' }); assert.ok(staged); assert.equal(staged.id, undefined); + assert.deepEqual(staged.labels, ['Person']); assert.equal(staged.props.name, 'Alice'); const result = await txn.commitAsync(); @@ -119,24 +176,24 @@ describe('async write transactions', () => { it('supports async builders and rollback', async () => { const txn = db.beginWriteTxn(); - const alice = await txn.upsertNodeAsAsync('async-alice', 1, 'async-alice', { + const alice = await txn.upsertNodeAsAsync('async-alice', 'Person', 'async-alice', { props: { mood: 'staged' }, }); - const bob = await txn.upsertNodeAsAsync('async-bob', 1, 'async-bob'); - await txn.upsertEdgeAsAsync('async-knows', alice, bob, 9); + const bob = await txn.upsertNodeAsAsync('async-bob', 'Person', 'async-bob'); + await txn.upsertEdgeAsAsync('async-knows', alice, bob, 'FOLLOWS'); - const staged = await txn.getNodeByKeyAsync(1, 'async-alice'); + const staged = await txn.getNodeByKeyAsync('Person', 'async-alice'); assert.ok(staged); assert.equal(staged.props.mood, 'staged'); await txn.rollbackAsync(); - assert.equal(await db.getNodeByKeyAsync(1, 'async-alice'), null); + assert.equal(await db.getNodeByKeyAsync('Person', 'async-alice'), null); }); it('preserves async transaction call order when promises are started together', async () => { const txn = await db.beginWriteTxnAsync(); const stage = txn.stageAsync([ - { op: 'upsertNode', alias: 'queued', typeId: 1, key: 'queued' }, + { op: 'upsertNode', alias: 'queued', labels: ['Person'], key: 'queued' }, ]); const read = txn.getNodeAsync({ local: 'queued' }); const commit = txn.commitAsync(); @@ -160,27 +217,27 @@ describe('async delete + neighbors + find', () => { after(async () => { await db.closeAsync(); rmSync(tmpDir, { recursive: true, force: true }); }); it('deleteNodeAsync removes a node', async () => { - const id = await db.upsertNodeAsync(1, 'doomed'); + const id = await db.upsertNodeAsync('Person', 'doomed'); assert.ok(await db.getNodeAsync(id)); await db.deleteNodeAsync(id); assert.equal(await db.getNodeAsync(id), null); }); it('deleteEdgeAsync removes an edge', async () => { - const a = await db.upsertNodeAsync(1, 'da'); - const b = await db.upsertNodeAsync(1, 'db'); - const eid = await db.upsertEdgeAsync(a, b, 1); + const a = await db.upsertNodeAsync('Person', 'da'); + const b = await db.upsertNodeAsync('Person', 'db'); + const eid = await db.upsertEdgeAsync(a, b, 'LINKS_TO'); assert.ok(await db.getEdgeAsync(eid)); await db.deleteEdgeAsync(eid); assert.equal(await db.getEdgeAsync(eid), null); }); it('neighborsAsync returns correct results', async () => { - const c = await db.upsertNodeAsync(1, 'center'); - const n1 = await db.upsertNodeAsync(1, 'nbr1'); - const n2 = await db.upsertNodeAsync(1, 'nbr2'); - await db.upsertEdgeAsync(c, n1, 10, { weight: 1 }); - await db.upsertEdgeAsync(c, n2, 10, { weight: 2 }); + const c = await db.upsertNodeAsync('Person', 'center'); + const n1 = await db.upsertNodeAsync('Person', 'nbr1'); + const n2 = await db.upsertNodeAsync('Person', 'nbr2'); + await db.upsertEdgeAsync(c, n1, 'WORKS_AT', { weight: 1 }); + await db.upsertEdgeAsync(c, n2, 'WORKS_AT', { weight: 2 }); const result = await db.neighborsAsync(c, { direction: 'outgoing' }); assert.ok(Array.isArray(result)); @@ -189,11 +246,11 @@ describe('async delete + neighbors + find', () => { }); it('topKNeighborsAsync returns plain neighbor entry arrays', async () => { - const c = await db.upsertNodeAsync(1, 'topk-center'); - const n1 = await db.upsertNodeAsync(1, 'topk-nbr1'); - const n2 = await db.upsertNodeAsync(1, 'topk-nbr2'); - await db.upsertEdgeAsync(c, n1, 10, { weight: 1 }); - await db.upsertEdgeAsync(c, n2, 10, { weight: 2 }); + const c = await db.upsertNodeAsync('Person', 'topk-center'); + const n1 = await db.upsertNodeAsync('Person', 'topk-nbr1'); + const n2 = await db.upsertNodeAsync('Person', 'topk-nbr2'); + await db.upsertEdgeAsync(c, n1, 'WORKS_AT', { weight: 1 }); + await db.upsertEdgeAsync(c, n2, 'WORKS_AT', { weight: 2 }); const result = await db.topKNeighborsAsync(c, 2, { direction: 'outgoing', scoring: 'weight' }); assert.ok(Array.isArray(result)); @@ -205,11 +262,11 @@ describe('async delete + neighbors + find', () => { }); it('traverseAsync returns 2nd-hop nodes', async () => { - const a = await db.upsertNodeAsync(1, 'hop-a'); - const b = await db.upsertNodeAsync(1, 'hop-b'); - const c = await db.upsertNodeAsync(1, 'hop-c'); - await db.upsertEdgeAsync(a, b, 10); - await db.upsertEdgeAsync(b, c, 10); + const a = await db.upsertNodeAsync('Person', 'hop-a'); + const b = await db.upsertNodeAsync('Person', 'hop-b'); + const c = await db.upsertNodeAsync('Person', 'hop-c'); + await db.upsertEdgeAsync(a, b, 'WORKS_AT'); + await db.upsertEdgeAsync(b, c, 'WORKS_AT'); const page = await db.traverseAsync(a, 2, { minDepth: 2, direction: 'outgoing' }); const nodeSet = new Set(page.items.map(hit => hit.nodeId)); @@ -219,11 +276,11 @@ describe('async delete + neighbors + find', () => { }); it('findNodesAsync returns matching ids', async () => { - await db.upsertNodeAsync(7, 'fa', { props: { city: 'NYC' } }); - await db.upsertNodeAsync(7, 'fb', { props: { city: 'NYC' } }); - await db.upsertNodeAsync(7, 'fc', { props: { city: 'LA' } }); + await db.upsertNodeAsync('CityResident', 'fa', { props: { city: 'NYC' } }); + await db.upsertNodeAsync('CityResident', 'fb', { props: { city: 'NYC' } }); + await db.upsertNodeAsync('CityResident', 'fc', { props: { city: 'LA' } }); - const ids = await db.findNodesAsync(7, 'city', 'NYC'); + const ids = await db.findNodesAsync('CityResident', 'city', 'NYC'); assert.ok(ids instanceof Float64Array); assert.equal(ids.length, 2); }); @@ -238,7 +295,7 @@ describe('async flush + compact', () => { after(async () => { await db.closeAsync(); rmSync(tmpDir, { recursive: true, force: true }); }); it('flushAsync resolves without error', async () => { - await db.upsertNodeAsync(1, 'flushed'); + await db.upsertNodeAsync('Person', 'flushed'); await db.flushAsync(); }); @@ -256,11 +313,11 @@ describe('async flush + compact', () => { try { for (let i = 0; i < 50; i++) { - testDb.upsertNode(1, `cn-${i}`, { props: { idx: i } }); + testDb.upsertNode('Person', `cn-${i}`, { props: { idx: i } }); } await testDb.flushAsync(); for (let i = 50; i < 100; i++) { - testDb.upsertNode(1, `cn-${i}`, { props: { idx: i } }); + testDb.upsertNode('Person', `cn-${i}`, { props: { idx: i } }); } await testDb.flushAsync(); @@ -292,11 +349,11 @@ describe('compactWithProgressAsync', () => { it('returns stats and calls progress callback', async () => { for (let i = 0; i < 50; i++) { - db.upsertNode(1, `cp-${i}`, { props: { idx: i } }); + db.upsertNode('Person', `cp-${i}`, { props: { idx: i } }); } await db.flushAsync(); for (let i = 50; i < 100; i++) { - db.upsertNode(1, `cp-${i}`, { props: { idx: i } }); + db.upsertNode('Person', `cp-${i}`, { props: { idx: i } }); } await db.flushAsync(); @@ -316,11 +373,11 @@ describe('compactWithProgressAsync', () => { it('does not block event loop during compaction', async () => { for (let i = 0; i < 100; i++) { - db.upsertNode(2, `nb2-${i}`, { props: { data: 'y'.repeat(50) } }); + db.upsertNode('Company', `nb2-${i}`, { props: { data: 'y'.repeat(50) } }); } await db.flushAsync(); for (let i = 100; i < 200; i++) { - db.upsertNode(2, `nb2-${i}`, { props: { data: 'y'.repeat(50) } }); + db.upsertNode('Company', `nb2-${i}`, { props: { data: 'y'.repeat(50) } }); } await db.flushAsync(); @@ -347,7 +404,7 @@ describe('async does not block event loop', () => { it('setTimeout fires during async flush', async () => { // Insert enough data to make flush take a moment for (let i = 0; i < 200; i++) { - db.upsertNode(1, `nb-${i}`, { props: { data: 'x'.repeat(100) } }); + db.upsertNode('Person', `nb-${i}`, { props: { data: 'x'.repeat(100) } }); } let timerFired = false; diff --git a/overgraph-node/__test__/benchmark-v2.mjs b/overgraph-node/__test__/benchmark-v2.mjs index 3e1ece9..c14e868 100755 --- a/overgraph-node/__test__/benchmark-v2.mjs +++ b/overgraph-node/__test__/benchmark-v2.mjs @@ -187,12 +187,20 @@ function traverseDeepBranching(fanout) { return [Math.max(8, Math.min(24, Math.floor(fanout / 4))), 4, 4]; } +function nodeInput(label, key, fields = {}) { + return { labels: [label], key, ...fields }; +} + +function nodeFilter(label, mode = 'all') { + return { labels: [label], mode }; +} + function buildDepthTwoTraversalGraph(db, cfg) { - const hopNodes = [{ typeId: 1, key: 'root' }]; + const hopNodes = [nodeInput('Person', 'root')]; for (let i = 0; i < cfg.two_hop_mid; i++) { - hopNodes.push({ typeId: 1, key: `m-${i}` }); + hopNodes.push(nodeInput('Person', `m-${i}`)); for (let j = 0; j < cfg.two_hop_leaves_per_mid; j++) { - hopNodes.push({ typeId: 1, key: `l-${i}-${j}` }); + hopNodes.push(nodeInput('Person', `l-${i}-${j}`)); } } const hopIds = db.batchUpsertNodes(hopNodes); @@ -201,10 +209,10 @@ function buildDepthTwoTraversalGraph(db, cfg) { const hopEdges = []; for (let i = 0; i < cfg.two_hop_mid; i++) { const midId = hopIds[1 + i * midStride]; - hopEdges.push({ from: root, to: midId, typeId: 1, weight: 1.0 }); + hopEdges.push({ from: root, to: midId, label: 'LINKS_TO', weight: 1.0 }); for (let j = 0; j < cfg.two_hop_leaves_per_mid; j++) { const leafId = hopIds[1 + i * midStride + 1 + j]; - hopEdges.push({ from: midId, to: leafId, typeId: 1, weight: 1.0 }); + hopEdges.push({ from: midId, to: leafId, label: 'LINKS_TO', weight: 1.0 }); } } db.batchUpsertEdges(hopEdges); @@ -213,19 +221,19 @@ function buildDepthTwoTraversalGraph(db, cfg) { function buildDeepTraversalGraph(db, cfg) { const [level1, level2, level3] = traverseDeepBranching(cfg.fanout); - const nodes = [{ typeId: 1, key: 'root' }]; + const nodes = [nodeInput('Person', 'root')]; for (let i = 0; i < level1; i++) { - nodes.push({ typeId: 11, key: `lvl1-${i}` }); + nodes.push(nodeInput('LevelOne', `lvl1-${i}`)); } for (let i = 0; i < level1; i++) { for (let j = 0; j < level2; j++) { - nodes.push({ typeId: (i + j) % 2 === 0 ? 2 : 3, key: `lvl2-${i}-${j}` }); + nodes.push(nodeInput((i + j) % 2 === 0 ? 'Company' : 'Document', `lvl2-${i}-${j}`)); } } for (let i = 0; i < level1; i++) { for (let j = 0; j < level2; j++) { for (let k = 0; k < level3; k++) { - nodes.push({ typeId: (i + j + k) % 2 === 0 ? 2 : 3, key: `lvl3-${i}-${j}-${k}` }); + nodes.push(nodeInput((i + j + k) % 2 === 0 ? 'Company' : 'Document', `lvl3-${i}-${j}-${k}`)); } } } @@ -237,14 +245,14 @@ function buildDeepTraversalGraph(db, cfg) { const edges = []; for (let i = 0; i < level1; i++) { const lvl1Id = ids[level1Offset + i]; - edges.push({ from: root, to: lvl1Id, typeId: 1, weight: 1.0 }); + edges.push({ from: root, to: lvl1Id, label: 'LINKS_TO', weight: 1.0 }); for (let j = 0; j < level2; j++) { const lvl2Idx = i * level2 + j; const lvl2Id = ids[level2Offset + lvl2Idx]; - edges.push({ from: lvl1Id, to: lvl2Id, typeId: 1, weight: 1.0 }); + edges.push({ from: lvl1Id, to: lvl2Id, label: 'LINKS_TO', weight: 1.0 }); for (let k = 0; k < level3; k++) { const lvl3Idx = lvl2Idx * level3 + k; - edges.push({ from: lvl2Id, to: ids[level3Offset + lvl3Idx], typeId: 1, weight: 1.0 }); + edges.push({ from: lvl2Id, to: ids[level3Offset + lvl3Idx], label: 'LINKS_TO', weight: 1.0 }); } } } @@ -331,6 +339,17 @@ function waitForPropertyIndexReady(db, indexId) { throw new Error(`Timed out waiting for property index ${indexId} to become ready`); } +function waitForEdgePropertyIndexReady(db, indexId) { + const deadline = performance.now() + 10_000; + while (performance.now() < deadline) { + if (db.listEdgePropertyIndexes().some(info => info.indexId === indexId && info.state === 'ready')) { + return; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + throw new Error(`Timed out waiting for edge property index ${indexId} to become ready`); +} + function queryBenchmarkLayout(preloadNodes) { const segments = preloadNodes >= 2 ? 1 : 0; const segmentNodes = segments === 0 ? 0 : Math.max(1, Math.floor(preloadNodes / (segments + 1))); @@ -344,21 +363,19 @@ function queryBenchmarkLayout(preloadNodes) { function queryBenchNodes(start, count) { return Array.from({ length: count }, (_, offset) => { const i = start + offset; - return { - typeId: 1, - key: `q-${i}`, + return nodeInput('Person', `q-${i}`, { props: queryBenchProps(i), - }; + }); }); } function buildQueryBenchmarkDb(path, preloadNodes) { const db = OverGraph.open(path); - const status = db.ensureNodePropertyIndex(1, 'status', { kind: 'equality' }); + const status = db.ensureNodePropertyIndex('Person', 'status', { kind: 'equality' }); waitForPropertyIndexReady(db, status.indexId); - const tier = db.ensureNodePropertyIndex(1, 'tier', { kind: 'equality' }); + const tier = db.ensureNodePropertyIndex('Person', 'tier', { kind: 'equality' }); waitForPropertyIndexReady(db, tier.indexId); - const score = db.ensureNodePropertyIndex(1, 'score', { kind: 'range', domain: 'int' }); + const score = db.ensureNodePropertyIndex('Person', 'score', { kind: 'range', domain: 'int' }); waitForPropertyIndexReady(db, score.indexId); const layout = queryBenchmarkLayout(preloadNodes); @@ -372,6 +389,57 @@ function buildQueryBenchmarkDb(path, preloadNodes) { return { db, layout }; } +function buildEdgeQueryBenchmarkDb(path, preloadEdges) { + const db = OverGraph.open(path); + const sourceCount = 1; + const targetCount = Math.max(1, preloadEdges); + const nodes = []; + for (let i = 0; i < sourceCount; i += 1) { + nodes.push(nodeInput('Person', `edge-source-${i}`)); + } + for (let i = 0; i < targetCount; i += 1) { + nodes.push(nodeInput('Company', `edge-target-${i}`)); + } + const ids = db.batchUpsertNodes(nodes); + const sourceIds = ids.slice(0, sourceCount); + const targetIds = ids.slice(sourceCount); + const sourceId = sourceIds[0]; + const segments = preloadEdges >= 2 ? 1 : 0; + const segmentEdges = segments === 0 ? 0 : Math.max(1, Math.floor(preloadEdges / 2)); + const memtableTailEdges = Math.max(0, preloadEdges - segmentEdges); + const makeEdges = (start, count) => Array.from({ length: count }, (_, offset) => { + const i = start + offset; + return { + from: sourceIds[i % sourceCount], + to: targetIds[i % targetIds.length], + label: 'WORKS_AT', + props: { role: i % 10 === 0 ? 'lead' : 'member', score: i % 100 }, + weight: i % 2 === 0 ? 2.0 : 0.5, + }; + }); + if (segmentEdges > 0) { + db.batchUpsertEdges(makeEdges(0, segmentEdges)); + db.flush(); + } + if (memtableTailEdges > 0) { + db.batchUpsertEdges(makeEdges(segmentEdges, memtableTailEdges)); + } + return { + db, + sourceId, + layout: { segments, segment_edges: segmentEdges, memtable_tail_edges: memtableTailEdges }, + }; +} + +function buildIndexedEdgeQueryBenchmarkDb(path, preloadEdges) { + const fixture = buildEdgeQueryBenchmarkDb(path, preloadEdges); + const role = fixture.db.ensureEdgePropertyIndex('WORKS_AT', 'role', { kind: 'equality' }); + waitForEdgePropertyIndexReady(fixture.db, role.indexId); + const score = fixture.db.ensureEdgePropertyIndex('WORKS_AT', 'score', { kind: 'range', domain: 'int' }); + waitForEdgePropertyIndexReady(fixture.db, score.indexId); + return fixture; +} + function pushQueryScenarios(args, scenarioContract, cfg, tmpRoot, scenarios) { const preloadNodes = cfg.time_range_nodes; const limit = 100; @@ -381,7 +449,7 @@ function pushQueryScenarios(args, scenarioContract, cfg, tmpRoot, scenarios) { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const { db, layout } = buildQueryBenchmarkDb(join(tmpRoot, 'query-node-ids-intersected'), preloadNodes); const request = { - typeId: 1, + labelFilter: nodeFilter('Person'), filter: { and: [ { property: 'status', eq: 'active' }, @@ -399,7 +467,7 @@ function pushQueryScenarios(args, scenarioContract, cfg, tmpRoot, scenarios) { s, iterCfg, { - type_id: 1, + label: 'Person', preload_nodes: preloadNodes, segments: layout.segments, segment_nodes: layout.segment_nodes, @@ -418,7 +486,7 @@ function pushQueryScenarios(args, scenarioContract, cfg, tmpRoot, scenarios) { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const { db, layout } = buildQueryBenchmarkDb(join(tmpRoot, 'query-nodes-hydrated-intersected'), preloadNodes); const request = { - typeId: 1, + labelFilter: nodeFilter('Person'), filter: { and: [ { property: 'status', eq: 'active' }, @@ -436,7 +504,7 @@ function pushQueryScenarios(args, scenarioContract, cfg, tmpRoot, scenarios) { s, iterCfg, { - type_id: 1, + label: 'Person', preload_nodes: preloadNodes, segments: layout.segments, segment_nodes: layout.segment_nodes, @@ -449,6 +517,202 @@ function pushQueryScenarios(args, scenarioContract, cfg, tmpRoot, scenarios) { ); db.close(); } + + { + const scenarioId = 'S-QUERY-003'; + const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); + const { db, layout, sourceId } = buildEdgeQueryBenchmarkDb( + join(tmpRoot, 'query-edge-ids-endpoint-metadata'), + preloadNodes + ); + const request = { + label: 'WORKS_AT', + fromIds: [sourceId], + filter: { weight: { gte: 1.0 } }, + limit, + }; + const s = runBench(() => db.queryEdgeIds(request), iterCfg.warmup, iterCfg.iters); + scenarios.push( + scenario( + scenarioId, + 'query_edge_ids_endpoint_metadata', + 'query', + s, + iterCfg, + { + label: 'WORKS_AT', + preload_edges: preloadNodes, + segments: layout.segments, + segment_edges: layout.segment_edges, + memtable_tail_edges: layout.memtable_tail_edges, + filter: 'weight_gte_1', + limit, + }, + scenarioComparability(scenarioContract, scenarioId) + ) + ); + db.close(); + } + + { + const scenarioId = 'S-QUERY-004'; + const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); + const { db, layout, sourceId } = buildEdgeQueryBenchmarkDb( + join(tmpRoot, 'query-edges-endpoint-property-hydrated'), + preloadNodes + ); + const request = { + label: 'WORKS_AT', + fromIds: [sourceId], + filter: { + and: [ + { weight: { gte: 1.0 } }, + { property: 'role', eq: 'lead' }, + ], + }, + limit, + }; + const s = runBench(() => db.queryEdges(request), iterCfg.warmup, iterCfg.iters); + scenarios.push( + scenario( + scenarioId, + 'query_edges_endpoint_property_hydrated', + 'query', + s, + iterCfg, + { + label: 'WORKS_AT', + preload_edges: preloadNodes, + segments: layout.segments, + segment_edges: layout.segment_edges, + memtable_tail_edges: layout.memtable_tail_edges, + filter: 'weight_gte_1_and_role_eq_lead', + limit, + }, + scenarioComparability(scenarioContract, scenarioId) + ) + ); + db.close(); + } + + { + const scenarioId = 'S-QUERY-005'; + const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); + const { db, layout, sourceId } = buildIndexedEdgeQueryBenchmarkDb( + join(tmpRoot, 'query-edge-ids-property-indexed-equality'), + preloadNodes + ); + const request = { + label: 'WORKS_AT', + fromIds: [sourceId], + filter: { property: 'role', eq: 'lead' }, + limit, + }; + const s = runBench(() => db.queryEdgeIds(request), iterCfg.warmup, iterCfg.iters); + scenarios.push( + scenario( + scenarioId, + 'query_edge_ids_property_indexed_equality', + 'query', + s, + iterCfg, + { + label: 'WORKS_AT', + preload_edges: preloadNodes, + segments: layout.segments, + segment_edges: layout.segment_edges, + memtable_tail_edges: layout.memtable_tail_edges, + filter: 'role_eq_lead', + limit, + }, + scenarioComparability(scenarioContract, scenarioId) + ) + ); + db.close(); + } + + { + const scenarioId = 'S-QUERY-006'; + const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); + const { db, layout, sourceId } = buildIndexedEdgeQueryBenchmarkDb( + join(tmpRoot, 'query-edge-ids-property-indexed-range'), + preloadNodes + ); + const request = { + label: 'WORKS_AT', + fromIds: [sourceId], + filter: { property: 'score', gte: 90 }, + limit, + }; + const s = runBench(() => db.queryEdgeIds(request), iterCfg.warmup, iterCfg.iters); + scenarios.push( + scenario( + scenarioId, + 'query_edge_ids_property_indexed_range', + 'query', + s, + iterCfg, + { + label: 'WORKS_AT', + preload_edges: preloadNodes, + segments: layout.segments, + segment_edges: layout.segment_edges, + memtable_tail_edges: layout.memtable_tail_edges, + filter: 'score_gte_90', + limit, + }, + scenarioComparability(scenarioContract, scenarioId) + ) + ); + db.close(); + } + + { + const scenarioId = 'S-QUERY-007'; + const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); + const { db, layout } = buildIndexedEdgeQueryBenchmarkDb( + join(tmpRoot, 'query-pattern-edge-property-anchor-indexed'), + preloadNodes + ); + const request = { + nodes: [ + { alias: 'source', labelFilter: nodeFilter('Person') }, + { alias: 'target', labelFilter: nodeFilter('Company') }, + ], + edges: [ + { + alias: 'edge', + fromAlias: 'source', + toAlias: 'target', + direction: 'outgoing', + edgeLabelFilter: ['WORKS_AT'], + filter: { property: 'role', eq: 'lead' }, + }, + ], + limit, + }; + const s = runBench(() => db.queryPattern(request), iterCfg.warmup, iterCfg.iters); + scenarios.push( + scenario( + scenarioId, + 'query_pattern_edge_property_anchor_indexed', + 'query', + s, + iterCfg, + { + label: 'WORKS_AT', + preload_edges: preloadNodes, + segments: layout.segments, + segment_edges: layout.segment_edges, + memtable_tail_edges: layout.memtable_tail_edges, + filter: 'role_eq_lead', + limit, + }, + scenarioComparability(scenarioContract, scenarioId) + ) + ); + db.close(); + } } const args = parseArgs(process.argv); @@ -469,7 +733,7 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'crud-upsert-node')); const s = runBench( - (i) => db.upsertNode(1, `node-${i}`, { props: { idx: i }, weight: 1.0 }), + (i) => db.upsertNode('Person', `node-${i}`, { props: { idx: i }, weight: 1.0 }), iterCfg.warmup, iterCfg.iters, true @@ -481,7 +745,7 @@ try { 'crud', s, iterCfg, - { type_id: 1, with_props: true, weight: 1.0 }, + { label: 'Person', with_props: true, weight: 1.0 }, scenarioComparability(scenarioContract, scenarioId) ) ); @@ -494,10 +758,10 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'crud-upsert-edge')); const nodeIds = db.batchUpsertNodes( - Array.from({ length: iterCfg.warmup + iterCfg.iters + 1 }, (_, i) => ({ typeId: 1, key: `e-${i}` })) + Array.from({ length: iterCfg.warmup + iterCfg.iters + 1 }, (_, i) => nodeInput('Person', `e-${i}`)) ); const s = runBench( - (i) => db.upsertEdge(nodeIds[i], nodeIds[i + 1], 1, { weight: 1.0 }), + (i) => db.upsertEdge(nodeIds[i], nodeIds[i + 1], 'LINKS_TO', { weight: 1.0 }), iterCfg.warmup, iterCfg.iters, true @@ -509,7 +773,7 @@ try { 'crud', s, iterCfg, - { edge_type_id: 1, weight: 1.0 }, + { label: 'LINKS_TO', weight: 1.0 }, scenarioComparability(scenarioContract, scenarioId) ) ); @@ -524,10 +788,7 @@ try { const s = runBench( (i) => { const nodes = Array.from({ length: cfg.batch_nodes }, (_, j) => ({ - typeId: 1, - key: `bn-${i}-${j}`, - props: { idx: j }, - weight: 1.0, + ...nodeInput('Person', `bn-${i}-${j}`, { props: { idx: j }, weight: 1.0 }), })); db.batchUpsertNodes(nodes); }, @@ -541,7 +802,7 @@ try { 'batch', s, iterCfg, - { batch_nodes: cfg.batch_nodes, type_id: 1, with_props: true }, + { batch_nodes: cfg.batch_nodes, label: 'Person', with_props: true }, scenarioComparability(scenarioContract, scenarioId), cfg.batch_nodes ) @@ -557,10 +818,7 @@ try { const s = runBench( (i) => { const nodes = Array.from({ length: cfg.batch_nodes }, (_, j) => ({ - typeId: 1, - key: `bb-${i}-${j}`, - props: { idx: j }, - weight: 1.0, + ...nodeInput('Person', `bb-${i}-${j}`, { props: { idx: j }, weight: 1.0 }), })); db.batchUpsertNodesBinary(packNodeBatch(nodes)); }, @@ -589,9 +847,7 @@ try { const db = OverGraph.open(join(tmpRoot, 'crud-get-node')); const ids = db.batchUpsertNodes( Array.from({ length: cfg.get_node_nodes }, (_, i) => ({ - typeId: 1, - key: `gn-${i}`, - props: { idx: i }, + ...nodeInput('Person', `gn-${i}`, { props: { idx: i } }), })) ); const s = runBench( @@ -618,9 +874,9 @@ try { const scenarioId = 'S-CRUD-004'; const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'crud-upsert-node-fixed')); - db.upsertNode(1, 'fixed-node', { props: { idx: 0 }, weight: 1.0 }); + db.upsertNode('Person', 'fixed-node', { props: { idx: 0 }, weight: 1.0 }); const s = runBench( - (i) => db.upsertNode(1, 'fixed-node', { props: { idx: i }, weight: 1.0 }), + (i) => db.upsertNode('Person', 'fixed-node', { props: { idx: i }, weight: 1.0 }), iterCfg.warmup, iterCfg.iters ); @@ -631,7 +887,7 @@ try { 'crud', s, iterCfg, - { type_id: 1, with_props: true, weight: 1.0, fixed_key: true }, + { label: 'Person', with_props: true, weight: 1.0, fixed_key: true }, scenarioComparability(scenarioContract, scenarioId) ) ); @@ -645,10 +901,10 @@ try { const db = OverGraph.open(join(tmpRoot, 'crud-upsert-edge-fixed'), { edgeUniqueness: true, }); - const nodeA = db.upsertNode(1, 'fixed-a'); - const nodeB = db.upsertNode(1, 'fixed-b'); + const nodeA = db.upsertNode('Person', 'fixed-a'); + const nodeB = db.upsertNode('Person', 'fixed-b'); const s = runBench( - () => db.upsertEdge(nodeA, nodeB, 1, { weight: 1.0 }), + () => db.upsertEdge(nodeA, nodeB, 'LINKS_TO', { weight: 1.0 }), iterCfg.warmup, iterCfg.iters ); @@ -659,7 +915,7 @@ try { 'crud', s, iterCfg, - { edge_type_id: 1, weight: 1.0, edge_uniqueness: true, fixed_triple: true }, + { label: 'LINKS_TO', weight: 1.0, edge_uniqueness: true, fixed_triple: true }, scenarioComparability(scenarioContract, scenarioId) ) ); @@ -672,15 +928,15 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'trav-neighbors')); const neighNodeIds = db.batchUpsertNodes([ - { typeId: 1, key: 'hub' }, - ...Array.from({ length: cfg.fanout }, (_, i) => ({ typeId: 1, key: `n-${i}` })), + nodeInput('Person', 'hub'), + ...Array.from({ length: cfg.fanout }, (_, i) => nodeInput('Person', `n-${i}`)), ]); const hub = neighNodeIds[0]; db.batchUpsertEdges( Array.from({ length: cfg.fanout }, (_, i) => ({ from: hub, to: neighNodeIds[i + 1], - typeId: 1, + label: 'LINKS_TO', weight: 1.0, })) ); @@ -757,7 +1013,7 @@ try { layout: 'memtable', min_depth: 1, max_depth: 3, - node_type_filter: null, + node_label_filter: null, branching, }, scenarioComparability(scenarioContract, scenarioId) @@ -790,7 +1046,7 @@ try { layout: 'segment', min_depth: 1, max_depth: 3, - node_type_filter: null, + node_label_filter: null, branching, }, scenarioComparability(scenarioContract, scenarioId) @@ -806,7 +1062,7 @@ try { const db = OverGraph.open(join(tmpRoot, 'trav-depth13-filtered-memtable')); const { root, branching } = buildDeepTraversalGraph(db, cfg); const s = runBench( - () => db.traverse(root, 3, { direction: 'outgoing', nodeTypeFilter: [2] }), + () => db.traverse(root, 3, { direction: 'outgoing', emitNodeLabelFilter: nodeFilter('Company') }), iterCfg.warmup, iterCfg.iters ); @@ -822,7 +1078,7 @@ try { layout: 'memtable', min_depth: 1, max_depth: 3, - node_type_filter: [2], + node_label_filter: ['Company'], branching, }, scenarioComparability(scenarioContract, scenarioId) @@ -839,7 +1095,7 @@ try { const { root, branching } = buildDeepTraversalGraph(db, cfg); db.flush(); const s = runBench( - () => db.traverse(root, 3, { direction: 'outgoing', nodeTypeFilter: [2] }), + () => db.traverse(root, 3, { direction: 'outgoing', emitNodeLabelFilter: nodeFilter('Company') }), iterCfg.warmup, iterCfg.iters ); @@ -855,7 +1111,7 @@ try { layout: 'segment', min_depth: 1, max_depth: 3, - node_type_filter: [2], + node_label_filter: ['Company'], branching, }, scenarioComparability(scenarioContract, scenarioId) @@ -870,15 +1126,15 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'trav-degree')); const degNodeIds = db.batchUpsertNodes([ - { typeId: 1, key: 'hub' }, - ...Array.from({ length: cfg.fanout }, (_, i) => ({ typeId: 1, key: `d-${i}` })), + nodeInput('Person', 'hub'), + ...Array.from({ length: cfg.fanout }, (_, i) => nodeInput('Person', `d-${i}`)), ]); const hub = degNodeIds[0]; db.batchUpsertEdges( Array.from({ length: cfg.fanout }, (_, i) => ({ from: hub, to: degNodeIds[i + 1], - typeId: 1, + label: 'LINKS_TO', weight: 1.0, })) ); @@ -909,9 +1165,9 @@ try { // Batch all hub + spoke nodes: [hub-0, spoke-0-0, spoke-0-1, ..., hub-1, spoke-1-0, ...] const allNodes = []; for (let h = 0; h < cfg.batch_nodes; h++) { - allNodes.push({ typeId: 1, key: `hub-${h}` }); + allNodes.push(nodeInput('Person', `hub-${h}`)); for (let i = 0; i < cfg.fanout; i++) { - allNodes.push({ typeId: 1, key: `dt-${h}-${i}` }); + allNodes.push(nodeInput('Person', `dt-${h}-${i}`)); } } const allNodeIds = db.batchUpsertNodes(allNodes); @@ -922,7 +1178,7 @@ try { const hubId = allNodeIds[h * stride]; hubIds.push(hubId); for (let i = 0; i < cfg.fanout; i++) { - degEdges.push({ from: hubId, to: allNodeIds[h * stride + 1 + i], typeId: 1, weight: 1.0 }); + degEdges.push({ from: hubId, to: allNodeIds[h * stride + 1 + i], label: 'LINKS_TO', weight: 1.0 }); } } db.batchUpsertEdges(degEdges); @@ -952,15 +1208,15 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'trav-shortest-path')); const ids = db.batchUpsertNodes( - Array.from({ length: cfg.shortest_path_nodes }, (_, i) => ({ typeId: 1, key: `sp-${i}` })) + Array.from({ length: cfg.shortest_path_nodes }, (_, i) => nodeInput('Person', `sp-${i}`)) ); const spEdges = []; for (let i = 0; i < ids.length; i++) { const from = ids[i]; const to1 = ids[(i + cfg.shortest_path_edge_offsets[0]) % ids.length]; const to2 = ids[(i + cfg.shortest_path_edge_offsets[1]) % ids.length]; - spEdges.push({ from, to: to1, typeId: 1, weight: 1.0 }); - spEdges.push({ from, to: to2, typeId: 1, weight: 1.0 }); + spEdges.push({ from, to: to1, label: 'LINKS_TO', weight: 1.0 }); + spEdges.push({ from, to: to2, label: 'LINKS_TO', weight: 1.0 }); } db.batchUpsertEdges(spEdges); const spFrom = ids[0]; @@ -995,15 +1251,15 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'trav-is-connected')); const ids = db.batchUpsertNodes( - Array.from({ length: cfg.shortest_path_nodes }, (_, i) => ({ typeId: 1, key: `ic-${i}` })) + Array.from({ length: cfg.shortest_path_nodes }, (_, i) => nodeInput('Person', `ic-${i}`)) ); const icEdges = []; for (let i = 0; i < ids.length; i++) { const from = ids[i]; const to1 = ids[(i + cfg.shortest_path_edge_offsets[0]) % ids.length]; const to2 = ids[(i + cfg.shortest_path_edge_offsets[1]) % ids.length]; - icEdges.push({ from, to: to1, typeId: 1, weight: 1.0 }); - icEdges.push({ from, to: to2, typeId: 1, weight: 1.0 }); + icEdges.push({ from, to: to1, label: 'LINKS_TO', weight: 1.0 }); + icEdges.push({ from, to: to2, label: 'LINKS_TO', weight: 1.0 }); } db.batchUpsertEdges(icEdges); const spFrom = ids[0]; @@ -1037,10 +1293,9 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'adv-top-k')); const topKNodeIds = db.batchUpsertNodes([ - { typeId: 1, key: 'hub' }, + nodeInput('Person', 'hub'), ...Array.from({ length: cfg.top_k_candidates }, (_, i) => ({ - typeId: 1, - key: `tk-${i}`, + ...nodeInput('Person', `tk-${i}`), })), ]); const hub = topKNodeIds[0]; @@ -1048,7 +1303,7 @@ try { Array.from({ length: cfg.top_k_candidates }, (_, i) => ({ from: hub, to: topKNodeIds[i + 1], - typeId: 1, + label: 'LINKS_TO', weight: 1.0 + ((i % 100) / 10.0), })) ); @@ -1083,16 +1338,13 @@ try { const db = OverGraph.open(join(tmpRoot, 'adv-time-range')); db.batchUpsertNodes( Array.from({ length: cfg.time_range_nodes }, (_, i) => ({ - typeId: 1, - key: `tr-${i}`, - props: { idx: i }, - weight: 1.0, + ...nodeInput('Person', `tr-${i}`, { props: { idx: i }, weight: 1.0 }), })) ); const fromMs = cfg.time_range_from_ms; const toMs = Date.now() + cfg.time_range_window_ms; const s = runBench( - () => db.findNodesByTimeRange(1, fromMs, toMs), + () => db.findNodesByTimeRange('Person', fromMs, toMs), iterCfg.warmup, iterCfg.iters ); @@ -1104,7 +1356,7 @@ try { s, iterCfg, { - type_id: 1, + label: 'Person', preload_nodes: cfg.time_range_nodes, from_ms: cfg.time_range_from_ms, to_ms_window: cfg.time_range_window_ms, @@ -1121,15 +1373,15 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'adv-ppr')); const ids = db.batchUpsertNodes( - Array.from({ length: cfg.ppr_nodes }, (_, i) => ({ typeId: 1, key: `ppr-${i}` })) + Array.from({ length: cfg.ppr_nodes }, (_, i) => nodeInput('Person', `ppr-${i}`)) ); const pprEdges = []; for (let i = 0; i < ids.length; i++) { const from = ids[i]; const to1 = ids[(i + cfg.ppr_edge_offsets[0]) % ids.length]; const to2 = ids[(i + cfg.ppr_edge_offsets[1]) % ids.length]; - pprEdges.push({ from, to: to1, typeId: 1, weight: 1.0 }); - pprEdges.push({ from, to: to2, typeId: 1, weight: 0.7 }); + pprEdges.push({ from, to: to1, label: 'LINKS_TO', weight: 1.0 }); + pprEdges.push({ from, to: to2, label: 'LINKS_TO', weight: 0.7 }); } db.batchUpsertEdges(pprEdges); const s = runBench( @@ -1168,13 +1420,13 @@ try { const iterCfg = scenarioIterations(args, scenarioContract, scenarioId); const db = OverGraph.open(join(tmpRoot, 'adv-export')); const ids = db.batchUpsertNodes( - Array.from({ length: cfg.export_nodes }, (_, i) => ({ typeId: 1, key: `ex-${i}` })) + Array.from({ length: cfg.export_nodes }, (_, i) => nodeInput('Person', `ex-${i}`)) ); const exportEdges = []; for (let i = 0; i < cfg.export_edges; i++) { const from = ids[i % ids.length]; const to = ids[(i * 13 + 7) % ids.length]; - if (from !== to) exportEdges.push({ from, to, typeId: 1, weight: 1.0 }); + if (from !== to) exportEdges.push({ from, to, label: 'LINKS_TO', weight: 1.0 }); } db.batchUpsertEdges(exportEdges); const s = runBench( @@ -1208,15 +1460,12 @@ try { const s = runBench( (i) => { const nodes = Array.from({ length: cfg.flush_nodes_per_iter }, (_, j) => ({ - typeId: 1, - key: `fl-${i}-${j}`, - props: { idx: j }, - weight: 1.0, + ...nodeInput('Person', `fl-${i}-${j}`, { props: { idx: j }, weight: 1.0 }), })); const ids = db.batchUpsertNodes(nodes); const edges = []; for (let j = 0; j < Math.min(cfg.flush_edges_per_iter_cap, ids.length - 1); j++) { - edges.push({ from: ids[j], to: ids[j + 1], typeId: 1, weight: 1.0 }); + edges.push({ from: ids[j], to: ids[j + 1], label: 'LINKS_TO', weight: 1.0 }); } db.batchUpsertEdges(edges); db.flush(); @@ -1252,7 +1501,7 @@ try { const nodes = Array.from({ length: cfg.vector_nodes }, (_, i) => { const seed = 1729 * (i + 1); return { - typeId: 1, + labels: ['Person'], key: `v-${i}`, denseVector: benchDenseVector(cfg.vector_dim, seed), sparseVector: benchSparseVector(cfg.vector_sparse_dims, cfg.vector_nnz, seed + 0xCAFE), diff --git a/overgraph-node/__test__/benchmark.mjs b/overgraph-node/__test__/benchmark.mjs index 62ac639..e11ca75 100644 --- a/overgraph-node/__test__/benchmark.mjs +++ b/overgraph-node/__test__/benchmark.mjs @@ -87,7 +87,7 @@ try { { const db = OverGraph.open(join(tmpDir, 'single-node')); - const s = bench((i) => db.upsertNode(1, `node-${i}`, { props: { idx: i }, weight: 1.0 })); + const s = bench((i) => db.upsertNode('Person', `node-${i}`, { props: { idx: i }, weight: 1.0 })); printRow('upsert_node', s); db.close(); } @@ -96,9 +96,9 @@ try { const db = OverGraph.open(join(tmpDir, 'single-edge')); const nodeIds = []; for (let i = 0; i < WARMUP_ITERS + BENCH_ITERS + 1; i++) { - nodeIds.push(db.upsertNode(1, `en-${i}`)); + nodeIds.push(db.upsertNode('Person', `en-${i}`)); } - const s = bench((i) => db.upsertEdge(nodeIds[i], nodeIds[i + 1], 1, { weight: 1.0 })); + const s = bench((i) => db.upsertEdge(nodeIds[i], nodeIds[i + 1], 'LINKS_TO', { weight: 1.0 })); printRow('upsert_edge', s); db.close(); } @@ -114,7 +114,7 @@ try { { const db = OverGraph.open(join(tmpDir, 'call-nodes-100')); const jsonArr = Array.from({ length: 100 }, (_, j) => ({ - typeId: 1, key: `cn100-${j}`, props: { idx: j }, weight: 1.0, + labels: ['Person'], key: `cn100-${j}`, props: { idx: j }, weight: 1.0, })); const binBuf = packNodeBatch(jsonArr); const sJson = bench(() => db.batchUpsertNodes(jsonArr)); @@ -128,7 +128,7 @@ try { { const db = OverGraph.open(join(tmpDir, 'call-nodes-1000')); const jsonArr = Array.from({ length: 1000 }, (_, j) => ({ - typeId: 1, key: `cn1k-${j}`, props: { idx: j }, weight: 1.0, + labels: ['Person'], key: `cn1k-${j}`, props: { idx: j }, weight: 1.0, })); const binBuf = packNodeBatch(jsonArr); const sJson = bench(() => db.batchUpsertNodes(jsonArr), { warmup: 10, iters: 50 }); @@ -142,10 +142,10 @@ try { { const db = OverGraph.open(join(tmpDir, 'call-edges-100')); const nids = db.batchUpsertNodes( - Array.from({ length: 200 }, (_, i) => ({ typeId: 1, key: `ce100-${i}` })) + Array.from({ length: 200 }, (_, i) => ({ labels: ['Person'], key: `ce100-${i}` })) ); const jsonArr = Array.from({ length: 100 }, (_, j) => ({ - from: nids[j], to: nids[j + 100], typeId: 1, weight: 1.0, + from: nids[j], to: nids[j + 100], label: 'LINKS_TO', weight: 1.0, })); const binBuf = packEdgeBatch(jsonArr); const sJson = bench(() => db.batchUpsertEdges(jsonArr)); @@ -159,10 +159,10 @@ try { { const db = OverGraph.open(join(tmpDir, 'call-edges-1000')); const nids = db.batchUpsertNodes( - Array.from({ length: 2000 }, (_, i) => ({ typeId: 1, key: `ce1k-${i}` })) + Array.from({ length: 2000 }, (_, i) => ({ labels: ['Person'], key: `ce1k-${i}` })) ); const jsonArr = Array.from({ length: 1000 }, (_, j) => ({ - from: nids[j], to: nids[j + 1000], typeId: 1, weight: 1.0, + from: nids[j], to: nids[j + 1000], label: 'LINKS_TO', weight: 1.0, })); const binBuf = packEdgeBatch(jsonArr); const sJson = bench(() => db.batchUpsertEdges(jsonArr), { warmup: 10, iters: 50 }); @@ -184,7 +184,7 @@ try { (i) => { const nodes = []; for (let j = 0; j < 1000; j++) { - nodes.push({ typeId: 1, key: `e2e-${i}-${j}`, props: { idx: j }, weight: 1.0 }); + nodes.push({ labels: ['Person'], key: `e2e-${i}-${j}`, props: { idx: j }, weight: 1.0 }); } db.batchUpsertNodes(nodes); }, @@ -201,7 +201,7 @@ try { (i) => { const nodes = []; for (let j = 0; j < 1000; j++) { - nodes.push({ typeId: 1, key: `e2e-${i}-${j}`, props: { idx: j }, weight: 1.0 }); + nodes.push({ labels: ['Person'], key: `e2e-${i}-${j}`, props: { idx: j }, weight: 1.0 }); } db.batchUpsertNodesBinary(packNodeBatch(nodes)); }, @@ -220,7 +220,7 @@ try { const db = OverGraph.open(join(tmpDir, 'get-node')); const ids = db.batchUpsertNodes( Array.from({ length: 1000 }, (_, i) => ({ - typeId: 1, key: `gn-${i}`, props: { idx: i, label: `node-${i}` }, + labels: ['Person'], key: `gn-${i}`, props: { idx: i, label: `node-${i}` }, })) ); const s = bench((i) => db.getNode(ids[i % 1000])); @@ -230,10 +230,10 @@ try { { const db = OverGraph.open(join(tmpDir, 'nbr-10')); - const hub = db.upsertNode(1, 'hub'); + const hub = db.upsertNode('Person', 'hub'); for (let i = 0; i < 10; i++) { - const n = db.upsertNode(1, `nbr10-${i}`); - db.upsertEdge(hub, n, 1, { weight: 1.0 }); + const n = db.upsertNode('Person', `nbr10-${i}`); + db.upsertEdge(hub, n, 'LINKS_TO', { weight: 1.0 }); } const s = bench(() => db.neighbors(hub, { direction: 'outgoing' })); printRow('neighbors (10 edges)', s); @@ -242,10 +242,10 @@ try { { const db = OverGraph.open(join(tmpDir, 'nbr-100')); - const hub = db.upsertNode(1, 'hub'); + const hub = db.upsertNode('Person', 'hub'); for (let i = 0; i < 100; i++) { - const n = db.upsertNode(1, `nbr100-${i}`); - db.upsertEdge(hub, n, 1, { weight: 1.0 }); + const n = db.upsertNode('Person', `nbr100-${i}`); + db.upsertEdge(hub, n, 'LINKS_TO', { weight: 1.0 }); } const s = bench(() => db.neighbors(hub, { direction: 'outgoing' })); printRow('neighbors (100 edges)', s); @@ -255,9 +255,9 @@ try { { const db = OverGraph.open(join(tmpDir, 'find')); for (let i = 0; i < 1000; i++) { - db.upsertNode(1, `fn-${i}`, { props: { bucket: i < 500 ? 'target' : 'other' } }); + db.upsertNode('Person', `fn-${i}`, { props: { bucket: i < 500 ? 'target' : 'other' } }); } - const s = bench(() => db.findNodes(1, 'bucket', 'target')); + const s = bench(() => db.findNodes('Person', 'bucket', 'target')); printRow('find_nodes (500/1000 match)', s); db.close(); } @@ -273,12 +273,12 @@ try { (i) => { const nodes = []; for (let j = 0; j < 100; j++) { - nodes.push({ typeId: 1, key: `fl-${i}-${j}` }); + nodes.push({ labels: ['Person'], key: `fl-${i}-${j}` }); } const ids = db.batchUpsertNodes(nodes); const edges = []; for (let j = 0; j < 20; j++) { - edges.push({ from: ids[j], to: ids[j + 1], typeId: 1 }); + edges.push({ from: ids[j], to: ids[j + 1], label: 'LINKS_TO'}); } db.batchUpsertEdges(edges); db.flush(); diff --git a/overgraph-node/__test__/close-stats.mjs b/overgraph-node/__test__/close-stats.mjs index 628d079..84727fd 100644 --- a/overgraph-node/__test__/close-stats.mjs +++ b/overgraph-node/__test__/close-stats.mjs @@ -22,24 +22,24 @@ describe('close({ force: true })', () => { it('should close normally without options (backwards compat)', () => { const db = OverGraph.open(join(tmpDir, 'normal')); - db.upsertNode(1, 'n1'); + db.upsertNode('Person', 'n1'); db.close(); }); it('should close with force: false (same as default)', () => { const db = OverGraph.open(join(tmpDir, 'force-false')); - db.upsertNode(1, 'n1'); + db.upsertNode('Person', 'n1'); db.close({ force: false }); }); it('should close with force: true', () => { const db = OverGraph.open(join(tmpDir, 'force-true')); - db.upsertNode(1, 'n1'); + db.upsertNode('Person', 'n1'); db.close({ force: true }); // Reopen to verify data is intact const db2 = OverGraph.open(join(tmpDir, 'force-true')); - const node = db2.getNodeByKey(1, 'n1'); + const node = db2.getNodeByKey('Person', 'n1'); assert.ok(node, 'node should survive close_fast'); db2.close(); }); @@ -47,31 +47,31 @@ describe('close({ force: true })', () => { it('should close_fast after flush', () => { const db = OverGraph.open(join(tmpDir, 'force-flushed')); for (let i = 0; i < 10; i++) { - db.upsertNode(1, `n${i}`); + db.upsertNode('Person', `n${i}`); } db.flush(); db.close({ force: true }); const db2 = OverGraph.open(join(tmpDir, 'force-flushed')); - const node = db2.getNodeByKey(1, 'n5'); + const node = db2.getNodeByKey('Person', 'n5'); assert.ok(node, 'flushed nodes should survive close_fast'); db2.close(); }); it('should work with closeAsync({ force: true })', async () => { const db = OverGraph.open(join(tmpDir, 'async-force')); - db.upsertNode(1, 'async_node'); + db.upsertNode('Person', 'async_node'); await db.closeAsync({ force: true }); const db2 = OverGraph.open(join(tmpDir, 'async-force')); - const node = db2.getNodeByKey(1, 'async_node'); + const node = db2.getNodeByKey('Person', 'async_node'); assert.ok(node, 'node should survive async close_fast'); db2.close(); }); it('should work with closeAsync() no options', async () => { const db = OverGraph.open(join(tmpDir, 'async-normal')); - db.upsertNode(1, 'n1'); + db.upsertNode('Person', 'n1'); await db.closeAsync(); }); }); @@ -126,11 +126,11 @@ describe('stats()', () => { assert.equal(db.stats().segmentCount, 0); - db.upsertNode(1, 'a'); + db.upsertNode('Person', 'a'); db.flush(); assert.equal(db.stats().segmentCount, 1); - db.upsertNode(1, 'b'); + db.upsertNode('Person', 'b'); db.flush(); assert.equal(db.stats().segmentCount, 2); @@ -139,9 +139,9 @@ describe('stats()', () => { it('should count tombstones after deletes', () => { const db = OverGraph.open(join(tmpDir, 'tombstones'), { walSyncMode: 'immediate' }); - const n1 = db.upsertNode(1, 'a'); - const n2 = db.upsertNode(1, 'b'); - db.upsertEdge(n1, n2, 1); + const n1 = db.upsertNode('Person', 'a'); + const n2 = db.upsertNode('Person', 'b'); + db.upsertEdge(n1, n2, 'LINKS_TO'); assert.equal(db.stats().nodeTombstoneCount, 0); assert.equal(db.stats().edgeTombstoneCount, 0); @@ -164,9 +164,9 @@ describe('stats()', () => { db.stats().lastCompactionMs === null || db.stats().lastCompactionMs === undefined, ); - db.upsertNode(1, 'a'); + db.upsertNode('Person', 'a'); db.flush(); - db.upsertNode(1, 'b'); + db.upsertNode('Person', 'b'); db.flush(); const before = Date.now(); @@ -183,7 +183,7 @@ describe('stats()', () => { it('should work with statsAsync()', async () => { const db = OverGraph.open(join(tmpDir, 'async-stats')); - db.upsertNode(1, 'x'); + db.upsertNode('Person', 'x'); const s = await db.statsAsync(); assert.equal(typeof s.segmentCount, 'number'); diff --git a/overgraph-node/__test__/connected-components.mjs b/overgraph-node/__test__/connected-components.mjs index 6e481d0..95df9f0 100644 --- a/overgraph-node/__test__/connected-components.mjs +++ b/overgraph-node/__test__/connected-components.mjs @@ -19,11 +19,11 @@ describe('connectedComponents (sync)', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('finds single component', () => { - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - const c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10); - db.upsertEdge(b, c, 10); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + const c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT'); + db.upsertEdge(b, c, 'WORKS_AT'); const comps = db.connectedComponents(); assert.equal(comps.length, 3); @@ -34,9 +34,9 @@ describe('connectedComponents (sync)', () => { }); it('finds multiple components', () => { - const d = db.upsertNode(1, 'd'); - const e = db.upsertNode(1, 'e'); - db.upsertEdge(d, e, 10); + const d = db.upsertNode('Person', 'd'); + const e = db.upsertNode('Person', 'e'); + db.upsertEdge(d, e, 'WORKS_AT'); const comps = db.connectedComponents(); const map = Object.fromEntries(comps.map(e => [e.nodeId, e.componentId])); @@ -48,20 +48,20 @@ describe('connectedComponents (sync)', () => { }); it('handles self-loops', () => { - const s = db.upsertNode(1, 'selfloop'); - db.upsertEdge(s, s, 10); + const s = db.upsertNode('Person', 'selfloop'); + db.upsertEdge(s, s, 'WORKS_AT'); const comps = db.connectedComponents(); const map = Object.fromEntries(comps.map(e => [e.nodeId, e.componentId])); assert.equal(map[s], s); // self-loop doesn't change membership }); it('handles parallel edges', () => { - const p1 = db.upsertNode(1, 'par1'); - const p2 = db.upsertNode(1, 'par2'); - const p3 = db.upsertNode(1, 'par3'); - db.upsertEdge(p1, p2, 10); - db.upsertEdge(p1, p2, 20); // parallel - db.upsertEdge(p2, p1, 10); // reverse parallel + const p1 = db.upsertNode('Person', 'par1'); + const p2 = db.upsertNode('Person', 'par2'); + const p3 = db.upsertNode('Person', 'par3'); + db.upsertEdge(p1, p2, 'WORKS_AT'); + db.upsertEdge(p1, p2, 'MENTIONS'); // parallel + db.upsertEdge(p2, p1, 'WORKS_AT'); // reverse parallel const comps = db.connectedComponents(); const map = Object.fromEntries(comps.map(e => [e.nodeId, e.componentId])); assert.equal(map[p1], map[p2]); @@ -69,37 +69,37 @@ describe('connectedComponents (sync)', () => { }); it('isolated nodes are singletons', () => { - const f = db.upsertNode(1, 'f_isolated'); + const f = db.upsertNode('Person', 'f_isolated'); const comps = db.connectedComponents(); const map = Object.fromEntries(comps.map(e => [e.nodeId, e.componentId])); assert.equal(map[f], f); }); - it('respects edge type filter', () => { - const g = db.upsertNode(1, 'g'); - const h = db.upsertNode(1, 'h'); - const i = db.upsertNode(1, 'i'); - db.upsertEdge(g, h, 10); - db.upsertEdge(h, i, 20); + it('respects edge label filter', () => { + const g = db.upsertNode('Person', 'g'); + const h = db.upsertNode('Person', 'h'); + const i = db.upsertNode('Person', 'i'); + db.upsertEdge(g, h, 'WORKS_AT'); + db.upsertEdge(h, i, 'MENTIONS'); - const comps = db.connectedComponents({ edgeTypeFilter: [10] }); + const comps = db.connectedComponents({ edgeLabelFilter: ['WORKS_AT'] }); const map = Object.fromEntries(comps.map(e => [e.nodeId, e.componentId])); - assert.equal(map[g], map[h]); // connected via type 10 - assert.notEqual(map[h], map[i]); // type 20 excluded + assert.equal(map[g], map[h]); // connected via WORKS_AT + assert.notEqual(map[h], map[i]); // MENTIONS excluded }); - it('respects node type filter', () => { - const j = db.upsertNode(1, 'j'); - const k = db.upsertNode(2, 'k'); - const l = db.upsertNode(1, 'l'); - db.upsertEdge(j, k, 10); - db.upsertEdge(k, l, 10); + it('respects node label filter', () => { + const j = db.upsertNode('Person', 'j'); + const k = db.upsertNode('Company', 'k'); + const l = db.upsertNode('Person', 'l'); + db.upsertEdge(j, k, 'WORKS_AT'); + db.upsertEdge(k, l, 'WORKS_AT'); - const comps = db.connectedComponents({ nodeTypeFilter: [1] }); + const comps = db.connectedComponents({ nodeLabelFilter: { labels: ['Person'], mode: 'all' } }); const map = Object.fromEntries(comps.map(e => [e.nodeId, e.componentId])); assert.ok(map[j] !== undefined); assert.ok(map[l] !== undefined); - assert.equal(map[k], undefined); // type 2 filtered out + assert.equal(map[k], undefined); // Company filtered out }); it('returns sorted by nodeId', () => { @@ -124,12 +124,12 @@ describe('componentOf (sync)', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-comp-')); db = freshDb(tmpDir, 'comp'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - d = db.upsertNode(1, 'd'); - db.upsertEdge(a, b, 10); - db.upsertEdge(b, c, 10); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + d = db.upsertNode('Person', 'd'); + db.upsertEdge(a, b, 'WORKS_AT'); + db.upsertEdge(b, c, 'WORKS_AT'); // d is isolated }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -157,34 +157,34 @@ describe('componentOf (sync)', () => { assert.equal(members.length, 0); }); - it('respects edge type filter', () => { - const e = db.upsertNode(1, 'e'); - const f = db.upsertNode(1, 'f'); - db.upsertEdge(e, f, 20); + it('respects edge label filter', () => { + const e = db.upsertNode('Person', 'e'); + const f = db.upsertNode('Person', 'f'); + db.upsertEdge(e, f, 'MENTIONS'); - const members = Array.from(db.componentOf(e, { edgeTypeFilter: [10] })); - // Edge type 20 excluded, so e is isolated. + const members = Array.from(db.componentOf(e, { edgeLabelFilter: ['WORKS_AT'] })); + // MENTIONS excluded, so e is isolated. assert.deepEqual(members, [e]); }); - it('respects node type filter', () => { - const g = db.upsertNode(1, 'g'); - const h = db.upsertNode(2, 'h'); - db.upsertEdge(g, h, 10); + it('respects node label filter', () => { + const g = db.upsertNode('Person', 'g'); + const h = db.upsertNode('Company', 'h'); + db.upsertEdge(g, h, 'WORKS_AT'); - const members = Array.from(db.componentOf(g, { nodeTypeFilter: [1] })); - // h is type 2, filtered out + const members = Array.from(db.componentOf(g, { nodeLabelFilter: { labels: ['Person'], mode: 'all' } })); + // h is Company, filtered out assert.deepEqual(members, [g]); }); - it('returns empty for start node excluded by type filter', () => { - const members = Array.from(db.componentOf(a, { nodeTypeFilter: [99] })); + it('returns empty for start node excluded by label filter', () => { + const members = Array.from(db.componentOf(a, { nodeLabelFilter: { labels: ['MissingLabel'], mode: 'all' } })); assert.equal(members.length, 0); }); it('handles self-loop', () => { - const s = db.upsertNode(1, 'selfloop_comp'); - db.upsertEdge(s, s, 10); + const s = db.upsertNode('Person', 'selfloop_comp'); + db.upsertEdge(s, s, 'WORKS_AT'); const members = Array.from(db.componentOf(s)); assert.deepEqual(members, [s]); }); @@ -200,10 +200,10 @@ describe('connectedComponents (async)', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('finds components async', async () => { - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 10); - const c = db.upsertNode(1, 'c'); // isolated + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'WORKS_AT'); + const c = db.upsertNode('Person', 'c'); // isolated const comps = await db.connectedComponentsAsync(); const map = Object.fromEntries(comps.map(e => [e.nodeId, e.componentId])); @@ -223,9 +223,9 @@ describe('componentOf (async)', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('finds component members async', async () => { - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 10); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'WORKS_AT'); const members = await db.componentOfAsync(a); assert.deepEqual(Array.from(members).sort((x, y) => x - y), [a, b].sort((x, y) => x - y)); diff --git a/overgraph-node/__test__/degree.mjs b/overgraph-node/__test__/degree.mjs index 48d8e00..9e3288b 100644 --- a/overgraph-node/__test__/degree.mjs +++ b/overgraph-node/__test__/degree.mjs @@ -23,11 +23,11 @@ describe('degree (sync)', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-degree-')); db = freshDb(tmpDir, 'deg'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10, { weight: 2.0 }); - db.upsertEdge(a, c, 20, { weight: 3.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 2.0 }); + db.upsertEdge(a, c, 'MENTIONS', { weight: 3.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -47,10 +47,10 @@ describe('degree (sync)', () => { }); it('filters by type', () => { - assert.equal(db.degree(a, { direction: 'outgoing', typeFilter: [10] }), 1); - assert.equal(db.degree(a, { direction: 'outgoing', typeFilter: [20] }), 1); - assert.equal(db.degree(a, { direction: 'outgoing', typeFilter: [10, 20] }), 2); - assert.equal(db.degree(a, { direction: 'outgoing', typeFilter: [99] }), 0); + assert.equal(db.degree(a, { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }), 1); + assert.equal(db.degree(a, { direction: 'outgoing', edgeLabelFilter: ['MENTIONS'] }), 1); + assert.equal(db.degree(a, { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT', 'MENTIONS'] }), 2); + assert.equal(db.degree(a, { direction: 'outgoing', edgeLabelFilter: ['MISSING_EDGE_TYPE'] }), 0); }); it('returns 0 for nonexistent node', () => { @@ -65,11 +65,11 @@ describe('sumEdgeWeights (sync)', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-sum-')); db = freshDb(tmpDir, 'sum'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10, { weight: 2.0 }); - db.upsertEdge(a, c, 10, { weight: 3.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 2.0 }); + db.upsertEdge(a, c, 'WORKS_AT', { weight: 3.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -83,21 +83,21 @@ describe('sumEdgeWeights (sync)', () => { it('filters by type', () => { const db2 = freshDb(tmpDir, 'sum-tf'); - const x = db2.upsertNode(1, 'x'); - const y = db2.upsertNode(1, 'y'); - const z = db2.upsertNode(1, 'z'); - db2.upsertEdge(x, y, 10, { weight: 2.0 }); - db2.upsertEdge(x, z, 20, { weight: 5.0 }); - assert.ok(Math.abs(db2.sumEdgeWeights(x, { direction: 'outgoing', typeFilter: [10] }) - 2.0) < 1e-6); - assert.ok(Math.abs(db2.sumEdgeWeights(x, { direction: 'outgoing', typeFilter: [20] }) - 5.0) < 1e-6); + const x = db2.upsertNode('Person', 'x'); + const y = db2.upsertNode('Person', 'y'); + const z = db2.upsertNode('Person', 'z'); + db2.upsertEdge(x, y, 'WORKS_AT', { weight: 2.0 }); + db2.upsertEdge(x, z, 'MENTIONS', { weight: 5.0 }); + assert.ok(Math.abs(db2.sumEdgeWeights(x, { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }) - 2.0) < 1e-6); + assert.ok(Math.abs(db2.sumEdgeWeights(x, { direction: 'outgoing', edgeLabelFilter: ['MENTIONS'] }) - 5.0) < 1e-6); db2.close(); }); it('respects at_epoch', () => { const db2 = freshDb(tmpDir, 'sum-ep'); - const x = db2.upsertNode(1, 'x'); - const y = db2.upsertNode(1, 'y'); - db2.upsertEdge(x, y, 10, { weight: 3.0, validFrom: 100, validTo: 200 }); + const x = db2.upsertNode('Person', 'x'); + const y = db2.upsertNode('Person', 'y'); + db2.upsertEdge(x, y, 'WORKS_AT', { weight: 3.0, validFrom: 100, validTo: 200 }); assert.ok(Math.abs(db2.sumEdgeWeights(x, { direction: 'outgoing', atEpoch: 150 }) - 3.0) < 1e-6); assert.equal(db2.sumEdgeWeights(x, { direction: 'outgoing', atEpoch: 250 }), 0.0); db2.close(); @@ -111,11 +111,11 @@ describe('avgEdgeWeight (sync)', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-avg-')); db = freshDb(tmpDir, 'avg'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10, { weight: 2.0 }); - db.upsertEdge(a, c, 10, { weight: 4.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 2.0 }); + db.upsertEdge(a, c, 'WORKS_AT', { weight: 4.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -131,12 +131,12 @@ describe('avgEdgeWeight (sync)', () => { it('filters by type', () => { const db2 = freshDb(tmpDir, 'avg-tf'); - const x = db2.upsertNode(1, 'x'); - const y = db2.upsertNode(1, 'y'); - const z = db2.upsertNode(1, 'z'); - db2.upsertEdge(x, y, 10, { weight: 2.0 }); - db2.upsertEdge(x, z, 20, { weight: 6.0 }); - const avg = db2.avgEdgeWeight(x, { direction: 'outgoing', typeFilter: [10] }); + const x = db2.upsertNode('Person', 'x'); + const y = db2.upsertNode('Person', 'y'); + const z = db2.upsertNode('Person', 'z'); + db2.upsertEdge(x, y, 'WORKS_AT', { weight: 2.0 }); + db2.upsertEdge(x, z, 'MENTIONS', { weight: 6.0 }); + const avg = db2.avgEdgeWeight(x, { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }); assert.ok(avg !== null); assert.ok(Math.abs(avg - 2.0) < 1e-6); db2.close(); @@ -144,9 +144,9 @@ describe('avgEdgeWeight (sync)', () => { it('respects at_epoch', () => { const db2 = freshDb(tmpDir, 'avg-ep'); - const x = db2.upsertNode(1, 'x'); - const y = db2.upsertNode(1, 'y'); - db2.upsertEdge(x, y, 10, { weight: 4.0, validFrom: 100, validTo: 200 }); + const x = db2.upsertNode('Person', 'x'); + const y = db2.upsertNode('Person', 'y'); + db2.upsertEdge(x, y, 'WORKS_AT', { weight: 4.0, validFrom: 100, validTo: 200 }); const avg = db2.avgEdgeWeight(x, { direction: 'outgoing', atEpoch: 150 }); assert.ok(avg !== null); assert.ok(Math.abs(avg - 4.0) < 1e-6); @@ -162,12 +162,12 @@ describe('degrees batch (sync)', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-degs-')); db = freshDb(tmpDir, 'degs'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10); - db.upsertEdge(a, c, 10); - db.upsertEdge(b, c, 10); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT'); + db.upsertEdge(a, c, 'WORKS_AT'); + db.upsertEdge(b, c, 'WORKS_AT'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -190,12 +190,12 @@ describe('degrees batch (sync)', () => { it('filters by type', () => { const db2 = freshDb(tmpDir, 'degs-tf'); - const x = db2.upsertNode(1, 'x'); - const y = db2.upsertNode(1, 'y'); - const z = db2.upsertNode(1, 'z'); - db2.upsertEdge(x, y, 10); - db2.upsertEdge(x, z, 20); - const results = db2.degrees([x], { direction: 'outgoing', typeFilter: [10] }); + const x = db2.upsertNode('Person', 'x'); + const y = db2.upsertNode('Person', 'y'); + const z = db2.upsertNode('Person', 'z'); + db2.upsertEdge(x, y, 'WORKS_AT'); + db2.upsertEdge(x, z, 'MENTIONS'); + const results = db2.degrees([x], { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }); const degX = results.find(r => r.nodeId === x); assert.ok(degX); assert.equal(degX.degree, 1); @@ -204,9 +204,9 @@ describe('degrees batch (sync)', () => { it('respects at_epoch', () => { const db2 = freshDb(tmpDir, 'degs-ep'); - const x = db2.upsertNode(1, 'x'); - const y = db2.upsertNode(1, 'y'); - db2.upsertEdge(x, y, 10, { weight: 1.0, validFrom: 100, validTo: 200 }); + const x = db2.upsertNode('Person', 'x'); + const y = db2.upsertNode('Person', 'y'); + db2.upsertEdge(x, y, 'WORKS_AT', { weight: 1.0, validFrom: 100, validTo: 200 }); const at150 = db2.degrees([x], { direction: 'outgoing', atEpoch: 150 }); const degAt150 = at150.find(r => r.nodeId === x); assert.ok(degAt150); @@ -223,12 +223,12 @@ describe('degree sidecar persistence (sync)', () => { const dbPath = join(tmpDir, 'sidecar'); let db = OverGraph.open(dbPath); - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - const c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10, { weight: 2.0 }); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + const c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 2.0 }); db.flush(); - db.upsertEdge(a, c, 10, { weight: 4.0 }); + db.upsertEdge(a, c, 'WORKS_AT', { weight: 4.0 }); db.flush(); db.compact(); @@ -252,9 +252,9 @@ describe('degree sidecar persistence (sync)', () => { const dbPath = join(tmpDir, 'corrupt'); let db = OverGraph.open(dbPath); - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 10, { weight: 5.0 }); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 5.0 }); db.flush(); db.close(); @@ -280,12 +280,12 @@ describe('degree matches neighbors length', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-dparity-')); db = freshDb(tmpDir, 'dparity'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10, { weight: 2.0 }); - db.upsertEdge(a, c, 20, { weight: 3.0 }); - db.upsertEdge(b, c, 10, { weight: 1.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 2.0 }); + db.upsertEdge(a, c, 'MENTIONS', { weight: 3.0 }); + db.upsertEdge(b, c, 'WORKS_AT', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -308,9 +308,9 @@ describe('degree async', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-degasync-')); db = freshDb(tmpDir, 'degasync'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 10, { weight: 5.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 5.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -351,20 +351,20 @@ describe('degree temporal', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-degtemporal-')); db = freshDb(tmpDir, 'degtemporal'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('ignores expired edge', () => { const now = Date.now(); - db.upsertEdge(a, b, 10, { weight: 1.0, validFrom: now - 2000, validTo: now - 1000 }); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 1.0, validFrom: now - 2000, validTo: now - 1000 }); assert.equal(db.degree(a), 0); }); it('at_epoch selects valid window', () => { - const c = db.upsertNode(1, 'c'); - db.upsertEdge(a, c, 20, { weight: 1.0, validFrom: 100, validTo: 200 }); + const c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, c, 'MENTIONS', { weight: 1.0, validFrom: 100, validTo: 200 }); assert.equal(db.degree(a, { direction: 'outgoing', atEpoch: 150 }), 1); assert.equal(db.degree(a, { direction: 'outgoing', atEpoch: 250 }), 0); assert.equal(db.degree(a, { direction: 'outgoing', atEpoch: 50 }), 0); diff --git a/overgraph-node/__test__/edge-cases.mjs b/overgraph-node/__test__/edge-cases.mjs index eb2fa53..2d441eb 100644 --- a/overgraph-node/__test__/edge-cases.mjs +++ b/overgraph-node/__test__/edge-cases.mjs @@ -27,14 +27,14 @@ describe('neighbors, decay_lambda', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-decay-')); db = freshDb(tmpDir, 'decay'); - center = db.upsertNode(1, 'center'); - spoke1 = db.upsertNode(1, 'spoke1'); - spoke2 = db.upsertNode(1, 'spoke2'); + center = db.upsertNode('Person', 'center'); + spoke1 = db.upsertNode('Person', 'spoke1'); + spoke2 = db.upsertNode('Person', 'spoke2'); // spoke1: recent edge (1 hour old) - db.upsertEdge(center, spoke1, 10, { weight: 1.0, validFrom: recentFrom }); + db.upsertEdge(center, spoke1, 'WORKS_AT', { weight: 1.0, validFrom: recentFrom }); // spoke2: old edge (100 hours old) - db.upsertEdge(center, spoke2, 10, { weight: 1.0, validFrom: oldFrom }); + db.upsertEdge(center, spoke2, 'WORKS_AT', { weight: 1.0, validFrom: oldFrom }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -78,14 +78,14 @@ describe('neighbors, at_epoch temporal filtering', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-epoch-')); db = freshDb(tmpDir, 'epoch'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); // A->B valid [1000, 5000) - db.upsertEdge(a, b, 10, { weight: 1.0, validFrom: 1000, validTo: 5000 }); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 1.0, validFrom: 1000, validTo: 5000 }); // A->C valid [3000, 9000) - db.upsertEdge(a, c, 10, { weight: 1.0, validFrom: 3000, validTo: 9000 }); + db.upsertEdge(a, c, 'WORKS_AT', { weight: 1.0, validFrom: 3000, validTo: 9000 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -129,8 +129,8 @@ describe('prune, maxAgeMs', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('maxAgeMs prunes nodes older than the threshold', async () => { - const id1 = db.upsertNode(1, 'age-a', { weight: 1.0 }); - const id2 = db.upsertNode(1, 'age-b', { weight: 1.0 }); + const id1 = db.upsertNode('Person', 'age-a', { weight: 1.0 }); + const id2 = db.upsertNode('Person', 'age-b', { weight: 1.0 }); // Wait so nodes are clearly older than the threshold await sleep(50); @@ -142,8 +142,8 @@ describe('prune, maxAgeMs', () => { }); it('maxAgeMs=999999999 keeps everything (nothing is that old)', () => { - const id1 = db.upsertNode(1, 'age-c', { weight: 1.0 }); - const id2 = db.upsertNode(1, 'age-d', { weight: 1.0 }); + const id1 = db.upsertNode('Person', 'age-c', { weight: 1.0 }); + const id2 = db.upsertNode('Person', 'age-d', { weight: 1.0 }); const result = db.prune({ maxAgeMs: 999999999 }); assert.equal(result.nodesPruned, 0); @@ -167,9 +167,9 @@ describe('prune, combined maxAgeMs AND maxWeight', () => { it('only prunes nodes matching BOTH criteria', async () => { // Low weight node (matches weight criterion) - const lowWeight = db.upsertNode(1, 'low-w', { weight: 0.1 }); + const lowWeight = db.upsertNode('Person', 'low-w', { weight: 0.1 }); // High weight node (does not match weight criterion) - const highWeight = db.upsertNode(1, 'high-w', { weight: 0.9 }); + const highWeight = db.upsertNode('Person', 'high-w', { weight: 0.9 }); // Wait so nodes are old enough await sleep(50); @@ -185,7 +185,7 @@ describe('prune, combined maxAgeMs AND maxWeight', () => { }); it('very large maxAgeMs prevents pruning even when weight matches', () => { - const node = db.upsertNode(1, 'combo-safe', { weight: 0.1 }); + const node = db.upsertNode('Person', 'combo-safe', { weight: 0.1 }); // Weight criterion matches (0.1 <= 0.5), but maxAgeMs is huge so node is too young const result = db.prune({ maxAgeMs: 999999999999, maxWeight: 0.5 }); @@ -207,13 +207,13 @@ describe('batchUpsertNodesBinary, format errors', () => { }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('throws on truncated buffer (less than 4 bytes for count)', () => { + it('throws on truncated binary header', () => { assert.throws( () => db.batchUpsertNodesBinary(Buffer.alloc(2)), ); }); - it('throws on absurd count header', () => { + it('throws on invalid binary magic header', () => { assert.throws( () => db.batchUpsertNodesBinary(Buffer.from([0xff, 0xff, 0xff, 0x7f])), ); @@ -232,11 +232,11 @@ describe('personalizedPagerank, dampingFactor edge cases', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-ppr-damp-')); db = freshDb(tmpDir, 'ppr-damp'); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); - db.upsertEdge(b, c, 1, { weight: 1.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); + db.upsertEdge(b, c, 'LINKS_TO', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -312,27 +312,27 @@ describe('personalizedPagerank, dampingFactor edge cases', () => { // 7. Pagination cursor tampering // ============================================================================= -describe('nodesByTypePaged, cursor edge cases', () => { +describe('nodesByLabelsPaged, cursor edge cases', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-cursor-')); db = freshDb(tmpDir, 'cursor'); for (let i = 0; i < 10; i++) { - db.upsertNode(1, `node-${i}`); + db.upsertNode('Person', `node-${i}`); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('cursor past all IDs returns empty result', () => { - const page = db.nodesByTypePaged(1, 3, 999999999); + const page = db.nodesByLabelsPaged('Person', 3, 999999999); assert.equal(page.items.length, 0); assert.equal(page.nextCursor, undefined); }); it('cursor=0 works like no cursor (returns first page)', () => { - const pageNoCursor = db.nodesByTypePaged(1, 3); - const pageCursor0 = db.nodesByTypePaged(1, 3, 0); + const pageNoCursor = db.nodesByLabelsPaged('Person', 3); + const pageCursor0 = db.nodesByLabelsPaged('Person', 3, 0); assert.equal(pageCursor0.items.length, pageNoCursor.items.length); // Both should return the same set of IDs @@ -353,8 +353,8 @@ describe('self-loops', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-selfloop-')); db = freshDb(tmpDir, 'selfloop'); - a = db.upsertNode(1, 'self'); - db.upsertEdge(a, a, 10, { weight: 1.0 }); + a = db.upsertNode('Person', 'self'); + db.upsertEdge(a, a, 'WORKS_AT', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -394,7 +394,7 @@ describe('concurrent async stress', () => { it('50 parallel upsertNodeAsync calls all succeed', async () => { const promises = []; for (let i = 0; i < 50; i++) { - promises.push(db.upsertNodeAsync(1, `stress-${i}`, { props: { idx: i } })); + promises.push(db.upsertNodeAsync('Person', `stress-${i}`, { props: { idx: i } })); } const ids = await Promise.all(promises); @@ -413,7 +413,7 @@ describe('concurrent async stress', () => { for (const id of ids) { const node = db.getNode(id); assert.ok(node, `node ${id} should be retrievable`); - assert.equal(node.typeId, 1); + assert.ok(node.labels.includes('Person')); } }); }); diff --git a/overgraph-node/__test__/export.mjs b/overgraph-node/__test__/export.mjs index b5165ec..b19ea3f 100644 --- a/overgraph-node/__test__/export.mjs +++ b/overgraph-node/__test__/export.mjs @@ -18,7 +18,8 @@ describe('exportAdjacency (sync)', () => { assert.equal(r.nodeIds.length, 0); assert.equal(r.edgeFrom.length, 0); assert.equal(r.edgeTo.length, 0); - assert.equal(r.edgeTypeIds.length, 0); + assert.equal(r.edgeLabels.length, 0); + assert.equal(r.edgeLabelIndexes.length, 0); // Default include_weights=true, so weights typed array is present (but empty) assert.ok(r.edgeWeights != null); assert.equal(r.edgeWeights.length, 0); @@ -30,12 +31,12 @@ describe('exportAdjacency, full graph', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-export-graph-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 1, { weight: 2.0 }); - db.upsertEdge(b, c, 1, { weight: 3.0 }); - db.upsertEdge(c, a, 2, { weight: 1.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 2.0 }); + db.upsertEdge(b, c, 'LINKS_TO', { weight: 3.0 }); + db.upsertEdge(c, a, 'REFERENCES', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -44,7 +45,8 @@ describe('exportAdjacency, full graph', () => { assert.equal(r.nodeIds.length, 3); assert.equal(r.edgeFrom.length, 3); assert.equal(r.edgeTo.length, 3); - assert.equal(r.edgeTypeIds.length, 3); + assert.equal(r.edgeLabels.length, 2); + assert.equal(r.edgeLabelIndexes.length, 3); assert.ok(r.edgeWeights !== null); assert.equal(r.edgeWeights.length, 3); }); @@ -62,7 +64,7 @@ describe('exportAdjacency, full graph', () => { let found = false; for (let i = 0; i < r.edgeFrom.length; i++) { if (r.edgeFrom[i] === a && r.edgeTo[i] === b) { - assert.equal(r.edgeTypeIds[i], 1); + assert.equal(r.edgeLabels[r.edgeLabelIndexes[i]], 'LINKS_TO'); assert.ok(Math.abs(r.edgeWeights[i] - 2.0) < 1e-6); found = true; } @@ -76,28 +78,31 @@ describe('exportAdjacency, filters', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-export-filter-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(2, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); - db.upsertEdge(a, c, 1, { weight: 1.0 }); - db.upsertEdge(a, c, 2, { weight: 2.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Company', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); + db.upsertEdge(a, c, 'LINKS_TO', { weight: 1.0 }); + db.upsertEdge(a, c, 'REFERENCES', { weight: 2.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('node type filter excludes nodes and their edges', () => { - const r = db.exportAdjacency({ nodeTypeFilter: [1], includeWeights: true }); + it('node label filter excludes nodes and their edges', () => { + const r = db.exportAdjacency({ + nodeLabelFilter: { labels: ['Person'], mode: 'all' }, + includeWeights: true, + }); assert.equal(r.nodeIds.length, 2); // a and c - // Edge a→b excluded (b is type 2) + // Edge a->b excluded because b is Company. for (let i = 0; i < r.edgeFrom.length; i++) { assert.notEqual(r.edgeTo[i], b); } }); - it('edge type filter restricts edges', () => { - const r = db.exportAdjacency({ edgeTypeFilter: [2], includeWeights: true }); + it('edge label filter restricts edges', () => { + const r = db.exportAdjacency({ edgeLabelFilter: ['REFERENCES'], includeWeights: true }); assert.equal(r.edgeFrom.length, 1); - assert.equal(r.edgeTypeIds[0], 2); + assert.equal(r.edgeLabels[r.edgeLabelIndexes[0]], 'REFERENCES'); }); it('includeWeights false gives null weights', () => { @@ -111,12 +116,12 @@ describe('exportAdjacency, across flush', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-export-flush-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); db.flush(); - const c = db.upsertNode(1, 'c'); - db.upsertEdge(b, c, 1, { weight: 2.0 }); + const c = db.upsertNode('Person', 'c'); + db.upsertEdge(b, c, 'LINKS_TO', { weight: 2.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -132,9 +137,9 @@ describe('exportAdjacencyAsync', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-export-async-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -143,7 +148,8 @@ describe('exportAdjacencyAsync', () => { assert.equal(r.nodeIds.length, 2); assert.equal(r.edgeFrom.length, 1); assert.equal(r.edgeTo.length, 1); - assert.equal(r.edgeTypeIds.length, 1); + assert.equal(r.edgeLabels.length, 1); + assert.equal(r.edgeLabelIndexes.length, 1); assert.ok(r.edgeWeights !== null); }); }); diff --git a/overgraph-node/__test__/integration-lifecycle.mjs b/overgraph-node/__test__/integration-lifecycle.mjs index 113ed6c..ef6f965 100644 --- a/overgraph-node/__test__/integration-lifecycle.mjs +++ b/overgraph-node/__test__/integration-lifecycle.mjs @@ -40,9 +40,9 @@ describe('Full lifecycle integration', () => { const nodes = []; for (let i = 0; i < 500; i++) { - const typeId = (i % 5) + 1; // types 1-5 + const label = ['Person', 'Company', 'Document', 'Post', 'User'][i % 5]; nodes.push({ - typeId, + labels: [label], key: `node-${i}`, props: { idx: i, label: `Node ${i}`, active: i % 3 !== 0 }, weight: (i % 10) / 10.0, @@ -72,7 +72,7 @@ describe('Full lifecycle integration', () => { edges.push({ from: nodeIds[fromIdx], to: nodeIds[toIdx], - typeId: (i % 3) + 10, // edge types 10, 11, 12 + label: ['WORKS_AT', 'MENTIONS', 'REFERENCES'][i % 3], props: { order: i }, weight: 1.0 + (i % 5) * 0.1, }); @@ -97,7 +97,7 @@ describe('Full lifecycle integration', () => { assert.ok(nbrs.length > 0, 'node 0 should have outgoing neighbors'); // Neighbors with type filter - const filtered = await db.neighborsAsync(nodeIds[0], { direction: 'outgoing', typeFilter: [10] }); + const filtered = await db.neighborsAsync(nodeIds[0], { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }); assert.ok(filtered.length <= nbrs.length); // Neighbors with limit @@ -134,19 +134,19 @@ describe('Full lifecycle integration', () => { // Find all type-1 nodes where active=true // Type 1 nodes: indices 0, 5, 10, ... (every 5th). Active: idx % 3 !== 0 - const activeType1 = await db.findNodesAsync(1, 'active', true); + const activeType1 = await db.findNodesAsync('Person', 'active', true); assert.ok(activeType1.length > 0); // Verify they're actually active type-1 nodes for (const id of activeType1) { const n = await db.getNodeAsync(id); assert.ok(n); - assert.equal(n.typeId, 1); + assert.ok(n.labels.includes('Person')); assert.equal(n.props.active, true); } // Find by string property - const specific = await db.findNodesAsync(1, 'label', 'Node 0'); + const specific = await db.findNodesAsync('Person', 'label', 'Node 0'); assert.equal(specific.length, 1); db.close(); @@ -178,7 +178,7 @@ describe('Full lifecycle integration', () => { // Insert a few more nodes to create a second segment for (let i = 0; i < 20; i++) { - db.upsertNode(1, `extra-${i}`); + db.upsertNode('Person', `extra-${i}`); } await db.flushAsync(); @@ -215,7 +215,7 @@ describe('Full lifecycle integration', () => { assert.ok(nbrs.length > 0, 'neighbors should work after reopen'); // Verify find_nodes still works after compaction + reopen - const found = await db.findNodesAsync(1, 'label', 'Node 0'); + const found = await db.findNodesAsync('Person', 'label', 'Node 0'); assert.equal(found.length, 1); // Verify edge data survived diff --git a/overgraph-node/__test__/integration-nonblocking.mjs b/overgraph-node/__test__/integration-nonblocking.mjs index e8fc45e..e6e1dba 100644 --- a/overgraph-node/__test__/integration-nonblocking.mjs +++ b/overgraph-node/__test__/integration-nonblocking.mjs @@ -20,7 +20,7 @@ describe('Async non-blocking verification', () => { // Pre-populate with enough data to make operations take measurable time for (let i = 0; i < 500; i++) { - db.upsertNode(1, `node-${i}`, { props: { + db.upsertNode('Person', `node-${i}`, { props: { payload: 'x'.repeat(200), idx: i, tags: ['a', 'b', 'c'], @@ -28,9 +28,9 @@ describe('Async non-blocking verification', () => { } // Create edges for (let i = 0; i < 400; i++) { - const from = db.upsertNode(1, `node-${i}`); - const to = db.upsertNode(1, `node-${(i + 1) % 500}`); - db.upsertEdge(from, to, 1, { props: { order: i } }); + const from = db.upsertNode('Person', `node-${i}`); + const to = db.upsertNode('Person', `node-${(i + 1) % 500}`); + db.upsertEdge(from, to, 'LINKS_TO', { props: { order: i } }); } }); @@ -64,7 +64,7 @@ describe('Async non-blocking verification', () => { it('event loop stays responsive during async compact', async () => { // Insert more data and flush again to create a second segment for (let i = 500; i < 800; i++) { - db.upsertNode(1, `node-${i}`, { props: { payload: 'y'.repeat(200) } }); + db.upsertNode('Person', `node-${i}`, { props: { payload: 'y'.repeat(200) } }); } await db.flushAsync(); @@ -85,9 +85,9 @@ describe('Async non-blocking verification', () => { // Fire multiple async operations at once. They should all resolve const results = await Promise.all([ db.getNodeAsync(1), - db.findNodesAsync(1, 'idx', 0), + db.findNodesAsync('Person', 'idx', 0), db.neighborsAsync(1, { direction: 'outgoing' }), - db.upsertNodeAsync(99, 'concurrent-test', { props: { ts: Date.now() } }), + db.upsertNodeAsync('MissingLabel', 'concurrent-test', { props: { ts: Date.now() } }), ]); // getNode @@ -102,10 +102,10 @@ describe('Async non-blocking verification', () => { it('async and sync can interleave without corruption', async () => { // Mix sync writes with async reads - const id1 = db.upsertNode(50, 'sync-write', { props: { val: 1 } }); + const id1 = db.upsertNode('SyncWrite', 'sync-write', { props: { val: 1 } }); const asyncRead = db.getNodeAsync(id1); - const id2 = db.upsertNode(50, 'sync-write-2', { props: { val: 2 } }); + const id2 = db.upsertNode('SyncWrite', 'sync-write-2', { props: { val: 2 } }); const node = await asyncRead; assert.ok(node); diff --git a/overgraph-node/__test__/neighbors-batch.mjs b/overgraph-node/__test__/neighbors-batch.mjs index 8fc8733..193cb58 100644 --- a/overgraph-node/__test__/neighbors-batch.mjs +++ b/overgraph-node/__test__/neighbors-batch.mjs @@ -17,13 +17,13 @@ describe('neighborsBatch (sync)', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-nbatch-')); db = freshDb(tmpDir, 'nbatch'); // Build a small graph: n1->n2 (type 10), n1->n3 (type 20), n2->n4 (type 10) - n1 = db.upsertNode(1, 'n1'); - n2 = db.upsertNode(1, 'n2'); - n3 = db.upsertNode(1, 'n3'); - n4 = db.upsertNode(1, 'n4'); - db.upsertEdge(n1, n2, 10); - db.upsertEdge(n1, n3, 20); - db.upsertEdge(n2, n4, 10); + n1 = db.upsertNode('Person', 'n1'); + n2 = db.upsertNode('Person', 'n2'); + n3 = db.upsertNode('Person', 'n3'); + n4 = db.upsertNode('Person', 'n4'); + db.upsertEdge(n1, n2, 'WORKS_AT'); + db.upsertEdge(n1, n3, 'MENTIONS'); + db.upsertEdge(n2, n4, 'WORKS_AT'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -54,7 +54,7 @@ describe('neighborsBatch (sync)', () => { }); it('respects type filter', () => { - const results = db.neighborsBatch([n1], { direction: 'outgoing', typeFilter: [10] }); + const results = db.neighborsBatch([n1], { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }); assert.equal(results.length, 1); // n1 has 1 outgoing edge of type 10 (to n2) assert.equal(results[0].neighbors.length, 1); @@ -68,7 +68,7 @@ describe('neighborsBatch (sync)', () => { }); it('handles node with no neighbors', () => { - const lonely = db.upsertNode(1, 'lonely'); + const lonely = db.upsertNode('Person', 'lonely'); const results = db.neighborsBatch([lonely]); // Engine filters empty entries: nodes with no neighbors are omitted assert.equal(results.length, 0); @@ -100,10 +100,10 @@ describe('neighborsBatch, temporal (at_epoch)', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-nbatch-temp-')); db = freshDb(tmpDir, 'nbatch-temp'); - n1 = db.upsertNode(1, 'a'); - n2 = db.upsertNode(1, 'b'); + n1 = db.upsertNode('Person', 'a'); + n2 = db.upsertNode('Person', 'b'); // Edge valid from 1000 to 2000 - db.upsertEdge(n1, n2, 10, { weight: 1.0, validFrom: 1000, validTo: 2000 }); + db.upsertEdge(n1, n2, 'WORKS_AT', { weight: 1.0, validFrom: 1000, validTo: 2000 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -126,11 +126,11 @@ describe('neighborsBatchAsync', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-nbatch-async-')); db = freshDb(tmpDir, 'nbatch-async'); - n1 = db.upsertNode(1, 'x1'); - n2 = db.upsertNode(1, 'x2'); - n3 = db.upsertNode(1, 'x3'); - db.upsertEdge(n1, n2, 10); - db.upsertEdge(n1, n3, 10); + n1 = db.upsertNode('Person', 'x1'); + n2 = db.upsertNode('Person', 'x2'); + n3 = db.upsertNode('Person', 'x3'); + db.upsertEdge(n1, n2, 'WORKS_AT'); + db.upsertEdge(n1, n3, 'WORKS_AT'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); diff --git a/overgraph-node/__test__/pagination.mjs b/overgraph-node/__test__/pagination.mjs index 7533e7f..6c1405f 100644 --- a/overgraph-node/__test__/pagination.mjs +++ b/overgraph-node/__test__/pagination.mjs @@ -21,89 +21,89 @@ function collectIdPages(db, method, ...args) { return allItems; } -describe('nodesByTypePaged', () => { +describe('nodesByLabelsPaged', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); // Create 10 nodes of type 1 for (let i = 0; i < 10; i++) { - db.upsertNode(1, `node-${i}`); + db.upsertNode('Person', `node-${i}`); } // 3 nodes of type 2 for (let i = 0; i < 3; i++) { - db.upsertNode(2, `other-${i}`); + db.upsertNode('Company', `other-${i}`); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns first page with correct size', () => { - const page = db.nodesByTypePaged(1, 3); + const page = db.nodesByLabelsPaged('Person', 3); assert.equal(page.items.length, 3); assert.ok(page.nextCursor !== null && page.nextCursor !== undefined); assert.equal(typeof page.nextCursor, 'number'); }); it('round-trip pagination collects all items', () => { - const all = collectIdPages(db, 'nodesByTypePaged', 1, 3); + const all = collectIdPages(db, 'nodesByLabelsPaged', 'Person', 3); assert.equal(all.length, 10); }); it('no limit returns all', () => { - const page = db.nodesByTypePaged(1); + const page = db.nodesByLabelsPaged('Person'); assert.equal(page.items.length, 10); assert.equal(page.nextCursor, undefined); }); it('empty type returns empty page', () => { - const page = db.nodesByTypePaged(999, 5); + const page = db.nodesByLabelsPaged('UnusedLabel', 5); assert.equal(page.items.length, 0); assert.equal(page.nextCursor, undefined); }); }); -describe('edgesByTypePaged', () => { +describe('edgesByLabelPaged', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); const nodes = []; for (let i = 0; i < 6; i++) { - nodes.push(db.upsertNode(1, `n${i}`)); + nodes.push(db.upsertNode('Person', `n${i}`)); } // 5 edges of type 10 for (let i = 0; i < 5; i++) { - db.upsertEdge(nodes[i], nodes[i + 1], 10); + db.upsertEdge(nodes[i], nodes[i + 1], 'WORKS_AT'); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('round-trip pagination', () => { - const all = collectIdPages(db, 'edgesByTypePaged', 10, 2); + const all = collectIdPages(db, 'edgesByLabelPaged', 'WORKS_AT', 2); assert.equal(all.length, 5); }); }); -describe('getNodesByTypePaged', () => { +describe('getNodesByLabelsPaged', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); for (let i = 0; i < 8; i++) { - db.upsertNode(1, `node-${i}`, { props: { idx: i } }); + db.upsertNode('Person', `node-${i}`, { props: { idx: i } }); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns hydrated records with pagination', () => { - const page = db.getNodesByTypePaged(1, 3); + const page = db.getNodesByLabelsPaged('Person', 3); assert.equal(page.items.length, 3); // Each item should be a full record for (const item of page.items) { assert.equal(typeof item.id, 'number'); assert.equal(typeof item.key, 'string'); assert.ok(item.props !== undefined); - assert.equal(item.typeId, 1); + assert.ok(item.labels.includes('Person')); } assert.ok(page.nextCursor != null); }); @@ -112,7 +112,7 @@ describe('getNodesByTypePaged', () => { const allItems = []; let cursor = null; for (;;) { - const page = db.getNodesByTypePaged(1, 3, cursor); + const page = db.getNodesByLabelsPaged('Person', 3, cursor); allItems.push(...page.items); cursor = page.nextCursor; if (cursor == null) break; @@ -124,25 +124,25 @@ describe('getNodesByTypePaged', () => { }); }); -describe('getEdgesByTypePaged', () => { +describe('getEdgesByLabelPaged', () => { let tmpDir, db; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - const n1 = db.upsertNode(1, 'a'); - const n2 = db.upsertNode(1, 'b'); + const n1 = db.upsertNode('Person', 'a'); + const n2 = db.upsertNode('Person', 'b'); for (let i = 0; i < 6; i++) { - db.upsertEdge(n1, n2, 10, { weight: 0.5 + i * 0.1 }); + db.upsertEdge(n1, n2, 'WORKS_AT', { weight: 0.5 + i * 0.1 }); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns hydrated edge records with cursor', () => { - const page = db.getEdgesByTypePaged(10, 2); + const page = db.getEdgesByLabelPaged('WORKS_AT', 2); assert.equal(page.items.length, 2); for (const item of page.items) { assert.equal(typeof item.id, 'number'); - assert.equal(item.typeId, 10); + assert.equal(item.label, 'WORKS_AT'); } assert.ok(page.nextCursor != null); }); @@ -154,9 +154,9 @@ describe('findNodesPaged', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); for (let i = 0; i < 7; i++) { - db.upsertNode(1, `node-${i}`, { props: { color: 'red' } }); + db.upsertNode('Person', `node-${i}`, { props: { color: 'red' } }); } - db.upsertNode(1, 'blue-node', { props: { color: 'blue' } }); + db.upsertNode('Person', 'blue-node', { props: { color: 'blue' } }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -164,7 +164,7 @@ describe('findNodesPaged', () => { const allItems = []; let cursor = null; for (;;) { - const page = db.findNodesPaged(1, 'color', 'red', { limit: 3, after: cursor ?? undefined }); + const page = db.findNodesPaged('Person', 'color', 'red', { limit: 3, after: cursor ?? undefined }); for (let i = 0; i < page.items.length; i++) { allItems.push(page.items[i]); } @@ -175,7 +175,7 @@ describe('findNodesPaged', () => { }); it('no matches returns empty', () => { - const page = db.findNodesPaged(1, 'color', 'green', { limit: 10 }); + const page = db.findNodesPaged('Person', 'color', 'green', { limit: 10 }); assert.equal(page.items.length, 0); assert.equal(page.nextCursor, undefined); }); @@ -186,10 +186,10 @@ describe('neighborsPaged', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - center = db.upsertNode(1, 'center'); + center = db.upsertNode('Person', 'center'); for (let i = 0; i < 12; i++) { - const n = db.upsertNode(1, `n${i}`); - db.upsertEdge(center, n, 10); + const n = db.upsertNode('Person', `n${i}`); + db.upsertEdge(center, n, 'WORKS_AT'); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -199,7 +199,7 @@ describe('neighborsPaged', () => { assert.equal(page.items.length, 4); assert.equal(typeof page.items[0].nodeId, 'number'); assert.equal(typeof page.items[0].edgeId, 'number'); - assert.equal(typeof page.items[0].edgeTypeId, 'number'); + assert.equal(typeof page.items[0].label, 'string'); assert.equal(typeof page.items[0].weight, 'number'); assert.ok(page.nextCursor != null); assert.equal(typeof page.nextCursor, 'number'); @@ -235,13 +235,13 @@ describe('traverse pagination', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); // a -> b1, b2, b3; each bi -> ci1, ci2 (6 2-hop targets) - a = db.upsertNode(1, 'a'); + a = db.upsertNode('Person', 'a'); for (let i = 0; i < 3; i++) { - const b = db.upsertNode(1, `b${i}`); - db.upsertEdge(a, b, 10); + const b = db.upsertNode('Person', `b${i}`); + db.upsertEdge(a, b, 'WORKS_AT'); for (let j = 0; j < 2; j++) { - const c = db.upsertNode(1, `c${i}_${j}`); - db.upsertEdge(b, c, 10); + const c = db.upsertNode('Person', `c${i}_${j}`); + db.upsertEdge(b, c, 'WORKS_AT'); } } }); @@ -284,32 +284,36 @@ describe('traverse pagination', () => { }); }); -describe('traverse node-type filtering', () => { +describe('traverse node-label filtering', () => { let tmpDir, db, a; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 10); - // 2-hop targets: 3 of type 2, 2 of type 3 + a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'WORKS_AT'); + // 2-hop targets: 3 Company nodes, 2 Document nodes for (let i = 0; i < 3; i++) { - const c = db.upsertNode(2, `c-type2-${i}`); - db.upsertEdge(b, c, 10); + const c = db.upsertNode('Company', `c-type2-${i}`); + db.upsertEdge(b, c, 'WORKS_AT'); } for (let i = 0; i < 2; i++) { - const d = db.upsertNode(3, `d-type3-${i}`); - db.upsertEdge(b, d, 10); + const d = db.upsertNode('Document', `d-type3-${i}`); + db.upsertEdge(b, d, 'WORKS_AT'); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('filters by target node type with pagination', () => { + it('filters by target node label with pagination', () => { const allNodeIds = []; let cursor = null; for (;;) { const page = db.traverse(a, 2, { - minDepth: 2, direction: 'outgoing', nodeTypeFilter: [2], limit: 2, cursor: cursor ?? undefined + minDepth: 2, + direction: 'outgoing', + emitNodeLabelFilter: { labels: ['Company'], mode: 'all' }, + limit: 2, + cursor: cursor ?? undefined, }); allNodeIds.push(...page.items.map(hit => hit.nodeId)); cursor = page.nextCursor ?? null; @@ -329,26 +333,26 @@ describe('pagination cross-segment', () => { }); // Create nodes in two segments for (let i = 0; i < 5; i++) { - db.upsertNode(1, `seg1-${i}`); + db.upsertNode('Person', `seg1-${i}`); } db.flush(); for (let i = 0; i < 5; i++) { - db.upsertNode(1, `seg2-${i}`); + db.upsertNode('Person', `seg2-${i}`); } // 5 in memtable, 5 in segment }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('nodesByTypePaged merges across memtable and segment', () => { - const all = collectIdPages(db, 'nodesByTypePaged', 1, 3); + it('nodesByLabelsPaged merges across memtable and segment', () => { + const all = collectIdPages(db, 'nodesByLabelsPaged', 'Person', 3); assert.equal(all.length, 10); }); - it('getNodesByTypePaged merges across sources', () => { + it('getNodesByLabelsPaged merges across sources', () => { const allItems = []; let cursor = null; for (;;) { - const page = db.getNodesByTypePaged(1, 4, cursor); + const page = db.getNodesByLabelsPaged('Person', 4, cursor); allItems.push(...page.items); cursor = page.nextCursor ?? null; if (cursor === null || cursor === undefined) break; @@ -365,22 +369,22 @@ describe('pagination async variants', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); for (let i = 0; i < 6; i++) { - db.upsertNode(1, `n${i}`, { props: { tag: 'a' } }); + db.upsertNode('Person', `n${i}`, { props: { tag: 'a' } }); } - const n0 = db.findNodes(1, 'tag', 'a')[0]; - const n1 = db.findNodes(1, 'tag', 'a')[1]; - db.upsertEdge(n0, n1, 10); + const n0 = db.findNodes('Person', 'tag', 'a')[0]; + const n1 = db.findNodes('Person', 'tag', 'a')[1]; + db.upsertEdge(n0, n1, 'WORKS_AT'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('nodesByTypePagedAsync returns promise', async () => { - const page = await db.nodesByTypePagedAsync(1, 2); + it('nodesByLabelsPagedAsync returns promise', async () => { + const page = await db.nodesByLabelsPagedAsync('Person', 2); assert.equal(page.items.length, 2); assert.ok(page.nextCursor !== undefined); }); - it('getNodesByTypePagedAsync returns hydrated records', async () => { - const page = await db.getNodesByTypePagedAsync(1, 3); + it('getNodesByLabelsPagedAsync returns hydrated records', async () => { + const page = await db.getNodesByLabelsPagedAsync('Person', 3); assert.equal(page.items.length, 3); for (const item of page.items) { assert.equal(typeof item.id, 'number'); @@ -389,31 +393,33 @@ describe('pagination async variants', () => { }); it('findNodesPagedAsync works', async () => { - const page = await db.findNodesPagedAsync(1, 'tag', 'a', { limit: 4 }); + const page = await db.findNodesPagedAsync('Person', 'tag', 'a', { limit: 4 }); assert.equal(page.items.length, 4); assert.ok(page.nextCursor !== undefined); }); - it('edgesByTypePagedAsync works', async () => { - const page = await db.edgesByTypePagedAsync(10, 10); + it('edgesByLabelPagedAsync works', async () => { + const page = await db.edgesByLabelPagedAsync('WORKS_AT', 10); assert.equal(page.items.length, 1); assert.equal(page.nextCursor, undefined); }); it('neighborsPagedAsync returns array result', async () => { - const page = await db.neighborsPagedAsync(db.findNodes(1, 'tag', 'a')[0], { direction: 'outgoing', limit: 10 }); + const page = await db.neighborsPagedAsync(db.findNodes('Person', 'tag', 'a')[0], { direction: 'outgoing', limit: 10 }); assert.equal(typeof page.items.length, 'number'); assert.equal(page.nextCursor, null); }); it('traverseAsync paginates exact-depth traversal hits', async () => { - const page = await db.traverseAsync(db.findNodes(1, 'tag', 'a')[0], 2, { direction: 'outgoing', limit: 10 }); + const page = await db.traverseAsync(db.findNodes('Person', 'tag', 'a')[0], 2, { direction: 'outgoing', limit: 10 }); assert.equal(typeof page.items.length, 'number'); }); - it('traverseAsync accepts node type filters and traversal cursor args', async () => { - const page = await db.traverseAsync(db.findNodes(1, 'tag', 'a')[0], 2, { - direction: 'outgoing', nodeTypeFilter: [1], limit: 10 + it('traverseAsync accepts node label filters and traversal cursor args', async () => { + const page = await db.traverseAsync(db.findNodes('Person', 'tag', 'a')[0], 2, { + direction: 'outgoing', + emitNodeLabelFilter: { labels: ['Person'], mode: 'all' }, + limit: 10, }); assert.equal(typeof page.items.length, 'number'); }); @@ -425,13 +431,13 @@ describe('pagination single-item pages', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); for (let i = 0; i < 4; i++) { - db.upsertNode(1, `n${i}`); + db.upsertNode('Person', `n${i}`); } }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('page size 1 yields one item per page', () => { - const all = collectIdPages(db, 'nodesByTypePaged', 1, 1); + const all = collectIdPages(db, 'nodesByLabelsPaged', 'Person', 1); assert.equal(all.length, 4); }); }); @@ -441,20 +447,20 @@ describe('pagination cursor at end', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-page-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - db.upsertNode(1, 'only'); + db.upsertNode('Person', 'only'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('cursor past all IDs returns empty page', () => { // Get the one item, then use its cursor - const p1 = db.nodesByTypePaged(1, 1); + const p1 = db.nodesByLabelsPaged('Person', 1); assert.equal(p1.items.length, 1); // nextCursor should be undefined since there's only one item assert.equal(p1.nextCursor, undefined); }); it('large cursor returns empty', () => { - const page = db.nodesByTypePaged(1, 10, 9999999999999); + const page = db.nodesByLabelsPaged('Person', 10, 9999999999999); assert.equal(page.items.length, 0); assert.equal(page.nextCursor, undefined); }); @@ -467,7 +473,7 @@ describe('pagination with deletions', () => { db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); const ids = []; for (let i = 0; i < 5; i++) { - ids.push(db.upsertNode(1, `n${i}`)); + ids.push(db.upsertNode('Person', `n${i}`)); } // Delete node at index 2 db.deleteNode(ids[2]); @@ -475,7 +481,7 @@ describe('pagination with deletions', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('deleted nodes excluded from paginated results', () => { - const all = collectIdPages(db, 'nodesByTypePaged', 1, 2); + const all = collectIdPages(db, 'nodesByLabelsPaged', 'Person', 2); assert.equal(all.length, 4); // 5 - 1 deleted }); }); diff --git a/overgraph-node/__test__/ppr.mjs b/overgraph-node/__test__/ppr.mjs index 5e8418a..2e22f07 100644 --- a/overgraph-node/__test__/ppr.mjs +++ b/overgraph-node/__test__/ppr.mjs @@ -22,7 +22,7 @@ describe('personalizedPagerank (sync)', () => { }); it('single seed no edges returns seed with rank 1.0', () => { - const id = db.upsertNode(1, 'lonely'); + const id = db.upsertNode('Person', 'lonely'); const r = db.personalizedPagerank([id]); assert.equal(r.nodeIds.length, 1); assert.equal(r.nodeIds[0], id); @@ -31,7 +31,7 @@ describe('personalizedPagerank (sync)', () => { }); it('approx mode returns algorithm metadata', () => { - const id = db.upsertNode(1, 'approx-seed'); + const id = db.upsertNode('Person', 'approx-seed'); const r = db.personalizedPagerank([id], { algorithm: 'approx', approxResidualTolerance: 1e-6, @@ -48,12 +48,12 @@ describe('personalizedPagerank, graph queries', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-ppr-graph-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); - db.upsertEdge(b, c, 1, { weight: 1.0 }); - db.upsertEdge(c, a, 1, { weight: 1.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); + db.upsertEdge(b, c, 'LINKS_TO', { weight: 1.0 }); + db.upsertEdge(c, a, 'LINKS_TO', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -73,10 +73,10 @@ describe('personalizedPagerank, graph queries', () => { assert.ok(r.nodeIds.length <= 2); }); - it('edge type filter restricts walk', () => { + it('edge label filter restricts walk', () => { // Only type-1 edges exist, filter to type-99 should give no neighbors const r = db.personalizedPagerank([a], { - edgeTypeFilter: [99], + edgeLabelFilter: ['MISSING_EDGE_TYPE'], maxIterations: 100, }); // Only seed should have rank (no edges to walk) @@ -90,11 +90,11 @@ describe('personalizedPagerank, weighted edges', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-ppr-weight-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); - db.upsertEdge(a, c, 1, { weight: 9.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); + db.upsertEdge(a, c, 'LINKS_TO', { weight: 9.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -112,12 +112,12 @@ describe('personalizedPagerank across flush', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-ppr-flush-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); db.flush(); - c = db.upsertNode(1, 'c'); - db.upsertEdge(b, c, 1, { weight: 1.0 }); + c = db.upsertNode('Person', 'c'); + db.upsertEdge(b, c, 'LINKS_TO', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -132,9 +132,9 @@ describe('personalizedPagerankAsync', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-ppr-async-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - a = db.upsertNode(1, 'a'); - b = db.upsertNode(1, 'b'); - db.upsertEdge(a, b, 1, { weight: 1.0 }); + a = db.upsertNode('Person', 'a'); + b = db.upsertNode('Person', 'b'); + db.upsertEdge(a, b, 'LINKS_TO', { weight: 1.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); diff --git a/overgraph-node/__test__/property-indexes.mjs b/overgraph-node/__test__/property-indexes.mjs index 035e549..35685bb 100644 --- a/overgraph-node/__test__/property-indexes.mjs +++ b/overgraph-node/__test__/property-indexes.mjs @@ -20,11 +20,32 @@ async function waitForIndexState(db, predicate, expectedState = 'ready', timeout } } +async function waitForEdgeIndexState(db, predicate, expectedState = 'ready', timeoutMs = 5000) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const info = predicate(db.listEdgePropertyIndexes()); + if (info?.state === expectedState) { + return info; + } + if (Date.now() >= deadline) { + throw new Error(`timed out waiting for edge secondary index state '${expectedState}'`); + } + await delay(20); + } +} + +function planHasKind(node, kind) { + if (!node) return false; + if (node.kind === kind) return true; + if (node.input && planHasKind(node.input, kind)) return true; + return Array.isArray(node.inputs) && node.inputs.some(input => planHasKind(input, kind)); +} + async function ensureRangeIndexReady(db, propKey = 'score') { - db.ensureNodePropertyIndex(1, propKey, { kind: 'range', domain: 'int' }); + db.ensureNodePropertyIndex('Person', propKey, { kind: 'range', domain: 'int' }); return waitForIndexState( db, - infos => infos.find(info => info.typeId === 1 && info.propKey === propKey && info.kind === 'range') + infos => infos.find(info => info.label === 'Person' && info.propKey === propKey && info.kind === 'range') ); } @@ -36,7 +57,7 @@ describe('node property index APIs', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-prop-index-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); for (let i = 0; i < 6; i++) { - db.upsertNode(1, `node-${i}`, { + db.upsertNode('Person', `node-${i}`, { props: { color: i % 2 === 0 ? 'red' : 'blue', score: (i + 1) * 10, @@ -52,23 +73,23 @@ describe('node property index APIs', () => { }); it('ensures, lists, and drops declared property indexes', async () => { - const eq = db.ensureNodePropertyIndex(1, 'color', { kind: 'equality' }); + const eq = db.ensureNodePropertyIndex('Person', 'color', { kind: 'equality' }); assert.equal(eq.kind, 'equality'); assert.equal(eq.domain, undefined); assert.equal(eq.state, 'building'); - const range = db.ensureNodePropertyIndex(1, 'score', { kind: 'range', domain: 'int' }); + const range = db.ensureNodePropertyIndex('Person', 'score', { kind: 'range', domain: 'int' }); assert.equal(range.kind, 'range'); assert.equal(range.domain, 'int'); assert.equal(range.state, 'building'); await waitForIndexState( db, - infos => infos.find(info => info.typeId === 1 && info.propKey === 'color' && info.kind === 'equality') + infos => infos.find(info => info.label === 'Person' && info.propKey === 'color' && info.kind === 'equality') ); const readyRange = await waitForIndexState( db, - infos => infos.find(info => info.typeId === 1 && info.propKey === 'score' && info.kind === 'range') + infos => infos.find(info => info.label === 'Person' && info.propKey === 'score' && info.kind === 'range') ); assert.equal(readyRange.domain, 'int'); @@ -83,20 +104,19 @@ describe('node property index APIs', () => { ); assert.throws( - () => db.ensureNodePropertyIndex(1, 'score', { kind: 'range', domain: 'float' }), + () => db.ensureNodePropertyIndex('Person', 'score', { kind: 'range', domain: 'float' }), /different domain/i ); - assert.equal(db.dropNodePropertyIndex(1, 'color', { kind: 'equality' }), true); - assert.equal(db.dropNodePropertyIndex(1, 'color', { kind: 'equality' }), false); + assert.equal(db.dropNodePropertyIndex('Person', 'color', { kind: 'equality' }), true); + assert.equal(db.dropNodePropertyIndex('Person', 'color', { kind: 'equality' }), false); }); it('runs range queries and paging through the public API', async () => { await ensureRangeIndexReady(db); const all = Array.from( - db.findNodesRange( - 1, + db.findNodesRange('Person', 'score', { value: 20, inclusive: true, domain: 'int' }, { value: 50, inclusive: false, domain: 'int' } @@ -104,8 +124,7 @@ describe('node property index APIs', () => { ); assert.equal(all.length, 3); - const first = db.findNodesRangePaged( - 1, + const first = db.findNodesRangePaged('Person', 'score', { value: 20, inclusive: true, domain: 'int' }, { value: 50, inclusive: false, domain: 'int' }, @@ -116,8 +135,7 @@ describe('node property index APIs', () => { assert.equal(typeof first.nextCursor?.value, 'number'); assert.equal(typeof first.nextCursor?.nodeId, 'number'); - const second = db.findNodesRangePaged( - 1, + const second = db.findNodesRangePaged('Person', 'score', { value: 20, inclusive: true, domain: 'int' }, { value: 50, inclusive: false, domain: 'int' }, @@ -127,8 +145,7 @@ describe('node property index APIs', () => { assert.ok(second.nextCursor == null); const fallback = Array.from( - db.findNodesRange( - 1, + db.findNodesRange('Person', 'temp', { value: 10, inclusive: true, domain: 'int' }, { value: 25, inclusive: true, domain: 'int' } @@ -139,25 +156,24 @@ describe('node property index APIs', () => { it('validates kind and domain inputs at the binding boundary', () => { assert.throws( - () => db.ensureNodePropertyIndex(1, 'score', { kind: 'bogus' }), + () => db.ensureNodePropertyIndex('Person', 'score', { kind: 'bogus' }), /Invalid index kind/i ); assert.throws( - () => db.ensureNodePropertyIndex(1, 'score', { kind: 'range' }), + () => db.ensureNodePropertyIndex('Person', 'score', { kind: 'range' }), /Range indexes require domain/i ); assert.throws( - () => db.ensureNodePropertyIndex(1, 'score', { kind: 'equality', domain: 'int' }), + () => db.ensureNodePropertyIndex('Person', 'score', { kind: 'equality', domain: 'int' }), /do not accept a range domain/i ); assert.throws( - () => db.findNodesRange(1, 'score', { value: 10, inclusive: true, domain: 'bogus' }), + () => db.findNodesRange('Person', 'score', { value: 10, inclusive: true, domain: 'bogus' }), /Invalid range domain/i ); assert.throws( () => - db.findNodesRange( - 1, + db.findNodesRange('Person', 'score', { value: 10, inclusive: true, domain: 'int' }, { value: 20, inclusive: true, domain: 'float' } @@ -166,8 +182,7 @@ describe('node property index APIs', () => { ); assert.throws( () => - db.findNodesRangePaged( - 1, + db.findNodesRangePaged('Person', 'score', { value: 10, inclusive: true, domain: 'int' }, { value: 20, inclusive: true, domain: 'int' }, @@ -181,27 +196,25 @@ describe('node property index APIs', () => { }); it('supports async property index and range APIs', async () => { - const asyncEq = await db.ensureNodePropertyIndexAsync(1, 'temp', { kind: 'equality' }); + const asyncEq = await db.ensureNodePropertyIndexAsync('Person', 'temp', { kind: 'equality' }); assert.equal(asyncEq.kind, 'equality'); await waitForIndexState( db, - infos => infos.find(info => info.typeId === 1 && info.propKey === 'temp' && info.kind === 'equality') + infos => infos.find(info => info.label === 'Person' && info.propKey === 'temp' && info.kind === 'equality') ); await ensureRangeIndexReady(db); const listed = await db.listNodePropertyIndexesAsync(); assert.ok(listed.some(info => info.propKey === 'temp' && info.kind === 'equality')); - const ids = await db.findNodesRangeAsync( - 1, + const ids = await db.findNodesRangeAsync('Person', 'score', { value: 20, inclusive: true, domain: 'int' }, { value: 30, inclusive: true, domain: 'int' } ); assert.equal(ids.length, 2); - const page = await db.findNodesRangePagedAsync( - 1, + const page = await db.findNodesRangePagedAsync('Person', 'score', { value: 20, inclusive: true, domain: 'int' }, { value: 40, inclusive: true, domain: 'int' }, @@ -210,6 +223,170 @@ describe('node property index APIs', () => { assert.equal(page.items.length, 2); assert.equal(page.nextCursor?.domain, 'int'); - assert.equal(await db.dropNodePropertyIndexAsync(1, 'temp', { kind: 'equality' }), true); + assert.equal(await db.dropNodePropertyIndexAsync('Person', 'temp', { kind: 'equality' }), true); + }); +}); + +describe('edge property index APIs', () => { + let tmpDir; + let db; + let source; + let hotTarget; + let coldTarget; + let hotEdge; + + before(async () => { + tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-edge-prop-index-')); + db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); + + const eq = db.ensureEdgePropertyIndex('WORKS_AT', 'status', { kind: 'equality' }); + assert.equal(eq.kind, 'equality'); + assert.equal(eq.domain, undefined); + assert.equal(eq.state, 'building'); + + const range = db.ensureEdgePropertyIndex('WORKS_AT', 'score', { kind: 'range', domain: 'int' }); + assert.equal(range.kind, 'range'); + assert.equal(range.domain, 'int'); + assert.equal(range.state, 'building'); + + source = db.upsertNode('Person', 'source'); + hotTarget = db.upsertNode('Company', 'hot-target'); + coldTarget = db.upsertNode('Company', 'cold-target'); + hotEdge = db.upsertEdge(source, hotTarget, 'WORKS_AT', { + props: { status: 'hot', score: 90 }, + weight: 2.0, + }); + db.upsertEdge(source, coldTarget, 'WORKS_AT', { + props: { status: 'cold', score: 10 }, + weight: 1.0, + }); + + await waitForEdgeIndexState( + db, + infos => infos.find(info => info.label === 'WORKS_AT' && info.propKey === 'status' && info.kind === 'equality') + ); + await waitForEdgeIndexState( + db, + infos => infos.find(info => info.label === 'WORKS_AT' && info.propKey === 'score' && info.kind === 'range') + ); + }); + + after(() => { + db.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('ensures, lists, validates, and drops declared edge property indexes', () => { + const listed = db.listEdgePropertyIndexes(); + assert.deepEqual( + listed.map(info => [info.propKey, info.kind, info.domain ?? null, info.state]).sort(), + [ + ['score', 'range', 'int', 'ready'], + ['status', 'equality', null, 'ready'], + ] + ); + + assert.throws( + () => db.ensureEdgePropertyIndex('WORKS_AT', 'score', { kind: 'range', domain: 'float' }), + /different domain/i + ); + assert.throws( + () => db.ensureEdgePropertyIndex('WORKS_AT', 'score', { kind: 'range' }), + /Range indexes require domain/i + ); + assert.throws( + () => db.ensureEdgePropertyIndex('WORKS_AT', 'status', { kind: 'equality', domain: 'int' }), + /do not accept a range domain/i + ); + + assert.equal(db.dropEdgePropertyIndex('WORKS_AT', 'missing', { kind: 'equality' }), false); + }); + + it('uses edge property indexes from direct edge queries and pattern explain', () => { + const direct = db.queryEdgeIds({ + label: 'WORKS_AT', + fromIds: [source], + filter: { property: 'status', eq: 'hot' }, + limit: 10, + }); + assert.deepEqual(Array.from(direct.items), [hotEdge]); + + const directPlan = db.explainEdgeQuery({ + label: 'WORKS_AT', + fromIds: [source], + filter: { property: 'status', eq: 'hot' }, + limit: 10, + }); + assert.ok(planHasKind(directPlan.root, 'edge_property_equality_index')); + + const directRange = db.queryEdgeIds({ + label: 'WORKS_AT', + fromIds: [source], + filter: { property: 'score', gte: 80 }, + limit: 10, + }); + assert.deepEqual(Array.from(directRange.items), [hotEdge]); + + const directRangePlan = db.explainEdgeQuery({ + label: 'WORKS_AT', + fromIds: [source], + filter: { property: 'score', gte: 80 }, + limit: 10, + }); + assert.ok(planHasKind(directRangePlan.root, 'edge_property_range_index')); + + const pattern = { + nodes: [ + { alias: 'a', labelFilter: { labels: ['Person'], mode: 'all' } }, + { alias: 'b', labelFilter: { labels: ['Company'], mode: 'all' } }, + ], + edges: [ + { + alias: 'e', + fromAlias: 'a', + toAlias: 'b', + direction: 'outgoing', + labelFilter: ['WORKS_AT'], + filter: { property: 'status', eq: 'hot' }, + }, + ], + limit: 10, + }; + assert.deepEqual(db.queryPattern(pattern).matches, [ + { nodes: { a: source, b: hotTarget }, edges: { e: hotEdge } }, + ]); + const patternPlan = db.explainPatternQuery(pattern); + assert.ok(planHasKind(patternPlan.root, 'pattern_edge_anchor')); + assert.ok(planHasKind(patternPlan.root, 'edge_property_equality_index')); + + const rangePattern = { + ...pattern, + edges: [ + { + ...pattern.edges[0], + filter: { property: 'score', gte: 80 }, + }, + ], + }; + assert.deepEqual(db.queryPattern(rangePattern).matches, [ + { nodes: { a: source, b: hotTarget }, edges: { e: hotEdge } }, + ]); + const rangePatternPlan = db.explainPatternQuery(rangePattern); + assert.ok(planHasKind(rangePatternPlan.root, 'pattern_edge_anchor')); + assert.ok(planHasKind(rangePatternPlan.root, 'edge_property_range_index')); + }); + + it('supports async edge property index APIs', async () => { + const asyncEq = await db.ensureEdgePropertyIndexAsync('WORKS_AT', 'temp', { kind: 'equality' }); + assert.equal(asyncEq.kind, 'equality'); + await waitForEdgeIndexState( + db, + infos => infos.find(info => info.label === 'WORKS_AT' && info.propKey === 'temp' && info.kind === 'equality') + ); + + const listed = await db.listEdgePropertyIndexesAsync(); + assert.ok(listed.some(info => info.propKey === 'temp' && info.kind === 'equality')); + + assert.equal(await db.dropEdgePropertyIndexAsync('WORKS_AT', 'temp', { kind: 'equality' }), true); }); }); diff --git a/overgraph-node/__test__/queries.mjs b/overgraph-node/__test__/queries.mjs index fb9c36a..491d1ff 100644 --- a/overgraph-node/__test__/queries.mjs +++ b/overgraph-node/__test__/queries.mjs @@ -13,10 +13,25 @@ function planHasKind(node, kind) { return Array.isArray(node.inputs) && node.inputs.some(input => planHasKind(input, kind)); } +function nodeLabels(label) { + return { labels: [label], mode: 'all' }; +} + function sortedIds(page) { return Array.from(page.items).sort((a, b) => a - b); } +async function rejectsOrThrows(fn, pattern) { + let result; + try { + result = fn(); + } catch (err) { + assert.match(String(err?.message ?? err), pattern); + return; + } + await assert.rejects(result, pattern); +} + async function waitForIndexState(db, predicate, expectedState = 'ready', timeoutMs = 5000) { const deadline = Date.now() + timeoutMs; for (;;) { @@ -49,30 +64,30 @@ describe('query API parity', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-query-node-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - activeHigh = db.upsertNode(1, 'active-high', { + activeHigh = db.upsertNode('Person', 'active-high', { props: { status: 'active', score: 90, team: 'core' }, }); - activeLow = db.upsertNode(1, 'active-low', { + activeLow = db.upsertNode('Person', 'active-low', { props: { status: 'active', score: 40, team: 'core' }, }); - inactive = db.upsertNode(1, 'inactive', { + inactive = db.upsertNode('Person', 'inactive', { props: { status: 'inactive', score: 95, team: 'core' }, }); - literalUpdatedAt = db.upsertNode(1, 'literal-updated-at', { + literalUpdatedAt = db.upsertNode('Person', 'literal-updated-at', { props: { updatedAt: 'literal-property-value', status: 'active', score: 70 }, }); - nullTag = db.upsertNode(1, 'null-tag', { + nullTag = db.upsertNode('Person', 'null-tag', { props: { status: 'nullish', tag: null, score: 10 }, }); - nested = db.upsertNode(1, 'nested', { + nested = db.upsertNode('Person', 'nested', { props: { status: 'nested', payload: { items: [1, '1', null] }, score: 15 }, }); - acme = db.upsertNode(2, 'acme', { props: { status: 'customer' } }); - beta = db.upsertNode(2, 'beta', { props: { status: 'prospect' } }); - worksAt = db.upsertEdge(activeHigh, acme, 10, { + acme = db.upsertNode('Company', 'acme', { props: { status: 'customer' } }); + beta = db.upsertNode('Company', 'beta', { props: { status: 'prospect' } }); + worksAt = db.upsertEdge(activeHigh, acme, 'WORKS_AT', { props: { role: 'engineer', since: 2020, updatedAt: 'edge-literal' }, }); - inactiveWorksAt = db.upsertEdge(inactive, beta, 10, { + inactiveWorksAt = db.upsertEdge(inactive, beta, 'WORKS_AT', { props: { role: 'engineer', since: 2022, updatedAt: 'edge-literal' }, }); }); @@ -84,7 +99,7 @@ describe('query API parity', () => { it('runs ID-only and hydrated compound node queries', () => { const request = { - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { and: [ { property: 'status', eq: 'active' }, @@ -102,9 +117,25 @@ describe('query API parity', () => { assert.equal(nodes.items[0].props.status, 'active'); }); + it('honors NodeLabelFilter Any versus All semantics', () => { + const alpha = db.upsertNode('AnyAlpha', 'any-alpha'); + const beta = db.upsertNode('AnyBeta', 'any-beta'); + const both = db.upsertNode(['AnyAlpha', 'AnyBeta'], 'any-both'); + + const any = sortedIds(db.queryNodeIds({ + labelFilter: { labels: ['AnyAlpha', 'AnyBeta'], mode: 'any' }, + })); + assert.deepEqual(any, [alpha, beta, both].sort((a, b) => a - b)); + + const all = sortedIds(db.queryNodeIds({ + labelFilter: { labels: ['AnyAlpha', 'AnyBeta'], mode: 'all' }, + })); + assert.deepEqual(all, [both]); + }); + it('ANDs filters while preserving literal built-in property names', () => { const result = db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { and: [ { property: 'status', eq: 'active' }, @@ -116,7 +147,7 @@ describe('query API parity', () => { assert.deepEqual(Array.from(result.items), [literalUpdatedAt]); const timestampResult = db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { and: [ { updatedAt: { gte: db.getNode(activeHigh).updatedAt - 1000 } }, @@ -168,8 +199,8 @@ describe('query API parity', () => { it('matches graph patterns and treats edge updatedAt as a literal property', () => { const result = db.queryPattern({ nodes: [ - { alias: 'person', typeId: 1, filter: { property: 'status', eq: 'active' } }, - { alias: 'company', typeId: 2, keys: ['acme'] }, + { alias: 'person', labelFilter: nodeLabels('Person'), filter: { property: 'status', eq: 'active' } }, + { alias: 'company', labelFilter: nodeLabels('Company'), keys: ['acme'] }, ], edges: [ { @@ -177,12 +208,14 @@ describe('query API parity', () => { fromAlias: 'person', toAlias: 'company', direction: 'outgoing', - typeFilter: [10], - where: { - role: { op: 'eq', value: 'engineer' }, - updatedAt: { op: 'eq', value: 'edge-literal' }, + labelFilter: ['WORKS_AT'], + filter: { + and: [ + { property: 'role', eq: 'engineer' }, + { property: 'updatedAt', eq: 'edge-literal' }, + { property: 'since', lte: 2021 }, + ], }, - predicates: [{ property: { key: 'since', op: 'range', lte: 2021 } }], }, ], limit: 10, @@ -198,20 +231,81 @@ describe('query API parity', () => { assert.notEqual(inactiveWorksAt, worksAt); }); + it('runs direct edge ID and hydrated edge queries', () => { + const edge = db.getEdge(worksAt); + const request = { + label: 'WORKS_AT', + fromIds: [activeHigh], + filter: { + and: [ + { weight: { gte: 1.0 } }, + { validAt: Date.now() }, + { updatedAt: { gte: edge.updatedAt - 1000 } }, + { property: 'role', eq: 'engineer' }, + ], + }, + limit: 0, + }; + + assert.deepEqual(Array.from(db.queryEdgeIds(request).items), [worksAt]); + + const edges = db.queryEdges({ ...request, limit: 1 }); + assert.deepEqual(edges.items.map(item => item.id), [worksAt]); + assert.equal(edges.nextCursor, null); + assert.equal(edges.items[0].props.role, 'engineer'); + + const plan = db.explainEdgeQuery(request); + assert.equal(plan.kind, 'edge_query'); + assert.ok(planHasKind(plan.root, 'verify_edge_filter')); + assert.ok(plan.warnings.includes('edge_property_post_filter')); + }); + + it('accepts canonical graph edge filters', () => { + const result = db.queryPattern({ + nodes: [ + { alias: 'person', ids: [activeHigh] }, + { alias: 'company', labelFilter: nodeLabels('Company'), keys: ['acme'] }, + ], + edges: [ + { + alias: 'employment', + fromAlias: 'person', + toAlias: 'company', + direction: 'outgoing', + labelFilter: ['WORKS_AT'], + filter: { + and: [ + { validAt: Date.now() }, + { property: 'role', eq: 'engineer' }, + ], + }, + }, + ], + limit: 10, + }); + + assert.deepEqual(result.matches, [ + { + nodes: { company: acme, person: activeHigh }, + edges: { employment: worksAt }, + }, + ]); + }); + it('serializes explain output with recursive lower_snake kinds and warnings', () => { const nodePlan = db.explainNodeQuery({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'status', eq: 'active' }, }); assert.equal(nodePlan.kind, 'node_query'); - assert.ok(planHasKind(nodePlan.root, 'fallback_type_scan')); + assert.ok(planHasKind(nodePlan.root, 'fallback_node_label_scan')); assert.ok(nodePlan.warnings.every(warning => /^[a-z_]+$/.test(warning))); assert.ok(nodePlan.warnings.includes('using_fallback_scan')); const patternPlan = db.explainPatternQuery({ nodes: [ - { alias: 'person', typeId: 1, filter: { property: 'status', eq: 'active' } }, - { alias: 'company', typeId: 2, keys: ['acme'] }, + { alias: 'person', labelFilter: nodeLabels('Person'), filter: { property: 'status', eq: 'active' } }, + { alias: 'company', labelFilter: nodeLabels('Company'), keys: ['acme'] }, ], edges: [ { @@ -219,8 +313,8 @@ describe('query API parity', () => { fromAlias: 'person', toAlias: 'company', direction: 'outgoing', - typeFilter: [10], - where: { role: { op: 'eq', value: 'engineer' } }, + labelFilter: ['WORKS_AT'], + filter: { property: 'role', eq: 'engineer' }, }, ], limit: 10, @@ -233,15 +327,15 @@ describe('query API parity', () => { it('rejects invalid predicate and pattern shapes at the binding boundary', () => { assert.throws( - () => db.queryNodeIds({ typeId: 1, predicates: [{ property: { key: 'status', op: 'eq' } }] }), + () => db.queryNodeIds({ labelFilter: nodeLabels('Person'), predicates: [{ property: { key: 'status', op: 'eq' } }] }), /use filter/i ); assert.throws( - () => db.queryNodeIds({ typeId: 1, filter: { property: 'score', gt: 1, gte: 2 } }), + () => db.queryNodeIds({ labelFilter: nodeLabels('Person'), filter: { property: 'score', gt: 1, gte: 2 } }), /both gt and gte/i ); assert.throws( - () => db.queryNodeIds({ typeId: 1, where: { status: { eq: 'active' } } }), + () => db.queryNodeIds({ labelFilter: nodeLabels('Person'), where: { status: { eq: 'active' } } }), /use filter/i ); assert.throws( @@ -249,8 +343,39 @@ describe('query API parity', () => { /use filter/i ); assert.throws( - () => db.queryPattern({ nodes: [{ alias: 'a' }], edges: [{ fromAlias: 'a', toAlias: 'b', filter: { property: 'role', eq: 'engineer' } }], limit: 1 }), - /edge pattern filter is not supported/i + () => db.queryEdgeIds({ filter: { weight: { gte: 1 } } }), + /full scan|anchor|allow_full_scan/i + ); + assert.throws( + () => db.queryEdgeIds({ label: 'WORKS_AT', filter: { weight: { gt: 1, gte: 2 } } }), + /both gt and gte/i + ); + for (const field of ['where', 'predicates']) { + assert.throws( + () => db.queryEdgeIds({ label: 'WORKS_AT', [field]: { role: { eq: 'engineer' } } }), + /use filter/i + ); + assert.throws( + () => db.queryEdges({ label: 'WORKS_AT', [field]: { role: { eq: 'engineer' } } }), + /use filter/i + ); + assert.throws( + () => db.explainEdgeQuery({ label: 'WORKS_AT', [field]: { role: { eq: 'engineer' } } }), + /use filter/i + ); + } + assert.throws( + () => db.queryPattern({ + nodes: [{ alias: 'a' }], + edges: [{ + fromAlias: 'a', + toAlias: 'b', + filter: { property: 'role', eq: 'engineer' }, + where: { role: { eq: 'engineer' } }, + }], + limit: 1, + }), + /use filter/i ); assert.throws( () => db.queryPattern({ nodes: [], edges: [], limit: 0 }), @@ -260,30 +385,58 @@ describe('query API parity', () => { it('supports async query and explain parity', async () => { const ids = await db.queryNodeIdsAsync({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'status', eq: 'active' }, }); assert.ok(Array.from(ids.items).includes(activeHigh)); const nodes = await db.queryNodesAsync({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'score', gte: 80 }, }); assert.deepEqual(nodes.items.map(node => node.id), [activeHigh, inactive]); const plan = await db.explainNodeQueryAsync({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'status', eq: 'active' }, }); assert.equal(plan.kind, 'node_query'); + const edgeIds = await db.queryEdgeIdsAsync({ + fromIds: [activeHigh], + filter: { property: 'role', eq: 'engineer' }, + }); + assert.deepEqual(Array.from(edgeIds.items), [worksAt]); + + const edges = await db.queryEdgesAsync({ + ids: [worksAt], + filter: { validAt: Date.now() }, + }); + assert.deepEqual(edges.items.map(edge => edge.id), [worksAt]); + + const edgePlan = await db.explainEdgeQueryAsync({ ids: [worksAt] }); + assert.equal(edgePlan.kind, 'edge_query'); + + await rejectsOrThrows( + () => db.queryEdgeIdsAsync({ label: 'WORKS_AT', where: { role: { eq: 'engineer' } } }), + /use filter/i + ); + await rejectsOrThrows( + () => db.queryEdgesAsync({ label: 'WORKS_AT', predicates: { role: { eq: 'engineer' } } }), + /use filter/i + ); + await rejectsOrThrows( + () => db.explainEdgeQueryAsync({ label: 'WORKS_AT', where: { role: { eq: 'engineer' } } }), + /use filter/i + ); + const pattern = await db.queryPatternAsync({ nodes: [ { alias: 'person', ids: [activeHigh], filter: { property: 'status', eq: 'active' } }, - { alias: 'company', typeId: 2, keys: ['acme'] }, + { alias: 'company', labelFilter: nodeLabels('Company'), keys: ['acme'] }, ], edges: [ - { alias: 'employment', fromAlias: 'person', toAlias: 'company', typeFilter: [10] }, + { alias: 'employment', fromAlias: 'person', toAlias: 'company', labelFilter: ['WORKS_AT'] }, ], limit: 10, }); @@ -292,10 +445,10 @@ describe('query API parity', () => { const patternPlan = await db.explainPatternQueryAsync({ nodes: [ { alias: 'person', ids: [activeHigh], filter: { property: 'status', eq: 'active' } }, - { alias: 'company', typeId: 2, keys: ['acme'] }, + { alias: 'company', labelFilter: nodeLabels('Company'), keys: ['acme'] }, ], edges: [ - { alias: 'employment', fromAlias: 'person', toAlias: 'company', typeFilter: [10] }, + { alias: 'employment', fromAlias: 'person', toAlias: 'company', labelFilter: ['WORKS_AT'] }, ], limit: 10, }); @@ -305,38 +458,38 @@ describe('query API parity', () => { it('supports boolean filters, null presence semantics, and nested values', () => { assert.deepEqual(sortedIds(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { or: [{ property: 'status', eq: 'active' }, { property: 'status', eq: 'nullish' }] }, })), [activeHigh, activeLow, literalUpdatedAt, nullTag]); assert.deepEqual(Array.from(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'status', in: ['nested'] }, }).items), [nested]); assert.deepEqual(Array.from(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'tag', eq: null }, }).items), [nullTag]); assert.deepEqual(Array.from(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'tag', in: [null] }, }).items), [nullTag]); assert.deepEqual(Array.from(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'tag', exists: true }, }).items), [nullTag]); assert.ok(!Array.from(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'tag', missing: true }, }).items).includes(nullTag)); assert.deepEqual(Array.from(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'payload', eq: { items: [1, '1', null] } }, }).items), [nested]); assert.deepEqual(Array.from(db.queryNodeIds({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'status', eq: '1' }, }).items), []); }); @@ -358,33 +511,33 @@ describe('query API parity', () => { [{ property: 'x' }, /exactly one operator family/i], ]; for (const [filter, pattern] of invalid) { - assert.throws(() => db.queryNodeIds({ typeId: 1, filter }), pattern); + assert.throws(() => db.queryNodeIds({ labelFilter: nodeLabels('Person'), filter }), pattern); } }); it('serializes boolean explain plans with lower_snake physical nodes and warnings', async () => { - db.ensureNodePropertyIndex(1, 'status', { kind: 'equality' }); + db.ensureNodePropertyIndex('Person', 'status', { kind: 'equality' }); await waitForIndexState( db, - infos => infos.find(info => info.typeId === 1 && info.propKey === 'status' && info.kind === 'equality') + infos => infos.find(info => info.label === 'Person' && info.propKey === 'status' && info.kind === 'equality') ); const indexedOr = db.explainNodeQuery({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { or: [{ property: 'status', eq: 'active' }, { property: 'status', eq: 'nullish' }] }, }); assert.ok(planHasKind(indexedOr.root, 'union')); assert.ok(planHasKind(indexedOr.root, 'verify_node_filter')); const fallbackOr = db.explainNodeQuery({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { or: [{ property: 'status', eq: 'active' }, { property: 'tag', missing: true }] }, }); assert.ok(fallbackOr.warnings.includes('boolean_branch_fallback')); assert.ok(fallbackOr.warnings.includes('verify_only_filter')); const empty = db.explainNodeQuery({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { and: [ { property: 'status', eq: 'active' }, @@ -395,7 +548,7 @@ describe('query API parity', () => { assert.ok(planHasKind(empty.root, 'empty_result')); const asyncPlan = await db.explainNodeQueryAsync({ - typeId: 1, + labelFilter: nodeLabels('Person'), filter: { property: 'status', eq: 'active' }, }); assert.equal(asyncPlan.kind, 'node_query'); diff --git a/overgraph-node/__test__/scrub.mjs b/overgraph-node/__test__/scrub.mjs new file mode 100644 index 0000000..851360b --- /dev/null +++ b/overgraph-node/__test__/scrub.mjs @@ -0,0 +1,63 @@ +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { OverGraph } from '../index.js'; + +describe('scrub', () => { + let tmpDir; + + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-scrub-')); + }); + + after(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('reports healthy database with no findings', () => { + const db = OverGraph.open(join(tmpDir, 'healthy')); + for (let i = 0; i < 5; i++) { + db.upsertNode('Person', `node_${i}`); + } + db.flush(); + + const report = db.scrub(); + assert.equal(report.totalComponentsFailed, 0); + assert(report.totalComponentsOk > 0); + assert(report.totalComponentsChecked > 0); + assert(report.durationMs >= 0); + assert.equal(report.segments.length, 1); + assert.equal(report.segments[0].segmentId, 1); + assert(report.segments[0].componentsOk > 0); + assert.equal(report.segments[0].findings.length, 0); + db.close(); + }); + + it('detects packed range corruption', () => { + const dbPath = join(tmpDir, 'corrupt'); + let db = OverGraph.open(dbPath); + for (let i = 0; i < 5; i++) { + db.upsertNode('Person', `node_${i}`); + } + db.flush(); + db.close(); + + const corePath = join(dbPath, 'segments', 'seg_0001', 'segment.core'); + const data = Buffer.from(readFileSync(corePath)); + data[Math.floor(data.length / 2)] ^= 0xFF; + writeFileSync(corePath, data); + + db = OverGraph.open(dbPath); + const report = db.scrub(); + assert(report.totalComponentsFailed > 0); + + const findings = report.segments[0].findings; + assert(findings.length > 0); + assert(findings.some(f => f.findingType === 'PayloadDigestMismatch')); + assert(findings[0].componentKind.length > 0); + assert(findings[0].detail.length > 0); + db.close(); + }); +}); diff --git a/overgraph-node/__test__/shortest-path.mjs b/overgraph-node/__test__/shortest-path.mjs index 29ffa64..d54ce43 100644 --- a/overgraph-node/__test__/shortest-path.mjs +++ b/overgraph-node/__test__/shortest-path.mjs @@ -10,27 +10,27 @@ function freshDb(tmpDir, name) { } // Helper: build a linear chain n0 -> n1 -> n2 -> ... -> n(len-1) -function buildChain(db, len, edgeType = 10) { +function buildChain(db, len, labelName = 'WORKS_AT') { const nodes = []; for (let i = 0; i < len; i++) { - nodes.push(db.upsertNode(1, `n${i}`)); + nodes.push(db.upsertNode('Person', `n${i}`)); } for (let i = 0; i < len - 1; i++) { - db.upsertEdge(nodes[i], nodes[i + 1], edgeType); + db.upsertEdge(nodes[i], nodes[i + 1], labelName); } return nodes; } // Helper: build a diamond A -> B -> D, A -> C -> D function buildDiamond(db) { - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - const c = db.upsertNode(1, 'c'); - const d = db.upsertNode(1, 'd'); - db.upsertEdge(a, b, 10); - db.upsertEdge(a, c, 10); - db.upsertEdge(b, d, 10); - db.upsertEdge(c, d, 10); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + const c = db.upsertNode('Person', 'c'); + const d = db.upsertNode('Person', 'd'); + db.upsertEdge(a, b, 'WORKS_AT'); + db.upsertEdge(a, c, 'WORKS_AT'); + db.upsertEdge(b, d, 'WORKS_AT'); + db.upsertEdge(c, d, 'WORKS_AT'); return { a, b, c, d }; } @@ -65,8 +65,8 @@ describe('shortestPath (sync)', () => { it('returns null for disconnected nodes', () => { const db2 = freshDb(tmpDir, 'sp-disc'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); const result = db2.shortestPath(a, b); assert.equal(result, null); db2.close(); @@ -74,7 +74,7 @@ describe('shortestPath (sync)', () => { it('returns self-path for from == to', () => { const db2 = freshDb(tmpDir, 'sp-self'); - const a = db2.upsertNode(1, 'a'); + const a = db2.upsertNode('Person', 'a'); const result = db2.shortestPath(a, a); assert.ok(result); assert.deepEqual(result.nodes, [a]); @@ -85,9 +85,9 @@ describe('shortestPath (sync)', () => { it('respects direction (incoming)', () => { const db2 = freshDb(tmpDir, 'sp-dir'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - db2.upsertEdge(a, b, 10); // a -> b + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + db2.upsertEdge(a, b, 'WORKS_AT'); // a -> b // outgoing from b: no path to a assert.equal(db2.shortestPath(b, a), null); // incoming from perspective of a: follow incoming edges = reverse, so b->a should work @@ -101,9 +101,9 @@ describe('shortestPath (sync)', () => { it('respects direction (both)', () => { const db2 = freshDb(tmpDir, 'sp-both'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - db2.upsertEdge(a, b, 10); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + db2.upsertEdge(a, b, 'WORKS_AT'); // With 'both', b can reach a const result = db2.shortestPath(b, a, { direction: 'both' }); assert.ok(result); @@ -111,18 +111,18 @@ describe('shortestPath (sync)', () => { db2.close(); }); - it('filters by edge type', () => { + it('filters by edge label', () => { const db2 = freshDb(tmpDir, 'sp-tf'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - const c = db2.upsertNode(1, 'c'); - db2.upsertEdge(a, b, 10); - db2.upsertEdge(b, c, 20); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + const c = db2.upsertNode('Person', 'c'); + db2.upsertEdge(a, b, 'WORKS_AT'); + db2.upsertEdge(b, c, 'MENTIONS'); // Only type 10 edges: can reach b but not c - const result = db2.shortestPath(a, c, { typeFilter: [10] }); + const result = db2.shortestPath(a, c, { edgeLabelFilter: ['WORKS_AT'] }); assert.equal(result, null); // Both types: can reach c - const result2 = db2.shortestPath(a, c, { typeFilter: [10, 20] }); + const result2 = db2.shortestPath(a, c, { edgeLabelFilter: ['WORKS_AT', 'MENTIONS'] }); assert.ok(result2); db2.close(); }); @@ -157,12 +157,12 @@ describe('shortestPath weighted (sync)', () => { it('finds weighted shortest path via weightField', () => { // A -> B (weight 1), A -> C (weight 10), B -> C (weight 1) // Shortest by weight: A->B->C (cost 2) not A->C (cost 10) - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - const c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10, { weight: 1.0 }); - db.upsertEdge(a, c, 10, { weight: 10.0 }); - db.upsertEdge(b, c, 10, { weight: 1.0 }); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + const c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 1.0 }); + db.upsertEdge(a, c, 'WORKS_AT', { weight: 10.0 }); + db.upsertEdge(b, c, 'WORKS_AT', { weight: 1.0 }); const result = db.shortestPath(a, c, { weightField: 'weight' }); assert.ok(result); assert.deepEqual(result.nodes, [a, b, c]); @@ -171,11 +171,11 @@ describe('shortestPath weighted (sync)', () => { it('respects maxCost', () => { const db2 = freshDb(tmpDir, 'spw-maxc'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - const c = db2.upsertNode(1, 'c'); - db2.upsertEdge(a, b, 10, { weight: 5.0 }); - db2.upsertEdge(b, c, 10, { weight: 5.0 }); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + const c = db2.upsertNode('Person', 'c'); + db2.upsertEdge(a, b, 'WORKS_AT', { weight: 5.0 }); + db2.upsertEdge(b, c, 'WORKS_AT', { weight: 5.0 }); // maxCost=8: can't afford A->B->C (cost 10) const result = db2.shortestPath(a, c, { weightField: 'weight', maxCost: 8.0 }); assert.equal(result, null); @@ -187,16 +187,16 @@ describe('shortestPath weighted (sync)', () => { it('uses the best constrained weighted path under maxDepth', () => { const db2 = freshDb(tmpDir, 'spw-depth'); - const s = db2.upsertNode(1, 's'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - const c = db2.upsertNode(1, 'c'); - const t = db2.upsertNode(1, 't'); - db2.upsertEdge(s, a, 10, { weight: 1.0 }); - db2.upsertEdge(a, b, 10, { weight: 1.0 }); - db2.upsertEdge(b, t, 10, { weight: 1.0 }); - db2.upsertEdge(s, c, 10, { weight: 3.0 }); - db2.upsertEdge(c, t, 10, { weight: 3.0 }); + const s = db2.upsertNode('Person', 's'); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + const c = db2.upsertNode('Person', 'c'); + const t = db2.upsertNode('Person', 't'); + db2.upsertEdge(s, a, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(a, b, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(b, t, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(s, c, 'WORKS_AT', { weight: 3.0 }); + db2.upsertEdge(c, t, 'WORKS_AT', { weight: 3.0 }); const result = db2.shortestPath(s, t, { weightField: 'weight', @@ -225,24 +225,24 @@ describe('isConnected (sync)', () => { it('returns false for disconnected nodes', () => { const db2 = freshDb(tmpDir, 'ic-disc'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); assert.equal(db2.isConnected(a, b), false); db2.close(); }); it('returns true for self', () => { const db2 = freshDb(tmpDir, 'ic-self'); - const a = db2.upsertNode(1, 'a'); + const a = db2.upsertNode('Person', 'a'); assert.equal(db2.isConnected(a, a), true); db2.close(); }); it('respects direction', () => { const db2 = freshDb(tmpDir, 'ic-dir'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - db2.upsertEdge(a, b, 10); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + db2.upsertEdge(a, b, 'WORKS_AT'); assert.equal(db2.isConnected(b, a, { direction: 'outgoing' }), false); assert.equal(db2.isConnected(b, a, { direction: 'incoming' }), true); assert.equal(db2.isConnected(b, a, { direction: 'both' }), true); @@ -259,13 +259,13 @@ describe('isConnected (sync)', () => { it('filters by type', () => { const db2 = freshDb(tmpDir, 'ic-tf'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - const c = db2.upsertNode(1, 'c'); - db2.upsertEdge(a, b, 10); - db2.upsertEdge(b, c, 20); - assert.equal(db2.isConnected(a, c, { typeFilter: [10] }), false); - assert.equal(db2.isConnected(a, c, { typeFilter: [10, 20] }), true); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + const c = db2.upsertNode('Person', 'c'); + db2.upsertEdge(a, b, 'WORKS_AT'); + db2.upsertEdge(b, c, 'MENTIONS'); + assert.equal(db2.isConnected(a, c, { edgeLabelFilter: ['WORKS_AT'] }), false); + assert.equal(db2.isConnected(a, c, { edgeLabelFilter: ['WORKS_AT', 'MENTIONS'] }), true); db2.close(); }); }); @@ -293,8 +293,8 @@ describe('allShortestPaths (sync)', () => { it('returns empty array for disconnected', () => { const db2 = freshDb(tmpDir, 'asp-disc'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); const paths = db2.allShortestPaths(a, b); assert.ok(Array.isArray(paths)); assert.equal(paths.length, 0); @@ -311,14 +311,14 @@ describe('allShortestPaths (sync)', () => { it('works with weighted paths', () => { const db2 = freshDb(tmpDir, 'asp-w'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - const c = db2.upsertNode(1, 'c'); - const d = db2.upsertNode(1, 'd'); - db2.upsertEdge(a, b, 10, { weight: 1.0 }); - db2.upsertEdge(a, c, 10, { weight: 1.0 }); - db2.upsertEdge(b, d, 10, { weight: 1.0 }); - db2.upsertEdge(c, d, 10, { weight: 1.0 }); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + const c = db2.upsertNode('Person', 'c'); + const d = db2.upsertNode('Person', 'd'); + db2.upsertEdge(a, b, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(a, c, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(b, d, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(c, d, 'WORKS_AT', { weight: 1.0 }); const paths = db2.allShortestPaths(a, d, { weightField: 'weight' }); assert.equal(paths.length, 2); for (const p of paths) { @@ -329,16 +329,16 @@ describe('allShortestPaths (sync)', () => { it('uses the best constrained weighted cost under maxDepth', () => { const db2 = freshDb(tmpDir, 'asp-w-depth'); - const s = db2.upsertNode(1, 's'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); - const c = db2.upsertNode(1, 'c'); - const t = db2.upsertNode(1, 't'); - db2.upsertEdge(s, a, 10, { weight: 1.0 }); - db2.upsertEdge(a, b, 10, { weight: 1.0 }); - db2.upsertEdge(b, t, 10, { weight: 1.0 }); - db2.upsertEdge(s, c, 10, { weight: 3.0 }); - db2.upsertEdge(c, t, 10, { weight: 3.0 }); + const s = db2.upsertNode('Person', 's'); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); + const c = db2.upsertNode('Person', 'c'); + const t = db2.upsertNode('Person', 't'); + db2.upsertEdge(s, a, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(a, b, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(b, t, 'WORKS_AT', { weight: 1.0 }); + db2.upsertEdge(s, c, 'WORKS_AT', { weight: 3.0 }); + db2.upsertEdge(c, t, 'WORKS_AT', { weight: 3.0 }); const paths = db2.allShortestPaths(s, t, { weightField: 'weight', @@ -352,7 +352,7 @@ describe('allShortestPaths (sync)', () => { it('returns single path for from == to', () => { const db2 = freshDb(tmpDir, 'asp-self'); - const a = db2.upsertNode(1, 'a'); + const a = db2.upsertNode('Person', 'a'); const paths = db2.allShortestPaths(a, a); assert.equal(paths.length, 1); assert.deepEqual(paths[0].nodes, [a]); @@ -380,8 +380,8 @@ describe('shortest path async', () => { it('shortestPathAsync returns null for disconnected', async () => { const db2 = freshDb(tmpDir, 'spa-disc'); - const a = db2.upsertNode(1, 'a'); - const b = db2.upsertNode(1, 'b'); + const a = db2.upsertNode('Person', 'a'); + const b = db2.upsertNode('Person', 'b'); const result = await db2.shortestPathAsync(a, b); assert.equal(result, null); db2.close(); @@ -391,8 +391,8 @@ describe('shortest path async', () => { const [a, , c] = buildChain(db, 3); assert.equal(await db.isConnectedAsync(a, c), true); const db2 = freshDb(tmpDir, 'spa-ic'); - const x = db2.upsertNode(1, 'x'); - const y = db2.upsertNode(1, 'y'); + const x = db2.upsertNode('Person', 'x'); + const y = db2.upsertNode('Person', 'y'); assert.equal(await db2.isConnectedAsync(x, y), false); db2.close(); }); @@ -416,11 +416,11 @@ describe('shortest path temporal', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('atEpoch excludes expired edges', () => { - const a = db.upsertNode(1, 'a'); - const b = db.upsertNode(1, 'b'); - const c = db.upsertNode(1, 'c'); - db.upsertEdge(a, b, 10, { weight: 1.0, validFrom: 100, validTo: 200 }); - db.upsertEdge(b, c, 10, { weight: 1.0, validFrom: 100, validTo: 200 }); + const a = db.upsertNode('Person', 'a'); + const b = db.upsertNode('Person', 'b'); + const c = db.upsertNode('Person', 'c'); + db.upsertEdge(a, b, 'WORKS_AT', { weight: 1.0, validFrom: 100, validTo: 200 }); + db.upsertEdge(b, c, 'WORKS_AT', { weight: 1.0, validFrom: 100, validTo: 200 }); // At epoch 150: edges valid const r1 = db.shortestPath(a, c, { atEpoch: 150 }); assert.ok(r1); diff --git a/overgraph-node/__test__/sync-api.mjs b/overgraph-node/__test__/sync-api.mjs index 7a60350..21ed3b4 100644 --- a/overgraph-node/__test__/sync-api.mjs +++ b/overgraph-node/__test__/sync-api.mjs @@ -11,6 +11,87 @@ function freshDb(tmpDir, name) { return OverGraph.open(join(tmpDir, name)); } +function legacyNodeBinaryBuffer() { + const key = Buffer.from('legacy-node', 'utf8'); + const buf = Buffer.alloc(4 + 4 + 4 + 2 + key.length + 4); + let off = 0; + buf.writeUInt32LE(1, off); off += 4; + buf.writeUInt32LE(0x4e500002, off); off += 4; // was accepted as label "PN" without a magic header + buf.writeFloatLE(1.0, off); off += 4; + buf.writeUInt16LE(key.length, off); off += 2; + key.copy(buf, off); off += key.length; + buf.writeUInt32LE(0, off); + return buf; +} + +function legacyNodeBinaryV1Buffer() { + const buf = Buffer.alloc(10); + let off = 0; + Buffer.from('OGNB').copy(buf, off); off += 4; + buf.writeUInt16LE(1, off); off += 2; + buf.writeUInt32LE(0, off); + return buf; +} + +function legacyEdgeBinaryBuffer(from, to) { + const buf = Buffer.alloc(4 + 8 + 8 + 4 + 4 + 8 + 8 + 4); + let off = 0; + buf.writeUInt32LE(1, off); off += 4; + buf.writeBigUInt64LE(BigInt(from), off); off += 8; + buf.writeBigUInt64LE(BigInt(to), off); off += 8; + buf.writeUInt32LE(0x544c0002, off); off += 4; // was accepted as edge label "LT" without a magic header + buf.writeFloatLE(1.0, off); off += 4; + buf.writeBigInt64LE(0n, off); off += 8; + buf.writeBigInt64LE(0n, off); off += 8; + buf.writeUInt32LE(0, off); + return buf; +} + +describe('catalog diagnostics', () => { + let tmpDir, db; + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-catalog-')); + db = freshDb(tmpDir, 'catalog'); + }); + after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); + + it('ensures, gets, and lists node labels and edge labels', () => { + assert.deepEqual(db.listNodeLabels(), []); + assert.deepEqual(db.listEdgeLabels(), []); + + const personId = db.ensureNodeLabel('Person'); + const companyId = db.ensureNodeLabel('Company'); + const worksAtId = db.ensureEdgeLabel('WORKS_AT'); + const knowsId = db.ensureEdgeLabel('KNOWS'); + + assert.equal(db.ensureNodeLabel('Person'), personId); + assert.equal(db.ensureEdgeLabel('WORKS_AT'), worksAtId); + assert.equal(db.getNodeLabelId('Person'), personId); + assert.equal(db.getNodeLabelId('MissingLabel'), null); + assert.equal(db.getEdgeLabelId('WORKS_AT'), worksAtId); + assert.equal(db.getEdgeLabelId('MISSING_EDGE'), null); + assert.equal(db.getNodeLabel(personId), 'Person'); + assert.equal(db.getNodeLabel(999_999), null); + assert.equal(db.getEdgeLabel(worksAtId), 'WORKS_AT'); + assert.equal(db.getEdgeLabel(999_999), null); + + assert.deepEqual( + db.listNodeLabels().sort((a, b) => a.labelId - b.labelId), + [ + { label: 'Person', labelId: personId }, + { label: 'Company', labelId: companyId }, + ], + ); + assert.deepEqual( + db.listEdgeLabels().sort((a, b) => a.labelId - b.labelId), + [ + { label: 'WORKS_AT', labelId: worksAtId }, + { label: 'KNOWS', labelId: knowsId }, + ], + ); + }); +}); + describe('upsert_node / upsert_edge', () => { let tmpDir, db; before(() => { @@ -20,39 +101,112 @@ describe('upsert_node / upsert_edge', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('upsertNode returns a number id', () => { - const id = db.upsertNode(1, 'alice'); + const id = db.upsertNode('Person', 'alice'); assert.equal(typeof id, 'number'); assert.ok(id > 0); }); it('upsertNode with props and weight', () => { - const id = db.upsertNode(1, 'bob', { props: { age: 30, name: 'Bob' }, weight: 0.8 }); + const id = db.upsertNode('Person', 'bob', { props: { age: 30, name: 'Bob' }, weight: 0.8 }); assert.equal(typeof id, 'number'); assert.ok(id > 0); }); - it('upsertNode deduplicates by (type_id, key)', () => { - const id1 = db.upsertNode(2, 'same-key'); - const id2 = db.upsertNode(2, 'same-key'); + it('upsertNode deduplicates by (label, key)', () => { + const id1 = db.upsertNode('Company', 'same-key'); + const id2 = db.upsertNode('Company', 'same-key'); assert.equal(id1, id2); }); it('upsertEdge returns a number id', () => { - const a = db.upsertNode(1, 'src'); - const b = db.upsertNode(1, 'dst'); - const eid = db.upsertEdge(a, b, 10); + const a = db.upsertNode('Person', 'src'); + const b = db.upsertNode('Person', 'dst'); + const eid = db.upsertEdge(a, b, 'WORKS_AT'); assert.equal(typeof eid, 'number'); assert.ok(eid > 0); }); it('upsertEdge with props and weight', () => { - const a = db.upsertNode(1, 'e-src'); - const b = db.upsertNode(1, 'e-dst'); - const eid = db.upsertEdge(a, b, 10, { props: { strength: 0.9 }, weight: 2.5 }); + const a = db.upsertNode('Person', 'e-src'); + const b = db.upsertNode('Person', 'e-dst'); + const eid = db.upsertEdge(a, b, 'WORKS_AT', { props: { strength: 0.9 }, weight: 2.5 }); assert.ok(eid > 0); }); }); +describe('node label mutations', () => { + let tmpDir, db; + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-node-label-mutate-')); + db = freshDb(tmpDir, 'labels'); + }); + after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); + + it('adds and removes node labels with changed flags', () => { + const id = db.upsertNode(['Person'], 'multi-label'); + assert.equal(db.addNodeLabel(id, 'Admin'), true); + assert.equal(db.addNodeLabel(id, 'Admin'), false); + assert.deepEqual([...db.getNode(id).labels].sort(), ['Admin', 'Person']); + assert.deepEqual(Array.from(db.nodesByLabels(['Person', 'Admin'])), [id]); + + assert.equal(db.removeNodeLabel(id, 'Admin'), true); + assert.equal(db.removeNodeLabel(id, 'Admin'), false); + assert.deepEqual(db.getNode(id).labels, ['Person']); + assert.throws( + () => db.removeNodeLabel(id, 'Person'), + /cannot remove the last node label/, + ); + }); +}); + +describe('multi-label node APIs', () => { + let tmpDir, db; + let both, personOnly, adminOnly; + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-node-multi-label-')); + db = freshDb(tmpDir, 'multi-labels'); + both = db.upsertNode(['Person', 'Admin'], 'both'); + personOnly = db.upsertNode('Person', 'person-only'); + adminOnly = db.upsertNode('Admin', 'admin-only'); + }); + after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); + + it('upsertNode accepts a label array and hydrates labels', () => { + assert.deepEqual([...db.getNode(both).labels].sort(), ['Admin', 'Person']); + }); + + it('plural convenience APIs treat arrays as All', () => { + assert.deepEqual(Array.from(db.nodesByLabels(['Person', 'Admin'])), [both]); + assert.deepEqual(db.getNodesByLabels(['Person', 'Admin']).map(node => node.id), [both]); + assert.equal(db.countNodesByLabels(['Person', 'Admin']), 1); + assert.equal(db.countNodesByLabels('Person'), 2); + assert.equal(db.countNodesByLabels('Admin'), 2); + }); + + it('paged plural convenience APIs treat arrays as All', () => { + const idPage = db.nodesByLabelsPaged(['Person', 'Admin'], 10); + assert.deepEqual(Array.from(idPage.items), [both]); + assert.equal(idPage.nextCursor ?? null, null); + + const nodePage = db.getNodesByLabelsPaged(['Person', 'Admin'], 10); + assert.deepEqual(nodePage.items.map(node => node.id), [both]); + assert.equal(nodePage.nextCursor ?? null, null); + }); + + it('async plural convenience APIs preserve All semantics', async () => { + assert.deepEqual(Array.from(await db.nodesByLabelsAsync(['Person', 'Admin'])), [both]); + assert.deepEqual((await db.getNodesByLabelsAsync(['Person', 'Admin'])).map(node => node.id), [both]); + assert.equal(await db.countNodesByLabelsAsync(['Person', 'Admin']), 1); + assert.deepEqual(Array.from((await db.nodesByLabelsPagedAsync(['Person', 'Admin'], 10)).items), [both]); + assert.deepEqual((await db.getNodesByLabelsPagedAsync(['Person', 'Admin'], 10)).items.map(node => node.id), [both]); + }); + + it('single-label queries still include multi-label nodes', () => { + assert.deepEqual(new Set(Array.from(db.nodesByLabels('Person'))), new Set([both, personOnly])); + assert.deepEqual(new Set(Array.from(db.nodesByLabels('Admin'))), new Set([both, adminOnly])); + }); +}); + describe('batch_upsert_nodes / batch_upsert_edges', () => { let tmpDir, db; before(() => { @@ -63,9 +217,9 @@ describe('batch_upsert_nodes / batch_upsert_edges', () => { it('batchUpsertNodes returns Float64Array of ids', () => { const ids = db.batchUpsertNodes([ - { typeId: 1, key: 'n1' }, - { typeId: 1, key: 'n2', props: { x: 1 }, weight: 0.5 }, - { typeId: 2, key: 'n3' }, + { labels: ['Person'], key: 'n1' }, + { labels: ['Person'], key: 'n2', props: { x: 1 }, weight: 0.5 }, + { labels: ['Company'], key: 'n3' }, ]); assert.ok(ids instanceof Float64Array); assert.equal(ids.length, 3); @@ -75,13 +229,13 @@ describe('batch_upsert_nodes / batch_upsert_edges', () => { it('batchUpsertEdges returns Float64Array of ids', () => { const [a, b, c] = db.batchUpsertNodes([ - { typeId: 1, key: 'ba' }, - { typeId: 1, key: 'bb' }, - { typeId: 1, key: 'bc' }, + { labels: 'Person', key: 'ba' }, + { labels: 'Person', key: 'bb' }, + { labels: 'Person', key: 'bc' }, ]); const eids = db.batchUpsertEdges([ - { from: a, to: b, typeId: 5 }, - { from: b, to: c, typeId: 5, props: { label: 'knows' }, weight: 1.2 }, + { from: a, to: b, label: 'DEPENDS_ON'}, + { from: b, to: c, label: 'DEPENDS_ON', props: { label: 'knows' }, weight: 1.2 }, ]); assert.ok(eids instanceof Float64Array); assert.equal(eids.length, 2); @@ -89,6 +243,43 @@ describe('batch_upsert_nodes / batch_upsert_edges', () => { }); }); +describe('vector hydration', () => { + let tmpDir, db; + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-vector-hydration-')); + db = OverGraph.open(join(tmpDir, 'db'), { + denseVector: { dimension: 2, metric: 'cosine' }, + }); + }); + after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); + + it('hydrates node vectors on full record paths', () => { + const id = db.upsertNode('VectorDoc', 'full-record-vector', { + denseVector: [1, 0], + sparseVector: [{ dimension: 0, value: 2.5 }, { dimension: 3, value: 1.25 }], + }); + + const node = db.getNode(id); + assert.deepEqual(node.denseVector, [1, 0]); + assert.deepEqual(node.sparseVector, [ + { dimension: 0, value: 2.5 }, + { dimension: 3, value: 1.25 }, + ]); + + const page = db.queryNodes({ + labelFilter: { labels: ['VectorDoc'], mode: 'all' }, + allowFullScan: true, + }); + const hydrated = page.items.find((candidate) => candidate.id === id); + assert.ok(hydrated); + assert.deepEqual(hydrated.denseVector, [1, 0]); + assert.deepEqual(hydrated.sparseVector, [ + { dimension: 0, value: 2.5 }, + { dimension: 3, value: 1.25 }, + ]); + }); +}); + describe('batch_upsert_nodes_binary / batch_upsert_edges_binary', () => { let tmpDir, db; before(() => { @@ -99,9 +290,9 @@ describe('batch_upsert_nodes_binary / batch_upsert_edges_binary', () => { it('batchUpsertNodesBinary returns Float64Array of ids', () => { const buf = packNodeBatch([ - { typeId: 1, key: 'bn1' }, - { typeId: 1, key: 'bn2', props: { x: 1 }, weight: 0.5 }, - { typeId: 2, key: 'bn3' }, + { labels: ['Person'], key: 'bn1' }, + { labels: ['Person'], key: 'bn2', props: { x: 1 }, weight: 0.5 }, + { labels: ['Company'], key: 'bn3' }, ]); const ids = db.batchUpsertNodesBinary(buf); assert.ok(ids instanceof Float64Array); @@ -111,22 +302,32 @@ describe('batch_upsert_nodes_binary / batch_upsert_edges_binary', () => { it('binary node upsert data matches getNode', () => { const buf = packNodeBatch([ - { typeId: 3, key: 'check-me', props: { color: 'red', score: 42 }, weight: 0.8 }, + { labels: ['Document'], key: 'check-me', props: { color: 'red', score: 42 }, weight: 0.8 }, ]); const [id] = db.batchUpsertNodesBinary(buf); const n = db.getNode(id); assert.ok(n); - assert.equal(n.typeId, 3); + assert.deepEqual(n.labels, ['Document']); assert.equal(n.key, 'check-me'); assert.equal(n.props.color, 'red'); assert.equal(n.props.score, 42); assert.ok(Math.abs(n.weight - 0.8) < 0.01); }); - it('binary node upsert deduplicates by (type_id, key)', () => { + it('binary node upsert preserves multiple labels', () => { + const [id] = db.batchUpsertNodesBinary(packNodeBatch([ + { labels: ['Person', 'Admin'], key: 'multi-bin', props: { role: 'owner' } }, + ])); + const node = db.getNode(id); + assert.deepEqual([...node.labels].sort(), ['Admin', 'Person']); + assert.equal(node.props.role, 'owner'); + assert.deepEqual(Array.from(db.nodesByLabels(['Person', 'Admin'])), [id]); + }); + + it('binary node upsert deduplicates by (label, key)', () => { const buf = packNodeBatch([ - { typeId: 4, key: 'dedup-bin', props: { v: 1 } }, - { typeId: 4, key: 'dedup-bin', props: { v: 2 } }, + { labels: ['Post'], key: 'dedup-bin', props: { v: 1 } }, + { labels: ['Post'], key: 'dedup-bin', props: { v: 2 } }, ]); const ids = db.batchUpsertNodesBinary(buf); assert.equal(ids[0], ids[1]); @@ -136,13 +337,13 @@ describe('batch_upsert_nodes_binary / batch_upsert_edges_binary', () => { it('batchUpsertEdgesBinary returns Float64Array of ids', () => { const nodeIds = db.batchUpsertNodesBinary(packNodeBatch([ - { typeId: 1, key: 'be-a' }, - { typeId: 1, key: 'be-b' }, - { typeId: 1, key: 'be-c' }, + { labels: ['Person'], key: 'be-a' }, + { labels: ['Person'], key: 'be-b' }, + { labels: ['Person'], key: 'be-c' }, ])); const buf = packEdgeBatch([ - { from: nodeIds[0], to: nodeIds[1], typeId: 5 }, - { from: nodeIds[1], to: nodeIds[2], typeId: 5, props: { label: 'knows' }, weight: 1.2 }, + { from: nodeIds[0], to: nodeIds[1], label: 'DEPENDS_ON'}, + { from: nodeIds[1], to: nodeIds[2], label: 'DEPENDS_ON', props: { label: 'knows' }, weight: 1.2 }, ]); const eids = db.batchUpsertEdgesBinary(buf); assert.ok(eids instanceof Float64Array); @@ -152,18 +353,18 @@ describe('batch_upsert_nodes_binary / batch_upsert_edges_binary', () => { it('binary edge upsert data matches getEdge', () => { const nodeIds = db.batchUpsertNodesBinary(packNodeBatch([ - { typeId: 1, key: 'edge-src' }, - { typeId: 1, key: 'edge-dst' }, + { labels: ['Person'], key: 'edge-src' }, + { labels: ['Person'], key: 'edge-dst' }, ])); const buf = packEdgeBatch([ - { from: nodeIds[0], to: nodeIds[1], typeId: 7, props: { kind: 'test' }, weight: 2.5 }, + { from: nodeIds[0], to: nodeIds[1], label: 'KNOWS', props: { kind: 'test' }, weight: 2.5 }, ]); const [eid] = db.batchUpsertEdgesBinary(buf); const e = db.getEdge(eid); assert.ok(e); assert.equal(e.from, nodeIds[0]); assert.equal(e.to, nodeIds[1]); - assert.equal(e.typeId, 7); + assert.equal(e.label, 'KNOWS'); assert.equal(e.props.kind, 'test'); assert.ok(Math.abs(e.weight - 2.5) < 0.01); }); @@ -175,15 +376,38 @@ describe('batch_upsert_nodes_binary / batch_upsert_edges_binary', () => { assert.equal(edgeIds.length, 0); }); + it('rejects old count-first numeric node and edge binary buffers', () => { + assert.throws( + () => db.batchUpsertNodesBinary(legacyNodeBinaryBuffer()), + /missing magic header/, + ); + + const [from, to] = db.batchUpsertNodes([ + { labels: ['Person'], key: 'legacy-edge-from' }, + { labels: ['Person'], key: 'legacy-edge-to' }, + ]); + assert.throws( + () => db.batchUpsertEdgesBinary(legacyEdgeBinaryBuffer(from, to)), + /missing magic header/, + ); + }); + + it('rejects OGNB v1 single-label binary buffers clearly', () => { + assert.throws( + () => db.batchUpsertNodesBinary(legacyNodeBinaryV1Buffer()), + /OGNB v1 single-label buffers are no longer supported/, + ); + }); + it('neighbors work with binary-inserted edges', () => { const nodeIds = db.batchUpsertNodesBinary(packNodeBatch([ - { typeId: 1, key: 'nbr-hub' }, - { typeId: 1, key: 'nbr-leaf1' }, - { typeId: 1, key: 'nbr-leaf2' }, + { labels: ['Person'], key: 'nbr-hub' }, + { labels: ['Person'], key: 'nbr-leaf1' }, + { labels: ['Person'], key: 'nbr-leaf2' }, ])); db.batchUpsertEdgesBinary(packEdgeBatch([ - { from: nodeIds[0], to: nodeIds[1], typeId: 10 }, - { from: nodeIds[0], to: nodeIds[2], typeId: 10 }, + { from: nodeIds[0], to: nodeIds[1], label: 'WORKS_AT'}, + { from: nodeIds[0], to: nodeIds[2], label: 'WORKS_AT'}, ])); const result = db.neighbors(nodeIds[0], { direction: 'outgoing' }); assert.equal(result.length, 2); @@ -195,9 +419,9 @@ describe('get_node / get_edge', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-get-')); db = freshDb(tmpDir, 'get'); - nodeId = db.upsertNode(3, 'getme', { props: { color: 'blue', score: 42 }, weight: 0.7 }); - const dst = db.upsertNode(3, 'dst'); - edgeId = db.upsertEdge(nodeId, dst, 8, { props: { rel: 'parent' }, weight: 1.5 }); + nodeId = db.upsertNode('Document', 'getme', { props: { color: 'blue', score: 42 }, weight: 0.7 }); + const dst = db.upsertNode('Document', 'dst'); + edgeId = db.upsertEdge(nodeId, dst, 'PARENT_OF', { props: { rel: 'parent' }, weight: 1.5 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -205,7 +429,7 @@ describe('get_node / get_edge', () => { const n = db.getNode(nodeId); assert.ok(n); assert.equal(n.id, nodeId); - assert.equal(n.typeId, 3); + assert.deepEqual(n.labels, ['Document']); assert.equal(n.key, 'getme'); assert.equal(n.props.color, 'blue'); assert.equal(n.props.score, 42); @@ -223,7 +447,7 @@ describe('get_node / get_edge', () => { assert.ok(e); assert.equal(e.id, edgeId); assert.equal(e.from, nodeId); - assert.equal(e.typeId, 8); + assert.equal(e.label, 'PARENT_OF'); assert.equal(e.props.rel, 'parent'); assert.ok(Math.abs(e.weight - 1.5) < 0.01); }); @@ -242,16 +466,16 @@ describe('delete_node / delete_edge', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('deleteNode removes a node', () => { - const id = db.upsertNode(1, 'todelete'); + const id = db.upsertNode('Person', 'todelete'); assert.ok(db.getNode(id)); db.deleteNode(id); assert.equal(db.getNode(id), null); }); it('deleteEdge removes an edge', () => { - const a = db.upsertNode(1, 'da'); - const b = db.upsertNode(1, 'db'); - const eid = db.upsertEdge(a, b, 1); + const a = db.upsertNode('Person', 'da'); + const b = db.upsertNode('Person', 'db'); + const eid = db.upsertEdge(a, b, 'LINKS_TO'); assert.ok(db.getEdge(eid)); db.deleteEdge(eid); assert.equal(db.getEdge(eid), null); @@ -263,14 +487,14 @@ describe('neighbors / traverse', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-nbr-')); db = freshDb(tmpDir, 'nbr'); - center = db.upsertNode(1, 'center'); - n1 = db.upsertNode(1, 'n1'); - n2 = db.upsertNode(1, 'n2'); - n3 = db.upsertNode(1, 'n3'); + center = db.upsertNode('Person', 'center'); + n1 = db.upsertNode('Person', 'n1'); + n2 = db.upsertNode('Person', 'n2'); + n3 = db.upsertNode('Person', 'n3'); // center -> n1 (type 10), center -> n2 (type 20), n1 -> n3 (type 10) - db.upsertEdge(center, n1, 10, { weight: 1.0 }); - db.upsertEdge(center, n2, 20, { weight: 2.0 }); - db.upsertEdge(n1, n3, 10, { weight: 3.0 }); + db.upsertEdge(center, n1, 'WORKS_AT', { weight: 1.0 }); + db.upsertEdge(center, n2, 'MENTIONS', { weight: 2.0 }); + db.upsertEdge(n1, n3, 'WORKS_AT', { weight: 3.0 }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -284,7 +508,7 @@ describe('neighbors / traverse', () => { }); it('neighbors with type filter', () => { - const result = db.neighbors(center, { direction: 'outgoing', typeFilter: [10] }); + const result = db.neighbors(center, { direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }); assert.equal(result.length, 1); assert.equal(result[0].nodeId, n1); }); @@ -327,7 +551,7 @@ describe('neighbors / traverse', () => { }); it('traverse supports edge filters and deterministic hit fields', () => { - const page = db.traverse(center, 2, { minDepth: 2, direction: 'outgoing', edgeTypeFilter: [10] }); + const page = db.traverse(center, 2, { minDepth: 2, direction: 'outgoing', edgeLabelFilter: ['WORKS_AT'] }); const nodeSet = new Set(page.items.map(hit => hit.nodeId)); assert.ok(nodeSet.has(n3)); assert.equal(page.items.length, 1); @@ -359,32 +583,32 @@ describe('find_nodes', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-find-')); db = freshDb(tmpDir, 'find'); - db.upsertNode(5, 'alice', { props: { city: 'NYC', active: true } }); - db.upsertNode(5, 'bob', { props: { city: 'NYC', active: false } }); - db.upsertNode(5, 'carol', { props: { city: 'LA', active: true } }); - db.upsertNode(6, 'dave', { props: { city: 'NYC' } }); + db.upsertNode('User', 'alice', { props: { city: 'NYC', active: true } }); + db.upsertNode('User', 'bob', { props: { city: 'NYC', active: false } }); + db.upsertNode('User', 'carol', { props: { city: 'LA', active: true } }); + db.upsertNode('Location', 'dave', { props: { city: 'NYC' } }); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('findNodes by string property', () => { - const ids = db.findNodes(5, 'city', 'NYC'); + const ids = db.findNodes('User', 'city', 'NYC'); assert.ok(ids instanceof Float64Array); assert.equal(ids.length, 2); }); it('findNodes by boolean property', () => { - const ids = db.findNodes(5, 'active', true); + const ids = db.findNodes('User', 'active', true); assert.equal(ids.length, 2); }); - it('findNodes respects type_id', () => { + it('findNodes respects label', () => { // type 6 has one NYC node - const ids = db.findNodes(6, 'city', 'NYC'); + const ids = db.findNodes('Location', 'city', 'NYC'); assert.equal(ids.length, 1); }); it('findNodes returns empty for no match', () => { - const ids = db.findNodes(5, 'city', 'Chicago'); + const ids = db.findNodes('User', 'city', 'Chicago'); assert.equal(ids.length, 0); }); }); @@ -398,7 +622,7 @@ describe('flush / compact', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('flush does not throw', () => { - db.upsertNode(1, 'flushed'); + db.upsertNode('Person', 'flushed'); db.flush(); }); @@ -416,11 +640,11 @@ describe('flush / compact', () => { try { for (let i = 0; i < 50; i++) { - testDb.upsertNode(1, `node-${i}`, { props: { idx: i } }); + testDb.upsertNode('Person', `node-${i}`, { props: { idx: i } }); } testDb.flush(); for (let i = 50; i < 100; i++) { - testDb.upsertNode(1, `node-${i}`, { props: { idx: i } }); + testDb.upsertNode('Person', `node-${i}`, { props: { idx: i } }); } testDb.flush(); @@ -447,23 +671,23 @@ describe('edge cases', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('upsertNode with empty props object', () => { - const id = db.upsertNode(1, 'empty-props', { props: {} }); + const id = db.upsertNode('Person', 'empty-props', { props: {} }); const n = db.getNode(id); assert.ok(n); assert.deepEqual(n.props, {}); }); it('upsertNode with null/undefined props', () => { - const id1 = db.upsertNode(1, 'null-props'); - const id2 = db.upsertNode(1, 'undef-props'); + const id1 = db.upsertNode('Person', 'null-props'); + const id2 = db.upsertNode('Person', 'undef-props'); assert.ok(db.getNode(id1)); assert.ok(db.getNode(id2)); }); it('findNodes with integer property value', () => { - db.upsertNode(9, 'scored', { props: { score: 100 } }); - db.upsertNode(9, 'scored2', { props: { score: 200 } }); - const ids = db.findNodes(9, 'score', 100); + db.upsertNode('ScoredNode', 'scored', { props: { score: 100 } }); + db.upsertNode('ScoredNode', 'scored2', { props: { score: 200 } }); + const ids = db.findNodes('ScoredNode', 'score', 100); assert.equal(ids.length, 1); }); @@ -480,7 +704,7 @@ describe('edge cases', () => { }); it('neighbors on node with no edges returns empty', () => { - const id = db.upsertNode(1, 'loner'); + const id = db.upsertNode('Person', 'loner'); const result = db.neighbors(id, { direction: 'outgoing' }); assert.equal(result.length, 0); }); @@ -495,7 +719,7 @@ describe('error handling', () => { after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('throws on invalid direction string', () => { - const id = db.upsertNode(1, 'x'); + const id = db.upsertNode('Person', 'x'); assert.throws(() => db.neighbors(id, { direction: 'sideways' }), /Invalid direction/); }); @@ -524,7 +748,7 @@ describe('error handling', () => { const dbPath = join(tmpDir, 'closed'); const closedDb = OverGraph.open(dbPath); closedDb.close(); - assert.throws(() => closedDb.upsertNode(1, 'fail'), /closed/i); + assert.throws(() => closedDb.upsertNode('Person', 'fail'), /closed/i); }); }); @@ -540,9 +764,9 @@ describe('persistence round-trip', () => { // Write phase const db1 = OverGraph.open(dbPath); - const nid = db1.upsertNode(1, 'persist-me', { props: { val: 'hello' } }); - const dst = db1.upsertNode(1, 'persist-dst'); - const eid = db1.upsertEdge(nid, dst, 5, { props: { kind: 'test' } }); + const nid = db1.upsertNode('Person', 'persist-me', { props: { val: 'hello' } }); + const dst = db1.upsertNode('Person', 'persist-dst'); + const eid = db1.upsertEdge(nid, dst, 'DEPENDS_ON', { props: { kind: 'test' } }); db1.close(); // Read phase @@ -554,29 +778,29 @@ describe('persistence round-trip', () => { const e = db2.getEdge(eid); assert.ok(e); - assert.equal(e.typeId, 5); + assert.equal(e.label, 'DEPENDS_ON'); assert.equal(e.props.kind, 'test'); db2.close(); }); }); -describe('nodesByType / edgesByType (ID-only)', () => { +describe('nodesByLabels / edgesByLabel (ID-only)', () => { let tmpDir, db; let n1, n2, n3, e1, e2; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-bytype-')); db = freshDb(tmpDir, 'bytype'); - n1 = db.upsertNode(1, 'a'); - n2 = db.upsertNode(1, 'b'); - n3 = db.upsertNode(2, 'c'); - e1 = db.upsertEdge(n1, n2, 10); - e2 = db.upsertEdge(n1, n3, 20); + n1 = db.upsertNode('Person', 'a'); + n2 = db.upsertNode('Person', 'b'); + n3 = db.upsertNode('Company', 'c'); + e1 = db.upsertEdge(n1, n2, 'WORKS_AT'); + e2 = db.upsertEdge(n1, n3, 'MENTIONS'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('nodesByType returns Float64Array of node IDs', () => { - const ids = db.nodesByType(1); + it('nodesByLabels returns Float64Array of node IDs', () => { + const ids = db.nodesByLabels('Person'); assert.ok(ids instanceof Float64Array); assert.equal(ids.length, 2); const idSet = new Set(Array.from(ids)); @@ -584,34 +808,51 @@ describe('nodesByType / edgesByType (ID-only)', () => { assert.ok(idSet.has(n2)); }); - it('nodesByType returns empty for unused type', () => { - const ids = db.nodesByType(999); + it('nodesByLabels returns empty for unused type', () => { + const ids = db.nodesByLabels('UnusedLabel'); assert.equal(ids.length, 0); }); - it('edgesByType returns Float64Array of edge IDs', () => { - const ids = db.edgesByType(10); + it('edgesByLabel returns Float64Array of edge IDs', () => { + const ids = db.edgesByLabel('WORKS_AT'); assert.ok(ids instanceof Float64Array); assert.equal(ids.length, 1); assert.equal(ids[0], e1); }); - it('edgesByType returns empty for unused type', () => { - const ids = db.edgesByType(999); + it('edgesByLabel returns empty for unused type', () => { + const ids = db.edgesByLabel('UNUSED_EDGE_TYPE'); assert.equal(ids.length, 0); }); - it('nodesByType distinguishes types', () => { - const type1Ids = db.nodesByType(1); - const type2Ids = db.nodesByType(2); + it('nodesByLabels distinguishes types', () => { + const type1Ids = db.nodesByLabels('Person'); + const type2Ids = db.nodesByLabels('Company'); assert.equal(type1Ids.length, 2); assert.equal(type2Ids.length, 1); assert.equal(type2Ids[0], n3); }); - it('edgesByType covers both types', () => { - const type10Ids = db.edgesByType(10); - const type20Ids = db.edgesByType(20); + it('does not expose singular node-label convenience aliases', () => { + for (const name of [ + 'nodesByLabel', + 'getNodesByLabel', + 'countNodesByLabel', + 'nodesByLabelPaged', + 'getNodesByLabelPaged', + 'nodesByLabelAsync', + 'getNodesByLabelAsync', + 'countNodesByLabelAsync', + 'nodesByLabelPagedAsync', + 'getNodesByLabelPagedAsync', + ]) { + assert.equal(db[name], undefined, `${name} should not be exported`); + } + }); + + it('edgesByLabel covers both types', () => { + const type10Ids = db.edgesByLabel('WORKS_AT'); + const type20Ids = db.edgesByLabel('MENTIONS'); assert.equal(type10Ids.length, 1); assert.equal(type10Ids[0], e1); assert.equal(type20Ids.length, 1); @@ -619,27 +860,27 @@ describe('nodesByType / edgesByType (ID-only)', () => { }); }); -describe('nodesByTypeAsync / edgesByTypeAsync', () => { +describe('nodesByLabelsAsync / edgesByLabelAsync', () => { let tmpDir, db; let n1, n2, e1; before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-bytype-async-')); db = freshDb(tmpDir, 'bytype-async'); - n1 = db.upsertNode(1, 'a'); - n2 = db.upsertNode(1, 'b'); - e1 = db.upsertEdge(n1, n2, 10); + n1 = db.upsertNode('Person', 'a'); + n2 = db.upsertNode('Person', 'b'); + e1 = db.upsertEdge(n1, n2, 'WORKS_AT'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it('nodesByTypeAsync returns same IDs as sync', async () => { - const syncIds = db.nodesByType(1); - const asyncIds = await db.nodesByTypeAsync(1); + it('nodesByLabelsAsync returns same IDs as sync', async () => { + const syncIds = db.nodesByLabels('Person'); + const asyncIds = await db.nodesByLabelsAsync('Person'); assert.deepEqual([...asyncIds].sort(), [...syncIds].sort()); }); - it('edgesByTypeAsync returns same IDs as sync', async () => { - const syncIds = db.edgesByType(10); - const asyncIds = await db.edgesByTypeAsync(10); + it('edgesByLabelAsync returns same IDs as sync', async () => { + const syncIds = db.edgesByLabel('WORKS_AT'); + const asyncIds = await db.edgesByLabelAsync('WORKS_AT'); assert.deepEqual([...asyncIds].sort(), [...syncIds].sort()); }); }); diff --git a/overgraph-node/__test__/time-range.mjs b/overgraph-node/__test__/time-range.mjs index 61bb282..a7bfcbe 100644 --- a/overgraph-node/__test__/time-range.mjs +++ b/overgraph-node/__test__/time-range.mjs @@ -11,26 +11,26 @@ describe('findNodesByTimeRange (sync)', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-time-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); // Create nodes: type 1 at different times - db.upsertNode(1, 'node-a'); - db.upsertNode(1, 'node-b'); - db.upsertNode(1, 'node-c'); - db.upsertNode(2, 'node-d'); // different type + db.upsertNode('Person', 'node-a'); + db.upsertNode('Person', 'node-b'); + db.upsertNode('Person', 'node-c'); + db.upsertNode('Company', 'node-d'); // different type }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns all nodes in a wide range', () => { - const ids = db.findNodesByTimeRange(1, 0, Number.MAX_SAFE_INTEGER); + const ids = db.findNodesByTimeRange('Person', 0, Number.MAX_SAFE_INTEGER); assert.ok(ids.length >= 3, `expected >= 3 nodes, got ${ids.length}`); }); it('returns empty for non-existent type', () => { - const ids = db.findNodesByTimeRange(99, 0, Number.MAX_SAFE_INTEGER); + const ids = db.findNodesByTimeRange('MissingLabel', 0, Number.MAX_SAFE_INTEGER); assert.equal(ids.length, 0); }); it('type filtering works', () => { - const type1 = db.findNodesByTimeRange(1, 0, Number.MAX_SAFE_INTEGER); - const type2 = db.findNodesByTimeRange(2, 0, Number.MAX_SAFE_INTEGER); + const type1 = db.findNodesByTimeRange('Person', 0, Number.MAX_SAFE_INTEGER); + const type2 = db.findNodesByTimeRange('Company', 0, Number.MAX_SAFE_INTEGER); assert.ok(type1.length >= 3); assert.ok(type2.length >= 1); }); @@ -42,17 +42,17 @@ describe('findNodesByTimeRange across flush', () => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-time-flush-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); // Batch 1: flush to segment - db.upsertNode(1, 'seg-a'); - db.upsertNode(1, 'seg-b'); + db.upsertNode('Person', 'seg-a'); + db.upsertNode('Person', 'seg-b'); db.flush(); // Batch 2: in memtable - db.upsertNode(1, 'mem-c'); - db.upsertNode(1, 'mem-d'); + db.upsertNode('Person', 'mem-c'); + db.upsertNode('Person', 'mem-d'); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('finds nodes across memtable and segments', () => { - const ids = db.findNodesByTimeRange(1, 0, Number.MAX_SAFE_INTEGER); + const ids = db.findNodesByTimeRange('Person', 0, Number.MAX_SAFE_INTEGER); assert.equal(ids.length, 4); }); }); @@ -62,16 +62,16 @@ describe('findNodesByTimeRange survives compaction', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-time-compact-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - for (let i = 0; i < 5; i++) db.upsertNode(1, `node-${i}`); + for (let i = 0; i < 5; i++) db.upsertNode('Person', `node-${i}`); db.flush(); - for (let i = 5; i < 10; i++) db.upsertNode(1, `node-${i}`); + for (let i = 5; i < 10; i++) db.upsertNode('Person', `node-${i}`); db.flush(); db.compact(); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('returns correct results after compaction', () => { - const ids = db.findNodesByTimeRange(1, 0, Number.MAX_SAFE_INTEGER); + const ids = db.findNodesByTimeRange('Person', 0, Number.MAX_SAFE_INTEGER); assert.equal(ids.length, 10); }); }); @@ -81,15 +81,15 @@ describe('findNodesByTimeRange respects tombstones', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-time-tomb-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - const id1 = db.upsertNode(1, 'keep'); - const id2 = db.upsertNode(1, 'delete-me'); + const id1 = db.upsertNode('Person', 'keep'); + const id2 = db.upsertNode('Person', 'delete-me'); db.flush(); db.deleteNode(id2); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('excludes deleted nodes', () => { - const ids = db.findNodesByTimeRange(1, 0, Number.MAX_SAFE_INTEGER); + const ids = db.findNodesByTimeRange('Person', 0, Number.MAX_SAFE_INTEGER); assert.equal(ids.length, 1); }); }); @@ -99,7 +99,7 @@ describe('findNodesByTimeRangePaged (sync)', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-time-paged-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - for (let i = 0; i < 10; i++) db.upsertNode(1, `node-${i}`); + for (let i = 0; i < 10; i++) db.upsertNode('Person', `node-${i}`); db.flush(); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -108,7 +108,7 @@ describe('findNodesByTimeRangePaged (sync)', () => { const allItems = []; let cursor = null; for (;;) { - const page = db.findNodesByTimeRangePaged(1, 0, Number.MAX_SAFE_INTEGER, { limit: 3, after: cursor ?? undefined }); + const page = db.findNodesByTimeRangePaged('Person', 0, Number.MAX_SAFE_INTEGER, { limit: 3, after: cursor ?? undefined }); for (let i = 0; i < page.items.length; i++) allItems.push(page.items[i]); cursor = page.nextCursor ?? null; if (cursor === null || cursor === undefined) break; @@ -117,13 +117,13 @@ describe('findNodesByTimeRangePaged (sync)', () => { }); it('first page has correct size', () => { - const page = db.findNodesByTimeRangePaged(1, 0, Number.MAX_SAFE_INTEGER, { limit: 3 }); + const page = db.findNodesByTimeRangePaged('Person', 0, Number.MAX_SAFE_INTEGER, { limit: 3 }); assert.equal(page.items.length, 3); assert.ok(page.nextCursor !== null && page.nextCursor !== undefined); }); it('unlimited page returns all', () => { - const page = db.findNodesByTimeRangePaged(1, 0, Number.MAX_SAFE_INTEGER); + const page = db.findNodesByTimeRangePaged('Person', 0, Number.MAX_SAFE_INTEGER); assert.equal(page.items.length, 10); assert.ok(page.nextCursor === null || page.nextCursor === undefined); }); @@ -134,18 +134,18 @@ describe('findNodesByTimeRange async', () => { before(() => { tmpDir = mkdtempSync(join(tmpdir(), 'overgraph-time-async-')); db = OverGraph.open(join(tmpDir, 'db'), { walSyncMode: 'immediate' }); - for (let i = 0; i < 5; i++) db.upsertNode(1, `node-${i}`); + for (let i = 0; i < 5; i++) db.upsertNode('Person', `node-${i}`); db.flush(); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true, force: true }); }); it('findNodesByTimeRangeAsync returns results', async () => { - const ids = await db.findNodesByTimeRangeAsync(1, 0, Number.MAX_SAFE_INTEGER); + const ids = await db.findNodesByTimeRangeAsync('Person', 0, Number.MAX_SAFE_INTEGER); assert.equal(ids.length, 5); }); it('findNodesByTimeRangePagedAsync paginates', async () => { - const page = await db.findNodesByTimeRangePagedAsync(1, 0, Number.MAX_SAFE_INTEGER, { limit: 2 }); + const page = await db.findNodesByTimeRangePagedAsync('Person', 0, Number.MAX_SAFE_INTEGER, { limit: 2 }); assert.equal(page.items.length, 2); assert.ok(page.nextCursor !== null && page.nextCursor !== undefined); }); diff --git a/overgraph-node/__test__/transactions.mjs b/overgraph-node/__test__/transactions.mjs index 403d1f8..ca3964a 100644 --- a/overgraph-node/__test__/transactions.mjs +++ b/overgraph-node/__test__/transactions.mjs @@ -28,14 +28,14 @@ describe('write transactions', () => { const txn = db.beginWriteTxn(); txn.stage([ - { op: 'upsertNode', alias: 'alice', typeId: 1, key: 'alice', props: { name: 'Alice' } }, - { op: 'upsertNode', alias: 'bob', typeId: 1, key: 'bob' }, + { op: 'upsertNode', alias: 'alice', labels: ['Person'], key: 'alice', props: { name: 'Alice' } }, + { op: 'upsertNode', alias: 'bob', labels: ['Person'], key: 'bob' }, { op: 'upsertEdge', alias: 'knows', from: { local: 'alice' }, to: { local: 'bob' }, - typeId: 7, + label: 'KNOWS', props: { since: 2026 }, }, ]); @@ -43,6 +43,7 @@ describe('write transactions', () => { const stagedAlice = txn.getNode({ local: 'alice' }); assert.equal(stagedAlice.id, undefined); assert.equal(stagedAlice.local, 'alice'); + assert.deepEqual(stagedAlice.labels, ['Person']); assert.equal(stagedAlice.props.name, 'Alice'); const stagedEdge = txn.getEdge({ local: 'knows' }); @@ -68,28 +69,62 @@ describe('write transactions', () => { const { db } = current; const txn = db.beginWriteTxn(); - const alice = txn.upsertNodeAs('alice', 1, 'alice', { props: { mood: 'staged' } }); - const bob = txn.upsertNodeAs('bob', 1, 'bob'); - txn.upsertEdgeAs('knows', alice, bob, 9); + const alice = txn.upsertNodeAs('alice', 'Person', 'alice', { props: { mood: 'staged' } }); + const bob = txn.upsertNodeAs('bob', 'Person', 'bob'); + txn.upsertEdgeAs('knows', alice, bob, 'FOLLOWS'); - assert.equal(txn.getNodeByKey(1, 'alice').props.mood, 'staged'); + assert.equal(txn.getNodeByKey('Person', 'alice').props.mood, 'staged'); txn.rollback(); - assert.equal(db.getNodeByKey(1, 'alice'), null); + assert.equal(db.getNodeByKey('Person', 'alice'), null); assert.throws(() => txn.commit(), /transaction is closed/); }); + it('stages node label mutations inside a transaction', () => { + current = freshDb(); + const { db } = current; + const id = db.upsertNode('Person', 'labeled'); + const txn = db.beginWriteTxn(); + + assert.equal(txn.addNodeLabel({ id }, 'Admin'), true); + assert.equal(txn.addNodeLabel({ id }, 'Admin'), false); + assert.deepEqual([...txn.getNode({ id }).labels].sort(), ['Admin', 'Person']); + assert.equal(txn.removeNodeLabel({ id }, 'Admin'), true); + assert.equal(txn.removeNodeLabel({ id }, 'Admin'), false); + assert.deepEqual(txn.getNode({ id }).labels, ['Person']); + assert.throws( + () => txn.removeNodeLabel({ id }, 'Person'), + /cannot remove the last node label/, + ); + + txn.commit(); + assert.deepEqual(db.getNode(id).labels, ['Person']); + }); + + it('async transaction node label mutations preserve order', async () => { + current = freshDb(); + const { db } = current; + const id = db.upsertNode('Person', 'async-labeled'); + const txn = db.beginWriteTxn(); + + assert.equal(await txn.addNodeLabelAsync({ id }, 'Admin'), true); + assert.deepEqual([...(await txn.getNodeAsync({ id })).labels].sort(), ['Admin', 'Person']); + assert.equal(await txn.removeNodeLabelAsync({ id }, 'Admin'), true); + await txn.commitAsync(); + assert.deepEqual(db.getNode(id).labels, ['Person']); + }); + it('unaliased builder refs can create and connect in one transaction', () => { current = freshDb(); const { db } = current; const txn = db.beginWriteTxn(); - const alice = txn.upsertNode(1, 'alice'); - const bob = txn.upsertNode(1, 'bob'); - const edgeRef = txn.upsertEdge(alice, bob, 7); + const alice = txn.upsertNode('Person', 'alice'); + const bob = txn.upsertNode('Person', 'bob'); + const edgeRef = txn.upsertEdge(alice, bob, 'KNOWS'); - assert.deepEqual(alice, { typeId: 1, key: 'alice' }); - assert.deepEqual(bob, { typeId: 1, key: 'bob' }); - assert.equal(txn.getEdge(edgeRef).typeId, 7); + assert.deepEqual(alice, { labels: ['Person'], key: 'alice' }); + assert.deepEqual(bob, { labels: ['Person'], key: 'bob' }); + assert.equal(txn.getEdge(edgeRef).label, 'KNOWS'); const result = txn.commit(); const edge = db.getEdge(result.edgeIds[0]); @@ -101,15 +136,15 @@ describe('write transactions', () => { current = freshDb(); const { db } = current; const [a, b, c, d] = Array.from(db.batchUpsertNodes([ - { typeId: 1, key: 'a' }, - { typeId: 1, key: 'b' }, - { typeId: 1, key: 'c' }, - { typeId: 1, key: 'd' }, + { labels: ['Person'], key: 'a' }, + { labels: ['Person'], key: 'b' }, + { labels: ['Person'], key: 'c' }, + { labels: ['Person'], key: 'd' }, ])); const [activeEdge, deletedEdge, cascadedEdge] = Array.from(db.batchUpsertEdges([ - { from: a, to: b, typeId: 7 }, - { from: b, to: c, typeId: 8 }, - { from: c, to: d, typeId: 9 }, + { from: a, to: b, label: 'KNOWS'}, + { from: b, to: c, label: 'PARENT_OF'}, + { from: c, to: d, label: 'FOLLOWS'}, ])); const txn = db.beginWriteTxn(); @@ -144,7 +179,7 @@ describe('write transactions', () => { const missingFieldTxn = db.beginWriteTxn(); assert.throws( - () => missingFieldTxn.stage([{ op: 'upsertNode', typeId: 1 }]), + () => missingFieldTxn.stage([{ op: 'upsertNode', labels: ['Person'] }]), /upsertNode requires key/, ); missingFieldTxn.rollback(); @@ -152,8 +187,8 @@ describe('write transactions', () => { const duplicateTxn = db.beginWriteTxn(); assert.throws( () => duplicateTxn.stage([ - { op: 'upsertNode', alias: 'n', typeId: 1, key: 'n1' }, - { op: 'upsertNode', alias: 'n', typeId: 1, key: 'n2' }, + { op: 'upsertNode', alias: 'n', labels: ['Person'], key: 'n1' }, + { op: 'upsertNode', alias: 'n', labels: ['Person'], key: 'n2' }, ]), /duplicate transaction node alias/, ); @@ -163,24 +198,24 @@ describe('write transactions', () => { it('conflicts with implicit writes and closes after failed commit', () => { current = freshDb(); const { db } = current; - db.upsertNode(1, 'base', { props: { v: 1 } }); + db.upsertNode('Person', 'base', { props: { v: 1 } }); const txn = db.beginWriteTxn(); - txn.upsertNode(1, 'base', { props: { v: 2 } }); - db.upsertNode(1, 'base', { props: { v: 3 } }); + txn.upsertNode('Person', 'base', { props: { v: 2 } }); + db.upsertNode('Person', 'base', { props: { v: 3 } }); assert.throws(() => txn.commit(), /transaction conflict/); assert.throws(() => txn.rollback(), /transaction is closed/); - assert.equal(db.getNodeByKey(1, 'base').props.v, 3); + assert.equal(db.getNodeByKey('Person', 'base').props.v, 3); }); it('begin and commit respect database and transaction close states', () => { current = freshDb(); const { db } = current; const commitTxn = db.beginWriteTxn(); - commitTxn.upsertNodeAs('n', 1, 'n'); + commitTxn.upsertNodeAs('n', 'Person', 'n'); const rollbackTxn = db.beginWriteTxn(); - rollbackTxn.upsertNodeAs('m', 1, 'm'); + rollbackTxn.upsertNodeAs('m', 'Person', 'm'); db.close(); assert.throws(() => db.beginWriteTxn(), /Database is closed/); diff --git a/overgraph-node/__test__/types/declarations.ts b/overgraph-node/__test__/types/declarations.ts new file mode 100644 index 0000000..76918f6 --- /dev/null +++ b/overgraph-node/__test__/types/declarations.ts @@ -0,0 +1,95 @@ +import type { + EdgeInput, + EdgeLabelInfo, + NeighborEntry, + NeighborsOptions, + NodeInput, + NodeLabelFilter, + NodeLabelInfo, + OverGraph, +} from '../../index.js' +import type { + GraphPatternRequest, + QueryEdgeRequest, + QueryPlanNode, +} from '../../query-types.js' + +declare const db: OverGraph + +const edgeInput: EdgeInput = { + from: 1, + to: 2, + label: 'WORKS_AT', + props: { since: 2026 }, +} + +const edgeInfo: EdgeLabelInfo = { + label: 'WORKS_AT', + labelId: 1, +} + +const nodeInfo: NodeLabelInfo = { + label: 'Person', + labelId: 1, +} + +const neighbor: NeighborEntry = { + nodeId: 2, + edgeId: 3, + label: 'WORKS_AT', + weight: 1, + validFrom: 0, + validTo: 0, +} + +const neighborOptions: NeighborsOptions = { + direction: 'outgoing', + edgeLabelFilter: ['WORKS_AT'], +} + +const edgeQuery: QueryEdgeRequest = { + label: 'WORKS_AT', + allowFullScan: true, +} + +const nodeInput: NodeInput = { + labels: ['Person', 'Admin'], + key: 'alice', + props: { active: true }, +} + +const nodeLabelFilter: NodeLabelFilter = { + labels: ['Person'], + mode: 'all', +} + +const graphPattern: GraphPatternRequest = { + nodes: [ + { alias: 'person', labelFilter: { labels: ['Person'], mode: 'all' } }, + { alias: 'company', labelFilter: { labels: ['Company'], mode: 'all' } }, + ], + edges: [ + { + fromAlias: 'person', + toAlias: 'company', + labelFilter: ['WORKS_AT'], + }, + ], + limit: 10, +} + +const fallbackEdgeLabelScan: QueryPlanNode = { + kind: 'fallback_edge_label_scan', +} + +void db +void edgeInput +void edgeInfo +void nodeInfo +void neighbor +void neighborOptions +void edgeQuery +void nodeInput +void nodeLabelFilter +void graphPattern +void fallbackEdgeLabelScan diff --git a/overgraph-node/__test__/vector-search.mjs b/overgraph-node/__test__/vector-search.mjs index 27fba3c..c70c6c2 100644 --- a/overgraph-node/__test__/vector-search.mjs +++ b/overgraph-node/__test__/vector-search.mjs @@ -14,19 +14,19 @@ function freshDb(dir, name) { function setupHybridDb(dir) { const db = freshDb(dir, 'hybrid'); // Node 1: dense rank #1, sparse rank #4 - const n1 = db.upsertNode(1, 'n1', { weight: 1.0, + const n1 = db.upsertNode('Person', 'n1', { weight: 1.0, denseVector: [0.95, 0.05, 0.05, 0.05], sparseVector: [{ dimension: 0, value: 0.2 }, { dimension: 1, value: 0.1 }] }); // Node 2: dense rank #4, sparse rank #1 - const n2 = db.upsertNode(1, 'n2', { weight: 1.0, + const n2 = db.upsertNode('Person', 'n2', { weight: 1.0, denseVector: [0.3, 0.5, 0.5, 0.5], sparseVector: [{ dimension: 0, value: 0.9 }, { dimension: 1, value: 0.8 }, { dimension: 2, value: 0.7 }] }); // Node 3: dense rank #2, sparse rank #2 (balanced) - const n3 = db.upsertNode(1, 'n3', { weight: 1.0, + const n3 = db.upsertNode('Person', 'n3', { weight: 1.0, denseVector: [0.85, 0.1, 0.1, 0.1], sparseVector: [{ dimension: 0, value: 0.7 }, { dimension: 1, value: 0.6 }] }); // Node 4: dense rank #3, sparse rank #3 - const n4 = db.upsertNode(1, 'n4', { weight: 1.0, + const n4 = db.upsertNode('Person', 'n4', { weight: 1.0, denseVector: [0.6, 0.3, 0.3, 0.3], sparseVector: [{ dimension: 0, value: 0.5 }, { dimension: 2, value: 0.3 }] }); // Node 5: dense rank #5, sparse rank #5 - const n5 = db.upsertNode(1, 'n5', { weight: 1.0, + const n5 = db.upsertNode('Person', 'n5', { weight: 1.0, denseVector: [0.1, 0.4, 0.6, 0.6], sparseVector: [{ dimension: 1, value: 0.1 }] }); db.flush(); return { db, ids: [n1, n2, n3, n4, n5] }; @@ -169,11 +169,11 @@ describe('vectorSearch with scope (sync)', () => { db = freshDb(tmpDir, 'scope'); ids = []; for (let i = 0; i < 4; i++) { - ids.push(db.upsertNode(1, `n${i}`, { weight: 1.0, + ids.push(db.upsertNode('Person', `n${i}`, { weight: 1.0, denseVector: [1, 0, 0, 0], sparseVector: [{ dimension: 0, value: (i + 1) * 0.3 }] })); } - db.upsertEdge(ids[0], ids[1], 1); - db.upsertEdge(ids[1], ids[2], 1); + db.upsertEdge(ids[0], ids[1], 'LINKS_TO'); + db.upsertEdge(ids[1], ids[2], 'LINKS_TO'); db.flush(); }); after(() => { db.close(); rmSync(tmpDir, { recursive: true }); }); diff --git a/overgraph-node/helpers/pack-binary.mjs b/overgraph-node/helpers/pack-binary.mjs index 5656fd9..b55e804 100644 --- a/overgraph-node/helpers/pack-binary.mjs +++ b/overgraph-node/helpers/pack-binary.mjs @@ -5,31 +5,62 @@ * batchUpsertNodesBinary() and batchUpsertEdgesBinary(). */ +const NODE_BATCH_MAGIC = Buffer.from('OGNB'); +const EDGE_BATCH_MAGIC = Buffer.from('OGEB'); +const NODE_BINARY_BATCH_VERSION = 2; +const EDGE_BINARY_BATCH_VERSION = 1; +const BINARY_BATCH_HEADER_BYTES = 10; +const MAX_NODE_LABELS_PER_NODE = 10; + /** * Pack an array of node objects into a binary Buffer. * * Format (little-endian): - * [count: u32] + * [magic: "OGNB"][version: u16 = 2][count: u32] * per node: - * [type_id: u32][weight: f32][key_len: u16][key: utf8] + * [label_count: u8] repeated [label_len: u16][label: utf8] + * [weight: f32][key_len: u16][key: utf8] * [props_len: u32][props: json utf8] * - * @param {Array<{typeId: number, key: string, props?: object, weight?: number}>} nodes + * @param {Array<{labels: string | string[], key: string, props?: object, weight?: number}>} nodes * @returns {Buffer} */ export function packNodeBatch(nodes) { - let size = 4; + let size = BINARY_BATCH_HEADER_BYTES; const encoded = nodes.map((n) => { + const labels = typeof n.labels === 'string' ? [n.labels] : n.labels; + if (!Array.isArray(labels)) { + throw new TypeError('node.labels must be a string or string array'); + } + if (labels.length < 1 || labels.length > MAX_NODE_LABELS_PER_NODE) { + throw new RangeError('node.labels must contain 1..10 labels'); + } + const labelBufs = labels.map((label, index) => { + if (typeof label !== 'string') { + throw new TypeError(`node.labels[${index}] must be a string`); + } + const labelBuf = Buffer.from(label, 'utf8'); + if (labelBuf.length < 1 || labelBuf.length > 255) { + throw new RangeError(`node.labels[${index}] must encode to 1..255 UTF-8 bytes`); + } + return labelBuf; + }); const keyBuf = Buffer.from(n.key, 'utf8'); const propsBuf = n.props ? Buffer.from(JSON.stringify(n.props), 'utf8') : null; - size += 4 + 4 + 2 + keyBuf.length + 4 + (propsBuf ? propsBuf.length : 0); - return { keyBuf, propsBuf, typeId: n.typeId, weight: n.weight ?? 1.0 }; + size += 1 + labelBufs.reduce((sum, labelBuf) => sum + 2 + labelBuf.length, 0) + 4 + 2 + keyBuf.length + 4 + (propsBuf ? propsBuf.length : 0); + return { labelBufs, keyBuf, propsBuf, weight: n.weight ?? 1.0 }; }); const buf = Buffer.alloc(size); let off = 0; + NODE_BATCH_MAGIC.copy(buf, off); off += 4; + buf.writeUInt16LE(NODE_BINARY_BATCH_VERSION, off); off += 2; buf.writeUInt32LE(nodes.length, off); off += 4; - for (const { typeId, weight, keyBuf, propsBuf } of encoded) { - buf.writeUInt32LE(typeId, off); off += 4; + for (const { labelBufs, weight, keyBuf, propsBuf } of encoded) { + buf.writeUInt8(labelBufs.length, off); off += 1; + for (const labelBuf of labelBufs) { + buf.writeUInt16LE(labelBuf.length, off); off += 2; + labelBuf.copy(buf, off); off += labelBuf.length; + } buf.writeFloatLE(weight, off); off += 4; buf.writeUInt16LE(keyBuf.length, off); off += 2; keyBuf.copy(buf, off); off += keyBuf.length; @@ -47,9 +78,9 @@ export function packNodeBatch(nodes) { * Pack an array of edge objects into a binary Buffer. * * Format (little-endian): - * [count: u32] + * [magic: "OGEB"][version: u16 = 1][count: u32] * per edge: - * [from: u64][to: u64][type_id: u32][weight: f32] + * [from: u64][to: u64][label_len: u16][label: utf8][weight: f32] * [valid_from: i64][valid_to: i64] * [props_len: u32][props: json utf8] * @@ -57,25 +88,32 @@ export function packNodeBatch(nodes) { * (created_at and i64::MAX respectively). Actual Unix epoch 0 cannot be * represented. This is acceptable for practical graph database timestamps. * - * @param {Array<{from: number, to: number, typeId: number, props?: object, weight?: number, validFrom?: number, validTo?: number}>} edges + * @param {Array<{from: number, to: number, label: string, props?: object, weight?: number, validFrom?: number, validTo?: number}>} edges * @returns {Buffer} */ export function packEdgeBatch(edges) { - let size = 4; + let size = BINARY_BATCH_HEADER_BYTES; const encoded = edges.map((e) => { + const labelBuf = Buffer.from(e.label, 'utf8'); const propsBuf = e.props ? Buffer.from(JSON.stringify(e.props), 'utf8') : null; - size += 8 + 8 + 4 + 4 + 8 + 8 + 4 + (propsBuf ? propsBuf.length : 0); - return { from: e.from, to: e.to, typeId: e.typeId, weight: e.weight, validFrom: e.validFrom, validTo: e.validTo, propsBuf }; + if (labelBuf.length < 1 || labelBuf.length > 255) { + throw new RangeError('edge.label must encode to 1..255 UTF-8 bytes'); + } + size += 8 + 8 + 2 + labelBuf.length + 4 + 8 + 8 + 4 + (propsBuf ? propsBuf.length : 0); + return { from: e.from, to: e.to, labelBuf, weight: e.weight, validFrom: e.validFrom, validTo: e.validTo, propsBuf }; }); const buf = Buffer.alloc(size); let off = 0; + EDGE_BATCH_MAGIC.copy(buf, off); off += 4; + buf.writeUInt16LE(EDGE_BINARY_BATCH_VERSION, off); off += 2; buf.writeUInt32LE(edges.length, off); off += 4; for (const e of encoded) { if (!Number.isSafeInteger(e.from) || e.from < 0) throw new RangeError('edge.from must be a safe non-negative integer'); if (!Number.isSafeInteger(e.to) || e.to < 0) throw new RangeError('edge.to must be a safe non-negative integer'); buf.writeBigUInt64LE(BigInt(e.from), off); off += 8; buf.writeBigUInt64LE(BigInt(e.to), off); off += 8; - buf.writeUInt32LE(e.typeId, off); off += 4; + buf.writeUInt16LE(e.labelBuf.length, off); off += 2; + e.labelBuf.copy(buf, off); off += e.labelBuf.length; buf.writeFloatLE(e.weight ?? 1.0, off); off += 4; buf.writeBigInt64LE(BigInt(e.validFrom ?? 0), off); off += 8; buf.writeBigInt64LE(BigInt(e.validTo ?? 0), off); off += 8; diff --git a/overgraph-node/index.d.ts b/overgraph-node/index.d.ts index 215ebc8..d353e0d 100644 --- a/overgraph-node/index.d.ts +++ b/overgraph-node/index.d.ts @@ -1,19 +1,19 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ -export declare class JsEdgePageResult { - get items(): Array +export declare class EdgePageResult { + get items(): Array get nextCursor(): number | null } /** - * Edge record: eager primitives, lazy props. Props are Arc-shared so + * Edge view: eager primitives, lazy props. Props are Arc-shared so * container getters (page results, subgraph) avoid cloning the BTreeMap. */ -export declare class JsEdgeRecord { +export declare class EdgeView { get id(): number get from(): number get to(): number - get typeId(): number + get label(): string get props(): Record get createdAt(): number get updatedAt(): number @@ -22,129 +22,145 @@ export declare class JsEdgeRecord { get validTo(): number } -export declare class JsNeighborPageResult { - get items(): Array +export declare class NeighborPageResult { + get items(): Array get nextCursor(): number | null } -export declare class JsNodePageResult { - get items(): Array +export declare class NodePageResult { + get items(): Array get nextCursor(): number | null } /** - * Node record: eager primitives, lazy props. Props are Arc-shared so + * Node view: eager primitives, lazy props. Props are Arc-shared so * container getters (page results, subgraph) avoid cloning the BTreeMap. */ -export declare class JsNodeRecord { +export declare class NodeView { get id(): number - get typeId(): number + get labels(): Array get key(): string get props(): Record get createdAt(): number get updatedAt(): number get weight(): number -} - -export declare class JsSubgraphResult { - get nodes(): Array - get edges(): Array + get denseVector(): Array | null + get sparseVector(): Array | null } export declare class OverGraph { - static open(path: string, options?: JsDbOptions | undefined | null): OverGraph - close(options?: JsCloseOptions | undefined | null): void - upsertNode(typeId: number, key: string, options?: JsUpsertNodeOptions | undefined | null): number - upsertEdge(from: number, to: number, typeId: number, options?: JsUpsertEdgeOptions | undefined | null): number - batchUpsertNodes(nodes: Array): Float64Array - batchUpsertEdges(edges: Array): Float64Array + static open(path: string, options?: DbOptions | undefined | null): OverGraph + close(options?: CloseOptions | undefined | null): void + ensureNodeLabel(label: string): number + ensureEdgeLabel(label: string): number + getNodeLabelId(label: string): number | null + getEdgeLabelId(label: string): number | null + getNodeLabel(labelId: number): string | null + getEdgeLabel(labelId: number): string | null + listNodeLabels(): Array + listEdgeLabels(): Array + upsertNode(labels: string | string[], key: string, options?: UpsertNodeOptions | null): number + addNodeLabel(nodeId: number, label: string): boolean + removeNodeLabel(nodeId: number, label: string): boolean + upsertEdge(from: number, to: number, label: string, options?: UpsertEdgeOptions | undefined | null): number + batchUpsertNodes(nodes: Array): Float64Array + batchUpsertEdges(edges: Array): Float64Array /** * Batch upsert nodes from a packed binary Buffer. See `packNodeBatch()` in JS. * * Binary format (little-endian): - * [count: u32] + * [magic: 4 bytes "OGNB"][version: u16 = 2][count: u32] * per node: - * [type_id: u32][weight: f32][key_len: u16][key: utf8][props_len: u32][props: json utf8] + * [label_count: u8] repeated [label_len: u16][label: utf8][weight: f32] + * [key_len: u16][key: utf8][props_len: u32][props: json utf8] */ batchUpsertNodesBinary(buffer: Buffer): Float64Array /** * Batch upsert edges from a packed binary Buffer. See `packEdgeBatch()` in JS. * * Binary format (little-endian): - * [count: u32] + * [magic: 4 bytes "OGEB"][version: u16 = 1][count: u32] * per edge: - * [from: u64][to: u64][type_id: u32][weight: f32] + * [from: u64][to: u64][label_len: u16][label: utf8][weight: f32] * [valid_from: i64][valid_to: i64][props_len: u32][props: json utf8] + * In this packed format, valid_from=0 and valid_to=0 are sentinels for + * engine defaults (created_at and no expiration), not explicit epoch 0. */ batchUpsertEdgesBinary(buffer: Buffer): Float64Array - getNode(id: number): JsNodeRecord | null - getEdge(id: number): JsEdgeRecord | null - getNodeByKey(typeId: number, key: string): JsNodeRecord | null - getEdgeByTriple(from: number, to: number, typeId: number): JsEdgeRecord | null - getNodes(ids: Array): Array - getNodesByKeys(keys: Array): Array - getEdges(ids: Array): Array + getNode(id: number): NodeView | null + getEdge(id: number): EdgeView | null + getNodeByKey(label: string, key: string): NodeView | null + getEdgeByTriple(from: number, to: number, label: string): EdgeView | null + getNodes(ids: Array): Array + getNodesByKeys(keys: Array): Array + getEdges(ids: Array): Array deleteNode(id: number): void deleteEdge(id: number): void - invalidateEdge(id: number, validTo: number): JsEdgeRecord | null - graphPatch(patch: JsGraphPatch): JsPatchResult - beginWriteTxn(): JsWriteTxn - beginWriteTxnAsync(): Promise - prune(policy: JsPrunePolicy): JsPruneResult - setPrunePolicy(name: string, policy: JsPrunePolicy): void + invalidateEdge(id: number, validTo: number): EdgeView | null + graphPatch(patch: GraphPatch): PatchResult + beginWriteTxn(): WriteTxn + beginWriteTxnAsync(): Promise + prune(policy: PrunePolicy): PruneResult + setPrunePolicy(name: string, policy: PrunePolicy): void removePrunePolicy(name: string): boolean - listPrunePolicies(): Array - setPrunePolicyAsync(name: string, policy: JsPrunePolicy): Promise + listPrunePolicies(): Array + setPrunePolicyAsync(name: string, policy: PrunePolicy): Promise removePrunePolicyAsync(name: string): Promise - listPrunePoliciesAsync(): Promise> - neighbors(nodeId: number, options?: JsNeighborsOptions | undefined | null): Array - traverse(startNodeId: number, maxDepth: number, options?: JsTraverseOptions | undefined | null): JsTraversalPageResult - topKNeighbors(nodeId: number, k: number, options?: JsTopKNeighborsOptions | undefined | null): Array - extractSubgraph(startNodeId: number, maxDepth: number, options?: JsExtractSubgraphOptions | undefined | null): JsSubgraphResult + listPrunePoliciesAsync(): Promise> + neighbors(nodeId: number, options?: NeighborsOptions | undefined | null): Array + traverse(startNodeId: number, maxDepth: number, options?: TraverseOptions | undefined | null): TraversalPageResult + topKNeighbors(nodeId: number, k: number, options?: TopKNeighborsOptions | undefined | null): Array + extractSubgraph(startNodeId: number, maxDepth: number, options?: ExtractSubgraphOptions | undefined | null): SubgraphResult /** * Batch neighbor query: fetch neighbors for multiple nodes in one call. * Returns an array of entries, each mapping a query node to its neighbors. */ - neighborsBatch(nodeIds: Array, options?: JsNeighborsBatchOptions | undefined | null): Array - degree(nodeId: number, options?: JsDegreeOptions | undefined | null): number - sumEdgeWeights(nodeId: number, options?: JsSumEdgeWeightsOptions | undefined | null): number - avgEdgeWeight(nodeId: number, options?: JsAvgEdgeWeightOptions | undefined | null): number | null - degrees(nodeIds: Array, options?: JsDegreesOptions | undefined | null): Array - shortestPath(from: number, to: number, options?: JsShortestPathOptions | undefined | null): JsShortestPath | null - isConnected(from: number, to: number, options?: JsIsConnectedOptions | undefined | null): boolean - allShortestPaths(from: number, to: number, options?: JsAllShortestPathsOptions | undefined | null): Array - findNodes(typeId: number, propKey: string, propValue: any): Float64Array - queryNodeIds(request: import('./query-types').QueryNodeRequest): JsIdPageResult - queryNodes(request: import('./query-types').QueryNodeRequest): JsNodePageResult + neighborsBatch(nodeIds: Array, options?: NeighborsBatchOptions | undefined | null): Array + degree(nodeId: number, options?: DegreeOptions | undefined | null): number + sumEdgeWeights(nodeId: number, options?: SumEdgeWeightsOptions | undefined | null): number + avgEdgeWeight(nodeId: number, options?: AvgEdgeWeightOptions | undefined | null): number | null + degrees(nodeIds: Array, options?: DegreesOptions | undefined | null): Array + shortestPath(from: number, to: number, options?: ShortestPathOptions | undefined | null): ShortestPath | null + isConnected(from: number, to: number, options?: IsConnectedOptions | undefined | null): boolean + allShortestPaths(from: number, to: number, options?: AllShortestPathsOptions | undefined | null): Array + findNodes(label: string, propKey: string, propValue: any): Float64Array + queryNodeIds(request: import('./query-types').QueryNodeRequest): IdPageResult + queryNodes(request: import('./query-types').QueryNodeRequest): NodePageResult + queryEdgeIds(request: import('./query-types').QueryEdgeRequest): IdPageResult + queryEdges(request: import('./query-types').QueryEdgeRequest): EdgePageResult queryPattern(request: import('./query-types').GraphPatternRequest): import('./query-types').QueryPatternResult explainNodeQuery(request: import('./query-types').QueryNodeRequest): import('./query-types').QueryPlan + explainEdgeQuery(request: import('./query-types').QueryEdgeRequest): import('./query-types').QueryPlan explainPatternQuery(request: import('./query-types').GraphPatternRequest): import('./query-types').QueryPlan - ensureNodePropertyIndex(typeId: number, propKey: string, kind: JsSecondaryIndexKind): JsNodePropertyIndexInfo - dropNodePropertyIndex(typeId: number, propKey: string, kind: JsSecondaryIndexKind): boolean - listNodePropertyIndexes(): Array - /** Return all node IDs of a given type (unpaged). */ - nodesByType(typeId: number): Float64Array - /** Return all edge IDs of a given type (unpaged). */ - edgesByType(typeId: number): Float64Array - getNodesByType(typeId: number): Array - getEdgesByType(typeId: number): Array - countNodesByType(typeId: number): number - countEdgesByType(typeId: number): number - nodesByTypePaged(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): JsIdPageResult - edgesByTypePaged(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): JsIdPageResult - getNodesByTypePaged(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): JsNodePageResult - getEdgesByTypePaged(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): JsEdgePageResult - findNodesPaged(typeId: number, propKey: string, propValue: any, options?: JsFindNodesPagedOptions | undefined | null): JsIdPageResult - findNodesByTimeRange(typeId: number, fromMs: number, toMs: number): Float64Array - findNodesRange(typeId: number, propKey: string, lower?: JsPropertyRangeBound | undefined | null, upper?: JsPropertyRangeBound | undefined | null): Float64Array - findNodesByTimeRangePaged(typeId: number, fromMs: number, toMs: number, options?: JsFindNodesByTimeRangePagedOptions | undefined | null): JsIdPageResult - findNodesRangePaged(typeId: number, propKey: string, lower?: JsPropertyRangeBound | undefined | null, upper?: JsPropertyRangeBound | undefined | null, options?: JsFindNodesRangePagedOptions | undefined | null): JsPropertyRangePageResult - personalizedPagerank(seedNodeIds: Array, options?: JsPersonalizedPagerankOptions | undefined | null): JsPprResult - exportAdjacency(options?: JsExportOptions | undefined | null): JsAdjacencyExport - neighborsPaged(nodeId: number, options?: JsNeighborsPagedOptions | undefined | null): JsNeighborPageResult - connectedComponents(options?: JsConnectedComponentsOptions | undefined | null): Array - componentOf(nodeId: number, options?: JsComponentOfOptions | undefined | null): Float64Array - vectorSearch(mode: string, options: JsVectorSearchOptions): Array + ensureNodePropertyIndex(label: string, propKey: string, kind: SecondaryIndexKind): NodePropertyIndexInfo + dropNodePropertyIndex(label: string, propKey: string, kind: SecondaryIndexKind): boolean + listNodePropertyIndexes(): Array + ensureEdgePropertyIndex(label: string, propKey: string, kind: SecondaryIndexKind): EdgePropertyIndexInfo + dropEdgePropertyIndex(label: string, propKey: string, kind: SecondaryIndexKind): boolean + listEdgePropertyIndexes(): Array + /** Return all node IDs containing every supplied node label (unpaged). */ + nodesByLabels(labels: string | string[]): Float64Array + /** Return all edge IDs of a given label (unpaged). */ + edgesByLabel(label: string): Float64Array + getNodesByLabels(labels: string | string[]): Array + getEdgesByLabel(label: string): Array + countNodesByLabels(labels: string | string[]): number + countEdgesByLabel(label: string): number + nodesByLabelsPaged(labels: string | string[], limit?: number | null, after?: number | null): IdPageResult + edgesByLabelPaged(label: string, limit?: number | undefined | null, after?: number | undefined | null): IdPageResult + getNodesByLabelsPaged(labels: string | string[], limit?: number | null, after?: number | null): NodePageResult + getEdgesByLabelPaged(label: string, limit?: number | undefined | null, after?: number | undefined | null): EdgePageResult + findNodesPaged(label: string, propKey: string, propValue: any, options?: FindNodesPagedOptions | undefined | null): IdPageResult + findNodesByTimeRange(label: string, fromMs: number, toMs: number): Float64Array + findNodesRange(label: string, propKey: string, lower?: PropertyRangeBound | undefined | null, upper?: PropertyRangeBound | undefined | null): Float64Array + findNodesByTimeRangePaged(label: string, fromMs: number, toMs: number, options?: FindNodesByTimeRangePagedOptions | undefined | null): IdPageResult + findNodesRangePaged(label: string, propKey: string, lower?: PropertyRangeBound | undefined | null, upper?: PropertyRangeBound | undefined | null, options?: FindNodesRangePagedOptions | undefined | null): PropertyRangePageResult + personalizedPagerank(seedNodeIds: Array, options?: PersonalizedPagerankOptions | undefined | null): PprResult + exportAdjacency(options?: ExportOptions | undefined | null): AdjacencyExport + neighborsPaged(nodeId: number, options?: NeighborsPagedOptions | undefined | null): NeighborPageResult + connectedComponents(options?: ConnectedComponentsOptions | undefined | null): Array + componentOf(nodeId: number, options?: ComponentOfOptions | undefined | null): Float64Array + vectorSearch(mode: string, options: VectorSearchOptions): Array /** * Force an immediate WAL fsync. In GroupCommit mode, blocks until all * buffered data is durable. In Immediate mode, this is a no-op. @@ -152,134 +168,161 @@ export declare class OverGraph { sync(): void flush(): void ingestMode(): void - endIngest(): JsCompactionStats | null - compact(): JsCompactionStats | null + endIngest(): CompactionStats | null + compact(): CompactionStats | null /** * Compact with a progress callback. The callback receives a progress object * and should return `true` to continue or `false` to cancel. * Runs synchronously. Blocks the event loop. */ - compactWithProgress(callback: (progress: JsCompactionProgress) => boolean): JsCompactionStats | null - stats(): JsDbStats - closeAsync(options?: JsCloseOptions | undefined | null): Promise - statsAsync(): Promise - upsertNodeAsync(typeId: number, key: string, options?: JsUpsertNodeOptions | undefined | null): Promise - upsertEdgeAsync(from: number, to: number, typeId: number, options?: JsUpsertEdgeOptions | undefined | null): Promise - batchUpsertNodesAsync(nodes: Array): Promise - batchUpsertEdgesAsync(edges: Array): Promise + compactWithProgress(callback: (progress: CompactionProgress) => boolean): CompactionStats | null + stats(): DbStats + scrub(): ScrubReport + closeAsync(options?: CloseOptions | undefined | null): Promise + statsAsync(): Promise + scrubAsync(): Promise + ensureNodeLabelAsync(label: string): Promise + ensureEdgeLabelAsync(label: string): Promise + getNodeLabelIdAsync(label: string): Promise + getEdgeLabelIdAsync(label: string): Promise + getNodeLabelAsync(labelId: number): Promise + getEdgeLabelAsync(labelId: number): Promise + listNodeLabelsAsync(): Promise> + listEdgeLabelsAsync(): Promise> + upsertNodeAsync(labels: string | string[], key: string, options?: UpsertNodeOptions | null): Promise + addNodeLabelAsync(nodeId: number, label: string): Promise + removeNodeLabelAsync(nodeId: number, label: string): Promise + upsertEdgeAsync(from: number, to: number, label: string, options?: UpsertEdgeOptions | undefined | null): Promise + batchUpsertNodesAsync(nodes: Array): Promise + batchUpsertEdgesAsync(edges: Array): Promise batchUpsertNodesBinaryAsync(buffer: Buffer): Promise batchUpsertEdgesBinaryAsync(buffer: Buffer): Promise - getNodeAsync(id: number): Promise - getEdgeAsync(id: number): Promise - getNodeByKeyAsync(typeId: number, key: string): Promise - getEdgeByTripleAsync(from: number, to: number, typeId: number): Promise - getNodesAsync(ids: Array): Promise> - getNodesByKeysAsync(keys: Array): Promise> - getEdgesAsync(ids: Array): Promise> + getNodeAsync(id: number): Promise + getEdgeAsync(id: number): Promise + getNodeByKeyAsync(label: string, key: string): Promise + getEdgeByTripleAsync(from: number, to: number, label: string): Promise + getNodesAsync(ids: Array): Promise> + getNodesByKeysAsync(keys: Array): Promise> + getEdgesAsync(ids: Array): Promise> deleteNodeAsync(id: number): Promise deleteEdgeAsync(id: number): Promise - invalidateEdgeAsync(id: number, validTo: number): Promise - graphPatchAsync(patch: JsGraphPatch): Promise - pruneAsync(policy: JsPrunePolicy): Promise - neighborsAsync(nodeId: number, options?: JsNeighborsOptions | undefined | null): Promise> - traverseAsync(startNodeId: number, maxDepth: number, options?: JsTraverseOptions | undefined | null): Promise - topKNeighborsAsync(nodeId: number, k: number, options?: JsTopKNeighborsOptions | undefined | null): Promise> - extractSubgraphAsync(startNodeId: number, maxDepth: number, options?: JsExtractSubgraphOptions | undefined | null): Promise - findNodesAsync(typeId: number, propKey: string, propValue: any): Promise - queryNodeIdsAsync(request: import('./query-types').QueryNodeRequest): Promise - queryNodesAsync(request: import('./query-types').QueryNodeRequest): Promise + invalidateEdgeAsync(id: number, validTo: number): Promise + graphPatchAsync(patch: GraphPatch): Promise + pruneAsync(policy: PrunePolicy): Promise + neighborsAsync(nodeId: number, options?: NeighborsOptions | undefined | null): Promise> + traverseAsync(startNodeId: number, maxDepth: number, options?: TraverseOptions | undefined | null): Promise + topKNeighborsAsync(nodeId: number, k: number, options?: TopKNeighborsOptions | undefined | null): Promise> + extractSubgraphAsync(startNodeId: number, maxDepth: number, options?: ExtractSubgraphOptions | undefined | null): Promise + findNodesAsync(label: string, propKey: string, propValue: any): Promise + queryNodeIdsAsync(request: import('./query-types').QueryNodeRequest): Promise + queryNodesAsync(request: import('./query-types').QueryNodeRequest): Promise + queryEdgeIdsAsync(request: import('./query-types').QueryEdgeRequest): Promise + queryEdgesAsync(request: import('./query-types').QueryEdgeRequest): Promise queryPatternAsync(request: import('./query-types').GraphPatternRequest): Promise explainNodeQueryAsync(request: import('./query-types').QueryNodeRequest): Promise + explainEdgeQueryAsync(request: import('./query-types').QueryEdgeRequest): Promise explainPatternQueryAsync(request: import('./query-types').GraphPatternRequest): Promise - ensureNodePropertyIndexAsync(typeId: number, propKey: string, kind: JsSecondaryIndexKind): Promise - dropNodePropertyIndexAsync(typeId: number, propKey: string, kind: JsSecondaryIndexKind): Promise - listNodePropertyIndexesAsync(): Promise> - findNodesRangeAsync(typeId: number, propKey: string, lower?: JsPropertyRangeBound | undefined | null, upper?: JsPropertyRangeBound | undefined | null): Promise - findNodesRangePagedAsync(typeId: number, propKey: string, lower?: JsPropertyRangeBound | undefined | null, upper?: JsPropertyRangeBound | undefined | null, options?: JsFindNodesRangePagedOptions | undefined | null): Promise - getNodesByTypeAsync(typeId: number): Promise> - getEdgesByTypeAsync(typeId: number): Promise> - countNodesByTypeAsync(typeId: number): Promise - countEdgesByTypeAsync(typeId: number): Promise - nodesByTypeAsync(typeId: number): Promise - edgesByTypeAsync(typeId: number): Promise - neighborsBatchAsync(nodeIds: Array, options?: JsNeighborsBatchOptions | undefined | null): Promise> - degreeAsync(nodeId: number, options?: JsDegreeOptions | undefined | null): Promise - sumEdgeWeightsAsync(nodeId: number, options?: JsSumEdgeWeightsOptions | undefined | null): Promise - avgEdgeWeightAsync(nodeId: number, options?: JsAvgEdgeWeightOptions | undefined | null): Promise - degreesAsync(nodeIds: Array, options?: JsDegreesOptions | undefined | null): Promise> - shortestPathAsync(from: number, to: number, options?: JsShortestPathOptions | undefined | null): Promise - isConnectedAsync(from: number, to: number, options?: JsIsConnectedOptions | undefined | null): Promise - allShortestPathsAsync(from: number, to: number, options?: JsAllShortestPathsOptions | undefined | null): Promise> - nodesByTypePagedAsync(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): Promise - edgesByTypePagedAsync(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): Promise - getNodesByTypePagedAsync(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): Promise - getEdgesByTypePagedAsync(typeId: number, limit?: number | undefined | null, after?: number | undefined | null): Promise - findNodesPagedAsync(typeId: number, propKey: string, propValue: any, options?: JsFindNodesPagedOptions | undefined | null): Promise - findNodesByTimeRangeAsync(typeId: number, fromMs: number, toMs: number): Promise - findNodesByTimeRangePagedAsync(typeId: number, fromMs: number, toMs: number, options?: JsFindNodesByTimeRangePagedOptions | undefined | null): Promise - personalizedPagerankAsync(seedNodeIds: Array, options?: JsPersonalizedPagerankOptions | undefined | null): Promise - exportAdjacencyAsync(options?: JsExportOptions | undefined | null): Promise - neighborsPagedAsync(nodeId: number, options?: JsNeighborsPagedOptions | undefined | null): Promise - connectedComponentsAsync(options?: JsConnectedComponentsOptions | undefined | null): Promise> - componentOfAsync(nodeId: number, options?: JsComponentOfOptions | undefined | null): Promise - vectorSearchAsync(mode: string, options: JsVectorSearchOptions): Promise> + ensureNodePropertyIndexAsync(label: string, propKey: string, kind: SecondaryIndexKind): Promise + dropNodePropertyIndexAsync(label: string, propKey: string, kind: SecondaryIndexKind): Promise + listNodePropertyIndexesAsync(): Promise> + ensureEdgePropertyIndexAsync(label: string, propKey: string, kind: SecondaryIndexKind): Promise + dropEdgePropertyIndexAsync(label: string, propKey: string, kind: SecondaryIndexKind): Promise + listEdgePropertyIndexesAsync(): Promise> + findNodesRangeAsync(label: string, propKey: string, lower?: PropertyRangeBound | undefined | null, upper?: PropertyRangeBound | undefined | null): Promise + findNodesRangePagedAsync(label: string, propKey: string, lower?: PropertyRangeBound | undefined | null, upper?: PropertyRangeBound | undefined | null, options?: FindNodesRangePagedOptions | undefined | null): Promise + getNodesByLabelsAsync(labels: string | string[]): Promise> + getEdgesByLabelAsync(label: string): Promise> + countNodesByLabelsAsync(labels: string | string[]): Promise + countEdgesByLabelAsync(label: string): Promise + nodesByLabelsAsync(labels: string | string[]): Promise + edgesByLabelAsync(label: string): Promise + neighborsBatchAsync(nodeIds: Array, options?: NeighborsBatchOptions | undefined | null): Promise> + degreeAsync(nodeId: number, options?: DegreeOptions | undefined | null): Promise + sumEdgeWeightsAsync(nodeId: number, options?: SumEdgeWeightsOptions | undefined | null): Promise + avgEdgeWeightAsync(nodeId: number, options?: AvgEdgeWeightOptions | undefined | null): Promise + degreesAsync(nodeIds: Array, options?: DegreesOptions | undefined | null): Promise> + shortestPathAsync(from: number, to: number, options?: ShortestPathOptions | undefined | null): Promise + isConnectedAsync(from: number, to: number, options?: IsConnectedOptions | undefined | null): Promise + allShortestPathsAsync(from: number, to: number, options?: AllShortestPathsOptions | undefined | null): Promise> + nodesByLabelsPagedAsync(labels: string | string[], limit?: number | null, after?: number | null): Promise + edgesByLabelPagedAsync(label: string, limit?: number | undefined | null, after?: number | undefined | null): Promise + getNodesByLabelsPagedAsync(labels: string | string[], limit?: number | null, after?: number | null): Promise + getEdgesByLabelPagedAsync(label: string, limit?: number | undefined | null, after?: number | undefined | null): Promise + findNodesPagedAsync(label: string, propKey: string, propValue: any, options?: FindNodesPagedOptions | undefined | null): Promise + findNodesByTimeRangeAsync(label: string, fromMs: number, toMs: number): Promise + findNodesByTimeRangePagedAsync(label: string, fromMs: number, toMs: number, options?: FindNodesByTimeRangePagedOptions | undefined | null): Promise + personalizedPagerankAsync(seedNodeIds: Array, options?: PersonalizedPagerankOptions | undefined | null): Promise + exportAdjacencyAsync(options?: ExportOptions | undefined | null): Promise + neighborsPagedAsync(nodeId: number, options?: NeighborsPagedOptions | undefined | null): Promise + connectedComponentsAsync(options?: ConnectedComponentsOptions | undefined | null): Promise> + componentOfAsync(nodeId: number, options?: ComponentOfOptions | undefined | null): Promise + vectorSearchAsync(mode: string, options: VectorSearchOptions): Promise> syncAsync(): Promise flushAsync(): Promise ingestModeAsync(): Promise - endIngestAsync(): Promise - compactAsync(): Promise + endIngestAsync(): Promise + compactAsync(): Promise /** * Async compaction with a fire-and-forget progress callback. * The callback receives progress updates but cannot cancel compaction (unlike the sync version). * Note: the database write lock is held for the entire compaction, so other operations on this * instance will block until compaction completes. The JS event loop remains responsive. */ - compactWithProgressAsync(callback: (progress: JsCompactionProgress) => void): Promise + compactWithProgressAsync(callback: (progress: CompactionProgress) => void): Promise +} + +export declare class SubgraphResult { + get nodes(): Array + get edges(): Array } export declare class WriteTxn { - upsertNode(typeId: number, key: string, options?: JsUpsertNodeOptions | undefined | null): JsTxnNodeRef - upsertNodeAs(alias: string, typeId: number, key: string, options?: JsUpsertNodeOptions | undefined | null): JsTxnNodeRef - upsertEdge(from: JsTxnNodeRef, to: JsTxnNodeRef, typeId: number, options?: JsUpsertEdgeOptions | undefined | null): JsTxnEdgeRef - upsertEdgeAs(alias: string, from: JsTxnNodeRef, to: JsTxnNodeRef, typeId: number, options?: JsUpsertEdgeOptions | undefined | null): JsTxnEdgeRef - deleteNode(target: JsTxnNodeRef): void - deleteEdge(target: JsTxnEdgeRef): void - invalidateEdge(target: JsTxnEdgeRef, validTo: number): void - stage(operations: Array): void - getNode(target: JsTxnNodeRef): JsTxnNodeView | null - getEdge(target: JsTxnEdgeRef): JsTxnEdgeView | null - getNodeByKey(typeId: number, key: string): JsTxnNodeView | null - getEdgeByTriple(from: JsTxnNodeRef, to: JsTxnNodeRef, typeId: number): JsTxnEdgeView | null - commit(): JsTxnCommitResult + upsertNode(labels: string | string[], key: string, options?: UpsertNodeOptions | null): TxnNodeRef + upsertNodeAs(alias: string, labels: string | string[], key: string, options?: UpsertNodeOptions | null): TxnNodeRef + addNodeLabel(target: TxnNodeRef, label: string): boolean + removeNodeLabel(target: TxnNodeRef, label: string): boolean + upsertEdge(from: TxnNodeRef, to: TxnNodeRef, label: string, options?: UpsertEdgeOptions | undefined | null): TxnEdgeRef + upsertEdgeAs(alias: string, from: TxnNodeRef, to: TxnNodeRef, label: string, options?: UpsertEdgeOptions | undefined | null): TxnEdgeRef + deleteNode(target: TxnNodeRef): void + deleteEdge(target: TxnEdgeRef): void + invalidateEdge(target: TxnEdgeRef, validTo: number): void + stage(operations: Array<{ op: 'upsertNode'; alias?: string; labels: string | string[]; key: string; props?: Record; weight?: number; denseVector?: Array; sparseVector?: Array } | { op: 'upsertEdge'; alias?: string; from: TxnNodeRef; to: TxnNodeRef; label: string; props?: Record; weight?: number; validFrom?: number; validTo?: number } | { op: 'deleteNode'; target: TxnEdgeOrNodeRef } | { op: 'deleteEdge'; target: TxnEdgeOrNodeRef } | { op: 'invalidateEdge'; target: TxnEdgeOrNodeRef; validTo: number }>): void + getNode(target: TxnNodeRef): TxnNodeView | null + getEdge(target: TxnEdgeRef): TxnEdgeView | null + getNodeByKey(label: string, key: string): TxnNodeView | null + getEdgeByTriple(from: TxnNodeRef, to: TxnNodeRef, label: string): TxnEdgeView | null + commit(): TxnCommitResult rollback(): void - upsertNodeAsync(typeId: number, key: string, options?: JsUpsertNodeOptions | undefined | null): Promise - upsertNodeAsAsync(alias: string, typeId: number, key: string, options?: JsUpsertNodeOptions | undefined | null): Promise - upsertEdgeAsync(from: JsTxnNodeRef, to: JsTxnNodeRef, typeId: number, options?: JsUpsertEdgeOptions | undefined | null): Promise - upsertEdgeAsAsync(alias: string, from: JsTxnNodeRef, to: JsTxnNodeRef, typeId: number, options?: JsUpsertEdgeOptions | undefined | null): Promise - deleteNodeAsync(target: JsTxnNodeRef): Promise - deleteEdgeAsync(target: JsTxnEdgeRef): Promise - invalidateEdgeAsync(target: JsTxnEdgeRef, validTo: number): Promise - stageAsync(operations: Array): Promise - getNodeAsync(target: JsTxnNodeRef): Promise - getEdgeAsync(target: JsTxnEdgeRef): Promise - getNodeByKeyAsync(typeId: number, key: string): Promise - getEdgeByTripleAsync(from: JsTxnNodeRef, to: JsTxnNodeRef, typeId: number): Promise - commitAsync(): Promise + upsertNodeAsync(labels: string | string[], key: string, options?: UpsertNodeOptions | null): Promise + upsertNodeAsAsync(alias: string, labels: string | string[], key: string, options?: UpsertNodeOptions | null): Promise + addNodeLabelAsync(target: TxnNodeRef, label: string): Promise + removeNodeLabelAsync(target: TxnNodeRef, label: string): Promise + upsertEdgeAsync(from: TxnNodeRef, to: TxnNodeRef, label: string, options?: UpsertEdgeOptions | undefined | null): Promise + upsertEdgeAsAsync(alias: string, from: TxnNodeRef, to: TxnNodeRef, label: string, options?: UpsertEdgeOptions | undefined | null): Promise + deleteNodeAsync(target: TxnNodeRef): Promise + deleteEdgeAsync(target: TxnEdgeRef): Promise + invalidateEdgeAsync(target: TxnEdgeRef, validTo: number): Promise + stageAsync(operations: Array<{ op: 'upsertNode'; alias?: string; labels: string | string[]; key: string; props?: Record; weight?: number; denseVector?: Array; sparseVector?: Array } | { op: 'upsertEdge'; alias?: string; from: TxnNodeRef; to: TxnNodeRef; label: string; props?: Record; weight?: number; validFrom?: number; validTo?: number } | { op: 'deleteNode'; target: TxnEdgeOrNodeRef } | { op: 'deleteEdge'; target: TxnEdgeOrNodeRef } | { op: 'invalidateEdge'; target: TxnEdgeOrNodeRef; validTo: number }>): Promise + getNodeAsync(target: TxnNodeRef): Promise + getEdgeAsync(target: TxnEdgeRef): Promise + getNodeByKeyAsync(label: string, key: string): Promise + getEdgeByTripleAsync(from: TxnNodeRef, to: TxnNodeRef, label: string): Promise + commitAsync(): Promise rollbackAsync(): Promise } -export type JsWriteTxn = WriteTxn -export interface JsAdjacencyExport { +export interface AdjacencyExport { nodeIds: Float64Array + edgeLabels: Array edgeFrom: Float64Array edgeTo: Float64Array - edgeTypeIds: Uint32Array + edgeLabelIndexes: Uint32Array edgeWeights?: Float64Array } -export interface JsAllShortestPathsOptions { +export interface AllShortestPathsOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array weightField?: string atEpoch?: number maxDepth?: number @@ -287,18 +330,18 @@ export interface JsAllShortestPathsOptions { maxPaths?: number } -export interface JsAvgEdgeWeightOptions { +export interface AvgEdgeWeightOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array atEpoch?: number } -export interface JsCloseOptions { +export interface CloseOptions { /** If true, cancel any in-progress background compaction instead of waiting. */ force?: boolean } -export interface JsCompactionProgress { +export interface CompactionProgress { phase: string segmentsProcessed: number totalSegments: number @@ -306,7 +349,7 @@ export interface JsCompactionProgress { totalRecords: number } -export interface JsCompactionStats { +export interface CompactionStats { segmentsMerged: number nodesKept: number nodesRemoved: number @@ -320,33 +363,39 @@ export interface JsCompactionStats { edgesAutoPruned: number } -export interface JsComponentEntry { +export interface ComponentEntry { nodeId: number componentId: number } -export interface JsComponentOfOptions { - edgeTypeFilter?: Array - nodeTypeFilter?: Array +export interface ComponentOfOptions { + edgeLabelFilter?: Array + nodeLabelFilter?: NodeLabelFilter atEpoch?: number } -export interface JsConnectedComponentsOptions { - edgeTypeFilter?: Array - nodeTypeFilter?: Array +export interface ComponentScrubFinding { + componentKind: string + findingType: string + detail: string +} + +export interface ConnectedComponentsOptions { + edgeLabelFilter?: Array + nodeLabelFilter?: NodeLabelFilter atEpoch?: number } -export interface JsDbOptions { +export interface DbOptions { createIfMissing?: boolean edgeUniqueness?: boolean memtableFlushThreshold?: number - /** Trigger compaction automatically after this many flushes. Default 5, 0 = disabled. */ + /** Trigger compaction automatically after this many flushes. Default 4, 0 = disabled. */ compactAfterNFlushes?: number - denseVector?: JsDenseVectorConfig + denseVector?: DenseVectorConfig /** WAL sync mode: 'immediate' or 'group-commit' (default). */ walSyncMode?: string - /** Group commit sync interval in milliseconds. Default: 10. */ + /** Group commit sync interval in milliseconds. Default: 50. */ groupCommitIntervalMs?: number /** Hard cap on memtable size in bytes. Writes trigger a flush when exceeded. 0 = disabled. */ memtableHardCapBytes?: number @@ -357,7 +406,7 @@ export interface JsDbOptions { maxImmutableMemtables?: number } -export interface JsDbStats { +export interface DbStats { /** Bytes buffered in WAL but not yet fsynced. Always 0 in immediate mode. */ pendingWalBytes: number /** Number of on-disk segments. */ @@ -384,151 +433,177 @@ export interface JsDbStats { oldestRetainedWalGenerationId: number } -export interface JsDegreeBatchEntry { +export interface DegreeBatchEntry { nodeId: number degree: number } -export interface JsDegreeOptions { +export interface DegreeOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array atEpoch?: number } -export interface JsDegreesOptions { +export interface DegreesOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array atEpoch?: number } -export interface JsDenseVectorConfig { +export interface DenseVectorConfig { dimension: number metric?: string } -export interface JsEdgeInput { +export interface EdgeInput { from: number to: number - typeId: number + label: string props?: Record weight?: number validFrom?: number validTo?: number } -export interface JsEdgeInvalidation { +export interface EdgeInvalidation { edgeId: number validTo: number } -export interface JsExportOptions { - nodeTypeFilter?: Array - edgeTypeFilter?: Array +export interface EdgeLabelInfo { + label: string + labelId: number +} + +export interface EdgePropertyIndexInfo { + indexId: number + label: string + propKey: string + kind: string + domain?: string + state: string + lastError?: string +} + +export interface ExportOptions { + nodeLabelFilter?: NodeLabelFilter + edgeLabelFilter?: Array includeWeights?: boolean } -export interface JsExtractSubgraphOptions { +export interface ExtractSubgraphOptions { direction?: string - edgeTypeFilter?: Array + edgeLabelFilter?: Array + nodeLabelFilter?: NodeLabelFilter atEpoch?: number } -export interface JsFindNodesByTimeRangePagedOptions { +export interface FindNodesByTimeRangePagedOptions { limit?: number after?: number } -export interface JsFindNodesPagedOptions { +export interface FindNodesPagedOptions { limit?: number after?: number } -export interface JsFindNodesRangePagedOptions { +export interface FindNodesRangePagedOptions { limit?: number - after?: JsPropertyRangeCursor + after?: PropertyRangeCursor } -export interface JsGraphPatch { - upsertNodes?: Array - upsertEdges?: Array - invalidateEdges?: Array +export interface GraphPatch { + upsertNodes?: Array + upsertEdges?: Array + invalidateEdges?: Array deleteNodeIds?: Array deleteEdgeIds?: Array } -export interface JsIdPageResult { +export interface IdPageResult { items: Float64Array nextCursor?: number } -export interface JsIsConnectedOptions { +export interface IsConnectedOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array atEpoch?: number maxDepth?: number } -export interface JsKeyQuery { - typeId: number +export interface KeyQuery { + label: string key: string } -export interface JsNamedPrunePolicy { +export interface NamedPrunePolicy { name: string - policy: JsPrunePolicy + policy: PrunePolicy } -export interface JsNeighborBatchEntry { +export interface NeighborBatchEntry { queryNodeId: number - neighbors: Array + neighbors: Array } /** A single neighbor entry as a plain JS object. */ -export interface JsNeighborEntry { +export interface NeighborEntry { nodeId: number edgeId: number - edgeTypeId: number + label: string weight: number validFrom: number validTo: number } -export interface JsNeighborsBatchOptions { +export interface NeighborsBatchOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array atEpoch?: number decayLambda?: number } -export interface JsNeighborsOptions { +export interface NeighborsOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array limit?: number atEpoch?: number decayLambda?: number } -export interface JsNeighborsPagedOptions { +export interface NeighborsPagedOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array limit?: number after?: number atEpoch?: number decayLambda?: number } -export interface JsNodeInput { - typeId: number +export interface NodeInput { + labels: string | string[] key: string props?: Record weight?: number denseVector?: Array - sparseVector?: Array + sparseVector?: Array } -export interface JsNodePropertyIndexInfo { +export interface NodeLabelFilter { + labels: Array + mode: 'any' | 'all' +} + +export interface NodeLabelInfo { + label: string + labelId: number +} + +export interface NodePropertyIndexInfo { indexId: number - typeId: number + label: string propKey: string kind: string domain?: string @@ -536,166 +611,183 @@ export interface JsNodePropertyIndexInfo { lastError?: string } -export interface JsPatchResult { +export interface PatchResult { nodeIds: Float64Array edgeIds: Float64Array } -export interface JsPersonalizedPagerankOptions { +export interface PersonalizedPagerankOptions { algorithm?: string dampingFactor?: number maxIterations?: number epsilon?: number approxResidualTolerance?: number - edgeTypeFilter?: Array + edgeLabelFilter?: Array maxResults?: number } -export interface JsPprApproxMeta { +export interface PprApproxMeta { residualTolerance: number pushes: number maxRemainingResidual: number } -export interface JsPprResult { +export interface PprResult { nodeIds: Float64Array scores: Float64Array iterations: number converged: boolean algorithm: string - approx?: JsPprApproxMeta + approx?: PprApproxMeta } -export interface JsPropertyRangeBound { +export interface PropertyRangeBound { value: number inclusive?: boolean domain: string } -export interface JsPropertyRangeCursor { +export interface PropertyRangeCursor { value: number nodeId: number domain: string } -export interface JsPropertyRangePageResult { +export interface PropertyRangePageResult { items: Float64Array - nextCursor?: JsPropertyRangeCursor + nextCursor?: PropertyRangeCursor } -export interface JsPrunePolicy { +export interface PrunePolicy { /** Prune nodes older than this many milliseconds. Optional. */ maxAgeMs?: number /** Prune nodes with weight <= this threshold. Optional. */ maxWeight?: number - /** Scope to a single node type. Optional. */ - typeId?: number + /** Scope to a single node label. Optional. */ + label?: string } -export interface JsPruneResult { +export interface PruneResult { /** Number of nodes pruned. */ nodesPruned: number /** Number of edges cascade-deleted. */ edgesPruned: number } -export interface JsSecondaryIndexKind { +export interface ScrubReport { + segments: Array + totalComponentsChecked: number + totalComponentsOk: number + totalComponentsFailed: number + totalBytesDigested: number + durationMs: number +} + +export interface SecondaryIndexKind { kind: string domain?: string } -export interface JsShortestPath { +export interface SegmentScrubResult { + segmentId: number + findings: Array + componentsOk: number + bytesDigested: number +} + +export interface ShortestPath { nodes: Array edges: Array totalCost: number } -export interface JsShortestPathOptions { +export interface ShortestPathOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array weightField?: string atEpoch?: number maxDepth?: number maxCost?: number } -export interface JsSparseEntry { +export interface SparseEntry { dimension: number value: number } -export interface JsSumEdgeWeightsOptions { +export interface SumEdgeWeightsOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array atEpoch?: number } -export interface JsTopKNeighborsOptions { +export interface TopKNeighborsOptions { direction?: string - typeFilter?: Array + edgeLabelFilter?: Array scoring?: string decayLambda?: number atEpoch?: number } -export interface JsTraversalCursor { +export interface TraversalCursor { depth: number lastNodeId: number } -export interface JsTraversalHit { +export interface TraversalHit { nodeId: number depth: number viaEdgeId?: number score?: number } -export interface JsTraversalPageResult { - items: Array - nextCursor?: JsTraversalCursor +export interface TraversalPageResult { + items: Array + nextCursor?: TraversalCursor } -export interface JsTraverseOptions { +export interface TraverseOptions { minDepth?: number direction?: string - edgeTypeFilter?: Array - nodeTypeFilter?: Array + edgeLabelFilter?: Array + emitNodeLabelFilter?: NodeLabelFilter atEpoch?: number decayLambda?: number limit?: number - cursor?: JsTraversalCursor + cursor?: TraversalCursor } -export interface JsTxnCommitResult { +export interface TxnCommitResult { nodeIds: Float64Array edgeIds: Float64Array nodeAliases: Record edgeAliases: Record } -export interface JsTxnEdgeOrNodeRef { +export interface TxnEdgeOrNodeRef { id?: number - typeId?: number + labels?: string | string[] + label?: string key?: string local?: string - from?: JsTxnNodeRef - to?: JsTxnNodeRef + from?: TxnNodeRef + to?: TxnNodeRef } -export interface JsTxnEdgeRef { +export interface TxnEdgeRef { id?: number - from?: JsTxnNodeRef - to?: JsTxnNodeRef - typeId?: number + from?: TxnNodeRef + to?: TxnNodeRef + label?: string local?: string } -export interface JsTxnEdgeView { +export interface TxnEdgeView { id?: number local?: string - from: JsTxnNodeRef - to: JsTxnNodeRef - typeId: number + from: TxnNodeRef + to: TxnNodeRef + label: string props: Record createdAt?: number updatedAt?: number @@ -704,77 +796,61 @@ export interface JsTxnEdgeView { validTo?: number } -export interface JsTxnNodeRef { +export interface TxnNodeRef { id?: number - typeId?: number + labels?: string | string[] key?: string local?: string } -export interface JsTxnNodeView { +export interface TxnNodeView { id?: number local?: string - typeId: number + labels: Array key: string props: Record createdAt?: number updatedAt?: number weight: number denseVector?: Array - sparseVector?: Array -} - -export interface JsTxnOperation { - op: string - alias?: string - typeId?: number - key?: string - props?: Record - weight?: number - denseVector?: Array - sparseVector?: Array - from?: JsTxnNodeRef - to?: JsTxnNodeRef - target?: JsTxnEdgeOrNodeRef - validFrom?: number - validTo?: number + sparseVector?: Array } -export interface JsUpsertEdgeOptions { +export interface UpsertEdgeOptions { props?: Record weight?: number validFrom?: number validTo?: number } -export interface JsUpsertNodeOptions { +export interface UpsertNodeOptions { props?: Record weight?: number denseVector?: Array - sparseVector?: Array + sparseVector?: Array } -export interface JsVectorHit { +export interface VectorHit { nodeId: number score: number } -export interface JsVectorSearchOptions { +export interface VectorSearchOptions { k: number denseQuery?: Array - sparseQuery?: Array - typeFilter?: Array + sparseQuery?: Array + labelFilter?: NodeLabelFilter efSearch?: number - scope?: JsVectorSearchScope + scope?: VectorSearchScope denseWeight?: number sparseWeight?: number fusionMode?: string } -export interface JsVectorSearchScope { +export interface VectorSearchScope { startNodeId: number maxDepth: number direction?: string - edgeTypeFilter?: Array + edgeLabelFilter?: Array atEpoch?: number } diff --git a/overgraph-node/index.js b/overgraph-node/index.js index 1b31fba..a1914ee 100644 --- a/overgraph-node/index.js +++ b/overgraph-node/index.js @@ -77,8 +77,8 @@ function requireNative() { try { const binding = require('overgraph-android-arm64') const bindingPackageVersion = require('overgraph-android-arm64/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -93,8 +93,8 @@ function requireNative() { try { const binding = require('overgraph-android-arm-eabi') const bindingPackageVersion = require('overgraph-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -114,8 +114,8 @@ function requireNative() { try { const binding = require('overgraph-win32-x64-gnu') const bindingPackageVersion = require('overgraph-win32-x64-gnu/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -130,8 +130,8 @@ function requireNative() { try { const binding = require('overgraph-win32-x64-msvc') const bindingPackageVersion = require('overgraph-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -147,8 +147,8 @@ function requireNative() { try { const binding = require('overgraph-win32-ia32-msvc') const bindingPackageVersion = require('overgraph-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -163,8 +163,8 @@ function requireNative() { try { const binding = require('overgraph-win32-arm64-msvc') const bindingPackageVersion = require('overgraph-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -182,8 +182,8 @@ function requireNative() { try { const binding = require('overgraph-darwin-universal') const bindingPackageVersion = require('overgraph-darwin-universal/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -198,8 +198,8 @@ function requireNative() { try { const binding = require('overgraph-darwin-x64') const bindingPackageVersion = require('overgraph-darwin-x64/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -214,8 +214,8 @@ function requireNative() { try { const binding = require('overgraph-darwin-arm64') const bindingPackageVersion = require('overgraph-darwin-arm64/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -234,8 +234,8 @@ function requireNative() { try { const binding = require('overgraph-freebsd-x64') const bindingPackageVersion = require('overgraph-freebsd-x64/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -250,8 +250,8 @@ function requireNative() { try { const binding = require('overgraph-freebsd-arm64') const bindingPackageVersion = require('overgraph-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -271,8 +271,8 @@ function requireNative() { try { const binding = require('overgraph-linux-x64-musl') const bindingPackageVersion = require('overgraph-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -287,8 +287,8 @@ function requireNative() { try { const binding = require('overgraph-linux-x64-gnu') const bindingPackageVersion = require('overgraph-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -305,8 +305,8 @@ function requireNative() { try { const binding = require('overgraph-linux-arm64-musl') const bindingPackageVersion = require('overgraph-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -321,8 +321,8 @@ function requireNative() { try { const binding = require('overgraph-linux-arm64-gnu') const bindingPackageVersion = require('overgraph-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -339,8 +339,8 @@ function requireNative() { try { const binding = require('overgraph-linux-arm-musleabihf') const bindingPackageVersion = require('overgraph-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -355,8 +355,8 @@ function requireNative() { try { const binding = require('overgraph-linux-arm-gnueabihf') const bindingPackageVersion = require('overgraph-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -373,8 +373,8 @@ function requireNative() { try { const binding = require('overgraph-linux-loong64-musl') const bindingPackageVersion = require('overgraph-linux-loong64-musl/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -389,8 +389,8 @@ function requireNative() { try { const binding = require('overgraph-linux-loong64-gnu') const bindingPackageVersion = require('overgraph-linux-loong64-gnu/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -407,8 +407,8 @@ function requireNative() { try { const binding = require('overgraph-linux-riscv64-musl') const bindingPackageVersion = require('overgraph-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -423,8 +423,8 @@ function requireNative() { try { const binding = require('overgraph-linux-riscv64-gnu') const bindingPackageVersion = require('overgraph-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -440,8 +440,8 @@ function requireNative() { try { const binding = require('overgraph-linux-ppc64-gnu') const bindingPackageVersion = require('overgraph-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -456,8 +456,8 @@ function requireNative() { try { const binding = require('overgraph-linux-s390x-gnu') const bindingPackageVersion = require('overgraph-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -476,8 +476,8 @@ function requireNative() { try { const binding = require('overgraph-openharmony-arm64') const bindingPackageVersion = require('overgraph-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -492,8 +492,8 @@ function requireNative() { try { const binding = require('overgraph-openharmony-x64') const bindingPackageVersion = require('overgraph-openharmony-x64/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -508,8 +508,8 @@ function requireNative() { try { const binding = require('overgraph-openharmony-arm') const bindingPackageVersion = require('overgraph-openharmony-arm/package.json').version - if (bindingPackageVersion !== '0.6.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.6.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.8.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.8.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -576,12 +576,11 @@ if (!nativeBinding) { } module.exports = nativeBinding -module.exports.JsEdgePageResult = nativeBinding.JsEdgePageResult -module.exports.JsEdgeRecord = nativeBinding.JsEdgeRecord -module.exports.JsNeighborPageResult = nativeBinding.JsNeighborPageResult -module.exports.JsNodePageResult = nativeBinding.JsNodePageResult -module.exports.JsNodeRecord = nativeBinding.JsNodeRecord -module.exports.JsSubgraphResult = nativeBinding.JsSubgraphResult +module.exports.EdgePageResult = nativeBinding.EdgePageResult +module.exports.EdgeView = nativeBinding.EdgeView +module.exports.NeighborPageResult = nativeBinding.NeighborPageResult +module.exports.NodePageResult = nativeBinding.NodePageResult +module.exports.NodeView = nativeBinding.NodeView module.exports.OverGraph = nativeBinding.OverGraph +module.exports.SubgraphResult = nativeBinding.SubgraphResult module.exports.WriteTxn = nativeBinding.WriteTxn -module.exports.JsWriteTxn = nativeBinding.JsWriteTxn diff --git a/overgraph-node/package-lock.json b/overgraph-node/package-lock.json index a8e1d3d..99a72fe 100644 --- a/overgraph-node/package-lock.json +++ b/overgraph-node/package-lock.json @@ -1,15 +1,17 @@ { "name": "overgraph", - "version": "0.7.0", + "version": "0.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "overgraph", - "version": "0.7.0", + "version": "0.8.0", "license": "MIT OR Apache-2.0", "devDependencies": { - "@napi-rs/cli": "^3.0.0" + "@napi-rs/cli": "^3.0.0", + "@types/node": "^25.8.0", + "typescript": "^6.0.3" } }, "node_modules/@emnapi/core": { @@ -1554,6 +1556,16 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/node": { + "version": "25.8.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.8.0.tgz", + "integrity": "sha512-TCFSk8IZh+iLX1xtksoBVtdmgL+1IX0fC9BeU4QqFSuNdN/K+HUlhqOzEmSYYpZUVsLYcPqc9KX+60iDuninSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1812,6 +1824,27 @@ "website" ] }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "node_modules/universal-user-agent": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.3.tgz", diff --git a/overgraph-node/package.json b/overgraph-node/package.json index 77c0af7..2eb79c5 100644 --- a/overgraph-node/package.json +++ b/overgraph-node/package.json @@ -1,6 +1,6 @@ { "name": "overgraph", - "version": "0.7.0", + "version": "0.8.0", "description": "An absurdly fast embedded graph database for Node.js. Sub-microsecond reads, pure Rust core.", "main": "index.js", "types": "index.d.ts", @@ -16,11 +16,14 @@ "scripts": { "build": "napi build --platform --release", "build:debug": "napi build --platform", + "typecheck": "tsc --noEmit -p tsconfig.types.json", "test": "node --test __test__/*.mjs", "bench": "node __test__/benchmark-v2.mjs" }, "devDependencies": { - "@napi-rs/cli": "^3.0.0" + "@napi-rs/cli": "^3.0.0", + "@types/node": "^25.8.0", + "typescript": "^6.0.3" }, "license": "MIT OR Apache-2.0", "repository": { @@ -30,12 +33,20 @@ }, "homepage": "https://overgraph.io", "docs": "https://overgraph.io/docs/api-reference", - "keywords": ["graph-database", "embedded", "performance", "typescript", "napi-rs", "rust"], + "keywords": [ + "graph-database", + "embedded", + "performance", + "typescript", + "napi-rs", + "rust" + ], "author": "Brandon Hensley", "files": [ "index.js", "index.d.ts", "query-types.d.ts", + "helpers/pack-binary.mjs", "*.node", "README.md" ] diff --git a/overgraph-node/query-types.d.ts b/overgraph-node/query-types.d.ts index f0a7bb7..ba22bc5 100644 --- a/overgraph-node/query-types.d.ts +++ b/overgraph-node/query-types.d.ts @@ -9,6 +9,47 @@ export type QueryNodeFilter = | QueryNodePropertyFilter | QueryNodeUpdatedAtFilter +export type QueryEdgeFilter = + | { + and: NonEmptyArray + or?: never + not?: never + property?: never + weight?: never + updatedAt?: never + validAt?: never + validFrom?: never + validTo?: never + } + | { + or: NonEmptyArray + and?: never + not?: never + property?: never + weight?: never + updatedAt?: never + validAt?: never + validFrom?: never + validTo?: never + } + | { + not: QueryEdgeFilter + and?: never + or?: never + property?: never + weight?: never + updatedAt?: never + validAt?: never + validFrom?: never + validTo?: never + } + | QueryEdgePropertyFilter + | QueryEdgeWeightFilter + | QueryEdgeUpdatedAtFilter + | QueryEdgeValidAtFilter + | QueryEdgeValidFromFilter + | QueryEdgeValidToFilter + export type QueryNodePropertyFilter = | { property: string @@ -64,6 +105,93 @@ export interface QueryNodeUpdatedAtFilter { property?: never } +export type QueryEdgePropertyFilter = + | { + property: string + eq: any + in?: never + exists?: never + missing?: never + gt?: never + gte?: never + lt?: never + lte?: never + } + | { + property: string + in: NonEmptyArray + eq?: never + exists?: never + missing?: never + gt?: never + gte?: never + lt?: never + lte?: never + } + | ({ property: string; eq?: never; in?: never; exists?: never; missing?: never } & QueryNodeRangePredicate) + | { + property: string + exists: true + eq?: never + in?: never + missing?: never + gt?: never + gte?: never + lt?: never + lte?: never + } + | { + property: string + missing: true + eq?: never + in?: never + exists?: never + gt?: never + gte?: never + lt?: never + lte?: never + } + +export interface QueryEdgeWeightFilter { + weight: QueryNodeRangePredicate + and?: never + or?: never + not?: never + property?: never +} + +export interface QueryEdgeUpdatedAtFilter { + updatedAt: QueryNodeRangePredicate + and?: never + or?: never + not?: never + property?: never +} + +export interface QueryEdgeValidAtFilter { + validAt: number + and?: never + or?: never + not?: never + property?: never +} + +export interface QueryEdgeValidFromFilter { + validFrom: QueryNodeRangePredicate + and?: never + or?: never + not?: never + property?: never +} + +export interface QueryEdgeValidToFilter { + validTo: QueryNodeRangePredicate + and?: never + or?: never + not?: never + property?: never +} + export type QueryLowerBound = | { gt: any; gte?: never } | { gte: any; gt?: never } @@ -110,8 +238,15 @@ export interface QueryUpdatedAtPredicate { export type QueryPredicate = QueryPropertyPredicate | QueryUpdatedAtPredicate +export type LabelMatchMode = 'any' | 'all' + +export interface NodeLabelFilter { + labels: Array + mode: LabelMatchMode +} + export interface QueryNodeRequest { - typeId?: number + labelFilter?: NodeLabelFilter ids?: Array keys?: Array filter?: QueryNodeFilter | null @@ -121,26 +256,33 @@ export interface QueryNodeRequest { allowFullScan?: boolean } +export interface QueryEdgeRequest { + label?: string + ids?: Array + fromIds?: Array + toIds?: Array + endpointIds?: Array + filter?: QueryEdgeFilter | null + limit?: number | null + after?: number + allowFullScan?: boolean +} + export interface GraphNodePattern { alias: string - typeId?: number + labelFilter?: NodeLabelFilter ids?: Array keys?: Array filter?: QueryNodeFilter | null } -export interface EdgePropertyPredicate { - property: QueryPropertyPredicatePayload -} - export interface GraphEdgePattern { alias?: string fromAlias: string toAlias: string direction?: 'outgoing' | 'incoming' | 'both' - typeFilter?: Array - where?: Record - predicates?: Array + labelFilter?: Array + filter?: QueryEdgeFilter | null } export interface GraphPatternRequest { @@ -160,7 +302,7 @@ export interface QueryPatternResult { truncated: boolean } -export type QueryPlanKind = 'node_query' | 'pattern_query' +export type QueryPlanKind = 'node_query' | 'edge_query' | 'pattern_query' export type QueryPlanWarning = | 'missing_ready_index' @@ -176,22 +318,57 @@ export type QueryPlanWarning = | 'verify_only_filter' | 'boolean_branch_fallback' | 'planning_probe_budget_exceeded' + | 'unknown_node_label' + | 'unknown_edge_label' + +export type QueryPlanNote = + | 'node_label_any_dedupe_before_pagination' + | 'node_label_any_final_verification' + | 'node_label_all_superset_verification' + | 'stale_node_label_membership_verification' + +export interface QueryPlanPublicName { + alias?: string | null + name: string + known: boolean + mode?: LabelMatchMode | null +} + +export interface QueryPlanPublicInputs { + nodeLabels: Array + edgeLabels: Array +} export type QueryPlanNode = | { kind: 'explicit_ids' } | { kind: 'key_lookup' } - | { kind: 'node_type_index' } + | { kind: 'node_label_index' } + | { kind: 'node_label_any_index' } | { kind: 'property_equality_index' } | { kind: 'property_range_index' } | { kind: 'timestamp_index' } | { kind: 'adjacency_expansion' } + | { kind: 'explicit_edge_ids' } + | { kind: 'edge_label_index' } + | { kind: 'edge_triple_index' } + | { kind: 'edge_endpoint_adjacency' } + | { kind: 'edge_weight_index' } + | { kind: 'edge_updated_at_index' } + | { kind: 'edge_validity_index' } + | { kind: 'edge_metadata_scan' } + | { kind: 'edge_property_equality_index' } + | { kind: 'edge_property_range_index' } | { kind: 'intersect'; inputs: Array } | { kind: 'union'; inputs: Array } | { kind: 'verify_node_filter'; input: QueryPlanNode } + | { kind: 'verify_edge_filter'; input: QueryPlanNode } | { kind: 'verify_edge_predicates'; input: QueryPlanNode } | { kind: 'pattern_expand'; anchorAlias: string; input: QueryPlanNode } - | { kind: 'fallback_type_scan' } + | { kind: 'pattern_edge_anchor'; edgeAlias?: string | null; input: QueryPlanNode } + | { kind: 'fallback_node_label_scan' } | { kind: 'fallback_full_node_scan' } + | { kind: 'fallback_edge_label_scan' } + | { kind: 'fallback_full_edge_scan' } | { kind: 'empty_result' } export interface QueryPlan { @@ -199,4 +376,6 @@ export interface QueryPlan { root: QueryPlanNode estimatedCandidates: number | null warnings: Array + notes: Array + publicInputs: QueryPlanPublicInputs } diff --git a/overgraph-node/src/lib.rs b/overgraph-node/src/lib.rs index daaf697..c2a4ff3 100644 --- a/overgraph-node/src/lib.rs +++ b/overgraph-node/src/lib.rs @@ -4,36 +4,51 @@ use napi::bindgen_prelude::*; use napi::threadsafe_function::ThreadsafeFunctionCallMode; use napi_derive::napi; use overgraph::{ - AdjacencyExport, AllShortestPathsOptions, CompactionPhase, CompactionStats, ComponentOptions, - DatabaseEngine, DbOptions, DbStats, DegreeOptions, DenseMetric, DenseVectorConfig, Direction, - EdgeInput, EdgePattern, EdgePostFilterPredicate, EdgeRecord, EngineError, ExportOptions, - FusionMode, GraphPatch, GraphPatternQuery, HnswConfig, IsConnectedOptions, NeighborEntry, - NeighborOptions, NodeFilterExpr, NodeIdMap, NodeInput, NodePattern, NodePropertyIndexInfo, - NodeQuery, NodeQueryOrder, NodeRecord, PageRequest, PageResult, PatternOrder, PprAlgorithm, - PprOptions, PprResult, PropValue, PropertyRangeBound, PropertyRangeCursor, - PropertyRangePageRequest, PropertyRangePageResult, PrunePolicy, PruneResult, QueryMatch, - QueryNodeIdsResult, QueryNodesResult, QueryPatternResult, QueryPlan, QueryPlanKind, - QueryPlanNode, QueryPlanWarning, ScoringMode, SecondaryIndexKind, SecondaryIndexRangeDomain, - SecondaryIndexState, ShortestPath, ShortestPathOptions, Subgraph, SubgraphOptions, TopKOptions, - TraversalCursor, TraversalHit, TraversalPageResult, TraverseOptions, TxnCommitResult, - TxnEdgeRef, TxnEdgeView, TxnIntent, TxnLocalRef, TxnNodeRef, TxnNodeView, UpsertEdgeOptions, - UpsertNodeOptions, VectorHit, VectorSearchMode, VectorSearchRequest, VectorSearchScope, - WalSyncMode, WriteTxn, + AdjacencyExport as CoreAdjacencyExport, AllShortestPathsOptions as CoreAllShortestPathsOptions, + CompactionPhase, CompactionStats as CoreCompactionStats, ComponentOptions, DatabaseEngine, + DbOptions as CoreDbOptions, DbStats as CoreDbStats, DegreeOptions as CoreDegreeOptions, + DenseMetric, DenseVectorConfig as CoreDenseVectorConfig, Direction, EdgeFilterExpr, + EdgeInput as CoreEdgeInput, EdgeLabelInfo as CoreEdgeLabelInfo, EdgePattern, + EdgePropertyIndexInfo as CoreEdgePropertyIndexInfo, EdgeQuery, EdgeQueryOrder, + EdgeView as CoreEdgeView, EngineError, ExportOptions as CoreExportOptions, FusionMode, + GraphPatch as CoreGraphPatch, GraphPatternQuery, HnswConfig, + IsConnectedOptions as CoreIsConnectedOptions, LabelMatchMode as CoreLabelMatchMode, + NeighborEntry as CoreNeighborEntry, NeighborOptions, NodeFilterExpr, NodeIdMap, + NodeInput as CoreNodeInput, NodeKeyQuery, NodeLabelFilter as CoreNodeLabelFilter, + NodeLabelInfo as CoreNodeLabelInfo, NodePattern, + NodePropertyIndexInfo as CoreNodePropertyIndexInfo, NodeQuery, NodeQueryOrder, + NodeView as CoreNodeView, PageRequest, PageResult, PatternOrder, PprAlgorithm, PprOptions, + PprResult as CorePprResult, PropValue, PropertyRangeBound as CorePropertyRangeBound, + PropertyRangeCursor as CorePropertyRangeCursor, PropertyRangePageRequest, + PropertyRangePageResult as CorePropertyRangePageResult, PrunePolicy as CorePrunePolicy, + PrunePolicyInfo, PruneResult as CorePruneResult, QueryEdgeIdsResult, QueryEdgesResult, + QueryMatch, QueryNodeIdsResult, QueryNodesResult, QueryPatternResult, QueryPlan, QueryPlanKind, + QueryPlanNode, QueryPlanWarning, ScoringMode, ScrubReport as CoreScrubReport, + SecondaryIndexKind as CoreSecondaryIndexKind, SecondaryIndexRangeDomain, SecondaryIndexState, + ShortestPath as CoreShortestPath, ShortestPathOptions as CoreShortestPathOptions, Subgraph, + SubgraphOptions, TopKOptions, TraversalCursor as CoreTraversalCursor, + TraversalHit as CoreTraversalHit, TraversalPageResult as CoreTraversalPageResult, + TraverseOptions as CoreTraverseOptions, TxnCommitResult as CoreTxnCommitResult, + TxnEdgeRef as CoreTxnEdgeRef, TxnEdgeView as CoreTxnEdgeView, TxnIntent, TxnLocalRef, + TxnNodeRef as CoreTxnNodeRef, TxnNodeView as CoreTxnNodeView, + UpsertEdgeOptions as CoreUpsertEdgeOptions, UpsertNodeOptions as CoreUpsertNodeOptions, + VectorHit as CoreVectorHit, VectorSearchMode, VectorSearchRequest, + VectorSearchScope as CoreVectorSearchScope, WalSyncMode, WriteTxn as CoreWriteTxn, }; /// ThreadsafeFunction with `CalleeHandled = false` so the JS callback /// receives `(progress)` directly, not error-first `(null, progress)`. type ProgressTsfn = napi::threadsafe_function::ThreadsafeFunction< - JsCompactionProgress, + CompactionProgress, Unknown<'static>, - JsCompactionProgress, + CompactionProgress, Status, false, >; -pub struct JsJsonValue(serde_json::Value); +pub struct JsonPayload(serde_json::Value); -impl TypeName for JsJsonValue { +impl TypeName for JsonPayload { fn type_name() -> &'static str { "Object" } @@ -43,7 +58,7 @@ impl TypeName for JsJsonValue { } } -impl ToNapiValue for JsJsonValue { +impl ToNapiValue for JsonPayload { unsafe fn to_napi_value(env: napi::sys::napi_env, val: Self) -> Result { unsafe { serde_json::Value::to_napi_value(env, val.0) } } @@ -70,7 +85,7 @@ impl OverGraph { // --- Lifecycle --- #[napi(factory)] - pub fn open(path: String, options: Option) -> Result { + pub fn open(path: String, options: Option) -> Result { let opts = options.map(|o| o.into()).unwrap_or_default(); let engine = DatabaseEngine::open(Path::new(&path), &opts) .map_err(|e| napi::Error::from_reason(e.to_string()))?; @@ -80,7 +95,7 @@ impl OverGraph { } #[napi] - pub fn close(&self, options: Option) -> Result<()> { + pub fn close(&self, options: Option) -> Result<()> { let force = options.as_ref().and_then(|o| o.force).unwrap_or(false); let engine = { let mut guard = self @@ -103,21 +118,68 @@ impl OverGraph { Ok(()) } - // --- Single upserts --- + // --- Catalog diagnostics --- + + #[napi] + pub fn ensure_node_label(&self, label: String) -> Result { + with_engine(self, |eng| eng.ensure_node_label(&label)) + } + + #[napi] + pub fn ensure_edge_label(&self, label: String) -> Result { + with_engine(self, |eng| eng.ensure_edge_label(&label)) + } + + #[napi] + pub fn get_node_label_id(&self, label: String) -> Result> { + with_engine_ref(self, |eng| eng.get_node_label_id(&label)) + } + + #[napi] + pub fn get_edge_label_id(&self, label: String) -> Result> { + with_engine_ref(self, |eng| eng.get_edge_label_id(&label)) + } + + #[napi] + pub fn get_node_label(&self, label_id: u32) -> Result> { + with_engine_ref(self, |eng| eng.get_node_label(label_id)) + } #[napi] + pub fn get_edge_label(&self, label_id: u32) -> Result> { + with_engine_ref(self, |eng| eng.get_edge_label(label_id)) + } + + #[napi] + pub fn list_node_labels(&self) -> Result> { + let infos = with_engine_ref(self, |eng| eng.list_node_labels())?; + Ok(infos.into_iter().map(Into::into).collect()) + } + + #[napi] + pub fn list_edge_labels(&self) -> Result> { + let infos = with_engine_ref(self, |eng| eng.list_edge_labels())?; + Ok(infos.into_iter().map(Into::into).collect()) + } + + // --- Single upserts --- + + #[napi( + ts_args_type = "labels: string | string[], key: string, options?: UpsertNodeOptions | null" + )] pub fn upsert_node( &self, - type_id: u32, + labels: serde_json::Value, key: String, - options: Option, + options: Option, ) -> Result { + let labels = parse_js_node_labels_arg(&labels, "upsertNode labels")?; let (props, weight, dense_vector, sparse_vector) = match options { Some(o) => (o.props, o.weight, o.dense_vector, o.sparse_vector), None => (None, None, None, None), }; let props = convert_js_props(props); - let opts = UpsertNodeOptions { + let opts = CoreUpsertNodeOptions { props, weight: weight.unwrap_or(1.0) as f32, dense_vector: dense_vector.map(|dv| dv.into_iter().map(|x| x as f32).collect()), @@ -127,17 +189,29 @@ impl OverGraph { .collect() }), }; - let id = with_engine(self, |eng| eng.upsert_node(type_id, &key, opts))?; + let id = with_engine(self, |eng| eng.upsert_node(labels, &key, opts))?; u64_to_f64(id) } + #[napi] + pub fn add_node_label(&self, node_id: f64, label: String) -> Result { + let node_id = f64_to_u64(node_id)?; + with_engine(self, |eng| eng.add_node_label(node_id, &label)) + } + + #[napi] + pub fn remove_node_label(&self, node_id: f64, label: String) -> Result { + let node_id = f64_to_u64(node_id)?; + with_engine(self, |eng| eng.remove_node_label(node_id, &label)) + } + #[napi] pub fn upsert_edge( &self, from: f64, to: f64, - type_id: u32, - options: Option, + label: String, + options: Option, ) -> Result { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; @@ -146,31 +220,34 @@ impl OverGraph { None => (None, None, None, None), }; let props = convert_js_props(props); - let opts = UpsertEdgeOptions { + let opts = CoreUpsertEdgeOptions { props, weight: weight.unwrap_or(1.0) as f32, valid_from, valid_to, }; - let id = with_engine(self, |eng| eng.upsert_edge(from, to, type_id, opts))?; + let id = with_engine(self, |eng| eng.upsert_edge(from, to, &label, opts))?; u64_to_f64(id) } // --- Batch upserts (JSON object path) --- #[napi] - pub fn batch_upsert_nodes(&self, nodes: Vec) -> Result { - let inputs: Vec = nodes.into_iter().map(|n| n.into()).collect(); - let ids = with_engine(self, |eng| eng.batch_upsert_nodes(&inputs))?; + pub fn batch_upsert_nodes(&self, nodes: Vec) -> Result { + let inputs: Vec = nodes + .into_iter() + .map(NodeInput::try_into) + .collect::>>()?; + let ids = with_engine(self, |eng| eng.batch_upsert_nodes(inputs))?; ids_to_float64_array(&ids) } #[napi] - pub fn batch_upsert_edges(&self, edges: Vec) -> Result { - let inputs: std::result::Result, _> = + pub fn batch_upsert_edges(&self, edges: Vec) -> Result { + let inputs: std::result::Result, _> = edges.into_iter().map(|e| e.try_into()).collect(); let inputs = inputs?; - let ids = with_engine(self, |eng| eng.batch_upsert_edges(&inputs))?; + let ids = with_engine(self, |eng| eng.batch_upsert_edges(inputs))?; ids_to_float64_array(&ids) } @@ -179,52 +256,55 @@ impl OverGraph { /// Batch upsert nodes from a packed binary Buffer. See `packNodeBatch()` in JS. /// /// Binary format (little-endian): - /// [count: u32] + /// [magic: 4 bytes "OGNB"][version: u16 = 2][count: u32] /// per node: - /// [type_id: u32][weight: f32][key_len: u16][key: utf8][props_len: u32][props: json utf8] + /// [label_count: u8] repeated [label_len: u16][label: utf8][weight: f32] + /// [key_len: u16][key: utf8][props_len: u32][props: json utf8] #[napi] pub fn batch_upsert_nodes_binary(&self, buffer: Buffer) -> Result { let inputs = decode_node_batch(&buffer)?; - let ids = with_engine(self, |eng| eng.batch_upsert_nodes(&inputs))?; + let ids = with_engine(self, |eng| eng.batch_upsert_nodes(inputs))?; ids_to_float64_array(&ids) } /// Batch upsert edges from a packed binary Buffer. See `packEdgeBatch()` in JS. /// /// Binary format (little-endian): - /// [count: u32] + /// [magic: 4 bytes "OGEB"][version: u16 = 1][count: u32] /// per edge: - /// [from: u64][to: u64][type_id: u32][weight: f32] + /// [from: u64][to: u64][label_len: u16][label: utf8][weight: f32] /// [valid_from: i64][valid_to: i64][props_len: u32][props: json utf8] + /// In this packed format, valid_from=0 and valid_to=0 are sentinels for + /// engine defaults (created_at and no expiration), not explicit epoch 0. #[napi] pub fn batch_upsert_edges_binary(&self, buffer: Buffer) -> Result { let inputs = decode_edge_batch(&buffer)?; - let ids = with_engine(self, |eng| eng.batch_upsert_edges(&inputs))?; + let ids = with_engine(self, |eng| eng.batch_upsert_edges(inputs))?; ids_to_float64_array(&ids) } // --- Gets --- #[napi] - pub fn get_node(&self, id: f64) -> Result> { + pub fn get_node(&self, id: f64) -> Result> { let id = f64_to_u64(id)?; let raw = with_engine_ref(self, |eng| eng.get_node(id))?; - raw.map(JsNodeRecord::try_from).transpose() + raw.map(NodeView::try_from).transpose() } #[napi] - pub fn get_edge(&self, id: f64) -> Result> { + pub fn get_edge(&self, id: f64) -> Result> { let id = f64_to_u64(id)?; let raw = with_engine_ref(self, |eng| eng.get_edge(id))?; - raw.map(JsEdgeRecord::try_from).transpose() + raw.map(EdgeView::try_from).transpose() } // --- Key/triple lookups --- #[napi] - pub fn get_node_by_key(&self, type_id: u32, key: String) -> Result> { - let raw = with_engine_ref(self, |eng| eng.get_node_by_key(type_id, &key))?; - raw.map(JsNodeRecord::try_from).transpose() + pub fn get_node_by_key(&self, label: String, key: String) -> Result> { + let raw = with_engine_ref(self, |eng| eng.get_node_by_key(&label, &key))?; + raw.map(NodeView::try_from).transpose() } #[napi] @@ -232,18 +312,18 @@ impl OverGraph { &self, from: f64, to: f64, - type_id: u32, - ) -> Result> { + label: String, + ) -> Result> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; - let raw = with_engine_ref(self, |eng| eng.get_edge_by_triple(from, to, type_id))?; - raw.map(JsEdgeRecord::try_from).transpose() + let raw = with_engine_ref(self, |eng| eng.get_edge_by_triple(from, to, &label))?; + raw.map(EdgeView::try_from).transpose() } // --- Bulk reads --- #[napi] - pub fn get_nodes(&self, ids: Vec) -> Result>> { + pub fn get_nodes(&self, ids: Vec) -> Result>> { let ids: Vec = ids .into_iter() .map(f64_to_u64) @@ -251,23 +331,25 @@ impl OverGraph { let results = with_engine_ref(self, |eng| eng.get_nodes(&ids))?; results .into_iter() - .map(|r| r.map(JsNodeRecord::try_from).transpose()) + .map(|r| r.map(NodeView::try_from).transpose()) .collect::>>() } #[napi] - pub fn get_nodes_by_keys(&self, keys: Vec) -> Result>> { - let owned: Vec<(u32, String)> = keys.into_iter().map(|k| (k.type_id, k.key)).collect(); - let refs: Vec<(u32, &str)> = owned.iter().map(|(t, k)| (*t, k.as_str())).collect(); - let results = with_engine_ref(self, |eng| eng.get_nodes_by_keys(&refs))?; + pub fn get_nodes_by_keys(&self, keys: Vec) -> Result>> { + let owned: Vec = keys + .into_iter() + .map(KeyQuery::try_into) + .collect::>>()?; + let results = with_engine_ref(self, |eng| eng.get_nodes_by_keys(&owned))?; results .into_iter() - .map(|r| r.map(JsNodeRecord::try_from).transpose()) + .map(|r| r.map(NodeView::try_from).transpose()) .collect::>>() } #[napi] - pub fn get_edges(&self, ids: Vec) -> Result>> { + pub fn get_edges(&self, ids: Vec) -> Result>> { let ids: Vec = ids .into_iter() .map(f64_to_u64) @@ -275,7 +357,7 @@ impl OverGraph { let results = with_engine_ref(self, |eng| eng.get_edges(&ids))?; results .into_iter() - .map(|r| r.map(JsEdgeRecord::try_from).transpose()) + .map(|r| r.map(EdgeView::try_from).transpose()) .collect::>>() } @@ -296,30 +378,30 @@ impl OverGraph { // --- Temporal invalidation --- #[napi] - pub fn invalidate_edge(&self, id: f64, valid_to: i64) -> Result> { + pub fn invalidate_edge(&self, id: f64, valid_to: i64) -> Result> { let id = f64_to_u64(id)?; let raw = with_engine(self, |eng| eng.invalidate_edge(id, valid_to))?; - raw.map(JsEdgeRecord::try_from).transpose() + raw.map(EdgeView::try_from).transpose() } #[napi] - pub fn graph_patch(&self, patch: JsGraphPatch) -> Result { + pub fn graph_patch(&self, patch: GraphPatch) -> Result { let rust_patch = js_patch_to_rust(patch)?; - let result = with_engine(self, |eng| eng.graph_patch(&rust_patch))?; - Ok(JsPatchResult { + let result = with_engine(self, |eng| eng.graph_patch(rust_patch))?; + Ok(PatchResult { node_ids: ids_to_float64_array(&result.node_ids)?, edge_ids: ids_to_float64_array(&result.edge_ids)?, }) } #[napi] - pub fn begin_write_txn(&self) -> Result { + pub fn begin_write_txn(&self) -> Result { let txn = with_engine_ref(self, |eng| eng.begin_write_txn())?; Ok(write_txn_to_js(txn)) } - #[napi(ts_return_type = "Promise")] - pub fn begin_write_txn_async(&self) -> AsyncTask> { + #[napi(ts_return_type = "Promise")] + pub fn begin_write_txn_async(&self) -> AsyncTask> { AsyncTask::new(EngineReadOp::new( self.inner.clone(), |eng| eng.begin_write_txn(), @@ -330,15 +412,11 @@ impl OverGraph { // --- Retention / Forgetting --- #[napi] - pub fn prune(&self, policy: JsPrunePolicy) -> Result { - let rust_policy = PrunePolicy { - max_age_ms: policy.max_age_ms.map(|v| v as i64), - max_weight: policy.max_weight.map(|v| v as f32), - type_id: policy.type_id, - }; + pub fn prune(&self, policy: PrunePolicy) -> Result { + let rust_policy = js_prune_policy_to_rust(policy, "prune")?; with_engine(self, |eng| { let result = eng.prune(&rust_policy)?; - Ok(JsPruneResult { + Ok(PruneResult { nodes_pruned: result.nodes_pruned as i64, edges_pruned: result.edges_pruned as i64, }) @@ -348,12 +426,8 @@ impl OverGraph { // --- Named prune policies (compaction-filter auto-prune) --- #[napi] - pub fn set_prune_policy(&self, name: String, policy: JsPrunePolicy) -> Result<()> { - let rust_policy = PrunePolicy { - max_age_ms: policy.max_age_ms.map(|v| v as i64), - max_weight: policy.max_weight.map(|v| v as f32), - type_id: policy.type_id, - }; + pub fn set_prune_policy(&self, name: String, policy: PrunePolicy) -> Result<()> { + let rust_policy = js_prune_policy_to_rust(policy, "setPrunePolicy")?; with_engine(self, |eng| { eng.set_prune_policy(&name, rust_policy)?; Ok(()) @@ -366,17 +440,17 @@ impl OverGraph { } #[napi] - pub fn list_prune_policies(&self) -> Result> { + pub fn list_prune_policies(&self) -> Result> { with_engine_ref(self, |eng| { Ok(eng .list_prune_policies()? .into_iter() - .map(|(name, p)| JsNamedPrunePolicy { - name, - policy: JsPrunePolicy { - max_age_ms: p.max_age_ms.map(|v| v as f64), - max_weight: p.max_weight.map(|v| v as f64), - type_id: p.type_id, + .map(|info| NamedPrunePolicy { + name: info.name, + policy: PrunePolicy { + max_age_ms: info.policy.max_age_ms.map(|v| v as f64), + max_weight: info.policy.max_weight.map(|v| v as f64), + label: info.policy.label, }, }) .collect()) @@ -387,13 +461,9 @@ impl OverGraph { pub fn set_prune_policy_async( &self, name: String, - policy: JsPrunePolicy, + policy: PrunePolicy, ) -> Result>> { - let rust_policy = PrunePolicy { - max_age_ms: policy.max_age_ms.map(|v| v as i64), - max_weight: policy.max_weight.map(|v| v as f32), - type_id: policy.type_id, - }; + let rust_policy = js_prune_policy_to_rust(policy, "setPrunePolicyAsync")?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), move |eng| { @@ -416,22 +486,22 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn list_prune_policies_async( &self, - ) -> Result, Vec>>> { + ) -> Result, Vec>>> { Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), move |eng| eng.list_prune_policies(), |policies| { Ok(policies .into_iter() - .map(|(name, p)| JsNamedPrunePolicy { - name, - policy: JsPrunePolicy { - max_age_ms: p.max_age_ms.map(|v| v as f64), - max_weight: p.max_weight.map(|v| v as f64), - type_id: p.type_id, + .map(|info| NamedPrunePolicy { + name: info.name, + policy: PrunePolicy { + max_age_ms: info.policy.max_age_ms.map(|v| v as f64), + max_weight: info.policy.max_weight.map(|v| v as f64), + label: info.policy.label, }, }) .collect()) @@ -445,13 +515,13 @@ impl OverGraph { pub fn neighbors( &self, node_id: f64, - options: Option, - ) -> Result> { + options: Option, + ) -> Result> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, limit, at_epoch, decay_lambda) = match options { + let (direction, edge_label_filter, limit, at_epoch, decay_lambda) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.limit, o.at_epoch, o.decay_lambda, @@ -463,7 +533,7 @@ impl OverGraph { let decay = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter, limit: lim, at_epoch, decay_lambda: decay, @@ -477,14 +547,14 @@ impl OverGraph { &self, start_node_id: f64, max_depth: u32, - options: Option, - ) -> Result { + options: Option, + ) -> Result { let start_node_id = f64_to_u64(start_node_id)?; let ( direction, min_depth, - edge_type_filter, - node_type_filter, + edge_label_filter, + emit_node_label_filter, at_epoch, decay_lambda, limit, @@ -493,8 +563,10 @@ impl OverGraph { Some(o) => ( o.direction, o.min_depth, - o.edge_type_filter, - o.node_type_filter, + o.edge_label_filter, + o.emit_node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, o.at_epoch, o.decay_lambda, o.limit, @@ -505,11 +577,11 @@ impl OverGraph { let dir = parse_direction(direction.as_deref())?; let min_depth = min_depth.unwrap_or(1); let cursor = cursor.map(js_traversal_cursor_to_rust).transpose()?; - let opts = TraverseOptions { + let opts = CoreTraverseOptions { min_depth, direction: dir, - edge_type_filter, - node_type_filter, + edge_label_filter, + emit_node_label_filter, at_epoch, decay_lambda, limit: limit.map(|v| v as usize), @@ -524,13 +596,13 @@ impl OverGraph { &self, node_id: f64, k: u32, - options: Option, - ) -> Result> { + options: Option, + ) -> Result> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, scoring, decay_lambda, at_epoch) = match options { + let (direction, edge_label_filter, scoring, decay_lambda, at_epoch) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.scoring, o.decay_lambda, o.at_epoch, @@ -541,7 +613,7 @@ impl OverGraph { let scoring_mode = parse_scoring_mode(scoring.as_deref(), decay_lambda)?; let opts = TopKOptions { direction: dir, - type_filter, + edge_label_filter, scoring: scoring_mode, at_epoch, }; @@ -554,17 +626,25 @@ impl OverGraph { &self, start_node_id: f64, max_depth: u32, - options: Option, - ) -> Result { + options: Option, + ) -> Result { let start = f64_to_u64(start_node_id)?; - let (direction, edge_type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.edge_type_filter, o.at_epoch), - None => (None, None, None), + let (direction, edge_label_filter, node_label_filter, at_epoch) = match options { + Some(o) => ( + o.direction, + o.edge_label_filter, + o.node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, + o.at_epoch, + ), + None => (None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; let opts = SubgraphOptions { direction: dir, - edge_type_filter, + edge_label_filter, + node_label_filter, at_epoch, }; let sg = with_engine_ref(self, |eng| eng.extract_subgraph(start, max_depth, &opts))?; @@ -577,21 +657,21 @@ impl OverGraph { pub fn neighbors_batch( &self, node_ids: Vec, - options: Option, - ) -> Result> { + options: Option, + ) -> Result> { let ids: Vec = node_ids .into_iter() .map(f64_to_u64) .collect::>>()?; - let (direction, type_filter, at_epoch, decay_lambda) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch, o.decay_lambda), + let (direction, edge_label_filter, at_epoch, decay_lambda) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch, o.decay_lambda), None => (None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; let decay = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter, limit: None, at_epoch, decay_lambda: decay, @@ -603,16 +683,16 @@ impl OverGraph { // --- Degree counts + aggregations (Phase 18a) --- #[napi] - pub fn degree(&self, node_id: f64, options: Option) -> Result { + pub fn degree(&self, node_id: f64, options: Option) -> Result { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; let count: u64 = with_engine_ref(self, |eng| eng.degree(node_id, &opts))?; @@ -623,17 +703,17 @@ impl OverGraph { pub fn sum_edge_weights( &self, node_id: f64, - options: Option, + options: Option, ) -> Result { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; with_engine_ref(self, |eng| eng.sum_edge_weights(node_id, &opts)) @@ -643,17 +723,17 @@ impl OverGraph { pub fn avg_edge_weight( &self, node_id: f64, - options: Option, + options: Option, ) -> Result> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; with_engine_ref(self, |eng| eng.avg_edge_weight(node_id, &opts)) @@ -663,27 +743,27 @@ impl OverGraph { pub fn degrees( &self, node_ids: Vec, - options: Option, - ) -> Result> { + options: Option, + ) -> Result> { let ids: Vec = node_ids .into_iter() .map(f64_to_u64) .collect::>>()?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; let map = with_engine_ref(self, |eng| eng.degrees(&ids, &opts))?; - let mut entries: Vec = map + let mut entries: Vec = map .into_iter() .map(|(node_id, degree)| { - Ok(JsDegreeBatchEntry { + Ok(DegreeBatchEntry { node_id: u64_to_f64(node_id)?, degree: u64_to_safe_i64(degree)?, }) @@ -700,31 +780,32 @@ impl OverGraph { &self, from: f64, to: f64, - options: Option, - ) -> Result> { + options: Option, + ) -> Result> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; - let (direction, type_filter, weight_field, at_epoch, max_depth, max_cost) = match options { - Some(o) => ( - o.direction, - o.type_filter, - o.weight_field, - o.at_epoch, - o.max_depth, - o.max_cost, - ), - None => (None, None, None, None, None, None), - }; + let (direction, edge_label_filter, weight_field, at_epoch, max_depth, max_cost) = + match options { + Some(o) => ( + o.direction, + o.edge_label_filter, + o.weight_field, + o.at_epoch, + o.max_depth, + o.max_cost, + ), + None => (None, None, None, None, None, None), + }; let dir = parse_direction(direction.as_deref())?; - let opts = ShortestPathOptions { + let opts = CoreShortestPathOptions { direction: dir, - type_filter, + edge_label_filter, weight_field, at_epoch, max_depth, max_cost, }; - let result: Option = + let result: Option = with_engine_ref(self, |eng| eng.shortest_path(from, to, &opts))?; result.map(shortest_path_to_js).transpose() } @@ -734,18 +815,18 @@ impl OverGraph { &self, from: f64, to: f64, - options: Option, + options: Option, ) -> Result { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; - let (direction, type_filter, at_epoch, max_depth) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch, o.max_depth), + let (direction, edge_label_filter, at_epoch, max_depth) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch, o.max_depth), None => (None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = IsConnectedOptions { + let opts = CoreIsConnectedOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, max_depth, }; @@ -757,15 +838,15 @@ impl OverGraph { &self, from: f64, to: f64, - options: Option, - ) -> Result> { + options: Option, + ) -> Result> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; - let (direction, type_filter, weight_field, at_epoch, max_depth, max_cost, max_paths) = + let (direction, edge_label_filter, weight_field, at_epoch, max_depth, max_cost, max_paths) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.weight_field, o.at_epoch, o.max_depth, @@ -775,16 +856,16 @@ impl OverGraph { None => (None, None, None, None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = AllShortestPathsOptions { + let opts = CoreAllShortestPathsOptions { direction: dir, - type_filter, + edge_label_filter, weight_field, at_epoch, max_depth, max_cost, max_paths: max_paths.map(|n| n as usize), }; - let paths: Vec = + let paths: Vec = with_engine_ref(self, |eng| eng.all_shortest_paths(from, to, &opts))?; paths.into_iter().map(shortest_path_to_js).collect() } @@ -792,20 +873,20 @@ impl OverGraph { #[napi] pub fn find_nodes( &self, - type_id: u32, + label: String, prop_key: String, prop_value: serde_json::Value, ) -> Result { let pv = json_to_prop_value(&prop_value); - let ids = with_engine_ref(self, |eng| eng.find_nodes(type_id, &prop_key, &pv))?; + let ids = with_engine_ref(self, |eng| eng.find_nodes(&label, &prop_key, &pv))?; ids_to_float64_array(&ids) } #[napi( ts_args_type = "request: import('./query-types').QueryNodeRequest", - ts_return_type = "JsIdPageResult" + ts_return_type = "IdPageResult" )] - pub fn query_node_ids(&self, request: serde_json::Value) -> Result { + pub fn query_node_ids(&self, request: serde_json::Value) -> Result { let query = parse_js_node_query(&request)?; let result = with_engine_ref(self, |eng| eng.query_node_ids(&query))?; query_node_ids_to_js(result) @@ -813,19 +894,39 @@ impl OverGraph { #[napi( ts_args_type = "request: import('./query-types').QueryNodeRequest", - ts_return_type = "JsNodePageResult" + ts_return_type = "NodePageResult" )] - pub fn query_nodes(&self, request: serde_json::Value) -> Result { + pub fn query_nodes(&self, request: serde_json::Value) -> Result { let query = parse_js_node_query(&request)?; let result = with_engine_ref(self, |eng| eng.query_nodes(&query))?; query_nodes_to_js(result) } + #[napi( + ts_args_type = "request: import('./query-types').QueryEdgeRequest", + ts_return_type = "IdPageResult" + )] + pub fn query_edge_ids(&self, request: serde_json::Value) -> Result { + let query = parse_js_edge_query(&request)?; + let result = with_engine_ref(self, |eng| eng.query_edge_ids(&query))?; + query_edge_ids_to_js(result) + } + + #[napi( + ts_args_type = "request: import('./query-types').QueryEdgeRequest", + ts_return_type = "EdgePageResult" + )] + pub fn query_edges(&self, request: serde_json::Value) -> Result { + let query = parse_js_edge_query(&request)?; + let result = with_engine_ref(self, |eng| eng.query_edges(&query))?; + query_edges_to_js(result) + } + #[napi( ts_args_type = "request: import('./query-types').GraphPatternRequest", ts_return_type = "import('./query-types').QueryPatternResult" )] - pub fn query_pattern(&self, request: serde_json::Value) -> Result { + pub fn query_pattern(&self, request: serde_json::Value) -> Result { let query = parse_js_graph_pattern_query(&request)?; let result = with_engine_ref(self, |eng| eng.query_pattern(&query))?; query_pattern_result_to_js(result) @@ -835,17 +936,27 @@ impl OverGraph { ts_args_type = "request: import('./query-types').QueryNodeRequest", ts_return_type = "import('./query-types').QueryPlan" )] - pub fn explain_node_query(&self, request: serde_json::Value) -> Result { + pub fn explain_node_query(&self, request: serde_json::Value) -> Result { let query = parse_js_node_query(&request)?; let plan = with_engine_ref(self, |eng| eng.explain_node_query(&query))?; query_plan_to_js(plan) } + #[napi( + ts_args_type = "request: import('./query-types').QueryEdgeRequest", + ts_return_type = "import('./query-types').QueryPlan" + )] + pub fn explain_edge_query(&self, request: serde_json::Value) -> Result { + let query = parse_js_edge_query(&request)?; + let plan = with_engine_ref(self, |eng| eng.explain_edge_query(&query))?; + query_plan_to_js(plan) + } + #[napi( ts_args_type = "request: import('./query-types').GraphPatternRequest", ts_return_type = "import('./query-types').QueryPlan" )] - pub fn explain_pattern_query(&self, request: serde_json::Value) -> Result { + pub fn explain_pattern_query(&self, request: serde_json::Value) -> Result { let query = parse_js_graph_pattern_query(&request)?; let plan = with_engine_ref(self, |eng| eng.explain_pattern_query(&query))?; query_plan_to_js(plan) @@ -854,13 +965,13 @@ impl OverGraph { #[napi] pub fn ensure_node_property_index( &self, - type_id: u32, + label: String, prop_key: String, - kind: JsSecondaryIndexKind, - ) -> Result { + kind: SecondaryIndexKind, + ) -> Result { let kind = js_secondary_index_kind_to_rust(kind)?; let info = with_engine(self, |eng| { - eng.ensure_node_property_index(type_id, &prop_key, kind.clone()) + eng.ensure_node_property_index(&label, &prop_key, kind.clone()) })?; node_property_index_info_to_js(info) } @@ -868,18 +979,18 @@ impl OverGraph { #[napi] pub fn drop_node_property_index( &self, - type_id: u32, + label: String, prop_key: String, - kind: JsSecondaryIndexKind, + kind: SecondaryIndexKind, ) -> Result { let kind = js_secondary_index_kind_to_rust(kind)?; with_engine(self, |eng| { - eng.drop_node_property_index(type_id, &prop_key, kind.clone()) + eng.drop_node_property_index(&label, &prop_key, kind.clone()) }) } #[napi] - pub fn list_node_property_indexes(&self) -> Result> { + pub fn list_node_property_indexes(&self) -> Result> { let infos = with_engine_ref(self, |eng| eng.list_node_property_indexes())?; infos .into_iter() @@ -887,106 +998,151 @@ impl OverGraph { .collect() } - /// Return all node IDs of a given type (unpaged). #[napi] - pub fn nodes_by_type(&self, type_id: u32) -> Result { - let ids = with_engine_ref(self, |eng| eng.nodes_by_type(type_id))?; - ids_to_float64_array(&ids) + pub fn ensure_edge_property_index( + &self, + label: String, + prop_key: String, + kind: SecondaryIndexKind, + ) -> Result { + let kind = js_secondary_index_kind_to_rust(kind)?; + let info = with_engine(self, |eng| { + eng.ensure_edge_property_index(&label, &prop_key, kind.clone()) + })?; + edge_property_index_info_to_js(info) + } + + #[napi] + pub fn drop_edge_property_index( + &self, + label: String, + prop_key: String, + kind: SecondaryIndexKind, + ) -> Result { + let kind = js_secondary_index_kind_to_rust(kind)?; + with_engine(self, |eng| { + eng.drop_edge_property_index(&label, &prop_key, kind.clone()) + }) } - /// Return all edge IDs of a given type (unpaged). #[napi] - pub fn edges_by_type(&self, type_id: u32) -> Result { - let ids = with_engine_ref(self, |eng| eng.edges_by_type(type_id))?; + pub fn list_edge_property_indexes(&self) -> Result> { + let infos = with_engine_ref(self, |eng| eng.list_edge_property_indexes())?; + infos + .into_iter() + .map(edge_property_index_info_to_js) + .collect() + } + + /// Return all node IDs containing every supplied node label (unpaged). + #[napi(ts_args_type = "labels: string | string[]")] + pub fn nodes_by_labels(&self, labels: serde_json::Value) -> Result { + let labels = parse_js_node_labels_arg(&labels, "nodesByLabels labels")?; + let ids = with_engine_ref(self, |eng| eng.nodes_by_labels(labels))?; ids_to_float64_array(&ids) } + /// Return all edge IDs of a given label (unpaged). #[napi] - pub fn get_nodes_by_type(&self, type_id: u32) -> Result> { - let records = with_engine_ref(self, |eng| eng.get_nodes_by_type(type_id))?; + pub fn edges_by_label(&self, label: String) -> Result { + let ids = with_engine_ref(self, |eng| eng.edges_by_label(&label))?; + ids_to_float64_array(&ids) + } + + #[napi(ts_args_type = "labels: string | string[]")] + pub fn get_nodes_by_labels(&self, labels: serde_json::Value) -> Result> { + let labels = parse_js_node_labels_arg(&labels, "getNodesByLabels labels")?; + let records = with_engine_ref(self, |eng| eng.get_nodes_by_labels(labels))?; records .into_iter() - .map(JsNodeRecord::try_from) + .map(NodeView::try_from) .collect::>>() } #[napi] - pub fn get_edges_by_type(&self, type_id: u32) -> Result> { - let records = with_engine_ref(self, |eng| eng.get_edges_by_type(type_id))?; + pub fn get_edges_by_label(&self, label: String) -> Result> { + let records = with_engine_ref(self, |eng| eng.get_edges_by_label(&label))?; records .into_iter() - .map(JsEdgeRecord::try_from) + .map(EdgeView::try_from) .collect::>>() } - #[napi] - pub fn count_nodes_by_type(&self, type_id: u32) -> Result { - with_engine_ref(self, |eng| Ok(eng.count_nodes_by_type(type_id)? as i64)) + #[napi(ts_args_type = "labels: string | string[]")] + pub fn count_nodes_by_labels(&self, labels: serde_json::Value) -> Result { + let labels = parse_js_node_labels_arg(&labels, "countNodesByLabels labels")?; + with_engine_ref(self, |eng| Ok(eng.count_nodes_by_labels(labels)? as i64)) } #[napi] - pub fn count_edges_by_type(&self, type_id: u32) -> Result { - with_engine_ref(self, |eng| Ok(eng.count_edges_by_type(type_id)? as i64)) + pub fn count_edges_by_label(&self, label: String) -> Result { + with_engine_ref(self, |eng| Ok(eng.count_edges_by_label(&label)? as i64)) } // --- Paginated queries (sync) --- - #[napi] - pub fn nodes_by_type_paged( + #[napi( + ts_args_type = "labels: string | string[], limit?: number | null, after?: number | null" + )] + pub fn nodes_by_labels_paged( &self, - type_id: u32, + labels: serde_json::Value, limit: Option, after: Option, - ) -> Result { + ) -> Result { + let labels = parse_js_node_labels_arg(&labels, "nodesByLabelsPaged labels")?; let page = make_page_request(limit, after)?; - let raw = with_engine_ref(self, |eng| eng.nodes_by_type_paged(type_id, &page))?; + let raw = with_engine_ref(self, |eng| eng.nodes_by_labels_paged(labels, &page))?; id_page_to_js(raw) } #[napi] - pub fn edges_by_type_paged( + pub fn edges_by_label_paged( &self, - type_id: u32, + label: String, limit: Option, after: Option, - ) -> Result { + ) -> Result { let page = make_page_request(limit, after)?; - let raw = with_engine_ref(self, |eng| eng.edges_by_type_paged(type_id, &page))?; + let raw = with_engine_ref(self, |eng| eng.edges_by_label_paged(&label, &page))?; id_page_to_js(raw) } - #[napi] - pub fn get_nodes_by_type_paged( + #[napi( + ts_args_type = "labels: string | string[], limit?: number | null, after?: number | null" + )] + pub fn get_nodes_by_labels_paged( &self, - type_id: u32, + labels: serde_json::Value, limit: Option, after: Option, - ) -> Result { + ) -> Result { + let labels = parse_js_node_labels_arg(&labels, "getNodesByLabelsPaged labels")?; let page = make_page_request(limit, after)?; - let raw = with_engine_ref(self, |eng| eng.get_nodes_by_type_paged(type_id, &page))?; + let raw = with_engine_ref(self, |eng| eng.get_nodes_by_labels_paged(labels, &page))?; node_page_to_js(raw) } #[napi] - pub fn get_edges_by_type_paged( + pub fn get_edges_by_label_paged( &self, - type_id: u32, + label: String, limit: Option, after: Option, - ) -> Result { + ) -> Result { let page = make_page_request(limit, after)?; - let raw = with_engine_ref(self, |eng| eng.get_edges_by_type_paged(type_id, &page))?; + let raw = with_engine_ref(self, |eng| eng.get_edges_by_label_paged(&label, &page))?; edge_page_to_js(raw) } #[napi] pub fn find_nodes_paged( &self, - type_id: u32, + label: String, prop_key: String, prop_value: serde_json::Value, - options: Option, - ) -> Result { + options: Option, + ) -> Result { let pv = json_to_prop_value(&prop_value); let (limit, after) = match options { Some(o) => (o.limit, o.after), @@ -994,7 +1150,7 @@ impl OverGraph { }; let page = make_page_request(limit, after)?; let raw = with_engine_ref(self, |eng| { - eng.find_nodes_paged(type_id, &prop_key, &pv, &page) + eng.find_nodes_paged(&label, &prop_key, &pv, &page) })?; id_page_to_js(raw) } @@ -1002,12 +1158,12 @@ impl OverGraph { #[napi] pub fn find_nodes_by_time_range( &self, - type_id: u32, + label: String, from_ms: i64, to_ms: i64, ) -> Result { let ids = with_engine_ref(self, |eng| { - eng.find_nodes_by_time_range(type_id, from_ms, to_ms) + eng.find_nodes_by_time_range(&label, from_ms, to_ms) })?; ids_to_float64_array(&ids) } @@ -1015,10 +1171,10 @@ impl OverGraph { #[napi] pub fn find_nodes_range( &self, - type_id: u32, + label: String, prop_key: String, - lower: Option, - upper: Option, + lower: Option, + upper: Option, ) -> Result { let lower = lower .as_ref() @@ -1029,7 +1185,7 @@ impl OverGraph { .map(js_property_range_bound_to_rust) .transpose()?; let ids = with_engine_ref(self, |eng| { - eng.find_nodes_range(type_id, &prop_key, lower.as_ref(), upper.as_ref()) + eng.find_nodes_range(&label, &prop_key, lower.as_ref(), upper.as_ref()) })?; ids_to_float64_array(&ids) } @@ -1037,18 +1193,18 @@ impl OverGraph { #[napi] pub fn find_nodes_by_time_range_paged( &self, - type_id: u32, + label: String, from_ms: i64, to_ms: i64, - options: Option, - ) -> Result { + options: Option, + ) -> Result { let (limit, after) = match options { Some(o) => (o.limit, o.after), None => (None, None), }; let page = make_page_request(limit, after)?; let raw = with_engine_ref(self, |eng| { - eng.find_nodes_by_time_range_paged(type_id, from_ms, to_ms, &page) + eng.find_nodes_by_time_range_paged(&label, from_ms, to_ms, &page) })?; id_page_to_js(raw) } @@ -1056,12 +1212,12 @@ impl OverGraph { #[napi] pub fn find_nodes_range_paged( &self, - type_id: u32, + label: String, prop_key: String, - lower: Option, - upper: Option, - options: Option, - ) -> Result { + lower: Option, + upper: Option, + options: Option, + ) -> Result { let lower = lower .as_ref() .map(js_property_range_bound_to_rust) @@ -1072,7 +1228,7 @@ impl OverGraph { .transpose()?; let page = make_property_range_page_request(options)?; let raw = with_engine_ref(self, |eng| { - eng.find_nodes_range_paged(type_id, &prop_key, lower.as_ref(), upper.as_ref(), &page) + eng.find_nodes_range_paged(&label, &prop_key, lower.as_ref(), upper.as_ref(), &page) })?; property_range_page_to_js(raw) } @@ -1081,8 +1237,8 @@ impl OverGraph { pub fn personalized_pagerank( &self, seed_node_ids: Vec, - options: Option, - ) -> Result { + options: Option, + ) -> Result { let seeds: Vec = seed_node_ids .into_iter() .map(f64_to_u64) @@ -1093,7 +1249,7 @@ impl OverGraph { max_iterations, epsilon, approx_residual_tolerance, - edge_type_filter, + edge_label_filter, max_results, ) = match &options { Some(o) => ( @@ -1102,7 +1258,7 @@ impl OverGraph { o.max_iterations, o.epsilon, o.approx_residual_tolerance, - o.edge_type_filter.clone(), + o.edge_label_filter.clone(), o.max_results, ), None => (None, None, None, None, None, None, None), @@ -1113,7 +1269,7 @@ impl OverGraph { &max_iterations, &epsilon, &approx_residual_tolerance, - &edge_type_filter, + &edge_label_filter, &max_results, )?; let result = with_engine_ref(self, |eng| eng.personalized_pagerank(&seeds, &opts))?; @@ -1121,12 +1277,12 @@ impl OverGraph { } #[napi] - pub fn export_adjacency(&self, options: Option) -> Result { + pub fn export_adjacency(&self, options: Option) -> Result { let include_weights = options .as_ref() .and_then(|o| o.include_weights) .unwrap_or(true); - let opts = js_export_options_to_rust(options); + let opts = js_export_options_to_rust(options)?; let result = with_engine_ref(self, |eng| eng.export_adjacency(&opts))?; adjacency_export_to_js(result, include_weights) } @@ -1135,13 +1291,13 @@ impl OverGraph { pub fn neighbors_paged( &self, node_id: f64, - options: Option, - ) -> Result { + options: Option, + ) -> Result { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, limit, after, at_epoch, decay_lambda) = match options { + let (direction, edge_label_filter, limit, after, at_epoch, decay_lambda) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.limit, o.after, o.at_epoch, @@ -1154,7 +1310,7 @@ impl OverGraph { let decay = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, limit: None, at_epoch, decay_lambda: decay, @@ -1168,22 +1324,28 @@ impl OverGraph { #[napi] pub fn connected_components( &self, - options: Option, - ) -> Result> { - let (edge_type_filter, node_type_filter, at_epoch) = match options { - Some(o) => (o.edge_type_filter, o.node_type_filter, o.at_epoch), + options: Option, + ) -> Result> { + let (edge_label_filter, node_label_filter, at_epoch) = match options { + Some(o) => ( + o.edge_label_filter, + o.node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, + o.at_epoch, + ), None => (None, None, None), }; let opts = ComponentOptions { - edge_type_filter, - node_type_filter, + edge_label_filter, + node_label_filter, at_epoch, }; let map = with_engine_ref(self, |eng| eng.connected_components(&opts))?; - let mut entries: Vec = map + let mut entries: Vec = map .into_iter() .map(|(node_id, component_id)| { - Ok(JsComponentEntry { + Ok(ComponentEntry { node_id: u64_to_f64(node_id)?, component_id: u64_to_f64(component_id)?, }) @@ -1197,16 +1359,22 @@ impl OverGraph { pub fn component_of( &self, node_id: f64, - options: Option, + options: Option, ) -> Result { let node_id = f64_to_u64(node_id)?; - let (edge_type_filter, node_type_filter, at_epoch) = match options { - Some(o) => (o.edge_type_filter, o.node_type_filter, o.at_epoch), + let (edge_label_filter, node_label_filter, at_epoch) = match options { + Some(o) => ( + o.edge_label_filter, + o.node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, + o.at_epoch, + ), None => (None, None, None), }; let opts = ComponentOptions { - edge_type_filter, - node_type_filter, + edge_label_filter, + node_label_filter, at_epoch, }; let members = with_engine_ref(self, |eng| eng.component_of(node_id, &opts))?; @@ -1219,13 +1387,16 @@ impl OverGraph { pub fn vector_search( &self, mode: String, - options: JsVectorSearchOptions, - ) -> Result> { + options: VectorSearchOptions, + ) -> Result> { let mode = parse_vector_search_mode(&mode)?; let k = options.k; let dense_query = options.dense_query; let sparse_query = options.sparse_query; - let type_filter = options.type_filter; + let label_filter = options + .label_filter + .map(js_node_label_filter_to_rust) + .transpose()?; let ef_search = options.ef_search; let scope = options.scope; let dense_weight = options.dense_weight; @@ -1240,11 +1411,11 @@ impl OverGraph { }); let scope = match scope { None => None, - Some(s) => Some(VectorSearchScope { + Some(s) => Some(CoreVectorSearchScope { start_node_id: f64_to_u64(s.start_node_id)?, max_depth: s.max_depth, direction: parse_direction(s.direction.as_deref())?, - edge_type_filter: s.edge_type_filter, + edge_label_filter: s.edge_label_filter, at_epoch: s.at_epoch, }), }; @@ -1253,7 +1424,7 @@ impl OverGraph { dense_query: dense_q, sparse_query: sparse_q, k: k as usize, - type_filter, + label_filter, ef_search: ef_search.map(|v| v as usize), scope, dense_weight: dense_weight.map(|v| v as f32), @@ -1263,7 +1434,7 @@ impl OverGraph { let hits = with_engine_ref(self, |eng| eng.vector_search(&request))?; hits.into_iter() .map(|h| { - Ok(JsVectorHit { + Ok(VectorHit { node_id: u64_to_f64(h.node_id)?, score: h.score as f64, }) @@ -1297,26 +1468,26 @@ impl OverGraph { } #[napi] - pub fn end_ingest(&self) -> Result> { + pub fn end_ingest(&self) -> Result> { with_engine(self, |eng| Ok(eng.end_ingest()?.map(|s| s.into()))) } #[napi] - pub fn compact(&self) -> Result> { + pub fn compact(&self) -> Result> { with_engine(self, |eng| Ok(eng.compact()?.map(|s| s.into()))) } /// Compact with a progress callback. The callback receives a progress object /// and should return `true` to continue or `false` to cancel. /// Runs synchronously. Blocks the event loop. - #[napi(ts_args_type = "callback: (progress: JsCompactionProgress) => boolean")] + #[napi(ts_args_type = "callback: (progress: CompactionProgress) => boolean")] pub fn compact_with_progress( &self, - callback: Function, - ) -> Result> { + callback: Function, + ) -> Result> { with_engine(self, |eng| { let result = eng.compact_with_progress(|progress| { - let js_progress = JsCompactionProgress { + let js_progress = CompactionProgress { phase: match progress.phase { CompactionPhase::CollectingTombstones => { "collecting_tombstones".to_string() @@ -1345,16 +1516,21 @@ impl OverGraph { } #[napi] - pub fn stats(&self) -> Result { + pub fn stats(&self) -> Result { with_engine_ref(self, |eng| Ok(eng.stats()?.into())) } + #[napi] + pub fn scrub(&self) -> Result { + with_engine_ref(self, |eng| Ok(eng.scrub()?.into())) + } + // ============================ // Async API (Promise-returning) // ============================ #[napi(ts_return_type = "Promise")] - pub fn close_async(&self, options: Option) -> AsyncTask { + pub fn close_async(&self, options: Option) -> AsyncTask { let force = options.as_ref().and_then(|o| o.force).unwrap_or(false); AsyncTask::new(CloseOp { db: self.inner.clone(), @@ -1362,8 +1538,8 @@ impl OverGraph { }) } - #[napi(ts_return_type = "Promise")] - pub fn stats_async(&self) -> AsyncTask> { + #[napi(ts_return_type = "Promise")] + pub fn stats_async(&self) -> AsyncTask> { AsyncTask::new(EngineReadOp::new( self.inner.clone(), |eng| eng.stats(), @@ -1371,19 +1547,120 @@ impl OverGraph { )) } + #[napi(ts_return_type = "Promise")] + pub fn scrub_async(&self) -> AsyncTask> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + |eng| eng.scrub(), + |r| Ok(r.into()), + )) + } + + #[napi(ts_return_type = "Promise")] + pub fn ensure_node_label_async(&self, label: String) -> AsyncTask> { + AsyncTask::new(EngineOp::new( + self.inner.clone(), + move |eng| eng.ensure_node_label(&label), + Ok, + )) + } + #[napi(ts_return_type = "Promise")] + pub fn ensure_edge_label_async(&self, label: String) -> AsyncTask> { + AsyncTask::new(EngineOp::new( + self.inner.clone(), + move |eng| eng.ensure_edge_label(&label), + Ok, + )) + } + + #[napi(ts_return_type = "Promise")] + pub fn get_node_label_id_async( + &self, + label: String, + ) -> AsyncTask, Option>> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + move |eng| eng.get_node_label_id(&label), + Ok, + )) + } + + #[napi(ts_return_type = "Promise")] + pub fn get_edge_label_id_async( + &self, + label: String, + ) -> AsyncTask, Option>> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + move |eng| eng.get_edge_label_id(&label), + Ok, + )) + } + + #[napi(ts_return_type = "Promise")] + pub fn get_node_label_async( + &self, + label_id: u32, + ) -> AsyncTask, Option>> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + move |eng| eng.get_node_label(label_id), + Ok, + )) + } + + #[napi(ts_return_type = "Promise")] + pub fn get_edge_label_async( + &self, + label_id: u32, + ) -> AsyncTask, Option>> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + move |eng| eng.get_edge_label(label_id), + Ok, + )) + } + + #[napi(ts_return_type = "Promise>")] + pub fn list_node_labels_async( + &self, + ) -> AsyncTask, Vec>> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + |eng| eng.list_node_labels(), + |infos| Ok(infos.into_iter().map(Into::into).collect()), + )) + } + + #[napi(ts_return_type = "Promise>")] + pub fn list_edge_labels_async( + &self, + ) -> AsyncTask, Vec>> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + |eng| eng.list_edge_labels(), + |infos| Ok(infos.into_iter().map(Into::into).collect()), + )) + } + + #[napi( + ts_args_type = "labels: string | string[], key: string, options?: UpsertNodeOptions | null", + ts_return_type = "Promise" + )] pub fn upsert_node_async( &self, - type_id: u32, + labels: serde_json::Value, key: String, - options: Option, - ) -> AsyncTask> { + options: Option, + ) -> Result>> { + let labels = parse_js_node_labels_arg(&labels, "upsertNodeAsync labels")?; let (props, weight, dense_vector, sparse_vector) = match options { Some(o) => (o.props, o.weight, o.dense_vector, o.sparse_vector), None => (None, None, None, None), }; let props = convert_js_props(props); - let opts = UpsertNodeOptions { + let opts = CoreUpsertNodeOptions { props, weight: weight.unwrap_or(1.0) as f32, dense_vector: dense_vector.map(|dv| dv.into_iter().map(|x| x as f32).collect()), @@ -1393,11 +1670,39 @@ impl OverGraph { .collect() }), }; - AsyncTask::new(EngineOp::new( + Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.upsert_node(type_id, &key, opts), + move |eng| eng.upsert_node(labels, &key, opts), u64_to_f64, - )) + ))) + } + + #[napi(ts_return_type = "Promise")] + pub fn add_node_label_async( + &self, + node_id: f64, + label: String, + ) -> Result>> { + let node_id = f64_to_u64(node_id)?; + Ok(AsyncTask::new(EngineOp::new( + self.inner.clone(), + move |eng| eng.add_node_label(node_id, &label), + Ok, + ))) + } + + #[napi(ts_return_type = "Promise")] + pub fn remove_node_label_async( + &self, + node_id: f64, + label: String, + ) -> Result>> { + let node_id = f64_to_u64(node_id)?; + Ok(AsyncTask::new(EngineOp::new( + self.inner.clone(), + move |eng| eng.remove_node_label(node_id, &label), + Ok, + ))) } #[napi(ts_return_type = "Promise")] @@ -1405,8 +1710,8 @@ impl OverGraph { &self, from: f64, to: f64, - type_id: u32, - options: Option, + label: String, + options: Option, ) -> Result>> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; @@ -1415,7 +1720,7 @@ impl OverGraph { None => (None, None, None, None), }; let props = convert_js_props(props); - let opts = UpsertEdgeOptions { + let opts = CoreUpsertEdgeOptions { props, weight: weight.unwrap_or(1.0) as f32, valid_from, @@ -1423,7 +1728,7 @@ impl OverGraph { }; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.upsert_edge(from, to, type_id, opts), + move |eng| eng.upsert_edge(from, to, &label, opts), u64_to_f64, ))) } @@ -1431,27 +1736,30 @@ impl OverGraph { #[napi(ts_return_type = "Promise")] pub fn batch_upsert_nodes_async( &self, - nodes: Vec, - ) -> AsyncTask, Float64Array>> { - let inputs: Vec = nodes.into_iter().map(|n| n.into()).collect(); - AsyncTask::new(EngineOp::new( + nodes: Vec, + ) -> Result, Float64Array>>> { + let inputs: Vec = nodes + .into_iter() + .map(NodeInput::try_into) + .collect::>>()?; + Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.batch_upsert_nodes(&inputs), + move |eng| eng.batch_upsert_nodes(inputs), |ids| ids_to_float64_array(&ids), - )) + ))) } #[napi(ts_return_type = "Promise")] pub fn batch_upsert_edges_async( &self, - edges: Vec, + edges: Vec, ) -> Result, Float64Array>>> { - let inputs: std::result::Result, _> = + let inputs: std::result::Result, _> = edges.into_iter().map(|e| e.try_into()).collect(); let inputs = inputs?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.batch_upsert_edges(&inputs), + move |eng| eng.batch_upsert_edges(inputs), |ids| ids_to_float64_array(&ids), ))) } @@ -1464,7 +1772,7 @@ impl OverGraph { let inputs = decode_node_batch(&buffer)?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.batch_upsert_nodes(&inputs), + move |eng| eng.batch_upsert_nodes(inputs), |ids| ids_to_float64_array(&ids), ))) } @@ -1477,71 +1785,71 @@ impl OverGraph { let inputs = decode_edge_batch(&buffer)?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.batch_upsert_edges(&inputs), + move |eng| eng.batch_upsert_edges(inputs), |ids| ids_to_float64_array(&ids), ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_node_async( &self, id: f64, - ) -> Result, Option>>> { + ) -> Result, Option>>> { let id = f64_to_u64(id)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), move |eng| eng.get_node(id), - |n| n.map(JsNodeRecord::try_from).transpose(), + |n| n.map(NodeView::try_from).transpose(), ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_edge_async( &self, id: f64, - ) -> Result, Option>>> { + ) -> Result, Option>>> { let id = f64_to_u64(id)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), move |eng| eng.get_edge(id), - |e| e.map(JsEdgeRecord::try_from).transpose(), + |e| e.map(EdgeView::try_from).transpose(), ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_node_by_key_async( &self, - type_id: u32, + label: String, key: String, - ) -> AsyncTask, Option>> { + ) -> AsyncTask, Option>> { AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.get_node_by_key(type_id, &key), - |n| n.map(JsNodeRecord::try_from).transpose(), + move |eng| eng.get_node_by_key(&label, &key), + |n| n.map(NodeView::try_from).transpose(), )) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_edge_by_triple_async( &self, from: f64, to: f64, - type_id: u32, - ) -> Result, Option>>> { + label: String, + ) -> Result, Option>>> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.get_edge_by_triple(from, to, type_id), - |e| e.map(JsEdgeRecord::try_from).transpose(), + move |eng| eng.get_edge_by_triple(from, to, &label), + |e| e.map(EdgeView::try_from).transpose(), ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn get_nodes_async( &self, ids: Vec, - ) -> Result>, Vec>>>> { + ) -> Result>, Vec>>>> { let ids: Vec = ids .into_iter() .map(f64_to_u64) @@ -1552,38 +1860,38 @@ impl OverGraph { |results| { results .into_iter() - .map(|r| r.map(JsNodeRecord::try_from).transpose()) + .map(|r| r.map(NodeView::try_from).transpose()) .collect::>>() }, ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn get_nodes_by_keys_async( &self, - keys: Vec, - ) -> Result>, Vec>>>> { - let owned: Vec<(u32, String)> = keys.into_iter().map(|k| (k.type_id, k.key)).collect(); + keys: Vec, + ) -> Result>, Vec>>>> { + let owned: Vec = keys + .into_iter() + .map(KeyQuery::try_into) + .collect::>>()?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| { - let refs: Vec<(u32, &str)> = owned.iter().map(|(t, k)| (*t, k.as_str())).collect(); - eng.get_nodes_by_keys(&refs) - }, + move |eng| eng.get_nodes_by_keys(&owned), |results| { results .into_iter() - .map(|r| r.map(JsNodeRecord::try_from).transpose()) + .map(|r| r.map(NodeView::try_from).transpose()) .collect::>>() }, ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn get_edges_async( &self, ids: Vec, - ) -> Result>, Vec>>>> { + ) -> Result>, Vec>>>> { let ids: Vec = ids .into_iter() .map(f64_to_u64) @@ -1594,7 +1902,7 @@ impl OverGraph { |results| { results .into_iter() - .map(|r| r.map(JsEdgeRecord::try_from).transpose()) + .map(|r| r.map(EdgeView::try_from).transpose()) .collect::>>() }, ))) @@ -1620,31 +1928,31 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn invalidate_edge_async( &self, id: f64, valid_to: i64, - ) -> Result, Option>>> { + ) -> Result, Option>>> { let id = f64_to_u64(id)?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), move |eng| eng.invalidate_edge(id, valid_to), - |e| e.map(JsEdgeRecord::try_from).transpose(), + |e| e.map(EdgeView::try_from).transpose(), ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn graph_patch_async( &self, - patch: JsGraphPatch, - ) -> Result>> { + patch: GraphPatch, + ) -> Result>> { let rust_patch = js_patch_to_rust(patch)?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.graph_patch(&rust_patch), + move |eng| eng.graph_patch(rust_patch), |result| { - Ok(JsPatchResult { + Ok(PatchResult { node_ids: ids_to_float64_array(&result.node_ids)?, edge_ids: ids_to_float64_array(&result.edge_ids)?, }) @@ -1652,21 +1960,17 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn prune_async( &self, - policy: JsPrunePolicy, - ) -> Result>> { - let rust_policy = PrunePolicy { - max_age_ms: policy.max_age_ms.map(|v| v as i64), - max_weight: policy.max_weight.map(|v| v as f32), - type_id: policy.type_id, - }; + policy: PrunePolicy, + ) -> Result>> { + let rust_policy = js_prune_policy_to_rust(policy, "pruneAsync")?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), move |eng| eng.prune(&rust_policy), |result| { - Ok(JsPruneResult { + Ok(PruneResult { nodes_pruned: result.nodes_pruned as i64, edges_pruned: result.edges_pruned as i64, }) @@ -1674,17 +1978,17 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn neighbors_async( &self, node_id: f64, - options: Option, - ) -> Result, Vec>>> { + options: Option, + ) -> Result, Vec>>> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, limit, at_epoch, decay_lambda) = match options { + let (direction, edge_label_filter, limit, at_epoch, decay_lambda) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.limit, o.at_epoch, o.decay_lambda, @@ -1696,7 +2000,7 @@ impl OverGraph { let decay = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, limit: lim, at_epoch, decay_lambda: decay, @@ -1708,19 +2012,19 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn traverse_async( &self, start_node_id: f64, max_depth: u32, - options: Option, - ) -> Result>> { + options: Option, + ) -> Result>> { let start_node_id = f64_to_u64(start_node_id)?; let ( direction, min_depth, - edge_type_filter, - node_type_filter, + edge_label_filter, + node_label_filter, at_epoch, decay_lambda, limit, @@ -1729,8 +2033,10 @@ impl OverGraph { Some(o) => ( o.direction, o.min_depth, - o.edge_type_filter, - o.node_type_filter, + o.edge_label_filter, + o.emit_node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, o.at_epoch, o.decay_lambda, o.limit, @@ -1741,11 +2047,11 @@ impl OverGraph { let dir = parse_direction(direction.as_deref())?; let min_depth = min_depth.unwrap_or(1); let cursor = cursor.map(js_traversal_cursor_to_rust).transpose()?; - let opts = TraverseOptions { + let opts = CoreTraverseOptions { min_depth, direction: dir, - edge_type_filter, - node_type_filter, + edge_label_filter, + emit_node_label_filter: node_label_filter, at_epoch, decay_lambda, limit: limit.map(|v| v as usize), @@ -1758,18 +2064,18 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn top_k_neighbors_async( &self, node_id: f64, k: u32, - options: Option, - ) -> Result, Vec>>> { + options: Option, + ) -> Result, Vec>>> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, scoring, decay_lambda, at_epoch) = match options { + let (direction, edge_label_filter, scoring, decay_lambda, at_epoch) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.scoring, o.decay_lambda, o.at_epoch, @@ -1780,7 +2086,7 @@ impl OverGraph { let scoring_mode = parse_scoring_mode(scoring.as_deref(), decay_lambda)?; let opts = TopKOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, scoring: scoring_mode, at_epoch, }; @@ -1791,22 +2097,30 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn extract_subgraph_async( &self, start_node_id: f64, max_depth: u32, - options: Option, - ) -> Result>> { + options: Option, + ) -> Result>> { let start = f64_to_u64(start_node_id)?; - let (direction, edge_type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.edge_type_filter, o.at_epoch), - None => (None, None, None), + let (direction, edge_label_filter, node_label_filter, at_epoch) = match options { + Some(o) => ( + o.direction, + o.edge_label_filter, + o.node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, + o.at_epoch, + ), + None => (None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; let opts = SubgraphOptions { direction: dir, - edge_type_filter, + edge_label_filter, + node_label_filter, at_epoch, }; Ok(AsyncTask::new(EngineReadOp::new( @@ -1819,26 +2133,26 @@ impl OverGraph { #[napi(ts_return_type = "Promise")] pub fn find_nodes_async( &self, - type_id: u32, + label: String, prop_key: String, prop_value: serde_json::Value, ) -> AsyncTask, Float64Array>> { let pv = json_to_prop_value(&prop_value); AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.find_nodes(type_id, &prop_key, &pv), + move |eng| eng.find_nodes(&label, &prop_key, &pv), |ids| ids_to_float64_array(&ids), )) } #[napi( ts_args_type = "request: import('./query-types').QueryNodeRequest", - ts_return_type = "Promise" + ts_return_type = "Promise" )] pub fn query_node_ids_async( &self, request: serde_json::Value, - ) -> Result>> { + ) -> Result>> { let query = parse_js_node_query(&request)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), @@ -1849,12 +2163,12 @@ impl OverGraph { #[napi( ts_args_type = "request: import('./query-types').QueryNodeRequest", - ts_return_type = "Promise" + ts_return_type = "Promise" )] pub fn query_nodes_async( &self, request: serde_json::Value, - ) -> Result>> { + ) -> Result>> { let query = parse_js_node_query(&request)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), @@ -1863,6 +2177,38 @@ impl OverGraph { ))) } + #[napi( + ts_args_type = "request: import('./query-types').QueryEdgeRequest", + ts_return_type = "Promise" + )] + pub fn query_edge_ids_async( + &self, + request: serde_json::Value, + ) -> Result>> { + let query = parse_js_edge_query(&request)?; + Ok(AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + move |eng| eng.query_edge_ids(&query), + query_edge_ids_to_js, + ))) + } + + #[napi( + ts_args_type = "request: import('./query-types').QueryEdgeRequest", + ts_return_type = "Promise" + )] + pub fn query_edges_async( + &self, + request: serde_json::Value, + ) -> Result>> { + let query = parse_js_edge_query(&request)?; + Ok(AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + move |eng| eng.query_edges(&query), + query_edges_to_js, + ))) + } + #[napi( ts_args_type = "request: import('./query-types').GraphPatternRequest", ts_return_type = "Promise" @@ -1870,7 +2216,7 @@ impl OverGraph { pub fn query_pattern_async( &self, request: serde_json::Value, - ) -> Result>> { + ) -> Result>> { let query = parse_js_graph_pattern_query(&request)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), @@ -1886,7 +2232,7 @@ impl OverGraph { pub fn explain_node_query_async( &self, request: serde_json::Value, - ) -> Result>> { + ) -> Result>> { let query = parse_js_node_query(&request)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), @@ -1896,32 +2242,48 @@ impl OverGraph { } #[napi( - ts_args_type = "request: import('./query-types').GraphPatternRequest", + ts_args_type = "request: import('./query-types').QueryEdgeRequest", ts_return_type = "Promise" )] - pub fn explain_pattern_query_async( + pub fn explain_edge_query_async( &self, request: serde_json::Value, - ) -> Result>> { - let query = parse_js_graph_pattern_query(&request)?; + ) -> Result>> { + let query = parse_js_edge_query(&request)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.explain_pattern_query(&query), + move |eng| eng.explain_edge_query(&query), query_plan_to_js, ))) } - #[napi(ts_return_type = "Promise")] - pub fn ensure_node_property_index_async( - &self, - type_id: u32, - prop_key: String, - kind: JsSecondaryIndexKind, - ) -> Result>> { - let kind = js_secondary_index_kind_to_rust(kind)?; - Ok(AsyncTask::new(EngineOp::new( + #[napi( + ts_args_type = "request: import('./query-types').GraphPatternRequest", + ts_return_type = "Promise" + )] + pub fn explain_pattern_query_async( + &self, + request: serde_json::Value, + ) -> Result>> { + let query = parse_js_graph_pattern_query(&request)?; + Ok(AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + move |eng| eng.explain_pattern_query(&query), + query_plan_to_js, + ))) + } + + #[napi(ts_return_type = "Promise")] + pub fn ensure_node_property_index_async( + &self, + label: String, + prop_key: String, + kind: SecondaryIndexKind, + ) -> Result>> { + let kind = js_secondary_index_kind_to_rust(kind)?; + Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.ensure_node_property_index(type_id, &prop_key, kind.clone()), + move |eng| eng.ensure_node_property_index(&label, &prop_key, kind.clone()), node_property_index_info_to_js, ))) } @@ -1929,22 +2291,22 @@ impl OverGraph { #[napi(ts_return_type = "Promise")] pub fn drop_node_property_index_async( &self, - type_id: u32, + label: String, prop_key: String, - kind: JsSecondaryIndexKind, + kind: SecondaryIndexKind, ) -> Result>> { let kind = js_secondary_index_kind_to_rust(kind)?; Ok(AsyncTask::new(EngineOp::new( self.inner.clone(), - move |eng| eng.drop_node_property_index(type_id, &prop_key, kind.clone()), + move |eng| eng.drop_node_property_index(&label, &prop_key, kind.clone()), Ok, ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn list_node_property_indexes_async( &self, - ) -> AsyncTask, Vec>> { + ) -> AsyncTask, Vec>> { AsyncTask::new(EngineReadOp::new( self.inner.clone(), |eng| eng.list_node_property_indexes(), @@ -1952,13 +2314,54 @@ impl OverGraph { )) } + #[napi(ts_return_type = "Promise")] + pub fn ensure_edge_property_index_async( + &self, + label: String, + prop_key: String, + kind: SecondaryIndexKind, + ) -> Result>> { + let kind = js_secondary_index_kind_to_rust(kind)?; + Ok(AsyncTask::new(EngineOp::new( + self.inner.clone(), + move |eng| eng.ensure_edge_property_index(&label, &prop_key, kind.clone()), + edge_property_index_info_to_js, + ))) + } + + #[napi(ts_return_type = "Promise")] + pub fn drop_edge_property_index_async( + &self, + label: String, + prop_key: String, + kind: SecondaryIndexKind, + ) -> Result>> { + let kind = js_secondary_index_kind_to_rust(kind)?; + Ok(AsyncTask::new(EngineOp::new( + self.inner.clone(), + move |eng| eng.drop_edge_property_index(&label, &prop_key, kind.clone()), + Ok, + ))) + } + + #[napi(ts_return_type = "Promise>")] + pub fn list_edge_property_indexes_async( + &self, + ) -> AsyncTask, Vec>> { + AsyncTask::new(EngineReadOp::new( + self.inner.clone(), + |eng| eng.list_edge_property_indexes(), + edge_property_index_infos_to_js, + )) + } + #[napi(ts_return_type = "Promise")] pub fn find_nodes_range_async( &self, - type_id: u32, + label: String, prop_key: String, - lower: Option, - upper: Option, + lower: Option, + upper: Option, ) -> Result, Float64Array>>> { let lower = lower .as_ref() @@ -1970,20 +2373,20 @@ impl OverGraph { .transpose()?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.find_nodes_range(type_id, &prop_key, lower.as_ref(), upper.as_ref()), + move |eng| eng.find_nodes_range(&label, &prop_key, lower.as_ref(), upper.as_ref()), |ids| ids_to_float64_array(&ids), ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn find_nodes_range_paged_async( &self, - type_id: u32, + label: String, prop_key: String, - lower: Option, - upper: Option, - options: Option, - ) -> Result, JsPropertyRangePageResult>>> + lower: Option, + upper: Option, + options: Option, + ) -> Result, PropertyRangePageResult>>> { let lower = lower .as_ref() @@ -1997,114 +2400,123 @@ impl OverGraph { Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), move |eng| { - eng.find_nodes_range_paged( - type_id, - &prop_key, - lower.as_ref(), - upper.as_ref(), - &page, - ) + eng.find_nodes_range_paged(&label, &prop_key, lower.as_ref(), upper.as_ref(), &page) }, property_range_page_to_js, ))) } - #[napi(ts_return_type = "Promise>")] - pub fn get_nodes_by_type_async( + #[napi( + ts_args_type = "labels: string | string[]", + ts_return_type = "Promise>" + )] + pub fn get_nodes_by_labels_async( &self, - type_id: u32, - ) -> AsyncTask, Vec>> { - AsyncTask::new(EngineReadOp::new( + labels: serde_json::Value, + ) -> Result, Vec>>> { + let labels = parse_js_node_labels_arg(&labels, "getNodesByLabelsAsync labels")?; + Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.get_nodes_by_type(type_id), + move |eng| eng.get_nodes_by_labels(labels), |records| { records .into_iter() - .map(JsNodeRecord::try_from) + .map(NodeView::try_from) .collect::>>() }, - )) + ))) } - #[napi(ts_return_type = "Promise>")] - pub fn get_edges_by_type_async( + #[napi(ts_return_type = "Promise>")] + pub fn get_edges_by_label_async( &self, - type_id: u32, - ) -> AsyncTask, Vec>> { + label: String, + ) -> AsyncTask, Vec>> { AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.get_edges_by_type(type_id), + move |eng| eng.get_edges_by_label(&label), |records| { records .into_iter() - .map(JsEdgeRecord::try_from) + .map(EdgeView::try_from) .collect::>>() }, )) } - #[napi(ts_return_type = "Promise")] - pub fn count_nodes_by_type_async(&self, type_id: u32) -> AsyncTask> { - AsyncTask::new(EngineReadOp::new( + #[napi( + ts_args_type = "labels: string | string[]", + ts_return_type = "Promise" + )] + pub fn count_nodes_by_labels_async( + &self, + labels: serde_json::Value, + ) -> Result>> { + let labels = parse_js_node_labels_arg(&labels, "countNodesByLabelsAsync labels")?; + Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.count_nodes_by_type(type_id), + move |eng| eng.count_nodes_by_labels(labels), |count| Ok(count as i64), - )) + ))) } #[napi(ts_return_type = "Promise")] - pub fn count_edges_by_type_async(&self, type_id: u32) -> AsyncTask> { + pub fn count_edges_by_label_async(&self, label: String) -> AsyncTask> { AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.count_edges_by_type(type_id), + move |eng| eng.count_edges_by_label(&label), |count| Ok(count as i64), )) } - #[napi(ts_return_type = "Promise")] - pub fn nodes_by_type_async( + #[napi( + ts_args_type = "labels: string | string[]", + ts_return_type = "Promise" + )] + pub fn nodes_by_labels_async( &self, - type_id: u32, - ) -> AsyncTask, Float64Array>> { - AsyncTask::new(EngineReadOp::new( + labels: serde_json::Value, + ) -> Result, Float64Array>>> { + let labels = parse_js_node_labels_arg(&labels, "nodesByLabelsAsync labels")?; + Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.nodes_by_type(type_id), + move |eng| eng.nodes_by_labels(labels), |ids| ids_to_float64_array(&ids), - )) + ))) } #[napi(ts_return_type = "Promise")] - pub fn edges_by_type_async( + pub fn edges_by_label_async( &self, - type_id: u32, + label: String, ) -> AsyncTask, Float64Array>> { AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.edges_by_type(type_id), + move |eng| eng.edges_by_label(&label), |ids| ids_to_float64_array(&ids), )) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn neighbors_batch_async( &self, node_ids: Vec, - options: Option, - ) -> Result>, Vec>>> + options: Option, + ) -> Result>, Vec>>> { let ids: Vec = node_ids .into_iter() .map(f64_to_u64) .collect::>>()?; - let (direction, type_filter, at_epoch, decay_lambda) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch, o.decay_lambda), + let (direction, edge_label_filter, at_epoch, decay_lambda) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch, o.decay_lambda), None => (None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; let decay = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, limit: None, at_epoch, decay_lambda: decay, @@ -2122,17 +2534,17 @@ impl OverGraph { pub fn degree_async( &self, node_id: f64, - options: Option, + options: Option, ) -> Result>> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, at_epoch, }; Ok(AsyncTask::new(EngineReadOp::new( @@ -2146,17 +2558,17 @@ impl OverGraph { pub fn sum_edge_weights_async( &self, node_id: f64, - options: Option, + options: Option, ) -> Result>> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, at_epoch, }; Ok(AsyncTask::new(EngineReadOp::new( @@ -2170,17 +2582,17 @@ impl OverGraph { pub fn avg_edge_weight_async( &self, node_id: f64, - options: Option, + options: Option, ) -> Result, Option>>> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, at_epoch, }; Ok(AsyncTask::new(EngineReadOp::new( @@ -2190,34 +2602,34 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn degrees_async( &self, node_ids: Vec, - options: Option, - ) -> Result, Vec>>> { + options: Option, + ) -> Result, Vec>>> { let ids: Vec = node_ids .into_iter() .map(f64_to_u64) .collect::>>()?; - let (direction, type_filter, at_epoch) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch), + let (direction, edge_label_filter, at_epoch) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch), None => (None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = DegreeOptions { + let opts = CoreDegreeOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, at_epoch, }; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), move |eng| eng.degrees(&ids, &opts), |map| { - let mut entries: Vec = map + let mut entries: Vec = map .into_iter() .map(|(node_id, degree)| { - Ok(JsDegreeBatchEntry { + Ok(DegreeBatchEntry { node_id: u64_to_f64(node_id)?, degree: u64_to_safe_i64(degree)?, }) @@ -2231,30 +2643,31 @@ impl OverGraph { // --- Shortest path (async, Phase 18b) --- - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn shortest_path_async( &self, from: f64, to: f64, - options: Option, - ) -> Result, Option>>> { + options: Option, + ) -> Result, Option>>> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; - let (direction, type_filter, weight_field, at_epoch, max_depth, max_cost) = match options { - Some(o) => ( - o.direction, - o.type_filter, - o.weight_field, - o.at_epoch, - o.max_depth, - o.max_cost, - ), - None => (None, None, None, None, None, None), - }; + let (direction, edge_label_filter, weight_field, at_epoch, max_depth, max_cost) = + match options { + Some(o) => ( + o.direction, + o.edge_label_filter, + o.weight_field, + o.at_epoch, + o.max_depth, + o.max_cost, + ), + None => (None, None, None, None, None, None), + }; let dir = parse_direction(direction.as_deref())?; - let opts = ShortestPathOptions { + let opts = CoreShortestPathOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, weight_field, at_epoch, max_depth, @@ -2272,18 +2685,18 @@ impl OverGraph { &self, from: f64, to: f64, - options: Option, + options: Option, ) -> Result>> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; - let (direction, type_filter, at_epoch, max_depth) = match options { - Some(o) => (o.direction, o.type_filter, o.at_epoch, o.max_depth), + let (direction, edge_label_filter, at_epoch, max_depth) = match options { + Some(o) => (o.direction, o.edge_label_filter, o.at_epoch, o.max_depth), None => (None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = IsConnectedOptions { + let opts = CoreIsConnectedOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, at_epoch, max_depth, }; @@ -2294,20 +2707,20 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn all_shortest_paths_async( &self, from: f64, to: f64, - options: Option, - ) -> Result, Vec>>> { + options: Option, + ) -> Result, Vec>>> { let from = f64_to_u64(from)?; let to = f64_to_u64(to)?; - let (direction, type_filter, weight_field, at_epoch, max_depth, max_cost, max_paths) = + let (direction, edge_label_filter, weight_field, at_epoch, max_depth, max_cost, max_paths) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.weight_field, o.at_epoch, o.max_depth, @@ -2317,9 +2730,9 @@ impl OverGraph { None => (None, None, None, None, None, None, None), }; let dir = parse_direction(direction.as_deref())?; - let opts = AllShortestPathsOptions { + let opts = CoreAllShortestPathsOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, weight_field, at_epoch, max_depth, @@ -2335,74 +2748,82 @@ impl OverGraph { // --- Paginated queries (async) --- - #[napi(ts_return_type = "Promise")] - pub fn nodes_by_type_paged_async( + #[napi( + ts_args_type = "labels: string | string[], limit?: number | null, after?: number | null", + ts_return_type = "Promise" + )] + pub fn nodes_by_labels_paged_async( &self, - type_id: u32, + labels: serde_json::Value, limit: Option, after: Option, - ) -> Result, JsIdPageResult>>> { + ) -> Result, IdPageResult>>> { + let labels = parse_js_node_labels_arg(&labels, "nodesByLabelsPagedAsync labels")?; let page = make_page_request(limit, after)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.nodes_by_type_paged(type_id, &page), + move |eng| eng.nodes_by_labels_paged(labels, &page), id_page_to_js, ))) } - #[napi(ts_return_type = "Promise")] - pub fn edges_by_type_paged_async( + #[napi(ts_return_type = "Promise")] + pub fn edges_by_label_paged_async( &self, - type_id: u32, + label: String, limit: Option, after: Option, - ) -> Result, JsIdPageResult>>> { + ) -> Result, IdPageResult>>> { let page = make_page_request(limit, after)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.edges_by_type_paged(type_id, &page), + move |eng| eng.edges_by_label_paged(&label, &page), id_page_to_js, ))) } - #[napi(ts_return_type = "Promise")] - pub fn get_nodes_by_type_paged_async( + #[napi( + ts_args_type = "labels: string | string[], limit?: number | null, after?: number | null", + ts_return_type = "Promise" + )] + pub fn get_nodes_by_labels_paged_async( &self, - type_id: u32, + labels: serde_json::Value, limit: Option, after: Option, - ) -> Result, JsNodePageResult>>> { + ) -> Result, NodePageResult>>> { + let labels = parse_js_node_labels_arg(&labels, "getNodesByLabelsPagedAsync labels")?; let page = make_page_request(limit, after)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.get_nodes_by_type_paged(type_id, &page), + move |eng| eng.get_nodes_by_labels_paged(labels, &page), node_page_to_js, ))) } - #[napi(ts_return_type = "Promise")] - pub fn get_edges_by_type_paged_async( + #[napi(ts_return_type = "Promise")] + pub fn get_edges_by_label_paged_async( &self, - type_id: u32, + label: String, limit: Option, after: Option, - ) -> Result, JsEdgePageResult>>> { + ) -> Result, EdgePageResult>>> { let page = make_page_request(limit, after)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.get_edges_by_type_paged(type_id, &page), + move |eng| eng.get_edges_by_label_paged(&label, &page), edge_page_to_js, ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn find_nodes_paged_async( &self, - type_id: u32, + label: String, prop_key: String, prop_value: serde_json::Value, - options: Option, - ) -> Result, JsIdPageResult>>> { + options: Option, + ) -> Result, IdPageResult>>> { let pv = json_to_prop_value(&prop_value); let (limit, after) = match options { Some(o) => (o.limit, o.after), @@ -2411,7 +2832,7 @@ impl OverGraph { let page = make_page_request(limit, after)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.find_nodes_paged(type_id, &prop_key, &pv, &page), + move |eng| eng.find_nodes_paged(&label, &prop_key, &pv, &page), id_page_to_js, ))) } @@ -2419,25 +2840,25 @@ impl OverGraph { #[napi(ts_return_type = "Promise")] pub fn find_nodes_by_time_range_async( &self, - type_id: u32, + label: String, from_ms: i64, to_ms: i64, ) -> Result, Float64Array>>> { Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.find_nodes_by_time_range(type_id, from_ms, to_ms), + move |eng| eng.find_nodes_by_time_range(&label, from_ms, to_ms), |ids| ids_to_float64_array(&ids), ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn find_nodes_by_time_range_paged_async( &self, - type_id: u32, + label: String, from_ms: i64, to_ms: i64, - options: Option, - ) -> Result, JsIdPageResult>>> { + options: Option, + ) -> Result, IdPageResult>>> { let (limit, after) = match options { Some(o) => (o.limit, o.after), None => (None, None), @@ -2445,17 +2866,17 @@ impl OverGraph { let page = make_page_request(limit, after)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), - move |eng| eng.find_nodes_by_time_range_paged(type_id, from_ms, to_ms, &page), + move |eng| eng.find_nodes_by_time_range_paged(&label, from_ms, to_ms, &page), id_page_to_js, ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn personalized_pagerank_async( &self, seed_node_ids: Vec, - options: Option, - ) -> Result>> { + options: Option, + ) -> Result>> { let seeds: Vec = seed_node_ids .into_iter() .map(f64_to_u64) @@ -2466,18 +2887,21 @@ impl OverGraph { max_iterations, epsilon, approx_residual_tolerance, - edge_type_filter, + edge_label_filter, max_results, ) = match &options { - Some(o) => ( - o.algorithm.as_deref(), - o.damping_factor, - o.max_iterations, - o.epsilon, - o.approx_residual_tolerance, - o.edge_type_filter.clone(), - o.max_results, - ), + Some(o) => { + let edge_label_filter = o.edge_label_filter.clone(); + ( + o.algorithm.as_deref(), + o.damping_factor, + o.max_iterations, + o.epsilon, + o.approx_residual_tolerance, + edge_label_filter, + o.max_results, + ) + } None => (None, None, None, None, None, None, None), }; let opts = js_ppr_options_to_ppr_options( @@ -2486,7 +2910,7 @@ impl OverGraph { &max_iterations, &epsilon, &approx_residual_tolerance, - &edge_type_filter, + &edge_label_filter, &max_results, )?; Ok(AsyncTask::new(EngineReadOp::new( @@ -2496,16 +2920,16 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn export_adjacency_async( &self, - options: Option, - ) -> Result>> { + options: Option, + ) -> Result>> { let include_weights = options .as_ref() .and_then(|o| o.include_weights) .unwrap_or(true); - let opts = js_export_options_to_rust(options); + let opts = js_export_options_to_rust(options)?; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), move |eng| Ok((eng.export_adjacency(&opts)?, include_weights)), @@ -2513,17 +2937,17 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn neighbors_paged_async( &self, node_id: f64, - options: Option, - ) -> Result, JsNeighborPageResult>>> { + options: Option, + ) -> Result, NeighborPageResult>>> { let node_id = f64_to_u64(node_id)?; - let (direction, type_filter, limit, after, at_epoch, decay_lambda) = match options { + let (direction, edge_label_filter, limit, after, at_epoch, decay_lambda) = match options { Some(o) => ( o.direction, - o.type_filter, + o.edge_label_filter, o.limit, o.after, o.at_epoch, @@ -2536,7 +2960,7 @@ impl OverGraph { let decay = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter: edge_label_filter, limit: None, at_epoch, decay_lambda: decay, @@ -2550,28 +2974,34 @@ impl OverGraph { // --- Connected Components (async, Phase 18d) --- - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn connected_components_async( &self, - options: Option, - ) -> Result, Vec>>> { - let (edge_type_filter, node_type_filter, at_epoch) = match options { - Some(o) => (o.edge_type_filter, o.node_type_filter, o.at_epoch), + options: Option, + ) -> Result, Vec>>> { + let (edge_label_filter, node_label_filter, at_epoch) = match options { + Some(o) => ( + o.edge_label_filter, + o.node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, + o.at_epoch, + ), None => (None, None, None), }; let opts = ComponentOptions { - edge_type_filter, - node_type_filter, + edge_label_filter, + node_label_filter: node_label_filter, at_epoch, }; Ok(AsyncTask::new(EngineReadOp::new( self.inner.clone(), move |eng| eng.connected_components(&opts), |map| { - let mut entries: Vec = map + let mut entries: Vec = map .into_iter() .map(|(node_id, component_id)| { - Ok(JsComponentEntry { + Ok(ComponentEntry { node_id: u64_to_f64(node_id)?, component_id: u64_to_f64(component_id)?, }) @@ -2587,16 +3017,22 @@ impl OverGraph { pub fn component_of_async( &self, node_id: f64, - options: Option, + options: Option, ) -> Result, Float64Array>>> { let node_id = f64_to_u64(node_id)?; - let (edge_type_filter, node_type_filter, at_epoch) = match options { - Some(o) => (o.edge_type_filter, o.node_type_filter, o.at_epoch), + let (edge_label_filter, node_label_filter, at_epoch) = match options { + Some(o) => ( + o.edge_label_filter, + o.node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, + o.at_epoch, + ), None => (None, None, None), }; let opts = ComponentOptions { - edge_type_filter, - node_type_filter, + edge_label_filter, + node_label_filter: node_label_filter, at_epoch, }; Ok(AsyncTask::new(EngineReadOp::new( @@ -2606,17 +3042,20 @@ impl OverGraph { ))) } - #[napi(ts_return_type = "Promise>")] + #[napi(ts_return_type = "Promise>")] pub fn vector_search_async( &self, mode: String, - options: JsVectorSearchOptions, - ) -> Result, Vec>>> { + options: VectorSearchOptions, + ) -> Result, Vec>>> { let mode = parse_vector_search_mode(&mode)?; let k = options.k; let dense_query = options.dense_query; let sparse_query = options.sparse_query; - let type_filter = options.type_filter; + let label_filter = options + .label_filter + .map(js_node_label_filter_to_rust) + .transpose()?; let ef_search = options.ef_search; let scope = options.scope; let dense_weight = options.dense_weight; @@ -2631,11 +3070,11 @@ impl OverGraph { }); let scope = match scope { None => None, - Some(s) => Some(VectorSearchScope { + Some(s) => Some(CoreVectorSearchScope { start_node_id: f64_to_u64(s.start_node_id)?, max_depth: s.max_depth, direction: parse_direction(s.direction.as_deref())?, - edge_type_filter: s.edge_type_filter, + edge_label_filter: s.edge_label_filter, at_epoch: s.at_epoch, }), }; @@ -2644,7 +3083,7 @@ impl OverGraph { dense_query: dense_q, sparse_query: sparse_q, k: k as usize, - type_filter, + label_filter, ef_search: ef_search.map(|v| v as usize), scope, dense_weight: dense_weight.map(|v| v as f32), @@ -2657,7 +3096,7 @@ impl OverGraph { |hits| { hits.into_iter() .map(|h| { - Ok(JsVectorHit { + Ok(VectorHit { node_id: u64_to_f64(h.node_id)?, score: h.score as f64, }) @@ -2700,10 +3139,10 @@ impl OverGraph { )) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn end_ingest_async( &self, - ) -> AsyncTask, Option>> { + ) -> AsyncTask, Option>> { AsyncTask::new(EngineOp::new( self.inner.clone(), |eng| eng.end_ingest(), @@ -2711,10 +3150,10 @@ impl OverGraph { )) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn compact_async( &self, - ) -> AsyncTask, Option>> { + ) -> AsyncTask, Option>> { AsyncTask::new(EngineOp::new( self.inner.clone(), |eng| eng.compact(), @@ -2727,8 +3166,8 @@ impl OverGraph { /// Note: the database write lock is held for the entire compaction, so other operations on this /// instance will block until compaction completes. The JS event loop remains responsive. #[napi( - ts_args_type = "callback: (progress: JsCompactionProgress) => void", - ts_return_type = "Promise" + ts_args_type = "callback: (progress: CompactionProgress) => void", + ts_return_type = "Promise" )] pub fn compact_with_progress_async( &self, @@ -2741,9 +3180,9 @@ impl OverGraph { } } -#[napi(js_name = "WriteTxn")] -pub struct JsWriteTxn { - inner: Arc>>, +#[napi] +pub struct WriteTxn { + inner: Arc>>, async_order: Arc, } @@ -2814,65 +3253,86 @@ impl Drop for TxnAsyncTurn { } } -fn write_txn_to_js(txn: WriteTxn) -> JsWriteTxn { - JsWriteTxn { +fn write_txn_to_js(txn: CoreWriteTxn) -> WriteTxn { + WriteTxn { inner: Arc::new(Mutex::new(Some(txn))), async_order: Arc::new(TxnAsyncOrder::new()), } } #[napi] -impl JsWriteTxn { - #[napi] +impl WriteTxn { + #[napi( + ts_args_type = "labels: string | string[], key: string, options?: UpsertNodeOptions | null" + )] pub fn upsert_node( &self, - type_id: u32, + labels: serde_json::Value, key: String, - options: Option, - ) -> Result { + options: Option, + ) -> Result { + let labels = parse_js_node_labels_arg(&labels, "transaction upsertNode labels")?; + let ref_label = labels.first().cloned().ok_or_else(|| { + napi::Error::from_reason("transaction upsertNode requires labels".to_string()) + })?; with_txn(&self.inner, |txn| { - txn.upsert_node(type_id, &key, js_upsert_node_options(options)) + txn.upsert_node(labels, &key, js_upsert_node_options(options)) })?; - Ok(JsTxnNodeRef { + Ok(TxnNodeRef { id: None, - type_id: Some(type_id), + labels: Some(txn_node_ref_labels_value(ref_label)), key: Some(key), local: None, }) } - #[napi] + #[napi( + ts_args_type = "alias: string, labels: string | string[], key: string, options?: UpsertNodeOptions | null" + )] pub fn upsert_node_as( &self, alias: String, - type_id: u32, + labels: serde_json::Value, key: String, - options: Option, - ) -> Result { + options: Option, + ) -> Result { + let labels = parse_js_node_labels_arg(&labels, "transaction upsertNodeAs labels")?; let node_ref = with_txn(&self.inner, |txn| { - txn.upsert_node_as(&alias, type_id, &key, js_upsert_node_options(options)) + txn.upsert_node_as(&alias, labels, &key, js_upsert_node_options(options)) })?; txn_node_ref_to_js(node_ref) } + #[napi] + pub fn add_node_label(&self, target: TxnNodeRef, label: String) -> Result { + let target = js_txn_node_ref_to_rust(target)?; + with_txn(&self.inner, |txn| txn.add_node_label(target, &label)) + } + + #[napi] + pub fn remove_node_label(&self, target: TxnNodeRef, label: String) -> Result { + let target = js_txn_node_ref_to_rust(target)?; + with_txn(&self.inner, |txn| txn.remove_node_label(target, &label)) + } + #[napi] pub fn upsert_edge( &self, - from: JsTxnNodeRef, - to: JsTxnNodeRef, - type_id: u32, - options: Option, - ) -> Result { + from: TxnNodeRef, + to: TxnNodeRef, + label: String, + options: Option, + ) -> Result { let from_rust = js_txn_node_ref_to_rust(from.clone())?; let to_rust = js_txn_node_ref_to_rust(to.clone())?; with_txn(&self.inner, |txn| { - txn.upsert_edge(from_rust, to_rust, type_id, js_upsert_edge_options(options)) + txn.upsert_edge(from_rust, to_rust, &label, js_upsert_edge_options(options)) })?; - Ok(JsTxnEdgeRef { + Ok(TxnEdgeRef { id: None, from: Some(from), to: Some(to), - type_id: Some(type_id), + label: Some(label), local: None, }) } @@ -2881,39 +3341,41 @@ impl JsWriteTxn { pub fn upsert_edge_as( &self, alias: String, - from: JsTxnNodeRef, - to: JsTxnNodeRef, - type_id: u32, - options: Option, - ) -> Result { + from: TxnNodeRef, + to: TxnNodeRef, + label: String, + options: Option, + ) -> Result { let from = js_txn_node_ref_to_rust(from)?; let to = js_txn_node_ref_to_rust(to)?; let edge_ref = with_txn(&self.inner, |txn| { - txn.upsert_edge_as(&alias, from, to, type_id, js_upsert_edge_options(options)) + txn.upsert_edge_as(&alias, from, to, &label, js_upsert_edge_options(options)) })?; txn_edge_ref_to_js(edge_ref) } #[napi] - pub fn delete_node(&self, target: JsTxnNodeRef) -> Result<()> { + pub fn delete_node(&self, target: TxnNodeRef) -> Result<()> { let target = js_txn_node_ref_to_rust(target)?; with_txn(&self.inner, |txn| txn.delete_node(target)) } #[napi] - pub fn delete_edge(&self, target: JsTxnEdgeRef) -> Result<()> { + pub fn delete_edge(&self, target: TxnEdgeRef) -> Result<()> { let target = js_txn_edge_ref_to_rust(target)?; with_txn(&self.inner, |txn| txn.delete_edge(target)) } #[napi] - pub fn invalidate_edge(&self, target: JsTxnEdgeRef, valid_to: i64) -> Result<()> { + pub fn invalidate_edge(&self, target: TxnEdgeRef, valid_to: i64) -> Result<()> { let target = js_txn_edge_ref_to_rust(target)?; with_txn(&self.inner, |txn| txn.invalidate_edge(target, valid_to)) } - #[napi] - pub fn stage(&self, operations: Vec) -> Result<()> { + #[napi( + ts_args_type = "operations: Array<{ op: 'upsertNode'; alias?: string; labels: string | string[]; key: string; props?: Record; weight?: number; denseVector?: Array; sparseVector?: Array } | { op: 'upsertEdge'; alias?: string; from: TxnNodeRef; to: TxnNodeRef; label: string; props?: Record; weight?: number; validFrom?: number; validTo?: number } | { op: 'deleteNode'; target: TxnEdgeOrNodeRef } | { op: 'deleteEdge'; target: TxnEdgeOrNodeRef } | { op: 'invalidateEdge'; target: TxnEdgeOrNodeRef; validTo: number }>" + )] + pub fn stage(&self, operations: Vec) -> Result<()> { let intents = operations .into_iter() .map(js_txn_operation_to_rust) @@ -2922,40 +3384,40 @@ impl JsWriteTxn { } #[napi] - pub fn get_node(&self, target: JsTxnNodeRef) -> Result> { + pub fn get_node(&self, target: TxnNodeRef) -> Result> { let target = js_txn_node_ref_to_rust(target)?; let view = with_txn_ref(&self.inner, |txn| txn.get_node(target))?; view.map(txn_node_view_to_js).transpose() } #[napi] - pub fn get_edge(&self, target: JsTxnEdgeRef) -> Result> { + pub fn get_edge(&self, target: TxnEdgeRef) -> Result> { let target = js_txn_edge_ref_to_rust(target)?; let view = with_txn_ref(&self.inner, |txn| txn.get_edge(target))?; view.map(txn_edge_view_to_js).transpose() } #[napi] - pub fn get_node_by_key(&self, type_id: u32, key: String) -> Result> { - let view = with_txn_ref(&self.inner, |txn| txn.get_node_by_key(type_id, &key))?; + pub fn get_node_by_key(&self, label: String, key: String) -> Result> { + let view = with_txn_ref(&self.inner, |txn| txn.get_node_by_key(&label, &key))?; view.map(txn_node_view_to_js).transpose() } #[napi] pub fn get_edge_by_triple( &self, - from: JsTxnNodeRef, - to: JsTxnNodeRef, - type_id: u32, - ) -> Result> { + from: TxnNodeRef, + to: TxnNodeRef, + label: String, + ) -> Result> { let from = js_txn_node_ref_to_rust(from)?; let to = js_txn_node_ref_to_rust(to)?; - let view = with_txn_ref(&self.inner, |txn| txn.get_edge_by_triple(from, to, type_id))?; + let view = with_txn_ref(&self.inner, |txn| txn.get_edge_by_triple(from, to, &label))?; view.map(txn_edge_view_to_js).transpose() } #[napi] - pub fn commit(&self) -> Result { + pub fn commit(&self) -> Result { let result = with_txn_take(&self.inner, |txn| txn.commit())?; txn_commit_result_to_js(result) } @@ -2965,21 +3427,28 @@ impl JsWriteTxn { with_txn_take(&self.inner, |txn| txn.rollback()) } - #[napi(ts_return_type = "Promise")] + #[napi( + ts_args_type = "labels: string | string[], key: string, options?: UpsertNodeOptions | null", + ts_return_type = "Promise" + )] pub fn upsert_node_async( &self, - type_id: u32, + labels: serde_json::Value, key: String, - options: Option, - ) -> Result>> { + options: Option, + ) -> Result>> { + let labels = parse_js_node_labels_arg(&labels, "transaction upsertNodeAsync labels")?; + let ref_label = labels.first().cloned().ok_or_else(|| { + napi::Error::from_reason("transaction upsertNodeAsync requires labels".to_string()) + })?; let opts = js_upsert_node_options(options); Ok(AsyncTask::new(TxnAsyncOp::new( self, move |txn| { - txn.upsert_node(type_id, &key, opts)?; - Ok(JsTxnNodeRef { + txn.upsert_node(labels, &key, opts)?; + Ok(TxnNodeRef { id: None, - type_id: Some(type_id), + labels: Some(txn_node_ref_labels_value(ref_label)), key: Some(key), local: None, }) @@ -2988,42 +3457,74 @@ impl JsWriteTxn { )?)) } - #[napi(ts_return_type = "Promise")] + #[napi( + ts_args_type = "alias: string, labels: string | string[], key: string, options?: UpsertNodeOptions | null", + ts_return_type = "Promise" + )] pub fn upsert_node_as_async( &self, alias: String, - type_id: u32, + labels: serde_json::Value, key: String, - options: Option, - ) -> Result>> { + options: Option, + ) -> Result>> { + let labels = parse_js_node_labels_arg(&labels, "transaction upsertNodeAsAsync labels")?; let opts = js_upsert_node_options(options); Ok(AsyncTask::new(TxnAsyncOp::new( self, - move |txn| txn.upsert_node_as(&alias, type_id, &key, opts), + move |txn| txn.upsert_node_as(&alias, labels, &key, opts), txn_node_ref_to_js, )?)) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] + pub fn add_node_label_async( + &self, + target: TxnNodeRef, + label: String, + ) -> Result>> { + let target = js_txn_node_ref_to_rust(target)?; + Ok(AsyncTask::new(TxnAsyncOp::new( + self, + move |txn| txn.add_node_label(target, &label), + Ok, + )?)) + } + + #[napi(ts_return_type = "Promise")] + pub fn remove_node_label_async( + &self, + target: TxnNodeRef, + label: String, + ) -> Result>> { + let target = js_txn_node_ref_to_rust(target)?; + Ok(AsyncTask::new(TxnAsyncOp::new( + self, + move |txn| txn.remove_node_label(target, &label), + Ok, + )?)) + } + + #[napi(ts_return_type = "Promise")] pub fn upsert_edge_async( &self, - from: JsTxnNodeRef, - to: JsTxnNodeRef, - type_id: u32, - options: Option, - ) -> Result>> { + from: TxnNodeRef, + to: TxnNodeRef, + label: String, + options: Option, + ) -> Result>> { let from_rust = js_txn_node_ref_to_rust(from.clone())?; let to_rust = js_txn_node_ref_to_rust(to.clone())?; let opts = js_upsert_edge_options(options); Ok(AsyncTask::new(TxnAsyncOp::new( self, move |txn| { - txn.upsert_edge(from_rust, to_rust, type_id, opts)?; - Ok(JsTxnEdgeRef { + txn.upsert_edge(from_rust, to_rust, &label, opts)?; + Ok(TxnEdgeRef { id: None, from: Some(from), to: Some(to), - type_id: Some(type_id), + label: Some(label), local: None, }) }, @@ -3031,27 +3532,27 @@ impl JsWriteTxn { )?)) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn upsert_edge_as_async( &self, alias: String, - from: JsTxnNodeRef, - to: JsTxnNodeRef, - type_id: u32, - options: Option, - ) -> Result>> { + from: TxnNodeRef, + to: TxnNodeRef, + label: String, + options: Option, + ) -> Result>> { let from = js_txn_node_ref_to_rust(from)?; let to = js_txn_node_ref_to_rust(to)?; let opts = js_upsert_edge_options(options); Ok(AsyncTask::new(TxnAsyncOp::new( self, - move |txn| txn.upsert_edge_as(&alias, from, to, type_id, opts), + move |txn| txn.upsert_edge_as(&alias, from, to, &label, opts), txn_edge_ref_to_js, )?)) } #[napi(ts_return_type = "Promise")] - pub fn delete_node_async(&self, target: JsTxnNodeRef) -> Result>> { + pub fn delete_node_async(&self, target: TxnNodeRef) -> Result>> { let target = js_txn_node_ref_to_rust(target)?; Ok(AsyncTask::new(TxnAsyncOp::new( self, @@ -3061,7 +3562,7 @@ impl JsWriteTxn { } #[napi(ts_return_type = "Promise")] - pub fn delete_edge_async(&self, target: JsTxnEdgeRef) -> Result>> { + pub fn delete_edge_async(&self, target: TxnEdgeRef) -> Result>> { let target = js_txn_edge_ref_to_rust(target)?; Ok(AsyncTask::new(TxnAsyncOp::new( self, @@ -3073,7 +3574,7 @@ impl JsWriteTxn { #[napi(ts_return_type = "Promise")] pub fn invalidate_edge_async( &self, - target: JsTxnEdgeRef, + target: TxnEdgeRef, valid_to: i64, ) -> Result>> { let target = js_txn_edge_ref_to_rust(target)?; @@ -3084,10 +3585,13 @@ impl JsWriteTxn { )?)) } - #[napi(ts_return_type = "Promise")] + #[napi( + ts_args_type = "operations: Array<{ op: 'upsertNode'; alias?: string; labels: string | string[]; key: string; props?: Record; weight?: number; denseVector?: Array; sparseVector?: Array } | { op: 'upsertEdge'; alias?: string; from: TxnNodeRef; to: TxnNodeRef; label: string; props?: Record; weight?: number; validFrom?: number; validTo?: number } | { op: 'deleteNode'; target: TxnEdgeOrNodeRef } | { op: 'deleteEdge'; target: TxnEdgeOrNodeRef } | { op: 'invalidateEdge'; target: TxnEdgeOrNodeRef; validTo: number }>", + ts_return_type = "Promise" + )] pub fn stage_async( &self, - operations: Vec, + operations: Vec, ) -> Result>> { let intents = operations .into_iter() @@ -3100,11 +3604,11 @@ impl JsWriteTxn { )?)) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_node_async( &self, - target: JsTxnNodeRef, - ) -> Result, Option>>> { + target: TxnNodeRef, + ) -> Result, Option>>> { let target = js_txn_node_ref_to_rust(target)?; Ok(AsyncTask::new(TxnAsyncOp::new( self, @@ -3113,11 +3617,11 @@ impl JsWriteTxn { )?)) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_edge_async( &self, - target: JsTxnEdgeRef, - ) -> Result, Option>>> { + target: TxnEdgeRef, + ) -> Result, Option>>> { let target = js_txn_edge_ref_to_rust(target)?; Ok(AsyncTask::new(TxnAsyncOp::new( self, @@ -3126,39 +3630,39 @@ impl JsWriteTxn { )?)) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_node_by_key_async( &self, - type_id: u32, + label: String, key: String, - ) -> Result, Option>>> { + ) -> Result, Option>>> { Ok(AsyncTask::new(TxnAsyncOp::new( self, - move |txn| txn.get_node_by_key(type_id, &key), + move |txn| txn.get_node_by_key(&label, &key), |view| view.map(txn_node_view_to_js).transpose(), )?)) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn get_edge_by_triple_async( &self, - from: JsTxnNodeRef, - to: JsTxnNodeRef, - type_id: u32, - ) -> Result, Option>>> { + from: TxnNodeRef, + to: TxnNodeRef, + label: String, + ) -> Result, Option>>> { let from = js_txn_node_ref_to_rust(from)?; let to = js_txn_node_ref_to_rust(to)?; Ok(AsyncTask::new(TxnAsyncOp::new( self, - move |txn| txn.get_edge_by_triple(from, to, type_id), + move |txn| txn.get_edge_by_triple(from, to, &label), |view| view.map(txn_edge_view_to_js).transpose(), )?)) } - #[napi(ts_return_type = "Promise")] + #[napi(ts_return_type = "Promise")] pub fn commit_async( &self, - ) -> Result>> { + ) -> Result>> { Ok(AsyncTask::new(TxnAsyncTakeOp::new( self, |txn| txn.commit(), @@ -3181,7 +3685,7 @@ impl JsWriteTxn { // ============================================================ #[napi(object)] -pub struct JsCloseOptions { +pub struct CloseOptions { /// If true, cancel any in-progress background compaction instead of waiting. pub force: Option, } @@ -3191,15 +3695,15 @@ pub struct JsCloseOptions { // ============================================================ #[napi(object)] -pub struct JsUpsertNodeOptions { +pub struct UpsertNodeOptions { pub props: Option>, pub weight: Option, pub dense_vector: Option>, - pub sparse_vector: Option>, + pub sparse_vector: Option>, } #[napi(object)] -pub struct JsUpsertEdgeOptions { +pub struct UpsertEdgeOptions { pub props: Option>, pub weight: Option, pub valid_from: Option, @@ -3207,18 +3711,18 @@ pub struct JsUpsertEdgeOptions { } #[napi(object)] -pub struct JsNeighborsOptions { +pub struct NeighborsOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub limit: Option, pub at_epoch: Option, pub decay_lambda: Option, } #[napi(object)] -pub struct JsNeighborsPagedOptions { +pub struct NeighborsPagedOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub limit: Option, pub after: Option, pub at_epoch: Option, @@ -3226,45 +3730,54 @@ pub struct JsNeighborsPagedOptions { } #[napi(object)] -pub struct JsNeighborsBatchOptions { +pub struct NeighborsBatchOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, pub decay_lambda: Option, } #[napi(object)] -pub struct JsTraverseOptions { +#[derive(Clone)] +pub struct NodeLabelFilter { + pub labels: Vec, + #[napi(ts_type = "'any' | 'all'")] + pub mode: String, +} + +#[napi(object)] +pub struct TraverseOptions { pub min_depth: Option, pub direction: Option, - pub edge_type_filter: Option>, - pub node_type_filter: Option>, + pub edge_label_filter: Option>, + pub emit_node_label_filter: Option, pub at_epoch: Option, pub decay_lambda: Option, pub limit: Option, - pub cursor: Option, + pub cursor: Option, } #[napi(object)] -pub struct JsTopKNeighborsOptions { +pub struct TopKNeighborsOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub scoring: Option, pub decay_lambda: Option, pub at_epoch: Option, } #[napi(object)] -pub struct JsExtractSubgraphOptions { +pub struct ExtractSubgraphOptions { pub direction: Option, - pub edge_type_filter: Option>, + pub edge_label_filter: Option>, + pub node_label_filter: Option, pub at_epoch: Option, } #[napi(object)] -pub struct JsShortestPathOptions { +pub struct ShortestPathOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub weight_field: Option, pub at_epoch: Option, pub max_depth: Option, @@ -3272,9 +3785,9 @@ pub struct JsShortestPathOptions { } #[napi(object)] -pub struct JsAllShortestPathsOptions { +pub struct AllShortestPathsOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub weight_field: Option, pub at_epoch: Option, pub max_depth: Option, @@ -3283,85 +3796,96 @@ pub struct JsAllShortestPathsOptions { } #[napi(object)] -pub struct JsIsConnectedOptions { +pub struct IsConnectedOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, pub max_depth: Option, } #[napi(object)] -pub struct JsConnectedComponentsOptions { - pub edge_type_filter: Option>, - pub node_type_filter: Option>, +pub struct ConnectedComponentsOptions { + pub edge_label_filter: Option>, + pub node_label_filter: Option, pub at_epoch: Option, } #[napi(object)] -pub struct JsComponentOfOptions { - pub edge_type_filter: Option>, - pub node_type_filter: Option>, +pub struct ComponentOfOptions { + pub edge_label_filter: Option>, + pub node_label_filter: Option, pub at_epoch: Option, } #[napi(object)] -pub struct JsDegreeOptions { +pub struct DegreeOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, } #[napi(object)] -pub struct JsSumEdgeWeightsOptions { +pub struct SumEdgeWeightsOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, } #[napi(object)] -pub struct JsAvgEdgeWeightOptions { +pub struct AvgEdgeWeightOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, } #[napi(object)] -pub struct JsDegreesOptions { +pub struct DegreesOptions { pub direction: Option, - pub type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, } #[napi(object)] -pub struct JsVectorSearchOptions { +pub struct VectorSearchOptions { pub k: u32, pub dense_query: Option>, - pub sparse_query: Option>, - pub type_filter: Option>, + pub sparse_query: Option>, + pub label_filter: Option, pub ef_search: Option, - pub scope: Option, + pub scope: Option, pub dense_weight: Option, pub sparse_weight: Option, pub fusion_mode: Option, } #[napi(object)] -pub struct JsFindNodesPagedOptions { +pub struct FindNodesPagedOptions { pub limit: Option, pub after: Option, } #[napi(object)] #[derive(Clone)] -pub struct JsSecondaryIndexKind { +pub struct SecondaryIndexKind { + pub kind: String, + pub domain: Option, +} + +#[napi(object)] +pub struct NodePropertyIndexInfo { + pub index_id: f64, + pub label: String, + pub prop_key: String, pub kind: String, pub domain: Option, + pub state: String, + pub last_error: Option, } #[napi(object)] -pub struct JsNodePropertyIndexInfo { +pub struct EdgePropertyIndexInfo { pub index_id: f64, - pub type_id: u32, + pub label: String, pub prop_key: String, pub kind: String, pub domain: Option, @@ -3369,9 +3893,39 @@ pub struct JsNodePropertyIndexInfo { pub last_error: Option, } +#[napi(object)] +pub struct NodeLabelInfo { + pub label: String, + pub label_id: u32, +} + +impl From for NodeLabelInfo { + fn from(info: CoreNodeLabelInfo) -> Self { + Self { + label: info.label, + label_id: info.label_id, + } + } +} + +#[napi(object)] +pub struct EdgeLabelInfo { + pub label: String, + pub label_id: u32, +} + +impl From for EdgeLabelInfo { + fn from(info: CoreEdgeLabelInfo) -> Self { + Self { + label: info.label, + label_id: info.label_id, + } + } +} + #[napi(object)] #[derive(Clone)] -pub struct JsPropertyRangeBound { +pub struct PropertyRangeBound { pub value: f64, pub inclusive: Option, pub domain: String, @@ -3379,7 +3933,7 @@ pub struct JsPropertyRangeBound { #[napi(object)] #[derive(Clone)] -pub struct JsPropertyRangeCursor { +pub struct PropertyRangeCursor { pub value: f64, pub node_id: f64, pub domain: String, @@ -3387,30 +3941,30 @@ pub struct JsPropertyRangeCursor { #[napi(object)] #[derive(Clone)] -pub struct JsFindNodesRangePagedOptions { +pub struct FindNodesRangePagedOptions { pub limit: Option, - pub after: Option, + pub after: Option, } #[napi(object)] -pub struct JsFindNodesByTimeRangePagedOptions { +pub struct FindNodesByTimeRangePagedOptions { pub limit: Option, pub after: Option, } #[napi(object)] -pub struct JsPersonalizedPagerankOptions { +pub struct PersonalizedPagerankOptions { pub algorithm: Option, pub damping_factor: Option, pub max_iterations: Option, pub epsilon: Option, pub approx_residual_tolerance: Option, - pub edge_type_filter: Option>, + pub edge_label_filter: Option>, pub max_results: Option, } #[napi(object)] -pub struct JsDbStats { +pub struct DbStats { /// Bytes buffered in WAL but not yet fsynced. Always 0 in immediate mode. pub pending_wal_bytes: u32, /// Number of on-disk segments. @@ -3437,9 +3991,9 @@ pub struct JsDbStats { pub oldest_retained_wal_generation_id: f64, } -impl From for JsDbStats { - fn from(s: DbStats) -> Self { - JsDbStats { +impl From for DbStats { + fn from(s: CoreDbStats) -> Self { + DbStats { pending_wal_bytes: s.pending_wal_bytes.min(u32::MAX as usize) as u32, segment_count: s.segment_count.min(u32::MAX as usize) as u32, node_tombstone_count: s.node_tombstone_count.min(u32::MAX as usize) as u32, @@ -3457,22 +4011,81 @@ impl From for JsDbStats { } #[napi(object)] -pub struct JsDenseVectorConfig { +pub struct ScrubReport { + pub segments: Vec, + pub total_components_checked: f64, + pub total_components_ok: f64, + pub total_components_failed: f64, + pub total_bytes_digested: f64, + pub duration_ms: f64, +} + +#[napi(object)] +pub struct SegmentScrubResult { + pub segment_id: f64, + pub findings: Vec, + pub components_ok: f64, + pub bytes_digested: f64, +} + +#[napi(object)] +pub struct ComponentScrubFinding { + pub component_kind: String, + pub finding_type: String, + pub detail: String, +} + +impl From for ScrubReport { + fn from(r: CoreScrubReport) -> Self { + ScrubReport { + segments: r.segments.into_iter().map(|s| s.into()).collect(), + total_components_checked: r.total_components_checked as f64, + total_components_ok: r.total_components_ok as f64, + total_components_failed: r.total_components_failed as f64, + total_bytes_digested: r.total_bytes_digested as f64, + duration_ms: r.duration_ms as f64, + } + } +} + +impl From for SegmentScrubResult { + fn from(s: overgraph::SegmentScrubResult) -> Self { + SegmentScrubResult { + segment_id: s.segment_id as f64, + findings: s.findings.into_iter().map(|f| f.into()).collect(), + components_ok: s.components_ok as f64, + bytes_digested: s.bytes_digested as f64, + } + } +} + +impl From for ComponentScrubFinding { + fn from(f: overgraph::ComponentScrubFinding) -> Self { + ComponentScrubFinding { + component_kind: f.component_kind, + finding_type: format!("{:?}", f.finding_type), + detail: f.detail, + } + } +} + +#[napi(object)] +pub struct DenseVectorConfig { pub dimension: u32, pub metric: Option, } #[napi(object)] -pub struct JsDbOptions { +pub struct DbOptions { pub create_if_missing: Option, pub edge_uniqueness: Option, pub memtable_flush_threshold: Option, - /// Trigger compaction automatically after this many flushes. Default 5, 0 = disabled. + /// Trigger compaction automatically after this many flushes. Default 4, 0 = disabled. pub compact_after_n_flushes: Option, - pub dense_vector: Option, + pub dense_vector: Option, /// WAL sync mode: 'immediate' or 'group-commit' (default). pub wal_sync_mode: Option, - /// Group commit sync interval in milliseconds. Default: 10. + /// Group commit sync interval in milliseconds. Default: 50. pub group_commit_interval_ms: Option, /// Hard cap on memtable size in bytes. Writes trigger a flush when exceeded. 0 = disabled. pub memtable_hard_cap_bytes: Option, @@ -3481,9 +4094,9 @@ pub struct JsDbOptions { pub max_immutable_memtables: Option, } -impl From for DbOptions { - fn from(js: JsDbOptions) -> Self { - let defaults = DbOptions::default(); +impl From for CoreDbOptions { + fn from(js: DbOptions) -> Self { + let defaults = CoreDbOptions::default(); let wal_sync_mode = match js.wal_sync_mode.as_deref() { Some("immediate") => WalSyncMode::Immediate, _ => { @@ -3502,13 +4115,13 @@ impl From for DbOptions { Some("dot_product") => DenseMetric::DotProduct, _ => DenseMetric::Cosine, }; - DenseVectorConfig { + CoreDenseVectorConfig { dimension: dv.dimension, metric, hnsw: HnswConfig::default(), } }); - DbOptions { + CoreDbOptions { create_if_missing: js.create_if_missing.unwrap_or(defaults.create_if_missing), edge_uniqueness: js.edge_uniqueness.unwrap_or(defaults.edge_uniqueness), memtable_flush_threshold: js @@ -3533,25 +4146,40 @@ impl From for DbOptions { } #[napi(object)] -pub struct JsKeyQuery { - pub type_id: u32, +pub struct KeyQuery { + pub label: String, pub key: String, } +impl TryFrom for NodeKeyQuery { + type Error = napi::Error; + + fn try_from(js: KeyQuery) -> std::result::Result { + Ok(NodeKeyQuery { + label: js.label, + key: js.key, + }) + } +} + #[napi(object)] -pub struct JsNodeInput { - pub type_id: u32, +pub struct NodeInput { + #[napi(ts_type = "string | string[]")] + pub labels: serde_json::Value, pub key: String, pub props: Option>, pub weight: Option, pub dense_vector: Option>, - pub sparse_vector: Option>, + pub sparse_vector: Option>, } -impl From for NodeInput { - fn from(js: JsNodeInput) -> Self { - NodeInput { - type_id: js.type_id, +impl TryFrom for CoreNodeInput { + type Error = napi::Error; + + fn try_from(js: NodeInput) -> std::result::Result { + let labels = parse_js_node_labels_arg(&js.labels, "NodeInput labels")?; + Ok(CoreNodeInput { + labels, key: js.key, props: convert_js_props(js.props), weight: js.weight.unwrap_or(1.0) as f32, @@ -3563,84 +4191,71 @@ impl From for NodeInput { .map(|e| (e.dimension, e.value as f32)) .collect() }), - } + }) } } #[napi(object)] -pub struct JsSparseEntry { +#[derive(Clone)] +pub struct SparseEntry { pub dimension: u32, pub value: f64, } #[napi(object)] #[derive(Clone)] -pub struct JsTxnNodeRef { +pub struct TxnNodeRef { pub id: Option, - pub type_id: Option, + #[napi(ts_type = "string | string[]")] + pub labels: Option, pub key: Option, pub local: Option, } #[napi(object)] #[derive(Clone)] -pub struct JsTxnEdgeRef { +pub struct TxnEdgeRef { pub id: Option, - pub from: Option, - pub to: Option, - pub type_id: Option, + pub from: Option, + pub to: Option, + pub label: Option, pub local: Option, } -#[napi(object)] -pub struct JsTxnOperation { - pub op: String, - pub alias: Option, - pub type_id: Option, - pub key: Option, - pub props: Option>, - pub weight: Option, - pub dense_vector: Option>, - pub sparse_vector: Option>, - pub from: Option, - pub to: Option, - pub target: Option, - pub valid_from: Option, - pub valid_to: Option, -} - #[napi(object)] #[derive(Clone)] -pub struct JsTxnEdgeOrNodeRef { +pub struct TxnEdgeOrNodeRef { pub id: Option, - pub type_id: Option, + #[napi(ts_type = "string | string[]")] + pub labels: Option, + pub label: Option, pub key: Option, pub local: Option, - pub from: Option, - pub to: Option, + pub from: Option, + pub to: Option, } #[napi(object)] -pub struct JsTxnNodeView { +pub struct TxnNodeView { pub id: Option, pub local: Option, - pub type_id: u32, + pub labels: Vec, pub key: String, pub props: HashMap, pub created_at: Option, pub updated_at: Option, pub weight: f64, pub dense_vector: Option>, - pub sparse_vector: Option>, + pub sparse_vector: Option>, } #[napi(object)] -pub struct JsTxnEdgeView { +pub struct TxnEdgeView { pub id: Option, pub local: Option, - pub from: JsTxnNodeRef, - pub to: JsTxnNodeRef, - pub type_id: u32, + pub from: TxnNodeRef, + pub to: TxnNodeRef, + pub label: String, pub props: HashMap, pub created_at: Option, pub updated_at: Option, @@ -3650,7 +4265,7 @@ pub struct JsTxnEdgeView { } #[napi(object)] -pub struct JsTxnCommitResult { +pub struct TxnCommitResult { pub node_ids: Float64Array, pub edge_ids: Float64Array, pub node_aliases: HashMap, @@ -3658,38 +4273,38 @@ pub struct JsTxnCommitResult { } #[napi(object)] -pub struct JsVectorSearchScope { +pub struct VectorSearchScope { pub start_node_id: f64, pub max_depth: u32, pub direction: Option, - pub edge_type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, } #[napi(object)] -pub struct JsVectorHit { +pub struct VectorHit { pub node_id: f64, pub score: f64, } #[napi(object)] -pub struct JsEdgeInput { +pub struct EdgeInput { pub from: f64, pub to: f64, - pub type_id: u32, + pub label: String, pub props: Option>, pub weight: Option, pub valid_from: Option, pub valid_to: Option, } -impl TryFrom for EdgeInput { +impl TryFrom for CoreEdgeInput { type Error = napi::Error; - fn try_from(js: JsEdgeInput) -> std::result::Result { - Ok(EdgeInput { + fn try_from(js: EdgeInput) -> std::result::Result { + Ok(CoreEdgeInput { from: f64_to_u64(js.from)?, to: f64_to_u64(js.to)?, - type_id: js.type_id, + label: js.label, props: convert_js_props(js.props), weight: js.weight.unwrap_or(1.0) as f32, valid_from: js.valid_from, @@ -3698,28 +4313,30 @@ impl TryFrom for EdgeInput { } } -/// Node record: eager primitives, lazy props. Props are Arc-shared so +/// Node view: eager primitives, lazy props. Props are Arc-shared so /// container getters (page results, subgraph) avoid cloning the BTreeMap. #[napi] -pub struct JsNodeRecord { +pub struct NodeView { id_val: f64, - type_id_val: u32, + labels_val: Vec, key_val: String, created_at_val: i64, updated_at_val: i64, weight_val: f64, + dense_vector_val: Option>, + sparse_vector_val: Option>, props_raw: Arc>, } #[napi] -impl JsNodeRecord { +impl NodeView { #[napi(getter)] pub fn id(&self) -> f64 { self.id_val } #[napi(getter)] - pub fn type_id(&self) -> u32 { - self.type_id_val + pub fn labels(&self) -> Vec { + self.labels_val.clone() } #[napi(getter)] pub fn key(&self) -> String { @@ -3741,31 +4358,51 @@ impl JsNodeRecord { pub fn weight(&self) -> f64 { self.weight_val } + #[napi(getter)] + pub fn dense_vector(&self) -> Option> { + self.dense_vector_val.clone() + } + #[napi(getter)] + pub fn sparse_vector(&self) -> Option> { + self.sparse_vector_val.clone() + } } -impl TryFrom for JsNodeRecord { +impl TryFrom for NodeView { type Error = napi::Error; - fn try_from(n: NodeRecord) -> Result { - Ok(JsNodeRecord { + fn try_from(n: CoreNodeView) -> Result { + Ok(NodeView { id_val: u64_to_f64(n.id)?, - type_id_val: n.type_id, + labels_val: n.labels, key_val: n.key, created_at_val: n.created_at, updated_at_val: n.updated_at, weight_val: n.weight as f64, + dense_vector_val: n + .dense_vector + .map(|values| values.into_iter().map(|v| v as f64).collect()), + sparse_vector_val: n.sparse_vector.map(|entries| { + entries + .into_iter() + .map(|(dimension, value)| SparseEntry { + dimension, + value: value as f64, + }) + .collect() + }), props_raw: Arc::new(n.props), }) } } -/// Edge record: eager primitives, lazy props. Props are Arc-shared so +/// Edge view: eager primitives, lazy props. Props are Arc-shared so /// container getters (page results, subgraph) avoid cloning the BTreeMap. #[napi] -pub struct JsEdgeRecord { +pub struct EdgeView { id_val: f64, from_val: f64, to_val: f64, - type_id_val: u32, + label_val: String, created_at_val: i64, updated_at_val: i64, weight_val: f64, @@ -3775,7 +4412,7 @@ pub struct JsEdgeRecord { } #[napi] -impl JsEdgeRecord { +impl EdgeView { #[napi(getter)] pub fn id(&self) -> f64 { self.id_val @@ -3789,8 +4426,8 @@ impl JsEdgeRecord { self.to_val } #[napi(getter)] - pub fn type_id(&self) -> u32 { - self.type_id_val + pub fn label(&self) -> String { + self.label_val.clone() } #[napi(getter)] pub fn props(&self) -> HashMap { @@ -3818,14 +4455,14 @@ impl JsEdgeRecord { } } -impl TryFrom for JsEdgeRecord { +impl TryFrom for EdgeView { type Error = napi::Error; - fn try_from(e: EdgeRecord) -> Result { - Ok(JsEdgeRecord { + fn try_from(e: CoreEdgeView) -> Result { + Ok(EdgeView { id_val: u64_to_f64(e.id)?, from_val: u64_to_f64(e.from)?, to_val: u64_to_f64(e.to)?, - type_id_val: e.type_id, + label_val: e.label, created_at_val: e.created_at, updated_at_val: e.updated_at, weight_val: e.weight as f64, @@ -3839,20 +4476,20 @@ impl TryFrom for JsEdgeRecord { /// A single neighbor entry as a plain JS object. #[napi(object)] #[derive(Clone)] -pub struct JsNeighborEntry { +pub struct NeighborEntry { pub node_id: f64, pub edge_id: f64, - pub edge_type_id: u32, + pub label: String, pub weight: f64, pub valid_from: i64, pub valid_to: i64, } -fn neighbor_to_js_entry(e: &NeighborEntry) -> Result { - Ok(JsNeighborEntry { +fn neighbor_to_js_entry(e: &CoreNeighborEntry) -> Result { + Ok(NeighborEntry { node_id: u64_to_f64(e.node_id)?, edge_id: u64_to_f64(e.edge_id)?, - edge_type_id: e.edge_type_id, + label: e.label.clone(), weight: e.weight as f64, valid_from: e.valid_from, valid_to: e.valid_to, @@ -3860,32 +4497,32 @@ fn neighbor_to_js_entry(e: &NeighborEntry) -> Result { } #[napi(object)] -pub struct JsNeighborBatchEntry { +pub struct NeighborBatchEntry { pub query_node_id: f64, - pub neighbors: Vec, + pub neighbors: Vec, } #[napi(object)] -pub struct JsDegreeBatchEntry { +pub struct DegreeBatchEntry { pub node_id: f64, pub degree: i64, } #[napi(object)] -pub struct JsComponentEntry { +pub struct ComponentEntry { pub node_id: f64, pub component_id: f64, } #[napi(object)] -pub struct JsShortestPath { +pub struct ShortestPath { pub nodes: Vec, pub edges: Vec, pub total_cost: f64, } -fn shortest_path_to_js(sp: ShortestPath) -> Result { - Ok(JsShortestPath { +fn shortest_path_to_js(sp: CoreShortestPath) -> Result { + Ok(ShortestPath { nodes: sp .nodes .into_iter() @@ -3901,7 +4538,7 @@ fn shortest_path_to_js(sp: ShortestPath) -> Result { } #[napi(object)] -pub struct JsTraversalHit { +pub struct TraversalHit { pub node_id: f64, pub depth: u32, pub via_edge_id: Option, @@ -3909,19 +4546,19 @@ pub struct JsTraversalHit { } #[napi(object)] -pub struct JsTraversalCursor { +pub struct TraversalCursor { pub depth: u32, pub last_node_id: f64, } #[napi(object)] -pub struct JsTraversalPageResult { - pub items: Vec, - pub next_cursor: Option, +pub struct TraversalPageResult { + pub items: Vec, + pub next_cursor: Option, } -fn traversal_hit_to_js(hit: TraversalHit) -> Result { - Ok(JsTraversalHit { +fn traversal_hit_to_js(hit: CoreTraversalHit) -> Result { + Ok(TraversalHit { node_id: u64_to_f64(hit.node_id)?, depth: hit.depth, via_edge_id: hit.via_edge_id.map(u64_to_f64).transpose()?, @@ -3929,22 +4566,22 @@ fn traversal_hit_to_js(hit: TraversalHit) -> Result { }) } -fn traversal_cursor_to_js(cursor: TraversalCursor) -> Result { - Ok(JsTraversalCursor { +fn traversal_cursor_to_js(cursor: CoreTraversalCursor) -> Result { + Ok(TraversalCursor { depth: cursor.depth, last_node_id: u64_to_f64(cursor.last_node_id)?, }) } -fn js_traversal_cursor_to_rust(cursor: JsTraversalCursor) -> Result { - Ok(TraversalCursor { +fn js_traversal_cursor_to_rust(cursor: TraversalCursor) -> Result { + Ok(CoreTraversalCursor { depth: cursor.depth, last_node_id: f64_to_u64(cursor.last_node_id)?, }) } -fn traversal_page_to_js(page: TraversalPageResult) -> Result { - Ok(JsTraversalPageResult { +fn traversal_page_to_js(page: CoreTraversalPageResult) -> Result { + Ok(TraversalPageResult { items: page .items .into_iter() @@ -3955,37 +4592,39 @@ fn traversal_page_to_js(page: TraversalPageResult) -> Result, - edges_vec: Vec, +pub struct SubgraphResult { + nodes_vec: Vec, + edges_vec: Vec, } #[napi] -impl JsSubgraphResult { +impl SubgraphResult { #[napi(getter)] - pub fn nodes(&self) -> Vec { + pub fn nodes(&self) -> Vec { self.nodes_vec .iter() - .map(|n| JsNodeRecord { + .map(|n| NodeView { id_val: n.id_val, - type_id_val: n.type_id_val, + labels_val: n.labels_val.clone(), key_val: n.key_val.clone(), created_at_val: n.created_at_val, updated_at_val: n.updated_at_val, weight_val: n.weight_val, + dense_vector_val: n.dense_vector_val.clone(), + sparse_vector_val: n.sparse_vector_val.clone(), props_raw: Arc::clone(&n.props_raw), }) .collect() } #[napi(getter)] - pub fn edges(&self) -> Vec { + pub fn edges(&self) -> Vec { self.edges_vec .iter() - .map(|e| JsEdgeRecord { + .map(|e| EdgeView { id_val: e.id_val, from_val: e.from_val, to_val: e.to_val, - type_id_val: e.type_id_val, + label_val: e.label_val.clone(), created_at_val: e.created_at_val, updated_at_val: e.updated_at_val, weight_val: e.weight_val, @@ -3997,17 +4636,17 @@ impl JsSubgraphResult { } } -fn subgraph_to_js(sg: Subgraph) -> Result { - Ok(JsSubgraphResult { +fn subgraph_to_js(sg: Subgraph) -> Result { + Ok(SubgraphResult { nodes_vec: sg .nodes .into_iter() - .map(JsNodeRecord::try_from) + .map(NodeView::try_from) .collect::>>()?, edges_vec: sg .edges .into_iter() - .map(JsEdgeRecord::try_from) + .map(EdgeView::try_from) .collect::>>()?, }) } @@ -4015,30 +4654,32 @@ fn subgraph_to_js(sg: Subgraph) -> Result { // --- Pagination result types --- #[napi(object)] -pub struct JsIdPageResult { +pub struct IdPageResult { pub items: Float64Array, pub next_cursor: Option, } #[napi] -pub struct JsNodePageResult { - items_vec: Vec, +pub struct NodePageResult { + items_vec: Vec, cursor: Option, } #[napi] -impl JsNodePageResult { +impl NodePageResult { #[napi(getter)] - pub fn items(&self) -> Vec { + pub fn items(&self) -> Vec { self.items_vec .iter() - .map(|n| JsNodeRecord { + .map(|n| NodeView { id_val: n.id_val, - type_id_val: n.type_id_val, + labels_val: n.labels_val.clone(), key_val: n.key_val.clone(), created_at_val: n.created_at_val, updated_at_val: n.updated_at_val, weight_val: n.weight_val, + dense_vector_val: n.dense_vector_val.clone(), + sparse_vector_val: n.sparse_vector_val.clone(), props_raw: Arc::clone(&n.props_raw), }) .collect() @@ -4050,22 +4691,22 @@ impl JsNodePageResult { } #[napi] -pub struct JsEdgePageResult { - items_vec: Vec, +pub struct EdgePageResult { + items_vec: Vec, cursor: Option, } #[napi] -impl JsEdgePageResult { +impl EdgePageResult { #[napi(getter)] - pub fn items(&self) -> Vec { + pub fn items(&self) -> Vec { self.items_vec .iter() - .map(|e| JsEdgeRecord { + .map(|e| EdgeView { id_val: e.id_val, from_val: e.from_val, to_val: e.to_val, - type_id_val: e.type_id_val, + label_val: e.label_val.clone(), created_at_val: e.created_at_val, updated_at_val: e.updated_at_val, weight_val: e.weight_val, @@ -4082,15 +4723,15 @@ impl JsEdgePageResult { } #[napi] -pub struct JsNeighborPageResult { - items_vec: Vec, +pub struct NeighborPageResult { + items_vec: Vec, cursor: Option, } #[napi] -impl JsNeighborPageResult { +impl NeighborPageResult { #[napi(getter)] - pub fn items(&self) -> Vec { + pub fn items(&self) -> Vec { self.items_vec.clone() } @@ -4100,53 +4741,55 @@ impl JsNeighborPageResult { } } -fn id_page_to_js(page: PageResult) -> Result { - Ok(JsIdPageResult { +fn id_page_to_js(page: PageResult) -> Result { + Ok(IdPageResult { items: ids_to_float64_array(&page.items)?, next_cursor: page.next_cursor.map(u64_to_f64).transpose()?, }) } -fn node_page_to_js(page: PageResult) -> Result { - Ok(JsNodePageResult { +fn node_page_to_js(page: PageResult) -> Result { + Ok(NodePageResult { items_vec: page .items .into_iter() - .map(JsNodeRecord::try_from) + .map(NodeView::try_from) .collect::>>()?, cursor: page.next_cursor, }) } -fn edge_page_to_js(page: PageResult) -> Result { - Ok(JsEdgePageResult { +fn edge_page_to_js(page: PageResult) -> Result { + Ok(EdgePageResult { items_vec: page .items .into_iter() - .map(JsEdgeRecord::try_from) + .map(EdgeView::try_from) .collect::>>()?, cursor: page.next_cursor, }) } -fn neighbor_page_to_js(page: PageResult) -> Result { - Ok(JsNeighborPageResult { +fn neighbor_page_to_js(page: PageResult) -> Result { + Ok(NeighborPageResult { items_vec: neighbor_entries_to_js(page.items)?, cursor: page.next_cursor, }) } #[napi(object)] -pub struct JsPropertyRangePageResult { +pub struct PropertyRangePageResult { pub items: Float64Array, - pub next_cursor: Option, + pub next_cursor: Option, } -fn node_property_index_info_to_js(info: NodePropertyIndexInfo) -> Result { +fn node_property_index_info_to_js( + info: CoreNodePropertyIndexInfo, +) -> Result { let (kind, domain) = secondary_index_kind_to_js(&info.kind); - Ok(JsNodePropertyIndexInfo { + Ok(NodePropertyIndexInfo { index_id: u64_to_f64(info.index_id)?, - type_id: info.type_id, + label: info.label, prop_key: info.prop_key, kind, domain, @@ -4156,35 +4799,61 @@ fn node_property_index_info_to_js(info: NodePropertyIndexInfo) -> Result, -) -> Result> { + infos: Vec, +) -> Result> { infos .into_iter() .map(node_property_index_info_to_js) .collect() } -fn property_range_cursor_to_js(cursor: PropertyRangeCursor) -> Result { +fn edge_property_index_info_to_js( + info: CoreEdgePropertyIndexInfo, +) -> Result { + let (kind, domain) = secondary_index_kind_to_js(&info.kind); + Ok(EdgePropertyIndexInfo { + index_id: u64_to_f64(info.index_id)?, + label: info.label, + prop_key: info.prop_key, + kind, + domain, + state: secondary_index_state_to_js(info.state).to_string(), + last_error: info.last_error, + }) +} + +fn edge_property_index_infos_to_js( + infos: Vec, +) -> Result> { + infos + .into_iter() + .map(edge_property_index_info_to_js) + .collect() +} + +fn property_range_cursor_to_js(cursor: CorePropertyRangeCursor) -> Result { let (value, domain) = prop_value_to_js_numeric_parts(&cursor.value)?; - Ok(JsPropertyRangeCursor { + Ok(PropertyRangeCursor { value, node_id: u64_to_f64(cursor.node_id)?, domain, }) } -fn js_property_range_cursor_to_rust(cursor: JsPropertyRangeCursor) -> Result { +fn js_property_range_cursor_to_rust( + cursor: PropertyRangeCursor, +) -> Result { let domain = parse_secondary_index_range_domain(Some(cursor.domain.as_str()))?; - Ok(PropertyRangeCursor { + Ok(CorePropertyRangeCursor { value: js_numeric_to_prop_value(cursor.value, domain)?, node_id: f64_to_u64(cursor.node_id)?, }) } fn property_range_page_to_js( - page: PropertyRangePageResult, -) -> Result { - Ok(JsPropertyRangePageResult { + page: CorePropertyRangePageResult, +) -> Result { + Ok(PropertyRangePageResult { items: ids_to_float64_array(&page.items)?, next_cursor: page .next_cursor @@ -4201,26 +4870,44 @@ fn make_page_request(limit: Option, after: Option) -> napi::Result Result { - Ok(JsIdPageResult { +fn query_node_ids_to_js(result: QueryNodeIdsResult) -> Result { + Ok(IdPageResult { items: ids_to_float64_array(&result.items)?, next_cursor: result.next_cursor.map(u64_to_f64).transpose()?, }) } -fn query_nodes_to_js(result: QueryNodesResult) -> Result { - Ok(JsNodePageResult { +fn query_edge_ids_to_js(result: QueryEdgeIdsResult) -> Result { + Ok(IdPageResult { + items: ids_to_float64_array(&result.edge_ids)?, + next_cursor: result.next_cursor.map(u64_to_f64).transpose()?, + }) +} + +fn query_nodes_to_js(result: QueryNodesResult) -> Result { + Ok(NodePageResult { items_vec: result .items .into_iter() - .map(JsNodeRecord::try_from) + .map(NodeView::try_from) + .collect::>>()?, + cursor: result.next_cursor, + }) +} + +fn query_edges_to_js(result: QueryEdgesResult) -> Result { + Ok(EdgePageResult { + items_vec: result + .edges + .into_iter() + .map(EdgeView::try_from) .collect::>>()?, cursor: result.next_cursor, }) } -fn query_pattern_result_to_js(result: QueryPatternResult) -> Result { - Ok(JsJsonValue(serde_json::json!({ +fn query_pattern_result_to_js(result: QueryPatternResult) -> Result { + Ok(JsonPayload(serde_json::json!({ "matches": result .matches .into_iter() @@ -4245,8 +4932,8 @@ fn query_match_to_js(match_: QueryMatch) -> Result { })) } -fn query_plan_to_js(plan: QueryPlan) -> Result { - Ok(JsJsonValue(serde_json::json!({ +fn query_plan_to_js(plan: QueryPlan) -> Result { + Ok(JsonPayload(serde_json::json!({ "kind": query_plan_kind_to_js(&plan.kind), "root": query_plan_node_to_js(plan.root), "estimatedCandidates": plan.estimated_candidates.map(|count| count as f64), @@ -4255,12 +4942,19 @@ fn query_plan_to_js(plan: QueryPlan) -> Result { .iter() .map(query_plan_warning_to_js) .collect::>(), + "notes": plan + .notes + .iter() + .map(query_plan_note_to_js) + .collect::>(), + "publicInputs": query_plan_public_inputs_to_js(plan.public_inputs), }))) } fn query_plan_kind_to_js(kind: &QueryPlanKind) -> &'static str { match kind { QueryPlanKind::NodeQuery => "node_query", + QueryPlanKind::EdgeQuery => "edge_query", QueryPlanKind::PatternQuery => "pattern_query", } } @@ -4269,7 +4963,8 @@ fn query_plan_node_to_js(node: QueryPlanNode) -> serde_json::Value { match node { QueryPlanNode::ExplicitIds => serde_json::json!({ "kind": "explicit_ids" }), QueryPlanNode::KeyLookup => serde_json::json!({ "kind": "key_lookup" }), - QueryPlanNode::NodeTypeIndex => serde_json::json!({ "kind": "node_type_index" }), + QueryPlanNode::NodeLabelIndex => serde_json::json!({ "kind": "node_label_index" }), + QueryPlanNode::NodeLabelAnyIndex => serde_json::json!({ "kind": "node_label_any_index" }), QueryPlanNode::PropertyEqualityIndex => { serde_json::json!({ "kind": "property_equality_index" }) } @@ -4278,6 +4973,24 @@ fn query_plan_node_to_js(node: QueryPlanNode) -> serde_json::Value { } QueryPlanNode::TimestampIndex => serde_json::json!({ "kind": "timestamp_index" }), QueryPlanNode::AdjacencyExpansion => serde_json::json!({ "kind": "adjacency_expansion" }), + QueryPlanNode::ExplicitEdgeIds => serde_json::json!({ "kind": "explicit_edge_ids" }), + QueryPlanNode::EdgeLabelIndex => serde_json::json!({ "kind": "edge_label_index" }), + QueryPlanNode::EdgeTripleIndex => serde_json::json!({ "kind": "edge_triple_index" }), + QueryPlanNode::EdgeEndpointAdjacency => { + serde_json::json!({ "kind": "edge_endpoint_adjacency" }) + } + QueryPlanNode::EdgeWeightIndex => serde_json::json!({ "kind": "edge_weight_index" }), + QueryPlanNode::EdgeUpdatedAtIndex => { + serde_json::json!({ "kind": "edge_updated_at_index" }) + } + QueryPlanNode::EdgeValidityIndex => serde_json::json!({ "kind": "edge_validity_index" }), + QueryPlanNode::EdgeMetadataScan => serde_json::json!({ "kind": "edge_metadata_scan" }), + QueryPlanNode::EdgePropertyEqualityIndex => { + serde_json::json!({ "kind": "edge_property_equality_index" }) + } + QueryPlanNode::EdgePropertyRangeIndex => { + serde_json::json!({ "kind": "edge_property_range_index" }) + } QueryPlanNode::Intersect { inputs } => serde_json::json!({ "kind": "intersect", "inputs": inputs.into_iter().map(query_plan_node_to_js).collect::>(), @@ -4290,6 +5003,10 @@ fn query_plan_node_to_js(node: QueryPlanNode) -> serde_json::Value { "kind": "verify_node_filter", "input": query_plan_node_to_js(*input), }), + QueryPlanNode::VerifyEdgeFilter { input } => serde_json::json!({ + "kind": "verify_edge_filter", + "input": query_plan_node_to_js(*input), + }), QueryPlanNode::VerifyEdgePredicates { input } => serde_json::json!({ "kind": "verify_edge_predicates", "input": query_plan_node_to_js(*input), @@ -4302,14 +5019,71 @@ fn query_plan_node_to_js(node: QueryPlanNode) -> serde_json::Value { "anchorAlias": anchor_alias, "input": query_plan_node_to_js(*input), }), - QueryPlanNode::FallbackTypeScan => serde_json::json!({ "kind": "fallback_type_scan" }), + QueryPlanNode::PatternEdgeAnchor { edge_alias, input } => serde_json::json!({ + "kind": "pattern_edge_anchor", + "edgeAlias": edge_alias, + "input": query_plan_node_to_js(*input), + }), + QueryPlanNode::FallbackNodeLabelScan => { + serde_json::json!({ "kind": "fallback_node_label_scan" }) + } QueryPlanNode::FallbackFullNodeScan => { serde_json::json!({ "kind": "fallback_full_node_scan" }) } + QueryPlanNode::FallbackEdgeLabelScan => { + serde_json::json!({ "kind": "fallback_edge_label_scan" }) + } + QueryPlanNode::FallbackFullEdgeScan => { + serde_json::json!({ "kind": "fallback_full_edge_scan" }) + } QueryPlanNode::EmptyResult => serde_json::json!({ "kind": "empty_result" }), } } +fn query_plan_note_to_js(note: &overgraph::QueryPlanNote) -> &'static str { + match note { + overgraph::QueryPlanNote::NodeLabelAnyDedupeBeforePagination => { + "node_label_any_dedupe_before_pagination" + } + overgraph::QueryPlanNote::NodeLabelAnyFinalVerification => { + "node_label_any_final_verification" + } + overgraph::QueryPlanNote::NodeLabelAllSupersetVerification => { + "node_label_all_superset_verification" + } + overgraph::QueryPlanNote::StaleNodeLabelMembershipVerification => { + "stale_node_label_membership_verification" + } + } +} + +fn query_plan_public_inputs_to_js(inputs: overgraph::QueryPlanPublicInputs) -> serde_json::Value { + serde_json::json!({ + "nodeLabels": inputs + .node_labels + .into_iter() + .map(query_plan_public_name_to_js) + .collect::>(), + "edgeLabels": inputs + .edge_labels + .into_iter() + .map(query_plan_public_name_to_js) + .collect::>(), + }) +} + +fn query_plan_public_name_to_js(name: overgraph::QueryPlanPublicName) -> serde_json::Value { + serde_json::json!({ + "alias": name.alias, + "name": name.name, + "known": name.known, + "mode": name.mode.map(|mode| match mode { + CoreLabelMatchMode::Any => "any", + CoreLabelMatchMode::All => "all", + }), + }) +} + fn query_plan_warning_to_js(warning: &QueryPlanWarning) -> &'static str { match warning { QueryPlanWarning::MissingReadyIndex => "missing_ready_index", @@ -4325,11 +5099,29 @@ fn query_plan_warning_to_js(warning: &QueryPlanWarning) -> &'static str { QueryPlanWarning::VerifyOnlyFilter => "verify_only_filter", QueryPlanWarning::BooleanBranchFallback => "boolean_branch_fallback", QueryPlanWarning::PlanningProbeBudgetExceeded => "planning_probe_budget_exceeded", + QueryPlanWarning::UnknownNodeLabel => "unknown_node_label", + QueryPlanWarning::UnknownEdgeLabel => "unknown_edge_label", } } fn parse_js_node_query(value: &serde_json::Value) -> Result { let object = js_object(value, "node query request")?; + ensure_only_js_fields( + object, + &[ + "labelFilter", + "ids", + "keys", + "filter", + "orderBy", + "limit", + "after", + "allowFullScan", + "where", + "predicates", + ], + "node query request", + )?; let page = PageRequest { limit: parse_js_limit(object, "node query limit")?, after: parse_js_optional_u64_field(object, "after", "node query after")?, @@ -4352,7 +5144,11 @@ fn parse_js_node_query(value: &serde_json::Value) -> Result { }, }; Ok(NodeQuery { - type_id: parse_js_optional_u32_field(object, "typeId", "node query typeId")?, + label_filter: parse_js_node_label_filter_field( + object, + "labelFilter", + "node query labelFilter", + )?, ids: parse_js_optional_u64_array_field(object, "ids", "node query ids")?, keys: parse_js_optional_string_array_field(object, "keys", "node query keys")?, filter: parse_js_node_filter(object, "updatedAt", "node query")?, @@ -4367,6 +5163,42 @@ fn parse_js_node_query(value: &serde_json::Value) -> Result { }) } +fn parse_js_edge_query(value: &serde_json::Value) -> Result { + let object = js_object(value, "edge query request")?; + reject_js_legacy_node_predicate_fields(object, "edge query")?; + let page = PageRequest { + limit: parse_js_limit(object, "edge query limit")?, + after: parse_js_optional_u64_field(object, "after", "edge query after")?, + }; + Ok(EdgeQuery { + label: parse_js_optional_string_field(object, "label", "edge query label")?, + ids: parse_js_optional_u64_array_field(object, "ids", "edge query ids")?, + from_ids: parse_js_optional_u64_array_field(object, "fromIds", "edge query fromIds")?, + to_ids: parse_js_optional_u64_array_field(object, "toIds", "edge query toIds")?, + endpoint_ids: parse_js_optional_u64_array_field( + object, + "endpointIds", + "edge query endpointIds", + )?, + filter: parse_js_edge_filter( + object, + "updatedAt", + "validAt", + "validFrom", + "validTo", + "edge query", + )?, + page, + order: EdgeQueryOrder::EdgeIdAsc, + allow_full_scan: parse_js_optional_bool_field( + object, + "allowFullScan", + "edge query allowFullScan", + )? + .unwrap_or(false), + }) +} + fn parse_js_graph_pattern_query(value: &serde_json::Value) -> Result { let object = js_object(value, "graph pattern request")?; let nodes = match js_non_null_field(object, "nodes") { @@ -4420,9 +5252,26 @@ fn parse_js_graph_pattern_query(value: &serde_json::Value) -> Result Result { let object = js_object(value, "node pattern")?; + ensure_only_js_fields( + object, + &[ + "alias", + "labelFilter", + "ids", + "keys", + "filter", + "where", + "predicates", + ], + "node pattern", + )?; Ok(NodePattern { alias: parse_js_required_string_field(object, "alias", "node pattern alias")?, - type_id: parse_js_optional_u32_field(object, "typeId", "node pattern typeId")?, + label_filter: parse_js_node_label_filter_field( + object, + "labelFilter", + "node pattern labelFilter", + )?, ids: parse_js_optional_u64_array_field(object, "ids", "node pattern ids")?, keys: parse_js_optional_string_array_field(object, "keys", "node pattern keys")?, filter: parse_js_node_filter(object, "updatedAt", "node pattern")?, @@ -4431,12 +5280,7 @@ fn parse_js_node_pattern(value: &serde_json::Value) -> Result { fn parse_js_edge_pattern(value: &serde_json::Value) -> Result { let object = js_object(value, "edge pattern")?; - if object.contains_key("filter") { - return Err(napi::Error::from_reason( - "edge pattern filter is not supported in Phase 24; use edge pattern where or predicates" - .to_string(), - )); - } + reject_js_legacy_node_predicate_fields(object, "edge pattern")?; let direction = match js_non_null_field(object, "direction") { None => Direction::Outgoing, Some(value) => parse_direction(Some(value.as_str().ok_or_else(|| { @@ -4448,15 +5292,73 @@ fn parse_js_edge_pattern(value: &serde_json::Value) -> Result { from_alias: parse_js_required_string_field(object, "fromAlias", "edge pattern fromAlias")?, to_alias: parse_js_required_string_field(object, "toAlias", "edge pattern toAlias")?, direction, - type_filter: parse_js_optional_u32_array_field( + label_filter: parse_js_optional_string_array_field( + object, + "labelFilter", + "edge pattern labelFilter", + )?, + filter: parse_js_edge_filter( object, - "typeFilter", - "edge pattern typeFilter", + "updatedAt", + "validAt", + "validFrom", + "validTo", + "edge pattern", )?, - property_predicates: parse_js_edge_predicates(object, "edge pattern")?, }) } +fn parse_js_node_labels_arg(value: &serde_json::Value, context: &str) -> Result> { + match value { + serde_json::Value::String(label) => Ok(vec![label.clone()]), + serde_json::Value::Array(labels) => labels + .iter() + .enumerate() + .map(|(index, value)| { + value.as_str().map(ToString::to_string).ok_or_else(|| { + napi::Error::from_reason(format!("{}[{}] must be a string", context, index)) + }) + }) + .collect(), + _ => Err(napi::Error::from_reason(format!( + "{} must be a string or string array", + context + ))), + } +} + +fn js_node_label_filter_to_rust(filter: NodeLabelFilter) -> Result { + let mode = match filter.mode.as_str() { + "any" => CoreLabelMatchMode::Any, + "all" => CoreLabelMatchMode::All, + other => { + return Err(napi::Error::from_reason(format!( + "node label filter mode must be 'any' or 'all', got '{}'", + other + ))); + } + }; + Ok(CoreNodeLabelFilter { + labels: filter.labels, + mode, + }) +} + +fn parse_js_node_label_filter_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result> { + let Some(value) = js_non_null_field(object, key) else { + return Ok(None); + }; + let filter = js_object(value, context)?; + ensure_only_js_fields(filter, &["labels", "mode"], context)?; + let labels = parse_js_required_string_array_field(filter, "labels", context)?; + let mode = parse_js_required_string_field(filter, "mode", context)?; + js_node_label_filter_to_rust(NodeLabelFilter { labels, mode }).map(Some) +} + fn reject_js_legacy_node_predicate_fields( object: &serde_json::Map, context: &str, @@ -4491,6 +5393,28 @@ fn parse_js_node_filter( } } +fn parse_js_edge_filter( + object: &serde_json::Map, + updated_at_key: &str, + valid_at_key: &str, + valid_from_key: &str, + valid_to_key: &str, + context: &str, +) -> Result> { + match object.get("filter") { + None | Some(serde_json::Value::Null) => Ok(None), + Some(value) => parse_js_edge_filter_expr( + value, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} filter", context), + ) + .map(Some), + } +} + fn parse_js_node_filter_expr( value: &serde_json::Value, updated_at_key: &str, @@ -4580,58 +5504,148 @@ fn parse_js_node_filter_expr( ))) } -fn parse_js_edge_predicates( - object: &serde_json::Map, +fn parse_js_edge_filter_expr( + value: &serde_json::Value, + updated_at_key: &str, + valid_at_key: &str, + valid_from_key: &str, + valid_to_key: &str, context: &str, -) -> Result> { - let mut predicates = Vec::new(); - if let Some(where_value) = js_non_null_field(object, "where") { - let where_object = js_object(where_value, &format!("{} where", context))?; - for (key, value) in where_object { - predicates.push(parse_js_property_edge_predicate( - key.clone(), - value, - &format!("{} where.{}", context, key), - )?); - } +) -> Result { + let object = js_object(value, context)?; + if object.is_empty() { + return Err(napi::Error::from_reason(format!( + "{} must not be an empty object", + context + ))); + } + + let selectors = [ + "and", + "or", + "not", + "property", + "weight", + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + ] + .iter() + .filter(|field| object.contains_key(**field)) + .count(); + if selectors != 1 { + return Err(napi::Error::from_reason(format!( + "{} must contain exactly one boolean tag or leaf selector", + context + ))); } - if let Some(predicates_value) = js_non_null_field(object, "predicates") { - for (index, value) in js_array(predicates_value, &format!("{} predicates", context))? + reject_js_uppercase_filter_fields(object, context)?; + + if let Some(value) = object.get("and") { + ensure_only_js_fields(object, &["and"], context)?; + let children = js_array(value, &format!("{} and", context))?; + if children.is_empty() { + return Err(napi::Error::from_reason(format!( + "{} and must contain at least one child", + context + ))); + } + return children .iter() .enumerate() - { - let predicate_object = js_object(value, &format!("{} predicates[{}]", context, index))?; - if predicate_object.len() != 1 { - return Err(napi::Error::from_reason(format!( - "{} predicates[{}] must contain exactly one top-level predicate tag", - context, index - ))); - } - let (tag, payload) = predicate_object.iter().next().unwrap(); - match tag.as_str() { - "property" => predicates.push(parse_js_explicit_property_edge_predicate( - payload, - &format!("{} predicates[{}].property", context, index), - )?), - other => { - return Err(napi::Error::from_reason(format!( - "Unknown edge predicate tag '{}'. Only 'property' is supported.", - other - ))); - } - } + .map(|(index, child)| { + parse_js_edge_filter_expr( + child, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} and[{}]", context, index), + ) + }) + .collect::>>() + .map(EdgeFilterExpr::And); + } + if let Some(value) = object.get("or") { + ensure_only_js_fields(object, &["or"], context)?; + let children = js_array(value, &format!("{} or", context))?; + if children.is_empty() { + return Err(napi::Error::from_reason(format!( + "{} or must contain at least one child", + context + ))); } + return children + .iter() + .enumerate() + .map(|(index, child)| { + parse_js_edge_filter_expr( + child, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} or[{}]", context, index), + ) + }) + .collect::>>() + .map(EdgeFilterExpr::Or); + } + if let Some(value) = object.get("not") { + ensure_only_js_fields(object, &["not"], context)?; + return parse_js_edge_filter_expr( + value, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} not", context), + ) + .map(Box::new) + .map(EdgeFilterExpr::Not); + } + if object.contains_key("property") { + return parse_js_property_edge_filter(object, context); + } + if let Some(value) = object.get("weight") { + ensure_only_js_fields(object, &["weight"], context)?; + let range = js_object(value, &format!("{} weight", context))?; + let (lower, upper) = parse_js_f32_range_bounds(range, &format!("{} weight", context))?; + return Ok(EdgeFilterExpr::WeightRange { lower, upper }); + } + if let Some(value) = object.get(updated_at_key) { + ensure_only_js_fields(object, &[updated_at_key], context)?; + let range = js_object(value, &format!("{} {}", context, updated_at_key))?; + let (lower_ms, upper_ms) = + parse_js_i64_range_bounds(range, &format!("{} {}", context, updated_at_key))?; + return Ok(EdgeFilterExpr::UpdatedAtRange { lower_ms, upper_ms }); + } + if let Some(value) = object.get(valid_at_key) { + ensure_only_js_fields(object, &[valid_at_key], context)?; + return Ok(EdgeFilterExpr::ValidAt { + epoch_ms: js_number_to_i64(value, &format!("{} {}", context, valid_at_key))?, + }); + } + if let Some(value) = object.get(valid_from_key) { + ensure_only_js_fields(object, &[valid_from_key], context)?; + let range = js_object(value, &format!("{} {}", context, valid_from_key))?; + let (lower_ms, upper_ms) = + parse_js_i64_range_bounds(range, &format!("{} {}", context, valid_from_key))?; + return Ok(EdgeFilterExpr::ValidFromRange { lower_ms, upper_ms }); + } + if let Some(value) = object.get(valid_to_key) { + ensure_only_js_fields(object, &[valid_to_key], context)?; + let range = js_object(value, &format!("{} {}", context, valid_to_key))?; + let (lower_ms, upper_ms) = + parse_js_i64_range_bounds(range, &format!("{} {}", context, valid_to_key))?; + return Ok(EdgeFilterExpr::ValidToRange { lower_ms, upper_ms }); } - Ok(predicates) -} -fn parse_js_explicit_property_edge_predicate( - value: &serde_json::Value, - context: &str, -) -> Result { - let object = js_object(value, context)?; - let key = parse_js_required_string_field(object, "key", &format!("{} key", context))?; - parse_js_property_edge_predicate(key, value, context) + Err(napi::Error::from_reason(format!( + "{} must contain a valid filter selector", + context + ))) } fn parse_js_property_node_filter( @@ -4704,74 +5718,74 @@ fn parse_js_property_node_filter( unreachable!("operator family count was checked above") } -fn parse_js_property_edge_predicate( - key: String, - value: &serde_json::Value, +fn parse_js_property_edge_filter( + object: &serde_json::Map, context: &str, -) -> Result { - let parsed = parse_js_property_predicate(value, context)?; - Ok(match parsed { - JsParsedPropertyPredicate::Equals(value) => { - EdgePostFilterPredicate::PropertyEquals { key, value } - } - JsParsedPropertyPredicate::Range { lower, upper } => { - EdgePostFilterPredicate::PropertyRange { key, lower, upper } - } - }) -} +) -> Result { + let key = parse_js_required_string_field(object, "property", &format!("{} property", context))?; + if key.is_empty() { + return Err(napi::Error::from_reason(format!( + "{} property must be non-empty", + context + ))); + } -enum JsParsedPropertyPredicate { - Equals(PropValue), - Range { - lower: Option, - upper: Option, - }, -} + let has_range = has_any_js_field(object, &["gt", "gte", "lt", "lte"]); + let families = [ + object.contains_key("eq"), + object.contains_key("in"), + has_range, + object.contains_key("exists"), + object.contains_key("missing"), + ] + .into_iter() + .filter(|present| *present) + .count(); + if families != 1 { + return Err(napi::Error::from_reason(format!( + "{} property filter must specify exactly one operator family", + context + ))); + } -fn parse_js_property_predicate( - value: &serde_json::Value, - context: &str, -) -> Result { - let object = js_object(value, context)?; - match js_non_null_field(object, "op") { - Some(op_value) => match op_value.as_str() { - Some("eq") => { - ensure_no_js_fields(object, &["gt", "gte", "lt", "lte", "eq"], context)?; - let value = object.get("value").ok_or_else(|| { - napi::Error::from_reason(format!("{} eq predicate requires value", context)) - })?; - Ok(JsParsedPropertyPredicate::Equals(json_to_prop_value(value))) - } - Some("range") => { - ensure_no_js_fields(object, &["value", "eq"], context)?; - let (lower, upper) = parse_js_property_range_bounds(object, context)?; - Ok(JsParsedPropertyPredicate::Range { lower, upper }) - } - Some(other) => Err(napi::Error::from_reason(format!( - "Unknown predicate op '{}'. Valid ops are 'eq' and 'range'.", - other - ))), - None => Err(napi::Error::from_reason(format!( - "{} predicate op must be a string", + if let Some(value) = object.get("eq") { + ensure_only_js_fields(object, &["property", "eq"], context)?; + return Ok(EdgeFilterExpr::PropertyEquals { + key, + value: json_to_prop_value(value), + }); + } + if let Some(value) = object.get("in") { + ensure_only_js_fields(object, &["property", "in"], context)?; + let values = js_array(value, &format!("{} in", context))?; + if values.is_empty() { + return Err(napi::Error::from_reason(format!( + "{} in must contain at least one value", context - ))), - }, - None if object.contains_key("eq") => { - ensure_no_js_fields(object, &["value", "gt", "gte", "lt", "lte"], context)?; - Ok(JsParsedPropertyPredicate::Equals(json_to_prop_value( - object.get("eq").unwrap(), - ))) - } - None if has_any_js_field(object, &["gt", "gte", "lt", "lte"]) => { - ensure_no_js_fields(object, &["value", "eq"], context)?; - let (lower, upper) = parse_js_property_range_bounds(object, context)?; - Ok(JsParsedPropertyPredicate::Range { lower, upper }) + ))); } - None => Err(napi::Error::from_reason(format!( - "{} predicate requires op, eq, or range bounds", - context - ))), + return Ok(EdgeFilterExpr::PropertyIn { + key, + values: values.iter().map(json_to_prop_value).collect(), + }); + } + if has_range { + ensure_only_js_fields(object, &["property", "gt", "gte", "lt", "lte"], context)?; + let (lower, upper) = parse_js_property_range_bounds(object, context)?; + return Ok(EdgeFilterExpr::PropertyRange { key, lower, upper }); + } + if object.contains_key("exists") { + ensure_only_js_fields(object, &["property", "exists"], context)?; + require_js_true_field(object, "exists", context)?; + return Ok(EdgeFilterExpr::PropertyExists { key }); + } + if object.contains_key("missing") { + ensure_only_js_fields(object, &["property", "missing"], context)?; + require_js_true_field(object, "missing", context)?; + return Ok(EdgeFilterExpr::PropertyMissing { key }); } + + unreachable!("operator family count was checked above") } fn parse_js_updated_at_filter( @@ -4792,7 +5806,10 @@ fn parse_js_updated_at_filter( fn parse_js_property_range_bounds( object: &serde_json::Map, context: &str, -) -> Result<(Option, Option)> { +) -> Result<( + Option, + Option, +)> { if object.contains_key("gt") && object.contains_key("gte") { return Err(napi::Error::from_reason(format!( "{} range predicate cannot specify both gt and gte", @@ -4806,18 +5823,18 @@ fn parse_js_property_range_bounds( ))); } let lower = if let Some(value) = object.get("gt") { - Some(PropertyRangeBound::Excluded(json_to_prop_value(value))) + Some(CorePropertyRangeBound::Excluded(json_to_prop_value(value))) } else { object .get("gte") - .map(|value| PropertyRangeBound::Included(json_to_prop_value(value))) + .map(|value| CorePropertyRangeBound::Included(json_to_prop_value(value))) }; let upper = if let Some(value) = object.get("lt") { - Some(PropertyRangeBound::Excluded(json_to_prop_value(value))) + Some(CorePropertyRangeBound::Excluded(json_to_prop_value(value))) } else { object .get("lte") - .map(|value| PropertyRangeBound::Included(json_to_prop_value(value))) + .map(|value| CorePropertyRangeBound::Included(json_to_prop_value(value))) }; if lower.is_none() && upper.is_none() { return Err(napi::Error::from_reason(format!( @@ -4887,6 +5904,53 @@ fn parse_js_i64_range_bounds( Ok((lower, upper)) } +fn parse_js_f32_range_bounds( + object: &serde_json::Map, + context: &str, +) -> Result<(Option, Option)> { + if object.contains_key("gt") && object.contains_key("gte") { + return Err(napi::Error::from_reason(format!( + "{} range predicate cannot specify both gt and gte", + context + ))); + } + if object.contains_key("lt") && object.contains_key("lte") { + return Err(napi::Error::from_reason(format!( + "{} range predicate cannot specify both lt and lte", + context + ))); + } + let lower = if let Some(value) = object.get("gt") { + Some(next_up_f32(js_number_to_f32( + value, + &format!("{} gt", context), + )?)) + } else { + object + .get("gte") + .map(|value| js_number_to_f32(value, &format!("{} gte", context))) + .transpose()? + }; + let upper = if let Some(value) = object.get("lt") { + Some(next_down_f32(js_number_to_f32( + value, + &format!("{} lt", context), + )?)) + } else { + object + .get("lte") + .map(|value| js_number_to_f32(value, &format!("{} lte", context))) + .transpose()? + }; + if lower.is_none() && upper.is_none() { + return Err(napi::Error::from_reason(format!( + "{} range predicate requires at least one of gt, gte, lt, or lte", + context + ))); + } + Ok((lower, upper)) +} + fn js_object<'a>( value: &'a serde_json::Value, context: &str, @@ -4948,20 +6012,6 @@ fn parse_js_optional_i64_field( .transpose() } -fn parse_js_optional_u32_field( - object: &serde_json::Map, - key: &str, - context: &str, -) -> Result> { - js_non_null_field(object, key) - .map(|value| { - let value = js_number_to_u64(value, context)?; - u32::try_from(value) - .map_err(|_| napi::Error::from_reason(format!("{} must fit in u32", context))) - }) - .transpose() -} - fn parse_js_optional_bool_field( object: &serde_json::Map, key: &str, @@ -5021,28 +6071,6 @@ fn parse_js_optional_u64_array_field( } } -fn parse_js_optional_u32_array_field( - object: &serde_json::Map, - key: &str, - context: &str, -) -> Result>> { - match js_non_null_field(object, key) { - None => Ok(None), - Some(value) => Ok(Some( - js_array(value, context)? - .iter() - .enumerate() - .map(|(index, value)| { - let value = js_number_to_u64(value, &format!("{}[{}]", context, index))?; - u32::try_from(value).map_err(|_| { - napi::Error::from_reason(format!("{}[{}] must fit in u32", context, index)) - }) - }) - .collect::>>()?, - )), - } -} - fn parse_js_optional_string_array_field( object: &serde_json::Map, key: &str, @@ -5062,6 +6090,24 @@ fn parse_js_optional_string_array_field( } } +fn parse_js_required_string_array_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result> { + let value = js_non_null_field(object, key) + .ok_or_else(|| napi::Error::from_reason(format!("{} {} is required", context, key)))?; + js_array(value, &format!("{} {}", context, key))? + .iter() + .enumerate() + .map(|(index, value)| { + value.as_str().map(ToString::to_string).ok_or_else(|| { + napi::Error::from_reason(format!("{} {}[{}] must be a string", context, key, index)) + }) + }) + .collect() +} + fn js_number_to_u64(value: &serde_json::Value, context: &str) -> Result { let number = value .as_f64() @@ -5086,6 +6132,56 @@ fn js_number_to_i64(value: &serde_json::Value, context: &str) -> Result { Ok(number as i64) } +fn js_number_to_f32(value: &serde_json::Value, context: &str) -> Result { + let number = value + .as_f64() + .ok_or_else(|| napi::Error::from_reason(format!("{} must be a number", context)))?; + if !number.is_finite() || number < f32::MIN as f64 || number > f32::MAX as f64 { + return Err(napi::Error::from_reason(format!( + "{} must be a finite f32 number", + context + ))); + } + let parsed = number as f32; + if parsed.is_nan() { + return Err(napi::Error::from_reason(format!( + "{} must not be NaN", + context + ))); + } + Ok(parsed) +} + +fn next_up_f32(value: f32) -> f32 { + if value == f32::INFINITY { + return value; + } + if value == -0.0 { + return f32::from_bits(1); + } + let bits = value.to_bits(); + if value >= 0.0 { + f32::from_bits(bits + 1) + } else { + f32::from_bits(bits - 1) + } +} + +fn next_down_f32(value: f32) -> f32 { + if value == f32::NEG_INFINITY { + return value; + } + if value == 0.0 { + return -f32::from_bits(1); + } + let bits = value.to_bits(); + if value > 0.0 { + f32::from_bits(bits - 1) + } else { + f32::from_bits(bits + 1) + } +} + fn has_any_js_field(object: &serde_json::Map, fields: &[&str]) -> bool { fields.iter().any(|field| object.contains_key(*field)) } @@ -5106,22 +6202,6 @@ fn ensure_only_js_fields( Ok(()) } -fn ensure_no_js_fields( - object: &serde_json::Map, - fields: &[&str], - context: &str, -) -> Result<()> { - for field in fields { - if object.contains_key(*field) { - return Err(napi::Error::from_reason(format!( - "{} does not accept field '{}'", - context, field - ))); - } - } - Ok(()) -} - fn require_js_true_field( object: &serde_json::Map, field: &str, @@ -5155,7 +6235,7 @@ fn reject_js_uppercase_filter_fields( } fn make_property_range_page_request( - options: Option, + options: Option, ) -> Result { let (limit, after) = match options { Some(options) => (options.limit, options.after), @@ -5168,7 +6248,7 @@ fn make_property_range_page_request( } #[napi(object)] -pub struct JsCompactionProgress { +pub struct CompactionProgress { pub phase: String, pub segments_processed: u32, pub total_segments: u32, @@ -5177,7 +6257,7 @@ pub struct JsCompactionProgress { } #[napi(object)] -pub struct JsCompactionStats { +pub struct CompactionStats { pub segments_merged: u32, pub nodes_kept: i64, pub nodes_removed: i64, @@ -5191,8 +6271,8 @@ pub struct JsCompactionStats { pub edges_auto_pruned: i64, } -impl From for JsCompactionStats { - fn from(s: CompactionStats) -> Self { +impl From for CompactionStats { + fn from(s: CoreCompactionStats) -> Self { // All casts are safe: segment counts are small, and node/edge counts from compaction // never approach i64::MAX in practice (sequential IDs from 1). debug_assert!(s.segments_merged <= u32::MAX as usize); @@ -5202,7 +6282,7 @@ impl From for JsCompactionStats { debug_assert!(s.edges_removed <= i64::MAX as u64); debug_assert!(s.duration_ms <= i64::MAX as u64); debug_assert!(s.output_segment_id <= i64::MAX as u64); - JsCompactionStats { + CompactionStats { segments_merged: s.segments_merged as u32, nodes_kept: s.nodes_kept as i64, nodes_removed: s.nodes_removed as i64, @@ -5217,23 +6297,23 @@ impl From for JsCompactionStats { } #[napi(object)] -pub struct JsPrunePolicy { +pub struct PrunePolicy { /// Prune nodes older than this many milliseconds. Optional. pub max_age_ms: Option, /// Prune nodes with weight <= this threshold. Optional. pub max_weight: Option, - /// Scope to a single node type. Optional. - pub type_id: Option, + /// Scope to a single node label. Optional. + pub label: Option, } #[napi(object)] -pub struct JsNamedPrunePolicy { +pub struct NamedPrunePolicy { pub name: String, - pub policy: JsPrunePolicy, + pub policy: PrunePolicy, } #[napi(object)] -pub struct JsPruneResult { +pub struct PruneResult { /// Number of nodes pruned. pub nodes_pruned: i64, /// Number of edges cascade-deleted. @@ -5241,22 +6321,22 @@ pub struct JsPruneResult { } #[napi(object)] -pub struct JsEdgeInvalidation { +pub struct EdgeInvalidation { pub edge_id: f64, pub valid_to: i64, } #[napi(object)] -pub struct JsGraphPatch { - pub upsert_nodes: Option>, - pub upsert_edges: Option>, - pub invalidate_edges: Option>, +pub struct GraphPatch { + pub upsert_nodes: Option>, + pub upsert_edges: Option>, + pub invalidate_edges: Option>, pub delete_node_ids: Option>, pub delete_edge_ids: Option>, } #[napi(object)] -pub struct JsPatchResult { +pub struct PatchResult { pub node_ids: Float64Array, pub edge_ids: Float64Array, } @@ -5264,36 +6344,36 @@ pub struct JsPatchResult { // --- PPR types --- #[napi(object)] -pub struct JsPprResult { +pub struct PprResult { pub node_ids: Float64Array, pub scores: Float64Array, pub iterations: u32, pub converged: bool, pub algorithm: String, - pub approx: Option, + pub approx: Option, } #[napi(object)] -pub struct JsPprApproxMeta { +pub struct PprApproxMeta { pub residual_tolerance: f64, pub pushes: f64, pub max_remaining_residual: f64, } -fn ppr_result_to_js(r: PprResult) -> Result { +fn ppr_result_to_js(r: CorePprResult) -> Result { let mut node_ids_raw = Vec::with_capacity(r.scores.len()); let mut scores = Vec::with_capacity(r.scores.len()); for (id, score) in &r.scores { node_ids_raw.push(u64_to_f64(*id)?); scores.push(*score); } - Ok(JsPprResult { + Ok(PprResult { node_ids: Float64Array::new(node_ids_raw), scores: Float64Array::new(scores), iterations: r.iterations, converged: r.converged, algorithm: ppr_algorithm_to_js(r.algorithm).to_string(), - approx: r.approx.map(|a| JsPprApproxMeta { + approx: r.approx.map(|a| PprApproxMeta { residual_tolerance: a.residual_tolerance, pushes: a.pushes as f64, max_remaining_residual: a.max_remaining_residual, @@ -5301,13 +6381,21 @@ fn ppr_result_to_js(r: PprResult) -> Result { }) } +fn js_prune_policy_to_rust(policy: PrunePolicy, _context: &str) -> Result { + Ok(CorePrunePolicy { + max_age_ms: policy.max_age_ms.map(|v| v as i64), + max_weight: policy.max_weight.map(|v| v as f32), + label: policy.label, + }) +} + fn js_ppr_options_to_ppr_options( algorithm: Option<&str>, damping_factor: &Option, max_iterations: &Option, epsilon: &Option, approx_residual_tolerance: &Option, - edge_type_filter: &Option>, + edge_label_filter: &Option>, max_results: &Option, ) -> Result { let defaults = PprOptions::default(); @@ -5318,7 +6406,7 @@ fn js_ppr_options_to_ppr_options( epsilon: epsilon.unwrap_or(1e-6), approx_residual_tolerance: approx_residual_tolerance .unwrap_or(defaults.approx_residual_tolerance), - edge_type_filter: edge_type_filter.clone(), + edge_label_filter: edge_label_filter.clone(), max_results: max_results.map(|v| v as usize), }) } @@ -5326,22 +6414,26 @@ fn js_ppr_options_to_ppr_options( // --- Export types --- #[napi(object)] -pub struct JsExportOptions { - pub node_type_filter: Option>, - pub edge_type_filter: Option>, +pub struct ExportOptions { + pub node_label_filter: Option, + pub edge_label_filter: Option>, pub include_weights: Option, } #[napi(object)] -pub struct JsAdjacencyExport { +pub struct AdjacencyExport { pub node_ids: Float64Array, + pub edge_labels: Vec, pub edge_from: Float64Array, pub edge_to: Float64Array, - pub edge_type_ids: Uint32Array, + pub edge_label_indexes: Uint32Array, pub edge_weights: Option, } -fn adjacency_export_to_js(r: AdjacencyExport, include_weights: bool) -> Result { +fn adjacency_export_to_js( + r: CoreAdjacencyExport, + include_weights: bool, +) -> Result { let node_ids_vec: Vec = r .node_ids .iter() @@ -5350,19 +6442,22 @@ fn adjacency_export_to_js(r: AdjacencyExport, include_weights: bool) -> Result Result) -> ExportOptions { +fn js_export_options_to_rust(opts: Option) -> Result { match opts { - None => ExportOptions::default(), - Some(o) => ExportOptions { - node_type_filter: o.node_type_filter, - edge_type_filter: o.edge_type_filter, + None => Ok(CoreExportOptions::default()), + Some(o) => Ok(CoreExportOptions { + node_label_filter: o + .node_label_filter + .map(js_node_label_filter_to_rust) + .transpose()?, + edge_label_filter: o.edge_label_filter, include_weights: o.include_weights.unwrap_or(true), - }, + }), } } -fn js_patch_to_rust(patch: JsGraphPatch) -> napi::Result { - let upsert_nodes: Vec = patch +fn js_patch_to_rust(patch: GraphPatch) -> napi::Result { + let upsert_nodes: Vec = patch .upsert_nodes .unwrap_or_default() .into_iter() - .map(|n| n.into()) - .collect(); + .map(NodeInput::try_into) + .collect::>>()?; - let upsert_edges: Vec = patch + let upsert_edges: Vec = patch .upsert_edges .unwrap_or_default() .into_iter() @@ -5418,7 +6516,7 @@ fn js_patch_to_rust(patch: JsGraphPatch) -> napi::Result { .map(f64_to_u64) .collect::>>()?; - Ok(GraphPatch { + Ok(CoreGraphPatch { upsert_nodes, upsert_edges, invalidate_edges, @@ -5552,14 +6650,14 @@ pub struct CompactProgressOp { } impl Task for CompactProgressOp { - type Output = Option; - type JsValue = Option; + type Output = Option; + type JsValue = Option; fn compute(&mut self) -> napi::Result { let engine = clone_engine_handle(&self.db)?; let tsfn = &self.tsfn; let result = engine.compact_with_progress(|progress| { - let js_progress = JsCompactionProgress { + let js_progress = CompactionProgress { phase: match progress.phase { CompactionPhase::CollectingTombstones => "collecting_tombstones".to_string(), CompactionPhase::MergingNodes => "merging_nodes".to_string(), @@ -5587,17 +6685,17 @@ impl Task for CompactProgressOp { /// Async task for stateful transaction operations. Tickets preserve JS call order /// even when libuv schedules multiple operations on the same transaction in parallel. pub struct TxnAsyncOp { - inner: Arc>>, + inner: Arc>>, order: Arc, ticket: u64, - op: Option std::result::Result + Send>>, + op: Option std::result::Result + Send>>, convert: fn(T) -> napi::Result, } impl TxnAsyncOp { fn new( - txn: &JsWriteTxn, - op: impl FnOnce(&mut WriteTxn) -> std::result::Result + Send + 'static, + txn: &WriteTxn, + op: impl FnOnce(&mut CoreWriteTxn) -> std::result::Result + Send + 'static, convert: fn(T) -> napi::Result, ) -> Result { let ticket = txn.async_order.reserve_ticket()?; @@ -5637,17 +6735,17 @@ impl Task for TxnAsyncOp /// Async task for transaction operations that consume the transaction handle. pub struct TxnAsyncTakeOp { - inner: Arc>>, + inner: Arc>>, order: Arc, ticket: u64, - op: Option std::result::Result + Send>>, + op: Option std::result::Result + Send>>, convert: fn(T) -> napi::Result, } impl TxnAsyncTakeOp { fn new( - txn: &JsWriteTxn, - op: impl FnOnce(&mut WriteTxn) -> std::result::Result + Send + 'static, + txn: &WriteTxn, + op: impl FnOnce(&mut CoreWriteTxn) -> std::result::Result + Send + 'static, convert: fn(T) -> napi::Result, ) -> Result { let ticket = txn.async_order.reserve_ticket()?; @@ -5721,9 +6819,9 @@ fn clone_engine_handle(db: &Arc>>) -> Result(inner: &Arc>>, f: F) -> Result +fn with_txn(inner: &Arc>>, f: F) -> Result where - F: FnOnce(&mut WriteTxn) -> std::result::Result, + F: FnOnce(&mut CoreWriteTxn) -> std::result::Result, { let mut guard = inner .lock() @@ -5734,9 +6832,9 @@ where f(txn).map_err(|e| napi::Error::from_reason(e.to_string())) } -fn with_txn_ref(inner: &Arc>>, f: F) -> Result +fn with_txn_ref(inner: &Arc>>, f: F) -> Result where - F: FnOnce(&WriteTxn) -> std::result::Result, + F: FnOnce(&CoreWriteTxn) -> std::result::Result, { let guard = inner .lock() @@ -5747,9 +6845,9 @@ where f(txn).map_err(|e| napi::Error::from_reason(e.to_string())) } -fn with_txn_take(inner: &Arc>>, f: F) -> Result +fn with_txn_take(inner: &Arc>>, f: F) -> Result where - F: FnOnce(&mut WriteTxn) -> std::result::Result, + F: FnOnce(&mut CoreWriteTxn) -> std::result::Result, { let mut txn = { let mut guard = inner @@ -5762,12 +6860,12 @@ where f(&mut txn).map_err(|e| napi::Error::from_reason(e.to_string())) } -fn js_upsert_node_options(options: Option) -> UpsertNodeOptions { +fn js_upsert_node_options(options: Option) -> CoreUpsertNodeOptions { let (props, weight, dense_vector, sparse_vector) = match options { Some(o) => (o.props, o.weight, o.dense_vector, o.sparse_vector), None => (None, None, None, None), }; - UpsertNodeOptions { + CoreUpsertNodeOptions { props: convert_js_props(props), weight: weight.unwrap_or(1.0) as f32, dense_vector: dense_vector.map(|dv| dv.into_iter().map(|x| x as f32).collect()), @@ -5779,12 +6877,12 @@ fn js_upsert_node_options(options: Option) -> UpsertNodeOpt } } -fn js_upsert_edge_options(options: Option) -> UpsertEdgeOptions { +fn js_upsert_edge_options(options: Option) -> CoreUpsertEdgeOptions { let (props, weight, valid_from, valid_to) = match options { Some(o) => (o.props, o.weight, o.valid_from, o.valid_to), None => (None, None, None, None), }; - UpsertEdgeOptions { + CoreUpsertEdgeOptions { props: convert_js_props(props), weight: weight.unwrap_or(1.0) as f32, valid_from, @@ -5792,96 +6890,117 @@ fn js_upsert_edge_options(options: Option) -> UpsertEdgeOpt } } -fn js_txn_node_ref_to_rust(value: JsTxnNodeRef) -> Result { +fn txn_node_ref_labels_value(label: String) -> serde_json::Value { + serde_json::Value::Array(vec![serde_json::Value::String(label)]) +} + +fn parse_txn_node_ref_label(labels: serde_json::Value, context: &str) -> Result { + let labels = parse_js_node_labels_arg(&labels, context)?; + if labels.len() != 1 { + return Err(napi::Error::from_reason(format!( + "{} must contain exactly one label", + context + ))); + } + Ok(labels.into_iter().next().unwrap()) +} + +fn js_txn_node_ref_to_rust(value: TxnNodeRef) -> Result { let has_id = value.id.is_some(); - let has_key = value.type_id.is_some() || value.key.is_some(); + let has_key = value.labels.is_some() || value.key.is_some(); let has_local = value.local.is_some(); match (has_id, has_key, has_local) { - (true, false, false) => Ok(TxnNodeRef::Id(f64_to_u64(value.id.unwrap())?)), - (false, true, false) => Ok(TxnNodeRef::Key { - type_id: value.type_id.ok_or_else(|| { - napi::Error::from_reason("node key ref requires typeId".to_string()) - })?, + (true, false, false) => Ok(CoreTxnNodeRef::Id(f64_to_u64(value.id.unwrap())?)), + (false, true, false) => Ok(CoreTxnNodeRef::Key { + label: parse_txn_node_ref_label( + value.labels.ok_or_else(|| { + napi::Error::from_reason("node key ref requires labels".to_string()) + })?, + "node key ref labels", + )?, key: value .key .ok_or_else(|| napi::Error::from_reason("node key ref requires key".to_string()))?, }), - (false, false, true) => Ok(TxnNodeRef::Local(TxnLocalRef::Alias(value.local.unwrap()))), + (false, false, true) => Ok(CoreTxnNodeRef::Local(TxnLocalRef::Alias( + value.local.unwrap(), + ))), _ => Err(napi::Error::from_reason( - "node ref must be exactly one of { id }, { typeId, key }, or { local }".to_string(), + "node ref must be exactly one of { id }, { labels, key }, or { local }".to_string(), )), } } -fn js_txn_edge_ref_to_rust(value: JsTxnEdgeRef) -> Result { +fn js_txn_edge_ref_to_rust(value: TxnEdgeRef) -> Result { let has_id = value.id.is_some(); - let has_triple = value.from.is_some() || value.to.is_some() || value.type_id.is_some(); + let has_triple = value.from.is_some() || value.to.is_some() || value.label.is_some(); let has_local = value.local.is_some(); match (has_id, has_triple, has_local) { - (true, false, false) => Ok(TxnEdgeRef::Id(f64_to_u64(value.id.unwrap())?)), - (false, true, false) => Ok(TxnEdgeRef::Triple { + (true, false, false) => Ok(CoreTxnEdgeRef::Id(f64_to_u64(value.id.unwrap())?)), + (false, true, false) => Ok(CoreTxnEdgeRef::Triple { from: js_txn_node_ref_to_rust(value.from.ok_or_else(|| { napi::Error::from_reason("edge triple ref requires from".to_string()) })?)?, to: js_txn_node_ref_to_rust(value.to.ok_or_else(|| { napi::Error::from_reason("edge triple ref requires to".to_string()) })?)?, - type_id: value.type_id.ok_or_else(|| { - napi::Error::from_reason("edge triple ref requires typeId".to_string()) + label: value.label.ok_or_else(|| { + napi::Error::from_reason("edge triple ref requires label".to_string()) })?, }), - (false, false, true) => Ok(TxnEdgeRef::Local(TxnLocalRef::Alias(value.local.unwrap()))), + (false, false, true) => Ok(CoreTxnEdgeRef::Local(TxnLocalRef::Alias( + value.local.unwrap(), + ))), _ => Err(napi::Error::from_reason( - "edge ref must be exactly one of { id }, { from, to, typeId }, or { local }" - .to_string(), + "edge ref must be exactly one of { id }, { from, to, label }, or { local }".to_string(), )), } } -fn txn_node_ref_to_js(value: TxnNodeRef) -> Result { +fn txn_node_ref_to_js(value: CoreTxnNodeRef) -> Result { match value { - TxnNodeRef::Id(id) => Ok(JsTxnNodeRef { + CoreTxnNodeRef::Id(id) => Ok(TxnNodeRef { id: Some(u64_to_f64(id)?), - type_id: None, + labels: None, key: None, local: None, }), - TxnNodeRef::Key { type_id, key } => Ok(JsTxnNodeRef { + CoreTxnNodeRef::Key { label, key } => Ok(TxnNodeRef { id: None, - type_id: Some(type_id), + labels: Some(txn_node_ref_labels_value(label)), key: Some(key), local: None, }), - TxnNodeRef::Local(local) => Ok(JsTxnNodeRef { + CoreTxnNodeRef::Local(local) => Ok(TxnNodeRef { id: None, - type_id: None, + labels: None, key: None, local: txn_local_ref_to_js(local), }), } } -fn txn_edge_ref_to_js(value: TxnEdgeRef) -> Result { +fn txn_edge_ref_to_js(value: CoreTxnEdgeRef) -> Result { match value { - TxnEdgeRef::Id(id) => Ok(JsTxnEdgeRef { + CoreTxnEdgeRef::Id(id) => Ok(TxnEdgeRef { id: Some(u64_to_f64(id)?), from: None, to: None, - type_id: None, + label: None, local: None, }), - TxnEdgeRef::Triple { from, to, type_id } => Ok(JsTxnEdgeRef { + CoreTxnEdgeRef::Triple { from, to, label } => Ok(TxnEdgeRef { id: None, from: Some(txn_node_ref_to_js(from)?), to: Some(txn_node_ref_to_js(to)?), - type_id: Some(type_id), + label: Some(label), local: None, }), - TxnEdgeRef::Local(local) => Ok(JsTxnEdgeRef { + CoreTxnEdgeRef::Local(local) => Ok(TxnEdgeRef { id: None, from: None, to: None, - type_id: None, + label: None, local: txn_local_ref_to_js(local), }), } @@ -5894,11 +7013,11 @@ fn txn_local_ref_to_js(local: TxnLocalRef) -> Option { } } -fn txn_node_view_to_js(view: TxnNodeView) -> Result { - Ok(JsTxnNodeView { +fn txn_node_view_to_js(view: CoreTxnNodeView) -> Result { + Ok(TxnNodeView { id: view.id.map(u64_to_f64).transpose()?, local: view.local.and_then(txn_local_ref_to_js), - type_id: view.type_id, + labels: view.labels, key: view.key, props: props_to_json(view.props), created_at: view.created_at, @@ -5909,7 +7028,7 @@ fn txn_node_view_to_js(view: TxnNodeView) -> Result { .map(|v| v.into_iter().map(|x| x as f64).collect()), sparse_vector: view.sparse_vector.map(|v| { v.into_iter() - .map(|(dimension, value)| JsSparseEntry { + .map(|(dimension, value)| SparseEntry { dimension, value: value as f64, }) @@ -5918,13 +7037,13 @@ fn txn_node_view_to_js(view: TxnNodeView) -> Result { }) } -fn txn_edge_view_to_js(view: TxnEdgeView) -> Result { - Ok(JsTxnEdgeView { +fn txn_edge_view_to_js(view: CoreTxnEdgeView) -> Result { + Ok(TxnEdgeView { id: view.id.map(u64_to_f64).transpose()?, local: view.local.and_then(txn_local_ref_to_js), from: txn_node_ref_to_js(view.from)?, to: txn_node_ref_to_js(view.to)?, - type_id: view.type_id, + label: view.label, props: props_to_json(view.props), created_at: view.created_at, updated_at: view.updated_at, @@ -5934,60 +7053,97 @@ fn txn_edge_view_to_js(view: TxnEdgeView) -> Result { }) } -fn js_txn_operation_to_rust(op: JsTxnOperation) -> Result { - match op.op.as_str() { +fn js_txn_operation_to_rust(value: serde_json::Value) -> Result { + let object = js_object(&value, "transaction operation")?; + let op = parse_js_required_string_field(object, "op", "transaction operation op")?; + match op.as_str() { "upsertNode" => Ok(TxnIntent::UpsertNode { - alias: op.alias, - type_id: op.type_id.ok_or_else(|| { - napi::Error::from_reason("upsertNode requires typeId".to_string()) - })?, - key: op - .key - .ok_or_else(|| napi::Error::from_reason("upsertNode requires key".to_string()))?, - options: UpsertNodeOptions { - props: convert_js_props(op.props), - weight: op.weight.unwrap_or(1.0) as f32, - dense_vector: op - .dense_vector - .map(|v| v.into_iter().map(|x| x as f32).collect()), - sparse_vector: op.sparse_vector.map(|v| { + alias: parse_js_optional_string_field(object, "alias", "upsertNode alias")?, + labels: parse_js_node_labels_arg( + js_non_null_field(object, "labels") + .ok_or_else(|| napi::Error::from_reason("upsertNode requires labels"))?, + "upsertNode labels", + )?, + key: js_non_null_field(object, "key") + .and_then(|value| value.as_str()) + .map(ToString::to_string) + .ok_or_else(|| napi::Error::from_reason("upsertNode requires key"))?, + options: CoreUpsertNodeOptions { + props: convert_js_props(parse_js_optional_props_field( + object, + "props", + "upsertNode props", + )?), + weight: parse_js_optional_f64_field(object, "weight", "upsertNode weight")? + .unwrap_or(1.0) as f32, + dense_vector: parse_js_optional_f64_array_field( + object, + "denseVector", + "upsertNode denseVector", + )? + .map(|v| v.into_iter().map(|x| x as f32).collect()), + sparse_vector: parse_js_optional_sparse_vector_field( + object, + "sparseVector", + "upsertNode sparseVector", + )? + .map(|v| { v.into_iter() .map(|e| (e.dimension, e.value as f32)) .collect() }), }, }), - "upsertEdge" => { - Ok(TxnIntent::UpsertEdge { - alias: op.alias, - from: js_txn_node_ref_to_rust(op.from.ok_or_else(|| { - napi::Error::from_reason("upsertEdge requires from".to_string()) - })?)?, - to: js_txn_node_ref_to_rust(op.to.ok_or_else(|| { - napi::Error::from_reason("upsertEdge requires to".to_string()) - })?)?, - type_id: op.type_id.ok_or_else(|| { - napi::Error::from_reason("upsertEdge requires typeId".to_string()) - })?, - options: UpsertEdgeOptions { - props: convert_js_props(op.props), - weight: op.weight.unwrap_or(1.0) as f32, - valid_from: op.valid_from, - valid_to: op.valid_to, - }, - }) - } + "upsertEdge" => Ok(TxnIntent::UpsertEdge { + alias: parse_js_optional_string_field(object, "alias", "upsertEdge alias")?, + from: js_txn_node_ref_to_rust(parse_js_required_txn_node_ref_field( + object, + "from", + "upsertEdge from", + )?)?, + to: js_txn_node_ref_to_rust(parse_js_required_txn_node_ref_field( + object, + "to", + "upsertEdge to", + )?)?, + label: js_non_null_field(object, "label") + .and_then(|value| value.as_str()) + .map(ToString::to_string) + .ok_or_else(|| napi::Error::from_reason("upsertEdge requires label"))?, + options: CoreUpsertEdgeOptions { + props: convert_js_props(parse_js_optional_props_field( + object, + "props", + "upsertEdge props", + )?), + weight: parse_js_optional_f64_field(object, "weight", "upsertEdge weight")? + .unwrap_or(1.0) as f32, + valid_from: parse_js_optional_i64_field( + object, + "validFrom", + "upsertEdge validFrom", + )?, + valid_to: parse_js_optional_i64_field(object, "validTo", "upsertEdge validTo")?, + }, + }), "deleteNode" => Ok(TxnIntent::DeleteNode { - target: js_txn_node_ref_to_rust(txn_target_as_node(op.target)?)?, + target: js_txn_node_ref_to_rust(txn_target_as_node( + parse_js_required_txn_target_field(object, "target", "deleteNode target")?, + )?)?, }), "deleteEdge" => Ok(TxnIntent::DeleteEdge { - target: js_txn_edge_ref_to_rust(txn_target_as_edge(op.target)?)?, + target: js_txn_edge_ref_to_rust(txn_target_as_edge( + parse_js_required_txn_target_field(object, "target", "deleteEdge target")?, + )?)?, }), "invalidateEdge" => Ok(TxnIntent::InvalidateEdge { - target: js_txn_edge_ref_to_rust(txn_target_as_edge(op.target)?)?, - valid_to: op.valid_to.ok_or_else(|| { - napi::Error::from_reason("invalidateEdge requires validTo".to_string()) - })?, + target: js_txn_edge_ref_to_rust(txn_target_as_edge( + parse_js_required_txn_target_field(object, "target", "invalidateEdge target")?, + )?)?, + valid_to: js_non_null_field(object, "validTo") + .map(|value| js_number_to_i64(value, "invalidateEdge validTo")) + .transpose()? + .ok_or_else(|| napi::Error::from_reason("invalidateEdge requires validTo"))?, }), other => Err(napi::Error::from_reason(format!( "invalid transaction op '{}'", @@ -5996,28 +7152,172 @@ fn js_txn_operation_to_rust(op: JsTxnOperation) -> Result { } } -fn txn_target_as_node(target: Option) -> Result { - let target = target.ok_or_else(|| napi::Error::from_reason("operation requires target"))?; - Ok(JsTxnNodeRef { +fn parse_js_optional_f64_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result> { + js_non_null_field(object, key) + .map(|value| { + value + .as_f64() + .ok_or_else(|| napi::Error::from_reason(format!("{} must be a number", context))) + }) + .transpose() +} + +fn parse_js_optional_props_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result>> { + js_non_null_field(object, key) + .map(|value| { + Ok(js_object(value, context)? + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect()) + }) + .transpose() +} + +fn parse_js_optional_f64_array_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result>> { + js_non_null_field(object, key) + .map(|value| { + js_array(value, context)? + .iter() + .enumerate() + .map(|(index, value)| { + value.as_f64().ok_or_else(|| { + napi::Error::from_reason(format!("{}[{}] must be a number", context, index)) + }) + }) + .collect() + }) + .transpose() +} + +fn parse_js_optional_sparse_vector_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result>> { + js_non_null_field(object, key) + .map(|value| { + js_array(value, context)? + .iter() + .enumerate() + .map(|(index, value)| { + let entry = js_object(value, &format!("{}[{}]", context, index))?; + let dimension = js_non_null_field(entry, "dimension") + .ok_or_else(|| { + napi::Error::from_reason(format!( + "{}[{}].dimension is required", + context, index + )) + }) + .and_then(|value| { + let dimension = js_number_to_u64( + value, + &format!("{}[{}].dimension", context, index), + )?; + u32::try_from(dimension).map_err(|_| { + napi::Error::from_reason(format!( + "{}[{}].dimension is too large", + context, index + )) + }) + })?; + let value = js_non_null_field(entry, "value") + .and_then(|value| value.as_f64()) + .ok_or_else(|| { + napi::Error::from_reason(format!( + "{}[{}].value must be a number", + context, index + )) + })?; + Ok(SparseEntry { dimension, value }) + }) + .collect() + }) + .transpose() +} + +fn parse_js_required_txn_node_ref_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result { + parse_js_txn_node_ref( + js_non_null_field(object, key) + .ok_or_else(|| napi::Error::from_reason(format!("{} is required", context)))?, + context, + ) +} + +fn parse_js_txn_node_ref(value: &serde_json::Value, context: &str) -> Result { + let object = js_object(value, context)?; + Ok(TxnNodeRef { + id: parse_js_optional_f64_field(object, "id", &format!("{} id", context))?, + labels: js_non_null_field(object, "labels").cloned(), + key: parse_js_optional_string_field(object, "key", &format!("{} key", context))?, + local: parse_js_optional_string_field(object, "local", &format!("{} local", context))?, + }) +} + +fn parse_js_required_txn_target_field( + object: &serde_json::Map, + key: &str, + context: &str, +) -> Result { + parse_js_txn_target( + js_non_null_field(object, key) + .ok_or_else(|| napi::Error::from_reason(format!("{} is required", context)))?, + context, + ) +} + +fn parse_js_txn_target(value: &serde_json::Value, context: &str) -> Result { + let object = js_object(value, context)?; + Ok(TxnEdgeOrNodeRef { + id: parse_js_optional_f64_field(object, "id", &format!("{} id", context))?, + labels: js_non_null_field(object, "labels").cloned(), + label: parse_js_optional_string_field(object, "label", &format!("{} label", context))?, + key: parse_js_optional_string_field(object, "key", &format!("{} key", context))?, + local: parse_js_optional_string_field(object, "local", &format!("{} local", context))?, + from: js_non_null_field(object, "from") + .map(|value| parse_js_txn_node_ref(value, &format!("{} from", context))) + .transpose()?, + to: js_non_null_field(object, "to") + .map(|value| parse_js_txn_node_ref(value, &format!("{} to", context))) + .transpose()?, + }) +} + +fn txn_target_as_node(target: TxnEdgeOrNodeRef) -> Result { + Ok(TxnNodeRef { id: target.id, - type_id: target.type_id, + labels: target.labels, key: target.key, local: target.local, }) } -fn txn_target_as_edge(target: Option) -> Result { - let target = target.ok_or_else(|| napi::Error::from_reason("operation requires target"))?; - Ok(JsTxnEdgeRef { +fn txn_target_as_edge(target: TxnEdgeOrNodeRef) -> Result { + Ok(TxnEdgeRef { id: target.id, from: target.from, to: target.to, - type_id: target.type_id, + label: target.label, local: target.local, }) } -fn txn_commit_result_to_js(result: TxnCommitResult) -> Result { +fn txn_commit_result_to_js(result: CoreTxnCommitResult) -> Result { let node_aliases = result .local_node_ids .into_iter() @@ -6034,7 +7334,7 @@ fn txn_commit_result_to_js(result: TxnCommitResult) -> Result TxnLocalRef::Slot(_) => None, }) .collect::>>()?; - Ok(JsTxnCommitResult { + Ok(TxnCommitResult { node_ids: ids_to_float64_array(&result.node_ids)?, edge_ids: ids_to_float64_array(&result.edge_ids)?, node_aliases, @@ -6173,17 +7473,17 @@ fn secondary_index_state_to_js(state: SecondaryIndexState) -> &'static str { } } -fn secondary_index_kind_to_js(kind: &SecondaryIndexKind) -> (String, Option) { +fn secondary_index_kind_to_js(kind: &CoreSecondaryIndexKind) -> (String, Option) { match kind { - SecondaryIndexKind::Equality => ("equality".to_string(), None), - SecondaryIndexKind::Range { domain } => ( + CoreSecondaryIndexKind::Equality => ("equality".to_string(), None), + CoreSecondaryIndexKind::Range { domain } => ( "range".to_string(), Some(secondary_index_domain_to_js(*domain).to_string()), ), } } -fn js_secondary_index_kind_to_rust(kind: JsSecondaryIndexKind) -> Result { +fn js_secondary_index_kind_to_rust(kind: SecondaryIndexKind) -> Result { match kind.kind.as_str() { "equality" => { if kind.domain.is_some() { @@ -6191,9 +7491,9 @@ fn js_secondary_index_kind_to_rust(kind: JsSecondaryIndexKind) -> Result Ok(SecondaryIndexKind::Range { + "range" => Ok(CoreSecondaryIndexKind::Range { domain: parse_secondary_index_range_domain(kind.domain.as_deref())?, }), other => Err(napi::Error::from_reason(format!( @@ -6251,13 +7551,13 @@ fn prop_value_to_js_numeric_parts(value: &PropValue) -> Result<(f64, String)> { } } -fn js_property_range_bound_to_rust(bound: &JsPropertyRangeBound) -> Result { +fn js_property_range_bound_to_rust(bound: &PropertyRangeBound) -> Result { let domain = parse_secondary_index_range_domain(Some(bound.domain.as_str()))?; let value = js_numeric_to_prop_value(bound.value, domain)?; if bound.inclusive.unwrap_or(true) { - Ok(PropertyRangeBound::Included(value)) + Ok(CorePropertyRangeBound::Included(value)) } else { - Ok(PropertyRangeBound::Excluded(value)) + Ok(CorePropertyRangeBound::Excluded(value)) } } @@ -6414,10 +7714,10 @@ impl<'a> BinaryReader<'a> { Ok(slice) } - fn read_utf8(&mut self, len: usize) -> napi::Result<&'a str> { + fn read_utf8_with_context(&mut self, len: usize, context: &str) -> napi::Result<&'a str> { let bytes = self.read_bytes(len)?; std::str::from_utf8(bytes) - .map_err(|e| napi::Error::from_reason(format!("Invalid UTF-8 in key: {}", e))) + .map_err(|e| napi::Error::from_reason(format!("Invalid UTF-8 in {}: {}", context, e))) } } @@ -6441,27 +7741,90 @@ fn decode_props_json(reader: &mut BinaryReader) -> napi::Result. +const NODE_BATCH_MAGIC: &[u8; 4] = b"OGNB"; +const EDGE_BATCH_MAGIC: &[u8; 4] = b"OGEB"; +const NODE_BINARY_BATCH_VERSION: u16 = 2; +const EDGE_BINARY_BATCH_VERSION: u16 = 1; +const BINARY_BATCH_HEADER_LEN: usize = 10; +const MAX_NODE_LABELS_PER_NODE: usize = 10; + +fn decode_binary_batch_header( + reader: &mut BinaryReader<'_>, + expected_magic: &[u8; 4], + expected_version: u16, + context: &str, +) -> napi::Result { + let magic = reader.read_bytes(4)?; + if magic != expected_magic { + return Err(napi::Error::from_reason(format!( + "Invalid {} binary batch format: missing magic header", + context + ))); + } + let version = reader.read_u16_le()?; + if version != expected_version { + if context == "node" && version == 1 { + return Err(napi::Error::from_reason( + "Unsupported node binary batch version 1; OGNB v1 single-label buffers are no longer supported, expected version 2".to_string(), + )); + } + return Err(napi::Error::from_reason(format!( + "Unsupported {} binary batch version {}; expected {}", + context, version, expected_version + ))); + } + Ok(reader.read_u32_le()? as usize) +} + +/// Decode a binary buffer into a Vec. /// /// Format (little-endian): -/// [count: u32] +/// [magic: 4 bytes "OGNB"][version: u16 = 2][count: u32] /// per node: -/// [type_id: u32][weight: f32][key_len: u16][key: utf8][props_len: u32][props: json utf8] -fn decode_node_batch(buf: &[u8]) -> napi::Result> { +/// [label_count: u8] repeated [label_len: u16][label: utf8][weight: f32] +/// [key_len: u16][key: utf8][props_len: u32][props: json utf8] +fn decode_node_batch(buf: &[u8]) -> napi::Result> { let mut reader = BinaryReader::new(buf); - let count = reader.read_u32_le()? as usize; - // Cap allocation: minimum node record is 14 bytes (type_id + weight + key_len + props_len) - let max_possible = buf.len().saturating_sub(4) / 14; + let count = decode_binary_batch_header( + &mut reader, + NODE_BATCH_MAGIC, + NODE_BINARY_BATCH_VERSION, + "node", + )?; + // Cap allocation: minimum v2 node record is 14 bytes. + let max_possible = buf.len().saturating_sub(BINARY_BATCH_HEADER_LEN) / 14; let mut inputs = Vec::with_capacity(count.min(max_possible)); for _ in 0..count { - let type_id = reader.read_u32_le()?; + let label_count = reader.read_bytes(1)?[0] as usize; + if label_count == 0 || label_count > MAX_NODE_LABELS_PER_NODE { + return Err(napi::Error::from_reason( + "Binary node label count must be between 1 and 10".to_string(), + )); + } + let mut labels = Vec::with_capacity(label_count); + for label_index in 0..label_count { + let label_len = reader.read_u16_le()? as usize; + if label_len == 0 || label_len > 255 { + return Err(napi::Error::from_reason(format!( + "Binary node label {} length must be between 1 and 255 bytes", + label_index + ))); + } + labels.push( + reader + .read_utf8_with_context(label_len, "node label")? + .to_string(), + ); + } let weight = reader.read_f32_le()?; let key_len = reader.read_u16_le()? as usize; - let key = reader.read_utf8(key_len)?.to_string(); + let key = reader + .read_utf8_with_context(key_len, "node key")? + .to_string(); let props = decode_props_json(&mut reader)?; - inputs.push(NodeInput { - type_id, + inputs.push(CoreNodeInput { + labels, key, props, weight, @@ -6481,35 +7844,48 @@ fn decode_node_batch(buf: &[u8]) -> napi::Result> { Ok(inputs) } -/// Decode a binary buffer into a Vec. +/// Decode a binary buffer into a Vec. /// /// Format (little-endian): -/// [count: u32] +/// [magic: 4 bytes "OGEB"][version: u16 = 1][count: u32] /// per edge: -/// [from: u64][to: u64][type_id: u32][weight: f32] +/// [from: u64][to: u64][label_len: u16][label: utf8][weight: f32] /// [valid_from: i64][valid_to: i64] /// [props_len: u32][props: json utf8] /// /// Sentinel values: valid_from=0 → None (engine default), valid_to=0 → None (engine default). -fn decode_edge_batch(buf: &[u8]) -> napi::Result> { +fn decode_edge_batch(buf: &[u8]) -> napi::Result> { let mut reader = BinaryReader::new(buf); - let count = reader.read_u32_le()? as usize; - // Cap allocation: minimum edge record is 36 bytes (from + to + type_id + weight + valid_from + valid_to + props_len) - let max_possible = buf.len().saturating_sub(4) / 36; + let count = decode_binary_batch_header( + &mut reader, + EDGE_BATCH_MAGIC, + EDGE_BINARY_BATCH_VERSION, + "edge", + )?; + // Cap allocation: minimum edge record is 35 bytes. + let max_possible = buf.len().saturating_sub(BINARY_BATCH_HEADER_LEN) / 35; let mut inputs = Vec::with_capacity(count.min(max_possible)); for _ in 0..count { let from = reader.read_u64_le()?; let to = reader.read_u64_le()?; - let type_id = reader.read_u32_le()?; + let label_len = reader.read_u16_le()? as usize; + if label_len == 0 || label_len > 255 { + return Err(napi::Error::from_reason( + "Binary edge label length must be between 1 and 255 bytes".to_string(), + )); + } + let label = reader + .read_utf8_with_context(label_len, "edge label")? + .to_string(); let weight = reader.read_f32_le()?; let valid_from_raw = reader.read_i64_le()?; let valid_to_raw = reader.read_i64_le()?; let props = decode_props_json(&mut reader)?; - inputs.push(EdgeInput { + inputs.push(CoreEdgeInput { from, to, - type_id, + label, props, weight, valid_from: if valid_from_raw == 0 { @@ -6536,17 +7912,17 @@ fn decode_edge_batch(buf: &[u8]) -> napi::Result> { Ok(inputs) } -fn neighbor_entries_to_js(entries: Vec) -> Result> { +fn neighbor_entries_to_js(entries: Vec) -> Result> { entries.iter().map(neighbor_to_js_entry).collect() } fn convert_batch_result( - map: impl IntoIterator)>, -) -> Result> { - let mut entries: Vec = map + map: impl IntoIterator)>, +) -> Result> { + let mut entries: Vec = map .into_iter() .map(|(query_id, neighbors)| { - Ok(JsNeighborBatchEntry { + Ok(NeighborBatchEntry { query_node_id: u64_to_f64(query_id)?, neighbors: neighbor_entries_to_js(neighbors)?, }) diff --git a/overgraph-node/tsconfig.types.json b/overgraph-node/tsconfig.types.json new file mode 100644 index 0000000..797847f --- /dev/null +++ b/overgraph-node/tsconfig.types.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "skipLibCheck": false, + "types": ["node"] + }, + "include": [ + "index.d.ts", + "query-types.d.ts", + "__test__/types/declarations.ts" + ] +} diff --git a/overgraph-python/Cargo.toml b/overgraph-python/Cargo.toml index 96a1737..ffcd2f0 100644 --- a/overgraph-python/Cargo.toml +++ b/overgraph-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "overgraph-python" -version = "0.7.0" +version = "0.8.0" edition = "2021" description = "Python bindings for OverGraph" diff --git a/overgraph-python/README.md b/overgraph-python/README.md index 29b1788..8b042c6 100644 --- a/overgraph-python/README.md +++ b/overgraph-python/README.md @@ -2,7 +2,7 @@

OverGraph

An absurdly fast embedded graph database with built-in vector search.
- Pure Rust. Sub-microsecond reads. Native connectors for Node.js and Python.
+ Pure Rust. Sub-microsecond reads. Native Python connector.
Built for AI agent memory, knowledge graphs, RAG pipelines, and semantic search.

@@ -22,7 +22,7 @@ OverGraph is a graph database that runs inside your process. No server, no netwo I built it because I wanted a graph database that was genuinely fast. Not "fast for a database," but fast enough that you forget it's there. Node lookups in 34 nanoseconds. Neighbor traversals in 2 microseconds. Batch writes at 1.29M+ nodes per second. And I wanted graph structure and vector similarity to live together in one engine. No separate vector database, no external index, no synchronization headaches. -It's written entirely in Rust and it ships native connectors for Node.js (napi-rs) and Python (PyO3) so you can use it from whatever you're building in. +It's written entirely in Rust and ships a native Python connector built with PyO3, so Python applications call the engine in-process without a server or REST layer. ## Built for @@ -40,13 +40,13 @@ It's written entirely in Rust and it ships native connectors for Node.js (napi-r - **Graph + vectors in one engine.** Dense HNSW and sparse inverted indexes live alongside graph adjacency indexes in the same storage engine. Vector search can be scoped to graph neighborhoods ("find similar nodes within 2 hops of X") without a second database or a synchronization layer. - **Rich graph primitives.** Weighted nodes and edges, temporal validity windows, exponential decay scoring, automatic retention policies. Model relationships that evolve over time, and let the graph clean up what's no longer relevant. - **Fast where it matters.** Node lookups in ~34ns. Neighbor traversal in ~2μs. Batch writes at 1.29M+ nodes/sec. The storage engine is a log-structured merge tree with mmap'd immutable segments, so reads never block writes. -- **Explicit write transactions.** Stage ordered node and edge mutations locally, read your own staged writes, then commit atomically with optimistic conflict detection. Available in Rust, Node.js, and Python. -- **Three languages, one engine.** Rust core with native bindings for Node.js (napi-rs) and Python (PyO3). Not a wrapper around a REST API. Actual FFI into the same Rust engine with minimal overhead. +- **Explicit write transactions.** Stage ordered node and edge mutations locally, read your own staged writes, then commit atomically with optimistic conflict detection through the Python API. +- **Native Python, one engine.** Rust core with PyO3 bindings. Not a wrapper around a REST API. Actual FFI into the same Rust engine with minimal overhead. - **Full queries as functions.** Use regular APIs for everything: `find_nodes` for direct property lookups, `query_node_ids` / `query_nodes` for full boolean node queries, and `query_pattern` for bounded graph pattern matching. No query strings to parse, escape, or generate. ## Performance -All numbers from a real benchmark suite running on the Rust core (group-commit durability mode, small profile: 10K nodes / 50K edges). Full methodology and reproducibility guide in [`docs/04-quality/Benchmark-Methodology.md`](docs/04-quality/Benchmark-Methodology.md). +All numbers from a real benchmark suite running on the Rust core (group-commit durability mode, small profile: 10K nodes / 50K edges). Full methodology and reproducibility guide in [`docs/04-quality/Benchmark-Methodology.md`](../docs/04-quality/Benchmark-Methodology.md). | Operation | Latency | Throughput | |---|---|---| @@ -57,52 +57,35 @@ All numbers from a real benchmark suite running on the Rust core (group-commit d | `top_k_neighbors` | 17.5 μs | 81K ops/s | | `personalized_pagerank` | 254 μs | 4.3K ops/s | -Node.js and Python connectors add minimal overhead. Batch operations are especially efficient because they amortize the FFI boundary cost. Full cross-language comparison in the [launch benchmark pack](docs/04-quality/reports/2026-03-04-launch-pack-parity/). +The Python connector adds minimal overhead. Batch operations are especially efficient because they amortize the FFI boundary cost. Full methodology is covered in the [launch benchmark pack](../docs/04-quality/reports/2026-03-04-launch-pack-parity/). ## Install Prebuilt binaries are available for macOS (ARM + Intel), Linux (x64), and Windows (x64). No Rust toolchain required. -**Python** ```bash pip install overgraph ``` -**Node.js** -```bash -npm install overgraph -``` - -**Rust** -```bash -cargo add overgraph -``` - ## Quick start -### Python - ```python from overgraph import OverGraph -USER = 1 -PROJECT = 2 -CREATED = 10 - with OverGraph.open("./my-graph", dense_vector_dimension=384) as db: - # Embeddings come from your model (sentence-transformers, OpenAI, etc.) - # dense: model.encode("Alice is an engineer") -> [f32; 384] - # sparse: splade.encode("Alice is an engineer") -> [(token_id, weight), ...] - alice = db.upsert_node(USER, "user:alice", + # Embeddings come from your model. Dense vectors must match the configured dimension. + # Sparse vectors use (dimension, weight) pairs from your sparse encoder. + # Also accepts multiple labels: ["User", "Engineer"] + alice = db.upsert_node("User", "alice", props={"name": "Alice"}, dense_vector=alice_embedding, sparse_vector=alice_sparse) - project = db.upsert_node(PROJECT, "project:overgraph", + project = db.upsert_node("Project", "overgraph", dense_vector=project_embedding, sparse_vector=project_sparse) - db.upsert_edge(alice, project, CREATED) + db.upsert_edge(alice, project, "CREATED") # Hybrid vector search scoped to a graph neighborhood hits = db.vector_search("hybrid", k=10, @@ -115,120 +98,9 @@ with OverGraph.open("./my-graph", dense_vector_dimension=384) as db: print(f"node {hit.node_id} score {hit.score:.4f}") ``` -### Node.js - -```javascript -import { OverGraph } from 'overgraph'; - -const USER = 1; -const PROJECT = 2; -const CREATED = 10; - -const db = OverGraph.open('./my-graph', { - denseVector: { dimension: 384 }, -}); - -// Embeddings come from your model (sentence-transformers, OpenAI, etc.) -// dense: model.encode("Alice is an engineer") -> Float32Array(384) -// sparse: splade.encode("Alice is an engineer") -> [{ dimension, value }, ...] -const alice = db.upsertNode(USER, 'user:alice', { - props: { name: 'Alice' }, - denseVector: aliceEmbedding, - sparseVector: aliceSparse, -}); - -const project = db.upsertNode(PROJECT, 'project:overgraph', { - denseVector: projectEmbedding, - sparseVector: projectSparse, -}); - -db.upsertEdge(alice, project, CREATED); - -// Hybrid vector search scoped to a graph neighborhood -const hits = db.vectorSearch('hybrid', { - k: 10, - denseQuery: queryEmbedding, - sparseQuery: querySparse, - scope: { startNodeId: alice, maxDepth: 3 }, -}); - -hits.forEach(h => console.log(`node ${h.nodeId} score ${h.score.toFixed(4)}`)); -db.close(); -``` - -### Rust - -```rust -use overgraph::*; -use std::collections::BTreeMap; -use std::path::Path; - -const USER: u32 = 1; -const PROJECT: u32 = 2; -const CREATED: u32 = 10; - -fn main() -> Result<(), Box> { - let opts = DbOptions { - dense_vector: Some(DenseVectorConfig { - dimension: 384, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }), - ..Default::default() - }; - let mut db = DatabaseEngine::open(Path::new("./my-graph"), &opts)?; - - // Embeddings come from your model (sentence-transformers, OpenAI, etc.) - // dense: model.encode("Alice is an engineer") -> Vec with 384 dims - // sparse: splade.encode("Alice is an engineer") -> Vec<(u32, f32)> - let mut props = BTreeMap::new(); - props.insert("name".into(), PropValue::String("Alice".into())); - let alice = db.upsert_node(USER, "user:alice", UpsertNodeOptions { - props, - dense_vector: Some(alice_embedding), - sparse_vector: Some(alice_sparse), - ..Default::default() - })?; - - let project = db.upsert_node(PROJECT, "project:overgraph", UpsertNodeOptions { - dense_vector: Some(project_embedding), - sparse_vector: Some(project_sparse), - ..Default::default() - })?; - - db.upsert_edge(alice, project, CREATED, UpsertEdgeOptions::default())?; - - // Hybrid vector search: dense + sparse with graph scoping - let hits = db.vector_search(&VectorSearchRequest { - mode: VectorSearchMode::Hybrid, - dense_query: Some(query_embedding), - sparse_query: Some(query_sparse), - k: 10, // required: 0 returns empty - type_filter: Some(vec![USER, PROJECT]), // default: None (no filtering) - ef_search: Some(200), // default: 128 - scope: Some(VectorSearchScope { // default: None (search all nodes) - start_node_id: alice, - max_depth: 3, - direction: Direction::Outgoing, // default: Outgoing - edge_type_filter: Some(vec![CREATED]), // default: None (all edge types) - at_epoch: None, // default: None (current time) - }), - dense_weight: Some(0.7), // default: 1.0 - sparse_weight: Some(0.3), // default: 1.0 - fusion_mode: Some(FusionMode::ReciprocalRankFusion), // default: WeightedRankFusion - })?; - for hit in &hits { - println!("node {} score {:.4}", hit.node_id, hit.score); - } - - db.close()?; - Ok(()) -} -``` - ### Async support -Both Python and Node.js connectors include full async variants of every API. Python provides `AsyncOverGraph` with native `asyncio` support. Node.js methods have `Async` suffixed variants (e.g. `upsertNodeAsync`, `vectorSearchAsync`). +The Python connector includes `AsyncOverGraph`, an asyncio wrapper that exposes async variants of every API and runs sync operations in a thread pool via `asyncio.to_thread()`. ## Features @@ -236,14 +108,14 @@ Both Python and Node.js connectors include full async variants of every API. Pyt - **Dense vector search.** Attach `f32` embedding vectors to any node. HNSW indexes are built per segment at flush time for fast approximate nearest neighbor search. Supports cosine, Euclidean, and dot-product distance metrics. One dense vector space per DB with configurable dimension. - **Sparse vector search.** Attach sparse vectors (dimension-value pairs) for keyword-weighted retrieval. Works with pre-computed sparse embeddings from models like SPLADE or BGE-M3. Inverted posting-list indexes for exact dot-product scoring. - **Hybrid search.** Combine dense and sparse results with built-in fusion modes: weighted rank fusion, reciprocal rank fusion, or weighted score fusion. Adjustable `dense_weight` and `sparse_weight` for tuning the blend. -- **Graph-scoped search.** Scope vector search to a graph neighborhood: "find the 10 most similar nodes within 3 hops of node X." Uses traversal-based reachable-node filtering with edge-type and temporal support. Combine graph structure with vector similarity in a single query. -- **Zero overhead when unused.** Nodes without vectors pay no storage or runtime cost. Vector index files are only created for segments that contain vectors. +- **Graph-scoped search.** Scope vector search to a graph neighborhood: "find the 10 most similar nodes within 3 hops of node X." Uses traversal-based reachable-node filtering with edge-label and temporal support. Combine graph structure with vector similarity in a single query. +- **Zero overhead when unused.** Nodes without vectors pay no storage or runtime cost. Vector source payloads and accelerators are only created for segments that contain vectors. ### Core graph operations -- **Upsert semantics.** Nodes are keyed by `(type_id, key)`. Upsert the same key twice and you get an update, not a duplicate. Edges can optionally enforce uniqueness on `(from, to, type_id)`. +- **Upsert semantics.** Nodes carry one or more labels and one key. Each live `(label, key)` membership is unique, so a multi-label node owns the same key in every label it carries. Python `upsert_node` accepts either a label string or a list of labels, while dict-based batch and patch inputs use `"labels": [...]`. Upserting a key that resolves to the same node through the supplied labels updates it; if supplied label memberships resolve to different nodes, the write is rejected as a conflict. Edges can optionally enforce uniqueness on `(from, to, label)`. - **Batch operations.** `batch_upsert_nodes` and `batch_upsert_edges` amortize WAL and memtable overhead. `get_nodes` and `get_nodes_by_keys` do batched reads with sorted merge-walks instead of per-item lookups. There's also a packed binary format for maximum write throughput. - **Atomic graph patch.** `graph_patch` lets you upsert nodes, upsert edges, delete nodes, delete edges, and invalidate edges in a single atomic operation. -- **Explicit transactions.** `begin_write_txn()` / `beginWriteTxn()` gives you ordered staging, rollback, read-own-writes point lookups, local aliases, atomic commit, and clean conflict errors for retry loops. +- **Explicit transactions.** `begin_write_txn()` gives you ordered staging, rollback, read-own-writes point lookups, local aliases, atomic commit, and clean conflict errors for retry loops. ### Temporal edges - **Validity windows.** Edges have optional `valid_from` and `valid_to` timestamps. Query at any point in time with the `at_epoch` parameter and only see edges that were valid at that moment. @@ -251,30 +123,30 @@ Both Python and Node.js connectors include full async variants of every API. Pyt - **Decay scoring.** Pass a `decay_lambda` to neighbor queries and edge weights are automatically scaled by `exp(-lambda * age_hours)`. Recent connections matter more. ### Queries and traversal -- **Neighbors and bounded traversal.** `neighbors()` handles 1-hop expansion and returns normal neighbor entry collections in every connector; `traverse()` covers deterministic breadth-first traversal across arbitrary depth windows with optional edge-type filtering, emission-only node-type filtering, and traversal-specific pagination. +- **Neighbors and bounded traversal.** `neighbors()` handles 1-hop expansion and returns normal neighbor entry collections in the Python API; `traverse()` covers deterministic breadth-first traversal across arbitrary depth windows with optional edge-label filtering, emission-only node-label filtering, and traversal-specific pagination. - **Depth slices without special-case APIs.** Exact depth-2 traversals are expressed as `traverse(start, 2, min_depth=2)`, so 2-hop use cases stay available without a separate public method family. - **Top-K neighbors.** Get the K highest-scoring neighbors by weight, recency, or decay-adjusted score. -- **Personalized PageRank.** Run PPR from seed nodes to find the most relevant nodes in the graph. Rust, Node.js, and Python expose both exact power-iteration PPR and a much faster approximate forward-push mode for seed-centric retrieval workloads. +- **Personalized PageRank.** Run PPR from seed nodes with `personalized_pagerank()`, using either exact power iteration or the faster approximate forward-push mode for seed-centric retrieval workloads. - **Subgraph extraction.** Pull out a connected subgraph up to N hops deep. Good for building local context windows. - **Shortest path.** BFS (unweighted) or bidirectional Dijkstra (weighted). `is_connected` for fast reachability checks. `all_shortest_paths` when there are ties. -- **Connected components.** `connected_components()` returns a global WCC labelling (union-find, near-linear). `component_of(node)` returns the members of a single node's component via BFS. Both support edge-type, node-type, and temporal filters. +- **Connected components.** `connected_components()` returns a global WCC labelling (union-find, near-linear). `component_of(node)` returns the members of a single node's component via BFS. Both support edge-label, node-label, and temporal filters. - **Degree counts.** Count edges, sum weights, and compute averages without materializing neighbor lists. Batch `degrees` for bulk analysis. - **Direct property queries.** `find_nodes` and `find_nodes_paged` do focused equality lookups. `find_nodes_range` and `find_nodes_range_paged` do numeric range scans with exact bound and cursor semantics. -- **Optional property indexes.** Declare equality or numeric range indexes only where they pay off. Use `ensure_node_property_index`, `list_node_property_indexes`, and `drop_node_property_index` to manage them. Public query APIs stay index-transparent: when a matching declaration is `Ready`, OverGraph uses the declaration-backed path; otherwise it falls back to the same public API. -- **Full query APIs.** `query_node_ids`, `query_nodes`, `query_pattern`, and explain APIs combine IDs, keys, types, property equality/IN/range/exists/missing filters, updated-at ranges, and bounded graph patterns without a query string. OverGraph chooses the cheapest legal path with available indexes and planner stats, then verifies results against visible records. +- **Optional property indexes.** Declare node or edge equality/range indexes only where they pay off. Use `ensure_node_property_index` / `ensure_edge_property_index`, list APIs, and drop APIs to manage them. Public query APIs stay index-transparent: when a matching declaration is `Ready`, OverGraph uses the declaration-backed path; otherwise it falls back to the same public API. +- **Full query APIs.** `query_node_ids`, `query_nodes`, `query_edge_ids`, `query_edges`, `query_pattern`, and explain APIs combine IDs, keys, explicit node label filters, edge labels, endpoint constraints, property equality/IN/range/exists/missing filters, edge metadata filters, updated-at ranges, and bounded graph patterns without a query string. OverGraph chooses the cheapest legal path with available indexes and planner stats, then verifies results against visible records. - **Time-range queries.** Find nodes created or updated within a time window. Sorted timestamp index for efficient range scans. ### Pagination ID-keyed collection APIs use keyset pagination with `limit` and `after`. `traverse()` uses `limit` plus a traversal cursor keyed by `(depth, node_id)`. No offset-based pagination. Traversal cursors assume the same query arguments and a stable logical graph state; strict snapshot isolation across intervening writes is not promised. ### Retention and pruning -- **Manual prune.** Drop nodes older than X, below weight Y, or matching type Z. Incident edges cascade automatically. +- **Manual prune.** Drop nodes older than X, below weight Y, or matching a label. Incident edges cascade automatically. - **Named prune policies.** Register policies like `"short_term_memory"` that run automatically during compaction. Nodes matching any policy are invisible to reads immediately (lazy expiration) and cleaned up during the next compaction pass. ### Storage engine - **Write-ahead log.** Every mutation hits the WAL before the memtable. Crash recovery replays the WAL on startup. - **Configurable durability.** `Immediate` mode fsyncs every write for maximum safety. `GroupCommit` mode (default) batches fsyncs on a 50ms timer for ~20x better write throughput with at most one timer interval of data at risk. -- **Background compaction.** Segments are merged automatically when thresholds are met. Compaction runs on a background thread and never blocks reads or writes. Uses metadata sidecars for fast filtered merging without full record decoding. +- **Background compaction.** Segments are merged automatically when thresholds are met. Compaction runs on a background thread and never blocks reads or writes. Uses packed metadata payloads for fast filtered merging without full record decoding. - **Bulk ingest mode.** Temporarily disable auto-compaction during large write bursts with `ingest_mode()`, then call `end_ingest()` to compact accumulated segments and restore normal behavior. This favors ingest throughput over read performance during the ingest window. - **mmap'd reads.** Immutable segments are memory-mapped. The OS page cache handles caching. Reads never block writes. - **Portable databases.** Each database is a self-contained directory. `cp -r ./my-db /backup/my-db` and you're done. @@ -285,73 +157,55 @@ OverGraph uses a log-structured storage engine purpose-built from scratch in pur **Write path:** Mutations are appended to a write-ahead log and applied to an in-memory memtable. When the memtable reaches its threshold, it's frozen and flushed to disk as an immutable segment in the background. Writes continue unblocked against a fresh memtable. Each segment ships with pre-built adjacency indexes (inbound and outbound), optional declared property-index sidecars, optional advisory planner statistics, optional signed degree-delta sidecars for degree/weight fast paths, and, when the segment contains vectors, HNSW and sparse posting-list indexes. -**Read path:** Queries check the memtable first (freshest data), then merge results across immutable segments using the per-segment indexes. Because every segment carries its own adjacency index, a neighbor query is a handful of index lookups, not a scan across sorted keys. Vector search follows the same model: memtable candidates are found by exact brute-force scan, segment candidates via HNSW or posting-list indexes, then the engine merges and deduplicates across all sources. Property equality and numeric range queries stay index-transparent too: if a matching optional property-index declaration is `Ready`, the engine uses the declaration-backed path, otherwise it falls back to a type-scoped scan through the same public API. Pagination uses early termination to avoid unnecessary work. +**Read path:** Queries check the memtable first (freshest data), then merge results across immutable segments using the per-segment indexes. Because every segment carries its own adjacency index, a neighbor query is a handful of index lookups, not a scan across sorted keys. Vector search follows the same model: memtable candidates are found by exact brute-force scan, segment candidates via HNSW or posting-list indexes, then the engine merges and deduplicates across all sources. Property equality and numeric range queries stay index-transparent too: if a matching optional property-index declaration is `Ready`, the engine uses the declaration-backed path, otherwise it falls back to a label-scoped scan through the same public API. Pagination uses early termination to avoid unnecessary work. -**Compaction:** A background thread merges older segments together, applying tombstones, prune policies, and deduplication. The compaction path uses metadata sidecars to plan merges and raw-copies winning records without full deserialization, then rebuilds unified indexes from metadata. This includes rebuilding HNSW and sparse posting-list indexes for the merged output. Fewer segments after compaction means fewer index lookups per query, but even before compaction, reads are fast because every segment is self-indexed. +**Compaction:** A background thread merges older segments together, applying tombstones, prune policies, and deduplication. The compaction path uses packed metadata payloads to plan merges and raw-copies winning records without full deserialization, then rebuilds unified indexes from metadata. This includes rebuilding HNSW and sparse posting-list indexes for the merged output. Fewer segments after compaction means fewer index lookups per query, but even before compaction, reads are fast because every segment is self-indexed. **On-disk layout:** ``` my-graph/ manifest.current # atomic checkpoint (JSON) - data.wal # append-only write-ahead log + wal_0.wal # append-only write-ahead log generation segments/ seg_0001/ - nodes.dat # node records - edges.dat # edge records - adj_out.idx # outgoing adjacency index - adj_in.idx # incoming adjacency index - key_index.dat # (type_id, key) -> node_id - type_index.dat # type_id -> [id...] - tombstones.dat # deleted IDs - secondary_indexes/ # optional declared property-index sidecars - planner_stats.dat # optional advisory planner statistics + segment_manifest.dat # component table of contents + segment.core # packed immutable core records, metadata, and maintained indexes + secondary_indexes/ # optional declared equality/range property-index sidecars + planner_stats.dat # optional advisory planner statistics, refreshable degree_delta.dat # optional signed degree deltas for fast degree/weight reads - node_dense_vectors.dat # dense vector blob (when present) - node_sparse_vectors.dat # sparse vector blob (when present) - dense_hnsw_graph.dat # HNSW graph index (when present) - sparse_postings.dat # sparse posting lists (when present) - node_vector_meta.dat # vector offsets/lengths per node + dense_hnsw_meta.dat # optional dense-vector HNSW metadata + dense_hnsw_graph.dat # optional dense-vector HNSW graph + sparse_posting_index.dat # optional sparse-vector posting index + sparse_postings.dat # optional sparse-vector posting lists seg_0002/ ... ``` -For a deeper dive, see the [architecture overview](docs/architecture-overview.md). +`segment.core` is addressed through `segment_manifest.dat`. It contains the logical +node/edge record payloads, tombstones, key/label-token/timestamp/triple indexes, adjacency +indexes/postings, node/edge metadata, vector source-truth blobs, and immutable edge +metadata indexes. Refreshable optional accelerators stay outside the packed core so +they can be rebuilt or dropped without rewriting source data. + +For a deeper dive, see the [architecture overview](../docs/architecture-overview.md). ## Documentation - **[overgraph.io/docs](https://overgraph.io/docs)** - full documentation, getting started guide, and API reference. -- **[API Reference](../docs/api-reference.md)** - every method, parameter, type, and return value across Python, Node.js, and Rust. +- **[API Reference](../docs/api-reference.md)** - every Python method, parameter, type, and return value. - **[Roadmap](../docs/roadmap.md)** - where OverGraph is headed and what's already shipped. ## Running the benchmarks ```bash -# Rust -scripts/bench/run-rust.sh --profile small --warmup 20 --iters 80 - -# Node.js -scripts/bench/run-node.sh --profile small --warmup 20 --iters 80 - -# Python scripts/bench/run-python.sh --profile small --warmup 20 --iters 80 ``` -Benchmark methodology, FAQ, and reproducibility instructions are in [`docs/04-quality/`](docs/04-quality/). +Benchmark methodology, FAQ, and reproducibility instructions are in [`docs/04-quality/`](../docs/04-quality/). ## Building from source ```bash -# Rust core -cargo build --release -cargo test - -# Node.js connector -cd overgraph-node -npm install -npm run build -npm test - -# Python connector cd overgraph-python pip install maturin maturin develop diff --git a/overgraph-python/pyproject.toml b/overgraph-python/pyproject.toml index 337fb5a..209a20c 100644 --- a/overgraph-python/pyproject.toml +++ b/overgraph-python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "overgraph" -version = "0.7.0" +version = "0.8.0" description = "An absurdly fast embedded graph database for Python. Sub-microsecond reads, pure Rust core." requires-python = ">=3.9" license = {text = "MIT OR Apache-2.0"} diff --git a/overgraph-python/python/overgraph/__init__.py b/overgraph-python/python/overgraph/__init__.py index c7c24a9..1c5b0bb 100644 --- a/overgraph-python/python/overgraph/__init__.py +++ b/overgraph-python/python/overgraph/__init__.py @@ -83,7 +83,7 @@ def _request_list(value: Sequence[Any], field: str) -> list[dict[str, Any]]: @dataclass(frozen=True) class NodeQueryRequest: - type_id: int | None = None + label_filter: Mapping[str, Any] | None = None ids: Sequence[int] | None = None keys: Sequence[str] | None = None filter: Mapping[str, Any] | None = None @@ -94,7 +94,7 @@ class NodeQueryRequest: def to_dict(self) -> dict[str, Any]: data: dict[str, Any] = {} - _set_if_not_none(data, "type_id", self.type_id) + _set_if_not_none(data, "label_filter", _optional_dict(self.label_filter)) _set_if_not_none(data, "ids", _optional_int_list(self.ids, "ids")) _set_if_not_none(data, "keys", _optional_str_list(self.keys, "keys")) _set_if_not_none(data, "filter", _optional_dict(self.filter)) @@ -109,17 +109,47 @@ def to_dict(self) -> dict[str, Any]: QueryNodeRequest = NodeQueryRequest +@dataclass(frozen=True) +class EdgeQueryRequest: + label: str | None = None + ids: Sequence[int] | None = None + from_ids: Sequence[int] | None = None + to_ids: Sequence[int] | None = None + endpoint_ids: Sequence[int] | None = None + filter: Mapping[str, Any] | None = None + limit: int | None = None + after: int | None = None + allow_full_scan: bool = False + + def to_dict(self) -> dict[str, Any]: + data: dict[str, Any] = {} + _set_if_not_none(data, "label", self.label) + _set_if_not_none(data, "ids", _optional_int_list(self.ids, "ids")) + _set_if_not_none(data, "from_ids", _optional_int_list(self.from_ids, "from_ids")) + _set_if_not_none(data, "to_ids", _optional_int_list(self.to_ids, "to_ids")) + _set_if_not_none(data, "endpoint_ids", _optional_int_list(self.endpoint_ids, "endpoint_ids")) + _set_if_not_none(data, "filter", _optional_dict(self.filter)) + _set_if_not_none(data, "limit", self.limit) + _set_if_not_none(data, "after", self.after) + if self.allow_full_scan: + data["allow_full_scan"] = True + return data + + +QueryEdgeRequest = EdgeQueryRequest + + @dataclass(frozen=True) class GraphNodePattern: alias: str - type_id: int | None = None + label_filter: Mapping[str, Any] | None = None ids: Sequence[int] | None = None keys: Sequence[str] | None = None filter: Mapping[str, Any] | None = None def to_dict(self) -> dict[str, Any]: data: dict[str, Any] = {"alias": self.alias} - _set_if_not_none(data, "type_id", self.type_id) + _set_if_not_none(data, "label_filter", _optional_dict(self.label_filter)) _set_if_not_none(data, "ids", _optional_int_list(self.ids, "ids")) _set_if_not_none(data, "keys", _optional_str_list(self.keys, "keys")) _set_if_not_none(data, "filter", _optional_dict(self.filter)) @@ -132,9 +162,8 @@ class GraphEdgePattern: to_alias: str alias: str | None = None direction: str | None = None - type_filter: Sequence[int] | None = None - where: Mapping[str, Any] | None = None - predicates: Sequence[Mapping[str, Any]] | None = None + label_filter: Sequence[str] | None = None + filter: Mapping[str, Any] | None = None def to_dict(self) -> dict[str, Any]: data: dict[str, Any] = { @@ -143,13 +172,12 @@ def to_dict(self) -> dict[str, Any]: } _set_if_not_none(data, "alias", self.alias) _set_if_not_none(data, "direction", self.direction) - _set_if_not_none(data, "type_filter", _optional_int_list(self.type_filter, "type_filter")) - _set_if_not_none(data, "where", _optional_dict(self.where)) _set_if_not_none( data, - "predicates", - _optional_mapping_list(self.predicates, "predicates"), + "label_filter", + _optional_str_list(self.label_filter, "label_filter"), ) + _set_if_not_none(data, "filter", _optional_dict(self.filter)) return data diff --git a/overgraph-python/python/overgraph/__init__.pyi b/overgraph-python/python/overgraph/__init__.pyi index e2e9ae9..71348dd 100644 --- a/overgraph-python/python/overgraph/__init__.pyi +++ b/overgraph-python/python/overgraph/__init__.pyi @@ -4,14 +4,41 @@ from typing import Any, Callable, Mapping, Sequence IntList = list[int] | tuple[int, ...] StrList = list[str] | tuple[str, ...] +NodeLabels = str | list[str] | tuple[str, ...] MappingList = list[Mapping[str, Any]] | tuple[Mapping[str, Any], ...] QueryNodeFilter = Mapping[str, Any] +QueryEdgeFilter = Mapping[str, Any] +NodeLabelFilter = Mapping[str, Any] # ============================================================ # Data types # ============================================================ -class PyDbStats: +class ScrubReport: + total_components_checked: int + total_components_ok: int + total_components_failed: int + total_bytes_digested: int + duration_ms: int + @property + def segments(self) -> list[SegmentScrubResult]: ... + def __repr__(self) -> str: ... + +class SegmentScrubResult: + segment_id: int + components_ok: int + bytes_digested: int + @property + def findings(self) -> list[ComponentScrubFinding]: ... + def __repr__(self) -> str: ... + +class ComponentScrubFinding: + component_kind: str + finding_type: str + detail: str + def __repr__(self) -> str: ... + +class DbStats: pending_wal_bytes: int segment_count: int node_tombstone_count: int @@ -26,21 +53,23 @@ class PyDbStats: oldest_retained_wal_generation_id: int def __repr__(self) -> str: ... -class PyNodeRecord: +class NodeView: id: int - type_id: int + labels: list[str] key: str props: dict[str, Any] weight: float + dense_vector: list[float] | None + sparse_vector: list[tuple[int, float]] | None created_at: int updated_at: int def __repr__(self) -> str: ... -class PyEdgeRecord: +class EdgeView: id: int from_id: int to_id: int - type_id: int + label: str props: dict[str, Any] weight: float valid_from: int @@ -49,42 +78,42 @@ class PyEdgeRecord: updated_at: int def __repr__(self) -> str: ... -class PyPatchResult: +class PatchResult: node_ids: list[int] edge_ids: list[int] def __repr__(self) -> str: ... -class PyTxnCommitResult: +class TxnCommitResult: node_ids: list[int] edge_ids: list[int] node_aliases: dict[str, int] edge_aliases: dict[str, int] def __repr__(self) -> str: ... -class PyNeighborEntry: +class NeighborEntry: node_id: int edge_id: int - edge_type_id: int + label: str weight: float valid_from: int valid_to: int def __repr__(self) -> str: ... -class PyTraversalHit: +class TraversalHit: node_id: int depth: int via_edge_id: int | None score: float | None def __repr__(self) -> str: ... -class PyVectorHit: +class VectorHit: node_id: int score: float def __repr__(self) -> str: ... -class PyNodePropertyIndexInfo: +class NodePropertyIndexInfo: index_id: int - type_id: int + label: str prop_key: str kind: str domain: str | None @@ -92,56 +121,76 @@ class PyNodePropertyIndexInfo: last_error: str | None def __repr__(self) -> str: ... -class PyPropertyRangeBound: +class EdgePropertyIndexInfo: + index_id: int + label: str + prop_key: str + kind: str + domain: str | None + state: str + last_error: str | None + def __repr__(self) -> str: ... + +class PropertyRangeBound: value: int | float inclusive: bool domain: str def __init__(self, value: int | float, *, inclusive: bool = True, domain: str) -> None: ... def __repr__(self) -> str: ... -class PyPropertyRangeCursor: +class PropertyRangeCursor: value: int | float node_id: int domain: str def __init__(self, value: int | float, node_id: int, *, domain: str) -> None: ... def __repr__(self) -> str: ... -class PyTraversalCursor: +class TraversalCursor: depth: int last_node_id: int def __init__(self, depth: int, last_node_id: int) -> None: ... def __repr__(self) -> str: ... -class PyShortestPath: +class ShortestPath: nodes: list[int] edges: list[int] total_cost: float def __repr__(self) -> str: ... -class PySubgraph: - nodes: list[PyNodeRecord] - edges: list[PyEdgeRecord] +class Subgraph: + nodes: list[NodeView] + edges: list[EdgeView] def __repr__(self) -> str: ... -class PyPruneResult: +class PruneResult: nodes_pruned: int edges_pruned: int def __repr__(self) -> str: ... -class PyNamedPrunePolicy: +class NamedPrunePolicy: name: str max_age_ms: int | None max_weight: float | None - type_id: int | None + label: str | None + def __repr__(self) -> str: ... + +class NodeLabelInfo: + label: str + label_id: int def __repr__(self) -> str: ... -class PySegmentInfo: +class EdgeLabelInfo: + label: str + label_id: int + def __repr__(self) -> str: ... + +class SegmentInfo: id: int node_count: int edge_count: int def __repr__(self) -> str: ... -class PyCompactionStats: +class CompactionStats: segments_merged: int nodes_kept: int nodes_removed: int @@ -153,7 +202,7 @@ class PyCompactionStats: edges_auto_pruned: int def __repr__(self) -> str: ... -class PyCompactionProgress: +class CompactionProgress: phase: str segments_processed: int total_segments: int @@ -165,88 +214,91 @@ class IdArray: """Lazy sequence wrapper. Data stays in Rust, converted on access.""" def __len__(self) -> int: ... def __getitem__(self, index: int) -> int: ... - def __iter__(self) -> PyIdArrayIter: ... + def __iter__(self) -> IdArrayIter: ... def __bool__(self) -> bool: ... def __contains__(self, val: int) -> bool: ... def __eq__(self, other: object) -> bool: ... def __repr__(self) -> str: ... def to_list(self) -> list[int]: ... -class PyIdArrayIter: - def __iter__(self) -> PyIdArrayIter: ... +class IdArrayIter: + def __iter__(self) -> IdArrayIter: ... def __next__(self) -> int: ... -class PyIdPageResult: +class IdPageResult: items: IdArray next_cursor: int | None def __len__(self) -> int: ... def __bool__(self) -> bool: ... def __repr__(self) -> str: ... -class PyNodePageResult: - items: list[PyNodeRecord] +class NodePageResult: + items: list[NodeView] next_cursor: int | None def __len__(self) -> int: ... def __bool__(self) -> bool: ... def __repr__(self) -> str: ... -class PyEdgePageResult: - items: list[PyEdgeRecord] +class EdgePageResult: + items: list[EdgeView] next_cursor: int | None def __len__(self) -> int: ... def __bool__(self) -> bool: ... def __repr__(self) -> str: ... -class PyNeighborPageResult: - items: list[PyNeighborEntry] +class NeighborPageResult: + items: list[NeighborEntry] next_cursor: int | None def __len__(self) -> int: ... def __bool__(self) -> bool: ... def __repr__(self) -> str: ... -class PyPropertyRangePageResult: +class PropertyRangePageResult: items: IdArray - next_cursor: PyPropertyRangeCursor | None + next_cursor: PropertyRangeCursor | None def __len__(self) -> int: ... def __bool__(self) -> bool: ... def __repr__(self) -> str: ... -class PyTraversalPageResult: - items: list[PyTraversalHit] - next_cursor: PyTraversalCursor | None +class TraversalPageResult: + items: list[TraversalHit] + next_cursor: TraversalCursor | None def __len__(self) -> int: ... def __bool__(self) -> bool: ... def __repr__(self) -> str: ... -class PyPprResult: +class PprResult: node_ids: list[int] scores: list[float] iterations: int converged: bool algorithm: str - approx: PyPprApproxMeta | None + approx: PprApproxMeta | None def __repr__(self) -> str: ... -class PyPprApproxMeta: +class PprApproxMeta: residual_tolerance: float pushes: int max_remaining_residual: float def __repr__(self) -> str: ... -class PyExportEdge: +class ExportEdge: from_id: int to_id: int - type_id: int - weight: float + edge_label_index: int + weight: float | None def __repr__(self) -> str: ... -class PyAdjacencyExport: +class AdjacencyExport: node_ids: list[int] - edges: list[PyExportEdge] + node_labels: list[str] + node_label_indexes: list[list[int]] + edge_labels: list[str] + edges: list[ExportEdge] def __repr__(self) -> str: ... class NodeQueryRequest: - type_id: int | None + label_filter: NodeLabelFilter | None ids: IntList | None keys: StrList | None filter: QueryNodeFilter | None @@ -256,7 +308,7 @@ class NodeQueryRequest: allow_full_scan: bool def __init__( self, - type_id: int | None = None, + label_filter: NodeLabelFilter | None = None, ids: IntList | None = None, keys: StrList | None = None, filter: QueryNodeFilter | None = None, @@ -269,16 +321,42 @@ class NodeQueryRequest: QueryNodeRequest = NodeQueryRequest +class EdgeQueryRequest: + label: str | None + ids: IntList | None + from_ids: IntList | None + to_ids: IntList | None + endpoint_ids: IntList | None + filter: QueryEdgeFilter | None + limit: int | None + after: int | None + allow_full_scan: bool + def __init__( + self, + label: str | None = None, + ids: IntList | None = None, + from_ids: IntList | None = None, + to_ids: IntList | None = None, + endpoint_ids: IntList | None = None, + filter: QueryEdgeFilter | None = None, + limit: int | None = None, + after: int | None = None, + allow_full_scan: bool = False, + ) -> None: ... + def to_dict(self) -> dict[str, Any]: ... + +QueryEdgeRequest = EdgeQueryRequest + class GraphNodePattern: alias: str - type_id: int | None + label_filter: NodeLabelFilter | None ids: IntList | None keys: StrList | None filter: QueryNodeFilter | None def __init__( self, alias: str, - type_id: int | None = None, + label_filter: NodeLabelFilter | None = None, ids: IntList | None = None, keys: StrList | None = None, filter: QueryNodeFilter | None = None, @@ -290,18 +368,16 @@ class GraphEdgePattern: to_alias: str alias: str | None direction: str | None - type_filter: IntList | None - where: Mapping[str, Any] | None - predicates: MappingList | None + label_filter: StrList | None + filter: QueryEdgeFilter | None def __init__( self, from_alias: str, to_alias: str, alias: str | None = None, direction: str | None = None, - type_filter: IntList | None = None, - where: Mapping[str, Any] | None = None, - predicates: MappingList | None = None, + label_filter: StrList | None = None, + filter: QueryEdgeFilter | None = None, ) -> None: ... def to_dict(self) -> dict[str, Any]: ... @@ -340,12 +416,23 @@ class OverGraph: exc_val: BaseException | None = None, exc_tb: Any = None, ) -> bool: ... - def stats(self) -> PyDbStats: ... + def stats(self) -> DbStats: ... + def scrub(self) -> ScrubReport: ... + + # Catalog + def ensure_node_label(self, label: str) -> int: ... + def ensure_edge_label(self, label: str) -> int: ... + def get_node_label_id(self, label: str) -> int | None: ... + def get_edge_label_id(self, label: str) -> int | None: ... + def get_node_label(self, label_id: int) -> str | None: ... + def get_edge_label(self, label_id: int) -> str | None: ... + def list_node_labels(self) -> list[NodeLabelInfo]: ... + def list_edge_labels(self) -> list[EdgeLabelInfo]: ... # Single CRUD def upsert_node( self, - type_id: int, + labels: NodeLabels, key: str, *, props: dict[str, Any] | None = None, @@ -353,52 +440,60 @@ class OverGraph: dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None, ) -> int: ... + def add_node_label(self, node_id: int, label: str) -> bool: ... + def remove_node_label(self, node_id: int, label: str) -> bool: ... def upsert_edge( self, from_id: int, to_id: int, - type_id: int, + label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, valid_from: int | None = None, valid_to: int | None = None, ) -> int: ... - def get_node(self, node_id: int) -> PyNodeRecord | None: ... - def get_edge(self, edge_id: int) -> PyEdgeRecord | None: ... - def get_node_by_key(self, type_id: int, key: str) -> PyNodeRecord | None: ... - def get_edge_by_triple(self, from_id: int, to_id: int, type_id: int) -> PyEdgeRecord | None: ... + def get_node(self, node_id: int) -> NodeView | None: ... + def get_edge(self, edge_id: int) -> EdgeView | None: ... + def get_node_by_key(self, label: str, key: str) -> NodeView | None: ... + def get_edge_by_triple(self, from_id: int, to_id: int, label: str) -> EdgeView | None: ... def delete_node(self, node_id: int) -> None: ... def delete_edge(self, edge_id: int) -> None: ... - def invalidate_edge(self, edge_id: int, valid_to: int) -> PyEdgeRecord | None: ... + def invalidate_edge(self, edge_id: int, valid_to: int) -> EdgeView | None: ... # Batch def batch_upsert_nodes(self, nodes: list[dict[str, Any]]) -> list[int]: ... def batch_upsert_edges(self, edges: list[dict[str, Any]]) -> list[int]: ... - def get_nodes(self, node_ids: list[int]) -> list[PyNodeRecord | None]: ... - def get_nodes_by_keys(self, keys: list[tuple[int, str]]) -> list[PyNodeRecord | None]: ... - def get_edges(self, edge_ids: list[int]) -> list[PyEdgeRecord | None]: ... - def graph_patch(self, patch: dict[str, Any]) -> PyPatchResult: ... - def begin_write_txn(self) -> PyWriteTxn: ... + def get_nodes(self, node_ids: list[int]) -> list[NodeView | None]: ... + def get_nodes_by_keys(self, keys: list[dict[str, Any]]) -> list[NodeView | None]: ... + def get_edges(self, edge_ids: list[int]) -> list[EdgeView | None]: ... + def graph_patch(self, patch: dict[str, Any]) -> PatchResult: ... + def begin_write_txn(self) -> WriteTxn: ... # Queries - def find_nodes(self, type_id: int, prop_key: str, prop_value: Any) -> IdArray: ... - def query_node_ids(self, request: dict[str, Any] | NodeQueryRequest) -> PyIdPageResult: ... - def query_nodes(self, request: dict[str, Any] | NodeQueryRequest) -> PyNodePageResult: ... + def find_nodes(self, label: str, prop_key: str, prop_value: Any) -> IdArray: ... + def query_node_ids(self, request: dict[str, Any] | NodeQueryRequest) -> IdPageResult: ... + def query_nodes(self, request: dict[str, Any] | NodeQueryRequest) -> NodePageResult: ... + def query_edge_ids(self, request: dict[str, Any] | EdgeQueryRequest) -> IdPageResult: ... + def query_edges(self, request: dict[str, Any] | EdgeQueryRequest) -> EdgePageResult: ... def query_pattern(self, request: dict[str, Any] | GraphPatternRequest) -> dict[str, Any]: ... def explain_node_query(self, request: dict[str, Any] | NodeQueryRequest) -> dict[str, Any]: ... + def explain_edge_query(self, request: dict[str, Any] | EdgeQueryRequest) -> dict[str, Any]: ... def explain_pattern_query(self, request: dict[str, Any] | GraphPatternRequest) -> dict[str, Any]: ... - def ensure_node_property_index(self, type_id: int, prop_key: str, kind: str, *, domain: str | None = None) -> PyNodePropertyIndexInfo: ... - def drop_node_property_index(self, type_id: int, prop_key: str, kind: str, *, domain: str | None = None) -> bool: ... - def list_node_property_indexes(self) -> list[PyNodePropertyIndexInfo]: ... - def nodes_by_type(self, type_id: int) -> IdArray: ... - def edges_by_type(self, type_id: int) -> IdArray: ... - def get_nodes_by_type(self, type_id: int) -> list[PyNodeRecord]: ... - def get_edges_by_type(self, type_id: int) -> list[PyEdgeRecord]: ... - def count_nodes_by_type(self, type_id: int) -> int: ... - def count_edges_by_type(self, type_id: int) -> int: ... - def find_nodes_by_time_range(self, type_id: int, from_ms: int, to_ms: int) -> IdArray: ... - def find_nodes_range(self, type_id: int, prop_key: str, lower: PyPropertyRangeBound | None = None, upper: PyPropertyRangeBound | None = None) -> IdArray: ... + def ensure_node_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> NodePropertyIndexInfo: ... + def drop_node_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> bool: ... + def list_node_property_indexes(self) -> list[NodePropertyIndexInfo]: ... + def ensure_edge_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> EdgePropertyIndexInfo: ... + def drop_edge_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> bool: ... + def list_edge_property_indexes(self) -> list[EdgePropertyIndexInfo]: ... + def nodes_by_labels(self, labels: NodeLabels) -> IdArray: ... + def edges_by_label(self, label: str) -> IdArray: ... + def get_nodes_by_labels(self, labels: NodeLabels) -> list[NodeView]: ... + def get_edges_by_label(self, label: str) -> list[EdgeView]: ... + def count_nodes_by_labels(self, labels: NodeLabels) -> int: ... + def count_edges_by_label(self, label: str) -> int: ... + def find_nodes_by_time_range(self, label: str, from_ms: int, to_ms: int) -> IdArray: ... + def find_nodes_range(self, label: str, prop_key: str, lower: PropertyRangeBound | None = None, upper: PropertyRangeBound | None = None) -> IdArray: ... # Binary batch def batch_upsert_nodes_binary(self, buffer: bytes) -> list[int]: ... @@ -410,11 +505,11 @@ class OverGraph: node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, limit: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> list[PyNeighborEntry]: ... + ) -> list[NeighborEntry]: ... def traverse( self, start: int, @@ -422,42 +517,43 @@ class OverGraph: *, min_depth: int = 1, direction: str = "outgoing", - edge_type_filter: list[int] | None = None, - node_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + emit_node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, limit: int | None = None, - cursor: PyTraversalCursor | None = None, - ) -> PyTraversalPageResult: ... + cursor: TraversalCursor | None = None, + ) -> TraversalPageResult: ... def top_k_neighbors( self, node_id: int, k: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, scoring: str = "weight", at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> list[PyNeighborEntry]: ... + ) -> list[NeighborEntry]: ... def extract_subgraph( self, start_node_id: int, max_depth: int, *, direction: str = "outgoing", - edge_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None, - ) -> PySubgraph: ... + ) -> Subgraph: ... def neighbors_batch( self, node_ids: list[int], *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> dict[int, list[PyNeighborEntry]]: ... + ) -> dict[int, list[NeighborEntry]]: ... # Degree counts + aggregations def degree( @@ -465,7 +561,7 @@ class OverGraph: node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> int: ... def sum_edge_weights( @@ -473,7 +569,7 @@ class OverGraph: node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> float: ... def avg_edge_weight( @@ -481,7 +577,7 @@ class OverGraph: node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> float | None: ... def degrees( @@ -489,7 +585,7 @@ class OverGraph: node_ids: list[int], *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> dict[int, int]: ... @@ -500,19 +596,19 @@ class OverGraph: to_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None, - ) -> PyShortestPath | None: ... + ) -> ShortestPath | None: ... def is_connected( self, from_id: int, to_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, max_depth: int | None = None, ) -> bool: ... @@ -522,13 +618,13 @@ class OverGraph: to_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None, max_paths: int | None = None, - ) -> list[PyShortestPath]: ... + ) -> list[ShortestPath]: ... # Retention def prune( @@ -536,46 +632,46 @@ class OverGraph: *, max_age_ms: int | None = None, max_weight: float | None = None, - type_id: int | None = None, - ) -> PyPruneResult: ... + label: str | None = None, + ) -> PruneResult: ... def set_prune_policy( self, name: str, *, max_age_ms: int | None = None, max_weight: float | None = None, - type_id: int | None = None, + label: str | None = None, ) -> None: ... def remove_prune_policy(self, name: str) -> bool: ... - def list_prune_policies(self) -> list[PyNamedPrunePolicy]: ... + def list_prune_policies(self) -> list[NamedPrunePolicy]: ... # Maintenance def sync(self) -> None: ... - def flush(self) -> PySegmentInfo | None: ... + def flush(self) -> SegmentInfo | None: ... def ingest_mode(self) -> None: ... - def end_ingest(self) -> PyCompactionStats | None: ... - def compact(self) -> PyCompactionStats | None: ... - def compact_with_progress(self, callback: Callable[[PyCompactionProgress], bool]) -> PyCompactionStats | None: ... + def end_ingest(self) -> CompactionStats | None: ... + def compact(self) -> CompactionStats | None: ... + def compact_with_progress(self, callback: Callable[[CompactionProgress], bool]) -> CompactionStats | None: ... # Pagination - def nodes_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - def edges_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - def get_nodes_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyNodePageResult: ... - def get_edges_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyEdgePageResult: ... - def find_nodes_paged(self, type_id: int, prop_key: str, prop_value: Any, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - def find_nodes_by_time_range_paged(self, type_id: int, from_ms: int, to_ms: int, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - def find_nodes_range_paged(self, type_id: int, prop_key: str, lower: PyPropertyRangeBound | None = None, upper: PyPropertyRangeBound | None = None, *, limit: int | None = None, after: PyPropertyRangeCursor | None = None) -> PyPropertyRangePageResult: ... + def nodes_by_labels_paged(self, labels: NodeLabels, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + def edges_by_label_paged(self, label: str, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + def get_nodes_by_labels_paged(self, labels: NodeLabels, *, limit: int | None = None, after: int | None = None) -> NodePageResult: ... + def get_edges_by_label_paged(self, label: str, *, limit: int | None = None, after: int | None = None) -> EdgePageResult: ... + def find_nodes_paged(self, label: str, prop_key: str, prop_value: Any, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + def find_nodes_by_time_range_paged(self, label: str, from_ms: int, to_ms: int, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + def find_nodes_range_paged(self, label: str, prop_key: str, lower: PropertyRangeBound | None = None, upper: PropertyRangeBound | None = None, *, limit: int | None = None, after: PropertyRangeCursor | None = None) -> PropertyRangePageResult: ... def neighbors_paged( self, node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, limit: int | None = None, after: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> PyNeighborPageResult: ... + ) -> NeighborPageResult: ... # Analytics def personalized_pagerank( @@ -587,31 +683,31 @@ class OverGraph: max_iterations: int | None = None, epsilon: float | None = None, approx_residual_tolerance: float | None = None, - edge_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, max_results: int | None = None, - ) -> PyPprResult: ... + ) -> PprResult: ... def export_adjacency( self, *, - node_type_filter: list[int] | None = None, - edge_type_filter: list[int] | None = None, + node_label_filter: NodeLabelFilter | None = None, + edge_label_filter: list[str] | None = None, include_weights: bool = True, - ) -> PyAdjacencyExport: ... + ) -> AdjacencyExport: ... # Connected components def connected_components( self, *, - edge_type_filter: list[int] | None = None, - node_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None, ) -> dict[int, int]: ... def component_of( self, node_id: int, *, - edge_type_filter: list[int] | None = None, - node_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None, ) -> list[int]: ... @@ -623,22 +719,22 @@ class OverGraph: *, dense_query: list[float] | None = None, sparse_query: list[tuple[int, float]] | None = None, - type_filter: list[int] | None = None, + label_filter: NodeLabelFilter | None = None, ef_search: int | None = None, scope_start_node_id: int | None = None, scope_max_depth: int | None = None, scope_direction: str | None = None, - scope_edge_type_filter: list[int] | None = None, + scope_edge_label_filter: list[str] | None = None, scope_at_epoch: int | None = None, dense_weight: float | None = None, sparse_weight: float | None = None, fusion_mode: str | None = None, - ) -> list[PyVectorHit]: ... + ) -> list[VectorHit]: ... -class PyWriteTxn: +class WriteTxn: def upsert_node( self, - type_id: int, + labels: NodeLabels, key: str, *, props: dict[str, Any] | None = None, @@ -649,7 +745,7 @@ class PyWriteTxn: def upsert_node_as( self, alias: str, - type_id: int, + labels: NodeLabels, key: str, *, props: dict[str, Any] | None = None, @@ -657,11 +753,13 @@ class PyWriteTxn: dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None, ) -> dict[str, Any]: ... + def add_node_label(self, target: dict[str, Any], label: str) -> bool: ... + def remove_node_label(self, target: dict[str, Any], label: str) -> bool: ... def upsert_edge( self, from_ref: dict[str, Any], to_ref: dict[str, Any], - type_id: int, + label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, @@ -673,7 +771,7 @@ class PyWriteTxn: alias: str, from_ref: dict[str, Any], to_ref: dict[str, Any], - type_id: int, + label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, @@ -686,14 +784,14 @@ class PyWriteTxn: def stage(self, operations: list[dict[str, Any]]) -> None: ... def get_node(self, target: dict[str, Any]) -> dict[str, Any] | None: ... def get_edge(self, target: dict[str, Any]) -> dict[str, Any] | None: ... - def get_node_by_key(self, type_id: int, key: str) -> dict[str, Any] | None: ... + def get_node_by_key(self, label: str, key: str) -> dict[str, Any] | None: ... def get_edge_by_triple( self, from_ref: dict[str, Any], to_ref: dict[str, Any], - type_id: int, + label: str, ) -> dict[str, Any] | None: ... - def commit(self) -> PyTxnCommitResult: ... + def commit(self) -> TxnCommitResult: ... def rollback(self) -> None: ... # ============================================================ @@ -711,110 +809,131 @@ class AsyncOverGraph: exc_val: BaseException | None, exc_tb: Any, ) -> None: ... - async def stats(self) -> PyDbStats: ... + async def stats(self) -> DbStats: ... + async def scrub(self) -> ScrubReport: ... + + # Catalog + async def ensure_node_label(self, label: str) -> int: ... + async def ensure_edge_label(self, label: str) -> int: ... + async def get_node_label_id(self, label: str) -> int | None: ... + async def get_edge_label_id(self, label: str) -> int | None: ... + async def get_node_label(self, label_id: int) -> str | None: ... + async def get_edge_label(self, label_id: int) -> str | None: ... + async def list_node_labels(self) -> list[NodeLabelInfo]: ... + async def list_edge_labels(self) -> list[EdgeLabelInfo]: ... # Single CRUD - async def upsert_node(self, type_id: int, key: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None) -> int: ... - async def upsert_edge(self, from_id: int, to_id: int, type_id: int, *, props: dict[str, Any] | None = None, weight: float = 1.0, valid_from: int | None = None, valid_to: int | None = None) -> int: ... - async def get_node(self, node_id: int) -> PyNodeRecord | None: ... - async def get_edge(self, edge_id: int) -> PyEdgeRecord | None: ... - async def get_node_by_key(self, type_id: int, key: str) -> PyNodeRecord | None: ... - async def get_edge_by_triple(self, from_id: int, to_id: int, type_id: int) -> PyEdgeRecord | None: ... + async def upsert_node(self, labels: NodeLabels, key: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None) -> int: ... + async def add_node_label(self, node_id: int, label: str) -> bool: ... + async def remove_node_label(self, node_id: int, label: str) -> bool: ... + async def upsert_edge(self, from_id: int, to_id: int, label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, valid_from: int | None = None, valid_to: int | None = None) -> int: ... + async def get_node(self, node_id: int) -> NodeView | None: ... + async def get_edge(self, edge_id: int) -> EdgeView | None: ... + async def get_node_by_key(self, label: str, key: str) -> NodeView | None: ... + async def get_edge_by_triple(self, from_id: int, to_id: int, label: str) -> EdgeView | None: ... async def delete_node(self, node_id: int) -> None: ... async def delete_edge(self, edge_id: int) -> None: ... - async def invalidate_edge(self, edge_id: int, valid_to: int) -> PyEdgeRecord | None: ... + async def invalidate_edge(self, edge_id: int, valid_to: int) -> EdgeView | None: ... # Batch async def batch_upsert_nodes(self, nodes: list[dict[str, Any]]) -> list[int]: ... async def batch_upsert_edges(self, edges: list[dict[str, Any]]) -> list[int]: ... async def batch_upsert_nodes_binary(self, buffer: bytes) -> list[int]: ... async def batch_upsert_edges_binary(self, buffer: bytes) -> list[int]: ... - async def get_nodes(self, node_ids: list[int]) -> list[PyNodeRecord | None]: ... - async def get_nodes_by_keys(self, keys: list[tuple[int, str]]) -> list[PyNodeRecord | None]: ... - async def get_edges(self, edge_ids: list[int]) -> list[PyEdgeRecord | None]: ... - async def graph_patch(self, patch: dict[str, Any]) -> PyPatchResult: ... + async def get_nodes(self, node_ids: list[int]) -> list[NodeView | None]: ... + async def get_nodes_by_keys(self, keys: list[dict[str, Any]]) -> list[NodeView | None]: ... + async def get_edges(self, edge_ids: list[int]) -> list[EdgeView | None]: ... + async def graph_patch(self, patch: dict[str, Any]) -> PatchResult: ... async def begin_write_txn(self) -> AsyncWriteTxn: ... # Queries - async def find_nodes(self, type_id: int, prop_key: str, prop_value: Any) -> IdArray: ... - async def query_node_ids(self, request: dict[str, Any] | NodeQueryRequest) -> PyIdPageResult: ... - async def query_nodes(self, request: dict[str, Any] | NodeQueryRequest) -> PyNodePageResult: ... + async def find_nodes(self, label: str, prop_key: str, prop_value: Any) -> IdArray: ... + async def query_node_ids(self, request: dict[str, Any] | NodeQueryRequest) -> IdPageResult: ... + async def query_nodes(self, request: dict[str, Any] | NodeQueryRequest) -> NodePageResult: ... + async def query_edge_ids(self, request: dict[str, Any] | EdgeQueryRequest) -> IdPageResult: ... + async def query_edges(self, request: dict[str, Any] | EdgeQueryRequest) -> EdgePageResult: ... async def query_pattern(self, request: dict[str, Any] | GraphPatternRequest) -> dict[str, Any]: ... async def explain_node_query(self, request: dict[str, Any] | NodeQueryRequest) -> dict[str, Any]: ... + async def explain_edge_query(self, request: dict[str, Any] | EdgeQueryRequest) -> dict[str, Any]: ... async def explain_pattern_query(self, request: dict[str, Any] | GraphPatternRequest) -> dict[str, Any]: ... - async def ensure_node_property_index(self, type_id: int, prop_key: str, kind: str, *, domain: str | None = None) -> PyNodePropertyIndexInfo: ... - async def drop_node_property_index(self, type_id: int, prop_key: str, kind: str, *, domain: str | None = None) -> bool: ... - async def list_node_property_indexes(self) -> list[PyNodePropertyIndexInfo]: ... - async def nodes_by_type(self, type_id: int) -> IdArray: ... - async def edges_by_type(self, type_id: int) -> IdArray: ... - async def get_nodes_by_type(self, type_id: int) -> list[PyNodeRecord]: ... - async def get_edges_by_type(self, type_id: int) -> list[PyEdgeRecord]: ... - async def count_nodes_by_type(self, type_id: int) -> int: ... - async def count_edges_by_type(self, type_id: int) -> int: ... - async def find_nodes_by_time_range(self, type_id: int, from_ms: int, to_ms: int) -> IdArray: ... - async def find_nodes_range(self, type_id: int, prop_key: str, lower: PyPropertyRangeBound | None = None, upper: PyPropertyRangeBound | None = None) -> IdArray: ... + async def ensure_node_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> NodePropertyIndexInfo: ... + async def drop_node_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> bool: ... + async def list_node_property_indexes(self) -> list[NodePropertyIndexInfo]: ... + async def ensure_edge_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> EdgePropertyIndexInfo: ... + async def drop_edge_property_index(self, label: str, prop_key: str, kind: str, *, domain: str | None = None) -> bool: ... + async def list_edge_property_indexes(self) -> list[EdgePropertyIndexInfo]: ... + async def nodes_by_labels(self, labels: NodeLabels) -> IdArray: ... + async def edges_by_label(self, label: str) -> IdArray: ... + async def get_nodes_by_labels(self, labels: NodeLabels) -> list[NodeView]: ... + async def get_edges_by_label(self, label: str) -> list[EdgeView]: ... + async def count_nodes_by_labels(self, labels: NodeLabels) -> int: ... + async def count_edges_by_label(self, label: str) -> int: ... + async def find_nodes_by_time_range(self, label: str, from_ms: int, to_ms: int) -> IdArray: ... + async def find_nodes_range(self, label: str, prop_key: str, lower: PropertyRangeBound | None = None, upper: PropertyRangeBound | None = None) -> IdArray: ... # Traversal - async def neighbors(self, node_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, limit: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None) -> list[PyNeighborEntry]: ... - async def traverse(self, start: int, max_depth: int, *, min_depth: int = 1, direction: str = "outgoing", edge_type_filter: list[int] | None = None, node_type_filter: list[int] | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, limit: int | None = None, cursor: PyTraversalCursor | None = None) -> PyTraversalPageResult: ... - async def top_k_neighbors(self, node_id: int, k: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, scoring: str = "weight", at_epoch: int | None = None, decay_lambda: float | None = None) -> list[PyNeighborEntry]: ... - async def extract_subgraph(self, start_node_id: int, max_depth: int, *, direction: str = "outgoing", edge_type_filter: list[int] | None = None, at_epoch: int | None = None) -> PySubgraph: ... - async def neighbors_batch(self, node_ids: list[int], *, direction: str = "outgoing", type_filter: list[int] | None = None, at_epoch: int | None = None, decay_lambda: float | None = None) -> dict[int, list[PyNeighborEntry]]: ... + async def neighbors(self, node_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, limit: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None) -> list[NeighborEntry]: ... + async def traverse(self, start: int, max_depth: int, *, min_depth: int = 1, direction: str = "outgoing", edge_label_filter: list[str] | None = None, emit_node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, limit: int | None = None, cursor: TraversalCursor | None = None) -> TraversalPageResult: ... + async def top_k_neighbors(self, node_id: int, k: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, scoring: str = "weight", at_epoch: int | None = None, decay_lambda: float | None = None) -> list[NeighborEntry]: ... + async def extract_subgraph(self, start_node_id: int, max_depth: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None) -> Subgraph: ... + async def neighbors_batch(self, node_ids: list[int], *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, at_epoch: int | None = None, decay_lambda: float | None = None) -> dict[int, list[NeighborEntry]]: ... # Degree counts + aggregations - async def degree(self, node_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, at_epoch: int | None = None) -> int: ... - async def sum_edge_weights(self, node_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, at_epoch: int | None = None) -> float: ... - async def avg_edge_weight(self, node_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, at_epoch: int | None = None) -> float | None: ... - async def degrees(self, node_ids: list[int], *, direction: str = "outgoing", type_filter: list[int] | None = None, at_epoch: int | None = None) -> dict[int, int]: ... + async def degree(self, node_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, at_epoch: int | None = None) -> int: ... + async def sum_edge_weights(self, node_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, at_epoch: int | None = None) -> float: ... + async def avg_edge_weight(self, node_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, at_epoch: int | None = None) -> float | None: ... + async def degrees(self, node_ids: list[int], *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, at_epoch: int | None = None) -> dict[int, int]: ... # Shortest path - async def shortest_path(self, from_id: int, to_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None) -> PyShortestPath | None: ... - async def is_connected(self, from_id: int, to_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, at_epoch: int | None = None, max_depth: int | None = None) -> bool: ... - async def all_shortest_paths(self, from_id: int, to_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None, max_paths: int | None = None) -> list[PyShortestPath]: ... + async def shortest_path(self, from_id: int, to_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None) -> ShortestPath | None: ... + async def is_connected(self, from_id: int, to_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, at_epoch: int | None = None, max_depth: int | None = None) -> bool: ... + async def all_shortest_paths(self, from_id: int, to_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None, max_paths: int | None = None) -> list[ShortestPath]: ... # Retention - async def prune(self, *, max_age_ms: int | None = None, max_weight: float | None = None, type_id: int | None = None) -> PyPruneResult: ... - async def set_prune_policy(self, name: str, *, max_age_ms: int | None = None, max_weight: float | None = None, type_id: int | None = None) -> None: ... + async def prune(self, *, max_age_ms: int | None = None, max_weight: float | None = None, label: str | None = None) -> PruneResult: ... + async def set_prune_policy(self, name: str, *, max_age_ms: int | None = None, max_weight: float | None = None, label: str | None = None) -> None: ... async def remove_prune_policy(self, name: str) -> bool: ... - async def list_prune_policies(self) -> list[PyNamedPrunePolicy]: ... + async def list_prune_policies(self) -> list[NamedPrunePolicy]: ... # Maintenance async def sync(self) -> None: ... - async def flush(self) -> PySegmentInfo | None: ... + async def flush(self) -> SegmentInfo | None: ... async def ingest_mode(self) -> None: ... - async def end_ingest(self) -> PyCompactionStats | None: ... - async def compact(self) -> PyCompactionStats | None: ... - async def compact_with_progress(self, callback: Callable[[PyCompactionProgress], bool]) -> PyCompactionStats | None: ... + async def end_ingest(self) -> CompactionStats | None: ... + async def compact(self) -> CompactionStats | None: ... + async def compact_with_progress(self, callback: Callable[[CompactionProgress], bool]) -> CompactionStats | None: ... # Pagination - async def nodes_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - async def edges_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - async def get_nodes_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyNodePageResult: ... - async def get_edges_by_type_paged(self, type_id: int, *, limit: int | None = None, after: int | None = None) -> PyEdgePageResult: ... - async def find_nodes_paged(self, type_id: int, prop_key: str, prop_value: Any, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - async def find_nodes_by_time_range_paged(self, type_id: int, from_ms: int, to_ms: int, *, limit: int | None = None, after: int | None = None) -> PyIdPageResult: ... - async def find_nodes_range_paged(self, type_id: int, prop_key: str, lower: PyPropertyRangeBound | None = None, upper: PyPropertyRangeBound | None = None, *, limit: int | None = None, after: PyPropertyRangeCursor | None = None) -> PyPropertyRangePageResult: ... - async def neighbors_paged(self, node_id: int, *, direction: str = "outgoing", type_filter: list[int] | None = None, limit: int | None = None, after: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None) -> PyNeighborPageResult: ... + async def nodes_by_labels_paged(self, labels: NodeLabels, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + async def edges_by_label_paged(self, label: str, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + async def get_nodes_by_labels_paged(self, labels: NodeLabels, *, limit: int | None = None, after: int | None = None) -> NodePageResult: ... + async def get_edges_by_label_paged(self, label: str, *, limit: int | None = None, after: int | None = None) -> EdgePageResult: ... + async def find_nodes_paged(self, label: str, prop_key: str, prop_value: Any, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + async def find_nodes_by_time_range_paged(self, label: str, from_ms: int, to_ms: int, *, limit: int | None = None, after: int | None = None) -> IdPageResult: ... + async def find_nodes_range_paged(self, label: str, prop_key: str, lower: PropertyRangeBound | None = None, upper: PropertyRangeBound | None = None, *, limit: int | None = None, after: PropertyRangeCursor | None = None) -> PropertyRangePageResult: ... + async def neighbors_paged(self, node_id: int, *, direction: str = "outgoing", edge_label_filter: list[str] | None = None, limit: int | None = None, after: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None) -> NeighborPageResult: ... # Analytics - async def personalized_pagerank(self, seed_node_ids: list[int], *, algorithm: str | None = None, damping_factor: float | None = None, max_iterations: int | None = None, epsilon: float | None = None, approx_residual_tolerance: float | None = None, edge_type_filter: list[int] | None = None, max_results: int | None = None) -> PyPprResult: ... - async def export_adjacency(self, *, node_type_filter: list[int] | None = None, edge_type_filter: list[int] | None = None, include_weights: bool = True) -> PyAdjacencyExport: ... - async def connected_components(self, *, edge_type_filter: list[int] | None = None, node_type_filter: list[int] | None = None, at_epoch: int | None = None) -> dict[int, int]: ... - async def component_of(self, node_id: int, *, edge_type_filter: list[int] | None = None, node_type_filter: list[int] | None = None, at_epoch: int | None = None) -> list[int]: ... - async def vector_search(self, mode: str, k: int, *, dense_query: list[float] | None = None, sparse_query: list[tuple[int, float]] | None = None, type_filter: list[int] | None = None, ef_search: int | None = None, scope_start_node_id: int | None = None, scope_max_depth: int | None = None, scope_direction: str | None = None, scope_edge_type_filter: list[int] | None = None, scope_at_epoch: int | None = None, dense_weight: float | None = None, sparse_weight: float | None = None, fusion_mode: str | None = None) -> list[PyVectorHit]: ... + async def personalized_pagerank(self, seed_node_ids: list[int], *, algorithm: str | None = None, damping_factor: float | None = None, max_iterations: int | None = None, epsilon: float | None = None, approx_residual_tolerance: float | None = None, edge_label_filter: list[str] | None = None, max_results: int | None = None) -> PprResult: ... + async def export_adjacency(self, *, node_label_filter: NodeLabelFilter | None = None, edge_label_filter: list[str] | None = None, include_weights: bool = True) -> AdjacencyExport: ... + async def connected_components(self, *, edge_label_filter: list[str] | None = None, node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None) -> dict[int, int]: ... + async def component_of(self, node_id: int, *, edge_label_filter: list[str] | None = None, node_label_filter: NodeLabelFilter | None = None, at_epoch: int | None = None) -> list[int]: ... + async def vector_search(self, mode: str, k: int, *, dense_query: list[float] | None = None, sparse_query: list[tuple[int, float]] | None = None, label_filter: NodeLabelFilter | None = None, ef_search: int | None = None, scope_start_node_id: int | None = None, scope_max_depth: int | None = None, scope_direction: str | None = None, scope_edge_label_filter: list[str] | None = None, scope_at_epoch: int | None = None, dense_weight: float | None = None, sparse_weight: float | None = None, fusion_mode: str | None = None) -> list[VectorHit]: ... class AsyncWriteTxn: - async def upsert_node(self, type_id: int, key: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None) -> dict[str, Any]: ... - async def upsert_node_as(self, alias: str, type_id: int, key: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None) -> dict[str, Any]: ... - async def upsert_edge(self, from_ref: dict[str, Any], to_ref: dict[str, Any], type_id: int, *, props: dict[str, Any] | None = None, weight: float = 1.0, valid_from: int | None = None, valid_to: int | None = None) -> dict[str, Any]: ... - async def upsert_edge_as(self, alias: str, from_ref: dict[str, Any], to_ref: dict[str, Any], type_id: int, *, props: dict[str, Any] | None = None, weight: float = 1.0, valid_from: int | None = None, valid_to: int | None = None) -> dict[str, Any]: ... + async def upsert_node(self, labels: NodeLabels, key: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None) -> dict[str, Any]: ... + async def upsert_node_as(self, alias: str, labels: NodeLabels, key: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, dense_vector: list[float] | None = None, sparse_vector: list[tuple[int, float]] | None = None) -> dict[str, Any]: ... + async def add_node_label(self, target: dict[str, Any], label: str) -> bool: ... + async def remove_node_label(self, target: dict[str, Any], label: str) -> bool: ... + async def upsert_edge(self, from_ref: dict[str, Any], to_ref: dict[str, Any], label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, valid_from: int | None = None, valid_to: int | None = None) -> dict[str, Any]: ... + async def upsert_edge_as(self, alias: str, from_ref: dict[str, Any], to_ref: dict[str, Any], label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, valid_from: int | None = None, valid_to: int | None = None) -> dict[str, Any]: ... async def delete_node(self, target: dict[str, Any]) -> None: ... async def delete_edge(self, target: dict[str, Any]) -> None: ... async def invalidate_edge(self, target: dict[str, Any], valid_to: int) -> None: ... async def stage(self, operations: list[dict[str, Any]]) -> None: ... async def get_node(self, target: dict[str, Any]) -> dict[str, Any] | None: ... async def get_edge(self, target: dict[str, Any]) -> dict[str, Any] | None: ... - async def get_node_by_key(self, type_id: int, key: str) -> dict[str, Any] | None: ... - async def get_edge_by_triple(self, from_ref: dict[str, Any], to_ref: dict[str, Any], type_id: int) -> dict[str, Any] | None: ... - async def commit(self) -> PyTxnCommitResult: ... + async def get_node_by_key(self, label: str, key: str) -> dict[str, Any] | None: ... + async def get_edge_by_triple(self, from_ref: dict[str, Any], to_ref: dict[str, Any], label: str) -> dict[str, Any] | None: ... + async def commit(self) -> TxnCommitResult: ... async def rollback(self) -> None: ... diff --git a/overgraph-python/python/overgraph/async_api.py b/overgraph-python/python/overgraph/async_api.py index 659e85f..1409a56 100644 --- a/overgraph-python/python/overgraph/async_api.py +++ b/overgraph-python/python/overgraph/async_api.py @@ -8,34 +8,38 @@ from .overgraph import ( OverGraph, IdArray, - PyAdjacencyExport, - PyCompactionProgress, - PyCompactionStats, - PyDbStats, - PyEdgePageResult, - PyEdgeRecord, - PyIdPageResult, - PyNamedPrunePolicy, - PyNeighborEntry, - PyNeighborPageResult, - PyNodePropertyIndexInfo, - PyNodePageResult, - PyNodeRecord, - PyPatchResult, - PyPprResult, - PyPropertyRangeBound, - PyPropertyRangeCursor, - PyPropertyRangePageResult, - PyPruneResult, - PySegmentInfo, - PyShortestPath, - PySubgraph, - PyTraversalCursor, - PyTraversalHit, - PyTraversalPageResult, - PyTxnCommitResult, - PyVectorHit, - PyWriteTxn, + AdjacencyExport, + CompactionProgress, + CompactionStats, + DbStats, + EdgePageResult, + EdgePropertyIndexInfo, + EdgeView, + EdgeLabelInfo, + IdPageResult, + NamedPrunePolicy, + NeighborEntry, + NeighborPageResult, + NodeLabelInfo, + NodePropertyIndexInfo, + NodePageResult, + NodeView, + PatchResult, + PprResult, + PropertyRangeBound, + PropertyRangeCursor, + PropertyRangePageResult, + PruneResult, + ScrubReport, + SegmentInfo, + ShortestPath, + Subgraph, + TraversalCursor, + TraversalHit, + TraversalPageResult, + TxnCommitResult, + VectorHit, + WriteTxn, ) @@ -48,7 +52,7 @@ class AsyncWriteTxn: __slots__ = ("_txn", "_lock") - def __init__(self, txn: PyWriteTxn) -> None: + def __init__(self, txn: WriteTxn) -> None: self._txn = txn self._lock = asyncio.Lock() @@ -57,7 +61,7 @@ def __repr__(self) -> str: async def upsert_node( self, - type_id: int, + labels: str | list[str] | tuple[str, ...], key: str, *, props: dict[str, Any] | None = None, @@ -68,7 +72,7 @@ async def upsert_node( async with self._lock: return await asyncio.to_thread( self._txn.upsert_node, - type_id, + labels, key, props=props, weight=weight, @@ -79,7 +83,7 @@ async def upsert_node( async def upsert_node_as( self, alias: str, - type_id: int, + labels: str | list[str] | tuple[str, ...], key: str, *, props: dict[str, Any] | None = None, @@ -91,7 +95,7 @@ async def upsert_node_as( return await asyncio.to_thread( self._txn.upsert_node_as, alias, - type_id, + labels, key, props=props, weight=weight, @@ -99,11 +103,19 @@ async def upsert_node_as( sparse_vector=sparse_vector, ) + async def add_node_label(self, target: dict[str, Any], label: str) -> bool: + async with self._lock: + return await asyncio.to_thread(self._txn.add_node_label, target, label) + + async def remove_node_label(self, target: dict[str, Any], label: str) -> bool: + async with self._lock: + return await asyncio.to_thread(self._txn.remove_node_label, target, label) + async def upsert_edge( self, from_ref: dict[str, Any], to_ref: dict[str, Any], - type_id: int, + label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, @@ -115,7 +127,7 @@ async def upsert_edge( self._txn.upsert_edge, from_ref, to_ref, - type_id, + label, props=props, weight=weight, valid_from=valid_from, @@ -127,7 +139,7 @@ async def upsert_edge_as( alias: str, from_ref: dict[str, Any], to_ref: dict[str, Any], - type_id: int, + label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, @@ -140,7 +152,7 @@ async def upsert_edge_as( alias, from_ref, to_ref, - type_id, + label, props=props, weight=weight, valid_from=valid_from, @@ -171,22 +183,22 @@ async def get_edge(self, target: dict[str, Any]) -> dict[str, Any] | None: async with self._lock: return await asyncio.to_thread(self._txn.get_edge, target) - async def get_node_by_key(self, type_id: int, key: str) -> dict[str, Any] | None: + async def get_node_by_key(self, label: str, key: str) -> dict[str, Any] | None: async with self._lock: - return await asyncio.to_thread(self._txn.get_node_by_key, type_id, key) + return await asyncio.to_thread(self._txn.get_node_by_key, label, key) async def get_edge_by_triple( self, from_ref: dict[str, Any], to_ref: dict[str, Any], - type_id: int, + label: str, ) -> dict[str, Any] | None: async with self._lock: return await asyncio.to_thread( - self._txn.get_edge_by_triple, from_ref, to_ref, type_id + self._txn.get_edge_by_triple, from_ref, to_ref, label ) - async def commit(self) -> PyTxnCommitResult: + async def commit(self) -> TxnCommitResult: async with self._lock: return await asyncio.to_thread(self._txn.commit) @@ -225,14 +237,43 @@ async def __aexit__( ) -> None: await self.close() - async def stats(self) -> PyDbStats: + async def stats(self) -> DbStats: return await asyncio.to_thread(self._db.stats) + async def scrub(self) -> "ScrubReport": + return await asyncio.to_thread(self._db.scrub) + + # --- Catalog --- + + async def ensure_node_label(self, label: str) -> int: + return await asyncio.to_thread(self._db.ensure_node_label, label) + + async def ensure_edge_label(self, label: str) -> int: + return await asyncio.to_thread(self._db.ensure_edge_label, label) + + async def get_node_label_id(self, label: str) -> int | None: + return await asyncio.to_thread(self._db.get_node_label_id, label) + + async def get_edge_label_id(self, label: str) -> int | None: + return await asyncio.to_thread(self._db.get_edge_label_id, label) + + async def get_node_label(self, label_id: int) -> str | None: + return await asyncio.to_thread(self._db.get_node_label, label_id) + + async def get_edge_label(self, label_id: int) -> str | None: + return await asyncio.to_thread(self._db.get_edge_label, label_id) + + async def list_node_labels(self) -> list[NodeLabelInfo]: + return await asyncio.to_thread(self._db.list_node_labels) + + async def list_edge_labels(self) -> list[EdgeLabelInfo]: + return await asyncio.to_thread(self._db.list_edge_labels) + # --- Single CRUD --- async def upsert_node( self, - type_id: int, + labels: str | list[str] | tuple[str, ...], key: str, *, props: dict[str, Any] | None = None, @@ -241,16 +282,22 @@ async def upsert_node( sparse_vector: list[tuple[int, float]] | None = None, ) -> int: return await asyncio.to_thread( - self._db.upsert_node, type_id, key, + self._db.upsert_node, labels, key, props=props, weight=weight, dense_vector=dense_vector, sparse_vector=sparse_vector, ) + async def add_node_label(self, node_id: int, label: str) -> bool: + return await asyncio.to_thread(self._db.add_node_label, node_id, label) + + async def remove_node_label(self, node_id: int, label: str) -> bool: + return await asyncio.to_thread(self._db.remove_node_label, node_id, label) + async def upsert_edge( self, from_id: int, to_id: int, - type_id: int, + label: str, *, props: dict[str, Any] | None = None, weight: float = 1.0, @@ -258,21 +305,21 @@ async def upsert_edge( valid_to: int | None = None, ) -> int: return await asyncio.to_thread( - self._db.upsert_edge, from_id, to_id, type_id, + self._db.upsert_edge, from_id, to_id, label, props=props, weight=weight, valid_from=valid_from, valid_to=valid_to, ) - async def get_node(self, node_id: int) -> PyNodeRecord | None: + async def get_node(self, node_id: int) -> NodeView | None: return await asyncio.to_thread(self._db.get_node, node_id) - async def get_edge(self, edge_id: int) -> PyEdgeRecord | None: + async def get_edge(self, edge_id: int) -> EdgeView | None: return await asyncio.to_thread(self._db.get_edge, edge_id) - async def get_node_by_key(self, type_id: int, key: str) -> PyNodeRecord | None: - return await asyncio.to_thread(self._db.get_node_by_key, type_id, key) + async def get_node_by_key(self, label: str, key: str) -> NodeView | None: + return await asyncio.to_thread(self._db.get_node_by_key, label, key) - async def get_edge_by_triple(self, from_id: int, to_id: int, type_id: int) -> PyEdgeRecord | None: - return await asyncio.to_thread(self._db.get_edge_by_triple, from_id, to_id, type_id) + async def get_edge_by_triple(self, from_id: int, to_id: int, label: str) -> EdgeView | None: + return await asyncio.to_thread(self._db.get_edge_by_triple, from_id, to_id, label) async def delete_node(self, node_id: int) -> None: await asyncio.to_thread(self._db.delete_node, node_id) @@ -280,7 +327,7 @@ async def delete_node(self, node_id: int) -> None: async def delete_edge(self, edge_id: int) -> None: await asyncio.to_thread(self._db.delete_edge, edge_id) - async def invalidate_edge(self, edge_id: int, valid_to: int) -> PyEdgeRecord | None: + async def invalidate_edge(self, edge_id: int, valid_to: int) -> EdgeView | None: return await asyncio.to_thread(self._db.invalidate_edge, edge_id, valid_to) # --- Batch --- @@ -291,16 +338,16 @@ async def batch_upsert_nodes(self, nodes: list[dict[str, Any]]) -> list[int]: async def batch_upsert_edges(self, edges: list[dict[str, Any]]) -> list[int]: return await asyncio.to_thread(self._db.batch_upsert_edges, edges) - async def get_nodes(self, node_ids: list[int]) -> list[PyNodeRecord | None]: + async def get_nodes(self, node_ids: list[int]) -> list[NodeView | None]: return await asyncio.to_thread(self._db.get_nodes, node_ids) - async def get_nodes_by_keys(self, keys: list[tuple[int, str]]) -> list[PyNodeRecord | None]: + async def get_nodes_by_keys(self, keys: list[dict[str, Any]]) -> list[NodeView | None]: return await asyncio.to_thread(self._db.get_nodes_by_keys, keys) - async def get_edges(self, edge_ids: list[int]) -> list[PyEdgeRecord | None]: + async def get_edges(self, edge_ids: list[int]) -> list[EdgeView | None]: return await asyncio.to_thread(self._db.get_edges, edge_ids) - async def graph_patch(self, patch: dict[str, Any]) -> PyPatchResult: + async def graph_patch(self, patch: dict[str, Any]) -> PatchResult: return await asyncio.to_thread(self._db.graph_patch, patch) async def begin_write_txn(self) -> AsyncWriteTxn: @@ -309,35 +356,44 @@ async def begin_write_txn(self) -> AsyncWriteTxn: # --- Queries --- - async def find_nodes(self, type_id: int, prop_key: str, prop_value: Any) -> IdArray: - return await asyncio.to_thread(self._db.find_nodes, type_id, prop_key, prop_value) + async def find_nodes(self, label: str, prop_key: str, prop_value: Any) -> IdArray: + return await asyncio.to_thread(self._db.find_nodes, label, prop_key, prop_value) - async def query_node_ids(self, request: Any) -> PyIdPageResult: + async def query_node_ids(self, request: Any) -> IdPageResult: return await asyncio.to_thread(self._db.query_node_ids, request) - async def query_nodes(self, request: Any) -> PyNodePageResult: + async def query_nodes(self, request: Any) -> NodePageResult: return await asyncio.to_thread(self._db.query_nodes, request) + async def query_edge_ids(self, request: Any) -> IdPageResult: + return await asyncio.to_thread(self._db.query_edge_ids, request) + + async def query_edges(self, request: Any) -> EdgePageResult: + return await asyncio.to_thread(self._db.query_edges, request) + async def query_pattern(self, request: Any) -> dict[str, Any]: return await asyncio.to_thread(self._db.query_pattern, request) async def explain_node_query(self, request: Any) -> dict[str, Any]: return await asyncio.to_thread(self._db.explain_node_query, request) + async def explain_edge_query(self, request: Any) -> dict[str, Any]: + return await asyncio.to_thread(self._db.explain_edge_query, request) + async def explain_pattern_query(self, request: Any) -> dict[str, Any]: return await asyncio.to_thread(self._db.explain_pattern_query, request) async def ensure_node_property_index( self, - type_id: int, + label: str, prop_key: str, kind: str, *, domain: str | None = None, - ) -> PyNodePropertyIndexInfo: + ) -> NodePropertyIndexInfo: return await asyncio.to_thread( self._db.ensure_node_property_index, - type_id, + label, prop_key, kind, domain=domain, @@ -345,7 +401,7 @@ async def ensure_node_property_index( async def drop_node_property_index( self, - type_id: int, + label: str, prop_key: str, kind: str, *, @@ -353,49 +409,84 @@ async def drop_node_property_index( ) -> bool: return await asyncio.to_thread( self._db.drop_node_property_index, - type_id, + label, prop_key, kind, domain=domain, ) - async def list_node_property_indexes(self) -> list[PyNodePropertyIndexInfo]: + async def list_node_property_indexes(self) -> list[NodePropertyIndexInfo]: return await asyncio.to_thread(self._db.list_node_property_indexes) - async def nodes_by_type(self, type_id: int) -> IdArray: - return await asyncio.to_thread(self._db.nodes_by_type, type_id) + async def ensure_edge_property_index( + self, + label: str, + prop_key: str, + kind: str, + *, + domain: str | None = None, + ) -> EdgePropertyIndexInfo: + return await asyncio.to_thread( + self._db.ensure_edge_property_index, + label, + prop_key, + kind, + domain=domain, + ) - async def edges_by_type(self, type_id: int) -> IdArray: - return await asyncio.to_thread(self._db.edges_by_type, type_id) + async def drop_edge_property_index( + self, + label: str, + prop_key: str, + kind: str, + *, + domain: str | None = None, + ) -> bool: + return await asyncio.to_thread( + self._db.drop_edge_property_index, + label, + prop_key, + kind, + domain=domain, + ) + + async def list_edge_property_indexes(self) -> list[EdgePropertyIndexInfo]: + return await asyncio.to_thread(self._db.list_edge_property_indexes) + + async def nodes_by_labels(self, labels: str | list[str] | tuple[str, ...]) -> IdArray: + return await asyncio.to_thread(self._db.nodes_by_labels, labels) - async def get_nodes_by_type(self, type_id: int) -> list[PyNodeRecord]: - return await asyncio.to_thread(self._db.get_nodes_by_type, type_id) + async def edges_by_label(self, label: str) -> IdArray: + return await asyncio.to_thread(self._db.edges_by_label, label) - async def get_edges_by_type(self, type_id: int) -> list[PyEdgeRecord]: - return await asyncio.to_thread(self._db.get_edges_by_type, type_id) + async def get_nodes_by_labels(self, labels: str | list[str] | tuple[str, ...]) -> list[NodeView]: + return await asyncio.to_thread(self._db.get_nodes_by_labels, labels) - async def count_nodes_by_type(self, type_id: int) -> int: - return await asyncio.to_thread(self._db.count_nodes_by_type, type_id) + async def get_edges_by_label(self, label: str) -> list[EdgeView]: + return await asyncio.to_thread(self._db.get_edges_by_label, label) - async def count_edges_by_type(self, type_id: int) -> int: - return await asyncio.to_thread(self._db.count_edges_by_type, type_id) + async def count_nodes_by_labels(self, labels: str | list[str] | tuple[str, ...]) -> int: + return await asyncio.to_thread(self._db.count_nodes_by_labels, labels) + + async def count_edges_by_label(self, label: str) -> int: + return await asyncio.to_thread(self._db.count_edges_by_label, label) async def find_nodes_by_time_range( - self, type_id: int, from_ms: int, to_ms: int + self, label: str, from_ms: int, to_ms: int ) -> IdArray: return await asyncio.to_thread( - self._db.find_nodes_by_time_range, type_id, from_ms, to_ms + self._db.find_nodes_by_time_range, label, from_ms, to_ms ) async def find_nodes_range( self, - type_id: int, + label: str, prop_key: str, - lower: PyPropertyRangeBound | None = None, - upper: PyPropertyRangeBound | None = None, + lower: PropertyRangeBound | None = None, + upper: PropertyRangeBound | None = None, ) -> IdArray: return await asyncio.to_thread( - self._db.find_nodes_range, type_id, prop_key, lower, upper + self._db.find_nodes_range, label, prop_key, lower, upper ) # --- Traversal --- @@ -405,14 +496,14 @@ async def neighbors( node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, limit: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> list[PyNeighborEntry]: + ) -> list[NeighborEntry]: return await asyncio.to_thread( self._db.neighbors, node_id, - direction=direction, type_filter=type_filter, limit=limit, + direction=direction, edge_label_filter=edge_label_filter, limit=limit, at_epoch=at_epoch, decay_lambda=decay_lambda, ) @@ -423,21 +514,21 @@ async def traverse( *, min_depth: int = 1, direction: str = "outgoing", - edge_type_filter: list[int] | None = None, - node_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + emit_node_label_filter: dict[str, Any] | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, limit: int | None = None, - cursor: PyTraversalCursor | None = None, - ) -> PyTraversalPageResult: + cursor: TraversalCursor | None = None, + ) -> TraversalPageResult: return await asyncio.to_thread( self._db.traverse, start, max_depth, min_depth=min_depth, direction=direction, - edge_type_filter=edge_type_filter, - node_type_filter=node_type_filter, + edge_label_filter=edge_label_filter, + emit_node_label_filter=emit_node_label_filter, at_epoch=at_epoch, decay_lambda=decay_lambda, limit=limit, @@ -449,13 +540,13 @@ async def neighbors_batch( node_ids: list[int], *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> dict[int, list[PyNeighborEntry]]: + ) -> dict[int, list[NeighborEntry]]: return await asyncio.to_thread( self._db.neighbors_batch, node_ids, - direction=direction, type_filter=type_filter, + direction=direction, edge_label_filter=edge_label_filter, at_epoch=at_epoch, decay_lambda=decay_lambda, ) @@ -464,12 +555,12 @@ async def degree( node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> int: return await asyncio.to_thread( self._db.degree, node_id, - direction=direction, type_filter=type_filter, at_epoch=at_epoch, + direction=direction, edge_label_filter=edge_label_filter, at_epoch=at_epoch, ) async def sum_edge_weights( @@ -477,12 +568,12 @@ async def sum_edge_weights( node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> float: return await asyncio.to_thread( self._db.sum_edge_weights, node_id, - direction=direction, type_filter=type_filter, at_epoch=at_epoch, + direction=direction, edge_label_filter=edge_label_filter, at_epoch=at_epoch, ) async def avg_edge_weight( @@ -490,12 +581,12 @@ async def avg_edge_weight( node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> float | None: return await asyncio.to_thread( self._db.avg_edge_weight, node_id, - direction=direction, type_filter=type_filter, at_epoch=at_epoch, + direction=direction, edge_label_filter=edge_label_filter, at_epoch=at_epoch, ) async def degrees( @@ -503,12 +594,12 @@ async def degrees( node_ids: list[int], *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, ) -> dict[int, int]: return await asyncio.to_thread( self._db.degrees, node_ids, - direction=direction, type_filter=type_filter, at_epoch=at_epoch, + direction=direction, edge_label_filter=edge_label_filter, at_epoch=at_epoch, ) async def shortest_path( @@ -517,15 +608,15 @@ async def shortest_path( to_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None, - ) -> PyShortestPath | None: + ) -> ShortestPath | None: return await asyncio.to_thread( self._db.shortest_path, from_id, to_id, - direction=direction, type_filter=type_filter, weight_field=weight_field, + direction=direction, edge_label_filter=edge_label_filter, weight_field=weight_field, at_epoch=at_epoch, max_depth=max_depth, max_cost=max_cost, ) @@ -535,13 +626,13 @@ async def is_connected( to_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, at_epoch: int | None = None, max_depth: int | None = None, ) -> bool: return await asyncio.to_thread( self._db.is_connected, from_id, to_id, - direction=direction, type_filter=type_filter, + direction=direction, edge_label_filter=edge_label_filter, at_epoch=at_epoch, max_depth=max_depth, ) @@ -551,16 +642,16 @@ async def all_shortest_paths( to_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, weight_field: str | None = None, at_epoch: int | None = None, max_depth: int | None = None, max_cost: float | None = None, max_paths: int | None = None, - ) -> list[PyShortestPath]: + ) -> list[ShortestPath]: return await asyncio.to_thread( self._db.all_shortest_paths, from_id, to_id, - direction=direction, type_filter=type_filter, weight_field=weight_field, + direction=direction, edge_label_filter=edge_label_filter, weight_field=weight_field, at_epoch=at_epoch, max_depth=max_depth, max_cost=max_cost, max_paths=max_paths, ) @@ -576,14 +667,14 @@ async def top_k_neighbors( k: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, scoring: str = "weight", at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> list[PyNeighborEntry]: + ) -> list[NeighborEntry]: return await asyncio.to_thread( self._db.top_k_neighbors, node_id, k, - direction=direction, type_filter=type_filter, scoring=scoring, + direction=direction, edge_label_filter=edge_label_filter, scoring=scoring, at_epoch=at_epoch, decay_lambda=decay_lambda, ) @@ -593,13 +684,15 @@ async def extract_subgraph( max_depth: int, *, direction: str = "outgoing", - edge_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + node_label_filter: dict[str, Any] | None = None, at_epoch: int | None = None, - ) -> PySubgraph: + ) -> Subgraph: return await asyncio.to_thread( self._db.extract_subgraph, start_node_id, max_depth, - direction=direction, edge_type_filter=edge_type_filter, at_epoch=at_epoch, + direction=direction, edge_label_filter=edge_label_filter, + node_label_filter=node_label_filter, at_epoch=at_epoch, ) # --- Retention --- @@ -609,11 +702,11 @@ async def prune( *, max_age_ms: int | None = None, max_weight: float | None = None, - type_id: int | None = None, - ) -> PyPruneResult: + label: str | None = None, + ) -> PruneResult: return await asyncio.to_thread( self._db.prune, - max_age_ms=max_age_ms, max_weight=max_weight, type_id=type_id, + max_age_ms=max_age_ms, max_weight=max_weight, label=label, ) async def set_prune_policy( @@ -622,17 +715,17 @@ async def set_prune_policy( *, max_age_ms: int | None = None, max_weight: float | None = None, - type_id: int | None = None, + label: str | None = None, ) -> None: await asyncio.to_thread( self._db.set_prune_policy, name, - max_age_ms=max_age_ms, max_weight=max_weight, type_id=type_id, + max_age_ms=max_age_ms, max_weight=max_weight, label=label, ) async def remove_prune_policy(self, name: str) -> bool: return await asyncio.to_thread(self._db.remove_prune_policy, name) - async def list_prune_policies(self) -> list[PyNamedPrunePolicy]: + async def list_prune_policies(self) -> list[NamedPrunePolicy]: return await asyncio.to_thread(self._db.list_prune_policies) # --- Maintenance --- @@ -640,94 +733,102 @@ async def list_prune_policies(self) -> list[PyNamedPrunePolicy]: async def sync(self) -> None: await asyncio.to_thread(self._db.sync) - async def flush(self) -> PySegmentInfo | None: + async def flush(self) -> SegmentInfo | None: return await asyncio.to_thread(self._db.flush) async def ingest_mode(self) -> None: return await asyncio.to_thread(self._db.ingest_mode) - async def end_ingest(self) -> PyCompactionStats | None: + async def end_ingest(self) -> CompactionStats | None: return await asyncio.to_thread(self._db.end_ingest) - async def compact(self) -> PyCompactionStats | None: + async def compact(self) -> CompactionStats | None: return await asyncio.to_thread(self._db.compact) async def compact_with_progress( - self, callback: Callable[[PyCompactionProgress], bool] - ) -> PyCompactionStats | None: + self, callback: Callable[[CompactionProgress], bool] + ) -> CompactionStats | None: return await asyncio.to_thread(self._db.compact_with_progress, callback) # --- Pagination --- - async def nodes_by_type_paged( - self, type_id: int, *, limit: int | None = None, after: int | None = None - ) -> PyIdPageResult: + async def nodes_by_labels_paged( + self, + labels: str | list[str] | tuple[str, ...], + *, + limit: int | None = None, + after: int | None = None, + ) -> IdPageResult: return await asyncio.to_thread( - self._db.nodes_by_type_paged, type_id, limit=limit, after=after, + self._db.nodes_by_labels_paged, labels, limit=limit, after=after, ) - async def edges_by_type_paged( - self, type_id: int, *, limit: int | None = None, after: int | None = None - ) -> PyIdPageResult: + async def edges_by_label_paged( + self, label: str, *, limit: int | None = None, after: int | None = None + ) -> IdPageResult: return await asyncio.to_thread( - self._db.edges_by_type_paged, type_id, limit=limit, after=after, + self._db.edges_by_label_paged, label, limit=limit, after=after, ) - async def get_nodes_by_type_paged( - self, type_id: int, *, limit: int | None = None, after: int | None = None - ) -> PyNodePageResult: + async def get_nodes_by_labels_paged( + self, + labels: str | list[str] | tuple[str, ...], + *, + limit: int | None = None, + after: int | None = None, + ) -> NodePageResult: return await asyncio.to_thread( - self._db.get_nodes_by_type_paged, type_id, limit=limit, after=after, + self._db.get_nodes_by_labels_paged, labels, limit=limit, after=after, ) - async def get_edges_by_type_paged( - self, type_id: int, *, limit: int | None = None, after: int | None = None - ) -> PyEdgePageResult: + async def get_edges_by_label_paged( + self, label: str, *, limit: int | None = None, after: int | None = None + ) -> EdgePageResult: return await asyncio.to_thread( - self._db.get_edges_by_type_paged, type_id, limit=limit, after=after, + self._db.get_edges_by_label_paged, label, limit=limit, after=after, ) async def find_nodes_paged( self, - type_id: int, + label: str, prop_key: str, prop_value: Any, *, limit: int | None = None, after: int | None = None, - ) -> PyIdPageResult: + ) -> IdPageResult: return await asyncio.to_thread( - self._db.find_nodes_paged, type_id, prop_key, prop_value, + self._db.find_nodes_paged, label, prop_key, prop_value, limit=limit, after=after, ) async def find_nodes_by_time_range_paged( self, - type_id: int, + label: str, from_ms: int, to_ms: int, *, limit: int | None = None, after: int | None = None, - ) -> PyIdPageResult: + ) -> IdPageResult: return await asyncio.to_thread( - self._db.find_nodes_by_time_range_paged, type_id, from_ms, to_ms, + self._db.find_nodes_by_time_range_paged, label, from_ms, to_ms, limit=limit, after=after, ) async def find_nodes_range_paged( self, - type_id: int, + label: str, prop_key: str, - lower: PyPropertyRangeBound | None = None, - upper: PyPropertyRangeBound | None = None, + lower: PropertyRangeBound | None = None, + upper: PropertyRangeBound | None = None, *, limit: int | None = None, - after: PyPropertyRangeCursor | None = None, - ) -> PyPropertyRangePageResult: + after: PropertyRangeCursor | None = None, + ) -> PropertyRangePageResult: return await asyncio.to_thread( self._db.find_nodes_range_paged, - type_id, + label, prop_key, lower, upper, @@ -740,15 +841,15 @@ async def neighbors_paged( node_id: int, *, direction: str = "outgoing", - type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, limit: int | None = None, after: int | None = None, at_epoch: int | None = None, decay_lambda: float | None = None, - ) -> PyNeighborPageResult: + ) -> NeighborPageResult: return await asyncio.to_thread( self._db.neighbors_paged, node_id, - direction=direction, type_filter=type_filter, limit=limit, + direction=direction, edge_label_filter=edge_label_filter, limit=limit, after=after, at_epoch=at_epoch, decay_lambda=decay_lambda, ) @@ -763,40 +864,40 @@ async def personalized_pagerank( max_iterations: int | None = None, epsilon: float | None = None, approx_residual_tolerance: float | None = None, - edge_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, max_results: int | None = None, - ) -> PyPprResult: + ) -> PprResult: return await asyncio.to_thread( self._db.personalized_pagerank, seed_node_ids, algorithm=algorithm, damping_factor=damping_factor, max_iterations=max_iterations, epsilon=epsilon, approx_residual_tolerance=approx_residual_tolerance, - edge_type_filter=edge_type_filter, max_results=max_results, + edge_label_filter=edge_label_filter, max_results=max_results, ) async def export_adjacency( self, *, - node_type_filter: list[int] | None = None, - edge_type_filter: list[int] | None = None, + node_label_filter: dict[str, Any] | None = None, + edge_label_filter: list[str] | None = None, include_weights: bool = True, - ) -> PyAdjacencyExport: + ) -> AdjacencyExport: return await asyncio.to_thread( self._db.export_adjacency, - node_type_filter=node_type_filter, edge_type_filter=edge_type_filter, + node_label_filter=node_label_filter, edge_label_filter=edge_label_filter, include_weights=include_weights, ) async def connected_components( self, *, - edge_type_filter: list[int] | None = None, - node_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + node_label_filter: dict[str, Any] | None = None, at_epoch: int | None = None, ) -> dict[int, int]: return await asyncio.to_thread( self._db.connected_components, - edge_type_filter=edge_type_filter, node_type_filter=node_type_filter, + edge_label_filter=edge_label_filter, node_label_filter=node_label_filter, at_epoch=at_epoch, ) @@ -804,13 +905,13 @@ async def component_of( self, node_id: int, *, - edge_type_filter: list[int] | None = None, - node_type_filter: list[int] | None = None, + edge_label_filter: list[str] | None = None, + node_label_filter: dict[str, Any] | None = None, at_epoch: int | None = None, ) -> list[int]: return await asyncio.to_thread( self._db.component_of, node_id, - edge_type_filter=edge_type_filter, node_type_filter=node_type_filter, + edge_label_filter=edge_label_filter, node_label_filter=node_label_filter, at_epoch=at_epoch, ) @@ -823,24 +924,24 @@ async def vector_search( *, dense_query: list[float] | None = None, sparse_query: list[tuple[int, float]] | None = None, - type_filter: list[int] | None = None, + label_filter: dict[str, Any] | None = None, ef_search: int | None = None, scope_start_node_id: int | None = None, scope_max_depth: int | None = None, scope_direction: str | None = None, - scope_edge_type_filter: list[int] | None = None, + scope_edge_label_filter: list[str] | None = None, scope_at_epoch: int | None = None, dense_weight: float | None = None, sparse_weight: float | None = None, fusion_mode: str | None = None, - ) -> list[PyVectorHit]: + ) -> list[VectorHit]: return await asyncio.to_thread( self._db.vector_search, mode, k, dense_query=dense_query, sparse_query=sparse_query, - type_filter=type_filter, ef_search=ef_search, + label_filter=label_filter, ef_search=ef_search, scope_start_node_id=scope_start_node_id, scope_max_depth=scope_max_depth, - scope_direction=scope_direction, scope_edge_type_filter=scope_edge_type_filter, + scope_direction=scope_direction, scope_edge_label_filter=scope_edge_label_filter, scope_at_epoch=scope_at_epoch, dense_weight=dense_weight, sparse_weight=sparse_weight, fusion_mode=fusion_mode, diff --git a/overgraph-python/src/lib.rs b/overgraph-python/src/lib.rs index ed5af2f..cb75c92 100644 --- a/overgraph-python/src/lib.rs +++ b/overgraph-python/src/lib.rs @@ -1,21 +1,31 @@ #![allow(clippy::too_many_arguments)] use eg::{ - AdjacencyExport, AllShortestPathsOptions, CompactionPhase, CompactionProgress, CompactionStats, - ComponentOptions, DatabaseEngine, DbOptions, DbStats, DegreeOptions, DenseMetric, - DenseVectorConfig, Direction, EdgeInput, EdgePattern, EdgePostFilterPredicate, EdgeRecord, - EngineError, ExportOptions, FusionMode, GraphPatch, GraphPatternQuery, HnswConfig, - IsConnectedOptions, NeighborEntry, NeighborOptions, NodeFilterExpr, NodeIdMap, NodeInput, - NodePattern, NodePropertyIndexInfo, NodeQuery, NodeQueryOrder, NodeRecord, PageRequest, - PatternOrder, PprAlgorithm, PprOptions, PprResult, PropValue, PropertyRangeBound, - PropertyRangeCursor, PropertyRangePageRequest, PropertyRangePageResult, PrunePolicy, - PruneResult, QueryMatch, QueryPatternResult, QueryPlan, QueryPlanKind, QueryPlanNode, - QueryPlanWarning, ScoringMode, SecondaryIndexKind, SecondaryIndexRangeDomain, - SecondaryIndexState, ShortestPath, ShortestPathOptions, Subgraph, SubgraphOptions, TopKOptions, - TraversalCursor, TraversalHit, TraversalPageResult, TraverseOptions, TxnCommitResult, - TxnEdgeRef, TxnEdgeView, TxnIntent, TxnLocalRef, TxnNodeRef, TxnNodeView, UpsertEdgeOptions, - UpsertNodeOptions, VectorSearchMode, VectorSearchRequest, VectorSearchScope, WalSyncMode, - WriteTxn, + AdjacencyExport as CoreAdjacencyExport, AllShortestPathsOptions, CompactionPhase, + CompactionProgress as CoreCompactionProgress, CompactionStats as CoreCompactionStats, + ComponentOptions, ComponentScrubFinding as CoreComponentScrubFinding, DatabaseEngine, + DbOptions, DbStats as CoreDbStats, DegreeOptions, DenseMetric, DenseVectorConfig, Direction, + EdgeFilterExpr, EdgeInput, EdgeLabelInfo as CoreEdgeLabelInfo, EdgePattern, + EdgePropertyIndexInfo as CoreEdgePropertyIndexInfo, EdgeQuery, EdgeQueryOrder, + EdgeView as CoreEdgeView, EngineError, ExportOptions, FusionMode, GraphPatch, + GraphPatternQuery, HnswConfig, IsConnectedOptions, LabelMatchMode, + NeighborEntry as CoreNeighborEntry, NeighborOptions, NodeFilterExpr, NodeIdMap, NodeInput, + NodeKeyQuery, NodeLabelFilter, NodeLabelInfo as CoreNodeLabelInfo, NodePattern, + NodePropertyIndexInfo as CoreNodePropertyIndexInfo, NodeQuery, NodeQueryOrder, + NodeView as CoreNodeView, PageRequest, PatternOrder, PprAlgorithm, PprOptions, + PprResult as CorePprResult, PropValue, PropertyRangeBound as CorePropertyRangeBound, + PropertyRangeCursor as CorePropertyRangeCursor, PropertyRangePageRequest, + PropertyRangePageResult as CorePropertyRangePageResult, PrunePolicy, PrunePolicyInfo, + PruneResult as CorePruneResult, QueryMatch, QueryPatternResult, QueryPlan, QueryPlanKind, + QueryPlanNode, QueryPlanNote, QueryPlanPublicInputs, QueryPlanPublicName, QueryPlanWarning, + ScoringMode, ScrubReport as CoreScrubReport, SecondaryIndexKind, SecondaryIndexRangeDomain, + SecondaryIndexState, SegmentScrubResult as CoreSegmentScrubResult, + ShortestPath as CoreShortestPath, ShortestPathOptions, Subgraph as CoreSubgraph, + SubgraphOptions, TopKOptions, TraversalCursor as CoreTraversalCursor, + TraversalHit as CoreTraversalHit, TraversalPageResult as CoreTraversalPageResult, + TraverseOptions, TxnCommitResult as CoreTxnCommitResult, TxnEdgeRef, TxnEdgeView, TxnIntent, + TxnLocalRef, TxnNodeRef, TxnNodeView, UpsertEdgeOptions, UpsertNodeOptions, VectorSearchMode, + VectorSearchRequest, VectorSearchScope, WalSyncMode, WriteTxn as CoreWriteTxn, }; use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; use pyo3::prelude::*; @@ -133,23 +143,74 @@ impl OverGraph { Ok(false) } - fn stats(&self, py: Python<'_>) -> PyResult { - with_engine_ref(self, py, |eng| Ok(PyDbStats::from(eng.stats()?))) + fn stats(&self, py: Python<'_>) -> PyResult { + with_engine_ref(self, py, |eng| Ok(DbStats::from(eng.stats()?))) + } + + fn scrub(&self, py: Python<'_>) -> PyResult { + with_engine_ref(self, py, |eng| Ok(ScrubReport::from(eng.scrub()?))) + } + + // --- Catalog diagnostics --- + + fn ensure_node_label(&self, py: Python<'_>, label: String) -> PyResult { + with_engine(self, py, move |eng| eng.ensure_node_label(&label)) + } + + fn ensure_edge_label(&self, py: Python<'_>, label: String) -> PyResult { + with_engine(self, py, move |eng| eng.ensure_edge_label(&label)) + } + + fn get_node_label_id(&self, py: Python<'_>, label: String) -> PyResult> { + with_engine_ref(self, py, move |eng| eng.get_node_label_id(&label)) + } + + fn get_edge_label_id(&self, py: Python<'_>, label: String) -> PyResult> { + with_engine_ref(self, py, move |eng| eng.get_edge_label_id(&label)) + } + + fn get_node_label(&self, py: Python<'_>, label_id: u32) -> PyResult> { + with_engine_ref(self, py, move |eng| eng.get_node_label(label_id)) + } + + fn get_edge_label(&self, py: Python<'_>, label_id: u32) -> PyResult> { + with_engine_ref(self, py, move |eng| eng.get_edge_label(label_id)) + } + + fn list_node_labels(&self, py: Python<'_>) -> PyResult> { + with_engine_ref(self, py, |eng| { + Ok(eng + .list_node_labels()? + .into_iter() + .map(NodeLabelInfo::from) + .collect()) + }) + } + + fn list_edge_labels(&self, py: Python<'_>) -> PyResult> { + with_engine_ref(self, py, |eng| { + Ok(eng + .list_edge_labels()? + .into_iter() + .map(EdgeLabelInfo::from) + .collect()) + }) } // --- Single CRUD --- - #[pyo3(signature = (type_id, key, *, props=None, weight=1.0, dense_vector=None, sparse_vector=None))] + #[pyo3(signature = (labels, key, *, props=None, weight=1.0, dense_vector=None, sparse_vector=None))] fn upsert_node( &self, py: Python<'_>, - type_id: u32, + labels: &Bound<'_, PyAny>, key: String, props: Option<&Bound<'_, PyDict>>, weight: f64, dense_vector: Option>, sparse_vector: Option>, ) -> PyResult { + let labels = parse_node_labels_arg(labels, "upsert_node labels")?; let props = convert_py_props(py, props)?; let opts = UpsertNodeOptions { props, @@ -157,16 +218,24 @@ impl OverGraph { dense_vector, sparse_vector, }; - with_engine(self, py, move |eng| eng.upsert_node(type_id, &key, opts)) + with_engine(self, py, move |eng| eng.upsert_node(labels, &key, opts)) + } + + fn add_node_label(&self, py: Python<'_>, node_id: u64, label: String) -> PyResult { + with_engine(self, py, move |eng| eng.add_node_label(node_id, &label)) + } + + fn remove_node_label(&self, py: Python<'_>, node_id: u64, label: String) -> PyResult { + with_engine(self, py, move |eng| eng.remove_node_label(node_id, &label)) } - #[pyo3(signature = (from_id, to_id, type_id, *, props=None, weight=1.0, valid_from=None, valid_to=None))] + #[pyo3(signature = (from_id, to_id, label, *, props=None, weight=1.0, valid_from=None, valid_to=None))] fn upsert_edge( &self, py: Python<'_>, from_id: u64, to_id: u64, - type_id: u32, + label: String, props: Option<&Bound<'_, PyDict>>, weight: f64, valid_from: Option, @@ -180,30 +249,30 @@ impl OverGraph { valid_to, }; with_engine(self, py, move |eng| { - eng.upsert_edge(from_id, to_id, type_id, opts) + eng.upsert_edge(from_id, to_id, &label, opts) }) } - fn get_node(&self, py: Python<'_>, id: u64) -> PyResult> { + fn get_node(&self, py: Python<'_>, id: u64) -> PyResult> { with_engine_ref(self, py, |eng| { - Ok(eng.get_node(id)?.map(PyNodeRecord::from)) + eng.get_node(id)?.map(NodeView::try_from).transpose() }) } - fn get_edge(&self, py: Python<'_>, id: u64) -> PyResult> { - with_engine_ref(self, py, |eng| { - Ok(eng.get_edge(id)?.map(PyEdgeRecord::from)) - }) + fn get_edge(&self, py: Python<'_>, id: u64) -> PyResult> { + with_engine_ref(self, py, |eng| Ok(eng.get_edge(id)?.map(EdgeView::from))) } fn get_node_by_key( &self, py: Python<'_>, - type_id: u32, + label: String, key: String, - ) -> PyResult> { + ) -> PyResult> { with_engine_ref(self, py, move |eng| { - Ok(eng.get_node_by_key(type_id, &key)?.map(PyNodeRecord::from)) + eng.get_node_by_key(&label, &key)? + .map(NodeView::try_from) + .transpose() }) } @@ -212,12 +281,12 @@ impl OverGraph { py: Python<'_>, from_id: u64, to_id: u64, - type_id: u32, - ) -> PyResult> { + label: String, + ) -> PyResult> { with_engine_ref(self, py, move |eng| { Ok(eng - .get_edge_by_triple(from_id, to_id, type_id)? - .map(PyEdgeRecord::from)) + .get_edge_by_triple(from_id, to_id, &label)? + .map(EdgeView::from)) }) } @@ -234,9 +303,9 @@ impl OverGraph { py: Python<'_>, id: u64, valid_to: i64, - ) -> PyResult> { + ) -> PyResult> { with_engine(self, py, move |eng| { - Ok(eng.invalidate_edge(id, valid_to)?.map(PyEdgeRecord::from)) + Ok(eng.invalidate_edge(id, valid_to)?.map(EdgeView::from)) }) } @@ -244,63 +313,60 @@ impl OverGraph { fn batch_upsert_nodes(&self, py: Python<'_>, nodes: &Bound<'_, PyList>) -> PyResult> { let inputs = parse_node_inputs(py, nodes)?; - with_engine(self, py, move |eng| eng.batch_upsert_nodes(&inputs)) + with_engine(self, py, move |eng| eng.batch_upsert_nodes(inputs)) } fn batch_upsert_edges(&self, py: Python<'_>, edges: &Bound<'_, PyList>) -> PyResult> { let inputs = parse_edge_inputs(py, edges)?; - with_engine(self, py, move |eng| eng.batch_upsert_edges(&inputs)) + with_engine(self, py, move |eng| eng.batch_upsert_edges(inputs)) } - fn get_nodes(&self, py: Python<'_>, ids: Vec) -> PyResult>> { + fn get_nodes(&self, py: Python<'_>, ids: Vec) -> PyResult>> { with_engine_ref(self, py, move |eng| { let results = eng.get_nodes(&ids)?; - Ok(results + results .into_iter() - .map(|r| r.map(PyNodeRecord::from)) - .collect()) + .map(|r| r.map(NodeView::try_from).transpose()) + .collect() }) } fn get_nodes_by_keys( &self, py: Python<'_>, - keys: Vec<(u32, String)>, - ) -> PyResult>> { + keys: &Bound<'_, PyList>, + ) -> PyResult>> { + let keys = parse_node_key_queries(keys)?; with_engine_ref(self, py, move |eng| { - let refs: Vec<(u32, &str)> = keys.iter().map(|(t, k)| (*t, k.as_str())).collect(); - let results = eng.get_nodes_by_keys(&refs)?; - Ok(results + let results = eng.get_nodes_by_keys(&keys)?; + results .into_iter() - .map(|r| r.map(PyNodeRecord::from)) - .collect()) + .map(|r| r.map(NodeView::try_from).transpose()) + .collect() }) } - fn get_edges(&self, py: Python<'_>, ids: Vec) -> PyResult>> { + fn get_edges(&self, py: Python<'_>, ids: Vec) -> PyResult>> { with_engine_ref(self, py, move |eng| { let results = eng.get_edges(&ids)?; - Ok(results - .into_iter() - .map(|r| r.map(PyEdgeRecord::from)) - .collect()) + Ok(results.into_iter().map(|r| r.map(EdgeView::from)).collect()) }) } - fn graph_patch(&self, py: Python<'_>, patch: &Bound<'_, PyDict>) -> PyResult { + fn graph_patch(&self, py: Python<'_>, patch: &Bound<'_, PyDict>) -> PyResult { let rust_patch = parse_graph_patch(py, patch)?; with_engine(self, py, move |eng| { - let result = eng.graph_patch(&rust_patch)?; - Ok(PyPatchResult { + let result = eng.graph_patch(rust_patch)?; + Ok(PatchResult { node_ids: result.node_ids, edge_ids: result.edge_ids, }) }) } - fn begin_write_txn(&self, py: Python<'_>) -> PyResult { + fn begin_write_txn(&self, py: Python<'_>) -> PyResult { let txn = with_engine_ref(self, py, |eng| eng.begin_write_txn())?; - Ok(PyWriteTxn { + Ok(WriteTxn { inner: Arc::new(Mutex::new(Some(txn))), }) } @@ -310,27 +376,23 @@ impl OverGraph { fn find_nodes( &self, py: Python<'_>, - type_id: u32, + label: String, prop_key: String, prop_value: &Bound<'_, pyo3::PyAny>, ) -> PyResult { let pv = py_to_prop_value(py, prop_value)?; with_engine_ref(self, py, move |eng| { Ok(IdArray { - ids: Arc::new(eng.find_nodes(type_id, &prop_key, &pv)?), + ids: Arc::new(eng.find_nodes(&label, &prop_key, &pv)?), }) }) } - fn query_node_ids( - &self, - py: Python<'_>, - request: &Bound<'_, PyAny>, - ) -> PyResult { + fn query_node_ids(&self, py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let query = parse_py_node_query(py, request)?; with_engine_ref(self, py, move |eng| { let result = eng.query_node_ids(&query)?; - Ok(PyIdPageResult { + Ok(IdPageResult { items: IdArray { ids: Arc::new(result.items), }, @@ -339,16 +401,40 @@ impl OverGraph { }) } - fn query_nodes( - &self, - py: Python<'_>, - request: &Bound<'_, PyAny>, - ) -> PyResult { + fn query_nodes(&self, py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let query = parse_py_node_query(py, request)?; with_engine_ref(self, py, move |eng| { let result = eng.query_nodes(&query)?; - Ok(PyNodePageResult { - items: result.items.into_iter().map(PyNodeRecord::from).collect(), + Ok(NodePageResult { + items: result + .items + .into_iter() + .map(NodeView::try_from) + .collect::, EngineError>>()?, + next_cursor: result.next_cursor, + }) + }) + } + + fn query_edge_ids(&self, py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let query = parse_py_edge_query(py, request)?; + with_engine_ref(self, py, move |eng| { + let result = eng.query_edge_ids(&query)?; + Ok(IdPageResult { + items: IdArray { + ids: Arc::new(result.edge_ids), + }, + next_cursor: result.next_cursor, + }) + }) + } + + fn query_edges(&self, py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let query = parse_py_edge_query(py, request)?; + with_engine_ref(self, py, move |eng| { + let result = eng.query_edges(&query)?; + Ok(EdgePageResult { + items: result.edges.into_iter().map(EdgeView::from).collect(), next_cursor: result.next_cursor, }) }) @@ -366,6 +452,12 @@ impl OverGraph { query_plan_to_py(py, plan) } + fn explain_edge_query(&self, py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let query = parse_py_edge_query(py, request)?; + let plan = with_engine_ref(self, py, move |eng| eng.explain_edge_query(&query))?; + query_plan_to_py(py, plan) + } + fn explain_pattern_query( &self, py: Python<'_>, @@ -376,121 +468,169 @@ impl OverGraph { query_plan_to_py(py, plan) } - #[pyo3(signature = (type_id, prop_key, kind, *, domain=None))] + #[pyo3(signature = (label, prop_key, kind, *, domain=None))] fn ensure_node_property_index( &self, py: Python<'_>, - type_id: u32, + label: String, prop_key: String, kind: &str, domain: Option<&str>, - ) -> PyResult { + ) -> PyResult { let kind = parse_secondary_index_kind(kind, domain)?; with_engine(self, py, move |eng| { - Ok(PyNodePropertyIndexInfo::from( - eng.ensure_node_property_index(type_id, &prop_key, kind.clone())?, + Ok(NodePropertyIndexInfo::from( + eng.ensure_node_property_index(&label, &prop_key, kind.clone())?, )) }) } - #[pyo3(signature = (type_id, prop_key, kind, *, domain=None))] + #[pyo3(signature = (label, prop_key, kind, *, domain=None))] fn drop_node_property_index( &self, py: Python<'_>, - type_id: u32, + label: String, prop_key: String, kind: &str, domain: Option<&str>, ) -> PyResult { let kind = parse_secondary_index_kind(kind, domain)?; with_engine(self, py, move |eng| { - eng.drop_node_property_index(type_id, &prop_key, kind.clone()) + eng.drop_node_property_index(&label, &prop_key, kind.clone()) }) } - fn list_node_property_indexes(&self, py: Python<'_>) -> PyResult> { + fn list_node_property_indexes(&self, py: Python<'_>) -> PyResult> { with_engine_ref(self, py, |eng| { Ok(eng .list_node_property_indexes()? .into_iter() - .map(PyNodePropertyIndexInfo::from) + .map(NodePropertyIndexInfo::from) + .collect()) + }) + } + + #[pyo3(signature = (label, prop_key, kind, *, domain=None))] + fn ensure_edge_property_index( + &self, + py: Python<'_>, + label: String, + prop_key: String, + kind: &str, + domain: Option<&str>, + ) -> PyResult { + let kind = parse_secondary_index_kind(kind, domain)?; + with_engine(self, py, move |eng| { + Ok(EdgePropertyIndexInfo::from( + eng.ensure_edge_property_index(&label, &prop_key, kind.clone())?, + )) + }) + } + + #[pyo3(signature = (label, prop_key, kind, *, domain=None))] + fn drop_edge_property_index( + &self, + py: Python<'_>, + label: String, + prop_key: String, + kind: &str, + domain: Option<&str>, + ) -> PyResult { + let kind = parse_secondary_index_kind(kind, domain)?; + with_engine(self, py, move |eng| { + eng.drop_edge_property_index(&label, &prop_key, kind.clone()) + }) + } + + fn list_edge_property_indexes(&self, py: Python<'_>) -> PyResult> { + with_engine_ref(self, py, |eng| { + Ok(eng + .list_edge_property_indexes()? + .into_iter() + .map(EdgePropertyIndexInfo::from) .collect()) }) } - fn nodes_by_type(&self, py: Python<'_>, type_id: u32) -> PyResult { + fn nodes_by_labels(&self, py: Python<'_>, labels: &Bound<'_, PyAny>) -> PyResult { + let labels = parse_node_labels_arg(labels, "nodes_by_labels labels")?; with_engine_ref(self, py, move |eng| { Ok(IdArray { - ids: Arc::new(eng.nodes_by_type(type_id)?), + ids: Arc::new(eng.nodes_by_labels(labels)?), }) }) } - fn get_nodes_by_type(&self, py: Python<'_>, type_id: u32) -> PyResult> { + fn get_nodes_by_labels( + &self, + py: Python<'_>, + labels: &Bound<'_, PyAny>, + ) -> PyResult> { + let labels = parse_node_labels_arg(labels, "get_nodes_by_labels labels")?; with_engine_ref(self, py, move |eng| { - Ok(eng - .get_nodes_by_type(type_id)? + eng.get_nodes_by_labels(labels)? .into_iter() - .map(PyNodeRecord::from) - .collect()) + .map(NodeView::try_from) + .collect() }) } - fn edges_by_type(&self, py: Python<'_>, type_id: u32) -> PyResult { + fn edges_by_label(&self, py: Python<'_>, label: String) -> PyResult { with_engine_ref(self, py, move |eng| { Ok(IdArray { - ids: Arc::new(eng.edges_by_type(type_id)?), + ids: Arc::new(eng.edges_by_label(&label)?), }) }) } - fn get_edges_by_type(&self, py: Python<'_>, type_id: u32) -> PyResult> { + fn get_edges_by_label(&self, py: Python<'_>, label: String) -> PyResult> { with_engine_ref(self, py, move |eng| { Ok(eng - .get_edges_by_type(type_id)? + .get_edges_by_label(&label)? .into_iter() - .map(PyEdgeRecord::from) + .map(EdgeView::from) .collect()) }) } - fn count_nodes_by_type(&self, py: Python<'_>, type_id: u32) -> PyResult { - with_engine_ref(self, py, move |eng| eng.count_nodes_by_type(type_id)) + fn count_nodes_by_labels(&self, py: Python<'_>, labels: &Bound<'_, PyAny>) -> PyResult { + let labels = parse_node_labels_arg(labels, "count_nodes_by_labels labels")?; + with_engine_ref(self, py, move |eng| eng.count_nodes_by_labels(labels)) } - fn count_edges_by_type(&self, py: Python<'_>, type_id: u32) -> PyResult { - with_engine_ref(self, py, move |eng| eng.count_edges_by_type(type_id)) + fn count_edges_by_label(&self, py: Python<'_>, label: String) -> PyResult { + with_engine_ref(self, py, move |eng| eng.count_edges_by_label(&label)) } fn find_nodes_by_time_range( &self, py: Python<'_>, - type_id: u32, + label: String, from_ms: i64, to_ms: i64, ) -> PyResult { with_engine_ref(self, py, move |eng| { Ok(IdArray { - ids: Arc::new(eng.find_nodes_by_time_range(type_id, from_ms, to_ms)?), + ids: Arc::new(eng.find_nodes_by_time_range(&label, from_ms, to_ms)?), }) }) } - #[pyo3(signature = (type_id, prop_key, lower=None, upper=None))] + #[pyo3(signature = (label, prop_key, lower=None, upper=None))] fn find_nodes_range( &self, py: Python<'_>, - type_id: u32, + label: String, prop_key: String, - lower: Option, - upper: Option, + lower: Option, + upper: Option, ) -> PyResult { - let lower = lower.map(PropertyRangeBound::from); - let upper = upper.map(PropertyRangeBound::from); + let lower = lower.map(CorePropertyRangeBound::from); + let upper = upper.map(CorePropertyRangeBound::from); with_engine_ref(self, py, move |eng| { Ok(IdArray { ids: Arc::new(eng.find_nodes_range( - type_id, + &label, &prop_key, lower.as_ref(), upper.as_ref(), @@ -501,22 +641,22 @@ impl OverGraph { // --- Traversal --- - #[pyo3(signature = (node_id, *, direction="outgoing", type_filter=None, limit=None, at_epoch=None, decay_lambda=None))] + #[pyo3(signature = (node_id, *, direction="outgoing", edge_label_filter=None, limit=None, at_epoch=None, decay_lambda=None))] fn neighbors( &self, py: Python<'_>, node_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, limit: Option, at_epoch: Option, decay_lambda: Option, - ) -> PyResult> { + ) -> PyResult> { let dir = parse_direction(direction)?; let dl = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter, limit, at_epoch, decay_lambda: dl, @@ -525,12 +665,12 @@ impl OverGraph { Ok(eng .neighbors(node_id, &opts)? .into_iter() - .map(PyNeighborEntry::from) + .map(NeighborEntry::from) .collect()) }) } - #[pyo3(signature = (start, max_depth, *, min_depth=1, direction="outgoing", edge_type_filter=None, node_type_filter=None, at_epoch=None, decay_lambda=None, limit=None, cursor=None))] + #[pyo3(signature = (start, max_depth, *, min_depth=1, direction="outgoing", edge_label_filter=None, emit_node_label_filter=None, at_epoch=None, decay_lambda=None, limit=None, cursor=None))] fn traverse( &self, py: Python<'_>, @@ -538,49 +678,53 @@ impl OverGraph { max_depth: u32, min_depth: u32, direction: &str, - edge_type_filter: Option>, - node_type_filter: Option>, + edge_label_filter: Option>, + emit_node_label_filter: Option<&Bound<'_, PyAny>>, at_epoch: Option, decay_lambda: Option, limit: Option, - cursor: Option, - ) -> PyResult { + cursor: Option, + ) -> PyResult { let dir = parse_direction(direction)?; - let cursor = cursor.map(TraversalCursor::from); + let cursor = cursor.map(CoreTraversalCursor::from); + let emit_node_label_filter = parse_optional_node_label_filter_arg( + emit_node_label_filter, + "traverse emit_node_label_filter", + )?; let opts = TraverseOptions { min_depth, direction: dir, - edge_type_filter, - node_type_filter, + edge_label_filter, + emit_node_label_filter, at_epoch, decay_lambda, limit, cursor, }; with_engine_ref(self, py, move |eng| { - Ok(PyTraversalPageResult::from( + Ok(TraversalPageResult::from( eng.traverse(start, max_depth, &opts)?, )) }) } - #[pyo3(signature = (node_id, k, *, direction="outgoing", type_filter=None, scoring="weight", at_epoch=None, decay_lambda=None))] + #[pyo3(signature = (node_id, k, *, direction="outgoing", edge_label_filter=None, scoring="weight", at_epoch=None, decay_lambda=None))] fn top_k_neighbors( &self, py: Python<'_>, node_id: u64, k: usize, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, scoring: &str, at_epoch: Option, decay_lambda: Option, - ) -> PyResult> { + ) -> PyResult> { let dir = parse_direction(direction)?; let sm = parse_scoring_mode(scoring, decay_lambda)?; let opts = TopKOptions { direction: dir, - type_filter, + edge_label_filter, scoring: sm, at_epoch, }; @@ -588,50 +732,56 @@ impl OverGraph { Ok(eng .top_k_neighbors(node_id, k, &opts)? .into_iter() - .map(PyNeighborEntry::from) + .map(NeighborEntry::from) .collect()) }) } - #[pyo3(signature = (start_node_id, max_depth, *, direction="outgoing", edge_type_filter=None, at_epoch=None))] + #[pyo3(signature = (start_node_id, max_depth, *, direction="outgoing", edge_label_filter=None, node_label_filter=None, at_epoch=None))] fn extract_subgraph( &self, py: Python<'_>, start_node_id: u64, max_depth: u32, direction: &str, - edge_type_filter: Option>, + edge_label_filter: Option>, + node_label_filter: Option<&Bound<'_, PyAny>>, at_epoch: Option, - ) -> PyResult { + ) -> PyResult { let dir = parse_direction(direction)?; + let node_label_filter = parse_optional_node_label_filter_arg( + node_label_filter, + "extract_subgraph node_label_filter", + )?; let opts = SubgraphOptions { direction: dir, - edge_type_filter, + edge_label_filter, + node_label_filter, at_epoch, }; with_engine_ref(self, py, move |eng| { let sg = eng.extract_subgraph(start_node_id, max_depth, &opts)?; - Ok(PySubgraph::from(sg)) + Subgraph::try_from(sg) }) } /// Batch neighbor query: fetch neighbors for multiple nodes in one call. - /// Returns dict[int, list[PyNeighborEntry]] mapping each queried node_id to its neighbors. - #[pyo3(signature = (node_ids, *, direction="outgoing", type_filter=None, at_epoch=None, decay_lambda=None))] + /// Returns dict[int, list[NeighborEntry]] mapping each queried node_id to its neighbors. + #[pyo3(signature = (node_ids, *, direction="outgoing", edge_label_filter=None, at_epoch=None, decay_lambda=None))] fn neighbors_batch( &self, py: Python<'_>, node_ids: Vec, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, at_epoch: Option, decay_lambda: Option, - ) -> PyResult>> { + ) -> PyResult>> { let dir = parse_direction(direction)?; let dl = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter, limit: None, at_epoch, decay_lambda: dl, @@ -640,80 +790,80 @@ impl OverGraph { let map = eng.neighbors_batch(&node_ids, &opts)?; Ok(map .into_iter() - .map(|(k, v)| (k, v.into_iter().map(PyNeighborEntry::from).collect())) + .map(|(k, v)| (k, v.into_iter().map(NeighborEntry::from).collect())) .collect()) }) } // --- Degree counts + aggregations (Phase 18a) --- - #[pyo3(signature = (node_id, *, direction="outgoing", type_filter=None, at_epoch=None))] + #[pyo3(signature = (node_id, *, direction="outgoing", edge_label_filter=None, at_epoch=None))] fn degree( &self, py: Python<'_>, node_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, at_epoch: Option, ) -> PyResult { let dir = parse_direction(direction)?; let opts = DegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; with_engine_ref(self, py, move |eng| eng.degree(node_id, &opts)) } - #[pyo3(signature = (node_id, *, direction="outgoing", type_filter=None, at_epoch=None))] + #[pyo3(signature = (node_id, *, direction="outgoing", edge_label_filter=None, at_epoch=None))] fn sum_edge_weights( &self, py: Python<'_>, node_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, at_epoch: Option, ) -> PyResult { let dir = parse_direction(direction)?; let opts = DegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; with_engine_ref(self, py, move |eng| eng.sum_edge_weights(node_id, &opts)) } - #[pyo3(signature = (node_id, *, direction="outgoing", type_filter=None, at_epoch=None))] + #[pyo3(signature = (node_id, *, direction="outgoing", edge_label_filter=None, at_epoch=None))] fn avg_edge_weight( &self, py: Python<'_>, node_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, at_epoch: Option, ) -> PyResult> { let dir = parse_direction(direction)?; let opts = DegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; with_engine_ref(self, py, move |eng| eng.avg_edge_weight(node_id, &opts)) } - #[pyo3(signature = (node_ids, *, direction="outgoing", type_filter=None, at_epoch=None))] + #[pyo3(signature = (node_ids, *, direction="outgoing", edge_label_filter=None, at_epoch=None))] fn degrees( &self, py: Python<'_>, node_ids: Vec, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, at_epoch: Option, ) -> PyResult> { let dir = parse_direction(direction)?; let opts = DegreeOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, }; with_engine_ref(self, py, move |eng| eng.degrees(&node_ids, &opts)) @@ -721,23 +871,23 @@ impl OverGraph { // --- Shortest path (Phase 18b) --- - #[pyo3(signature = (from_id, to_id, *, direction="outgoing", type_filter=None, weight_field=None, at_epoch=None, max_depth=None, max_cost=None))] + #[pyo3(signature = (from_id, to_id, *, direction="outgoing", edge_label_filter=None, weight_field=None, at_epoch=None, max_depth=None, max_cost=None))] fn shortest_path( &self, py: Python<'_>, from_id: u64, to_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, weight_field: Option<&str>, at_epoch: Option, max_depth: Option, max_cost: Option, - ) -> PyResult> { + ) -> PyResult> { let dir = parse_direction(direction)?; let opts = ShortestPathOptions { direction: dir, - type_filter, + edge_label_filter, weight_field: weight_field.map(|s| s.to_string()), at_epoch, max_depth, @@ -746,49 +896,49 @@ impl OverGraph { with_engine_ref(self, py, move |eng| { Ok(eng .shortest_path(from_id, to_id, &opts)? - .map(PyShortestPath::from)) + .map(ShortestPath::from)) }) } - #[pyo3(signature = (from_id, to_id, *, direction="outgoing", type_filter=None, at_epoch=None, max_depth=None))] + #[pyo3(signature = (from_id, to_id, *, direction="outgoing", edge_label_filter=None, at_epoch=None, max_depth=None))] fn is_connected( &self, py: Python<'_>, from_id: u64, to_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, at_epoch: Option, max_depth: Option, ) -> PyResult { let dir = parse_direction(direction)?; let opts = IsConnectedOptions { direction: dir, - type_filter, + edge_label_filter, at_epoch, max_depth, }; with_engine_ref(self, py, move |eng| eng.is_connected(from_id, to_id, &opts)) } - #[pyo3(signature = (from_id, to_id, *, direction="outgoing", type_filter=None, weight_field=None, at_epoch=None, max_depth=None, max_cost=None, max_paths=None))] + #[pyo3(signature = (from_id, to_id, *, direction="outgoing", edge_label_filter=None, weight_field=None, at_epoch=None, max_depth=None, max_cost=None, max_paths=None))] fn all_shortest_paths( &self, py: Python<'_>, from_id: u64, to_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, weight_field: Option<&str>, at_epoch: Option, max_depth: Option, max_cost: Option, max_paths: Option, - ) -> PyResult> { + ) -> PyResult> { let dir = parse_direction(direction)?; let opts = AllShortestPathsOptions { direction: dir, - type_filter, + edge_label_filter, weight_field: weight_field.map(|s| s.to_string()), at_epoch, max_depth, @@ -799,7 +949,7 @@ impl OverGraph { Ok(eng .all_shortest_paths(from_id, to_id, &opts)? .into_iter() - .map(PyShortestPath::from) + .map(ShortestPath::from) .collect()) }) } @@ -809,12 +959,13 @@ impl OverGraph { /// Batch upsert nodes from a packed binary buffer. /// /// Binary format (little-endian): - /// [count: u32] + /// [magic: "OGNB"][version: u16 = 2][count: u32] /// per node: - /// [type_id: u32][weight: f32][key_len: u16][key: utf8][props_len: u32][props: json utf8] + /// [label_count: u8] repeated labels: [label_len: u16][label: utf8] + /// [weight: f32][key_len: u16][key: utf8][props_len: u32][props: json utf8] fn batch_upsert_nodes_binary(&self, py: Python<'_>, buffer: &[u8]) -> PyResult> { let inputs = decode_node_batch_py(buffer)?; - with_engine(self, py, move |eng| eng.batch_upsert_nodes(&inputs)) + with_engine(self, py, move |eng| eng.batch_upsert_nodes(inputs)) } /// Batch upsert edges from a packed binary buffer. @@ -822,46 +973,46 @@ impl OverGraph { /// Binary format (little-endian): /// [count: u32] /// per edge: - /// [from: u64][to: u64][type_id: u32][weight: f32] + /// [from: u64][to: u64][label_len: u16][label: utf8][weight: f32] /// [valid_from: i64][valid_to: i64][props_len: u32][props: json utf8] fn batch_upsert_edges_binary(&self, py: Python<'_>, buffer: &[u8]) -> PyResult> { let inputs = decode_edge_batch_py(buffer)?; - with_engine(self, py, move |eng| eng.batch_upsert_edges(&inputs)) + with_engine(self, py, move |eng| eng.batch_upsert_edges(inputs)) } // --- Retention --- - #[pyo3(signature = (*, max_age_ms=None, max_weight=None, type_id=None))] + #[pyo3(signature = (*, max_age_ms=None, max_weight=None, label=None))] fn prune( &self, py: Python<'_>, max_age_ms: Option, max_weight: Option, - type_id: Option, - ) -> PyResult { + label: Option, + ) -> PyResult { let policy = PrunePolicy { max_age_ms, max_weight: max_weight.map(|v| v as f32), - type_id, + label, }; with_engine(self, py, move |eng| { - Ok(PyPruneResult::from(eng.prune(&policy)?)) + Ok(PruneResult::from(eng.prune(&policy)?)) }) } - #[pyo3(signature = (name, *, max_age_ms=None, max_weight=None, type_id=None))] + #[pyo3(signature = (name, *, max_age_ms=None, max_weight=None, label=None))] fn set_prune_policy( &self, py: Python<'_>, name: String, max_age_ms: Option, max_weight: Option, - type_id: Option, + label: Option, ) -> PyResult<()> { let policy = PrunePolicy { max_age_ms, max_weight: max_weight.map(|v| v as f32), - type_id, + label, }; with_engine(self, py, move |eng| eng.set_prune_policy(&name, policy)) } @@ -870,17 +1021,12 @@ impl OverGraph { with_engine(self, py, move |eng| eng.remove_prune_policy(&name)) } - fn list_prune_policies(&self, py: Python<'_>) -> PyResult> { + fn list_prune_policies(&self, py: Python<'_>) -> PyResult> { with_engine_ref(self, py, |eng| { Ok(eng .list_prune_policies()? .into_iter() - .map(|(name, policy)| PyNamedPrunePolicy { - name, - max_age_ms: policy.max_age_ms, - max_weight: policy.max_weight.map(|w| w as f64), - type_id: policy.type_id, - }) + .map(NamedPrunePolicy::from) .collect()) }) } @@ -891,9 +1037,9 @@ impl OverGraph { with_engine_ref(self, py, |eng| eng.sync()) } - fn flush(&self, py: Python<'_>) -> PyResult> { + fn flush(&self, py: Python<'_>) -> PyResult> { with_engine(self, py, |eng| { - Ok(eng.flush()?.map(|si| PySegmentInfo { + Ok(eng.flush()?.map(|si| SegmentInfo { id: si.id, node_count: si.node_count, edge_count: si.edge_count, @@ -905,15 +1051,15 @@ impl OverGraph { with_engine(self, py, |eng| eng.ingest_mode()) } - fn end_ingest(&self, py: Python<'_>) -> PyResult> { + fn end_ingest(&self, py: Python<'_>) -> PyResult> { with_engine(self, py, |eng| { - Ok(eng.end_ingest()?.map(PyCompactionStats::from)) + Ok(eng.end_ingest()?.map(CompactionStats::from)) }) } - fn compact(&self, py: Python<'_>) -> PyResult> { + fn compact(&self, py: Python<'_>) -> PyResult> { with_engine(self, py, |eng| { - Ok(eng.compact()?.map(PyCompactionStats::from)) + Ok(eng.compact()?.map(CompactionStats::from)) }) } @@ -921,7 +1067,7 @@ impl OverGraph { &self, py: Python<'_>, callback: PyObject, - ) -> PyResult> { + ) -> PyResult> { let engine = clone_engine_handle(&self.inner)?; let captured_err: Arc>> = Arc::new(std::sync::Mutex::new(None)); @@ -933,7 +1079,7 @@ impl OverGraph { let result = engine .compact_with_progress(|progress| { Python::with_gil(|py| { - let py_progress = PyCompactionProgress::from(progress); + let py_progress = CompactionProgress::from(progress); match callback.call1(py, (py_progress,)) { Ok(result) => result.extract::(py).unwrap_or(true), Err(e) => { @@ -944,7 +1090,7 @@ impl OverGraph { }) }) .map_err(to_py_err)?; - Ok(result.map(PyCompactionStats::from)) + Ok(result.map(CompactionStats::from)) }); // If cancellation was due to a Python error, re-raise that instead if let Some(py_err) = captured_err.lock().unwrap().take() { @@ -955,158 +1101,160 @@ impl OverGraph { // --- Pagination --- - #[pyo3(signature = (type_id, *, limit=None, after=None))] - fn nodes_by_type_paged( + #[pyo3(signature = (labels, *, limit=None, after=None))] + fn nodes_by_labels_paged( &self, py: Python<'_>, - type_id: u32, + labels: &Bound<'_, PyAny>, limit: Option, after: Option, - ) -> PyResult { + ) -> PyResult { + let labels = parse_node_labels_arg(labels, "nodes_by_labels_paged labels")?; let page = PageRequest { limit, after }; with_engine_ref(self, py, move |eng| { - Ok(PyIdPageResult::from( - eng.nodes_by_type_paged(type_id, &page)?, + Ok(IdPageResult::from( + eng.nodes_by_labels_paged(labels, &page)?, )) }) } - #[pyo3(signature = (type_id, *, limit=None, after=None))] - fn edges_by_type_paged( + #[pyo3(signature = (label, *, limit=None, after=None))] + fn edges_by_label_paged( &self, py: Python<'_>, - type_id: u32, + label: String, limit: Option, after: Option, - ) -> PyResult { + ) -> PyResult { let page = PageRequest { limit, after }; with_engine_ref(self, py, move |eng| { - Ok(PyIdPageResult::from( - eng.edges_by_type_paged(type_id, &page)?, - )) + Ok(IdPageResult::from(eng.edges_by_label_paged(&label, &page)?)) }) } - #[pyo3(signature = (type_id, *, limit=None, after=None))] - fn get_nodes_by_type_paged( + #[pyo3(signature = (labels, *, limit=None, after=None))] + fn get_nodes_by_labels_paged( &self, py: Python<'_>, - type_id: u32, + labels: &Bound<'_, PyAny>, limit: Option, after: Option, - ) -> PyResult { + ) -> PyResult { + let labels = parse_node_labels_arg(labels, "get_nodes_by_labels_paged labels")?; let page = PageRequest { limit, after }; with_engine_ref(self, py, move |eng| { - let result = eng.get_nodes_by_type_paged(type_id, &page)?; - Ok(PyNodePageResult { - items: result.items.into_iter().map(PyNodeRecord::from).collect(), + let result = eng.get_nodes_by_labels_paged(labels, &page)?; + Ok(NodePageResult { + items: result + .items + .into_iter() + .map(NodeView::try_from) + .collect::, EngineError>>()?, next_cursor: result.next_cursor, }) }) } - #[pyo3(signature = (type_id, *, limit=None, after=None))] - fn get_edges_by_type_paged( + #[pyo3(signature = (label, *, limit=None, after=None))] + fn get_edges_by_label_paged( &self, py: Python<'_>, - type_id: u32, + label: String, limit: Option, after: Option, - ) -> PyResult { + ) -> PyResult { let page = PageRequest { limit, after }; with_engine_ref(self, py, move |eng| { - let result = eng.get_edges_by_type_paged(type_id, &page)?; - Ok(PyEdgePageResult { - items: result.items.into_iter().map(PyEdgeRecord::from).collect(), + let result = eng.get_edges_by_label_paged(&label, &page)?; + Ok(EdgePageResult { + items: result.items.into_iter().map(EdgeView::from).collect(), next_cursor: result.next_cursor, }) }) } - #[pyo3(signature = (type_id, prop_key, prop_value, *, limit=None, after=None))] + #[pyo3(signature = (label, prop_key, prop_value, *, limit=None, after=None))] fn find_nodes_paged( &self, py: Python<'_>, - type_id: u32, + label: String, prop_key: String, prop_value: PyObject, limit: Option, after: Option, - ) -> PyResult { + ) -> PyResult { let pv = py_to_prop_value(py, prop_value.bind(py))?; let page = PageRequest { limit, after }; with_engine_ref(self, py, move |eng| { - Ok(PyIdPageResult::from( - eng.find_nodes_paged(type_id, &prop_key, &pv, &page)?, + Ok(IdPageResult::from( + eng.find_nodes_paged(&label, &prop_key, &pv, &page)?, )) }) } - #[pyo3(signature = (type_id, from_ms, to_ms, *, limit=None, after=None))] + #[pyo3(signature = (label, from_ms, to_ms, *, limit=None, after=None))] fn find_nodes_by_time_range_paged( &self, py: Python<'_>, - type_id: u32, + label: String, from_ms: i64, to_ms: i64, limit: Option, after: Option, - ) -> PyResult { + ) -> PyResult { let page = PageRequest { limit, after }; with_engine_ref(self, py, move |eng| { - Ok(PyIdPageResult::from(eng.find_nodes_by_time_range_paged( - type_id, from_ms, to_ms, &page, + Ok(IdPageResult::from(eng.find_nodes_by_time_range_paged( + &label, from_ms, to_ms, &page, )?)) }) } - #[pyo3(signature = (type_id, prop_key, lower=None, upper=None, *, limit=None, after=None))] + #[pyo3(signature = (label, prop_key, lower=None, upper=None, *, limit=None, after=None))] fn find_nodes_range_paged( &self, py: Python<'_>, - type_id: u32, + label: String, prop_key: String, - lower: Option, - upper: Option, + lower: Option, + upper: Option, limit: Option, - after: Option, - ) -> PyResult { - let lower = lower.map(PropertyRangeBound::from); - let upper = upper.map(PropertyRangeBound::from); + after: Option, + ) -> PyResult { + let lower = lower.map(CorePropertyRangeBound::from); + let upper = upper.map(CorePropertyRangeBound::from); let page = PropertyRangePageRequest { limit, - after: after.map(PropertyRangeCursor::from), + after: after.map(CorePropertyRangeCursor::from), }; with_engine_ref(self, py, move |eng| { - Ok(PyPropertyRangePageResult::from( - eng.find_nodes_range_paged( - type_id, - &prop_key, - lower.as_ref(), - upper.as_ref(), - &page, - )?, - )) + Ok(PropertyRangePageResult::from(eng.find_nodes_range_paged( + &label, + &prop_key, + lower.as_ref(), + upper.as_ref(), + &page, + )?)) }) } - #[pyo3(signature = (node_id, *, direction="outgoing", type_filter=None, limit=None, after=None, at_epoch=None, decay_lambda=None))] + #[pyo3(signature = (node_id, *, direction="outgoing", edge_label_filter=None, limit=None, after=None, at_epoch=None, decay_lambda=None))] fn neighbors_paged( &self, py: Python<'_>, node_id: u64, direction: &str, - type_filter: Option>, + edge_label_filter: Option>, limit: Option, after: Option, at_epoch: Option, decay_lambda: Option, - ) -> PyResult { + ) -> PyResult { let dir = parse_direction(direction)?; let dl = decay_lambda.map(|v| v as f32); let opts = NeighborOptions { direction: dir, - type_filter, + edge_label_filter, limit: None, at_epoch, decay_lambda: dl, @@ -1114,12 +1262,8 @@ impl OverGraph { let page = PageRequest { limit, after }; with_engine_ref(self, py, move |eng| { let result = eng.neighbors_paged(node_id, &opts, &page)?; - Ok(PyNeighborPageResult { - items: result - .items - .into_iter() - .map(PyNeighborEntry::from) - .collect(), + Ok(NeighborPageResult { + items: result.items.into_iter().map(NeighborEntry::from).collect(), next_cursor: result.next_cursor, }) }) @@ -1127,7 +1271,7 @@ impl OverGraph { // --- Analytics --- - #[pyo3(signature = (seed_node_ids, *, algorithm=None, damping_factor=None, max_iterations=None, epsilon=None, approx_residual_tolerance=None, edge_type_filter=None, max_results=None))] + #[pyo3(signature = (seed_node_ids, *, algorithm=None, damping_factor=None, max_iterations=None, epsilon=None, approx_residual_tolerance=None, edge_label_filter=None, max_results=None))] fn personalized_pagerank( &self, py: Python<'_>, @@ -1137,9 +1281,9 @@ impl OverGraph { max_iterations: Option, epsilon: Option, approx_residual_tolerance: Option, - edge_type_filter: Option>, + edge_label_filter: Option>, max_results: Option, - ) -> PyResult { + ) -> PyResult { let defaults = PprOptions::default(); let options = PprOptions { algorithm: parse_ppr_algorithm(algorithm)?, @@ -1148,31 +1292,35 @@ impl OverGraph { epsilon: epsilon.unwrap_or(defaults.epsilon), approx_residual_tolerance: approx_residual_tolerance .unwrap_or(defaults.approx_residual_tolerance), - edge_type_filter, + edge_label_filter, max_results, }; with_engine_ref(self, py, move |eng| { - Ok(PyPprResult::from( + Ok(PprResult::from( eng.personalized_pagerank(&seed_node_ids, &options)?, )) }) } - #[pyo3(signature = (*, node_type_filter=None, edge_type_filter=None, include_weights=true))] + #[pyo3(signature = (*, node_label_filter=None, edge_label_filter=None, include_weights=true))] fn export_adjacency( &self, py: Python<'_>, - node_type_filter: Option>, - edge_type_filter: Option>, + node_label_filter: Option<&Bound<'_, PyAny>>, + edge_label_filter: Option>, include_weights: bool, - ) -> PyResult { + ) -> PyResult { + let node_label_filter = parse_optional_node_label_filter_arg( + node_label_filter, + "export_adjacency node_label_filter", + )?; let options = ExportOptions { - node_type_filter, - edge_type_filter, + node_label_filter, + edge_label_filter, include_weights, }; with_engine_ref(self, py, move |eng| { - Ok(PyAdjacencyExport::from(eng.export_adjacency(&options)?)) + Ok(AdjacencyExport::from(eng.export_adjacency(&options)?)) }) } @@ -1183,17 +1331,21 @@ impl OverGraph { /// Returns a dict mapping each visible node ID to its component ID /// (the minimum node ID in that component). WCC treats all edges as /// undirected. Isolated nodes become singleton components. - #[pyo3(signature = (*, edge_type_filter=None, node_type_filter=None, at_epoch=None))] + #[pyo3(signature = (*, edge_label_filter=None, node_label_filter=None, at_epoch=None))] fn connected_components( &self, py: Python<'_>, - edge_type_filter: Option>, - node_type_filter: Option>, + edge_label_filter: Option>, + node_label_filter: Option<&Bound<'_, PyAny>>, at_epoch: Option, ) -> PyResult> { + let node_label_filter = parse_optional_node_label_filter_arg( + node_label_filter, + "connected_components node_label_filter", + )?; let opts = ComponentOptions { - edge_type_filter, - node_type_filter, + edge_label_filter, + node_label_filter, at_epoch, }; with_engine_ref(self, py, move |eng| eng.connected_components(&opts)) @@ -1202,18 +1354,22 @@ impl OverGraph { /// Returns the sorted list of node IDs in the same weakly connected /// component as the given node. Returns an empty list if the node /// doesn't exist, is deleted, or is hidden by prune policy. - #[pyo3(signature = (node_id, *, edge_type_filter=None, node_type_filter=None, at_epoch=None))] + #[pyo3(signature = (node_id, *, edge_label_filter=None, node_label_filter=None, at_epoch=None))] fn component_of( &self, py: Python<'_>, node_id: u64, - edge_type_filter: Option>, - node_type_filter: Option>, + edge_label_filter: Option>, + node_label_filter: Option<&Bound<'_, PyAny>>, at_epoch: Option, ) -> PyResult> { + let node_label_filter = parse_optional_node_label_filter_arg( + node_label_filter, + "component_of node_label_filter", + )?; let opts = ComponentOptions { - edge_type_filter, - node_type_filter, + edge_label_filter, + node_label_filter, at_epoch, }; with_engine_ref(self, py, move |eng| eng.component_of(node_id, &opts)) @@ -1221,7 +1377,7 @@ impl OverGraph { // --- Vector search (Phase 19) --- - #[pyo3(signature = (mode, k, *, dense_query=None, sparse_query=None, type_filter=None, ef_search=None, scope_start_node_id=None, scope_max_depth=None, scope_direction=None, scope_edge_type_filter=None, scope_at_epoch=None, dense_weight=None, sparse_weight=None, fusion_mode=None))] + #[pyo3(signature = (mode, k, *, dense_query=None, sparse_query=None, label_filter=None, ef_search=None, scope_start_node_id=None, scope_max_depth=None, scope_direction=None, scope_edge_label_filter=None, scope_at_epoch=None, dense_weight=None, sparse_weight=None, fusion_mode=None))] fn vector_search( &self, py: Python<'_>, @@ -1229,18 +1385,20 @@ impl OverGraph { k: usize, dense_query: Option>, sparse_query: Option>, - type_filter: Option>, + label_filter: Option<&Bound<'_, PyAny>>, ef_search: Option, scope_start_node_id: Option, scope_max_depth: Option, scope_direction: Option<&str>, - scope_edge_type_filter: Option>, + scope_edge_label_filter: Option>, scope_at_epoch: Option, dense_weight: Option, sparse_weight: Option, fusion_mode: Option<&str>, - ) -> PyResult> { + ) -> PyResult> { let mode = parse_vector_search_mode(mode)?; + let label_filter = + parse_optional_node_label_filter_arg(label_filter, "vector_search label_filter")?; let fusion = parse_fusion_mode(fusion_mode)?; let scope = match scope_start_node_id { None => None, @@ -1252,7 +1410,7 @@ impl OverGraph { ) })?, direction: parse_direction(scope_direction.unwrap_or("outgoing"))?, - edge_type_filter: scope_edge_type_filter, + edge_label_filter: scope_edge_label_filter, at_epoch: scope_at_epoch, }), }; @@ -1261,7 +1419,7 @@ impl OverGraph { dense_query, sparse_query, k, - type_filter, + label_filter, ef_search, scope, dense_weight, @@ -1272,7 +1430,7 @@ impl OverGraph { let hits = eng.vector_search(&request)?; Ok(hits .into_iter() - .map(|h| PyVectorHit { + .map(|h| VectorHit { node_id: h.node_id, score: h.score as f64, }) @@ -1282,23 +1440,27 @@ impl OverGraph { } #[pyclass] -pub struct PyWriteTxn { - inner: Arc>>, +pub struct WriteTxn { + inner: Arc>>, } #[pymethods] -impl PyWriteTxn { - #[pyo3(signature = (type_id, key, *, props=None, weight=1.0, dense_vector=None, sparse_vector=None))] +impl WriteTxn { + #[pyo3(signature = (labels, key, *, props=None, weight=1.0, dense_vector=None, sparse_vector=None))] fn upsert_node( &self, py: Python<'_>, - type_id: u32, + labels: &Bound<'_, PyAny>, key: String, props: Option<&Bound<'_, PyDict>>, weight: f64, dense_vector: Option>, sparse_vector: Option>, ) -> PyResult { + let labels = parse_node_labels_arg(labels, "transaction upsert_node labels")?; + let key_ref_label = labels.first().cloned().ok_or_else(|| { + PyValueError::new_err("transaction upsert_node labels must not be empty") + })?; let options = UpsertNodeOptions { props: convert_py_props(py, props)?, weight: weight as f32, @@ -1306,23 +1468,30 @@ impl PyWriteTxn { sparse_vector, }; with_py_txn(&self.inner, |txn| { - txn.upsert_node(type_id, &key, options).map_err(to_py_err) + txn.upsert_node(labels, &key, options).map_err(to_py_err) })?; - txn_node_ref_to_py(py, TxnNodeRef::Key { type_id, key }) + txn_node_ref_to_py( + py, + TxnNodeRef::Key { + label: key_ref_label, + key, + }, + ) } - #[pyo3(signature = (alias, type_id, key, *, props=None, weight=1.0, dense_vector=None, sparse_vector=None))] + #[pyo3(signature = (alias, labels, key, *, props=None, weight=1.0, dense_vector=None, sparse_vector=None))] fn upsert_node_as( &self, py: Python<'_>, alias: String, - type_id: u32, + labels: &Bound<'_, PyAny>, key: String, props: Option<&Bound<'_, PyDict>>, weight: f64, dense_vector: Option>, sparse_vector: Option>, ) -> PyResult { + let labels = parse_node_labels_arg(labels, "transaction upsert_node_as labels")?; let options = UpsertNodeOptions { props: convert_py_props(py, props)?, weight: weight as f32, @@ -1330,19 +1499,43 @@ impl PyWriteTxn { sparse_vector, }; with_py_txn(&self.inner, |txn| { - txn.upsert_node_as(&alias, type_id, &key, options) + txn.upsert_node_as(&alias, labels, &key, options) .map_err(to_py_err) .and_then(|r| txn_node_ref_to_py(py, r)) }) } - #[pyo3(signature = (from_ref, to_ref, type_id, *, props=None, weight=1.0, valid_from=None, valid_to=None))] + fn add_node_label( + &self, + _py: Python<'_>, + target: &Bound<'_, PyDict>, + label: String, + ) -> PyResult { + let target = parse_txn_node_ref(target)?; + with_py_txn(&self.inner, |txn| { + txn.add_node_label(target, &label).map_err(to_py_err) + }) + } + + fn remove_node_label( + &self, + _py: Python<'_>, + target: &Bound<'_, PyDict>, + label: String, + ) -> PyResult { + let target = parse_txn_node_ref(target)?; + with_py_txn(&self.inner, |txn| { + txn.remove_node_label(target, &label).map_err(to_py_err) + }) + } + + #[pyo3(signature = (from_ref, to_ref, label, *, props=None, weight=1.0, valid_from=None, valid_to=None))] fn upsert_edge( &self, py: Python<'_>, from_ref: &Bound<'_, PyDict>, to_ref: &Bound<'_, PyDict>, - type_id: u32, + label: String, props: Option<&Bound<'_, PyDict>>, weight: f64, valid_from: Option, @@ -1357,20 +1550,20 @@ impl PyWriteTxn { valid_to, }; with_py_txn(&self.inner, |txn| { - txn.upsert_edge(from.clone(), to.clone(), type_id, options) + txn.upsert_edge(from.clone(), to.clone(), &label, options) .map_err(to_py_err) })?; - txn_edge_ref_to_py(py, TxnEdgeRef::Triple { from, to, type_id }) + txn_edge_ref_to_py(py, TxnEdgeRef::Triple { from, to, label }) } - #[pyo3(signature = (alias, from_ref, to_ref, type_id, *, props=None, weight=1.0, valid_from=None, valid_to=None))] + #[pyo3(signature = (alias, from_ref, to_ref, label, *, props=None, weight=1.0, valid_from=None, valid_to=None))] fn upsert_edge_as( &self, py: Python<'_>, alias: String, from_ref: &Bound<'_, PyDict>, to_ref: &Bound<'_, PyDict>, - type_id: u32, + label: String, props: Option<&Bound<'_, PyDict>>, weight: f64, valid_from: Option, @@ -1385,7 +1578,7 @@ impl PyWriteTxn { valid_to, }; with_py_txn(&self.inner, |txn| { - txn.upsert_edge_as(&alias, from, to, type_id, options) + txn.upsert_edge_as(&alias, from, to, &label, options) .map_err(to_py_err) .and_then(|r| txn_edge_ref_to_py(py, r)) }) @@ -1442,11 +1635,11 @@ impl PyWriteTxn { fn get_node_by_key( &self, py: Python<'_>, - type_id: u32, + label: String, key: String, ) -> PyResult> { with_py_txn_ref(&self.inner, |txn| { - txn.get_node_by_key(type_id, &key) + txn.get_node_by_key(&label, &key) .map_err(to_py_err)? .map(|v| txn_node_view_to_py(py, v)) .transpose() @@ -1458,19 +1651,19 @@ impl PyWriteTxn { py: Python<'_>, from_ref: &Bound<'_, PyDict>, to_ref: &Bound<'_, PyDict>, - type_id: u32, + label: String, ) -> PyResult> { let from = parse_txn_node_ref(from_ref)?; let to = parse_txn_node_ref(to_ref)?; with_py_txn_ref(&self.inner, |txn| { - txn.get_edge_by_triple(from, to, type_id) + txn.get_edge_by_triple(from, to, &label) .map_err(to_py_err)? .map(|v| txn_edge_view_to_py(py, v)) .transpose() }) } - fn commit(&self, py: Python<'_>) -> PyResult { + fn commit(&self, py: Python<'_>) -> PyResult { let mut txn = { let mut guard = self.inner.lock().map_err(lock_err)?; guard @@ -1478,7 +1671,7 @@ impl PyWriteTxn { .ok_or_else(|| OverGraphError::new_err(EngineError::TxnClosed.to_string()))? }; let result = py.allow_threads(move || txn.commit()).map_err(to_py_err)?; - Ok(PyTxnCommitResult::from(result)) + Ok(TxnCommitResult::from(result)) } fn rollback(&self) -> PyResult<()> { @@ -1492,7 +1685,7 @@ impl PyWriteTxn { #[pyclass] #[derive(Clone)] -pub struct PyDbStats { +pub struct DbStats { #[pyo3(get)] pub pending_wal_bytes: usize, #[pyo3(get)] @@ -1519,9 +1712,9 @@ pub struct PyDbStats { pub oldest_retained_wal_generation_id: u64, } -impl From for PyDbStats { - fn from(s: DbStats) -> Self { - PyDbStats { +impl From for DbStats { + fn from(s: CoreDbStats) -> Self { + DbStats { pending_wal_bytes: s.pending_wal_bytes, segment_count: s.segment_count, node_tombstone_count: s.node_tombstone_count, @@ -1539,7 +1732,7 @@ impl From for PyDbStats { } #[pymethods] -impl PyDbStats { +impl DbStats { fn __repr__(&self) -> String { format!( "DbStats(segments={}, wal_bytes={}, tombstones=({}, {}), sync='{}', \ @@ -1558,11 +1751,138 @@ impl PyDbStats { #[pyclass] #[derive(Clone)] -pub struct PyNodeRecord { +pub struct ScrubReport { + #[pyo3(get)] + pub total_components_checked: u64, + #[pyo3(get)] + pub total_components_ok: u64, + #[pyo3(get)] + pub total_components_failed: u64, + #[pyo3(get)] + pub total_bytes_digested: u64, + #[pyo3(get)] + pub duration_ms: u64, + segments_internal: Vec, +} + +impl From for ScrubReport { + fn from(r: CoreScrubReport) -> Self { + ScrubReport { + total_components_checked: r.total_components_checked, + total_components_ok: r.total_components_ok, + total_components_failed: r.total_components_failed, + total_bytes_digested: r.total_bytes_digested, + duration_ms: r.duration_ms, + segments_internal: r + .segments + .into_iter() + .map(SegmentScrubResult::from) + .collect(), + } + } +} + +#[pymethods] +impl ScrubReport { + #[getter] + fn segments(&self) -> Vec { + self.segments_internal.clone() + } + + fn __repr__(&self) -> String { + format!( + "ScrubReport(segments={}, checked={}, ok={}, failed={}, duration_ms={})", + self.segments_internal.len(), + self.total_components_checked, + self.total_components_ok, + self.total_components_failed, + self.duration_ms, + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct SegmentScrubResult { + #[pyo3(get)] + pub segment_id: u64, + #[pyo3(get)] + pub components_ok: u64, + #[pyo3(get)] + pub bytes_digested: u64, + findings_internal: Vec, +} + +impl From for SegmentScrubResult { + fn from(s: CoreSegmentScrubResult) -> Self { + SegmentScrubResult { + segment_id: s.segment_id, + components_ok: s.components_ok, + bytes_digested: s.bytes_digested, + findings_internal: s + .findings + .into_iter() + .map(ComponentScrubFinding::from) + .collect(), + } + } +} + +#[pymethods] +impl SegmentScrubResult { + #[getter] + fn findings(&self) -> Vec { + self.findings_internal.clone() + } + + fn __repr__(&self) -> String { + format!( + "SegmentScrubResult(segment_id={}, ok={}, findings={})", + self.segment_id, + self.components_ok, + self.findings_internal.len(), + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct ComponentScrubFinding { + #[pyo3(get)] + pub component_kind: String, + #[pyo3(get)] + pub finding_type: String, + #[pyo3(get)] + pub detail: String, +} + +impl From for ComponentScrubFinding { + fn from(f: CoreComponentScrubFinding) -> Self { + ComponentScrubFinding { + component_kind: f.component_kind, + finding_type: format!("{:?}", f.finding_type), + detail: f.detail, + } + } +} + +#[pymethods] +impl ComponentScrubFinding { + fn __repr__(&self) -> String { + format!( + "ScrubFinding(kind='{}', type='{}', detail='{}')", + self.component_kind, self.finding_type, self.detail, + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct NodeView { #[pyo3(get)] pub id: u64, #[pyo3(get)] - pub type_id: u32, + pub labels: Vec, #[pyo3(get)] pub key: String, #[pyo3(get)] @@ -1571,25 +1891,33 @@ pub struct PyNodeRecord { pub updated_at: i64, #[pyo3(get)] pub weight: f64, + #[pyo3(get)] + pub dense_vector: Option>, + #[pyo3(get)] + pub sparse_vector: Option>, props_internal: BTreeMap, } -impl From for PyNodeRecord { - fn from(n: NodeRecord) -> Self { - PyNodeRecord { +impl TryFrom for NodeView { + type Error = EngineError; + + fn try_from(n: CoreNodeView) -> Result { + Ok(NodeView { id: n.id, - type_id: n.type_id, + labels: n.labels, key: n.key, created_at: n.created_at, updated_at: n.updated_at, weight: n.weight as f64, + dense_vector: n.dense_vector, + sparse_vector: n.sparse_vector, props_internal: n.props, - } + }) } } #[pymethods] -impl PyNodeRecord { +impl NodeView { #[getter] fn props(&self, py: Python<'_>) -> PyResult { props_to_py(py, &self.props_internal) @@ -1597,15 +1925,15 @@ impl PyNodeRecord { fn __repr__(&self) -> String { format!( - "NodeRecord(id={}, type_id={}, key='{}')", - self.id, self.type_id, self.key + "NodeView(id={}, labels={:?}, key='{}')", + self.id, self.labels, self.key ) } } #[pyclass] #[derive(Clone)] -pub struct PyEdgeRecord { +pub struct EdgeView { #[pyo3(get)] pub id: u64, #[pyo3(get)] @@ -1613,7 +1941,7 @@ pub struct PyEdgeRecord { #[pyo3(get)] pub to_id: u64, #[pyo3(get)] - pub type_id: u32, + pub label: String, #[pyo3(get)] pub created_at: i64, #[pyo3(get)] @@ -1627,13 +1955,13 @@ pub struct PyEdgeRecord { props_internal: BTreeMap, } -impl From for PyEdgeRecord { - fn from(e: EdgeRecord) -> Self { - PyEdgeRecord { +impl From for EdgeView { + fn from(e: CoreEdgeView) -> Self { + EdgeView { id: e.id, from_id: e.from, to_id: e.to, - type_id: e.type_id, + label: e.label, created_at: e.created_at, updated_at: e.updated_at, weight: e.weight as f64, @@ -1645,7 +1973,7 @@ impl From for PyEdgeRecord { } #[pymethods] -impl PyEdgeRecord { +impl EdgeView { #[getter] fn props(&self, py: Python<'_>) -> PyResult { props_to_py(py, &self.props_internal) @@ -1653,15 +1981,15 @@ impl PyEdgeRecord { fn __repr__(&self) -> String { format!( - "EdgeRecord(id={}, {}->{})", - self.id, self.from_id, self.to_id + "EdgeView(id={}, {}->{}, label='{}')", + self.id, self.from_id, self.to_id, self.label ) } } #[pyclass] #[derive(Clone)] -pub struct PyPatchResult { +pub struct PatchResult { #[pyo3(get)] pub node_ids: Vec, #[pyo3(get)] @@ -1669,7 +1997,7 @@ pub struct PyPatchResult { } #[pymethods] -impl PyPatchResult { +impl PatchResult { fn __repr__(&self) -> String { format!( "PatchResult(nodes={}, edges={})", @@ -1681,7 +2009,7 @@ impl PyPatchResult { #[pyclass] #[derive(Clone)] -pub struct PyTxnCommitResult { +pub struct TxnCommitResult { #[pyo3(get)] pub node_ids: Vec, #[pyo3(get)] @@ -1692,8 +2020,8 @@ pub struct PyTxnCommitResult { pub edge_aliases: HashMap, } -impl From for PyTxnCommitResult { - fn from(result: TxnCommitResult) -> Self { +impl From for TxnCommitResult { + fn from(result: CoreTxnCommitResult) -> Self { let node_aliases = result .local_node_ids .into_iter() @@ -1710,7 +2038,7 @@ impl From for PyTxnCommitResult { TxnLocalRef::Slot(_) => None, }) .collect(); - PyTxnCommitResult { + TxnCommitResult { node_ids: result.node_ids, edge_ids: result.edge_ids, node_aliases, @@ -1720,7 +2048,7 @@ impl From for PyTxnCommitResult { } #[pymethods] -impl PyTxnCommitResult { +impl TxnCommitResult { fn __repr__(&self) -> String { format!( "TxnCommitResult(nodes={}, edges={})", @@ -1734,13 +2062,13 @@ impl PyTxnCommitResult { #[pyclass] #[derive(Clone)] -pub struct PyNeighborEntry { +pub struct NeighborEntry { #[pyo3(get)] pub node_id: u64, #[pyo3(get)] pub edge_id: u64, #[pyo3(get)] - pub edge_type_id: u32, + pub label: String, #[pyo3(get)] pub weight: f64, #[pyo3(get)] @@ -1749,12 +2077,12 @@ pub struct PyNeighborEntry { pub valid_to: i64, } -impl From for PyNeighborEntry { - fn from(n: NeighborEntry) -> Self { - PyNeighborEntry { +impl From for NeighborEntry { + fn from(n: CoreNeighborEntry) -> Self { + NeighborEntry { node_id: n.node_id, edge_id: n.edge_id, - edge_type_id: n.edge_type_id, + label: n.label, weight: n.weight as f64, valid_from: n.valid_from, valid_to: n.valid_to, @@ -1763,18 +2091,18 @@ impl From for PyNeighborEntry { } #[pymethods] -impl PyNeighborEntry { +impl NeighborEntry { fn __repr__(&self) -> String { format!( - "NeighborEntry(node_id={}, edge_id={}, type={})", - self.node_id, self.edge_id, self.edge_type_id + "NeighborEntry(node_id={}, edge_id={}, label='{}')", + self.node_id, self.edge_id, self.label ) } } #[pyclass] #[derive(Clone)] -pub struct PyTraversalHit { +pub struct TraversalHit { #[pyo3(get)] pub node_id: u64, #[pyo3(get)] @@ -1785,9 +2113,9 @@ pub struct PyTraversalHit { pub score: Option, } -impl From for PyTraversalHit { - fn from(hit: TraversalHit) -> Self { - PyTraversalHit { +impl From for TraversalHit { + fn from(hit: CoreTraversalHit) -> Self { + TraversalHit { node_id: hit.node_id, depth: hit.depth, via_edge_id: hit.via_edge_id, @@ -1797,7 +2125,7 @@ impl From for PyTraversalHit { } #[pymethods] -impl PyTraversalHit { +impl TraversalHit { fn __repr__(&self) -> String { format!( "TraversalHit(node_id={}, depth={}, via_edge_id={:?})", @@ -1808,7 +2136,7 @@ impl PyTraversalHit { #[pyclass] #[derive(Clone)] -pub struct PyVectorHit { +pub struct VectorHit { #[pyo3(get)] pub node_id: u64, #[pyo3(get)] @@ -1816,7 +2144,7 @@ pub struct PyVectorHit { } #[pymethods] -impl PyVectorHit { +impl VectorHit { fn __repr__(&self) -> String { format!( "VectorHit(node_id={}, score={:.4})", @@ -1827,11 +2155,11 @@ impl PyVectorHit { #[pyclass] #[derive(Clone)] -pub struct PyNodePropertyIndexInfo { +pub struct NodePropertyIndexInfo { #[pyo3(get)] pub index_id: u64, #[pyo3(get)] - pub type_id: u32, + pub label: String, #[pyo3(get)] pub prop_key: String, #[pyo3(get)] @@ -1844,12 +2172,12 @@ pub struct PyNodePropertyIndexInfo { pub last_error: Option, } -impl From for PyNodePropertyIndexInfo { - fn from(info: NodePropertyIndexInfo) -> Self { +impl From for NodePropertyIndexInfo { + fn from(info: CoreNodePropertyIndexInfo) -> Self { let (kind, domain) = secondary_index_kind_to_py(&info.kind); - PyNodePropertyIndexInfo { + NodePropertyIndexInfo { index_id: info.index_id, - type_id: info.type_id, + label: info.label, prop_key: info.prop_key, kind: kind.to_string(), domain: domain.map(str::to_string), @@ -1860,75 +2188,175 @@ impl From for PyNodePropertyIndexInfo { } #[pymethods] -impl PyNodePropertyIndexInfo { +impl NodePropertyIndexInfo { fn __repr__(&self) -> String { format!( - "NodePropertyIndexInfo(index_id={}, type_id={}, prop_key='{}', kind='{}', domain={:?}, state='{}')", - self.index_id, self.type_id, self.prop_key, self.kind, self.domain, self.state + "NodePropertyIndexInfo(index_id={}, label='{}', prop_key='{}', kind='{}', domain={:?}, state='{}')", + self.index_id, self.label, self.prop_key, self.kind, self.domain, self.state ) } } #[pyclass] #[derive(Clone)] -pub struct PyPropertyRangeBound { - value_internal: PropValue, +pub struct NodeLabelInfo { #[pyo3(get)] - pub inclusive: bool, + pub label: String, + #[pyo3(get)] + pub label_id: u32, } -impl From for PropertyRangeBound { - fn from(bound: PyPropertyRangeBound) -> Self { - if bound.inclusive { - PropertyRangeBound::Included(bound.value_internal) - } else { - PropertyRangeBound::Excluded(bound.value_internal) +impl From for NodeLabelInfo { + fn from(info: CoreNodeLabelInfo) -> Self { + NodeLabelInfo { + label: info.label, + label_id: info.label_id, } } } -impl From for PyPropertyRangeBound { - fn from(bound: PropertyRangeBound) -> Self { - match bound { - PropertyRangeBound::Included(value_internal) => PyPropertyRangeBound { - value_internal, - inclusive: true, - }, - PropertyRangeBound::Excluded(value_internal) => PyPropertyRangeBound { - value_internal, - inclusive: false, - }, - } +#[pymethods] +impl NodeLabelInfo { + fn __repr__(&self) -> String { + format!( + "NodeLabelInfo(label='{}', label_id={})", + self.label, self.label_id + ) } } -#[pymethods] -impl PyPropertyRangeBound { - #[new] - #[pyo3(signature = (value, *, inclusive=true, domain))] - fn new(value: &Bound<'_, pyo3::PyAny>, inclusive: bool, domain: &str) -> PyResult { - let domain = parse_secondary_index_range_domain(domain)?; - Ok(PyPropertyRangeBound { - value_internal: py_numeric_to_prop_value(value.py(), value, domain)?, - inclusive, - }) - } +#[pyclass] +#[derive(Clone)] +pub struct EdgeLabelInfo { + #[pyo3(get)] + pub label: String, + #[pyo3(get)] + pub label_id: u32, +} - #[getter] - fn value(&self, py: Python<'_>) -> PyResult { - prop_value_to_py_obj(py, &self.value_internal) +impl From for EdgeLabelInfo { + fn from(info: CoreEdgeLabelInfo) -> Self { + EdgeLabelInfo { + label: info.label, + label_id: info.label_id, + } } +} - #[getter] - fn domain(&self) -> PyResult { - Ok(range_domain_to_py(range_domain_from_prop_value( - &self.value_internal, - "property range bound", - )?) - .to_string()) +#[pymethods] +impl EdgeLabelInfo { + fn __repr__(&self) -> String { + format!( + "EdgeLabelInfo(label='{}', label_id={})", + self.label, self.label_id + ) } +} - fn __repr__(&self) -> PyResult { +#[pyclass] +#[derive(Clone)] +pub struct EdgePropertyIndexInfo { + #[pyo3(get)] + pub index_id: u64, + #[pyo3(get)] + pub label: String, + #[pyo3(get)] + pub prop_key: String, + #[pyo3(get)] + pub kind: String, + #[pyo3(get)] + pub domain: Option, + #[pyo3(get)] + pub state: String, + #[pyo3(get)] + pub last_error: Option, +} + +impl From for EdgePropertyIndexInfo { + fn from(info: CoreEdgePropertyIndexInfo) -> Self { + let (kind, domain) = secondary_index_kind_to_py(&info.kind); + EdgePropertyIndexInfo { + index_id: info.index_id, + label: info.label, + prop_key: info.prop_key, + kind: kind.to_string(), + domain: domain.map(str::to_string), + state: secondary_index_state_to_py(info.state).to_string(), + last_error: info.last_error, + } + } +} + +#[pymethods] +impl EdgePropertyIndexInfo { + fn __repr__(&self) -> String { + format!( + "EdgePropertyIndexInfo(index_id={}, label='{}', prop_key='{}', kind='{}', domain={:?}, state='{}')", + self.index_id, self.label, self.prop_key, self.kind, self.domain, self.state + ) + } +} + +#[pyclass] +#[derive(Clone)] +pub struct PropertyRangeBound { + value_internal: PropValue, + #[pyo3(get)] + pub inclusive: bool, +} + +impl From for CorePropertyRangeBound { + fn from(bound: PropertyRangeBound) -> Self { + if bound.inclusive { + CorePropertyRangeBound::Included(bound.value_internal) + } else { + CorePropertyRangeBound::Excluded(bound.value_internal) + } + } +} + +impl From for PropertyRangeBound { + fn from(bound: CorePropertyRangeBound) -> Self { + match bound { + CorePropertyRangeBound::Included(value_internal) => PropertyRangeBound { + value_internal, + inclusive: true, + }, + CorePropertyRangeBound::Excluded(value_internal) => PropertyRangeBound { + value_internal, + inclusive: false, + }, + } + } +} + +#[pymethods] +impl PropertyRangeBound { + #[new] + #[pyo3(signature = (value, *, inclusive=true, domain))] + fn new(value: &Bound<'_, pyo3::PyAny>, inclusive: bool, domain: &str) -> PyResult { + let domain = parse_secondary_index_range_domain(domain)?; + Ok(PropertyRangeBound { + value_internal: py_numeric_to_prop_value(value.py(), value, domain)?, + inclusive, + }) + } + + #[getter] + fn value(&self, py: Python<'_>) -> PyResult { + prop_value_to_py_obj(py, &self.value_internal) + } + + #[getter] + fn domain(&self) -> PyResult { + Ok(range_domain_to_py(range_domain_from_prop_value( + &self.value_internal, + "property range bound", + )?) + .to_string()) + } + + fn __repr__(&self) -> PyResult { let value = Python::with_gil(|py| prop_value_debug_repr(py, &self.value_internal))?; Ok(format!( "PropertyRangeBound(value={}, inclusive={}, domain='{}')", @@ -1941,24 +2369,24 @@ impl PyPropertyRangeBound { #[pyclass] #[derive(Clone)] -pub struct PyPropertyRangeCursor { +pub struct PropertyRangeCursor { value_internal: PropValue, #[pyo3(get)] pub node_id: u64, } -impl From for PropertyRangeCursor { - fn from(cursor: PyPropertyRangeCursor) -> Self { - PropertyRangeCursor { +impl From for CorePropertyRangeCursor { + fn from(cursor: PropertyRangeCursor) -> Self { + CorePropertyRangeCursor { value: cursor.value_internal, node_id: cursor.node_id, } } } -impl From for PyPropertyRangeCursor { - fn from(cursor: PropertyRangeCursor) -> Self { - PyPropertyRangeCursor { +impl From for PropertyRangeCursor { + fn from(cursor: CorePropertyRangeCursor) -> Self { + PropertyRangeCursor { value_internal: cursor.value, node_id: cursor.node_id, } @@ -1966,12 +2394,12 @@ impl From for PyPropertyRangeCursor { } #[pymethods] -impl PyPropertyRangeCursor { +impl PropertyRangeCursor { #[new] #[pyo3(signature = (value, node_id, *, domain))] fn new(value: &Bound<'_, pyo3::PyAny>, node_id: u64, domain: &str) -> PyResult { let domain = parse_secondary_index_range_domain(domain)?; - Ok(PyPropertyRangeCursor { + Ok(PropertyRangeCursor { value_internal: py_numeric_to_prop_value(value.py(), value, domain)?, node_id, }) @@ -2004,24 +2432,24 @@ impl PyPropertyRangeCursor { #[pyclass] #[derive(Clone)] -pub struct PyTraversalCursor { +pub struct TraversalCursor { #[pyo3(get)] pub depth: u32, #[pyo3(get)] pub last_node_id: u64, } -impl From for PyTraversalCursor { +impl From for CoreTraversalCursor { fn from(cursor: TraversalCursor) -> Self { - PyTraversalCursor { + CoreTraversalCursor { depth: cursor.depth, last_node_id: cursor.last_node_id, } } } -impl From for TraversalCursor { - fn from(cursor: PyTraversalCursor) -> Self { +impl From for TraversalCursor { + fn from(cursor: CoreTraversalCursor) -> Self { TraversalCursor { depth: cursor.depth, last_node_id: cursor.last_node_id, @@ -2030,10 +2458,10 @@ impl From for TraversalCursor { } #[pymethods] -impl PyTraversalCursor { +impl TraversalCursor { #[new] fn new(depth: u32, last_node_id: u64) -> Self { - PyTraversalCursor { + TraversalCursor { depth, last_node_id, } @@ -2049,7 +2477,7 @@ impl PyTraversalCursor { #[pyclass] #[derive(Clone)] -pub struct PyShortestPath { +pub struct ShortestPath { #[pyo3(get)] pub nodes: Vec, #[pyo3(get)] @@ -2058,9 +2486,9 @@ pub struct PyShortestPath { pub total_cost: f64, } -impl From for PyShortestPath { - fn from(sp: ShortestPath) -> Self { - PyShortestPath { +impl From for ShortestPath { + fn from(sp: CoreShortestPath) -> Self { + ShortestPath { nodes: sp.nodes, edges: sp.edges, total_cost: sp.total_cost, @@ -2069,7 +2497,7 @@ impl From for PyShortestPath { } #[pymethods] -impl PyShortestPath { +impl ShortestPath { fn __repr__(&self) -> String { format!( "ShortestPath(nodes={}, edges={}, cost={:.4})", @@ -2082,28 +2510,34 @@ impl PyShortestPath { #[pyclass] #[derive(Clone)] -pub struct PySubgraph { - nodes: Vec, - edges: Vec, +pub struct Subgraph { + nodes: Vec, + edges: Vec, } -impl From for PySubgraph { - fn from(sg: Subgraph) -> Self { - PySubgraph { - nodes: sg.nodes.into_iter().map(PyNodeRecord::from).collect(), - edges: sg.edges.into_iter().map(PyEdgeRecord::from).collect(), - } +impl TryFrom for Subgraph { + type Error = EngineError; + + fn try_from(sg: CoreSubgraph) -> Result { + Ok(Subgraph { + nodes: sg + .nodes + .into_iter() + .map(NodeView::try_from) + .collect::, EngineError>>()?, + edges: sg.edges.into_iter().map(EdgeView::from).collect(), + }) } } #[pymethods] -impl PySubgraph { +impl Subgraph { #[getter] - fn nodes(&self) -> Vec { + fn nodes(&self) -> Vec { self.nodes.clone() } #[getter] - fn edges(&self) -> Vec { + fn edges(&self) -> Vec { self.edges.clone() } fn __repr__(&self) -> String { @@ -2117,16 +2551,16 @@ impl PySubgraph { #[pyclass] #[derive(Clone)] -pub struct PyPruneResult { +pub struct PruneResult { #[pyo3(get)] pub nodes_pruned: u64, #[pyo3(get)] pub edges_pruned: u64, } -impl From for PyPruneResult { - fn from(r: PruneResult) -> Self { - PyPruneResult { +impl From for PruneResult { + fn from(r: CorePruneResult) -> Self { + PruneResult { nodes_pruned: r.nodes_pruned, edges_pruned: r.edges_pruned, } @@ -2134,7 +2568,7 @@ impl From for PyPruneResult { } #[pymethods] -impl PyPruneResult { +impl PruneResult { fn __repr__(&self) -> String { format!( "PruneResult(nodes={}, edges={})", @@ -2145,7 +2579,7 @@ impl PyPruneResult { #[pyclass] #[derive(Clone)] -pub struct PyNamedPrunePolicy { +pub struct NamedPrunePolicy { #[pyo3(get)] pub name: String, #[pyo3(get)] @@ -2153,22 +2587,33 @@ pub struct PyNamedPrunePolicy { #[pyo3(get)] pub max_weight: Option, #[pyo3(get)] - pub type_id: Option, + pub label: Option, +} + +impl From for NamedPrunePolicy { + fn from(info: PrunePolicyInfo) -> Self { + NamedPrunePolicy { + name: info.name, + max_age_ms: info.policy.max_age_ms, + max_weight: info.policy.max_weight.map(|w| w as f64), + label: info.policy.label, + } + } } #[pymethods] -impl PyNamedPrunePolicy { +impl NamedPrunePolicy { fn __repr__(&self) -> String { format!( - "PrunePolicy(name='{}', max_age_ms={:?}, max_weight={:?}, type_id={:?})", - self.name, self.max_age_ms, self.max_weight, self.type_id + "PrunePolicy(name='{}', max_age_ms={:?}, max_weight={:?}, label={:?})", + self.name, self.max_age_ms, self.max_weight, self.label ) } } #[pyclass] #[derive(Clone)] -pub struct PySegmentInfo { +pub struct SegmentInfo { #[pyo3(get)] pub id: u64, #[pyo3(get)] @@ -2178,7 +2623,7 @@ pub struct PySegmentInfo { } #[pymethods] -impl PySegmentInfo { +impl SegmentInfo { fn __repr__(&self) -> String { format!( "SegmentInfo(id={}, nodes={}, edges={})", @@ -2189,7 +2634,7 @@ impl PySegmentInfo { #[pyclass] #[derive(Clone)] -pub struct PyCompactionStats { +pub struct CompactionStats { #[pyo3(get)] pub segments_merged: usize, #[pyo3(get)] @@ -2210,9 +2655,9 @@ pub struct PyCompactionStats { pub edges_auto_pruned: u64, } -impl From for PyCompactionStats { - fn from(s: CompactionStats) -> Self { - PyCompactionStats { +impl From for CompactionStats { + fn from(s: CoreCompactionStats) -> Self { + CompactionStats { segments_merged: s.segments_merged, nodes_kept: s.nodes_kept, nodes_removed: s.nodes_removed, @@ -2227,7 +2672,7 @@ impl From for PyCompactionStats { } #[pymethods] -impl PyCompactionStats { +impl CompactionStats { fn __repr__(&self) -> String { format!( "CompactionStats(merged={}, kept={}/{}, removed={}/{}, {}ms)", @@ -2243,7 +2688,7 @@ impl PyCompactionStats { #[pyclass] #[derive(Clone)] -pub struct PyCompactionProgress { +pub struct CompactionProgress { #[pyo3(get)] pub phase: String, #[pyo3(get)] @@ -2256,8 +2701,8 @@ pub struct PyCompactionProgress { pub total_records: u64, } -impl From<&CompactionProgress> for PyCompactionProgress { - fn from(p: &CompactionProgress) -> Self { +impl From<&CoreCompactionProgress> for CompactionProgress { + fn from(p: &CoreCompactionProgress) -> Self { let phase_str = match p.phase { CompactionPhase::CollectingTombstones => "collecting_tombstones", CompactionPhase::MergingNodes => "merging_nodes", @@ -2265,7 +2710,7 @@ impl From<&CompactionProgress> for PyCompactionProgress { CompactionPhase::WritingOutput => "writing_output", } .to_string(); - PyCompactionProgress { + CompactionProgress { phase: phase_str, segments_processed: p.segments_processed, total_segments: p.total_segments, @@ -2276,7 +2721,7 @@ impl From<&CompactionProgress> for PyCompactionProgress { } #[pymethods] -impl PyCompactionProgress { +impl CompactionProgress { fn __repr__(&self) -> String { format!( "CompactionProgress(phase='{}', {}/{})", @@ -2316,8 +2761,8 @@ impl IdArray { } } - fn __iter__(slf: PyRef<'_, Self>) -> PyIdArrayIter { - PyIdArrayIter { + fn __iter__(slf: PyRef<'_, Self>) -> IdArrayIter { + IdArrayIter { ids: Arc::clone(&slf.ids), index: 0, } @@ -2352,13 +2797,13 @@ impl IdArray { } #[pyclass] -pub struct PyIdArrayIter { +pub struct IdArrayIter { ids: Arc>, index: usize, } #[pymethods] -impl PyIdArrayIter { +impl IdArrayIter { fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -2380,16 +2825,16 @@ impl PyIdArrayIter { #[pyclass] #[derive(Clone)] -pub struct PyIdPageResult { +pub struct IdPageResult { #[pyo3(get)] pub items: IdArray, #[pyo3(get)] pub next_cursor: Option, } -impl From> for PyIdPageResult { +impl From> for IdPageResult { fn from(r: eg::PageResult) -> Self { - PyIdPageResult { + IdPageResult { items: IdArray { ids: Arc::new(r.items), }, @@ -2399,7 +2844,7 @@ impl From> for PyIdPageResult { } #[pymethods] -impl PyIdPageResult { +impl IdPageResult { fn __repr__(&self) -> String { format!( "IdPageResult(count={}, has_next={})", @@ -2417,16 +2862,16 @@ impl PyIdPageResult { #[pyclass] #[derive(Clone)] -pub struct PyNodePageResult { - items: Vec, +pub struct NodePageResult { + items: Vec, #[pyo3(get)] pub next_cursor: Option, } #[pymethods] -impl PyNodePageResult { +impl NodePageResult { #[getter] - fn items(&self) -> Vec { + fn items(&self) -> Vec { self.items.clone() } fn __repr__(&self) -> String { @@ -2446,16 +2891,16 @@ impl PyNodePageResult { #[pyclass] #[derive(Clone)] -pub struct PyEdgePageResult { - items: Vec, +pub struct EdgePageResult { + items: Vec, #[pyo3(get)] pub next_cursor: Option, } #[pymethods] -impl PyEdgePageResult { +impl EdgePageResult { #[getter] - fn items(&self) -> Vec { + fn items(&self) -> Vec { self.items.clone() } fn __repr__(&self) -> String { @@ -2475,16 +2920,16 @@ impl PyEdgePageResult { #[pyclass] #[derive(Clone)] -pub struct PyNeighborPageResult { - items: Vec, +pub struct NeighborPageResult { + items: Vec, #[pyo3(get)] pub next_cursor: Option, } #[pymethods] -impl PyNeighborPageResult { +impl NeighborPageResult { #[getter] - fn items(&self) -> Vec { + fn items(&self) -> Vec { self.items.clone() } fn __repr__(&self) -> String { @@ -2504,27 +2949,27 @@ impl PyNeighborPageResult { #[pyclass] #[derive(Clone)] -pub struct PyPropertyRangePageResult { +pub struct PropertyRangePageResult { #[pyo3(get)] pub items: IdArray, - next_cursor: Option, + next_cursor: Option, } -impl From> for PyPropertyRangePageResult { - fn from(result: PropertyRangePageResult) -> Self { - PyPropertyRangePageResult { +impl From> for PropertyRangePageResult { + fn from(result: CorePropertyRangePageResult) -> Self { + PropertyRangePageResult { items: IdArray { ids: Arc::new(result.items), }, - next_cursor: result.next_cursor.map(PyPropertyRangeCursor::from), + next_cursor: result.next_cursor.map(PropertyRangeCursor::from), } } } #[pymethods] -impl PyPropertyRangePageResult { +impl PropertyRangePageResult { #[getter] - fn next_cursor(&self) -> Option { + fn next_cursor(&self) -> Option { self.next_cursor.clone() } @@ -2547,29 +2992,29 @@ impl PyPropertyRangePageResult { #[pyclass] #[derive(Clone)] -pub struct PyTraversalPageResult { - items: Vec, - next_cursor: Option, +pub struct TraversalPageResult { + items: Vec, + next_cursor: Option, } -impl From for PyTraversalPageResult { - fn from(result: TraversalPageResult) -> Self { - PyTraversalPageResult { - items: result.items.into_iter().map(PyTraversalHit::from).collect(), - next_cursor: result.next_cursor.map(PyTraversalCursor::from), +impl From for TraversalPageResult { + fn from(result: CoreTraversalPageResult) -> Self { + TraversalPageResult { + items: result.items.into_iter().map(TraversalHit::from).collect(), + next_cursor: result.next_cursor.map(TraversalCursor::from), } } } #[pymethods] -impl PyTraversalPageResult { +impl TraversalPageResult { #[getter] - fn items(&self) -> Vec { + fn items(&self) -> Vec { self.items.clone() } #[getter] - fn next_cursor(&self) -> Option { + fn next_cursor(&self) -> Option { self.next_cursor.clone() } @@ -2596,7 +3041,7 @@ impl PyTraversalPageResult { #[pyclass] #[derive(Clone)] -pub struct PyPprResult { +pub struct PprResult { #[pyo3(get)] pub node_ids: Vec, #[pyo3(get)] @@ -2608,12 +3053,12 @@ pub struct PyPprResult { #[pyo3(get)] pub algorithm: String, #[pyo3(get)] - pub approx: Option, + pub approx: Option, } #[pyclass] #[derive(Clone)] -pub struct PyPprApproxMeta { +pub struct PprApproxMeta { #[pyo3(get)] pub residual_tolerance: f64, #[pyo3(get)] @@ -2623,7 +3068,7 @@ pub struct PyPprApproxMeta { } #[pymethods] -impl PyPprApproxMeta { +impl PprApproxMeta { fn __repr__(&self) -> String { format!( "PprApproxMeta(residual_tolerance={}, pushes={}, max_remaining_residual={})", @@ -2632,16 +3077,16 @@ impl PyPprApproxMeta { } } -impl From for PyPprResult { - fn from(r: PprResult) -> Self { +impl From for PprResult { + fn from(r: CorePprResult) -> Self { let (node_ids, scores): (Vec, Vec) = r.scores.into_iter().unzip(); - PyPprResult { + PprResult { node_ids, scores, iterations: r.iterations, converged: r.converged, algorithm: ppr_algorithm_to_py(r.algorithm).to_string(), - approx: r.approx.map(|a| PyPprApproxMeta { + approx: r.approx.map(|a| PprApproxMeta { residual_tolerance: a.residual_tolerance, pushes: a.pushes, max_remaining_residual: a.max_remaining_residual, @@ -2651,7 +3096,7 @@ impl From for PyPprResult { } #[pymethods] -impl PyPprResult { +impl PprResult { fn __repr__(&self) -> String { format!( "PprResult(nodes={}, iterations={}, converged={}, algorithm='{}')", @@ -2665,47 +3110,56 @@ impl PyPprResult { #[pyclass] #[derive(Clone)] -pub struct PyExportEdge { +pub struct ExportEdge { #[pyo3(get)] pub from_id: u64, #[pyo3(get)] pub to_id: u64, #[pyo3(get)] - pub type_id: u32, + pub edge_label_index: u32, #[pyo3(get)] - pub weight: f64, + pub weight: Option, } #[pymethods] -impl PyExportEdge { +impl ExportEdge { fn __repr__(&self) -> String { format!( - "ExportEdge(from={}, to={}, type={}, weight={})", - self.from_id, self.to_id, self.type_id, self.weight + "ExportEdge(from={}, to={}, edge_label_index={}, weight={:?})", + self.from_id, self.to_id, self.edge_label_index, self.weight ) } } #[pyclass] #[derive(Clone)] -pub struct PyAdjacencyExport { +pub struct AdjacencyExport { #[pyo3(get)] pub node_ids: Vec, - edges: Vec, + #[pyo3(get)] + pub node_labels: Vec, + #[pyo3(get)] + pub node_label_indexes: Vec>, + #[pyo3(get)] + pub edge_labels: Vec, + edges: Vec, } -impl From for PyAdjacencyExport { - fn from(a: AdjacencyExport) -> Self { - PyAdjacencyExport { +impl From for AdjacencyExport { + fn from(a: CoreAdjacencyExport) -> Self { + AdjacencyExport { node_ids: a.node_ids, + node_labels: a.node_labels, + node_label_indexes: a.node_label_indexes, + edge_labels: a.edge_labels, edges: a .edges .into_iter() - .map(|(from_id, to_id, type_id, weight)| PyExportEdge { - from_id, - to_id, - type_id, - weight: weight as f64, + .map(|edge| ExportEdge { + from_id: edge.from, + to_id: edge.to, + edge_label_index: edge.edge_label_index, + weight: edge.weight.map(|w| w as f64), }) .collect(), } @@ -2713,9 +3167,9 @@ impl From for PyAdjacencyExport { } #[pymethods] -impl PyAdjacencyExport { +impl AdjacencyExport { #[getter] - fn edges(&self) -> Vec { + fn edges(&self) -> Vec { self.edges.clone() } fn __repr__(&self) -> String { @@ -2839,12 +3293,24 @@ fn query_plan_to_py(py: Python<'_>, plan: QueryPlan) -> PyResult { .map(query_plan_warning_to_py) .collect::>(), )?; + dict.set_item( + "notes", + plan.notes + .iter() + .map(query_plan_note_to_py) + .collect::>(), + )?; + dict.set_item( + "public_inputs", + query_plan_public_inputs_to_py(py, plan.public_inputs)?, + )?; Ok(dict.into_any().unbind()) } fn query_plan_kind_to_py(kind: &QueryPlanKind) -> &'static str { match kind { QueryPlanKind::NodeQuery => "node_query", + QueryPlanKind::EdgeQuery => "edge_query", QueryPlanKind::PatternQuery => "pattern_query", } } @@ -2854,11 +3320,26 @@ fn query_plan_node_to_py(py: Python<'_>, node: QueryPlanNode) -> PyResult dict.set_item("kind", "explicit_ids")?, QueryPlanNode::KeyLookup => dict.set_item("kind", "key_lookup")?, - QueryPlanNode::NodeTypeIndex => dict.set_item("kind", "node_type_index")?, + QueryPlanNode::NodeLabelIndex => dict.set_item("kind", "node_label_index")?, + QueryPlanNode::NodeLabelAnyIndex => dict.set_item("kind", "node_label_any_index")?, QueryPlanNode::PropertyEqualityIndex => dict.set_item("kind", "property_equality_index")?, QueryPlanNode::PropertyRangeIndex => dict.set_item("kind", "property_range_index")?, QueryPlanNode::TimestampIndex => dict.set_item("kind", "timestamp_index")?, QueryPlanNode::AdjacencyExpansion => dict.set_item("kind", "adjacency_expansion")?, + QueryPlanNode::ExplicitEdgeIds => dict.set_item("kind", "explicit_edge_ids")?, + QueryPlanNode::EdgeLabelIndex => dict.set_item("kind", "edge_label_index")?, + QueryPlanNode::EdgeTripleIndex => dict.set_item("kind", "edge_triple_index")?, + QueryPlanNode::EdgeEndpointAdjacency => dict.set_item("kind", "edge_endpoint_adjacency")?, + QueryPlanNode::EdgeWeightIndex => dict.set_item("kind", "edge_weight_index")?, + QueryPlanNode::EdgeUpdatedAtIndex => dict.set_item("kind", "edge_updated_at_index")?, + QueryPlanNode::EdgeValidityIndex => dict.set_item("kind", "edge_validity_index")?, + QueryPlanNode::EdgeMetadataScan => dict.set_item("kind", "edge_metadata_scan")?, + QueryPlanNode::EdgePropertyEqualityIndex => { + dict.set_item("kind", "edge_property_equality_index")? + } + QueryPlanNode::EdgePropertyRangeIndex => { + dict.set_item("kind", "edge_property_range_index")? + } QueryPlanNode::Intersect { inputs } => { dict.set_item("kind", "intersect")?; let inputs: PyResult> = inputs @@ -2879,6 +3360,10 @@ fn query_plan_node_to_py(py: Python<'_>, node: QueryPlanNode) -> PyResult { + dict.set_item("kind", "verify_edge_filter")?; + dict.set_item("input", query_plan_node_to_py(py, *input)?)?; + } QueryPlanNode::VerifyEdgePredicates { input } => { dict.set_item("kind", "verify_edge_predicates")?; dict.set_item("input", query_plan_node_to_py(py, *input)?)?; @@ -2891,13 +3376,72 @@ fn query_plan_node_to_py(py: Python<'_>, node: QueryPlanNode) -> PyResult dict.set_item("kind", "fallback_type_scan")?, + QueryPlanNode::PatternEdgeAnchor { edge_alias, input } => { + dict.set_item("kind", "pattern_edge_anchor")?; + dict.set_item("edge_alias", edge_alias)?; + dict.set_item("input", query_plan_node_to_py(py, *input)?)?; + } + QueryPlanNode::FallbackNodeLabelScan => { + dict.set_item("kind", "fallback_node_label_scan")? + } QueryPlanNode::FallbackFullNodeScan => dict.set_item("kind", "fallback_full_node_scan")?, + QueryPlanNode::FallbackEdgeLabelScan => { + dict.set_item("kind", "fallback_edge_label_scan")? + } + QueryPlanNode::FallbackFullEdgeScan => dict.set_item("kind", "fallback_full_edge_scan")?, QueryPlanNode::EmptyResult => dict.set_item("kind", "empty_result")?, } Ok(dict.into_any().unbind()) } +fn query_plan_note_to_py(note: &QueryPlanNote) -> &'static str { + match note { + QueryPlanNote::NodeLabelAnyDedupeBeforePagination => { + "node_label_any_dedupe_before_pagination" + } + QueryPlanNote::NodeLabelAnyFinalVerification => "node_label_any_final_verification", + QueryPlanNote::NodeLabelAllSupersetVerification => "node_label_all_superset_verification", + QueryPlanNote::StaleNodeLabelMembershipVerification => { + "stale_node_label_membership_verification" + } + } +} + +fn query_plan_public_inputs_to_py( + py: Python<'_>, + inputs: QueryPlanPublicInputs, +) -> PyResult { + let dict = PyDict::new(py); + let node_labels = inputs + .node_labels + .into_iter() + .map(|name| query_plan_public_name_to_py(py, name)) + .collect::>>()?; + let edge_labels = inputs + .edge_labels + .into_iter() + .map(|name| query_plan_public_name_to_py(py, name)) + .collect::>>()?; + dict.set_item("node_labels", node_labels)?; + dict.set_item("edge_labels", edge_labels)?; + Ok(dict.into_any().unbind()) +} + +fn query_plan_public_name_to_py(py: Python<'_>, name: QueryPlanPublicName) -> PyResult { + let dict = PyDict::new(py); + dict.set_item("alias", name.alias)?; + dict.set_item("name", name.name)?; + dict.set_item("known", name.known)?; + dict.set_item( + "mode", + name.mode.map(|mode| match mode { + LabelMatchMode::Any => "any", + LabelMatchMode::All => "all", + }), + )?; + Ok(dict.into_any().unbind()) +} + fn query_plan_warning_to_py(warning: &QueryPlanWarning) -> &'static str { match warning { QueryPlanWarning::MissingReadyIndex => "missing_ready_index", @@ -2913,6 +3457,8 @@ fn query_plan_warning_to_py(warning: &QueryPlanWarning) -> &'static str { QueryPlanWarning::VerifyOnlyFilter => "verify_only_filter", QueryPlanWarning::BooleanBranchFallback => "boolean_branch_fallback", QueryPlanWarning::PlanningProbeBudgetExceeded => "planning_probe_budget_exceeded", + QueryPlanWarning::UnknownNodeLabel => "unknown_node_label", + QueryPlanWarning::UnknownEdgeLabel => "unknown_edge_label", } } @@ -2931,6 +3477,7 @@ fn parse_py_node_query(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult, dict: &Bound<'_, PyDict>) -> PyResult { + reject_py_legacy_node_label_field(dict, "node query")?; let page = PageRequest { limit: parse_py_query_limit(dict, "node query limit")?, after: py_optional_query_u64(dict, "after", "node query after")?, @@ -2951,7 +3498,11 @@ fn parse_py_node_query_dict(py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResul } }; Ok(NodeQuery { - type_id: py_optional_query_u32(dict, "type_id", "node query type_id")?, + label_filter: parse_optional_node_label_filter_field( + dict, + "label_filter", + "node query label_filter", + )?, ids: py_optional_query_u64_vec(dict, "ids", "node query ids")?, keys: py_optional_extract::>(dict, "keys")?.unwrap_or_default(), filter: parse_py_node_filter(py, dict, "updated_at", "node query")?, @@ -2961,6 +3512,47 @@ fn parse_py_node_query_dict(py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResul }) } +fn parse_py_edge_query(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(dict) = value.downcast::() { + return parse_py_edge_query_dict(py, dict); + } + if value.hasattr("to_dict")? { + let dict_value = value.call_method0("to_dict")?; + let dict = dict_value.downcast::()?; + return parse_py_edge_query_dict(py, dict); + } + Err(PyTypeError::new_err( + "edge query request must be a dict or expose to_dict()", + )) +} + +fn parse_py_edge_query_dict(py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResult { + reject_py_legacy_node_predicate_fields(dict, "edge query")?; + let page = PageRequest { + limit: parse_py_query_limit(dict, "edge query limit")?, + after: py_optional_query_u64(dict, "after", "edge query after")?, + }; + Ok(EdgeQuery { + label: py_optional_extract::(dict, "label")?, + ids: py_optional_query_u64_vec(dict, "ids", "edge query ids")?, + from_ids: py_optional_query_u64_vec(dict, "from_ids", "edge query from_ids")?, + to_ids: py_optional_query_u64_vec(dict, "to_ids", "edge query to_ids")?, + endpoint_ids: py_optional_query_u64_vec(dict, "endpoint_ids", "edge query endpoint_ids")?, + filter: parse_py_edge_filter( + py, + dict, + "updated_at", + "valid_at", + "valid_from", + "valid_to", + "edge query", + )?, + page, + order: EdgeQueryOrder::EdgeIdAsc, + allow_full_scan: py_optional_extract::(dict, "allow_full_scan")?.unwrap_or(false), + }) +} + fn parse_py_graph_pattern_query( py: Python<'_>, value: &Bound<'_, PyAny>, @@ -3012,9 +3604,14 @@ fn parse_py_graph_pattern_query_dict( } fn parse_py_node_pattern(py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResult { + reject_py_legacy_node_label_field(dict, "node pattern")?; Ok(NodePattern { alias: py_required_extract(dict, "alias")?, - type_id: py_optional_query_u32(dict, "type_id", "node pattern type_id")?, + label_filter: parse_optional_node_label_filter_field( + dict, + "label_filter", + "node pattern label_filter", + )?, ids: py_optional_query_u64_vec(dict, "ids", "node pattern ids")?, keys: py_optional_extract::>(dict, "keys")?.unwrap_or_default(), filter: parse_py_node_filter(py, dict, "updated_at", "node pattern")?, @@ -3022,11 +3619,7 @@ fn parse_py_node_pattern(py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResult, dict: &Bound<'_, PyDict>) -> PyResult { - if py_has_field(dict, "filter")? { - return Err(PyValueError::new_err( - "edge pattern filter is not supported in Phase 24; use edge pattern where or predicates", - )); - } + reject_py_legacy_node_predicate_fields(dict, "edge pattern")?; let direction = match py_non_none_item(dict, "direction")? { None => Direction::Outgoing, Some(value) => parse_direction(&value.extract::()?)?, @@ -3036,8 +3629,17 @@ fn parse_py_edge_pattern(py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResult, context: &st Ok(()) } +fn reject_py_legacy_node_label_field(dict: &Bound<'_, PyDict>, context: &str) -> PyResult<()> { + if py_has_field(dict, "label")? { + return Err(PyValueError::new_err(format!( + "{} label is no longer supported; use label_filter", + context + ))); + } + Ok(()) +} + fn parse_py_node_filter( py: Python<'_>, dict: &Bound<'_, PyDict>, @@ -3073,6 +3685,30 @@ fn parse_py_node_filter( } } +fn parse_py_edge_filter( + py: Python<'_>, + dict: &Bound<'_, PyDict>, + updated_at_key: &str, + valid_at_key: &str, + valid_from_key: &str, + valid_to_key: &str, + context: &str, +) -> PyResult> { + match py_non_none_item(dict, "filter")? { + None => Ok(None), + Some(value) => parse_py_edge_filter_expr( + py, + &value, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} filter", context), + ) + .map(Some), + } +} + fn parse_py_node_filter_expr( py: Python<'_>, value: &Bound<'_, PyAny>, @@ -3167,62 +3803,149 @@ fn parse_py_node_filter_expr( ))) } -fn parse_py_edge_predicates( +fn parse_py_edge_filter_expr( py: Python<'_>, - dict: &Bound<'_, PyDict>, + value: &Bound<'_, PyAny>, + updated_at_key: &str, + valid_at_key: &str, + valid_from_key: &str, + valid_to_key: &str, context: &str, -) -> PyResult> { - let mut predicates = Vec::new(); - if let Some(where_value) = py_non_none_item(dict, "where")? { - let where_dict = where_value.downcast::()?; - for (key, value) in where_dict.iter() { - let key: String = key.extract()?; - predicates.push(parse_py_property_edge_predicate( +) -> PyResult { + let dict = value + .downcast::() + .map_err(|_| PyTypeError::new_err(format!("{} must be a dict", context)))?; + if dict.is_empty() { + return Err(PyValueError::new_err(format!( + "{} must not be an empty object", + context + ))); + } + + let selectors = [ + "and", + "or", + "not", + "property", + "weight", + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + ] + .iter() + .map(|field| py_has_field(dict, field)) + .collect::>>()? + .into_iter() + .filter(|present| *present) + .count(); + if selectors != 1 { + return Err(PyValueError::new_err(format!( + "{} must contain exactly one boolean tag or leaf selector", + context + ))); + } + reject_py_uppercase_filter_fields(dict, context)?; + + if let Some(value) = dict.get_item("and")? { + ensure_only_py_fields(dict, &["and"], context)?; + let children = value.downcast::()?; + if children.is_empty() { + return Err(PyValueError::new_err(format!( + "{} and must contain at least one child", + context + ))); + } + let mut parsed = Vec::with_capacity(children.len()); + for (index, child) in children.iter().enumerate() { + parsed.push(parse_py_edge_filter_expr( py, - key.clone(), - &value, - &format!("{} where.{}", context, key), + &child, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} and[{}]", context, index), )?); } + return Ok(EdgeFilterExpr::And(parsed)); } - if let Some(predicates_value) = py_non_none_item(dict, "predicates")? { - let predicates_list = predicates_value.downcast::()?; - for (index, item) in predicates_list.iter().enumerate() { - let predicate_dict = item.downcast::()?; - if predicate_dict.len() != 1 { - return Err(PyValueError::new_err(format!( - "{} predicates[{}] must contain exactly one top-level predicate tag", - context, index - ))); - } - let (tag, payload) = predicate_dict.iter().next().unwrap(); - let tag: String = tag.extract()?; - match tag.as_str() { - "property" => predicates.push(parse_py_explicit_property_edge_predicate( - py, - &payload, - &format!("{} predicates[{}].property", context, index), - )?), - other => { - return Err(PyValueError::new_err(format!( - "Unknown edge predicate tag '{}'. Only 'property' is supported.", - other - ))); - } - } + if let Some(value) = dict.get_item("or")? { + ensure_only_py_fields(dict, &["or"], context)?; + let children = value.downcast::()?; + if children.is_empty() { + return Err(PyValueError::new_err(format!( + "{} or must contain at least one child", + context + ))); + } + let mut parsed = Vec::with_capacity(children.len()); + for (index, child) in children.iter().enumerate() { + parsed.push(parse_py_edge_filter_expr( + py, + &child, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} or[{}]", context, index), + )?); } + return Ok(EdgeFilterExpr::Or(parsed)); + } + if let Some(value) = dict.get_item("not")? { + ensure_only_py_fields(dict, &["not"], context)?; + return Ok(EdgeFilterExpr::Not(Box::new(parse_py_edge_filter_expr( + py, + &value, + updated_at_key, + valid_at_key, + valid_from_key, + valid_to_key, + &format!("{} not", context), + )?))); + } + if py_has_field(dict, "property")? { + return parse_py_property_edge_filter(py, dict, context); + } + if let Some(value) = dict.get_item("weight")? { + ensure_only_py_fields(dict, &["weight"], context)?; + let range = value.downcast::()?; + let (lower, upper) = parse_py_f32_range_bounds(range, &format!("{} weight", context))?; + return Ok(EdgeFilterExpr::WeightRange { lower, upper }); + } + if let Some(value) = dict.get_item(updated_at_key)? { + ensure_only_py_fields(dict, &[updated_at_key], context)?; + let range = value.downcast::()?; + let (lower_ms, upper_ms) = + parse_py_i64_range_bounds(range, &format!("{} {}", context, updated_at_key))?; + return Ok(EdgeFilterExpr::UpdatedAtRange { lower_ms, upper_ms }); + } + if let Some(value) = dict.get_item(valid_at_key)? { + ensure_only_py_fields(dict, &[valid_at_key], context)?; + return Ok(EdgeFilterExpr::ValidAt { + epoch_ms: py_query_i64(&value, &format!("{} {}", context, valid_at_key))?, + }); + } + if let Some(value) = dict.get_item(valid_from_key)? { + ensure_only_py_fields(dict, &[valid_from_key], context)?; + let range = value.downcast::()?; + let (lower_ms, upper_ms) = + parse_py_i64_range_bounds(range, &format!("{} {}", context, valid_from_key))?; + return Ok(EdgeFilterExpr::ValidFromRange { lower_ms, upper_ms }); + } + if let Some(value) = dict.get_item(valid_to_key)? { + ensure_only_py_fields(dict, &[valid_to_key], context)?; + let range = value.downcast::()?; + let (lower_ms, upper_ms) = + parse_py_i64_range_bounds(range, &format!("{} {}", context, valid_to_key))?; + return Ok(EdgeFilterExpr::ValidToRange { lower_ms, upper_ms }); } - Ok(predicates) -} -fn parse_py_explicit_property_edge_predicate( - py: Python<'_>, - value: &Bound<'_, PyAny>, - context: &str, -) -> PyResult { - let dict = value.downcast::()?; - let key: String = py_required_extract(dict, "key")?; - parse_py_property_edge_predicate(py, key, value, context) + Err(PyValueError::new_err(format!( + "{} must contain a valid filter selector", + context + ))) } fn parse_py_property_node_filter( @@ -3302,78 +4025,81 @@ fn parse_py_property_node_filter( unreachable!("operator family count was checked above") } -fn parse_py_property_edge_predicate( +fn parse_py_property_edge_filter( py: Python<'_>, - key: String, - value: &Bound<'_, PyAny>, + dict: &Bound<'_, PyDict>, context: &str, -) -> PyResult { - let parsed = parse_py_property_predicate(py, value, context)?; - Ok(match parsed { - PyParsedPropertyPredicate::Equals(value) => { - EdgePostFilterPredicate::PropertyEquals { key, value } - } - PyParsedPropertyPredicate::Range { lower, upper } => { - EdgePostFilterPredicate::PropertyRange { key, lower, upper } - } - }) -} +) -> PyResult { + let key_value = py_non_none_item(dict, "property")? + .ok_or_else(|| PyValueError::new_err(format!("{} property is required", context)))?; + let key: String = key_value.extract()?; + if key.is_empty() { + return Err(PyValueError::new_err(format!( + "{} property must be non-empty", + context + ))); + } -enum PyParsedPropertyPredicate { - Equals(PropValue), - Range { - lower: Option, - upper: Option, - }, -} + let has_range = py_has_any_field(dict, &["gt", "gte", "lt", "lte"])?; + let families = [ + py_has_field(dict, "eq")?, + py_has_field(dict, "in")?, + has_range, + py_has_field(dict, "exists")?, + py_has_field(dict, "missing")?, + ] + .into_iter() + .filter(|present| *present) + .count(); + if families != 1 { + return Err(PyValueError::new_err(format!( + "{} property filter must specify exactly one operator family", + context + ))); + } -fn parse_py_property_predicate( - py: Python<'_>, - value: &Bound<'_, PyAny>, - context: &str, -) -> PyResult { - let dict = value.downcast::()?; - match py_non_none_item(dict, "op")? { - Some(op_value) => { - let op: String = op_value.extract()?; - match op.as_str() { - "eq" => { - ensure_no_py_fields(dict, &["gt", "gte", "lt", "lte", "eq"], context)?; - let value = dict.get_item("value")?.ok_or_else(|| { - PyValueError::new_err(format!("{} eq predicate requires value", context)) - })?; - Ok(PyParsedPropertyPredicate::Equals(py_to_prop_value( - py, &value, - )?)) - } - "range" => { - ensure_no_py_fields(dict, &["value", "eq"], context)?; - let (lower, upper) = parse_py_property_range_bounds(py, dict, context)?; - Ok(PyParsedPropertyPredicate::Range { lower, upper }) - } - other => Err(PyValueError::new_err(format!( - "Unknown predicate op '{}'. Valid ops are 'eq' and 'range'.", - other - ))), - } - } - None if py_has_field(dict, "eq")? => { - ensure_no_py_fields(dict, &["value", "gt", "gte", "lt", "lte"], context)?; - let value = dict.get_item("eq")?.unwrap(); - Ok(PyParsedPropertyPredicate::Equals(py_to_prop_value( - py, &value, - )?)) - } - None if py_has_any_field(dict, &["gt", "gte", "lt", "lte"])? => { - ensure_no_py_fields(dict, &["value", "eq"], context)?; - let (lower, upper) = parse_py_property_range_bounds(py, dict, context)?; - Ok(PyParsedPropertyPredicate::Range { lower, upper }) + if let Some(value) = dict.get_item("eq")? { + ensure_only_py_fields(dict, &["property", "eq"], context)?; + return Ok(EdgeFilterExpr::PropertyEquals { + key, + value: py_to_prop_value(py, &value)?, + }); + } + if let Some(value) = dict.get_item("in")? { + ensure_only_py_fields(dict, &["property", "in"], context)?; + let values = value.downcast::()?; + if values.is_empty() { + return Err(PyValueError::new_err(format!( + "{} in must contain at least one value", + context + ))); } - None => Err(PyValueError::new_err(format!( - "{} predicate requires op, eq, or range bounds", - context - ))), + let parsed = values + .iter() + .map(|value| py_to_prop_value(py, &value)) + .collect::>>()?; + return Ok(EdgeFilterExpr::PropertyIn { + key, + values: parsed, + }); + } + if has_range { + ensure_only_py_fields(dict, &["property", "gt", "gte", "lt", "lte"], context)?; + let (lower, upper) = parse_py_property_range_bounds(py, dict, context)?; + return Ok(EdgeFilterExpr::PropertyRange { key, lower, upper }); + } + if py_has_field(dict, "exists")? { + ensure_only_py_fields(dict, &["property", "exists"], context)?; + require_py_true_field(dict, "exists", context)?; + return Ok(EdgeFilterExpr::PropertyExists { key }); + } + if py_has_field(dict, "missing")? { + ensure_only_py_fields(dict, &["property", "missing"], context)?; + require_py_true_field(dict, "missing", context)?; + return Ok(EdgeFilterExpr::PropertyMissing { key }); } + + unreachable!("operator family count was checked above") } fn parse_py_updated_at_filter( @@ -3395,7 +4121,10 @@ fn parse_py_property_range_bounds( py: Python<'_>, dict: &Bound<'_, PyDict>, context: &str, -) -> PyResult<(Option, Option)> { +) -> PyResult<( + Option, + Option, +)> { if py_has_field(dict, "gt")? && py_has_field(dict, "gte")? { return Err(PyValueError::new_err(format!( "{} range predicate cannot specify both gt and gte", @@ -3409,17 +4138,21 @@ fn parse_py_property_range_bounds( ))); } let lower = if let Some(value) = dict.get_item("gt")? { - Some(PropertyRangeBound::Excluded(py_to_prop_value(py, &value)?)) + Some(CorePropertyRangeBound::Excluded(py_to_prop_value( + py, &value, + )?)) } else { dict.get_item("gte")? - .map(|value| py_to_prop_value(py, &value).map(PropertyRangeBound::Included)) + .map(|value| py_to_prop_value(py, &value).map(CorePropertyRangeBound::Included)) .transpose()? }; let upper = if let Some(value) = dict.get_item("lt")? { - Some(PropertyRangeBound::Excluded(py_to_prop_value(py, &value)?)) + Some(CorePropertyRangeBound::Excluded(py_to_prop_value( + py, &value, + )?)) } else { dict.get_item("lte")? - .map(|value| py_to_prop_value(py, &value).map(PropertyRangeBound::Included)) + .map(|value| py_to_prop_value(py, &value).map(CorePropertyRangeBound::Included)) .transpose()? }; if lower.is_none() && upper.is_none() { @@ -3488,11 +4221,56 @@ fn parse_py_i64_range_bounds( Ok((lower, upper)) } -fn parse_py_query_limit(dict: &Bound<'_, PyDict>, _context: &str) -> PyResult> { +fn parse_py_f32_range_bounds( + dict: &Bound<'_, PyDict>, + context: &str, +) -> PyResult<(Option, Option)> { + if py_has_field(dict, "gt")? && py_has_field(dict, "gte")? { + return Err(PyValueError::new_err(format!( + "{} range predicate cannot specify both gt and gte", + context + ))); + } + if py_has_field(dict, "lt")? && py_has_field(dict, "lte")? { + return Err(PyValueError::new_err(format!( + "{} range predicate cannot specify both lt and lte", + context + ))); + } + let lower = if let Some(value) = dict.get_item("gt")? { + Some(next_up_f32(py_query_f32( + &value, + &format!("{} gt", context), + )?)) + } else { + dict.get_item("gte")? + .map(|value| py_query_f32(&value, &format!("{} gte", context))) + .transpose()? + }; + let upper = if let Some(value) = dict.get_item("lt")? { + Some(next_down_f32(py_query_f32( + &value, + &format!("{} lt", context), + )?)) + } else { + dict.get_item("lte")? + .map(|value| py_query_f32(&value, &format!("{} lte", context))) + .transpose()? + }; + if lower.is_none() && upper.is_none() { + return Err(PyValueError::new_err(format!( + "{} range predicate requires at least one of gt, gte, lt, or lte", + context + ))); + } + Ok((lower, upper)) +} + +fn parse_py_query_limit(dict: &Bound<'_, PyDict>, context: &str) -> PyResult> { match py_non_none_item(dict, "limit")? { None => Ok(None), Some(value) => { - let limit = py_query_usize(&value, "node query limit")?; + let limit = py_query_usize(&value, context)?; if limit == 0 { Ok(None) } else { @@ -3502,16 +4280,6 @@ fn parse_py_query_limit(dict: &Bound<'_, PyDict>, _context: &str) -> PyResult, - key: &str, - context: &str, -) -> PyResult> { - py_non_none_item(dict, key)? - .map(|value| py_query_u32(&value, context)) - .transpose() -} - fn py_optional_query_u64( dict: &Bound<'_, PyDict>, key: &str, @@ -3550,27 +4318,95 @@ fn py_optional_query_u64_vec( } } -fn py_optional_query_u32_vec( +fn py_optional_string_vec( dict: &Bound<'_, PyDict>, key: &str, context: &str, -) -> PyResult>> { +) -> PyResult>> { match py_non_none_item(dict, key)? { None => Ok(None), Some(value) => { let items = value.downcast::()?; let mut parsed = Vec::with_capacity(items.len()); for (index, item) in items.iter().enumerate() { - parsed.push(py_query_u32(&item, &format!("{}[{}]", context, index))?); + parsed.push(item.extract::().map_err(|_| { + PyTypeError::new_err(format!("{}[{}] must be str", context, index)) + })?); } Ok(Some(parsed)) } } } -fn py_query_u32(value: &Bound<'_, PyAny>, context: &str) -> PyResult { - let parsed = py_query_u64(value, context)?; - u32::try_from(parsed).map_err(|_| PyValueError::new_err(format!("{} must fit in u32", context))) +fn parse_node_labels_arg(value: &Bound<'_, PyAny>, context: &str) -> PyResult> { + if let Ok(label) = value.extract::() { + return Ok(vec![label]); + } + value + .extract::>() + .map_err(|_| PyTypeError::new_err(format!("{context} must be str or a sequence of str"))) +} + +fn parse_node_labels_list_field( + dict: &Bound<'_, PyDict>, + key: &str, + context: &str, +) -> PyResult> { + let value = py_non_none_item(dict, key)? + .ok_or_else(|| PyValueError::new_err(format!("{context} requires {key}")))?; + if value.extract::().is_ok() { + return Err(PyTypeError::new_err(format!( + "{context} {key} must be a sequence of str, not str" + ))); + } + value + .extract::>() + .map_err(|_| PyTypeError::new_err(format!("{context} {key} must be a sequence of str"))) +} + +fn parse_optional_node_label_filter_arg( + value: Option<&Bound<'_, PyAny>>, + context: &str, +) -> PyResult> { + value + .filter(|value| !value.is_none()) + .map(|value| parse_node_label_filter_arg(value, context)) + .transpose() +} + +fn parse_optional_node_label_filter_field( + dict: &Bound<'_, PyDict>, + key: &str, + context: &str, +) -> PyResult> { + py_non_none_item(dict, key)? + .map(|value| parse_node_label_filter_arg(&value, context)) + .transpose() +} + +fn parse_node_label_filter_arg( + value: &Bound<'_, PyAny>, + context: &str, +) -> PyResult { + let dict = value + .downcast::() + .map_err(|_| PyTypeError::new_err(format!("{context} must be a dict")))?; + ensure_only_py_fields(dict, &["labels", "mode"], context)?; + let labels = parse_node_labels_list_field(dict, "labels", context)?; + let mode_value = py_non_none_item(dict, "mode")? + .ok_or_else(|| PyValueError::new_err(format!("{context} requires mode")))?; + let mode = parse_label_match_mode(&mode_value.extract::()?, context)?; + Ok(NodeLabelFilter { labels, mode }) +} + +fn parse_label_match_mode(value: &str, context: &str) -> PyResult { + match value { + "any" => Ok(LabelMatchMode::Any), + "all" => Ok(LabelMatchMode::All), + other => Err(PyValueError::new_err(format!( + "{context} mode must be 'any' or 'all', got '{other}'" + ))), + } } fn py_query_u64(value: &Bound<'_, PyAny>, context: &str) -> PyResult { @@ -3583,6 +4419,57 @@ fn py_query_i64(value: &Bound<'_, PyAny>, context: &str) -> PyResult { value.extract::() } +fn py_query_f32(value: &Bound<'_, PyAny>, context: &str) -> PyResult { + reject_py_bool(value, context)?; + let parsed = value + .extract::() + .map_err(|_| PyValueError::new_err(format!("{} must be a number", context)))?; + if !parsed.is_finite() || parsed < f32::MIN as f64 || parsed > f32::MAX as f64 { + return Err(PyValueError::new_err(format!( + "{} must be a finite f32 number", + context + ))); + } + let parsed = parsed as f32; + if parsed.is_nan() { + return Err(PyValueError::new_err(format!( + "{} must not be NaN", + context + ))); + } + Ok(parsed) +} + +fn next_up_f32(value: f32) -> f32 { + if value == f32::INFINITY { + return value; + } + if value == -0.0 { + return f32::from_bits(1); + } + let bits = value.to_bits(); + if value >= 0.0 { + f32::from_bits(bits + 1) + } else { + f32::from_bits(bits - 1) + } +} + +fn next_down_f32(value: f32) -> f32 { + if value == f32::NEG_INFINITY { + return value; + } + if value == 0.0 { + return -f32::from_bits(1); + } + let bits = value.to_bits(); + if value > 0.0 { + f32::from_bits(bits - 1) + } else { + f32::from_bits(bits + 1) + } +} + fn py_query_usize(value: &Bound<'_, PyAny>, context: &str) -> PyResult { reject_py_bool(value, context)?; value.extract::() @@ -3653,18 +4540,6 @@ fn ensure_only_py_fields( Ok(()) } -fn ensure_no_py_fields(dict: &Bound<'_, PyDict>, fields: &[&str], context: &str) -> PyResult<()> { - for field in fields { - if py_has_field(dict, field)? { - return Err(PyValueError::new_err(format!( - "{} does not accept field '{}'", - context, field - ))); - } - } - Ok(()) -} - fn require_py_true_field(dict: &Bound<'_, PyDict>, field: &str, context: &str) -> PyResult<()> { let value = dict .get_item(field)? @@ -3814,9 +4689,9 @@ fn convert_py_props( } } -fn with_py_txn(inner: &Arc>>, f: F) -> PyResult +fn with_py_txn(inner: &Arc>>, f: F) -> PyResult where - F: FnOnce(&mut WriteTxn) -> PyResult, + F: FnOnce(&mut CoreWriteTxn) -> PyResult, { let mut guard = inner.lock().map_err(lock_err)?; let txn = guard @@ -3825,9 +4700,9 @@ where f(txn) } -fn with_py_txn_ref(inner: &Arc>>, f: F) -> PyResult +fn with_py_txn_ref(inner: &Arc>>, f: F) -> PyResult where - F: FnOnce(&WriteTxn) -> PyResult, + F: FnOnce(&CoreWriteTxn) -> PyResult, { let guard = inner.lock().map_err(lock_err)?; let txn = guard @@ -3836,9 +4711,9 @@ where f(txn) } -fn with_py_txn_take(inner: &Arc>>, f: F) -> PyResult +fn with_py_txn_take(inner: &Arc>>, f: F) -> PyResult where - F: FnOnce(&mut WriteTxn) -> PyResult, + F: FnOnce(&mut CoreWriteTxn) -> PyResult, { let mut txn = { let mut guard = inner.lock().map_err(lock_err)?; @@ -3851,27 +4726,33 @@ where fn parse_txn_node_ref(d: &Bound<'_, PyDict>) -> PyResult { let id = d.get_item("id")?; - let type_id = d.get_item("type_id")?; + let labels = d.get_item("labels")?; let key = d.get_item("key")?; let local = d.get_item("local")?; let has_id = id.is_some(); - let has_key = type_id.is_some() || key.is_some(); + let has_key = labels.is_some() || key.is_some(); let has_local = local.is_some(); match (has_id, has_key, has_local) { (true, false, false) => Ok(TxnNodeRef::Id(id.unwrap().extract()?)), - (false, true, false) => Ok(TxnNodeRef::Key { - type_id: type_id - .ok_or_else(|| PyValueError::new_err("node key ref requires type_id"))? - .extract()?, - key: key - .ok_or_else(|| PyValueError::new_err("node key ref requires key"))? - .extract()?, - }), + (false, true, false) => { + let labels_value = + labels.ok_or_else(|| PyValueError::new_err("node key ref requires labels"))?; + let labels = parse_node_labels_arg(&labels_value, "node key ref labels")?; + let [label]: [String; 1] = labels.try_into().map_err(|_| { + PyValueError::new_err("node key ref labels must contain exactly one label") + })?; + Ok(TxnNodeRef::Key { + label, + key: key + .ok_or_else(|| PyValueError::new_err("node key ref requires key"))? + .extract()?, + }) + } (false, false, true) => Ok(TxnNodeRef::Local(TxnLocalRef::Alias( local.unwrap().extract()?, ))), _ => Err(PyValueError::new_err( - "node ref must be exactly one of {'id'}, {'type_id', 'key'}, or {'local'}", + "node ref must be exactly one of {'id'}, {'labels', 'key'}, or {'local'}", )), } } @@ -3880,10 +4761,10 @@ fn parse_txn_edge_ref(d: &Bound<'_, PyDict>) -> PyResult { let id = d.get_item("id")?; let from = d.get_item("from")?; let to = d.get_item("to")?; - let type_id = d.get_item("type_id")?; + let label = d.get_item("label")?; let local = d.get_item("local")?; let has_id = id.is_some(); - let has_triple = from.is_some() || to.is_some() || type_id.is_some(); + let has_triple = from.is_some() || to.is_some() || label.is_some(); let has_local = local.is_some(); match (has_id, has_triple, has_local) { (true, false, false) => Ok(TxnEdgeRef::Id(id.unwrap().extract()?)), @@ -3893,8 +4774,8 @@ fn parse_txn_edge_ref(d: &Bound<'_, PyDict>) -> PyResult { Ok(TxnEdgeRef::Triple { from: parse_txn_node_ref(from.downcast::()?)?, to: parse_txn_node_ref(to.downcast::()?)?, - type_id: type_id - .ok_or_else(|| PyValueError::new_err("edge ref requires type_id"))? + label: label + .ok_or_else(|| PyValueError::new_err("edge ref requires label"))? .extract()?, }) } @@ -3902,7 +4783,7 @@ fn parse_txn_edge_ref(d: &Bound<'_, PyDict>) -> PyResult { local.unwrap().extract()?, ))), _ => Err(PyValueError::new_err( - "edge ref must be exactly one of {'id'}, {'from', 'to', 'type_id'}, or {'local'}", + "edge ref must be exactly one of {'id'}, {'from', 'to', 'label'}, or {'local'}", )), } } @@ -3911,8 +4792,8 @@ fn txn_node_ref_to_py(py: Python<'_>, value: TxnNodeRef) -> PyResult { let dict = PyDict::new(py); match value { TxnNodeRef::Id(id) => dict.set_item("id", id)?, - TxnNodeRef::Key { type_id, key } => { - dict.set_item("type_id", type_id)?; + TxnNodeRef::Key { label, key } => { + dict.set_item("labels", vec![label])?; dict.set_item("key", key)?; } TxnNodeRef::Local(local) => { @@ -3926,10 +4807,10 @@ fn txn_edge_ref_to_py(py: Python<'_>, value: TxnEdgeRef) -> PyResult { let dict = PyDict::new(py); match value { TxnEdgeRef::Id(id) => dict.set_item("id", id)?, - TxnEdgeRef::Triple { from, to, type_id } => { + TxnEdgeRef::Triple { from, to, label } => { dict.set_item("from", txn_node_ref_to_py(py, from)?)?; dict.set_item("to", txn_node_ref_to_py(py, to)?)?; - dict.set_item("type_id", type_id)?; + dict.set_item("label", label)?; } TxnEdgeRef::Local(local) => { dict.set_item("local", txn_local_ref_to_py(local))?; @@ -3949,7 +4830,7 @@ fn txn_node_view_to_py(py: Python<'_>, view: TxnNodeView) -> PyResult let dict = PyDict::new(py); dict.set_item("id", view.id)?; dict.set_item("local", view.local.and_then(txn_local_ref_to_py))?; - dict.set_item("type_id", view.type_id)?; + dict.set_item("labels", view.labels)?; dict.set_item("key", view.key)?; dict.set_item("props", props_to_py(py, &view.props)?)?; dict.set_item("created_at", view.created_at)?; @@ -3970,7 +4851,7 @@ fn txn_edge_view_to_py(py: Python<'_>, view: TxnEdgeView) -> PyResult dict.set_item("local", view.local.and_then(txn_local_ref_to_py))?; dict.set_item("from", txn_node_ref_to_py(py, view.from)?)?; dict.set_item("to", txn_node_ref_to_py(py, view.to)?)?; - dict.set_item("type_id", view.type_id)?; + dict.set_item("label", view.label)?; dict.set_item("props", props_to_py(py, &view.props)?)?; dict.set_item("created_at", view.created_at)?; dict.set_item("updated_at", view.updated_at)?; @@ -3991,10 +4872,7 @@ fn parse_txn_operations(py: Python<'_>, list: &Bound<'_, PyList>) -> PyResult TxnIntent::UpsertNode { alias: op.get_item("alias")?.map(|v| v.extract()).transpose()?, - type_id: op - .get_item("type_id")? - .ok_or_else(|| PyValueError::new_err("upsert_node requires type_id"))? - .extract()?, + labels: parse_node_labels_list_field(op, "labels", "upsert_node")?, key: op .get_item("key")? .ok_or_else(|| PyValueError::new_err("upsert_node requires key"))? @@ -4033,9 +4911,9 @@ fn parse_txn_operations(py: Python<'_>, list: &Bound<'_, PyList>) -> PyResult()?, )?, - type_id: op - .get_item("type_id")? - .ok_or_else(|| PyValueError::new_err("upsert_edge requires type_id"))? + label: op + .get_item("label")? + .ok_or_else(|| PyValueError::new_err("upsert_edge requires label"))? .extract()?, options: UpsertEdgeOptions { props: match op.get_item("props")? { @@ -4216,10 +5094,7 @@ fn parse_node_inputs(py: Python<'_>, list: &Bound<'_, PyList>) -> PyResult()?; - let type_id: u32 = d - .get_item("type_id")? - .ok_or_else(|| PyValueError::new_err("Node input missing 'type_id'"))? - .extract()?; + let labels = parse_node_labels_list_field(d, "labels", "Node input")?; let key: String = d .get_item("key")? .ok_or_else(|| PyValueError::new_err("Node input missing 'key'"))? @@ -4245,7 +5120,7 @@ fn parse_node_inputs(py: Python<'_>, list: &Bound<'_, PyList>) -> PyResult, list: &Bound<'_, PyList>) -> PyResult { @@ -4297,7 +5172,7 @@ fn parse_edge_inputs(py: Python<'_>, list: &Bound<'_, PyList>) -> PyResult, list: &Bound<'_, PyList>) -> PyResult) -> PyResult> { + let mut queries = Vec::with_capacity(list.len()); + for (index, item) in list.iter().enumerate() { + let d = item.downcast::().map_err(|_| { + PyTypeError::new_err(format!( + "get_nodes_by_keys[{}] must be a dict with 'labels' and 'key'", + index + )) + })?; + let labels_value = d + .get_item("labels")? + .ok_or_else(|| PyValueError::new_err("node key query requires labels"))?; + let labels = parse_node_labels_arg(&labels_value, "node key query labels")?; + let [label]: [String; 1] = labels.try_into().map_err(|_| { + PyValueError::new_err("node key query labels must contain exactly one label") + })?; + let key: String = d + .get_item("key")? + .ok_or_else(|| PyValueError::new_err("node key query requires key"))? + .extract()?; + queries.push(NodeKeyQuery { label, key }); + } + Ok(queries) +} + fn parse_graph_patch(py: Python<'_>, d: &Bound<'_, PyDict>) -> PyResult { let upsert_nodes = match d.get_item("upsert_nodes")? { Some(v) if !v.is_none() => { @@ -4474,6 +5374,13 @@ impl<'a> BinaryReader<'a> { } } + fn read_u8(&mut self) -> PyResult { + self.ensure(1)?; + let v = self.buf[self.pos]; + self.pos += 1; + Ok(v) + } + fn read_u16_le(&mut self) -> PyResult { self.ensure(2)?; let v = u16::from_le_bytes([self.buf[self.pos], self.buf[self.pos + 1]]); @@ -4544,12 +5451,47 @@ fn json_to_prop_value(v: &serde_json::Value) -> eg::PropValue { fn decode_node_batch_py(buf: &[u8]) -> PyResult> { let mut r = BinaryReader::new(buf); + let magic = r.read_bytes(4)?; + if magic != b"OGNB" { + return Err(PyValueError::new_err( + "Unsupported node binary batch format: expected OGNB version 2 header; old version 1 buffers are not supported", + )); + } + let version = r.read_u16_le()?; + if version != 2 { + return Err(PyValueError::new_err(format!( + "Unsupported node binary batch version {}; expected version 2", + version + ))); + } let count = r.read_u32_le()? as usize; - // Cap allocation: minimum node record is 14 bytes (type_id + weight + key_len + props_len) - let max_possible = buf.len().saturating_sub(4) / 14; + // Cap allocation: minimum v2 node record is label_count + one 1-byte label + weight + key_len + props_len. + let max_possible = buf.len().saturating_sub(10) / 14; let mut inputs = Vec::with_capacity(count.min(max_possible)); for _ in 0..count { - let type_id = r.read_u32_le()?; + let label_count = r.read_u8()? as usize; + if label_count == 0 || label_count > 10 { + return Err(PyValueError::new_err( + "node binary label_count must be between 1 and 10", + )); + } + let mut labels = Vec::with_capacity(label_count); + for _ in 0..label_count { + let label_len = r.read_u16_le()? as usize; + validate_binary_token_len(label_len, "label")?; + let label_bytes = r.read_bytes(label_len)?; + let label = std::str::from_utf8(label_bytes) + .map_err(|e| PyValueError::new_err(format!("Invalid UTF-8 in node label: {}", e)))? + .to_string(); + validate_py_type_token_name(&label, "node label")?; + if labels.iter().any(|existing| existing == &label) { + return Err(PyValueError::new_err(format!( + "node binary labels contain duplicate label '{}'", + label + ))); + } + labels.push(label); + } let weight = r.read_f32_le()?; let key_len = r.read_u16_le()? as usize; let key_bytes = r.read_bytes(key_len)?; @@ -4572,7 +5514,7 @@ fn decode_node_batch_py(buf: &[u8]) -> PyResult> { } }; inputs.push(NodeInput { - type_id, + labels, key, props, weight, @@ -4593,13 +5535,19 @@ fn decode_node_batch_py(buf: &[u8]) -> PyResult> { fn decode_edge_batch_py(buf: &[u8]) -> PyResult> { let mut r = BinaryReader::new(buf); let count = r.read_u32_le()? as usize; - // Cap allocation: minimum edge record is 36 bytes (from + to + type_id + weight + valid_from + valid_to + props_len) - let max_possible = buf.len().saturating_sub(4) / 36; + // Cap allocation: minimum edge record is 34 bytes (from + to + label_len + weight + valid_from + valid_to + props_len) + let max_possible = buf.len().saturating_sub(4) / 34; let mut inputs = Vec::with_capacity(count.min(max_possible)); for _ in 0..count { let from = r.read_u64_le()?; let to = r.read_u64_le()?; - let type_id = r.read_u32_le()?; + let label_len = r.read_u16_le()? as usize; + validate_binary_token_len(label_len, "label")?; + let label_bytes = r.read_bytes(label_len)?; + let label = std::str::from_utf8(label_bytes) + .map_err(|e| PyValueError::new_err(format!("Invalid UTF-8 in edge label: {}", e)))? + .to_string(); + validate_py_type_token_name(&label, "edge label")?; let weight = r.read_f32_le()?; let valid_from_raw = r.read_i64_le()?; let valid_to_raw = r.read_i64_le()?; @@ -4631,7 +5579,7 @@ fn decode_edge_batch_py(buf: &[u8]) -> PyResult> { inputs.push(EdgeInput { from, to, - type_id, + label, props, weight, valid_from, @@ -4648,6 +5596,47 @@ fn decode_edge_batch_py(buf: &[u8]) -> PyResult> { Ok(inputs) } +fn validate_binary_token_len(len: usize, field: &str) -> PyResult<()> { + if len == 0 || len > 255 { + return Err(PyValueError::new_err(format!( + "{} length must be between 1 and 255 bytes", + field + ))); + } + Ok(()) +} + +fn validate_py_type_token_name(name: &str, context: &str) -> PyResult<()> { + if name.is_empty() { + return Err(PyValueError::new_err(format!( + "{} must not be empty", + context + ))); + } + if name.len() > 255 { + return Err(PyValueError::new_err(format!( + "{} must be at most 255 UTF-8 bytes", + context + ))); + } + if name.trim_matches(char::is_whitespace).len() != name.len() { + return Err(PyValueError::new_err(format!( + "{} must not contain leading or trailing whitespace", + context + ))); + } + if name + .chars() + .any(|ch| ch == '\0' || (ch.is_ascii() && ch.is_control())) + { + return Err(PyValueError::new_err(format!( + "{} must not contain ASCII control characters or NUL", + context + ))); + } + Ok(()) +} + // ============================================================ // Module registration // ============================================================ @@ -4655,38 +5644,44 @@ fn decode_edge_batch_py(buf: &[u8]) -> PyResult> { #[pymodule] fn overgraph(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add("OverGraphError", m.py().get_type::())?; Ok(()) } diff --git a/overgraph-python/tests/conftest.py b/overgraph-python/tests/conftest.py index a896536..80e6c52 100644 --- a/overgraph-python/tests/conftest.py +++ b/overgraph-python/tests/conftest.py @@ -38,25 +38,25 @@ async def async_db(tmp_dir): await database.close() -def make_chain(db, n=5, edge_type=10): +def make_chain(db, n=5, label="RELATES_TO"): """Create a chain: n0 -> n1 -> n2 -> ... -> n(n-1)""" nodes = [] for i in range(n): - nid = db.upsert_node(1, f"node_{i}") + nid = db.upsert_node("Person", f"node_{i}") nodes.append(nid) edges = [] for i in range(len(nodes) - 1): - eid = db.upsert_edge(nodes[i], nodes[i + 1], edge_type) + eid = db.upsert_edge(nodes[i], nodes[i + 1], label) edges.append(eid) return nodes, edges -def make_star(db, center_key="center", spokes=5, edge_type=10): +def make_star(db, center_key="center", spokes=5, label="RELATES_TO"): """Create a star: center -> spoke_0, center -> spoke_1, ...""" - center = db.upsert_node(1, center_key) + center = db.upsert_node("Person", center_key) spoke_ids = [] for i in range(spokes): - sid = db.upsert_node(1, f"spoke_{i}") - db.upsert_edge(center, sid, edge_type) + sid = db.upsert_node("Person", f"spoke_{i}") + db.upsert_edge(center, sid, label) spoke_ids.append(sid) return center, spoke_ids diff --git a/overgraph-python/tests/test_analytics.py b/overgraph-python/tests/test_analytics.py index 6acc4e9..4c3172a 100644 --- a/overgraph-python/tests/test_analytics.py +++ b/overgraph-python/tests/test_analytics.py @@ -20,30 +20,30 @@ def test_ppr_seed_has_highest_score(self, db): def test_ppr_multiple_seeds(self, db): # Two disconnected stars - c1 = db.upsert_node(1, "c1") + c1 = db.upsert_node("Person", "c1") for i in range(3): - s = db.upsert_node(1, f"s1_{i}") - db.upsert_edge(c1, s, 10) - c2 = db.upsert_node(1, "c2") + s = db.upsert_node("Person", f"s1_{i}") + db.upsert_edge(c1, s, "RELATES_TO") + c2 = db.upsert_node("Person", "c2") for i in range(3): - s = db.upsert_node(1, f"s2_{i}") - db.upsert_edge(c2, s, 10) + s = db.upsert_node("Person", f"s2_{i}") + db.upsert_edge(c2, s, "RELATES_TO") result = db.personalized_pagerank([c1, c2]) assert len(result.node_ids) > 0 # Both seeds should appear assert c1 in result.node_ids assert c2 in result.node_ids - def test_ppr_edge_type_filter(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 20) - # Only follow type 10 edges - result = db.personalized_pagerank([n1], edge_type_filter=[10]) + def test_ppr_edge_label_filter(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "WORKS_AT") + # Only follow RELATES_TO edges. + result = db.personalized_pagerank([n1], edge_label_filter=["RELATES_TO"]) assert n2 in result.node_ids - # n3 should not be reachable via type 10 + # n3 should not be reachable through the filtered edge label. assert n3 not in result.node_ids def test_ppr_max_results(self, db): @@ -77,7 +77,7 @@ def test_ppr_approx_mode_exposes_metadata(self, db): assert len(result.node_ids) > 0 def test_ppr_isolated_node(self, db): - nid = db.upsert_node(1, "lonely") + nid = db.upsert_node("Person", "lonely") result = db.personalized_pagerank([nid]) assert len(result.node_ids) == 1 assert result.node_ids[0] == nid @@ -98,7 +98,7 @@ def test_ppr_chain(self, db): assert result.scores[prev_idx] >= result.scores[curr_idx] def test_ppr_empty_seeds(self, db): - db.upsert_node(1, "a") + db.upsert_node("Person", "a") result = db.personalized_pagerank([]) assert len(result.node_ids) == 0 assert len(result.scores) == 0 @@ -117,49 +117,48 @@ def test_basic_export(self, db): assert e.from_id in export.node_ids assert e.to_id in export.node_ids - def test_export_node_type_filter(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(2, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 10) - # Only type 1 nodes - export = db.export_adjacency(node_type_filter=[1]) + def test_export_node_label_filter(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Company", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "RELATES_TO") + # Only Person nodes. + export = db.export_adjacency(node_label_filter={"labels": ["Person"], "mode": "all"}) assert n1 in export.node_ids assert n3 in export.node_ids assert n2 not in export.node_ids + assert "Person" in export.node_labels # Edge n1->n2 should be excluded (n2 not in subgraph) for e in export.edges: assert e.to_id != n2 - def test_export_edge_type_filter(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 20) - export = db.export_adjacency(edge_type_filter=[10]) + def test_export_edge_label_filter(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "WORKS_AT") + export = db.export_adjacency(edge_label_filter=["RELATES_TO"]) # All nodes should be present (node filter is separate) assert len(export.node_ids) == 3 - # Only type 10 edges - assert all(e.type_id == 10 for e in export.edges) + assert all(export.edge_labels[e.edge_label_index] == "RELATES_TO" for e in export.edges) def test_export_include_weights(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - db.upsert_edge(n1, n2, 10, weight=3.5) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + db.upsert_edge(n1, n2, "RELATES_TO", weight=3.5) export = db.export_adjacency(include_weights=True) assert len(export.edges) == 1 assert abs(export.edges[0].weight - 3.5) < 0.01 def test_export_without_weights(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - db.upsert_edge(n1, n2, 10, weight=3.5) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + db.upsert_edge(n1, n2, "RELATES_TO", weight=3.5) export = db.export_adjacency(include_weights=False) assert len(export.edges) == 1 - # Without weights, weight should be 0 - assert export.edges[0].weight == 0.0 + assert export.edges[0].weight is None def test_export_empty(self, db): export = db.export_adjacency() @@ -168,11 +167,11 @@ def test_export_empty(self, db): def test_export_consistent_subgraph(self, db): """Edges should only appear if both endpoints are in the node set.""" - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(2, "b") - db.upsert_edge(n1, n2, 10) - # Filter to only type 1 (n2 excluded) - export = db.export_adjacency(node_type_filter=[1]) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Company", "b") + db.upsert_edge(n1, n2, "RELATES_TO") + # Filter to only Person nodes (n2 excluded). + export = db.export_adjacency(node_label_filter={"labels": ["Person"], "mode": "all"}) assert n1 in export.node_ids assert n2 not in export.node_ids # No edges should be present since n2 is not in node set diff --git a/overgraph-python/tests/test_async.py b/overgraph-python/tests/test_async.py index 57101dd..388c4f5 100644 --- a/overgraph-python/tests/test_async.py +++ b/overgraph-python/tests/test_async.py @@ -7,7 +7,7 @@ import pytest import pytest_asyncio -from overgraph import AsyncOverGraph +from overgraph import AsyncOverGraph, OverGraphError @pytest_asyncio.fixture @@ -42,26 +42,90 @@ async def test_context_manager(self): path = os.path.join(d, "testdb") try: async with await AsyncOverGraph.open(path) as db: - nid = await db.upsert_node(1, "test") + nid = await db.upsert_node("Person", "test") assert nid > 0 finally: shutil.rmtree(d, ignore_errors=True) class TestAsyncCrud: + @pytest.mark.asyncio + async def test_catalog_apis(self, async_db): + person_id = await async_db.ensure_node_label("Person") + company_id = await async_db.ensure_node_label("Company") + relates_to_id = await async_db.ensure_edge_label("RELATES_TO") + works_at_id = await async_db.ensure_edge_label("WORKS_AT") + + assert await async_db.ensure_node_label("Person") == person_id + assert await async_db.ensure_edge_label("RELATES_TO") == relates_to_id + assert await async_db.get_node_label_id("Person") == person_id + assert await async_db.get_node_label_id("Company") == company_id + assert await async_db.get_edge_label_id("RELATES_TO") == relates_to_id + assert await async_db.get_edge_label_id("WORKS_AT") == works_at_id + assert await async_db.get_node_label(person_id) == "Person" + assert await async_db.get_node_label(company_id) == "Company" + assert await async_db.get_edge_label(relates_to_id) == "RELATES_TO" + assert await async_db.get_edge_label(works_at_id) == "WORKS_AT" + assert await async_db.get_edge_label(label_id=relates_to_id) == "RELATES_TO" + assert await async_db.get_node_label_id("Document") is None + assert await async_db.get_edge_label_id("LIKES") is None + assert await async_db.get_node_label(999999) is None + assert await async_db.get_edge_label(999999) is None + + old_field_name = "type" + "_id" + with pytest.raises(TypeError): + await async_db.get_edge_label(**{old_field_name: relates_to_id}) + + node_labels = { + entry.label: entry.label_id + for entry in await async_db.list_node_labels() + } + edge_label_entries = await async_db.list_edge_labels() + edge_labels = { + entry.label: entry.label_id + for entry in edge_label_entries + } + assert node_labels["Person"] == person_id + assert node_labels["Company"] == company_id + assert edge_labels["RELATES_TO"] == relates_to_id + assert edge_labels["WORKS_AT"] == works_at_id + for entry in edge_label_entries: + assert not hasattr(entry, old_field_name) + assert "label_id=" in repr(edge_label_entries[0]) + @pytest.mark.asyncio async def test_upsert_get_node(self, async_db): - nid = await async_db.upsert_node(1, "hello", props={"x": 42}) + nid = await async_db.upsert_node("Person", "hello", props={"x": 42}) node = await async_db.get_node(nid) assert node is not None assert node.key == "hello" - assert node.type_id == 1 + assert node.labels == ["Person"] + assert await async_db.add_node_label(nid, "Admin") is True + assert await async_db.add_node_label(nid, "Admin") is False + assert (await async_db.get_node(nid)).labels == ["Person", "Admin"] + assert await async_db.remove_node_label(nid, "Admin") is True + assert await async_db.remove_node_label(nid, "Admin") is False + + @pytest.mark.asyncio + async def test_node_label_failure_paths(self, async_db): + solo = await async_db.upsert_node("Person", "solo") + with pytest.raises(OverGraphError, match="last node label"): + await async_db.remove_node_label(solo, "Person") + assert (await async_db.get_node(solo)).labels == ["Person"] + + alice = await async_db.upsert_node("Person", "shared") + other = await async_db.upsert_node("Admin", "shared") + with pytest.raises(OverGraphError, match="node key conflict"): + await async_db.add_node_label(alice, "Admin") + + assert (await async_db.get_node(alice)).labels == ["Person"] + assert (await async_db.get_node(other)).labels == ["Admin"] @pytest.mark.asyncio async def test_upsert_get_edge(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - eid = await async_db.upsert_edge(n1, n2, 10, weight=2.5) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + eid = await async_db.upsert_edge(n1, n2, "RELATES_TO", weight=2.5) edge = await async_db.get_edge(eid) assert edge is not None assert edge.from_id == n1 @@ -69,74 +133,74 @@ async def test_upsert_get_edge(self, async_db): @pytest.mark.asyncio async def test_delete_node(self, async_db): - nid = await async_db.upsert_node(1, "bye") + nid = await async_db.upsert_node("Person", "bye") await async_db.delete_node(nid) assert await async_db.get_node(nid) is None @pytest.mark.asyncio async def test_delete_edge(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - eid = await async_db.upsert_edge(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + eid = await async_db.upsert_edge(n1, n2, "RELATES_TO") await async_db.delete_edge(eid) assert await async_db.get_edge(eid) is None @pytest.mark.asyncio async def test_get_node_by_key(self, async_db): - nid = await async_db.upsert_node(1, "mykey") - node = await async_db.get_node_by_key(1, "mykey") + nid = await async_db.upsert_node("Person", "mykey") + node = await async_db.get_node_by_key("Person", "mykey") assert node is not None assert node.id == nid @pytest.mark.asyncio async def test_get_edge_by_triple(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - eid = await async_db.upsert_edge(n1, n2, 10) - edge = await async_db.get_edge_by_triple(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + eid = await async_db.upsert_edge(n1, n2, "RELATES_TO") + edge = await async_db.get_edge_by_triple(n1, n2, "RELATES_TO") assert edge is not None assert edge.id == eid @pytest.mark.asyncio async def test_invalidate_edge(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - eid = await async_db.upsert_edge(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + eid = await async_db.upsert_edge(n1, n2, "RELATES_TO") result = await async_db.invalidate_edge(eid, 1000) - assert result is not None # returns updated EdgeRecord + assert result is not None # returns updated EdgeView class TestAsyncBatch: @pytest.mark.asyncio async def test_batch_upsert_nodes(self, async_db): - nodes = [{"type_id": 1, "key": f"n{i}"} for i in range(5)] + nodes = [{"labels": ["Person"], "key": f"n{i}"} for i in range(5)] ids = await async_db.batch_upsert_nodes(nodes) assert len(ids) == 5 @pytest.mark.asyncio async def test_batch_upsert_edges(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - n3 = await async_db.upsert_node(1, "c") + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + n3 = await async_db.upsert_node("Person", "c") edges = [ - {"from_id": n1, "to_id": n2, "type_id": 10}, - {"from_id": n2, "to_id": n3, "type_id": 10}, + {"from_id": n1, "to_id": n2, "label": "RELATES_TO"}, + {"from_id": n2, "to_id": n3, "label": "RELATES_TO"}, ] ids = await async_db.batch_upsert_edges(edges) assert len(ids) == 2 @pytest.mark.asyncio async def test_get_nodes(self, async_db): - nids = [await async_db.upsert_node(1, f"n{i}") for i in range(3)] + nids = [await async_db.upsert_node("Person", f"n{i}") for i in range(3)] nodes = await async_db.get_nodes(nids) assert len(nodes) == 3 assert all(n is not None for n in nodes) @pytest.mark.asyncio async def test_get_edges(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - eid = await async_db.upsert_edge(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + eid = await async_db.upsert_edge(n1, n2, "RELATES_TO") edges = await async_db.get_edges([eid]) assert len(edges) == 1 assert edges[0] is not None @@ -144,9 +208,13 @@ async def test_get_edges(self, async_db): @pytest.mark.asyncio async def test_get_nodes_by_keys(self, async_db): - await async_db.upsert_node(1, "alice") - await async_db.upsert_node(1, "bob") - results = await async_db.get_nodes_by_keys([(1, "alice"), (1, "bob"), (1, "missing")]) + await async_db.upsert_node("Person", "alice") + await async_db.upsert_node("Person", "bob") + results = await async_db.get_nodes_by_keys([ + {"labels": ["Person"], "key": "alice"}, + {"labels": ["Person"], "key": "bob"}, + {"labels": ["Person"], "key": "missing"}, + ]) assert len(results) == 3 assert results[0].key == "alice" assert results[1].key == "bob" @@ -156,8 +224,8 @@ async def test_get_nodes_by_keys(self, async_db): async def test_graph_patch(self, async_db): result = await async_db.graph_patch({ "upsert_nodes": [ - {"type_id": 1, "key": "a"}, - {"type_id": 1, "key": "b"}, + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, ], }) assert len(result.node_ids) == 2 @@ -172,17 +240,17 @@ async def test_async_stage_read_and_commit(self, async_db): { "op": "upsert_node", "alias": "alice", - "type_id": 1, + "labels": ["Person"], "key": "alice", "props": {"name": "Alice"}, }, - {"op": "upsert_node", "alias": "bob", "type_id": 1, "key": "bob"}, + {"op": "upsert_node", "alias": "bob", "labels": ["Person"], "key": "bob"}, { "op": "upsert_edge", "alias": "knows", "from": {"local": "alice"}, "to": {"local": "bob"}, - "type_id": 7, + "label": "KNOWS", }, ] ) @@ -190,6 +258,7 @@ async def test_async_stage_read_and_commit(self, async_db): staged = await txn.get_node({"local": "alice"}) assert staged is not None assert staged["id"] is None + assert staged["labels"] == ["Person"] assert staged["props"]["name"] == "Alice" result = await txn.commit() @@ -201,16 +270,48 @@ async def test_async_stage_read_and_commit(self, async_db): @pytest.mark.asyncio async def test_async_builders_and_rollback(self, async_db): txn = await async_db.begin_write_txn() - alice = await txn.upsert_node_as("alice", 1, "alice", props={"mood": "staged"}) - bob = await txn.upsert_node_as("bob", 1, "bob") - await txn.upsert_edge_as("knows", alice, bob, 9) + alice = await txn.upsert_node_as("alice", "Person", "alice", props={"mood": "staged"}) + bob = await txn.upsert_node_as("bob", "Person", "bob") + await txn.upsert_edge_as("knows", alice, bob, "KNOWS") - staged = await txn.get_node_by_key(1, "alice") + staged = await txn.get_node_by_key("Person", "alice") assert staged is not None assert staged["props"]["mood"] == "staged" await txn.rollback() - assert await async_db.get_node_by_key(1, "alice") is None + assert await async_db.get_node_by_key("Person", "alice") is None + + @pytest.mark.asyncio + async def test_async_transaction_add_remove_node_label(self, async_db): + node_id = await async_db.upsert_node("Person", "alice") + + txn = await async_db.begin_write_txn() + assert await txn.add_node_label({"id": node_id}, "Admin") is True + assert await txn.add_node_label({"id": node_id}, "Admin") is False + assert (await txn.get_node({"id": node_id}))["labels"] == ["Person", "Admin"] + assert await txn.remove_node_label({"id": node_id}, "Admin") is True + assert await txn.remove_node_label({"id": node_id}, "Admin") is False + await txn.commit() + + assert (await async_db.get_node(node_id)).labels == ["Person"] + + @pytest.mark.asyncio + async def test_async_transaction_node_label_failure_paths(self, async_db): + solo = await async_db.upsert_node("Person", "solo") + txn = await async_db.begin_write_txn() + with pytest.raises(OverGraphError, match="last node label"): + await txn.remove_node_label({"id": solo}, "Person") + await txn.rollback() + + alice = await async_db.upsert_node("Person", "shared") + other = await async_db.upsert_node("Admin", "shared") + conflict_txn = await async_db.begin_write_txn() + with pytest.raises(OverGraphError, match="node key conflict"): + await conflict_txn.add_node_label({"id": alice}, "Admin") + await conflict_txn.rollback() + + assert (await async_db.get_node(alice)).labels == ["Person"] + assert (await async_db.get_node(other)).labels == ["Admin"] @pytest.mark.asyncio async def test_async_transaction_operations_preserve_call_order(self, async_db): @@ -221,7 +322,7 @@ async def test_async_transaction_operations_preserve_call_order(self, async_db): { "op": "upsert_node", "alias": "queued", - "type_id": 1, + "labels": ["Person"], "key": "queued", } ] @@ -242,42 +343,42 @@ async def test_async_transaction_operations_preserve_call_order(self, async_db): class TestAsyncQueries: @pytest.mark.asyncio async def test_find_nodes(self, async_db): - await async_db.upsert_node(1, "x", props={"color": "red"}) - ids = await async_db.find_nodes(1, "color", "red") + await async_db.upsert_node("Person", "x", props={"color": "red"}) + ids = await async_db.find_nodes("Person", "color", "red") assert len(ids) == 1 @pytest.mark.asyncio async def test_count_by_type(self, async_db): for i in range(3): - await async_db.upsert_node(1, f"n{i}") - count = await async_db.count_nodes_by_type(1) + await async_db.upsert_node("Person", f"n{i}") + count = await async_db.count_nodes_by_labels("Person") assert count == 3 class TestAsyncTraversal: @pytest.mark.asyncio async def test_neighbors(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(n1, n2, "RELATES_TO") nbrs = await async_db.neighbors(n1, direction="outgoing") assert len(nbrs) == 1 assert nbrs[0].node_id == n2 @pytest.mark.asyncio async def test_top_k_neighbors(self, async_db): - center = await async_db.upsert_node(1, "center") + center = await async_db.upsert_node("Person", "center") for i in range(3): - s = await async_db.upsert_node(1, f"s{i}") - await async_db.upsert_edge(center, s, 10, weight=float(i + 1)) + s = await async_db.upsert_node("Person", f"s{i}") + await async_db.upsert_edge(center, s, "RELATES_TO", weight=float(i + 1)) top = await async_db.top_k_neighbors(center, k=2, scoring="weight") assert len(top) == 2 @pytest.mark.asyncio async def test_extract_subgraph(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(n1, n2, "RELATES_TO") sg = await async_db.extract_subgraph(n1, 1) assert len(sg.nodes) == 2 assert len(sg.edges) == 1 @@ -286,8 +387,8 @@ async def test_extract_subgraph(self, async_db): class TestAsyncRetention: @pytest.mark.asyncio async def test_prune(self, async_db): - await async_db.upsert_node(1, "low", weight=0.1) - await async_db.upsert_node(1, "high", weight=5.0) + await async_db.upsert_node("Person", "low", weight=0.1) + await async_db.upsert_node("Person", "high", weight=5.0) result = await async_db.prune(max_weight=0.5) assert result.nodes_pruned == 1 @@ -303,26 +404,26 @@ async def test_prune_policies(self, async_db): class TestAsyncTimeRange: @pytest.mark.asyncio async def test_find_nodes_by_time_range(self, async_db): - await async_db.upsert_node(1, "a") - await async_db.upsert_node(1, "b") + await async_db.upsert_node("Person", "a") + await async_db.upsert_node("Person", "b") # Use a wide range to catch all nodes - ids = await async_db.find_nodes_by_time_range(1, 0, 2**53) + ids = await async_db.find_nodes_by_time_range("Person", 0, 2**53) assert len(ids) == 2 class TestAsyncMaintenance: @pytest.mark.asyncio async def test_sync_flush(self, async_db): - await async_db.upsert_node(1, "a") + await async_db.upsert_node("Person", "a") await async_db.sync() result = await async_db.flush() assert result is not None @pytest.mark.asyncio async def test_compact(self, async_db): - await async_db.upsert_node(1, "a") + await async_db.upsert_node("Person", "a") await async_db.flush() - await async_db.upsert_node(1, "b") + await async_db.upsert_node("Person", "b") await async_db.flush() result = await async_db.compact() assert result is not None @@ -330,19 +431,19 @@ async def test_compact(self, async_db): class TestAsyncPagination: @pytest.mark.asyncio - async def test_nodes_by_type_paged(self, async_db): + async def test_nodes_by_labels_paged(self, async_db): for i in range(5): - await async_db.upsert_node(1, f"n{i}") - page = await async_db.nodes_by_type_paged(1, limit=3) + await async_db.upsert_node("Person", f"n{i}") + page = await async_db.nodes_by_labels_paged("Person", limit=3) assert len(page.items) == 3 assert page.next_cursor is not None @pytest.mark.asyncio async def test_neighbors_paged(self, async_db): - center = await async_db.upsert_node(1, "center") + center = await async_db.upsert_node("Person", "center") for i in range(5): - s = await async_db.upsert_node(1, f"s{i}") - await async_db.upsert_edge(center, s, 10) + s = await async_db.upsert_node("Person", f"s{i}") + await async_db.upsert_edge(center, s, "RELATES_TO") page = await async_db.neighbors_paged(center, direction="outgoing", limit=3) assert len(page.items) == 3 @@ -350,28 +451,28 @@ async def test_neighbors_paged(self, async_db): class TestAsyncTraversal2: @pytest.mark.asyncio async def test_traverse(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - n3 = await async_db.upsert_node(1, "c") - await async_db.upsert_edge(n1, n2, 10) - await async_db.upsert_edge(n2, n3, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + n3 = await async_db.upsert_node("Person", "c") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + await async_db.upsert_edge(n2, n3, "RELATES_TO") page = await async_db.traverse(n1, 2, min_depth=2, direction="outgoing") assert [(hit.node_id, hit.depth) for hit in page.items] == [(n3, 2)] @pytest.mark.asyncio - async def test_traverse_node_type_filter(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(2, "b") - n3 = await async_db.upsert_node(3, "c") - await async_db.upsert_edge(n1, n2, 10) - await async_db.upsert_edge(n2, n3, 10) + async def test_traverse_node_label_filter(self, async_db): + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Company", "b") + n3 = await async_db.upsert_node("Document", "c") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + await async_db.upsert_edge(n2, n3, "RELATES_TO") page = await async_db.traverse( n1, 2, min_depth=2, direction="outgoing", - edge_type_filter=[10], - node_type_filter=[3], + edge_label_filter=["RELATES_TO"], + emit_node_label_filter={"labels": ["Document"], "mode": "all"}, ) assert [(hit.node_id, hit.depth) for hit in page.items] == [(n3, 2)] @@ -384,11 +485,11 @@ async def test_removed_two_hop_async_apis_stay_absent(self, async_db): @pytest.mark.asyncio async def test_neighbors_batch(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - n3 = await async_db.upsert_node(1, "c") - await async_db.upsert_edge(n1, n2, 10) - await async_db.upsert_edge(n1, n3, 20) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + n3 = await async_db.upsert_node("Person", "c") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + await async_db.upsert_edge(n1, n3, "WORKS_AT") result = await async_db.neighbors_batch([n1]) assert n1 in result assert len(result[n1]) == 2 @@ -396,113 +497,113 @@ async def test_neighbors_batch(self, async_db): class TestAsyncQueries2: @pytest.mark.asyncio - async def test_count_edges_by_type(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(n1, n2, 10) - count = await async_db.count_edges_by_type(10) + async def test_count_edges_by_label(self, async_db): + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + count = await async_db.count_edges_by_label("RELATES_TO") assert count == 1 @pytest.mark.asyncio - async def test_nodes_by_type(self, async_db): - await async_db.upsert_node(1, "a") - await async_db.upsert_node(1, "b") - ids = await async_db.nodes_by_type(1) + async def test_nodes_by_labels(self, async_db): + await async_db.upsert_node("Person", "a") + await async_db.upsert_node("Person", "b") + ids = await async_db.nodes_by_labels("Person") assert len(ids) == 2 @pytest.mark.asyncio - async def test_edges_by_type(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(n1, n2, 10) - ids = await async_db.edges_by_type(10) + async def test_edges_by_label(self, async_db): + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + ids = await async_db.edges_by_label("RELATES_TO") assert len(ids) == 1 @pytest.mark.asyncio - async def test_get_nodes_by_type(self, async_db): - await async_db.upsert_node(1, "a") - await async_db.upsert_node(1, "b") - nodes = await async_db.get_nodes_by_type(1) + async def test_get_nodes_by_labels(self, async_db): + await async_db.upsert_node("Person", "a") + await async_db.upsert_node("Person", "b") + nodes = await async_db.get_nodes_by_labels("Person") assert len(nodes) == 2 keys = {n.key for n in nodes} assert keys == {"a", "b"} @pytest.mark.asyncio - async def test_get_edges_by_type(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(n1, n2, 10) - edges = await async_db.get_edges_by_type(10) + async def test_get_edges_by_label(self, async_db): + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + edges = await async_db.get_edges_by_label("RELATES_TO") assert len(edges) == 1 assert edges[0].from_id == n1 class TestAsyncPagination2: @pytest.mark.asyncio - async def test_edges_by_type_paged(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - n3 = await async_db.upsert_node(1, "c") - await async_db.upsert_edge(n1, n2, 10) - await async_db.upsert_edge(n1, n3, 10) - await async_db.upsert_edge(n2, n3, 10) - page = await async_db.edges_by_type_paged(10, limit=2) + async def test_edges_by_label_paged(self, async_db): + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + n3 = await async_db.upsert_node("Person", "c") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + await async_db.upsert_edge(n1, n3, "RELATES_TO") + await async_db.upsert_edge(n2, n3, "RELATES_TO") + page = await async_db.edges_by_label_paged("RELATES_TO", limit=2) assert len(page.items) == 2 assert page.next_cursor is not None @pytest.mark.asyncio - async def test_get_nodes_by_type_paged(self, async_db): + async def test_get_nodes_by_labels_paged(self, async_db): for i in range(5): - await async_db.upsert_node(1, f"n{i}") - page = await async_db.get_nodes_by_type_paged(1, limit=3) + await async_db.upsert_node("Person", f"n{i}") + page = await async_db.get_nodes_by_labels_paged("Person", limit=3) assert len(page.items) == 3 assert page.next_cursor is not None @pytest.mark.asyncio - async def test_get_edges_by_type_paged(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - n3 = await async_db.upsert_node(1, "c") - await async_db.upsert_edge(n1, n2, 10) - await async_db.upsert_edge(n2, n3, 10) - page = await async_db.get_edges_by_type_paged(10, limit=1) + async def test_get_edges_by_label_paged(self, async_db): + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + n3 = await async_db.upsert_node("Person", "c") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + await async_db.upsert_edge(n2, n3, "RELATES_TO") + page = await async_db.get_edges_by_label_paged("RELATES_TO", limit=1) assert len(page.items) == 1 assert page.next_cursor is not None @pytest.mark.asyncio async def test_find_nodes_paged(self, async_db): for i in range(5): - await async_db.upsert_node(1, f"fp{i}", props={"color": "blue"}) - page = await async_db.find_nodes_paged(1, "color", "blue", limit=3) + await async_db.upsert_node("Person", f"fp{i}", props={"color": "blue"}) + page = await async_db.find_nodes_paged("Person", "color", "blue", limit=3) assert len(page.items) == 3 assert page.next_cursor is not None @pytest.mark.asyncio async def test_find_nodes_by_time_range_paged(self, async_db): for i in range(5): - await async_db.upsert_node(1, f"tr{i}") - page = await async_db.find_nodes_by_time_range_paged(1, 0, 2**53, limit=3) + await async_db.upsert_node("Person", f"tr{i}") + page = await async_db.find_nodes_by_time_range_paged("Person", 0, 2**53, limit=3) assert len(page.items) == 3 @pytest.mark.asyncio async def test_traverse_paged(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - n3 = await async_db.upsert_node(1, "c") - await async_db.upsert_edge(n1, n2, 10) - await async_db.upsert_edge(n2, n3, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + n3 = await async_db.upsert_node("Person", "c") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + await async_db.upsert_edge(n2, n3, "RELATES_TO") page = await async_db.traverse(n1, 2, min_depth=2, direction="outgoing") assert [(hit.node_id, hit.depth) for hit in page.items] == [(n3, 2)] @pytest.mark.asyncio async def test_traverse_cursor_roundtrip(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - n3 = await async_db.upsert_node(1, "c") - n4 = await async_db.upsert_node(1, "d") - await async_db.upsert_edge(n1, n2, 10) - await async_db.upsert_edge(n2, n3, 10) - await async_db.upsert_edge(n2, n4, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + n3 = await async_db.upsert_node("Person", "c") + n4 = await async_db.upsert_node("Person", "d") + await async_db.upsert_edge(n1, n2, "RELATES_TO") + await async_db.upsert_edge(n2, n3, "RELATES_TO") + await async_db.upsert_edge(n2, n4, "RELATES_TO") p1 = await async_db.traverse(n1, 2, min_depth=2, direction="outgoing", limit=1) assert len(p1.items) == 1 assert p1.next_cursor is not None @@ -521,9 +622,9 @@ async def test_traverse_cursor_roundtrip(self, async_db): class TestAsyncMaintenance2: @pytest.mark.asyncio async def test_compact_with_progress(self, async_db): - await async_db.upsert_node(1, "a") + await async_db.upsert_node("Person", "a") await async_db.flush() - await async_db.upsert_node(1, "b") + await async_db.upsert_node("Person", "b") await async_db.flush() events = [] result = await async_db.compact_with_progress(lambda p: (events.append(p) or True)) @@ -535,11 +636,14 @@ class TestAsyncBatch2: @pytest.mark.asyncio async def test_batch_upsert_nodes_binary(self, async_db): import struct - import json - buf = struct.pack(" 0 assert n1 in result.node_ids @pytest.mark.asyncio async def test_personalized_pagerank_approx(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(n1, n2, "RELATES_TO") result = await async_db.personalized_pagerank( [n1], algorithm="approx", @@ -584,9 +689,9 @@ async def test_personalized_pagerank_approx(self, async_db): @pytest.mark.asyncio async def test_export_adjacency(self, async_db): - n1 = await async_db.upsert_node(1, "a") - n2 = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(n1, n2, 10) + n1 = await async_db.upsert_node("Person", "a") + n2 = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(n1, n2, "RELATES_TO") export = await async_db.export_adjacency() assert len(export.node_ids) == 2 assert len(export.edges) == 1 @@ -595,25 +700,25 @@ async def test_export_adjacency(self, async_db): class TestAsyncDegree: @pytest.mark.asyncio async def test_degree(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(a, b, 10, weight=5.0) + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(a, b, "RELATES_TO", weight=5.0) assert await async_db.degree(a) == 1 assert await async_db.degree(b) == 0 @pytest.mark.asyncio async def test_sum_edge_weights(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(a, b, 10, weight=5.0) + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(a, b, "RELATES_TO", weight=5.0) s = await async_db.sum_edge_weights(a) assert abs(s - 5.0) < 1e-6 @pytest.mark.asyncio async def test_avg_edge_weight(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(a, b, 10, weight=5.0) + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(a, b, "RELATES_TO", weight=5.0) avg = await async_db.avg_edge_weight(a) assert avg is not None assert abs(avg - 5.0) < 1e-6 @@ -621,9 +726,9 @@ async def test_avg_edge_weight(self, async_db): @pytest.mark.asyncio async def test_degrees_batch(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(a, b, 10) + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(a, b, "RELATES_TO") result = await async_db.degrees([a, b]) assert isinstance(result, dict) assert result[a] == 1 diff --git a/overgraph-python/tests/test_batch.py b/overgraph-python/tests/test_batch.py index d8f9b74..15638ac 100644 --- a/overgraph-python/tests/test_batch.py +++ b/overgraph-python/tests/test_batch.py @@ -7,24 +7,24 @@ class TestBatchUpsertNodes: def test_batch_upsert(self, db): ids = db.batch_upsert_nodes([ - {"type_id": 1, "key": "a"}, - {"type_id": 1, "key": "b"}, - {"type_id": 2, "key": "c"}, + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, + {"labels": ["Company"], "key": "c"}, ]) assert len(ids) == 3 assert len(set(ids)) == 3 # All unique def test_batch_upsert_with_props(self, db): ids = db.batch_upsert_nodes([ - {"type_id": 1, "key": "a", "props": {"name": "Alice"}}, - {"type_id": 1, "key": "b", "props": {"name": "Bob"}}, + {"labels": ["Person"], "key": "a", "props": {"name": "Alice"}}, + {"labels": ["Person"], "key": "b", "props": {"name": "Bob"}}, ]) assert db.get_node(ids[0]).props["name"] == "Alice" assert db.get_node(ids[1]).props["name"] == "Bob" def test_batch_upsert_with_weight(self, db): ids = db.batch_upsert_nodes([ - {"type_id": 1, "key": "a", "weight": 2.0}, + {"labels": ["Person"], "key": "a", "weight": 2.0}, ]) assert abs(db.get_node(ids[0]).weight - 2.0) < 0.01 @@ -34,45 +34,44 @@ def test_batch_upsert_empty(self, db): def test_batch_upsert_idempotent(self, db): ids1 = db.batch_upsert_nodes([ - {"type_id": 1, "key": "a"}, - {"type_id": 1, "key": "b"}, + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, ]) ids2 = db.batch_upsert_nodes([ - {"type_id": 1, "key": "a"}, - {"type_id": 1, "key": "b"}, + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, ]) assert ids1 == ids2 def test_batch_upsert_missing_field(self, db): with pytest.raises(Exception): - db.batch_upsert_nodes([{"type_id": 1}]) # missing 'key' - + db.batch_upsert_nodes([{"labels": ["Person"]}]) # missing 'key' class TestBatchUpsertEdges: def test_batch_upsert(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") ids = db.batch_upsert_edges([ - {"from_id": n1, "to_id": n2, "type_id": 10}, - {"from_id": n2, "to_id": n3, "type_id": 10}, + {"from_id": n1, "to_id": n2, "label": "RELATES_TO"}, + {"from_id": n2, "to_id": n3, "label": "RELATES_TO"}, ]) assert len(ids) == 2 assert len(set(ids)) == 2 def test_batch_upsert_with_props(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") ids = db.batch_upsert_edges([ - {"from_id": n1, "to_id": n2, "type_id": 10, "props": {"rel": "friend"}}, + {"from_id": n1, "to_id": n2, "label": "RELATES_TO", "props": {"rel": "friend"}}, ]) assert db.get_edge(ids[0]).props["rel"] == "friend" def test_batch_upsert_with_temporal(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") ids = db.batch_upsert_edges([ - {"from_id": n1, "to_id": n2, "type_id": 10, "valid_from": 100, "valid_to": 200}, + {"from_id": n1, "to_id": n2, "label": "RELATES_TO", "valid_from": 100, "valid_to": 200}, ]) edge = db.get_edge(ids[0]) assert edge.valid_from == 100 @@ -82,18 +81,17 @@ def test_batch_upsert_empty(self, db): ids = db.batch_upsert_edges([]) assert ids == [] - class TestGetNodes: def test_get_multiple(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") results = db.get_nodes([n1, n2]) assert len(results) == 2 assert results[0].id == n1 assert results[1].id == n2 def test_get_with_missing(self, db): - n1 = db.upsert_node(1, "a") + n1 = db.upsert_node("Person", "a") results = db.get_nodes([n1, 999999]) assert len(results) == 2 assert results[0] is not None @@ -106,20 +104,28 @@ def test_get_empty(self, db): class TestGetNodesByKeys: def test_basic(self, db): - db.upsert_node(1, "alice") - db.upsert_node(1, "bob") - db.upsert_node(2, "charlie") - results = db.get_nodes_by_keys([(1, "alice"), (1, "bob"), (2, "charlie")]) + db.upsert_node("Person", "alice") + db.upsert_node("Person", "bob") + db.upsert_node("Company", "charlie") + results = db.get_nodes_by_keys([ + {"labels": ["Person"], "key": "alice"}, + {"labels": ["Person"], "key": "bob"}, + {"labels": ["Company"], "key": "charlie"}, + ]) assert len(results) == 3 assert results[0].key == "alice" assert results[1].key == "bob" assert results[2].key == "charlie" def test_mixed_found_missing(self, db): - db.upsert_node(1, "a") - bid = db.upsert_node(1, "b") + db.upsert_node("Person", "a") + bid = db.upsert_node("Person", "b") db.delete_node(bid) - results = db.get_nodes_by_keys([(1, "a"), (1, "b"), (1, "nonexistent")]) + results = db.get_nodes_by_keys([ + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, + {"labels": ["Person"], "key": "nonexistent"}, + ]) assert len(results) == 3 assert results[0] is not None assert results[1] is None @@ -129,23 +135,22 @@ def test_empty(self, db): results = db.get_nodes_by_keys([]) assert results == [] - class TestGetEdges: def test_get_multiple(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - e1 = db.upsert_edge(n1, n2, 10) - e2 = db.upsert_edge(n2, n3, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + e1 = db.upsert_edge(n1, n2, "RELATES_TO") + e2 = db.upsert_edge(n2, n3, "RELATES_TO") results = db.get_edges([e1, e2]) assert len(results) == 2 assert results[0].id == e1 assert results[1].id == e2 def test_get_with_missing(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - e1 = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + e1 = db.upsert_edge(n1, n2, "RELATES_TO") results = db.get_edges([e1, 999999]) assert results[0] is not None assert results[1] is None @@ -159,19 +164,19 @@ class TestGraphPatch: def test_upsert_nodes_only(self, db): result = db.graph_patch({ "upsert_nodes": [ - {"type_id": 1, "key": "a"}, - {"type_id": 1, "key": "b"}, + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, ], }) assert len(result.node_ids) == 2 assert len(result.edge_ids) == 0 def test_upsert_edges_only(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") result = db.graph_patch({ "upsert_edges": [ - {"from_id": n1, "to_id": n2, "type_id": 10}, + {"from_id": n1, "to_id": n2, "label": "RELATES_TO"}, ], }) assert len(result.node_ids) == 0 @@ -180,37 +185,37 @@ def test_upsert_edges_only(self, db): def test_mixed_patch(self, db): result = db.graph_patch({ "upsert_nodes": [ - {"type_id": 1, "key": "x"}, - {"type_id": 1, "key": "y"}, + {"labels": ["Person"], "key": "x"}, + {"labels": ["Person"], "key": "y"}, ], }) n1, n2 = result.node_ids result2 = db.graph_patch({ "upsert_edges": [ - {"from_id": n1, "to_id": n2, "type_id": 10}, + {"from_id": n1, "to_id": n2, "label": "RELATES_TO"}, ], }) assert len(result2.edge_ids) == 1 assert db.get_edge(result2.edge_ids[0]) is not None def test_delete_nodes(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") db.graph_patch({"delete_node_ids": [n1]}) assert db.get_node(n1) is None assert db.get_node(n2) is not None def test_delete_edges(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") db.graph_patch({"delete_edge_ids": [eid]}) assert db.get_edge(eid) is None def test_invalidate_edges(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") db.graph_patch({ "invalidate_edges": [{"edge_id": eid, "valid_to": 5000}], }) @@ -224,7 +229,7 @@ def test_empty_patch(self, db): def test_patch_result_repr(self, db): result = db.graph_patch({ - "upsert_nodes": [{"type_id": 1, "key": "a"}], + "upsert_nodes": [{"labels": ["Person"], "key": "a"}], }) r = repr(result) assert "PatchResult" in r @@ -237,16 +242,23 @@ def test_patch_result_repr(self, db): def pack_node_batch(nodes): """Pack a list of node dicts into the binary wire format. - Format: [count:u32] per node: [type_id:u32][weight:f32][key_len:u16][key:utf8][props_len:u32][props:json] + Format: [magic:4][version:u16][count:u32] per node: + [label_count:u8] repeated [label_len:u16][label:utf8] + [weight:f32][key_len:u16][key:utf8][props_len:u32][props:json] """ import json - buf = struct.pack(" 0 def test_upsert_same_key_returns_same_id(self, db): - id1 = db.upsert_node(1, "alice") - id2 = db.upsert_node(1, "alice") + id1 = db.upsert_node("Person", "alice") + id2 = db.upsert_node("Person", "alice") assert id1 == id2 def test_upsert_different_keys(self, db): - id1 = db.upsert_node(1, "alice") - id2 = db.upsert_node(1, "bob") + id1 = db.upsert_node("Person", "alice") + id2 = db.upsert_node("Person", "bob") assert id1 != id2 def test_upsert_with_props(self, db): - nid = db.upsert_node(1, "alice", props={"age": 30, "name": "Alice"}) + nid = db.upsert_node("Person", "alice", props={"age": 30, "name": "Alice"}) node = db.get_node(nid) assert node.props["age"] == 30 assert node.props["name"] == "Alice" def test_upsert_with_weight(self, db): - nid = db.upsert_node(1, "alice", weight=2.5) + nid = db.upsert_node("Person", "alice", weight=2.5) node = db.get_node(nid) assert abs(node.weight - 2.5) < 0.01 def test_upsert_with_nested_props(self, db): - nid = db.upsert_node(1, "alice", props={ + nid = db.upsert_node("Person", "alice", props={ "tags": ["a", "b"], "meta": {"nested": True, "count": 42}, }) @@ -40,19 +79,56 @@ def test_upsert_with_nested_props(self, db): assert node.props["meta"]["count"] == 42 def test_upsert_updates_props(self, db): - nid = db.upsert_node(1, "alice", props={"v": 1}) - db.upsert_node(1, "alice", props={"v": 2}) + nid = db.upsert_node("Person", "alice", props={"v": 1}) + db.upsert_node("Person", "alice", props={"v": 2}) node = db.get_node(nid) assert node.props["v"] == 2 + def test_upsert_accepts_label_string_or_sequence(self, db): + single = db.upsert_node("Person", "alice") + multi = db.upsert_node(["Person", "Admin"], "root") + + assert db.get_node(single).labels == ["Person"] + assert db.get_node(multi).labels == ["Person", "Admin"] + assert db.count_nodes_by_labels("Person") == 2 + assert db.nodes_by_labels(["Person", "Admin"]).to_list() == [multi] + + def test_add_remove_node_label_returns_changed(self, db): + nid = db.upsert_node("Person", "alice") + + assert db.add_node_label(nid, "Admin") is True + assert db.add_node_label(nid, "Admin") is False + assert db.get_node(nid).labels == ["Person", "Admin"] + assert db.remove_node_label(nid, "Admin") is True + assert db.remove_node_label(nid, "Admin") is False + assert db.get_node(nid).labels == ["Person"] + + def test_remove_last_node_label_rejected(self, db): + nid = db.upsert_node("Person", "alice") + + with pytest.raises(OverGraphError, match="last node label"): + db.remove_node_label(nid, "Person") + + assert db.get_node(nid).labels == ["Person"] + + def test_add_node_label_conflict_rejected(self, db): + alice = db.upsert_node("Person", "shared") + other = db.upsert_node("Admin", "shared") + + with pytest.raises(OverGraphError, match="node key conflict"): + db.add_node_label(alice, "Admin") + + assert db.get_node(alice).labels == ["Person"] + assert db.get_node(other).labels == ["Admin"] + class TestGetNode: def test_get_existing(self, db): - nid = db.upsert_node(1, "alice") + nid = db.upsert_node("Person", "alice") node = db.get_node(nid) assert node is not None assert node.id == nid - assert node.type_id == 1 + assert node.labels == ["Person"] assert node.key == "alice" assert node.created_at > 0 assert node.updated_at > 0 @@ -62,33 +138,33 @@ def test_get_nonexistent(self, db): assert node is None def test_node_repr(self, db): - nid = db.upsert_node(5, "bob") + nid = db.upsert_node("Person", "bob") node = db.get_node(nid) r = repr(node) - assert "NodeRecord" in r + assert "NodeView" in r assert "bob" in r class TestGetNodeByKey: def test_get_by_key(self, db): - nid = db.upsert_node(1, "alice") - node = db.get_node_by_key(1, "alice") + nid = db.upsert_node("Person", "alice") + node = db.get_node_by_key("Person", "alice") assert node is not None assert node.id == nid def test_get_by_key_wrong_type(self, db): - db.upsert_node(1, "alice") - node = db.get_node_by_key(2, "alice") + db.upsert_node("Person", "alice") + node = db.get_node_by_key("Company", "alice") assert node is None def test_get_by_key_nonexistent(self, db): - node = db.get_node_by_key(1, "nobody") + node = db.get_node_by_key("Person", "nobody") assert node is None class TestDeleteNode: def test_delete_node(self, db): - nid = db.upsert_node(1, "alice") + nid = db.upsert_node("Person", "alice") db.delete_node(nid) assert db.get_node(nid) is None @@ -99,30 +175,30 @@ def test_delete_nonexistent(self, db): class TestUpsertEdge: def test_upsert_returns_id(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") assert isinstance(eid, int) assert eid > 0 def test_upsert_with_props(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10, props={"rel": "friend"}) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO", props={"rel": "friend"}) edge = db.get_edge(eid) assert edge.props["rel"] == "friend" def test_upsert_with_weight(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10, weight=3.14) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO", weight=3.14) edge = db.get_edge(eid) assert abs(edge.weight - 3.14) < 0.01 def test_upsert_with_temporal(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10, valid_from=1000, valid_to=2000) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO", valid_from=1000, valid_to=2000) edge = db.get_edge(eid) assert edge.valid_from == 1000 assert edge.valid_to == 2000 @@ -130,58 +206,58 @@ def test_upsert_with_temporal(self, db): class TestGetEdge: def test_get_existing(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") edge = db.get_edge(eid) assert edge is not None assert edge.id == eid assert edge.from_id == n1 assert edge.to_id == n2 - assert edge.type_id == 10 + assert edge.label == "RELATES_TO" def test_get_nonexistent(self, db): assert db.get_edge(999999) is None def test_edge_repr(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") edge = db.get_edge(eid) r = repr(edge) - assert "EdgeRecord" in r + assert "EdgeView" in r class TestGetEdgeByTriple: def test_get_by_triple(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) - edge = db.get_edge_by_triple(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") + edge = db.get_edge_by_triple(n1, n2, "RELATES_TO") assert edge is not None assert edge.id == eid def test_get_by_triple_wrong_type(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - db.upsert_edge(n1, n2, 10) - assert db.get_edge_by_triple(n1, n2, 99) is None + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + db.upsert_edge(n1, n2, "RELATES_TO") + assert db.get_edge_by_triple(n1, n2, "DOES_NOT_EXIST") is None class TestDeleteEdge: def test_delete_edge(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") db.delete_edge(eid) assert db.get_edge(eid) is None class TestInvalidateEdge: def test_invalidate(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") result = db.invalidate_edge(eid, 5000) assert result is not None assert result.valid_to == 5000 @@ -195,48 +271,48 @@ class TestPropertyTypes: """Test all supported property types roundtrip through Python -> Rust -> Python.""" def test_string(self, db): - nid = db.upsert_node(1, "k", props={"s": "hello"}) + nid = db.upsert_node("Person", "k", props={"s": "hello"}) assert db.get_node(nid).props["s"] == "hello" def test_int(self, db): - nid = db.upsert_node(1, "k", props={"i": 42}) + nid = db.upsert_node("Person", "k", props={"i": 42}) assert db.get_node(nid).props["i"] == 42 def test_float(self, db): - nid = db.upsert_node(1, "k", props={"f": 3.14}) + nid = db.upsert_node("Person", "k", props={"f": 3.14}) assert abs(db.get_node(nid).props["f"] - 3.14) < 0.001 def test_bool(self, db): - nid = db.upsert_node(1, "k", props={"b": True, "c": False}) + nid = db.upsert_node("Person", "k", props={"b": True, "c": False}) p = db.get_node(nid).props assert p["b"] is True assert p["c"] is False def test_null(self, db): - nid = db.upsert_node(1, "k", props={"n": None}) + nid = db.upsert_node("Person", "k", props={"n": None}) assert db.get_node(nid).props["n"] is None def test_bytes(self, db): - nid = db.upsert_node(1, "k", props={"data": b"\x00\x01\x02"}) + nid = db.upsert_node("Person", "k", props={"data": b"\x00\x01\x02"}) assert db.get_node(nid).props["data"] == b"\x00\x01\x02" def test_array(self, db): - nid = db.upsert_node(1, "k", props={"arr": [1, "two", 3.0]}) + nid = db.upsert_node("Person", "k", props={"arr": [1, "two", 3.0]}) p = db.get_node(nid).props assert p["arr"][0] == 1 assert p["arr"][1] == "two" assert abs(p["arr"][2] - 3.0) < 0.001 def test_nested_map(self, db): - nid = db.upsert_node(1, "k", props={"m": {"a": 1, "b": {"c": 2}}}) + nid = db.upsert_node("Person", "k", props={"m": {"a": 1, "b": {"c": 2}}}) p = db.get_node(nid).props assert p["m"]["a"] == 1 assert p["m"]["b"]["c"] == 2 def test_empty_props(self, db): - nid = db.upsert_node(1, "k", props={}) + nid = db.upsert_node("Person", "k", props={}) assert db.get_node(nid).props == {} def test_no_props(self, db): - nid = db.upsert_node(1, "k") + nid = db.upsert_node("Person", "k") assert db.get_node(nid).props == {} diff --git a/overgraph-python/tests/test_degree.py b/overgraph-python/tests/test_degree.py index ac2ce27..3f78c7a 100644 --- a/overgraph-python/tests/test_degree.py +++ b/overgraph-python/tests/test_degree.py @@ -6,53 +6,53 @@ class TestDegree: def test_outgoing(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=2.0) - db.upsert_edge(a, c, 20, weight=3.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=2.0) + db.upsert_edge(a, c, "WORKS_AT", weight=3.0) assert db.degree(a) == 2 assert db.degree(b) == 0 def test_incoming(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10) - db.upsert_edge(a, c, 20) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(a, c, "WORKS_AT") assert db.degree(a, direction="incoming") == 0 assert db.degree(b, direction="incoming") == 1 def test_both(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10) - db.upsert_edge(a, c, 20) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(a, c, "WORKS_AT") assert db.degree(a, direction="both") == 2 assert db.degree(b, direction="both") == 1 - def test_type_filter(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10) - db.upsert_edge(a, c, 20) - assert db.degree(a, direction="outgoing", type_filter=[10]) == 1 - assert db.degree(a, direction="outgoing", type_filter=[20]) == 1 - assert db.degree(a, direction="outgoing", type_filter=[10, 20]) == 2 - assert db.degree(a, direction="outgoing", type_filter=[99]) == 0 + def test_edge_label_filter(self, db): + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(a, c, "WORKS_AT") + assert db.degree(a, direction="outgoing", edge_label_filter=["RELATES_TO"]) == 1 + assert db.degree(a, direction="outgoing", edge_label_filter=["WORKS_AT"]) == 1 + assert db.degree(a, direction="outgoing", edge_label_filter=["RELATES_TO", "WORKS_AT"]) == 2 + assert db.degree(a, direction="outgoing", edge_label_filter=["DOES_NOT_EXIST"]) == 0 def test_nonexistent_node(self, db): assert db.degree(999999) == 0 def test_matches_neighbors_length(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=2.0) - db.upsert_edge(a, c, 20, weight=3.0) - db.upsert_edge(b, c, 10, weight=1.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=2.0) + db.upsert_edge(a, c, "WORKS_AT", weight=3.0) + db.upsert_edge(b, c, "RELATES_TO", weight=1.0) for direction in ["outgoing", "incoming", "both"]: for nid in [a, b, c]: deg = db.degree(nid, direction=direction) @@ -64,40 +64,40 @@ def test_matches_neighbors_length(self, db): class TestSumEdgeWeights: def test_sum(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=2.0) - db.upsert_edge(a, c, 10, weight=3.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=2.0) + db.upsert_edge(a, c, "RELATES_TO", weight=3.0) assert abs(db.sum_edge_weights(a) - 5.0) < 1e-6 def test_zero_for_no_edges(self, db): assert db.sum_edge_weights(999999) == 0.0 - def test_type_filter(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=2.0) - db.upsert_edge(a, c, 20, weight=5.0) - assert abs(db.sum_edge_weights(a, type_filter=[10]) - 2.0) < 1e-6 - assert abs(db.sum_edge_weights(a, type_filter=[20]) - 5.0) < 1e-6 + def test_edge_label_filter(self, db): + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=2.0) + db.upsert_edge(a, c, "WORKS_AT", weight=5.0) + assert abs(db.sum_edge_weights(a, edge_label_filter=["RELATES_TO"]) - 2.0) < 1e-6 + assert abs(db.sum_edge_weights(a, edge_label_filter=["WORKS_AT"]) - 5.0) < 1e-6 def test_at_epoch(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10, weight=3.0, valid_from=100, valid_to=200) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO", weight=3.0, valid_from=100, valid_to=200) assert abs(db.sum_edge_weights(a, at_epoch=150) - 3.0) < 1e-6 assert db.sum_edge_weights(a, at_epoch=250) == 0.0 class TestAvgEdgeWeight: def test_avg(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=2.0) - db.upsert_edge(a, c, 10, weight=4.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=2.0) + db.upsert_edge(a, c, "RELATES_TO", weight=4.0) avg = db.avg_edge_weight(a) assert avg is not None assert abs(avg - 3.0) < 1e-6 @@ -105,20 +105,20 @@ def test_avg(self, db): def test_none_for_no_edges(self, db): assert db.avg_edge_weight(999999) is None - def test_type_filter(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=2.0) - db.upsert_edge(a, c, 20, weight=6.0) - avg = db.avg_edge_weight(a, type_filter=[10]) + def test_edge_label_filter(self, db): + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=2.0) + db.upsert_edge(a, c, "WORKS_AT", weight=6.0) + avg = db.avg_edge_weight(a, edge_label_filter=["RELATES_TO"]) assert avg is not None assert abs(avg - 2.0) < 1e-6 def test_at_epoch(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10, weight=4.0, valid_from=100, valid_to=200) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO", weight=4.0, valid_from=100, valid_to=200) avg = db.avg_edge_weight(a, at_epoch=150) assert avg is not None assert abs(avg - 4.0) < 1e-6 @@ -127,12 +127,12 @@ def test_at_epoch(self, db): class TestDegreesBatch: def test_batch(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10) - db.upsert_edge(a, c, 10) - db.upsert_edge(b, c, 10) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(a, c, "RELATES_TO") + db.upsert_edge(b, c, "RELATES_TO") result = db.degrees([a, b, c]) assert isinstance(result, dict) assert result[a] == 2 @@ -144,19 +144,19 @@ def test_empty_input(self, db): assert isinstance(result, dict) assert len(result) == 0 - def test_type_filter(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10) - db.upsert_edge(a, c, 20) - result = db.degrees([a], type_filter=[10]) + def test_edge_label_filter(self, db): + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(a, c, "WORKS_AT") + result = db.degrees([a], edge_label_filter=["RELATES_TO"]) assert result[a] == 1 def test_at_epoch(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10, valid_from=100, valid_to=200) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO", valid_from=100, valid_to=200) result = db.degrees([a], at_epoch=150) assert result[a] == 1 result2 = db.degrees([a], at_epoch=250) @@ -166,12 +166,12 @@ def test_at_epoch(self, db): class TestDegreeSidecars: def test_flush_compact_reopen_preserves_degree_family_results(self, db_path): db = OverGraph.open(db_path) - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=2.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=2.0) db.flush() - db.upsert_edge(a, c, 10, weight=4.0) + db.upsert_edge(a, c, "RELATES_TO", weight=4.0) db.flush() db.compact() @@ -190,9 +190,9 @@ def test_flush_compact_reopen_preserves_degree_family_results(self, db_path): def test_corrupt_degree_sidecar_falls_back(self, db_path): db = OverGraph.open(db_path) - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10, weight=5.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO", weight=5.0) db.flush() db.close() @@ -217,23 +217,23 @@ def test_corrupt_degree_sidecar_falls_back(self, db_path): class TestDegreeTemporal: def test_ignores_expired_edge(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") now = int(time.time() * 1000) - db.upsert_edge(a, b, 10, valid_from=now - 2000, valid_to=now - 1000) + db.upsert_edge(a, b, "RELATES_TO", valid_from=now - 2000, valid_to=now - 1000) assert db.degree(a) == 0 def test_ignores_future_edge(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") now = int(time.time() * 1000) - db.upsert_edge(a, b, 10, valid_from=now + 10000) + db.upsert_edge(a, b, "RELATES_TO", valid_from=now + 10000) assert db.degree(a) == 0 def test_at_epoch(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10, valid_from=100, valid_to=200) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO", valid_from=100, valid_to=200) assert db.degree(a, at_epoch=150) == 1 assert db.degree(a, at_epoch=250) == 0 assert db.degree(a, at_epoch=50) == 0 diff --git a/overgraph-python/tests/test_edge_cases.py b/overgraph-python/tests/test_edge_cases.py index 4dea92c..ec10900 100644 --- a/overgraph-python/tests/test_edge_cases.py +++ b/overgraph-python/tests/test_edge_cases.py @@ -25,12 +25,12 @@ class TestTemporalFiltering: def test_neighbors_at_epoch_filters_by_validity(self, db): """Edges with non-overlapping validity windows are filtered by epoch.""" - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") # A->B valid [1000, 5000), A->C valid [3000, 9000) - db.upsert_edge(a, b, 10, valid_from=1000, valid_to=5000) - db.upsert_edge(a, c, 10, valid_from=3000, valid_to=9000) + db.upsert_edge(a, b, "RELATES_TO", valid_from=1000, valid_to=5000) + db.upsert_edge(a, c, "RELATES_TO", valid_from=3000, valid_to=9000) # epoch 2000: only A->B is valid nbrs = db.neighbors(a, direction="outgoing", at_epoch=2000) @@ -53,13 +53,13 @@ def test_neighbors_at_epoch_filters_by_validity(self, db): def test_neighbors_at_epoch_without_filter_uses_current_time(self, db): """Without at_epoch, defaults to current time -- expired edges excluded.""" - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") now = int(time.time() * 1000) # One edge valid now, one expired - db.upsert_edge(a, b, 10, valid_from=0, valid_to=now + 60000) # valid - db.upsert_edge(a, c, 10, valid_from=1000, valid_to=5000) # expired + db.upsert_edge(a, b, "RELATES_TO", valid_from=0, valid_to=now + 60000) # valid + db.upsert_edge(a, c, "RELATES_TO", valid_from=1000, valid_to=5000) # expired nbrs = db.neighbors(a, direction="outgoing") ids = {n.node_id for n in nbrs} @@ -68,12 +68,12 @@ def test_neighbors_at_epoch_without_filter_uses_current_time(self, db): def test_traverse_two_hop_at_epoch(self, db): """Traversal respects at_epoch across both hops.""" - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") # A->B valid [1000, 5000), B->C valid [2000, 6000) - db.upsert_edge(a, b, 10, valid_from=1000, valid_to=5000) - db.upsert_edge(b, c, 10, valid_from=2000, valid_to=6000) + db.upsert_edge(a, b, "RELATES_TO", valid_from=1000, valid_to=5000) + db.upsert_edge(b, c, "RELATES_TO", valid_from=2000, valid_to=6000) # epoch 3000: both hops valid, C reachable page = db.traverse(a, 2, min_depth=2, direction="outgoing", at_epoch=3000) @@ -93,11 +93,11 @@ def test_traverse_two_hop_at_epoch(self, db): def test_neighbors_batch_at_epoch(self, db): """Batch neighbor queries respect at_epoch.""" - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, valid_from=1000, valid_to=5000) - db.upsert_edge(a, c, 10, valid_from=3000, valid_to=9000) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", valid_from=1000, valid_to=5000) + db.upsert_edge(a, c, "RELATES_TO", valid_from=3000, valid_to=9000) # epoch 2000: only A->B valid result = db.neighbors_batch([a], at_epoch=2000) @@ -114,13 +114,13 @@ def test_neighbors_batch_at_epoch(self, db): def test_top_k_at_epoch(self, db): """top_k_neighbors with at_epoch filters correctly.""" - center = db.upsert_node(1, "center") - s1 = db.upsert_node(1, "s1") - s2 = db.upsert_node(1, "s2") - s3 = db.upsert_node(1, "s3") - db.upsert_edge(center, s1, 10, weight=3.0, valid_from=1000, valid_to=5000) - db.upsert_edge(center, s2, 10, weight=2.0, valid_from=3000, valid_to=9000) - db.upsert_edge(center, s3, 10, weight=1.0, valid_from=3000, valid_to=9000) + center = db.upsert_node("Person", "center") + s1 = db.upsert_node("Person", "s1") + s2 = db.upsert_node("Person", "s2") + s3 = db.upsert_node("Person", "s3") + db.upsert_edge(center, s1, "RELATES_TO", weight=3.0, valid_from=1000, valid_to=5000) + db.upsert_edge(center, s2, "RELATES_TO", weight=2.0, valid_from=3000, valid_to=9000) + db.upsert_edge(center, s3, "RELATES_TO", weight=1.0, valid_from=3000, valid_to=9000) # epoch 2000: only s1 valid top = db.top_k_neighbors(center, 3, direction="outgoing", scoring="weight", at_epoch=2000) @@ -140,11 +140,11 @@ def test_top_k_at_epoch(self, db): def test_extract_subgraph_at_epoch(self, db): """Subgraph extraction respects at_epoch.""" - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, valid_from=1000, valid_to=5000) - db.upsert_edge(b, c, 10, valid_from=2000, valid_to=6000) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", valid_from=1000, valid_to=5000) + db.upsert_edge(b, c, "RELATES_TO", valid_from=2000, valid_to=6000) # epoch 3000: both edges valid sg = db.extract_subgraph(a, 2, at_epoch=3000) @@ -213,27 +213,27 @@ class TestPaginationEdgeCases: def test_cursor_past_all_ids_returns_empty(self, db): """A cursor beyond all existing IDs should return empty results.""" for i in range(5): - db.upsert_node(1, f"n{i}") - page = db.nodes_by_type_paged(1, limit=50, after=999999999) + db.upsert_node("Person", f"n{i}") + page = db.nodes_by_labels_paged("Person", limit=50, after=999999999) assert len(page.items) == 0 assert page.next_cursor is None def test_cursor_zero_returns_first_page(self, db): """after=0 should behave like the first page (all IDs are > 0).""" for i in range(5): - db.upsert_node(1, f"n{i}") - page_no_cursor = db.nodes_by_type_paged(1, limit=50) - page_zero = db.nodes_by_type_paged(1, limit=50, after=0) + db.upsert_node("Person", f"n{i}") + page_no_cursor = db.nodes_by_labels_paged("Person", limit=50) + page_zero = db.nodes_by_labels_paged("Person", limit=50, after=0) assert page_no_cursor.items == page_zero.items def test_find_nodes_paged_cursor_traversal(self, db): """Exhaustive cursor traversal via find_nodes_paged.""" for i in range(7): - db.upsert_node(1, f"n{i}", props={"tag": "x"}) + db.upsert_node("Person", f"n{i}", props={"tag": "x"}) all_ids = [] cursor = None while True: - page = db.find_nodes_paged(1, "tag", "x", limit=3, after=cursor) + page = db.find_nodes_paged("Person", "tag", "x", limit=3, after=cursor) all_ids.extend(page.items) if page.next_cursor is None: break @@ -245,7 +245,7 @@ def test_find_nodes_paged_cursor_traversal(self, db): def test_edges_paged_cursor_past_all_returns_empty(self, db): """Edge pagination with cursor past all IDs returns empty.""" nodes, edges = make_chain(db, 4) - page = db.edges_by_type_paged(10, limit=50, after=999999999) + page = db.edges_by_label_paged("RELATES_TO", limit=50, after=999999999) assert len(page.items) == 0 @@ -257,24 +257,24 @@ def test_edges_paged_cursor_past_all_returns_empty(self, db): class TestSelfLoops: def test_self_loop_in_neighbors_outgoing(self, db): """A self-loop should appear in outgoing neighbors.""" - a = db.upsert_node(1, "a") - db.upsert_edge(a, a, 10) + a = db.upsert_node("Person", "a") + db.upsert_edge(a, a, "RELATES_TO") nbrs = db.neighbors(a, direction="outgoing") assert len(nbrs) == 1 assert nbrs[0].node_id == a def test_self_loop_in_neighbors_incoming(self, db): """A self-loop should appear in incoming neighbors.""" - a = db.upsert_node(1, "a") - db.upsert_edge(a, a, 10) + a = db.upsert_node("Person", "a") + db.upsert_edge(a, a, "RELATES_TO") nbrs = db.neighbors(a, direction="incoming") assert len(nbrs) == 1 assert nbrs[0].node_id == a def test_self_loop_both_direction_no_duplicate(self, db): """A self-loop queried with 'both' should not produce duplicates.""" - a = db.upsert_node(1, "a") - db.upsert_edge(a, a, 10) + a = db.upsert_node("Person", "a") + db.upsert_edge(a, a, "RELATES_TO") nbrs = db.neighbors(a, direction="both") # Self-loop should appear exactly once (deduplicated) assert len(nbrs) == 1 @@ -282,26 +282,26 @@ def test_self_loop_both_direction_no_duplicate(self, db): def test_self_loop_in_subgraph(self, db): """Subgraph extraction handles self-loops without infinite recursion.""" - a = db.upsert_node(1, "a") - db.upsert_edge(a, a, 10) + a = db.upsert_node("Person", "a") + db.upsert_edge(a, a, "RELATES_TO") sg = db.extract_subgraph(a, 2) assert len(sg.nodes) == 1 assert len(sg.edges) == 1 def test_self_loop_with_regular_edges(self, db): """Self-loop combined with regular edges should all appear.""" - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, a, 10) - db.upsert_edge(a, b, 10) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, a, "RELATES_TO") + db.upsert_edge(a, b, "RELATES_TO") nbrs = db.neighbors(a, direction="outgoing") ids = {n.node_id for n in nbrs} assert ids == {a, b} def test_self_loop_in_batch_neighbors(self, db): """Batch neighbor query includes self-loops.""" - a = db.upsert_node(1, "a") - db.upsert_edge(a, a, 10) + a = db.upsert_node("Person", "a") + db.upsert_edge(a, a, "RELATES_TO") result = db.neighbors_batch([a]) assert a in result ids = {n.node_id for n in result[a]} @@ -319,7 +319,7 @@ def test_callback_receives_valid_progress_fields(self, db): # Create data and flush multiple times to guarantee segments to compact for batch in range(3): for i in range(20): - db.upsert_node(1, f"b{batch}_n{i}") + db.upsert_node("Person", f"b{batch}_n{i}") db.flush() progress_events = [] @@ -345,7 +345,7 @@ def test_callback_cancellation_stops_compaction(self, db): """Returning False from callback cancels compaction without data loss.""" for batch in range(3): for i in range(20): - db.upsert_node(1, f"b{batch}_n{i}") + db.upsert_node("Person", f"b{batch}_n{i}") db.flush() def cancel_first_call(_p): @@ -359,14 +359,14 @@ def cancel_first_call(_p): # Data should still be intact after cancelled compaction # We inserted 60 unique nodes total (3 batches x 20) # Each batch re-upserts the same keys within its group, but keys are unique - page = db.nodes_by_type_paged(1, limit=100) + page = db.nodes_by_labels_paged("Person", limit=100) assert len(page.items) == 60 def test_callback_exception_propagates(self, db): """A RuntimeError raised in the callback should propagate.""" for batch in range(2): for i in range(20): - db.upsert_node(1, f"b{batch}_n{i}") + db.upsert_node("Person", f"b{batch}_n{i}") db.flush() def raise_error(_p): @@ -379,7 +379,7 @@ def test_callback_receives_monotonic_progress(self, db): """records_processed should be monotonically non-decreasing within a phase.""" for batch in range(3): for i in range(20): - db.upsert_node(1, f"b{batch}_n{i}") + db.upsert_node("Person", f"b{batch}_n{i}") db.flush() events_by_phase = {} @@ -416,7 +416,7 @@ async def test_concurrent_upserts(self, tmp_dir): db_path = os.path.join(tmp_dir, "stress_db") db = await AsyncOverGraph.open(db_path) try: - tasks = [db.upsert_node(1, f"node-{i}") for i in range(50)] + tasks = [db.upsert_node("Person", f"node-{i}") for i in range(50)] ids = await asyncio.gather(*tasks) assert len(ids) == 50 assert len(set(ids)) == 50 # all unique @@ -440,11 +440,11 @@ async def test_concurrent_mixed_operations(self, tmp_dir): # Pre-create some nodes pre_ids = [] for i in range(10): - nid = await db.upsert_node(1, f"pre-{i}") + nid = await db.upsert_node("Person", f"pre-{i}") pre_ids.append(nid) # Mix upserts and reads concurrently - upsert_tasks = [db.upsert_node(1, f"new-{i}") for i in range(20)] + upsert_tasks = [db.upsert_node("Person", f"new-{i}") for i in range(20)] read_tasks = [db.get_node(nid) for nid in pre_ids] all_results = await asyncio.gather(*upsert_tasks, *read_tasks) @@ -468,12 +468,12 @@ async def test_concurrent_edge_creation(self, tmp_dir): # Create nodes first nodes = [] for i in range(10): - nid = await db.upsert_node(1, f"n{i}") + nid = await db.upsert_node("Person", f"n{i}") nodes.append(nid) # Create edges concurrently (each from nodes[i] to nodes[i+1]) edge_tasks = [ - db.upsert_edge(nodes[i], nodes[(i + 1) % 10], 10) + db.upsert_edge(nodes[i], nodes[(i + 1) % 10], "RELATES_TO") for i in range(10) ] edge_ids = await asyncio.gather(*edge_tasks) @@ -492,7 +492,7 @@ class TestPruneByAge: def test_prune_max_age_removes_old_nodes(self, db): """Pruning with max_age_ms=1 should remove everything after a short sleep.""" for i in range(5): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") # Wait just enough so all nodes are older than 1ms time.sleep(0.05) result = db.prune(max_age_ms=1) @@ -501,38 +501,38 @@ def test_prune_max_age_removes_old_nodes(self, db): def test_prune_max_age_keeps_recent(self, db): """Pruning with a very large max_age_ms should keep everything.""" for i in range(5): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") result = db.prune(max_age_ms=999_999_999) assert result.nodes_pruned == 0 def test_prune_max_age_selective(self, db): """Only nodes older than threshold are pruned.""" - db.upsert_node(1, "old") + db.upsert_node("Person", "old") time.sleep(0.05) # 50ms - db.upsert_node(1, "new") + db.upsert_node("Person", "new") # Prune anything older than 10ms result = db.prune(max_age_ms=10) assert result.nodes_pruned >= 1 # "new" should survive - assert db.get_node_by_key(1, "new") is not None + assert db.get_node_by_key("Person", "new") is not None def test_prune_max_age_cascades_edges(self, db): """Pruning a node by age should also remove its edges.""" - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO") time.sleep(0.05) result = db.prune(max_age_ms=1) assert result.nodes_pruned >= 1 assert result.edges_pruned >= 1 assert db.get_edge(eid) is None - def test_prune_max_age_with_type_filter(self, db): - """max_age_ms combined with type_id only prunes matching type.""" - db.upsert_node(1, "t1") - db.upsert_node(2, "t2") + def test_prune_max_age_with_label_filter(self, db): + """max_age_ms combined with node label only prunes matching label.""" + db.upsert_node("Person", "t1") + db.upsert_node("Company", "t2") time.sleep(0.05) - result = db.prune(max_age_ms=1, type_id=1) + result = db.prune(max_age_ms=1, label="Person") assert result.nodes_pruned == 1 # Type 2 should survive - assert db.get_node_by_key(2, "t2") is not None + assert db.get_node_by_key("Company", "t2") is not None diff --git a/overgraph-python/tests/test_lifecycle.py b/overgraph-python/tests/test_lifecycle.py index 7c45d67..d6c5129 100644 --- a/overgraph-python/tests/test_lifecycle.py +++ b/overgraph-python/tests/test_lifecycle.py @@ -68,32 +68,32 @@ def test_operations_after_close_raise(self, db_path): db = OverGraph.open(db_path) db.close() with pytest.raises(OverGraphError): - db.upsert_node(1, "k") + db.upsert_node("Person", "k") class TestContextManager: def test_context_manager_basic(self, db_path): with OverGraph.open(db_path) as db: - nid = db.upsert_node(1, "hello") + nid = db.upsert_node("Person", "hello") assert nid > 0 def test_context_manager_closes_on_exit(self, db_path): db = OverGraph.open(db_path) with db: - db.upsert_node(1, "hello") + db.upsert_node("Person", "hello") with pytest.raises(OverGraphError): - db.upsert_node(1, "world") + db.upsert_node("Person", "world") def test_context_manager_closes_on_exception(self, db_path): db = OverGraph.open(db_path) try: with db: - db.upsert_node(1, "hello") + db.upsert_node("Person", "hello") raise ValueError("boom") except ValueError: pass with pytest.raises(OverGraphError): - db.upsert_node(1, "world") + db.upsert_node("Person", "world") class TestStats: @@ -110,7 +110,7 @@ def test_stats_repr(self, db): assert "DbStats" in r def test_stats_after_writes(self, db): - db.upsert_node(1, "a") - db.upsert_node(1, "b") + db.upsert_node("Person", "a") + db.upsert_node("Person", "b") s = db.stats() assert s.pending_wal_bytes > 0 diff --git a/overgraph-python/tests/test_maintenance.py b/overgraph-python/tests/test_maintenance.py index 354fb5e..493b270 100644 --- a/overgraph-python/tests/test_maintenance.py +++ b/overgraph-python/tests/test_maintenance.py @@ -3,7 +3,7 @@ class TestSync: def test_sync(self, db): - db.upsert_node(1, "a") + db.upsert_node("Person", "a") db.sync() # Should not raise @@ -14,7 +14,7 @@ def test_flush_empty(self, db): def test_flush_with_data(self, db): for i in range(10): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") result = db.flush() assert result is not None assert result.node_count == 10 @@ -30,10 +30,10 @@ def test_compact_no_segments(self, db): def test_compact_with_segments(self, db): # Create enough data to flush at least twice for i in range(50): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() for i in range(50, 100): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() result = db.compact() assert result is not None @@ -47,10 +47,10 @@ class TestCompactWithProgress: def test_progress_callback(self, db): # Create enough data to flush at least twice for i in range(50): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() for i in range(50, 100): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() progress_calls = [] @@ -70,10 +70,10 @@ def on_progress(p): def test_cancel_via_callback(self, db): for i in range(50): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() for i in range(50, 100): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() def cancel_immediately(_p): @@ -91,10 +91,10 @@ def test_callback_exception_reraises(self, db): import pytest for i in range(50): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() for i in range(50, 100): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") db.flush() def buggy_callback(_p): diff --git a/overgraph-python/tests/test_pagination.py b/overgraph-python/tests/test_pagination.py index cd75db4..61294ca 100644 --- a/overgraph-python/tests/test_pagination.py +++ b/overgraph-python/tests/test_pagination.py @@ -4,8 +4,8 @@ class TestNodesByTypePaged: def test_single_page(self, db): for i in range(5): - db.upsert_node(1, f"n{i}") - page = db.nodes_by_type_paged(1) + db.upsert_node("Person", f"n{i}") + page = db.nodes_by_labels_paged("Person") assert len(page.items) == 5 assert len(page) == 5 assert page # __bool__ truthy @@ -13,33 +13,33 @@ def test_single_page(self, db): assert "IdPageResult" in repr(page) def test_empty_page_is_falsy(self, db): - page = db.nodes_by_type_paged(99) + page = db.nodes_by_labels_paged("Missing") assert len(page) == 0 assert not page # __bool__ falsy def test_pagination(self, db): for i in range(10): - db.upsert_node(1, f"n{i}") - p1 = db.nodes_by_type_paged(1, limit=3) + db.upsert_node("Person", f"n{i}") + p1 = db.nodes_by_labels_paged("Person", limit=3) assert len(p1.items) == 3 assert p1.next_cursor is not None - p2 = db.nodes_by_type_paged(1, limit=3, after=p1.next_cursor) + p2 = db.nodes_by_labels_paged("Person", limit=3, after=p1.next_cursor) assert len(p2.items) == 3 # No overlap assert set(p1.items).isdisjoint(set(p2.items)) def test_empty(self, db): - page = db.nodes_by_type_paged(99) + page = db.nodes_by_labels_paged("Missing") assert len(page.items) == 0 assert page.next_cursor is None def test_exhaust_all_pages(self, db): for i in range(7): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") all_ids = [] cursor = None while True: - page = db.nodes_by_type_paged(1, limit=3, after=cursor) + page = db.nodes_by_labels_paged("Person", limit=3, after=cursor) all_ids.extend(page.items) if page.next_cursor is None: break @@ -50,13 +50,13 @@ def test_exhaust_all_pages(self, db): class TestEdgesByTypePaged: def test_single_page(self, db): nodes, edges = make_chain(db, 5) - page = db.edges_by_type_paged(10) + page = db.edges_by_label_paged("RELATES_TO") assert len(page.items) == 4 # 5 nodes, 4 edges assert page.next_cursor is None def test_pagination(self, db): nodes, edges = make_chain(db, 6) - p1 = db.edges_by_type_paged(10, limit=2) + p1 = db.edges_by_label_paged("RELATES_TO", limit=2) assert len(p1.items) == 2 assert p1.next_cursor is not None @@ -64,23 +64,23 @@ def test_pagination(self, db): class TestGetNodesByTypePaged: def test_returns_records(self, db): for i in range(5): - db.upsert_node(1, f"n{i}") - page = db.get_nodes_by_type_paged(1) + db.upsert_node("Person", f"n{i}") + page = db.get_nodes_by_labels_paged("Person") assert len(page.items) == 5 assert page.next_cursor is None assert "NodePageResult" in repr(page) # Items are full records for item in page.items: - assert item.type_id == 1 + assert item.labels == ["Person"] assert item.key.startswith("n") def test_pagination(self, db): for i in range(8): - db.upsert_node(1, f"n{i}") - p1 = db.get_nodes_by_type_paged(1, limit=3) + db.upsert_node("Person", f"n{i}") + p1 = db.get_nodes_by_labels_paged("Person", limit=3) assert len(p1.items) == 3 assert p1.next_cursor is not None - p2 = db.get_nodes_by_type_paged(1, limit=3, after=p1.next_cursor) + p2 = db.get_nodes_by_labels_paged("Person", limit=3, after=p1.next_cursor) assert len(p2.items) == 3 ids1 = {r.id for r in p1.items} ids2 = {r.id for r in p2.items} @@ -89,10 +89,10 @@ def test_pagination(self, db): class TestGetEdgesByTypePaged: def test_returns_records(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10, weight=2.5) - page = db.get_edges_by_type_paged(10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO", weight=2.5) + page = db.get_edges_by_label_paged("RELATES_TO") assert len(page.items) == 1 assert page.next_cursor is None assert "EdgePageResult" in repr(page) @@ -103,7 +103,7 @@ def test_returns_records(self, db): def test_pagination(self, db): nodes, edges = make_chain(db, 6) - p1 = db.get_edges_by_type_paged(10, limit=2) + p1 = db.get_edges_by_label_paged("RELATES_TO", limit=2) assert len(p1.items) == 2 assert p1.next_cursor is not None @@ -111,21 +111,21 @@ def test_pagination(self, db): class TestFindNodesPaged: def test_find_paged(self, db): for i in range(6): - db.upsert_node(1, f"n{i}", props={"color": "red"}) - page = db.find_nodes_paged(1, "color", "red") + db.upsert_node("Person", f"n{i}", props={"color": "red"}) + page = db.find_nodes_paged("Person", "color", "red") assert len(page.items) == 6 assert page.next_cursor is None def test_find_paged_with_limit(self, db): for i in range(6): - db.upsert_node(1, f"n{i}", props={"color": "red"}) - p1 = db.find_nodes_paged(1, "color", "red", limit=2) + db.upsert_node("Person", f"n{i}", props={"color": "red"}) + p1 = db.find_nodes_paged("Person", "color", "red", limit=2) assert len(p1.items) == 2 assert p1.next_cursor is not None def test_find_paged_no_match(self, db): - db.upsert_node(1, "a", props={"color": "blue"}) - page = db.find_nodes_paged(1, "color", "red") + db.upsert_node("Person", "a", props={"color": "blue"}) + page = db.find_nodes_paged("Person", "color", "red") assert len(page.items) == 0 @@ -134,18 +134,18 @@ def test_time_range_paged(self, db): import time start = int(time.time() * 1000) - 1000 for i in range(5): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") end = int(time.time() * 1000) + 1000 - page = db.find_nodes_by_time_range_paged(1, start, end) + page = db.find_nodes_by_time_range_paged("Person", start, end) assert len(page.items) == 5 def test_time_range_paged_with_limit(self, db): import time start = int(time.time() * 1000) - 1000 for i in range(5): - db.upsert_node(1, f"n{i}") + db.upsert_node("Person", f"n{i}") end = int(time.time() * 1000) + 1000 - p1 = db.find_nodes_by_time_range_paged(1, start, end, limit=2) + p1 = db.find_nodes_by_time_range_paged("Person", start, end, limit=2) assert len(p1.items) == 2 assert p1.next_cursor is not None @@ -169,13 +169,13 @@ def test_pagination(self, db): ids2 = {n.node_id for n in p2.items} assert ids1.isdisjoint(ids2) - def test_type_filter(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 20) - page = db.neighbors_paged(n1, direction="outgoing", type_filter=[10]) + def test_edge_label_filter(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "WORKS_AT") + page = db.neighbors_paged(n1, direction="outgoing", edge_label_filter=["RELATES_TO"]) assert len(page.items) == 1 assert page.items[0].node_id == n2 @@ -187,16 +187,16 @@ def test_depth_window(self, db): assert [(hit.node_id, hit.depth) for hit in page.items] == [(nodes[2], 2)] def test_pagination(self, db): - center = db.upsert_node(1, "center") + center = db.upsert_node("Person", "center") middles = [] for i in range(3): - m = db.upsert_node(1, f"mid_{i}") - db.upsert_edge(center, m, 10) + m = db.upsert_node("Person", f"mid_{i}") + db.upsert_edge(center, m, "RELATES_TO") middles.append(m) for m in middles: for j in range(3): - leaf = db.upsert_node(1, f"leaf_{m}_{j}") - db.upsert_edge(m, leaf, 10) + leaf = db.upsert_node("Person", f"leaf_{m}_{j}") + db.upsert_edge(m, leaf, "RELATES_TO") p1 = db.traverse(center, 2, min_depth=2, direction="outgoing", limit=4) assert len(p1.items) == 4 assert p1.next_cursor is not None @@ -214,17 +214,17 @@ def test_pagination(self, db): assert ids1.isdisjoint(ids2) def test_filtered_page(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(2, "b") - n3 = db.upsert_node(3, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n2, n3, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Company", "b") + n3 = db.upsert_node("Document", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n2, n3, "RELATES_TO") page = db.traverse( n1, 2, min_depth=2, direction="outgoing", - edge_type_filter=[10], - node_type_filter=[3], + edge_label_filter=["RELATES_TO"], + emit_node_label_filter={"labels": ["Document"], "mode": "all"}, ) assert [(hit.node_id, hit.depth) for hit in page.items] == [(n3, 2)] diff --git a/overgraph-python/tests/test_persistence.py b/overgraph-python/tests/test_persistence.py index d29985c..0fee62f 100644 --- a/overgraph-python/tests/test_persistence.py +++ b/overgraph-python/tests/test_persistence.py @@ -26,8 +26,8 @@ def open_db(db_dir, name="testdb"): class TestWalReplay: def test_nodes_survive_reopen(self, db_dir): db = open_db(db_dir) - n1 = db.upsert_node(1, "alice", props={"age": 30}) - n2 = db.upsert_node(2, "bob") + n1 = db.upsert_node("Person", "alice", props={"age": 30}) + n2 = db.upsert_node("Company", "bob") db.close() db2 = open_db(db_dir) @@ -40,9 +40,9 @@ def test_nodes_survive_reopen(self, db_dir): def test_edges_survive_reopen(self, db_dir): db = open_db(db_dir) - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10, props={"rel": "friend"}, weight=2.5) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO", props={"rel": "friend"}, weight=2.5) db.close() db2 = open_db(db_dir) @@ -50,16 +50,16 @@ def test_edges_survive_reopen(self, db_dir): assert edge is not None assert edge.from_id == n1 assert edge.to_id == n2 - assert edge.type_id == 10 + assert edge.label == "RELATES_TO" assert edge.props["rel"] == "friend" assert abs(edge.weight - 2.5) < 0.01 db2.close() def test_temporal_edges_survive_reopen(self, db_dir): db = open_db(db_dir) - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10, valid_from=100, valid_to=200) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO", valid_from=100, valid_to=200) db.close() db2 = open_db(db_dir) @@ -70,8 +70,8 @@ def test_temporal_edges_survive_reopen(self, db_dir): def test_deletes_persist(self, db_dir): db = open_db(db_dir) - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") db.delete_node(n1) db.close() @@ -82,11 +82,11 @@ def test_deletes_persist(self, db_dir): def test_neighbors_work_after_reopen(self, db_dir): db = open_db(db_dir) - n1 = db.upsert_node(1, "center") + n1 = db.upsert_node("Person", "center") spokes = [] for i in range(3): - s = db.upsert_node(1, f"spoke_{i}") - db.upsert_edge(n1, s, 10) + s = db.upsert_node("Person", f"spoke_{i}") + db.upsert_edge(n1, s, "RELATES_TO") spokes.append(s) db.close() @@ -99,22 +99,22 @@ def test_neighbors_work_after_reopen(self, db_dir): def test_find_nodes_works_after_reopen(self, db_dir): db = open_db(db_dir) - db.upsert_node(1, "x", props={"color": "red"}) - db.upsert_node(1, "y", props={"color": "blue"}) + db.upsert_node("Person", "x", props={"color": "red"}) + db.upsert_node("Person", "y", props={"color": "blue"}) db.close() db2 = open_db(db_dir) - ids = db2.find_nodes(1, "color", "red") + ids = db2.find_nodes("Person", "color", "red") assert len(ids) == 1 db2.close() def test_key_lookup_after_reopen(self, db_dir): db = open_db(db_dir) - nid = db.upsert_node(1, "mykey") + nid = db.upsert_node("Person", "mykey") db.close() db2 = open_db(db_dir) - node = db2.get_node_by_key(1, "mykey") + node = db2.get_node_by_key("Person", "mykey") assert node is not None assert node.id == nid db2.close() @@ -123,7 +123,7 @@ def test_key_lookup_after_reopen(self, db_dir): class TestFlushReopen: def test_flushed_data_survives(self, db_dir): db = open_db(db_dir) - n1 = db.upsert_node(1, "a", props={"v": 1}) + n1 = db.upsert_node("Person", "a", props={"v": 1}) db.flush() db.close() @@ -137,17 +137,17 @@ def test_compact_reopen(self, db_dir): db = open_db(db_dir) # Create two segments for i in range(10): - db.upsert_node(1, f"seg1_{i}") + db.upsert_node("Person", f"seg1_{i}") db.flush() for i in range(10): - db.upsert_node(1, f"seg2_{i}") + db.upsert_node("Person", f"seg2_{i}") db.flush() db.compact() db.close() db2 = open_db(db_dir) # All 20 nodes should be present - count = db2.count_nodes_by_type(1) + count = db2.count_nodes_by_labels("Person") assert count == 20 db2.close() @@ -155,21 +155,21 @@ def test_multi_cycle_flush_compact_reopen(self, db_dir): db = open_db(db_dir) # Cycle 1 for i in range(5): - db.upsert_node(1, f"c1_{i}") + db.upsert_node("Person", f"c1_{i}") db.flush() for i in range(5): - db.upsert_node(1, f"c2_{i}") + db.upsert_node("Person", f"c2_{i}") db.flush() db.compact() # Cycle 2: add more data after compaction for i in range(5): - db.upsert_node(1, f"c3_{i}") + db.upsert_node("Person", f"c3_{i}") db.flush() db.close() db2 = open_db(db_dir) - count = db2.count_nodes_by_type(1) + count = db2.count_nodes_by_labels("Person") assert count == 15 db2.close() @@ -179,24 +179,24 @@ def test_wal_and_segment_merged_on_reopen(self, db_dir): db = open_db(db_dir) # Segment data for i in range(5): - db.upsert_node(1, f"seg_{i}") + db.upsert_node("Person", f"seg_{i}") db.flush() # WAL-only data (not flushed) for i in range(3): - db.upsert_node(1, f"wal_{i}") + db.upsert_node("Person", f"wal_{i}") db.close() db2 = open_db(db_dir) - count = db2.count_nodes_by_type(1) + count = db2.count_nodes_by_labels("Person") assert count == 8 db2.close() def test_batch_upsert_survives(self, db_dir): db = open_db(db_dir) ids = db.batch_upsert_nodes([ - {"type_id": 1, "key": "a"}, - {"type_id": 1, "key": "b"}, - {"type_id": 1, "key": "c"}, + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, + {"labels": ["Person"], "key": "c"}, ]) db.close() @@ -209,14 +209,14 @@ def test_graph_patch_survives(self, db_dir): db = open_db(db_dir) result = db.graph_patch({ "upsert_nodes": [ - {"type_id": 1, "key": "p1"}, - {"type_id": 1, "key": "p2"}, + {"labels": ["Person"], "key": "p1"}, + {"labels": ["Person"], "key": "p2"}, ], }) n1, n2 = result.node_ids result2 = db.graph_patch({ "upsert_edges": [ - {"from_id": n1, "to_id": n2, "type_id": 10}, + {"from_id": n1, "to_id": n2, "label": "RELATES_TO"}, ], }) eid = result2.edge_ids[0] diff --git a/overgraph-python/tests/test_property_indexes.py b/overgraph-python/tests/test_property_indexes.py index 32787a6..f14a208 100644 --- a/overgraph-python/tests/test_property_indexes.py +++ b/overgraph-python/tests/test_property_indexes.py @@ -2,7 +2,7 @@ import pytest -from overgraph import OverGraphError, PyPropertyRangeBound, PyPropertyRangeCursor +from overgraph import OverGraphError, PropertyRangeBound, PropertyRangeCursor def wait_for_index_state(db, predicate, expected_state="ready", timeout_s=5.0): @@ -15,11 +15,31 @@ def wait_for_index_state(db, predicate, expected_state="ready", timeout_s=5.0): raise AssertionError(f"timed out waiting for secondary index state '{expected_state}'") +def wait_for_edge_index_state(db, predicate, expected_state="ready", timeout_s=5.0): + deadline = time.time() + timeout_s + while time.time() < deadline: + info = predicate(db.list_edge_property_indexes()) + if info is not None and info.state == expected_state: + return info + time.sleep(0.02) + raise AssertionError(f"timed out waiting for edge secondary index state '{expected_state}'") + + +def plan_has_kind(node, kind): + if not node: + return False + if node.get("kind") == kind: + return True + if "input" in node and plan_has_kind(node["input"], kind): + return True + return any(plan_has_kind(child, kind) for child in node.get("inputs", [])) + + class TestPropertyIndexes: def test_ensure_list_drop(self, db): for i in range(6): db.upsert_node( - 1, + "Person", f"node-{i}", props={ "color": "red" if i % 2 == 0 else "blue", @@ -28,12 +48,12 @@ def test_ensure_list_drop(self, db): }, ) - eq = db.ensure_node_property_index(1, "color", "equality") + eq = db.ensure_node_property_index("Person", "color", "equality") assert eq.kind == "equality" assert eq.domain is None assert eq.state == "building" - range_info = db.ensure_node_property_index(1, "score", "range", domain="int") + range_info = db.ensure_node_property_index("Person", "score", "range", domain="int") assert range_info.kind == "range" assert range_info.domain == "int" assert range_info.state == "building" @@ -61,23 +81,23 @@ def test_ensure_list_drop(self, db): ] with pytest.raises(OverGraphError, match="different domain"): - db.ensure_node_property_index(1, "score", "range", domain="float") + db.ensure_node_property_index("Person", "score", "range", domain="float") - assert db.drop_node_property_index(1, "color", "equality") is True - assert db.drop_node_property_index(1, "color", "equality") is False + assert db.drop_node_property_index("Person", "color", "equality") is True + assert db.drop_node_property_index("Person", "color", "equality") is False def test_range_queries_and_paging(self, db): inserted = [] for i in range(6): inserted.append( db.upsert_node( - 1, + "Person", f"node-{i}", props={"score": (i + 1) * 10, "temp": (i + 1) * 5}, ) ) - db.ensure_node_property_index(1, "score", "range", domain="int") + db.ensure_node_property_index("Person", "score", "range", domain="int") wait_for_index_state( db, lambda infos: next( @@ -86,19 +106,19 @@ def test_range_queries_and_paging(self, db): ), ) - lower = PyPropertyRangeBound(20, domain="int") - upper = PyPropertyRangeBound(50, inclusive=False, domain="int") - all_ids = db.find_nodes_range(1, "score", lower, upper).to_list() + lower = PropertyRangeBound(20, domain="int") + upper = PropertyRangeBound(50, inclusive=False, domain="int") + all_ids = db.find_nodes_range("Person", "score", lower, upper).to_list() assert len(all_ids) == 3 - first = db.find_nodes_range_paged(1, "score", lower, upper, limit=2) + first = db.find_nodes_range_paged("Person", "score", lower, upper, limit=2) assert first.items.to_list() == all_ids[:2] assert first.next_cursor is not None assert first.next_cursor.domain == "int" assert isinstance(first.next_cursor.node_id, int) second = db.find_nodes_range_paged( - 1, + "Person", "score", lower, upper, @@ -109,42 +129,42 @@ def test_range_queries_and_paging(self, db): assert second.next_cursor is None fallback = db.find_nodes_range( - 1, + "Person", "temp", - PyPropertyRangeBound(10, domain="int"), - PyPropertyRangeBound(25, domain="int"), + PropertyRangeBound(10, domain="int"), + PropertyRangeBound(25, domain="int"), ) assert len(fallback) == 4 def test_binding_validation_errors(self, db): with pytest.raises(ValueError, match="Invalid index kind"): - db.ensure_node_property_index(1, "score", "bogus") + db.ensure_node_property_index("Person", "score", "bogus") with pytest.raises(ValueError, match="require domain"): - db.ensure_node_property_index(1, "score", "range") + db.ensure_node_property_index("Person", "score", "range") with pytest.raises(ValueError, match="do not accept a range domain"): - db.ensure_node_property_index(1, "score", "equality", domain="int") + db.ensure_node_property_index("Person", "score", "equality", domain="int") with pytest.raises(ValueError, match="Invalid range domain"): - PyPropertyRangeBound(10, domain="bogus") + PropertyRangeBound(10, domain="bogus") with pytest.raises(OverGraphError, match="same PropValue variant"): db.find_nodes_range( - 1, + "Person", "score", - PyPropertyRangeBound(10, domain="int"), - PyPropertyRangeBound(20.0, domain="float"), + PropertyRangeBound(10, domain="int"), + PropertyRangeBound(20.0, domain="float"), ) with pytest.raises(OverGraphError, match="cursor must use the same PropValue variant"): db.find_nodes_range_paged( - 1, + "Person", "score", - PyPropertyRangeBound(10, domain="int"), - PyPropertyRangeBound(20, domain="int"), + PropertyRangeBound(10, domain="int"), + PropertyRangeBound(20, domain="int"), limit=2, - after=PyPropertyRangeCursor(15.0, 1, domain="float"), + after=PropertyRangeCursor(15.0, 1, domain="float"), ) @@ -153,12 +173,12 @@ class TestPropertyIndexesAsync: async def test_async_property_index_and_range_apis(self, async_db): for i in range(6): await async_db.upsert_node( - 1, + "Person", f"node-{i}", props={"score": (i + 1) * 10, "temp": (i + 1) * 5}, ) - eq = await async_db.ensure_node_property_index(1, "temp", "equality") + eq = await async_db.ensure_node_property_index("Person", "temp", "equality") assert eq.kind == "equality" deadline = time.time() + 5.0 @@ -175,22 +195,196 @@ async def test_async_property_index_and_range_apis(self, async_db): raise AssertionError("timed out waiting for async equality index to become ready") ids = await async_db.find_nodes_range( - 1, + "Person", "score", - PyPropertyRangeBound(20, domain="int"), - PyPropertyRangeBound(30, domain="int"), + PropertyRangeBound(20, domain="int"), + PropertyRangeBound(30, domain="int"), ) assert ids.to_list() and len(ids) == 2 page = await async_db.find_nodes_range_paged( - 1, + "Person", "score", - PyPropertyRangeBound(20, domain="int"), - PyPropertyRangeBound(40, domain="int"), + PropertyRangeBound(20, domain="int"), + PropertyRangeBound(40, domain="int"), limit=2, ) assert len(page.items) == 2 assert page.next_cursor is not None assert page.next_cursor.domain == "int" - assert await async_db.drop_node_property_index(1, "temp", "equality") is True + assert await async_db.drop_node_property_index("Person", "temp", "equality") is True + + +class TestEdgePropertyIndexes: + def test_ensure_list_validate_and_drop_edge_property_indexes(self, db): + eq = db.ensure_edge_property_index("RELATES_TO", "status", "equality") + assert eq.kind == "equality" + assert eq.domain is None + assert eq.state == "building" + + range_info = db.ensure_edge_property_index("RELATES_TO", "score", "range", domain="int") + assert range_info.kind == "range" + assert range_info.domain == "int" + assert range_info.state == "building" + + wait_for_edge_index_state( + db, + lambda infos: next( + (info for info in infos if info.prop_key == "status" and info.kind == "equality"), + None, + ), + ) + wait_for_edge_index_state( + db, + lambda infos: next( + (info for info in infos if info.prop_key == "score" and info.kind == "range"), + None, + ), + ) + + listed = db.list_edge_property_indexes() + assert sorted((info.prop_key, info.kind, info.domain, info.state) for info in listed) == [ + ("score", "range", "int", "ready"), + ("status", "equality", None, "ready"), + ] + + with pytest.raises(OverGraphError, match="different domain"): + db.ensure_edge_property_index("RELATES_TO", "score", "range", domain="float") + + with pytest.raises(ValueError, match="require domain"): + db.ensure_edge_property_index("RELATES_TO", "score", "range") + + with pytest.raises(ValueError, match="do not accept a range domain"): + db.ensure_edge_property_index("RELATES_TO", "status", "equality", domain="int") + + assert db.drop_edge_property_index("RELATES_TO", "missing", "equality") is False + + def test_edge_property_index_queries_and_pattern_explain(self, db): + db.ensure_edge_property_index("RELATES_TO", "status", "equality") + db.ensure_edge_property_index("RELATES_TO", "score", "range", domain="int") + source = db.upsert_node("Person", "source") + hot_target = db.upsert_node("Company", "hot-target") + cold_target = db.upsert_node("Company", "cold-target") + hot_edge = db.upsert_edge(source, hot_target, "RELATES_TO", props={"status": "hot", "score": 90}) + db.upsert_edge(source, cold_target, "RELATES_TO", props={"status": "cold", "score": 10}) + + wait_for_edge_index_state( + db, + lambda infos: next( + (info for info in infos if info.prop_key == "status" and info.kind == "equality"), + None, + ), + ) + wait_for_edge_index_state( + db, + lambda infos: next( + (info for info in infos if info.prop_key == "score" and info.kind == "range"), + None, + ), + ) + + direct = db.query_edge_ids( + { + "label": "RELATES_TO", + "from_ids": [source], + "filter": {"property": "status", "eq": "hot"}, + "limit": 10, + } + ) + assert direct.items.to_list() == [hot_edge] + + direct_plan = db.explain_edge_query( + { + "label": "RELATES_TO", + "from_ids": [source], + "filter": {"property": "status", "eq": "hot"}, + "limit": 10, + } + ) + assert plan_has_kind(direct_plan["root"], "edge_property_equality_index") + + direct_range = db.query_edge_ids( + { + "label": "RELATES_TO", + "from_ids": [source], + "filter": {"property": "score", "gte": 80}, + "limit": 10, + } + ) + assert direct_range.items.to_list() == [hot_edge] + + direct_range_plan = db.explain_edge_query( + { + "label": "RELATES_TO", + "from_ids": [source], + "filter": {"property": "score", "gte": 80}, + "limit": 10, + } + ) + assert plan_has_kind(direct_range_plan["root"], "edge_property_range_index") + + pattern = { + "nodes": [ + {"alias": "a", "label_filter": {"labels": ["Person"], "mode": "all"}}, + {"alias": "b", "label_filter": {"labels": ["Company"], "mode": "all"}}, + ], + "edges": [ + { + "alias": "e", + "from_alias": "a", + "to_alias": "b", + "direction": "outgoing", + "label_filter": ["RELATES_TO"], + "filter": {"property": "status", "eq": "hot"}, + } + ], + "limit": 10, + } + assert db.query_pattern(pattern)["matches"] == [ + {"nodes": {"a": source, "b": hot_target}, "edges": {"e": hot_edge}} + ] + pattern_plan = db.explain_pattern_query(pattern) + assert plan_has_kind(pattern_plan["root"], "pattern_edge_anchor") + assert plan_has_kind(pattern_plan["root"], "edge_property_equality_index") + + range_pattern = { + **pattern, + "edges": [ + { + **pattern["edges"][0], + "filter": {"property": "score", "gte": 80}, + } + ], + } + assert db.query_pattern(range_pattern)["matches"] == [ + {"nodes": {"a": source, "b": hot_target}, "edges": {"e": hot_edge}} + ] + range_pattern_plan = db.explain_pattern_query(range_pattern) + assert plan_has_kind(range_pattern_plan["root"], "pattern_edge_anchor") + assert plan_has_kind(range_pattern_plan["root"], "edge_property_range_index") + + +@pytest.mark.asyncio +class TestEdgePropertyIndexesAsync: + async def test_async_edge_property_index_apis(self, async_db): + info = await async_db.ensure_edge_property_index("RELATES_TO", "temp", "equality") + assert info.kind == "equality" + + deadline = time.time() + 5.0 + while time.time() < deadline: + infos = await async_db.list_edge_property_indexes() + ready = next( + (info for info in infos if info.prop_key == "temp" and info.kind == "equality"), + None, + ) + if ready is not None and ready.state == "ready": + break + await __import__("asyncio").sleep(0.02) + else: + raise AssertionError("timed out waiting for async edge equality index to become ready") + + listed = await async_db.list_edge_property_indexes() + assert any(info.prop_key == "temp" and info.kind == "equality" for info in listed) + + assert await async_db.drop_edge_property_index("RELATES_TO", "temp", "equality") is True diff --git a/overgraph-python/tests/test_queries.py b/overgraph-python/tests/test_queries.py index e63f05d..a6ddded 100644 --- a/overgraph-python/tests/test_queries.py +++ b/overgraph-python/tests/test_queries.py @@ -3,117 +3,125 @@ class TestFindNodes: def test_find_by_prop(self, db): - db.upsert_node(1, "a", props={"color": "red"}) - db.upsert_node(1, "b", props={"color": "blue"}) - db.upsert_node(1, "c", props={"color": "red"}) - ids = db.find_nodes(1, "color", "red") + db.upsert_node("Person", "a", props={"color": "red"}) + db.upsert_node("Person", "b", props={"color": "blue"}) + db.upsert_node("Person", "c", props={"color": "red"}) + ids = db.find_nodes("Person", "color", "red") assert len(ids) == 2 def test_find_no_match(self, db): - db.upsert_node(1, "a", props={"color": "red"}) - ids = db.find_nodes(1, "color", "green") + db.upsert_node("Person", "a", props={"color": "red"}) + ids = db.find_nodes("Person", "color", "green") assert len(ids) == 0 def test_find_wrong_type(self, db): - db.upsert_node(1, "a", props={"color": "red"}) - ids = db.find_nodes(2, "color", "red") + db.upsert_node("Person", "a", props={"color": "red"}) + ids = db.find_nodes("Company", "color", "red") assert len(ids) == 0 def test_find_int_prop(self, db): - db.upsert_node(1, "a", props={"level": 5}) - db.upsert_node(1, "b", props={"level": 10}) - ids = db.find_nodes(1, "level", 5) + db.upsert_node("Person", "a", props={"level": 5}) + db.upsert_node("Person", "b", props={"level": 10}) + ids = db.find_nodes("Person", "level", 5) assert len(ids) == 1 class TestNodesByType: - def test_nodes_by_type(self, db): - db.upsert_node(1, "a") - db.upsert_node(1, "b") - db.upsert_node(2, "c") - ids = db.nodes_by_type(1) + def test_nodes_by_labels(self, db): + db.upsert_node("Person", "a") + db.upsert_node("Person", "b") + db.upsert_node("Company", "c") + ids = db.nodes_by_labels("Person") assert len(ids) == 2 - def test_nodes_by_type_empty(self, db): - ids = db.nodes_by_type(99) + def test_nodes_by_labels_empty(self, db): + ids = db.nodes_by_labels("Missing") assert len(ids) == 0 + def test_nodes_by_labels_list_requires_all_labels(self, db): + admin = db.upsert_node(["Person", "Admin"], "admin") + db.upsert_node("Person", "regular") + + assert db.nodes_by_labels(["Person", "Admin"]).to_list() == [admin] + assert not hasattr(db, "nodes_by_label") + class TestGetNodesByType: - def test_get_nodes_by_type(self, db): - db.upsert_node(1, "a", props={"name": "Alice"}) - db.upsert_node(1, "b", props={"name": "Bob"}) - db.upsert_node(2, "c") - nodes = db.get_nodes_by_type(1) + def test_get_nodes_by_labels(self, db): + db.upsert_node("Person", "a", props={"name": "Alice"}) + db.upsert_node("Person", "b", props={"name": "Bob"}) + db.upsert_node("Company", "c") + nodes = db.get_nodes_by_labels("Person") assert len(nodes) == 2 - assert all(n.type_id == 1 for n in nodes) + assert all(n.labels == ["Person"] for n in nodes) - def test_get_nodes_by_type_empty(self, db): - assert db.get_nodes_by_type(99) == [] + def test_get_nodes_by_labels_empty(self, db): + assert db.get_nodes_by_labels("Missing") == [] class TestEdgesByType: - def test_edges_by_type(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n2, n3, 20) - db.upsert_edge(n1, n3, 10) - ids = db.edges_by_type(10) + def test_edges_by_label(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n2, n3, "WORKS_AT") + db.upsert_edge(n1, n3, "RELATES_TO") + ids = db.edges_by_label("RELATES_TO") assert len(ids) == 2 - def test_edges_by_type_empty(self, db): - assert len(db.edges_by_type(99)) == 0 + def test_edges_by_label_empty(self, db): + assert len(db.edges_by_label("FOLLOWS")) == 0 class TestGetEdgesByType: - def test_get_edges_by_type(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - db.upsert_edge(n1, n2, 10, props={"rel": "friend"}) - edges = db.get_edges_by_type(10) + def test_get_edges_by_label(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + db.upsert_edge(n1, n2, "RELATES_TO", props={"rel": "friend"}) + edges = db.get_edges_by_label("RELATES_TO") assert len(edges) == 1 assert edges[0].props["rel"] == "friend" - def test_get_edges_by_type_empty(self, db): - assert db.get_edges_by_type(99) == [] + def test_get_edges_by_label_empty(self, db): + assert db.get_edges_by_label("FOLLOWS") == [] class TestCountByType: def test_count_nodes(self, db): - db.upsert_node(1, "a") - db.upsert_node(1, "b") - db.upsert_node(2, "c") - assert db.count_nodes_by_type(1) == 2 - assert db.count_nodes_by_type(2) == 1 - assert db.count_nodes_by_type(99) == 0 + db.upsert_node("Person", "a") + db.upsert_node("Person", "b") + db.upsert_node("Company", "c") + assert db.count_nodes_by_labels("Person") == 2 + assert db.count_nodes_by_labels("Company") == 1 + assert db.count_nodes_by_labels("Missing") == 0 + assert not hasattr(db, "count_nodes_by_label") def test_count_edges(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n2, 20) - assert db.count_edges_by_type(10) == 1 - assert db.count_edges_by_type(20) == 1 - assert db.count_edges_by_type(99) == 0 + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n2, "WORKS_AT") + assert db.count_edges_by_label("RELATES_TO") == 1 + assert db.count_edges_by_label("WORKS_AT") == 1 + assert db.count_edges_by_label("FOLLOWS") == 0 class TestFindNodesByTimeRange: def test_time_range(self, db): - id1 = db.upsert_node(1, "a") - id2 = db.upsert_node(1, "b") + id1 = db.upsert_node("Person", "a") + id2 = db.upsert_node("Person", "b") # Both nodes were just created, so their timestamps should be recent node1 = db.get_node(id1) node2 = db.get_node(id2) # Search with a range that covers both lo = min(node1.updated_at, node2.updated_at) - 1000 hi = max(node1.updated_at, node2.updated_at) + 1000 - ids = db.find_nodes_by_time_range(1, lo, hi) + ids = db.find_nodes_by_time_range("Person", lo, hi) assert len(ids) == 2 def test_time_range_empty(self, db): - db.upsert_node(1, "a") + db.upsert_node("Person", "a") # Search far in the past - ids = db.find_nodes_by_time_range(1, 0, 1) + ids = db.find_nodes_by_time_range("Person", 0, 1) assert len(ids) == 0 diff --git a/overgraph-python/tests/test_query_api.py b/overgraph-python/tests/test_query_api.py index 2186dfd..a528a92 100644 --- a/overgraph-python/tests/test_query_api.py +++ b/overgraph-python/tests/test_query_api.py @@ -1,7 +1,13 @@ import pytest import time -from overgraph import GraphEdgePattern, GraphNodePattern, GraphPatternRequest, NodeQueryRequest +from overgraph import ( + EdgeQueryRequest, + GraphEdgePattern, + GraphNodePattern, + GraphPatternRequest, + NodeQueryRequest, +) def plan_has_kind(node, kind): @@ -14,6 +20,10 @@ def plan_has_kind(node, kind): return any(plan_has_kind(child, kind) for child in node.get("inputs", [])) +def node_lf(*labels, mode="all"): + return {"labels": list(labels), "mode": mode} + + def wait_for_index_state(db, predicate, expected_state="ready", timeout_s=5.0): deadline = time.time() + timeout_s while time.time() < deadline: @@ -26,41 +36,41 @@ def wait_for_index_state(db, predicate, expected_state="ready", timeout_s=5.0): def seed_query_graph(db): active_high = db.upsert_node( - 1, "active-high", props={"status": "active", "score": 90, "team": "core"} + "Person", "active-high", props={"status": "active", "score": 90, "team": "core"} ) active_low = db.upsert_node( - 1, "active-low", props={"status": "active", "score": 40, "team": "core"} + "Person", "active-low", props={"status": "active", "score": 40, "team": "core"} ) inactive = db.upsert_node( - 1, "inactive", props={"status": "inactive", "score": 95, "team": "core"} + "Person", "inactive", props={"status": "inactive", "score": 95, "team": "core"} ) literal_updated_at = db.upsert_node( - 1, + "Person", "literal-updated-at", props={"updated_at": "literal-property-value", "status": "active", "score": 70}, ) null_tag = db.upsert_node( - 1, + "Person", "null-tag", props={"status": "nullish", "tag": None, "score": 10}, ) nested = db.upsert_node( - 1, + "Person", "nested", props={"status": "nested", "payload": {"items": [1, "1", None]}, "score": 15}, ) - acme = db.upsert_node(2, "acme", props={"status": "customer"}) - beta = db.upsert_node(2, "beta", props={"status": "prospect"}) + acme = db.upsert_node("Company", "acme", props={"status": "customer"}) + beta = db.upsert_node("Company", "beta", props={"status": "prospect"}) works_at = db.upsert_edge( active_high, acme, - 10, + "WORKS_AT", props={"role": "engineer", "since": 2020, "updated_at": "edge-literal"}, ) inactive_works_at = db.upsert_edge( inactive, beta, - 10, + "WORKS_AT", props={"role": "engineer", "since": 2022, "updated_at": "edge-literal"}, ) return { @@ -80,7 +90,7 @@ def seed_query_graph(db): def test_query_node_ids_and_hydrated_nodes(db): ids = seed_query_graph(db) request = { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": { "and": [ {"property": "status", "eq": "active"}, @@ -104,7 +114,7 @@ def test_query_predicates_and_literal_builtin_name_collision(db): result = db.query_node_ids( { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": { "and": [ {"property": "status", "eq": "active"}, @@ -118,7 +128,7 @@ def test_query_predicates_and_literal_builtin_name_collision(db): updated_at = db.get_node(ids["active_high"]).updated_at timestamp_result = db.query_node_ids( { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": { "and": [ {"updated_at": {"gte": updated_at - 1000}}, @@ -130,6 +140,22 @@ def test_query_predicates_and_literal_builtin_name_collision(db): assert ids["active_high"] in timestamp_result.items.to_list() +def test_query_node_label_filter_any_and_all_modes(db): + admin = db.upsert_node(["Person", "Admin"], "admin") + person = db.upsert_node("Person", "person") + company = db.upsert_node("Company", "company") + + assert db.query_node_ids({"label_filter": node_lf("Person", "Admin")}).items.to_list() == [ + admin + ] + assert sorted( + db.query_node_ids({"label_filter": node_lf("Admin", "Company", mode="any")}).items.to_list() + ) == [admin, company] + assert person not in db.query_node_ids( + {"label_filter": node_lf("Admin", "Company", mode="any")} + ).items.to_list() + + def test_query_updated_at_exclusive_boundaries(db): ids = seed_query_graph(db) updated_at = db.get_node(ids["active_high"]).updated_at @@ -196,10 +222,10 @@ def test_query_pattern_and_edge_literal_updated_at(db): "nodes": [ { "alias": "person", - "type_id": 1, + "label_filter": node_lf("Person"), "filter": {"property": "status", "eq": "active"}, }, - {"alias": "company", "type_id": 2, "keys": ["acme"]}, + {"alias": "company", "label_filter": node_lf("Company"), "keys": ["acme"]}, ], "edges": [ { @@ -207,14 +233,14 @@ def test_query_pattern_and_edge_literal_updated_at(db): "from_alias": "person", "to_alias": "company", "direction": "outgoing", - "type_filter": [10], - "where": { - "role": {"op": "eq", "value": "engineer"}, - "updated_at": {"op": "eq", "value": "edge-literal"}, + "label_filter": ["WORKS_AT"], + "filter": { + "and": [ + {"property": "role", "eq": "engineer"}, + {"property": "updated_at", "eq": "edge-literal"}, + {"property": "since", "lte": 2021}, + ] }, - "predicates": [ - {"property": {"key": "since", "op": "range", "lte": 2021}} - ], } ], "limit": 10, @@ -233,11 +259,70 @@ def test_query_pattern_and_edge_literal_updated_at(db): assert ids["inactive_works_at"] != ids["works_at"] +def test_direct_edge_queries_and_canonical_pattern_filter(db): + ids = seed_query_graph(db) + edge = db.get_edge(ids["works_at"]) + request = EdgeQueryRequest( + label="WORKS_AT", + from_ids=[ids["active_high"]], + filter={ + "and": [ + {"weight": {"gte": 1.0}}, + {"valid_at": int(time.time() * 1000)}, + {"updated_at": {"gte": edge.updated_at - 1000}}, + {"property": "role", "eq": "engineer"}, + ] + }, + limit=0, + ) + + edge_ids = db.query_edge_ids(request) + assert edge_ids.items.to_list() == [ids["works_at"]] + + edges = db.query_edges({**request.to_dict(), "limit": 1}) + assert [edge.id for edge in edges.items] == [ids["works_at"]] + assert edges.next_cursor is None + assert edges.items[0].props["role"] == "engineer" + + plan = db.explain_edge_query(request) + assert plan["kind"] == "edge_query" + assert plan_has_kind(plan["root"], "verify_edge_filter") + assert "edge_property_post_filter" in plan["warnings"] + + pattern = GraphPatternRequest( + nodes=[ + GraphNodePattern("person", ids=[ids["active_high"]]), + GraphNodePattern("company", label_filter=node_lf("Company"), keys=["acme"]), + ], + edges=[ + GraphEdgePattern( + "person", + "company", + alias="employment", + label_filter=["WORKS_AT"], + filter={ + "and": [ + {"valid_at": int(time.time() * 1000)}, + {"property": "role", "eq": "engineer"}, + ] + }, + ) + ], + limit=10, + ) + assert db.query_pattern(pattern)["matches"] == [ + { + "nodes": {"company": ids["acme"], "person": ids["active_high"]}, + "edges": {"employment": ids["works_at"]}, + } + ] + + def test_query_request_helpers_are_directly_usable(db): ids = seed_query_graph(db) request = NodeQueryRequest( - type_id=1, + label_filter=node_lf("Person"), filter={ "and": [ {"property": "status", "eq": "active"}, @@ -253,18 +338,18 @@ def test_query_request_helpers_are_directly_usable(db): nodes=[ GraphNodePattern( "person", - type_id=1, + label_filter=node_lf("Person"), filter={"property": "status", "eq": "active"}, ), - GraphNodePattern("company", type_id=2, keys=["acme"]), + GraphNodePattern("company", label_filter=node_lf("Company"), keys=["acme"]), ], edges=[ GraphEdgePattern( "person", "company", alias="employment", - type_filter=[10], - where={"role": {"eq": "engineer"}}, + label_filter=["WORKS_AT"], + filter={"property": "role", "eq": "engineer"}, ) ], limit=10, @@ -277,40 +362,53 @@ def test_query_request_helpers_are_directly_usable(db): def test_query_request_helpers_reject_string_list_fields(db): with pytest.raises(TypeError, match="keys"): - db.query_node_ids(NodeQueryRequest(type_id=1, keys="acme")) + db.query_node_ids(NodeQueryRequest(label_filter=node_lf("Person"), keys="acme")) with pytest.raises(TypeError, match="keys"): - GraphNodePattern("company", type_id=2, keys="acme").to_dict() + GraphNodePattern("company", label_filter=node_lf("Company"), keys="acme").to_dict() def test_query_explain_uses_lower_snake_recursive_strings(db): - seed_query_graph(db) + ids = seed_query_graph(db) node_plan = db.explain_node_query( - {"type_id": 1, "filter": {"property": "status", "eq": "active"}} + {"label_filter": node_lf("Person"), "filter": {"property": "status", "eq": "active"}} ) assert node_plan["kind"] == "node_query" - assert plan_has_kind(node_plan["root"], "fallback_type_scan") + assert plan_has_kind(node_plan["root"], "fallback_node_label_scan") assert all(warning.replace("_", "").islower() for warning in node_plan["warnings"]) assert "using_fallback_scan" in node_plan["warnings"] + assert "stale_node_label_membership_verification" in node_plan["notes"] + assert node_plan["public_inputs"]["node_labels"] == [ + {"alias": None, "name": "Person", "known": True, "mode": "all"} + ] + assert node_plan["public_inputs"]["edge_labels"] == [] + + db.upsert_node(["Person", "Admin"], "admin") + any_plan = db.explain_node_query({"label_filter": node_lf("Person", "Admin", mode="any")}) + assert "node_label_any_final_verification" in any_plan["notes"] + assert any_plan["public_inputs"]["node_labels"] == [ + {"alias": None, "name": "Person", "known": True, "mode": "any"}, + {"alias": None, "name": "Admin", "known": True, "mode": "any"}, + ] pattern_plan = db.explain_pattern_query( { "nodes": [ { "alias": "person", - "type_id": 1, + "label_filter": node_lf("Person"), "filter": {"property": "status", "eq": "active"}, }, - {"alias": "company", "type_id": 2, "keys": ["acme"]}, + {"alias": "company", "label_filter": node_lf("Company"), "keys": ["acme"]}, ], "edges": [ { "alias": "employment", "from_alias": "person", "to_alias": "company", - "type_filter": [10], - "where": {"role": {"op": "eq", "value": "engineer"}}, + "label_filter": ["WORKS_AT"], + "filter": {"property": "role", "eq": "engineer"}, } ], "limit": 10, @@ -321,21 +419,48 @@ def test_query_explain_uses_lower_snake_recursive_strings(db): assert plan_has_kind(pattern_plan["root"], "verify_edge_predicates") assert "edge_property_post_filter" in pattern_plan["warnings"] + full_edge_plan = db.explain_edge_query( + { + "filter": {"property": "role", "eq": "engineer"}, + "allow_full_scan": True, + } + ) + assert plan_has_kind(full_edge_plan["root"], "fallback_full_edge_scan") + + missing_edge_plan = db.explain_edge_query( + {"label": "MISSING", "from_ids": [ids["active_high"]]} + ) + assert "unknown_edge_label" in missing_edge_plan["warnings"] + assert missing_edge_plan["public_inputs"]["edge_labels"] == [ + {"alias": None, "name": "MISSING", "known": False, "mode": None} + ] + def test_query_validation_errors(db): seed_query_graph(db) with pytest.raises(Exception, match="use filter"): db.query_node_ids( - {"type_id": 1, "predicates": [{"property": {"key": "status", "op": "eq"}}]} + {"label_filter": node_lf("Person"), "predicates": [{"property": {"key": "status", "op": "eq"}}]} ) with pytest.raises(Exception, match="both gt and gte"): - db.query_node_ids({"type_id": 1, "filter": {"property": "score", "gt": 1, "gte": 2}}) + db.query_node_ids({"label_filter": node_lf("Person"), "filter": {"property": "score", "gt": 1, "gte": 2}}) with pytest.raises(Exception, match="use filter"): - db.query_node_ids({"type_id": 1, "where": {"status": {"eq": "active"}}}) + db.query_node_ids({"label_filter": node_lf("Person"), "where": {"status": {"eq": "active"}}}) with pytest.raises(Exception, match="use filter"): db.query_pattern({"nodes": [{"alias": "a", "where": {"status": {"eq": "active"}}}], "edges": [], "limit": 1}) - with pytest.raises(Exception, match="edge pattern filter is not supported"): + with pytest.raises(Exception, match="full scan|anchor|allow_full_scan"): + db.query_edge_ids({"filter": {"weight": {"gte": 1}}}) + with pytest.raises(Exception, match="both gt and gte"): + db.query_edge_ids({"label": "WORKS_AT", "filter": {"weight": {"gt": 1, "gte": 2}}}) + for field in ("where", "predicates"): + with pytest.raises(Exception, match="use filter"): + db.query_edge_ids({"label": "WORKS_AT", field: {"role": {"eq": "engineer"}}}) + with pytest.raises(Exception, match="use filter"): + db.query_edges({"label": "WORKS_AT", field: {"role": {"eq": "engineer"}}}) + with pytest.raises(Exception, match="use filter"): + db.explain_edge_query({"label": "WORKS_AT", field: {"role": {"eq": "engineer"}}}) + with pytest.raises(Exception, match="use filter"): db.query_pattern( { "nodes": [{"alias": "a"}], @@ -344,6 +469,7 @@ def test_query_validation_errors(db): "from_alias": "a", "to_alias": "b", "filter": {"property": "role", "eq": "engineer"}, + "where": {"role": {"eq": "engineer"}}, } ], "limit": 1, @@ -357,30 +483,63 @@ def test_query_numeric_fields_reject_bool(db): seed_query_graph(db) invalid_node_requests = [ - {"type_id": True}, {"ids": [True]}, - {"type_id": 1, "after": True}, - {"type_id": 1, "limit": True}, - {"type_id": 1, "filter": {"updated_at": {"gte": True}}}, + {"label_filter": node_lf("Person"), "after": True}, + {"label_filter": node_lf("Person"), "limit": True}, + {"label_filter": node_lf("Person"), "filter": {"updated_at": {"gte": True}}}, ] for request in invalid_node_requests: with pytest.raises(TypeError, match="bool"): db.query_node_ids(request) - invalid_pattern_requests = [ + invalid_edge_requests = [ + {"label": True}, + {"ids": [True]}, + {"from_ids": [True]}, + {"to_ids": [True]}, + {"endpoint_ids": [True]}, + {"label": "WORKS_AT", "after": True}, + {"label": "WORKS_AT", "limit": True}, + {"label": "WORKS_AT", "filter": {"valid_at": True}}, + {"label": "WORKS_AT", "filter": {"weight": {"gte": True}}}, + ] + for request in invalid_edge_requests: + with pytest.raises(TypeError, match="bool"): + db.query_edge_ids(request) + + invalid_pattern_bool_requests = [ {"nodes": [], "edges": [], "limit": True}, {"nodes": [], "edges": [], "limit": 1, "at_epoch": True}, - {"nodes": [{"alias": "a", "type_id": True}], "edges": [], "limit": 1}, {"nodes": [{"alias": "a", "ids": [True]}], "edges": [], "limit": 1}, - { - "nodes": [], - "edges": [{"from_alias": "a", "to_alias": "b", "type_filter": [True]}], - "limit": 1, - }, ] - for request in invalid_pattern_requests: + for request in invalid_pattern_bool_requests: with pytest.raises(TypeError, match="bool"): db.query_pattern(request) + with pytest.raises(TypeError, match="str"): + db.query_node_ids({"label_filter": {"labels": [True], "mode": "all"}}) + with pytest.raises(TypeError, match="str"): + db.query_pattern( + { + "nodes": [ + {"alias": "a", "label_filter": {"labels": [True], "mode": "all"}} + ], + "edges": [], + "limit": 1, + } + ) + with pytest.raises(TypeError, match="str"): + db.query_pattern( + { + "nodes": [], + "edges": [{"from_alias": "a", "to_alias": "b", "label_filter": [True]}], + "limit": 1, + } + ) + + with pytest.raises(ValueError, match="label.*label_filter"): + db.query_node_ids({"label": "Person"}) + with pytest.raises(ValueError, match="label.*label_filter"): + db.query_pattern({"nodes": [{"alias": "a", "label": "Person"}], "edges": [], "limit": 1}) def test_query_boolean_filter_and_value_semantics(db): @@ -389,7 +548,7 @@ def test_query_boolean_filter_and_value_semantics(db): assert sorted( db.query_node_ids( { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": { "or": [ {"property": "status", "eq": "active"}, @@ -406,38 +565,38 @@ def test_query_boolean_filter_and_value_semantics(db): ] assert db.query_node_ids( - {"type_id": 1, "filter": {"property": "status", "in": ["nested"]}} + {"label_filter": node_lf("Person"), "filter": {"property": "status", "in": ["nested"]}} ).items.to_list() == [ids["nested"]] assert db.query_node_ids( - {"type_id": 1, "filter": {"property": "tag", "eq": None}} + {"label_filter": node_lf("Person"), "filter": {"property": "tag", "eq": None}} ).items.to_list() == [ids["null_tag"]] assert db.query_node_ids( - {"type_id": 1, "filter": {"property": "tag", "in": [None]}} + {"label_filter": node_lf("Person"), "filter": {"property": "tag", "in": [None]}} ).items.to_list() == [ids["null_tag"]] assert db.query_node_ids( - {"type_id": 1, "filter": {"property": "tag", "exists": True}} + {"label_filter": node_lf("Person"), "filter": {"property": "tag", "exists": True}} ).items.to_list() == [ids["null_tag"]] assert ids["null_tag"] not in db.query_node_ids( - {"type_id": 1, "filter": {"property": "tag", "missing": True}} + {"label_filter": node_lf("Person"), "filter": {"property": "tag", "missing": True}} ).items.to_list() assert db.query_node_ids( { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": {"property": "payload", "eq": {"items": [1, "1", None]}}, } ).items.to_list() == [ids["nested"]] assert db.query_node_ids( - {"type_id": 1, "filter": {"property": "status", "eq": "1"}} + {"label_filter": node_lf("Person"), "filter": {"property": "status", "eq": "1"}} ).items.to_list() == [] - int_node = db.upsert_node(1, "int-value", props={"kind": 1}) - float_node = db.upsert_node(1, "float-value", props={"kind": 1.0}) + int_node = db.upsert_node("Person", "int-value", props={"kind": 1}) + float_node = db.upsert_node("Person", "float-value", props={"kind": 1.0}) assert db.query_node_ids( - {"type_id": 1, "filter": {"property": "kind", "eq": 1}} + {"label_filter": node_lf("Person"), "filter": {"property": "kind", "eq": 1}} ).items.to_list() == [int_node] assert db.query_node_ids( - {"type_id": 1, "filter": {"property": "kind", "eq": 1.0}} + {"label_filter": node_lf("Person"), "filter": {"property": "kind", "eq": 1.0}} ).items.to_list() == [float_node] @@ -464,7 +623,7 @@ def test_query_invalid_canonical_filter_shapes(db): ] for filter_expr, pattern in invalid_filters: with pytest.raises(Exception, match=pattern): - db.query_node_ids({"type_id": 1, "filter": filter_expr}) + db.query_node_ids({"label_filter": node_lf("Person"), "filter": filter_expr}) with pytest.raises(Exception, match="use filter"): db.query_pattern( @@ -485,18 +644,18 @@ def test_query_invalid_canonical_filter_shapes(db): def test_query_boolean_explain_serialization(db): seed_query_graph(db) - db.ensure_node_property_index(1, "status", "equality") + db.ensure_node_property_index("Person", "status", "equality") wait_for_index_state( db, lambda infos: next( - (info for info in infos if info.type_id == 1 and info.prop_key == "status"), + (info for info in infos if info.label == "Person" and info.prop_key == "status"), None, ), ) indexed_or = db.explain_node_query( { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": { "or": [ {"property": "status", "eq": "active"}, @@ -510,7 +669,7 @@ def test_query_boolean_explain_serialization(db): fallback_or = db.explain_node_query( { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": { "or": [ {"property": "status", "eq": "active"}, @@ -524,7 +683,7 @@ def test_query_boolean_explain_serialization(db): empty = db.explain_node_query( { - "type_id": 1, + "label_filter": node_lf("Person"), "filter": { "and": [ {"property": "status", "eq": "active"}, @@ -539,29 +698,57 @@ def test_query_boolean_explain_serialization(db): @pytest.mark.asyncio async def test_async_query_parity(async_db): active_high = await async_db.upsert_node( - 1, "active-high", props={"status": "active", "score": 90} + "Person", "active-high", props={"status": "active", "score": 90} ) inactive = await async_db.upsert_node( - 1, "inactive", props={"status": "inactive", "score": 95} + "Person", "inactive", props={"status": "inactive", "score": 95} + ) + acme = await async_db.upsert_node("Company", "acme") + works_at = await async_db.upsert_edge( + active_high, acme, "WORKS_AT", props={"role": "engineer"} ) - acme = await async_db.upsert_node(2, "acme") - await async_db.upsert_edge(active_high, acme, 10) ids = await async_db.query_node_ids( - NodeQueryRequest(type_id=1, filter={"property": "status", "eq": "active"}) + NodeQueryRequest(label_filter=node_lf("Person"), filter={"property": "status", "eq": "active"}) ) assert ids.items.to_list() == [active_high] nodes = await async_db.query_nodes( - {"type_id": 1, "filter": {"property": "score", "gte": 80}} + {"label_filter": node_lf("Person"), "filter": {"property": "score", "gte": 80}} ) assert [node.id for node in nodes.items] == [active_high, inactive] plan = await async_db.explain_node_query( - {"type_id": 1, "filter": {"property": "status", "eq": "active"}} + {"label_filter": node_lf("Person"), "filter": {"property": "status", "eq": "active"}} ) assert plan["kind"] == "node_query" + edge_ids = await async_db.query_edge_ids( + {"from_ids": [active_high], "filter": {"property": "role", "eq": "engineer"}} + ) + assert edge_ids.items.to_list() == [works_at] + + edges = await async_db.query_edges( + {"ids": [works_at], "filter": {"valid_at": int(time.time() * 1000)}} + ) + assert [edge.id for edge in edges.items] == [works_at] + + edge_plan = await async_db.explain_edge_query({"ids": [works_at]}) + assert edge_plan["kind"] == "edge_query" + + with pytest.raises(Exception, match="use filter"): + await async_db.query_edge_ids( + {"label": "WORKS_AT", "where": {"role": {"eq": "engineer"}}} + ) + with pytest.raises(Exception, match="use filter"): + await async_db.query_edges( + {"label": "WORKS_AT", "predicates": {"role": {"eq": "engineer"}}} + ) + with pytest.raises(Exception, match="use filter"): + await async_db.explain_edge_query( + {"label": "WORKS_AT", "where": {"role": {"eq": "engineer"}}} + ) + pattern = await async_db.query_pattern( { "nodes": [ @@ -570,14 +757,14 @@ async def test_async_query_parity(async_db): "ids": [active_high], "filter": {"property": "status", "eq": "active"}, }, - {"alias": "company", "type_id": 2, "keys": ["acme"]}, + {"alias": "company", "label_filter": node_lf("Company"), "keys": ["acme"]}, ], "edges": [ { "alias": "employment", "from_alias": "person", "to_alias": "company", - "type_filter": [10], + "label_filter": ["WORKS_AT"], } ], "limit": 10, @@ -593,14 +780,14 @@ async def test_async_query_parity(async_db): "ids": [active_high], "filter": {"property": "status", "eq": "active"}, }, - {"alias": "company", "type_id": 2, "keys": ["acme"]}, + {"alias": "company", "label_filter": node_lf("Company"), "keys": ["acme"]}, ], "edges": [ { "alias": "employment", "from_alias": "person", "to_alias": "company", - "type_filter": [10], + "label_filter": ["WORKS_AT"], } ], "limit": 10, diff --git a/overgraph-python/tests/test_retention.py b/overgraph-python/tests/test_retention.py index fffc9f6..866947a 100644 --- a/overgraph-python/tests/test_retention.py +++ b/overgraph-python/tests/test_retention.py @@ -6,26 +6,26 @@ class TestPrune: def test_prune_by_weight(self, db): - db.upsert_node(1, "low", weight=0.1) - db.upsert_node(1, "high", weight=5.0) + db.upsert_node("Person", "low", weight=0.1) + db.upsert_node("Person", "high", weight=5.0) result = db.prune(max_weight=0.5) assert result.nodes_pruned == 1 assert result.edges_pruned == 0 - assert db.get_node_by_key(1, "low") is None - assert db.get_node_by_key(1, "high") is not None + assert db.get_node_by_key("Person", "low") is None + assert db.get_node_by_key("Person", "high") is not None def test_prune_by_type(self, db): - db.upsert_node(1, "a", weight=0.1) - db.upsert_node(2, "b", weight=0.1) - result = db.prune(max_weight=0.5, type_id=1) + db.upsert_node("Person", "a", weight=0.1) + db.upsert_node("Company", "b", weight=0.1) + result = db.prune(max_weight=0.5, label="Person") assert result.nodes_pruned == 1 # Type 2 should survive - assert db.get_node_by_key(2, "b") is not None + assert db.get_node_by_key("Company", "b") is not None def test_prune_cascades_edges(self, db): - n1 = db.upsert_node(1, "a", weight=0.1) - n2 = db.upsert_node(1, "b", weight=5.0) - eid = db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a", weight=0.1) + n2 = db.upsert_node("Person", "b", weight=5.0) + eid = db.upsert_edge(n1, n2, "RELATES_TO") result = db.prune(max_weight=0.5) assert result.nodes_pruned == 1 assert result.edges_pruned >= 1 @@ -33,19 +33,19 @@ def test_prune_cascades_edges(self, db): def test_prune_no_match(self, db): # weight 5.0 > threshold 0.1, so node survives - db.upsert_node(1, "a", weight=5.0) + db.upsert_node("Person", "a", weight=5.0) result = db.prune(max_weight=0.1) assert result.nodes_pruned == 0 def test_prune_no_criteria_rejects(self, db): - db.upsert_node(1, "a") + db.upsert_node("Person", "a") with pytest.raises(Exception, match="at least max_age_ms or max_weight"): db.prune() # no criteria = error def test_prune_by_age(self, db): - db.upsert_node(1, "old", weight=5.0) + db.upsert_node("Person", "old", weight=5.0) time.sleep(0.05) # 50ms - db.upsert_node(1, "new", weight=5.0) + db.upsert_node("Person", "new", weight=5.0) result = db.prune(max_age_ms=10) # Prune anything older than 10ms # The "old" node should be pruned (50ms old > 10ms threshold) assert result.nodes_pruned >= 1 @@ -81,15 +81,15 @@ def test_remove_nonexistent(self, db): removed = db.remove_prune_policy("nope") assert removed is False - def test_policy_with_type_id(self, db): - db.set_prune_policy("typed", max_weight=0.5, type_id=1) + def test_policy_with_label(self, db): + db.set_prune_policy("typed", max_weight=0.5, label="Person") policies = db.list_prune_policies() - assert policies[0].type_id == 1 + assert policies[0].label == "Person" def test_policy_filtering_reads(self, db): """Registered policies should filter reads immediately.""" - n1 = db.upsert_node(1, "low", weight=0.1) - n2 = db.upsert_node(1, "high", weight=5.0) + n1 = db.upsert_node("Person", "low", weight=0.1) + n2 = db.upsert_node("Person", "high", weight=5.0) # Before policy: both visible assert db.get_node(n1) is not None assert db.get_node(n2) is not None diff --git a/overgraph-python/tests/test_scrub.py b/overgraph-python/tests/test_scrub.py new file mode 100644 index 0000000..d4450f1 --- /dev/null +++ b/overgraph-python/tests/test_scrub.py @@ -0,0 +1,59 @@ +import os + +from overgraph import OverGraph + + +def test_scrub_healthy_database(tmp_dir): + db_path = os.path.join(tmp_dir, "scrub_db") + db = OverGraph.open(db_path) + for i in range(5): + db.upsert_node("Person", f"node_{i}") + db.flush() + + report = db.scrub() + assert report.total_components_failed == 0 + assert report.total_components_ok > 0 + assert report.total_components_checked > 0 + assert report.duration_ms >= 0 + assert len(report.segments) == 1 + assert report.segments[0].segment_id == 1 + assert report.segments[0].components_ok > 0 + assert len(report.segments[0].findings) == 0 + db.close() + + +def test_scrub_detects_corruption(tmp_dir): + db_path = os.path.join(tmp_dir, "scrub_corrupt_db") + db = OverGraph.open(db_path) + for i in range(5): + db.upsert_node("Person", f"node_{i}") + db.flush() + db.close() + + core_path = os.path.join(db_path, "segments", "seg_0001", "segment.core") + data = bytearray(open(core_path, "rb").read()) + data[len(data) // 2] ^= 0xFF + open(core_path, "wb").write(data) + + db = OverGraph.open(db_path) + report = db.scrub() + assert report.total_components_failed > 0 + findings = report.segments[0].findings + assert len(findings) > 0 + assert any(f.finding_type == "PayloadDigestMismatch" for f in findings) + assert findings[0].component_kind != "" + assert findings[0].detail != "" + db.close() + + +def test_scrub_repr(tmp_dir): + db_path = os.path.join(tmp_dir, "scrub_repr_db") + db = OverGraph.open(db_path) + db.upsert_node("Person", "x") + db.flush() + + report = db.scrub() + r = repr(report) + assert "ScrubReport" in r + assert "segments=1" in r + db.close() diff --git a/overgraph-python/tests/test_shortest_path.py b/overgraph-python/tests/test_shortest_path.py index 982a272..31a3172 100644 --- a/overgraph-python/tests/test_shortest_path.py +++ b/overgraph-python/tests/test_shortest_path.py @@ -1,27 +1,27 @@ import pytest -from overgraph import OverGraph, AsyncOverGraph, PyShortestPath +from overgraph import OverGraph, AsyncOverGraph, ShortestPath -def build_chain(db, n=5, edge_type=10): +def build_chain(db, n=5, label="RELATES_TO"): """Create a chain: n0 -> n1 -> ... -> n(n-1). Returns node IDs.""" nodes = [] for i in range(n): - nodes.append(db.upsert_node(1, f"n{i}")) + nodes.append(db.upsert_node("Person", f"n{i}")) for i in range(n - 1): - db.upsert_edge(nodes[i], nodes[i + 1], edge_type) + db.upsert_edge(nodes[i], nodes[i + 1], label) return nodes def build_diamond(db): """A -> B -> D, A -> C -> D.""" - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - d = db.upsert_node(1, "d") - db.upsert_edge(a, b, 10) - db.upsert_edge(a, c, 10) - db.upsert_edge(b, d, 10) - db.upsert_edge(c, d, 10) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + d = db.upsert_node("Person", "d") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(a, c, "RELATES_TO") + db.upsert_edge(b, d, "RELATES_TO") + db.upsert_edge(c, d, "RELATES_TO") return a, b, c, d @@ -43,12 +43,12 @@ def test_multi_hop(self, db): assert result.total_cost == 4.0 def test_disconnected(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") assert db.shortest_path(a, b) is None def test_self_path(self, db): - a = db.upsert_node(1, "a") + a = db.upsert_node("Person", "a") result = db.shortest_path(a, a) assert result is not None assert result.nodes == [a] @@ -56,9 +56,9 @@ def test_self_path(self, db): assert result.total_cost == 0.0 def test_direction_incoming(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO") # outgoing from b to a: no path assert db.shortest_path(b, a) is None # incoming: b has incoming from a, so b<-a should work @@ -67,23 +67,23 @@ def test_direction_incoming(self, db): assert result.nodes == [b, a] def test_direction_both(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO") result = db.shortest_path(b, a, direction="both") assert result is not None assert result.nodes == [b, a] - def test_type_filter(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10) - db.upsert_edge(b, c, 20) - # Only type 10: can't reach c - assert db.shortest_path(a, c, type_filter=[10]) is None + def test_edge_label_filter(self, db): + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(b, c, "WORKS_AT") + # Only Person0: can't reach c + assert db.shortest_path(a, c, edge_label_filter=["RELATES_TO"]) is None # Both types: can reach c - result = db.shortest_path(a, c, type_filter=[10, 20]) + result = db.shortest_path(a, c, edge_label_filter=["RELATES_TO", "WORKS_AT"]) assert result is not None def test_max_depth(self, db): @@ -96,45 +96,45 @@ def test_nonexistent_nodes(self, db): assert db.shortest_path(999998, 999999) is None def test_repr(self, db): - a = db.upsert_node(1, "a") + a = db.upsert_node("Person", "a") result = db.shortest_path(a, a) assert "ShortestPath" in repr(result) class TestShortestPathWeighted: def test_weighted_shortest(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=1.0) - db.upsert_edge(a, c, 10, weight=10.0) - db.upsert_edge(b, c, 10, weight=1.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=1.0) + db.upsert_edge(a, c, "RELATES_TO", weight=10.0) + db.upsert_edge(b, c, "RELATES_TO", weight=1.0) result = db.shortest_path(a, c, weight_field="weight") assert result is not None assert result.nodes == [a, b, c] assert abs(result.total_cost - 2.0) < 1e-6 def test_max_cost(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, weight=5.0) - db.upsert_edge(b, c, 10, weight=5.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", weight=5.0) + db.upsert_edge(b, c, "RELATES_TO", weight=5.0) assert db.shortest_path(a, c, weight_field="weight", max_cost=8.0) is None result = db.shortest_path(a, c, weight_field="weight", max_cost=10.0) assert result is not None def test_max_depth_uses_best_constrained_weighted_path(self, db): - s = db.upsert_node(1, "s") - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - t = db.upsert_node(1, "t") - db.upsert_edge(s, a, 10, weight=1.0) - db.upsert_edge(a, b, 10, weight=1.0) - db.upsert_edge(b, t, 10, weight=1.0) - db.upsert_edge(s, c, 10, weight=3.0) - db.upsert_edge(c, t, 10, weight=3.0) + s = db.upsert_node("Person", "s") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + t = db.upsert_node("Person", "t") + db.upsert_edge(s, a, "RELATES_TO", weight=1.0) + db.upsert_edge(a, b, "RELATES_TO", weight=1.0) + db.upsert_edge(b, t, "RELATES_TO", weight=1.0) + db.upsert_edge(s, c, "RELATES_TO", weight=3.0) + db.upsert_edge(c, t, "RELATES_TO", weight=3.0) result = db.shortest_path(s, t, weight_field="weight", max_depth=2) assert result is not None @@ -148,18 +148,18 @@ def test_connected(self, db): assert db.is_connected(nodes[0], nodes[2]) is True def test_disconnected(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") assert db.is_connected(a, b) is False def test_self(self, db): - a = db.upsert_node(1, "a") + a = db.upsert_node("Person", "a") assert db.is_connected(a, a) is True def test_direction(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO") assert db.is_connected(b, a) is False assert db.is_connected(b, a, direction="incoming") is True assert db.is_connected(b, a, direction="both") is True @@ -169,14 +169,14 @@ def test_max_depth(self, db): assert db.is_connected(nodes[0], nodes[4], max_depth=2) is False assert db.is_connected(nodes[0], nodes[4], max_depth=4) is True - def test_type_filter(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10) - db.upsert_edge(b, c, 20) - assert db.is_connected(a, c, type_filter=[10]) is False - assert db.is_connected(a, c, type_filter=[10, 20]) is True + def test_edge_label_filter(self, db): + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO") + db.upsert_edge(b, c, "WORKS_AT") + assert db.is_connected(a, c, edge_label_filter=["RELATES_TO"]) is False + assert db.is_connected(a, c, edge_label_filter=["RELATES_TO", "WORKS_AT"]) is True class TestAllShortestPaths: @@ -191,8 +191,8 @@ def test_diamond(self, db): assert len(p.edges) == 2 def test_disconnected(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") paths = db.all_shortest_paths(a, b) assert isinstance(paths, list) assert len(paths) == 0 @@ -203,30 +203,30 @@ def test_max_paths(self, db): assert len(paths) == 1 def test_weighted(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - d = db.upsert_node(1, "d") - db.upsert_edge(a, b, 10, weight=1.0) - db.upsert_edge(a, c, 10, weight=1.0) - db.upsert_edge(b, d, 10, weight=1.0) - db.upsert_edge(c, d, 10, weight=1.0) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + d = db.upsert_node("Person", "d") + db.upsert_edge(a, b, "RELATES_TO", weight=1.0) + db.upsert_edge(a, c, "RELATES_TO", weight=1.0) + db.upsert_edge(b, d, "RELATES_TO", weight=1.0) + db.upsert_edge(c, d, "RELATES_TO", weight=1.0) paths = db.all_shortest_paths(a, d, weight_field="weight") assert len(paths) == 2 for p in paths: assert abs(p.total_cost - 2.0) < 1e-6 def test_weighted_max_depth_uses_best_constrained_cost(self, db): - s = db.upsert_node(1, "s") - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - t = db.upsert_node(1, "t") - db.upsert_edge(s, a, 10, weight=1.0) - db.upsert_edge(a, b, 10, weight=1.0) - db.upsert_edge(b, t, 10, weight=1.0) - db.upsert_edge(s, c, 10, weight=3.0) - db.upsert_edge(c, t, 10, weight=3.0) + s = db.upsert_node("Person", "s") + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + t = db.upsert_node("Person", "t") + db.upsert_edge(s, a, "RELATES_TO", weight=1.0) + db.upsert_edge(a, b, "RELATES_TO", weight=1.0) + db.upsert_edge(b, t, "RELATES_TO", weight=1.0) + db.upsert_edge(s, c, "RELATES_TO", weight=3.0) + db.upsert_edge(c, t, "RELATES_TO", weight=3.0) paths = db.all_shortest_paths(s, t, weight_field="weight", max_depth=2) assert len(paths) == 1 @@ -234,7 +234,7 @@ def test_weighted_max_depth_uses_best_constrained_cost(self, db): assert abs(paths[0].total_cost - 6.0) < 1e-6 def test_self_path(self, db): - a = db.upsert_node(1, "a") + a = db.upsert_node("Person", "a") paths = db.all_shortest_paths(a, a) assert len(paths) == 1 assert paths[0].nodes == [a] @@ -243,18 +243,18 @@ def test_self_path(self, db): class TestShortestPathTemporal: def test_at_epoch_excludes_expired(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - c = db.upsert_node(1, "c") - db.upsert_edge(a, b, 10, valid_from=100, valid_to=200) - db.upsert_edge(b, c, 10, valid_from=100, valid_to=200) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + c = db.upsert_node("Person", "c") + db.upsert_edge(a, b, "RELATES_TO", valid_from=100, valid_to=200) + db.upsert_edge(b, c, "RELATES_TO", valid_from=100, valid_to=200) assert db.shortest_path(a, c, at_epoch=150) is not None assert db.shortest_path(a, c, at_epoch=250) is None def test_is_connected_temporal(self, db): - a = db.upsert_node(1, "a") - b = db.upsert_node(1, "b") - db.upsert_edge(a, b, 10, valid_from=100, valid_to=200) + a = db.upsert_node("Person", "a") + b = db.upsert_node("Person", "b") + db.upsert_edge(a, b, "RELATES_TO", valid_from=100, valid_to=200) assert db.is_connected(a, b, at_epoch=150) is True assert db.is_connected(a, b, at_epoch=250) is False @@ -262,36 +262,36 @@ def test_is_connected_temporal(self, db): @pytest.mark.asyncio class TestShortestPathAsync: async def test_shortest_path(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") - c = await async_db.upsert_node(1, "c") - await async_db.upsert_edge(a, b, 10) - await async_db.upsert_edge(b, c, 10) + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") + c = await async_db.upsert_node("Person", "c") + await async_db.upsert_edge(a, b, "RELATES_TO") + await async_db.upsert_edge(b, c, "RELATES_TO") result = await async_db.shortest_path(a, c) assert result is not None assert result.nodes == [a, b, c] assert result.total_cost == 2.0 async def test_disconnected(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") assert await async_db.shortest_path(a, b) is None async def test_is_connected(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") - await async_db.upsert_edge(a, b, 10) + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") + await async_db.upsert_edge(a, b, "RELATES_TO") assert await async_db.is_connected(a, b) is True assert await async_db.is_connected(b, a) is False async def test_all_shortest_paths(self, async_db): - a = await async_db.upsert_node(1, "a") - b = await async_db.upsert_node(1, "b") - c = await async_db.upsert_node(1, "c") - d = await async_db.upsert_node(1, "d") - await async_db.upsert_edge(a, b, 10) - await async_db.upsert_edge(a, c, 10) - await async_db.upsert_edge(b, d, 10) - await async_db.upsert_edge(c, d, 10) + a = await async_db.upsert_node("Person", "a") + b = await async_db.upsert_node("Person", "b") + c = await async_db.upsert_node("Person", "c") + d = await async_db.upsert_node("Person", "d") + await async_db.upsert_edge(a, b, "RELATES_TO") + await async_db.upsert_edge(a, c, "RELATES_TO") + await async_db.upsert_edge(b, d, "RELATES_TO") + await async_db.upsert_edge(c, d, "RELATES_TO") paths = await async_db.all_shortest_paths(a, d) assert len(paths) == 2 diff --git a/overgraph-python/tests/test_transactions.py b/overgraph-python/tests/test_transactions.py index a9aeb3f..8c6694b 100644 --- a/overgraph-python/tests/test_transactions.py +++ b/overgraph-python/tests/test_transactions.py @@ -11,17 +11,17 @@ def test_stage_alias_payload_commits_atomically(db): { "op": "upsert_node", "alias": "alice", - "type_id": 1, + "labels": ["Person"], "key": "alice", "props": {"name": "Alice"}, }, - {"op": "upsert_node", "alias": "bob", "type_id": 1, "key": "bob"}, + {"op": "upsert_node", "alias": "bob", "labels": ["Person"], "key": "bob"}, { "op": "upsert_edge", "alias": "knows", "from": {"local": "alice"}, "to": {"local": "bob"}, - "type_id": 7, + "label": "KNOWS", "props": {"since": 2026}, }, ] @@ -30,6 +30,7 @@ def test_stage_alias_payload_commits_atomically(db): staged_alice = txn.get_node({"local": "alice"}) assert staged_alice["id"] is None assert staged_alice["local"] == "alice" + assert staged_alice["labels"] == ["Person"] assert staged_alice["props"]["name"] == "Alice" staged_edge = txn.get_edge({"local": "knows"}) @@ -51,26 +52,26 @@ def test_stage_alias_payload_commits_atomically(db): def test_builder_aliases_read_own_writes_and_rollback(db): txn = db.begin_write_txn() - alice = txn.upsert_node_as("alice", 1, "alice", props={"mood": "staged"}) - bob = txn.upsert_node_as("bob", 1, "bob") - txn.upsert_edge_as("knows", alice, bob, 9) + alice = txn.upsert_node_as("alice", "Person", "alice", props={"mood": "staged"}) + bob = txn.upsert_node_as("bob", "Person", "bob") + txn.upsert_edge_as("knows", alice, bob, "KNOWS") - assert txn.get_node_by_key(1, "alice")["props"]["mood"] == "staged" + assert txn.get_node_by_key("Person", "alice")["props"]["mood"] == "staged" txn.rollback() - assert db.get_node_by_key(1, "alice") is None + assert db.get_node_by_key("Person", "alice") is None with pytest.raises(OverGraphError, match="transaction is closed"): txn.commit() def test_unaliased_builder_refs_create_and_connect(db): txn = db.begin_write_txn() - alice = txn.upsert_node(1, "alice") - bob = txn.upsert_node(1, "bob") - edge_ref = txn.upsert_edge(alice, bob, 7) + alice = txn.upsert_node("Person", "alice") + bob = txn.upsert_node("Person", "bob") + edge_ref = txn.upsert_edge(alice, bob, "KNOWS") - assert alice == {"type_id": 1, "key": "alice"} - assert bob == {"type_id": 1, "key": "bob"} - assert txn.get_edge(edge_ref)["type_id"] == 7 + assert alice == {"labels": ["Person"], "key": "alice"} + assert bob == {"labels": ["Person"], "key": "bob"} + assert txn.get_edge(edge_ref)["label"] == "KNOWS" result = txn.commit() edge = db.get_edge(result.edge_ids[0]) @@ -78,20 +79,52 @@ def test_unaliased_builder_refs_create_and_connect(db): assert edge.to_id == result.node_ids[1] +def test_transaction_add_remove_node_label(db): + node_id = db.upsert_node("Person", "alice") + + txn = db.begin_write_txn() + assert txn.add_node_label({"id": node_id}, "Admin") is True + assert txn.add_node_label({"id": node_id}, "Admin") is False + assert txn.get_node({"id": node_id})["labels"] == ["Person", "Admin"] + assert txn.remove_node_label({"id": node_id}, "Admin") is True + assert txn.remove_node_label({"id": node_id}, "Admin") is False + txn.commit() + + assert db.get_node(node_id).labels == ["Person"] + + +def test_transaction_node_label_failure_paths(db): + solo = db.upsert_node("Person", "solo") + txn = db.begin_write_txn() + with pytest.raises(OverGraphError, match="last node label"): + txn.remove_node_label({"id": solo}, "Person") + txn.rollback() + + alice = db.upsert_node("Person", "shared") + other = db.upsert_node("Admin", "shared") + conflict_txn = db.begin_write_txn() + with pytest.raises(OverGraphError, match="node key conflict"): + conflict_txn.add_node_label({"id": alice}, "Admin") + conflict_txn.rollback() + + assert db.get_node(alice).labels == ["Person"] + assert db.get_node(other).labels == ["Admin"] + + def test_stage_delete_and_invalidate_operations_read_own_writes(db): a, b, c, d = db.batch_upsert_nodes( [ - {"type_id": 1, "key": "a"}, - {"type_id": 1, "key": "b"}, - {"type_id": 1, "key": "c"}, - {"type_id": 1, "key": "d"}, + {"labels": ["Person"], "key": "a"}, + {"labels": ["Person"], "key": "b"}, + {"labels": ["Person"], "key": "c"}, + {"labels": ["Person"], "key": "d"}, ] ) active_edge, deleted_edge, cascaded_edge = db.batch_upsert_edges( [ - {"from_id": a, "to_id": b, "type_id": 7}, - {"from_id": b, "to_id": c, "type_id": 8}, - {"from_id": c, "to_id": d, "type_id": 9}, + {"from_id": a, "to_id": b, "label": "KNOWS"}, + {"from_id": b, "to_id": c, "label": "LIKES"}, + {"from_id": c, "to_id": d, "label": "FOLLOWS"}, ] ) @@ -124,38 +157,38 @@ def test_rejects_malformed_refs_missing_fields_and_duplicate_aliases(db): missing_field_txn = db.begin_write_txn() with pytest.raises(ValueError, match="upsert_node requires key"): - missing_field_txn.stage([{"op": "upsert_node", "type_id": 1}]) + missing_field_txn.stage([{"op": "upsert_node", "labels": ["Person"]}]) missing_field_txn.rollback() duplicate_txn = db.begin_write_txn() with pytest.raises(OverGraphError, match="duplicate transaction node alias"): duplicate_txn.stage( [ - {"op": "upsert_node", "alias": "n", "type_id": 1, "key": "n1"}, - {"op": "upsert_node", "alias": "n", "type_id": 1, "key": "n2"}, + {"op": "upsert_node", "alias": "n", "labels": ["Person"], "key": "n1"}, + {"op": "upsert_node", "alias": "n", "labels": ["Person"], "key": "n2"}, ] ) duplicate_txn.rollback() def test_conflict_with_implicit_write_closes_transaction(db): - db.upsert_node(1, "base", props={"v": 1}) + db.upsert_node("Person", "base", props={"v": 1}) txn = db.begin_write_txn() - txn.upsert_node(1, "base", props={"v": 2}) - db.upsert_node(1, "base", props={"v": 3}) + txn.upsert_node("Person", "base", props={"v": 2}) + db.upsert_node("Person", "base", props={"v": 3}) with pytest.raises(OverGraphError, match="transaction conflict"): txn.commit() with pytest.raises(OverGraphError, match="transaction is closed"): txn.rollback() - assert db.get_node_by_key(1, "base").props["v"] == 3 + assert db.get_node_by_key("Person", "base").props["v"] == 3 def test_reopen_and_closed_state_behavior(db_path): db = OverGraph.open(db_path) txn = db.begin_write_txn() - txn.stage([{"op": "upsert_node", "alias": "n", "type_id": 1, "key": "n"}]) + txn.stage([{"op": "upsert_node", "alias": "n", "labels": ["Person"], "key": "n"}]) committed = txn.commit() node_id = committed.node_aliases["n"] db.close() @@ -164,9 +197,9 @@ def test_reopen_and_closed_state_behavior(db_path): assert reopened.get_node(node_id).key == "n" commit_txn = reopened.begin_write_txn() - commit_txn.upsert_node_as("m", 1, "m") + commit_txn.upsert_node_as("m", "Person", "m") rollback_txn = reopened.begin_write_txn() - rollback_txn.upsert_node_as("r", 1, "r") + rollback_txn.upsert_node_as("r", "Person", "r") reopened.close() with pytest.raises(OverGraphError, match="[Dd]atabase is closed"): diff --git a/overgraph-python/tests/test_traversal.py b/overgraph-python/tests/test_traversal.py index bfbe787..f050eb5 100644 --- a/overgraph-python/tests/test_traversal.py +++ b/overgraph-python/tests/test_traversal.py @@ -23,13 +23,13 @@ def test_both(self, db): nbrs = db.neighbors(nodes[1], direction="both") assert len(nbrs) == 2 - def test_type_filter(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 20) - nbrs = db.neighbors(n1, direction="outgoing", type_filter=[10]) + def test_edge_label_filter(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "WORKS_AT") + nbrs = db.neighbors(n1, direction="outgoing", edge_label_filter=["RELATES_TO"]) assert len(nbrs) == 1 assert nbrs[0].node_id == n2 @@ -39,38 +39,38 @@ def test_limit(self, db): assert len(nbrs) == 3 def test_empty(self, db): - nid = db.upsert_node(1, "lonely") + nid = db.upsert_node("Person", "lonely") nbrs = db.neighbors(nid, direction="outgoing") assert nbrs == [] def test_invalid_direction(self, db): - nid = db.upsert_node(1, "a") + nid = db.upsert_node("Person", "a") with pytest.raises(ValueError, match="Invalid direction"): db.neighbors(nid, direction="sideways") def test_neighbor_entry_fields(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - eid = db.upsert_edge(n1, n2, 10, weight=2.5) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + eid = db.upsert_edge(n1, n2, "RELATES_TO", weight=2.5) nbrs = db.neighbors(n1, direction="outgoing") assert len(nbrs) == 1 entry = nbrs[0] assert entry.node_id == n2 assert entry.edge_id == eid - assert entry.edge_type_id == 10 + assert entry.label == "RELATES_TO" assert abs(entry.weight - 2.5) < 0.01 assert "NeighborEntry" in repr(entry) class TestTraverse: def test_basic_depth_and_order(self, db): - start = db.upsert_node(1, "start") - depth1_b = db.upsert_node(1, "depth1-b") - depth1_a = db.upsert_node(1, "depth1-a") - depth2 = db.upsert_node(1, "depth2") - db.upsert_edge(start, depth1_b, 10) - db.upsert_edge(start, depth1_a, 10) - db.upsert_edge(depth1_a, depth2, 10) + start = db.upsert_node("Person", "start") + depth1_b = db.upsert_node("Person", "depth1-b") + depth1_a = db.upsert_node("Person", "depth1-a") + depth2 = db.upsert_node("Person", "depth2") + db.upsert_edge(start, depth1_b, "RELATES_TO") + db.upsert_edge(start, depth1_a, "RELATES_TO") + db.upsert_edge(depth1_a, depth2, "RELATES_TO") page = db.traverse(start, 2, min_depth=0, direction="outgoing") @@ -94,11 +94,11 @@ def test_two_hop_window(self, db): assert [(hit.node_id, hit.depth) for hit in page.items] == [(nodes[2], 2)] def test_cursor_constructor(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n2, n3, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n2, n3, "RELATES_TO") page = db.traverse(n1, 2, direction="outgoing", limit=1) assert len(page.items) == 1 assert page.next_cursor is not None @@ -113,39 +113,39 @@ def test_removed_two_hop_apis_stay_absent(self, db): assert not hasattr(db, "neighbors_2hop_constrained_paged") def test_rejects_raw_id_cursor(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n2, n3, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n2, n3, "RELATES_TO") with pytest.raises(TypeError): db.traverse(n1, 2, min_depth=2, direction="outgoing", cursor=123) class TestTraverseFilters: - def test_node_type_filter_is_emission_only(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(2, "b") # type 2 - n3 = db.upsert_node(3, "c") # type 3 - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n2, n3, 10) + def test_node_label_filter_is_emission_only(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Company", "b") + n3 = db.upsert_node("Document", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n2, n3, "RELATES_TO") page = db.traverse( n1, 2, min_depth=2, direction="outgoing", - edge_type_filter=[10], - node_type_filter=[3], + edge_label_filter=["RELATES_TO"], + emit_node_label_filter={"labels": ["Document"], "mode": "all"}, ) assert [(hit.node_id, hit.depth) for hit in page.items] == [(n3, 2)] class TestTopKNeighbors: def test_top_k_by_weight(self, db): - center = db.upsert_node(1, "center") + center = db.upsert_node("Person", "center") for i in range(10): - spoke = db.upsert_node(1, f"s{i}") - db.upsert_edge(center, spoke, 10, weight=float(i)) + spoke = db.upsert_node("Person", f"s{i}") + db.upsert_edge(center, spoke, "RELATES_TO", weight=float(i)) top3 = db.top_k_neighbors(center, 3, direction="outgoing", scoring="weight") assert len(top3) == 3 # Highest weights first @@ -153,45 +153,45 @@ def test_top_k_by_weight(self, db): assert weights == sorted(weights, reverse=True) def test_top_k_by_recency(self, db): - center = db.upsert_node(1, "center") + center = db.upsert_node("Person", "center") for i in range(5): - spoke = db.upsert_node(1, f"s{i}") - db.upsert_edge(center, spoke, 10) + spoke = db.upsert_node("Person", f"s{i}") + db.upsert_edge(center, spoke, "RELATES_TO") top2 = db.top_k_neighbors(center, 2, direction="outgoing", scoring="recency") assert len(top2) == 2 def test_invalid_scoring(self, db): - nid = db.upsert_node(1, "a") + nid = db.upsert_node("Person", "a") with pytest.raises(ValueError, match="Invalid scoring"): db.top_k_neighbors(nid, 3, direction="outgoing", scoring="magic") def test_decay_requires_lambda(self, db): - nid = db.upsert_node(1, "a") + nid = db.upsert_node("Person", "a") with pytest.raises(ValueError, match="decay_lambda"): db.top_k_neighbors(nid, 3, direction="outgoing", scoring="decay") def test_decay_with_lambda(self, db): - center = db.upsert_node(1, "center") + center = db.upsert_node("Person", "center") for i in range(5): - spoke = db.upsert_node(1, f"s{i}") - db.upsert_edge(center, spoke, 10, weight=float(i)) + spoke = db.upsert_node("Person", f"s{i}") + db.upsert_edge(center, spoke, "RELATES_TO", weight=float(i)) top2 = db.top_k_neighbors( center, 2, direction="outgoing", scoring="decay", decay_lambda=0.01 ) assert len(top2) == 2 def test_decay_negative_lambda_rejected(self, db): - nid = db.upsert_node(1, "a") + nid = db.upsert_node("Person", "a") with pytest.raises(ValueError, match="decay_lambda"): db.top_k_neighbors( nid, 3, direction="outgoing", scoring="decay", decay_lambda=-0.5 ) def test_top_k_at_epoch(self, db): - center = db.upsert_node(1, "center") + center = db.upsert_node("Person", "center") for i in range(5): - spoke = db.upsert_node(1, f"s{i}") - db.upsert_edge(center, spoke, 10, weight=float(i)) + spoke = db.upsert_node("Person", f"s{i}") + db.upsert_edge(center, spoke, "RELATES_TO", weight=float(i)) # at_epoch in the far future should still return results import time future_ms = int(time.time() * 1000) + 60_000 @@ -203,11 +203,11 @@ def test_top_k_at_epoch(self, db): class TestNeighborsBatch: def test_basic(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 20) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "WORKS_AT") result = db.neighbors_batch([n1]) assert isinstance(result, dict) assert n1 in result @@ -216,11 +216,11 @@ def test_basic(self, db): assert node_ids == {n2, n3} def test_multiple_nodes(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n2, n3, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n2, n3, "RELATES_TO") result = db.neighbors_batch([n1, n2]) assert n1 in result assert n2 in result @@ -230,20 +230,20 @@ def test_multiple_nodes(self, db): assert result[n2][0].node_id == n3 def test_direction(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - db.upsert_edge(n1, n2, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + db.upsert_edge(n1, n2, "RELATES_TO") result = db.neighbors_batch([n2], direction="incoming") assert n2 in result assert result[n2][0].node_id == n1 - def test_type_filter(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 20) - result = db.neighbors_batch([n1], type_filter=[10]) + def test_edge_label_filter(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "WORKS_AT") + result = db.neighbors_batch([n1], edge_label_filter=["RELATES_TO"]) assert len(result[n1]) == 1 assert result[n1][0].node_id == n2 @@ -252,18 +252,18 @@ def test_empty_input(self, db): assert result == {} def test_no_neighbors(self, db): - lonely = db.upsert_node(1, "lonely") + lonely = db.upsert_node("Person", "lonely") result = db.neighbors_batch([lonely]) # Engine filters empty entries assert lonely not in result def test_matches_individual(self, db): """Batch results match individual neighbors calls.""" - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 10) + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "RELATES_TO") individual = db.neighbors(n1, direction="outgoing") batch = db.neighbors_batch([n1]) @@ -292,13 +292,13 @@ def test_subgraph_repr(self, db): r = repr(sg) assert "Subgraph" in r - def test_subgraph_edge_type_filter(self, db): - n1 = db.upsert_node(1, "a") - n2 = db.upsert_node(1, "b") - n3 = db.upsert_node(1, "c") - db.upsert_edge(n1, n2, 10) - db.upsert_edge(n1, n3, 20) - sg = db.extract_subgraph(n1, 1, edge_type_filter=[10]) + def test_subgraph_edge_label_filter(self, db): + n1 = db.upsert_node("Person", "a") + n2 = db.upsert_node("Person", "b") + n3 = db.upsert_node("Person", "c") + db.upsert_edge(n1, n2, "RELATES_TO") + db.upsert_edge(n1, n3, "WORKS_AT") + sg = db.extract_subgraph(n1, 1, edge_label_filter=["RELATES_TO"]) assert len(sg.edges) == 1 sg_ids = {n.id for n in sg.nodes} assert n2 in sg_ids diff --git a/overgraph-python/tests/test_vector_search.py b/overgraph-python/tests/test_vector_search.py index bb05681..1ce0398 100644 --- a/overgraph-python/tests/test_vector_search.py +++ b/overgraph-python/tests/test_vector_search.py @@ -1,7 +1,7 @@ """Vector search tests for the Python connector (Phase 19f).""" import pytest -from overgraph import OverGraph, PyVectorHit +from overgraph import OverGraph, VectorHit @pytest.fixture @@ -12,19 +12,19 @@ def hybrid_db(tmp_path): dense_vector_metric="cosine", ) # Node 1: dense rank #1, sparse rank #4 - n1 = db.upsert_node(1, "n1", dense_vector=[0.95, 0.05, 0.05, 0.05], + n1 = db.upsert_node("Person", "n1", dense_vector=[0.95, 0.05, 0.05, 0.05], sparse_vector=[(0, 0.2), (1, 0.1)]) # Node 2: dense rank #4, sparse rank #1 - n2 = db.upsert_node(1, "n2", dense_vector=[0.3, 0.5, 0.5, 0.5], + n2 = db.upsert_node("Person", "n2", dense_vector=[0.3, 0.5, 0.5, 0.5], sparse_vector=[(0, 0.9), (1, 0.8), (2, 0.7)]) # Node 3: dense rank #2, sparse rank #2. Balanced. - n3 = db.upsert_node(1, "n3", dense_vector=[0.85, 0.1, 0.1, 0.1], + n3 = db.upsert_node("Person", "n3", dense_vector=[0.85, 0.1, 0.1, 0.1], sparse_vector=[(0, 0.7), (1, 0.6)]) # Node 4: dense rank #3, sparse rank #3 - n4 = db.upsert_node(1, "n4", dense_vector=[0.6, 0.3, 0.3, 0.3], + n4 = db.upsert_node("Person", "n4", dense_vector=[0.6, 0.3, 0.3, 0.3], sparse_vector=[(0, 0.5), (2, 0.3)]) # Node 5: dense rank #5, sparse rank #5 - n5 = db.upsert_node(1, "n5", dense_vector=[0.1, 0.4, 0.6, 0.6], + n5 = db.upsert_node("Person", "n5", dense_vector=[0.1, 0.4, 0.6, 0.6], sparse_vector=[(1, 0.1)]) db.flush() yield db, [n1, n2, n3, n4, n5] @@ -32,6 +32,19 @@ def hybrid_db(tmp_path): class TestDenseSearch: + def test_hydrated_node_records_preserve_vectors(self, hybrid_db): + db, ids = hybrid_db + node = db.get_node(ids[0]) + assert node.dense_vector == pytest.approx([0.95, 0.05, 0.05, 0.05]) + assert [dim for dim, _ in node.sparse_vector] == [0, 1] + assert [weight for _, weight in node.sparse_vector] == pytest.approx([0.2, 0.1]) + + queried = db.query_nodes({"label_filter": {"labels": ["Person"], "mode": "all"}, "keys": ["n1"]}) + assert len(queried.items) == 1 + assert queried.items[0].dense_vector == pytest.approx([0.95, 0.05, 0.05, 0.05]) + assert [dim for dim, _ in queried.items[0].sparse_vector] == [0, 1] + assert [weight for _, weight in queried.items[0].sparse_vector] == pytest.approx([0.2, 0.1]) + def test_returns_sorted_results(self, hybrid_db): db, ids = hybrid_db hits = db.vector_search("dense", 5, dense_query=[1, 0, 0, 0]) @@ -156,12 +169,12 @@ def test_scope_limits_results(self, tmp_path): ids = [] for i in range(4): ids.append(db.upsert_node( - 1, f"n{i}", + "Person", f"n{i}", dense_vector=[1, 0, 0, 0], sparse_vector=[(0, (i + 1) * 0.3)], )) - db.upsert_edge(ids[0], ids[1], 1) - db.upsert_edge(ids[1], ids[2], 1) + db.upsert_edge(ids[0], ids[1], "RELATES_TO") + db.upsert_edge(ids[1], ids[2], "RELATES_TO") db.flush() hits = db.vector_search( diff --git a/src/bin/benchmark_harness.rs b/src/bin/benchmark_harness.rs index c66c4a3..a60f36d 100644 --- a/src/bin/benchmark_harness.rs +++ b/src/bin/benchmark_harness.rs @@ -1,10 +1,10 @@ use overgraph::{ - DatabaseEngine, DbOptions, DegreeOptions, DenseMetric, DenseVectorConfig, EdgeInput, - ExportOptions, HnswConfig, IsConnectedOptions, NeighborOptions, NodeFilterExpr, NodeInput, - NodeQuery, PageRequest, PprOptions, PropValue, PropertyRangeBound, SecondaryIndexKind, - SecondaryIndexRangeDomain, SecondaryIndexState, ShortestPathOptions, TopKOptions, - TraverseOptions, UpsertEdgeOptions, UpsertNodeOptions, VectorSearchMode, VectorSearchRequest, - WalSyncMode, + DatabaseEngine, DbOptions, DegreeOptions, DenseMetric, DenseVectorConfig, EdgeFilterExpr, + EdgeInput, EdgeQuery, ExportOptions, HnswConfig, IsConnectedOptions, LabelMatchMode, + NeighborOptions, NodeFilterExpr, NodeInput, NodeLabelFilter, NodeQuery, PageRequest, + PprOptions, PropValue, PropertyRangeBound, SecondaryIndexKind, SecondaryIndexRangeDomain, + SecondaryIndexState, ShortestPathOptions, TopKOptions, TraverseOptions, UpsertEdgeOptions, + UpsertNodeOptions, VectorSearchMode, VectorSearchRequest, WalSyncMode, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -317,7 +317,7 @@ fn main() -> Result<(), String> { let stats = run_bench_growth(iter_cfg, |i| { engine .upsert_node( - 1, + "BenchNode", &format!("node-{i}"), UpsertNodeOptions { props: idx_props(i), @@ -335,7 +335,7 @@ fn main() -> Result<(), String> { iter_cfg, 1, stats, - json!({"type_id": 1, "with_props": true, "weight": 1.0}), + json!({"label_id": 1, "with_props": true, "weight": 1.0}), scenario_comparability(&scenario_contract, scenario_id), )); } @@ -347,7 +347,7 @@ fn main() -> Result<(), String> { let engine = open_db(&tmp_root.db_path("crud-upsert-edge"))?; let node_inputs: Vec = (0..(iter_cfg.warmup + iter_cfg.iters + 1)) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("e-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -356,7 +356,7 @@ fn main() -> Result<(), String> { }) .collect(); let node_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs) .map_err(|e| e.to_string())?; let stats = run_bench_growth(iter_cfg, |i| { @@ -364,7 +364,7 @@ fn main() -> Result<(), String> { .upsert_edge( node_ids[i], node_ids[i + 1], - 1, + "BenchEdge", UpsertEdgeOptions::default(), ) .map(|_| ()) @@ -378,7 +378,7 @@ fn main() -> Result<(), String> { iter_cfg, 1, stats, - json!({"edge_type_id": 1, "weight": 1.0}), + json!({"edge_label": "BenchEdge", "weight": 1.0}), scenario_comparability(&scenario_contract, scenario_id), )); } @@ -391,7 +391,7 @@ fn main() -> Result<(), String> { let stats = run_bench(iter_cfg, |i| { let inputs: Vec = (0..cfg.batch_nodes) .map(|j| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("bn-{i}-{j}"), props: idx_props(j), weight: 1.0, @@ -399,7 +399,7 @@ fn main() -> Result<(), String> { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).map(|_| ()) + engine.batch_upsert_nodes(inputs).map(|_| ()) })?; engine.close().map_err(|e| e.to_string())?; @@ -410,7 +410,7 @@ fn main() -> Result<(), String> { iter_cfg, cfg.batch_nodes, stats, - json!({"batch_nodes": cfg.batch_nodes, "type_id": 1, "with_props": true}), + json!({"batch_nodes": cfg.batch_nodes, "label_id": 1, "with_props": true}), scenario_comparability(&scenario_contract, scenario_id), )); } @@ -422,7 +422,7 @@ fn main() -> Result<(), String> { let engine = open_db(&tmp_root.db_path("crud-get-node"))?; let node_inputs: Vec = (0..cfg.get_node_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("gn-{i}"), props: idx_props(i), weight: 1.0, @@ -431,7 +431,7 @@ fn main() -> Result<(), String> { }) .collect(); let node_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let stats = run_bench(iter_cfg, |i| { @@ -459,7 +459,7 @@ fn main() -> Result<(), String> { let engine = open_db(&tmp_root.db_path("crud-upsert-node-fixed"))?; engine .upsert_node( - 1, + "BenchNode", "fixed-node", UpsertNodeOptions { props: idx_props(0), @@ -470,7 +470,7 @@ fn main() -> Result<(), String> { let stats = run_bench(iter_cfg, |i| { engine .upsert_node( - 1, + "BenchNode", "fixed-node", UpsertNodeOptions { props: idx_props(i), @@ -488,7 +488,7 @@ fn main() -> Result<(), String> { iter_cfg, 1, stats, - json!({"type_id": 1, "with_props": true, "weight": 1.0, "fixed_key": true}), + json!({"label_id": 1, "with_props": true, "weight": 1.0, "fixed_key": true}), scenario_comparability(&scenario_contract, scenario_id), )); } @@ -502,14 +502,14 @@ fn main() -> Result<(), String> { let engine = DatabaseEngine::open(&tmp_root.db_path("crud-upsert-edge-fixed"), &opts) .map_err(|e| e.to_string())?; let node_a = engine - .upsert_node(1, "fixed-a", UpsertNodeOptions::default()) + .upsert_node("BenchNode", "fixed-a", UpsertNodeOptions::default()) .map_err(|e| e.to_string())?; let node_b = engine - .upsert_node(1, "fixed-b", UpsertNodeOptions::default()) + .upsert_node("BenchNode", "fixed-b", UpsertNodeOptions::default()) .map_err(|e| e.to_string())?; let stats = run_bench(iter_cfg, |_i| { engine - .upsert_edge(node_a, node_b, 1, UpsertEdgeOptions::default()) + .upsert_edge(node_a, node_b, "BenchEdge", UpsertEdgeOptions::default()) .map(|_| ()) })?; engine.close().map_err(|e| e.to_string())?; @@ -521,7 +521,7 @@ fn main() -> Result<(), String> { iter_cfg, 1, stats, - json!({"edge_type_id": 1, "weight": 1.0, "edge_uniqueness": true, "fixed_triple": true}), + json!({"edge_label": "BenchEdge", "weight": 1.0, "edge_uniqueness": true, "fixed_triple": true}), scenario_comparability(&scenario_contract, scenario_id), )); } @@ -532,7 +532,7 @@ fn main() -> Result<(), String> { let iter_cfg = scenario_iterations(&args, &scenario_contract, scenario_id); let engine = open_db(&tmp_root.db_path("trav-neighbors"))?; let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -540,7 +540,7 @@ fn main() -> Result<(), String> { sparse_vector: None, }]; node_inputs.extend((0..cfg.fanout).map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("n-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -548,7 +548,7 @@ fn main() -> Result<(), String> { sparse_vector: None, })); let ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let hub = ids[0]; let edge_inputs: Vec = ids[1..] @@ -556,7 +556,7 @@ fn main() -> Result<(), String> { .map(|&n| EdgeInput { from: hub, to: n, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -564,7 +564,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let stats = run_bench(iter_cfg, |_i| { engine @@ -649,7 +649,7 @@ fn main() -> Result<(), String> { "layout": "memtable", "min_depth": 1, "max_depth": 3, - "node_type_filter": null, + "node_label_filter": null, "branching": [level1, level2, level3] }), scenario_comparability(&scenario_contract, scenario_id), @@ -682,7 +682,7 @@ fn main() -> Result<(), String> { "layout": "segment", "min_depth": 1, "max_depth": 3, - "node_type_filter": null, + "node_label_filter": null, "branching": [level1, level2, level3] }), scenario_comparability(&scenario_contract, scenario_id), @@ -701,7 +701,10 @@ fn main() -> Result<(), String> { root, 3, &TraverseOptions { - node_type_filter: Some(vec![2u32]), + emit_node_label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::Any, + }), ..Default::default() }, ) @@ -721,7 +724,10 @@ fn main() -> Result<(), String> { "layout": "memtable", "min_depth": 1, "max_depth": 3, - "node_type_filter": [2], + "node_label_filter": { + "labels": [bench_node_label(2)], + "mode": "any" + }, "branching": [level1, level2, level3] }), scenario_comparability(&scenario_contract, scenario_id), @@ -741,7 +747,10 @@ fn main() -> Result<(), String> { root, 3, &TraverseOptions { - node_type_filter: Some(vec![2u32]), + emit_node_label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(2)], + mode: LabelMatchMode::Any, + }), ..Default::default() }, ) @@ -761,7 +770,10 @@ fn main() -> Result<(), String> { "layout": "segment", "min_depth": 1, "max_depth": 3, - "node_type_filter": [2], + "node_label_filter": { + "labels": [bench_node_label(2)], + "mode": "any" + }, "branching": [level1, level2, level3] }), scenario_comparability(&scenario_contract, scenario_id), @@ -774,7 +786,7 @@ fn main() -> Result<(), String> { let iter_cfg = scenario_iterations(&args, &scenario_contract, scenario_id); let engine = open_db(&tmp_root.db_path("trav-degree"))?; let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -782,7 +794,7 @@ fn main() -> Result<(), String> { sparse_vector: None, }]; node_inputs.extend((0..cfg.fanout).map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("d-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -790,7 +802,7 @@ fn main() -> Result<(), String> { sparse_vector: None, })); let ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let hub = ids[0]; let edge_inputs: Vec = ids[1..] @@ -798,7 +810,7 @@ fn main() -> Result<(), String> { .map(|&n| EdgeInput { from: hub, to: n, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -806,7 +818,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let stats = run_bench(iter_cfg, |_i| { engine.degree(hub, &DegreeOptions::default()).map(|_| ()) @@ -834,7 +846,7 @@ fn main() -> Result<(), String> { Vec::with_capacity(cfg.batch_nodes * (1 + cfg.fanout)); for h in 0..cfg.batch_nodes { node_inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("hub-{h}"), props: BTreeMap::new(), weight: 1.0, @@ -843,7 +855,7 @@ fn main() -> Result<(), String> { }); for i in 0..cfg.fanout { node_inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("dt-{h}-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -853,7 +865,7 @@ fn main() -> Result<(), String> { } } let all_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let stride = 1 + cfg.fanout; let hub_ids: Vec = (0..cfg.batch_nodes).map(|h| all_ids[h * stride]).collect(); @@ -865,7 +877,7 @@ fn main() -> Result<(), String> { edge_inputs.push(EdgeInput { from: hub, to: spoke, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -874,7 +886,7 @@ fn main() -> Result<(), String> { } } engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let stats = run_bench(iter_cfg, |_i| { engine @@ -903,7 +915,7 @@ fn main() -> Result<(), String> { let node_inputs: Vec = (0..cfg.shortest_path_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("sp-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -912,7 +924,7 @@ fn main() -> Result<(), String> { }) .collect(); let node_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let offset_a = *cfg @@ -932,7 +944,7 @@ fn main() -> Result<(), String> { EdgeInput { from, to: to1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -941,7 +953,7 @@ fn main() -> Result<(), String> { EdgeInput { from, to: to2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -951,7 +963,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let sp_from = node_ids[0]; @@ -988,7 +1000,7 @@ fn main() -> Result<(), String> { let node_inputs: Vec = (0..cfg.shortest_path_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("ic-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -997,7 +1009,7 @@ fn main() -> Result<(), String> { }) .collect(); let node_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let offset_a = *cfg @@ -1017,7 +1029,7 @@ fn main() -> Result<(), String> { EdgeInput { from, to: to1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1026,7 +1038,7 @@ fn main() -> Result<(), String> { EdgeInput { from, to: to2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1036,7 +1048,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let sp_from = node_ids[0]; @@ -1070,7 +1082,7 @@ fn main() -> Result<(), String> { let iter_cfg = scenario_iterations(&args, &scenario_contract, scenario_id); let engine = open_db(&tmp_root.db_path("adv-top-k"))?; let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "hub".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1078,7 +1090,7 @@ fn main() -> Result<(), String> { sparse_vector: None, }]; node_inputs.extend((0..cfg.top_k_candidates).map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("tk-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1086,7 +1098,7 @@ fn main() -> Result<(), String> { sparse_vector: None, })); let ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let hub = ids[0]; let edge_inputs: Vec = ids[1..] @@ -1097,7 +1109,7 @@ fn main() -> Result<(), String> { EdgeInput { from: hub, to: n, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight, valid_from: None, @@ -1106,7 +1118,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let stats = run_bench(iter_cfg, |_i| { @@ -1140,7 +1152,7 @@ fn main() -> Result<(), String> { let engine = open_db(&tmp_root.db_path("adv-time-range"))?; let node_inputs: Vec = (0..cfg.time_range_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("tr-{i}"), props: idx_props(i), weight: 1.0, @@ -1149,13 +1161,14 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let to_ms = now_millis() + cfg.time_range_window_ms; + let label = bench_node_label(1); let stats = run_bench(iter_cfg, |_i| { engine - .find_nodes_by_time_range(1, cfg.time_range_from_ms, to_ms) + .find_nodes_by_time_range(&label, cfg.time_range_from_ms, to_ms) .map(|_| ()) })?; engine.close().map_err(|e| e.to_string())?; @@ -1168,7 +1181,7 @@ fn main() -> Result<(), String> { 1, stats, json!({ - "type_id": 1, + "label_id": 1, "preload_nodes": cfg.time_range_nodes, "from_ms": cfg.time_range_from_ms, "to_ms_window": cfg.time_range_window_ms @@ -1185,7 +1198,7 @@ fn main() -> Result<(), String> { let node_inputs: Vec = (0..cfg.ppr_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("ppr-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1194,7 +1207,7 @@ fn main() -> Result<(), String> { }) .collect(); let node_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let offset_a = *cfg @@ -1214,7 +1227,7 @@ fn main() -> Result<(), String> { EdgeInput { from, to: to1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1223,7 +1236,7 @@ fn main() -> Result<(), String> { EdgeInput { from, to: to2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 0.7, valid_from: None, @@ -1233,7 +1246,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let seeds: Vec = node_ids @@ -1279,7 +1292,7 @@ fn main() -> Result<(), String> { let node_inputs: Vec = (0..cfg.export_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("ex-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1288,7 +1301,7 @@ fn main() -> Result<(), String> { }) .collect(); let node_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let edge_inputs: Vec = (0..cfg.export_edges) @@ -1299,7 +1312,7 @@ fn main() -> Result<(), String> { Some(EdgeInput { from, to, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1311,7 +1324,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; let export_opts = ExportOptions { @@ -1347,7 +1360,7 @@ fn main() -> Result<(), String> { let stats = run_bench(iter_cfg, |i| { let nodes: Vec = (0..cfg.flush_nodes_per_iter) .map(|j| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("fl-{i}-{j}"), props: idx_props(j), weight: 1.0, @@ -1355,7 +1368,7 @@ fn main() -> Result<(), String> { sparse_vector: None, }) .collect(); - let node_ids = engine.batch_upsert_nodes(&nodes)?; + let node_ids = engine.batch_upsert_nodes(nodes.clone())?; let mut edges = Vec::new(); let edge_count = cfg @@ -1365,14 +1378,14 @@ fn main() -> Result<(), String> { edges.push(EdgeInput { from: node_ids[j], to: node_ids[j + 1], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }); } - engine.batch_upsert_edges(&edges)?; + engine.batch_upsert_edges(edges.clone())?; engine.flush().map(|_| ()) })?; engine.close().map_err(|e| e.to_string())?; @@ -1402,7 +1415,7 @@ fn main() -> Result<(), String> { .map(|i| { let seed = 1729u64.wrapping_mul(i as u64 + 1); NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("v-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1416,7 +1429,7 @@ fn main() -> Result<(), String> { }) .collect(); engine - .batch_upsert_nodes(&inputs) + .batch_upsert_nodes(inputs.clone()) .map_err(|e| e.to_string())?; engine.flush().map_err(|e| e.to_string())?; @@ -1432,7 +1445,7 @@ fn main() -> Result<(), String> { dense_query: Some(dense_query), sparse_query: Some(sparse_query), k: cfg.vector_k, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -1609,7 +1622,7 @@ fn build_depth_two_traversal_graph( cfg: &EffectiveConfigResolved, ) -> Result { let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "root".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1618,7 +1631,7 @@ fn build_depth_two_traversal_graph( }]; for i in 0..cfg.two_hop_mid { node_inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("m-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1627,7 +1640,7 @@ fn build_depth_two_traversal_graph( }); for j in 0..cfg.two_hop_leaves_per_mid { node_inputs.push(NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("l-{i}-{j}"), props: BTreeMap::new(), weight: 1.0, @@ -1637,7 +1650,7 @@ fn build_depth_two_traversal_graph( } } let all_ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let root = all_ids[0]; let mid_stride = 1 + cfg.two_hop_leaves_per_mid; @@ -1647,7 +1660,7 @@ fn build_depth_two_traversal_graph( edge_inputs.push(EdgeInput { from: root, to: mid, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1658,7 +1671,7 @@ fn build_depth_two_traversal_graph( edge_inputs.push(EdgeInput { from: mid, to: leaf, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1667,7 +1680,7 @@ fn build_depth_two_traversal_graph( } } engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; Ok(root) } @@ -1678,7 +1691,7 @@ fn build_deep_traversal_graph( ) -> Result<(u64, usize, usize, usize), String> { let (level1, level2, level3) = traverse_deep_branching(fanout); let mut node_inputs = vec![NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: "root".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -1687,7 +1700,7 @@ fn build_deep_traversal_graph( }]; for i in 0..level1 { node_inputs.push(NodeInput { - type_id: 11, + labels: vec![bench_node_label(11)], key: format!("lvl1-{i}"), props: BTreeMap::new(), weight: 1.0, @@ -1698,7 +1711,7 @@ fn build_deep_traversal_graph( for i in 0..level1 { for j in 0..level2 { node_inputs.push(NodeInput { - type_id: if (i + j) % 2 == 0 { 2 } else { 3 }, + labels: vec![bench_node_label(if (i + j) % 2 == 0 { 2 } else { 3 })], key: format!("lvl2-{i}-{j}"), props: BTreeMap::new(), weight: 1.0, @@ -1711,7 +1724,7 @@ fn build_deep_traversal_graph( for j in 0..level2 { for k in 0..level3 { node_inputs.push(NodeInput { - type_id: if (i + j + k) % 2 == 0 { 2 } else { 3 }, + labels: vec![bench_node_label(if (i + j + k) % 2 == 0 { 2 } else { 3 })], key: format!("lvl3-{i}-{j}-{k}"), props: BTreeMap::new(), weight: 1.0, @@ -1722,7 +1735,7 @@ fn build_deep_traversal_graph( } } let ids = engine - .batch_upsert_nodes(&node_inputs) + .batch_upsert_nodes(node_inputs.clone()) .map_err(|e| e.to_string())?; let root = ids[0]; let level1_offset = 1usize; @@ -1734,7 +1747,7 @@ fn build_deep_traversal_graph( edge_inputs.push(EdgeInput { from: root, to: lvl1, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1746,7 +1759,7 @@ fn build_deep_traversal_graph( edge_inputs.push(EdgeInput { from: lvl1, to: lvl2, - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1757,7 +1770,7 @@ fn build_deep_traversal_graph( edge_inputs.push(EdgeInput { from: lvl2, to: ids[level3_offset + lvl3_idx], - type_id: 1, + label: "BenchEdge1".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -1767,7 +1780,7 @@ fn build_deep_traversal_graph( } } engine - .batch_upsert_edges(&edge_inputs) + .batch_upsert_edges(edge_inputs.clone()) .map_err(|e| e.to_string())?; Ok((root, level1, level2, level3)) } @@ -1847,7 +1860,9 @@ fn benchmark_db_options() -> DbOptions { fn open_db(path: &Path) -> Result { let opts = benchmark_db_options(); - DatabaseEngine::open(path, &opts).map_err(|e| e.to_string()) + let engine = DatabaseEngine::open(path, &opts).map_err(|e| e.to_string())?; + seed_bench_label_tokens(&engine)?; + Ok(engine) } fn open_vector_db(path: &Path, dim: u32) -> Result { @@ -1857,18 +1872,55 @@ fn open_vector_db(path: &Path, dim: u32) -> Result { metric: DenseMetric::Cosine, hnsw: HnswConfig::default(), }); - DatabaseEngine::open(path, &opts).map_err(|e| e.to_string()) + let engine = DatabaseEngine::open(path, &opts).map_err(|e| e.to_string())?; + seed_bench_label_tokens(&engine)?; + Ok(engine) +} + +fn seed_bench_label_tokens(engine: &DatabaseEngine) -> Result<(), String> { + for label_id in 1..=256 { + let node_id = engine + .ensure_node_label(&bench_node_label(label_id)) + .map_err(|e| e.to_string())?; + let edge_label_id = engine + .ensure_edge_label(&format!("BenchEdge{label_id}")) + .map_err(|e| e.to_string())?; + if node_id != label_id || edge_label_id != label_id { + return Err(format!( + "benchmark label-token seed drifted for label_id {label_id}: node={node_id}, edge={edge_label_id}" + )); + } + } + Ok(()) +} + +fn bench_node_label(label_id: u32) -> String { + format!("BenchNode{label_id}") } fn query_bench_props(i: usize) -> BTreeMap { let mut props = BTreeMap::new(); props.insert( "status".to_string(), - PropValue::String(if i % 10 == 0 { "active" } else { "inactive" }.to_string()), + PropValue::String( + if i.is_multiple_of(10) { + "active" + } else { + "inactive" + } + .to_string(), + ), ); props.insert( "tier".to_string(), - PropValue::String(if i % 20 == 0 { "gold" } else { "standard" }.to_string()), + PropValue::String( + if i.is_multiple_of(20) { + "gold" + } else { + "standard" + } + .to_string(), + ), ); props.insert("score".to_string(), PropValue::Int((i % 100) as i64)); props @@ -1926,19 +1978,20 @@ fn build_query_benchmark_engine( preload_nodes: usize, ) -> Result<(DatabaseEngine, QueryBenchmarkLayout), String> { let engine = open_db(path)?; + let node_label = bench_node_label(1); let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index(&node_label, "status", SecondaryIndexKind::Equality) .map_err(|e| e.to_string())?; wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready)?; let tier = engine - .ensure_node_property_index(1, "tier", SecondaryIndexKind::Equality) + .ensure_node_property_index(&node_label, "tier", SecondaryIndexKind::Equality) .map_err(|e| e.to_string())?; wait_for_property_index_state(&engine, tier.index_id, SecondaryIndexState::Ready)?; let score = engine .ensure_node_property_index( - 1, + &node_label, "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -1952,7 +2005,7 @@ fn build_query_benchmark_engine( let start = segment * layout.segment_nodes; let inputs: Vec = (start..start + layout.segment_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("q-{i}"), props: query_bench_props(i), weight: 1.0, @@ -1961,7 +2014,7 @@ fn build_query_benchmark_engine( }) .collect(); engine - .batch_upsert_nodes(&inputs) + .batch_upsert_nodes(inputs.clone()) .map_err(|e| e.to_string())?; engine.flush().map_err(|e| e.to_string())?; } @@ -1969,7 +2022,7 @@ fn build_query_benchmark_engine( let tail_start = layout.segments * layout.segment_nodes; let tail_inputs: Vec = (tail_start..tail_start + layout.memtable_tail_nodes) .map(|i| NodeInput { - type_id: 1, + labels: vec![bench_node_label(1)], key: format!("q-{i}"), props: query_bench_props(i), weight: 1.0, @@ -1978,7 +2031,7 @@ fn build_query_benchmark_engine( }) .collect(); engine - .batch_upsert_nodes(&tail_inputs) + .batch_upsert_nodes(tail_inputs.clone()) .map_err(|e| e.to_string())?; Ok((engine, layout)) @@ -1986,7 +2039,10 @@ fn build_query_benchmark_engine( fn query_ids_intersected_request(limit: usize) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -2007,7 +2063,10 @@ fn query_ids_intersected_request(limit: usize) -> NodeQuery { fn query_nodes_hydrated_request(limit: usize) -> NodeQuery { NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { + labels: vec![bench_node_label(1)], + mode: LabelMatchMode::All, + }), filter: filter_and![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -2027,6 +2086,137 @@ fn query_nodes_hydrated_request(limit: usize) -> NodeQuery { } } +struct EdgeBenchmarkLayout { + segments: usize, + segment_edges: usize, + memtable_tail_edges: usize, +} + +struct EdgeBenchmarkFixture { + engine: DatabaseEngine, + layout: EdgeBenchmarkLayout, + source_id: u64, +} + +fn build_edge_query_benchmark_engine( + path: &Path, + preload_edges: usize, +) -> Result { + let engine = open_db(path)?; + let source_count = 1usize; + let target_count = preload_edges.max(1); + let mut nodes = Vec::with_capacity(source_count + target_count); + nodes.extend((0..source_count).map(|i| NodeInput { + labels: vec![bench_node_label(1)], + key: format!("edge-source-{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + })); + nodes.extend((0..target_count).map(|i| NodeInput { + labels: vec![bench_node_label(2)], + key: format!("edge-target-{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + })); + let ids = engine + .batch_upsert_nodes(nodes.clone()) + .map_err(|e| e.to_string())?; + let source_ids = &ids[..source_count]; + let target_ids = &ids[source_count..]; + let source_id = source_ids[0]; + + let segments = if preload_edges >= 2 { 1 } else { 0 }; + let segment_edges = if segments == 0 { + 0 + } else { + (preload_edges / 2).max(1) + }; + let memtable_tail_edges = preload_edges.saturating_sub(segment_edges); + let make_edges = |start: usize, count: usize| -> Vec { + (start..start + count) + .map(|i| { + let mut props = BTreeMap::new(); + props.insert( + "role".to_string(), + PropValue::String(if i % 10 == 0 { "lead" } else { "member" }.to_string()), + ); + EdgeInput { + from: source_ids[i % source_count], + to: target_ids[i % target_ids.len()], + label: "BenchEdge10".to_string(), + props, + weight: if i % 2 == 0 { 2.0 } else { 0.5 }, + valid_from: None, + valid_to: None, + } + }) + .collect() + }; + if segment_edges > 0 { + engine + .batch_upsert_edges(make_edges(0, segment_edges)) + .map_err(|e| e.to_string())?; + engine.flush().map_err(|e| e.to_string())?; + } + if memtable_tail_edges > 0 { + engine + .batch_upsert_edges(make_edges(segment_edges, memtable_tail_edges)) + .map_err(|e| e.to_string())?; + } + + Ok(EdgeBenchmarkFixture { + engine, + layout: EdgeBenchmarkLayout { + segments, + segment_edges, + memtable_tail_edges, + }, + source_id, + }) +} + +fn query_edge_ids_request(source_id: u64, limit: usize) -> EdgeQuery { + EdgeQuery { + label: Some("BenchEdge10".to_string()), + from_ids: vec![source_id], + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(1.0), + upper: None, + }), + page: PageRequest { + limit: Some(limit), + after: None, + }, + ..Default::default() + } +} + +fn query_edges_hydrated_request(source_id: u64, limit: usize) -> EdgeQuery { + EdgeQuery { + label: Some("BenchEdge10".to_string()), + from_ids: vec![source_id], + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::WeightRange { + lower: Some(1.0), + upper: None, + }, + EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("lead".to_string()), + }, + ])), + page: PageRequest { + limit: Some(limit), + after: None, + }, + ..Default::default() + } +} + fn push_query_scenarios( args: &CliArgs, scenario_contract: &ScenarioContract, @@ -2056,7 +2246,7 @@ fn push_query_scenarios( 1, stats, json!({ - "type_id": 1, + "label_id": 1, "preload_nodes": preload_nodes, "segments": layout.segments, "segment_nodes": layout.segment_nodes, @@ -2087,7 +2277,7 @@ fn push_query_scenarios( 1, stats, json!({ - "type_id": 1, + "label_id": 1, "preload_nodes": preload_nodes, "segments": layout.segments, "segment_nodes": layout.segment_nodes, @@ -2099,6 +2289,72 @@ fn push_query_scenarios( )); } + { + let scenario_id = "S-QUERY-003"; + let iter_cfg = scenario_iterations(args, scenario_contract, scenario_id); + let fixture = build_edge_query_benchmark_engine( + &tmp_root.db_path("query-edge-ids-endpoint-metadata"), + preload_nodes, + )?; + let request = query_edge_ids_request(fixture.source_id, limit); + let stats = run_bench(iter_cfg, |_i| { + fixture.engine.query_edge_ids(&request).map(|_| ()) + })?; + fixture.engine.close().map_err(|e| e.to_string())?; + + scenarios.push(make_scenario( + scenario_id, + "query_edge_ids_endpoint_metadata", + "query", + iter_cfg, + 1, + stats, + json!({ + "label_id": 10, + "preload_edges": preload_nodes, + "segments": fixture.layout.segments, + "segment_edges": fixture.layout.segment_edges, + "memtable_tail_edges": fixture.layout.memtable_tail_edges, + "filter": "weight_gte_1", + "limit": limit + }), + scenario_comparability(scenario_contract, scenario_id), + )); + } + + { + let scenario_id = "S-QUERY-004"; + let iter_cfg = scenario_iterations(args, scenario_contract, scenario_id); + let fixture = build_edge_query_benchmark_engine( + &tmp_root.db_path("query-edges-endpoint-property-hydrated"), + preload_nodes, + )?; + let request = query_edges_hydrated_request(fixture.source_id, limit); + let stats = run_bench(iter_cfg, |_i| { + fixture.engine.query_edges(&request).map(|_| ()) + })?; + fixture.engine.close().map_err(|e| e.to_string())?; + + scenarios.push(make_scenario( + scenario_id, + "query_edges_endpoint_property_hydrated", + "query", + iter_cfg, + 1, + stats, + json!({ + "label_id": 10, + "preload_edges": preload_nodes, + "segments": fixture.layout.segments, + "segment_edges": fixture.layout.segment_edges, + "memtable_tail_edges": fixture.layout.memtable_tail_edges, + "filter": "weight_gte_1_and_role_eq_lead", + "limit": limit + }), + scenario_comparability(scenario_contract, scenario_id), + )); + } + Ok(()) } diff --git a/src/bin/inspect.rs b/src/bin/inspect.rs index 8b35744..1e0d469 100644 --- a/src/bin/inspect.rs +++ b/src/bin/inspect.rs @@ -81,7 +81,7 @@ fn inspect_json(db_path: &Path) -> Result<(), Box> { json!({ "max_age_ms": policy.max_age_ms, "max_weight": policy.max_weight, - "type_id": policy.type_id, + "label": policy.label, }), ) }) @@ -187,8 +187,8 @@ fn inspect_text(db_path: &Path) -> Result<(), Box> { if let Some(w) = policy.max_weight { criteria.push(format!("max_weight={}", w)); } - if let Some(t) = policy.type_id { - criteria.push(format!("type_id={}", t)); + if let Some(label) = policy.label.as_deref() { + criteria.push(format!("label={}", label)); } println!(" {}: {}", name, criteria.join(", ")); } diff --git a/src/degree_cache.rs b/src/degree_cache.rs index 98ee7d2..85c2cd5 100644 --- a/src/degree_cache.rs +++ b/src/degree_cache.rs @@ -1,12 +1,17 @@ use crate::engine::DegreeEntry; use crate::error::EngineError; +use crate::segment_components::{ + decode_identity_header, COMPONENT_IDENTITY_HEADER_LEN, COMPONENT_IDENTITY_HEADER_MAGIC, +}; #[cfg(test)] use crate::types::NodeIdBuildHasher; use crate::types::NodeIdMap; use memmap2::Mmap; use std::collections::BinaryHeap; use std::fs::File; -use std::io::{BufWriter, Write}; +#[cfg(test)] +use std::io::BufWriter; +use std::io::Write; use std::path::Path; use std::sync::Arc; @@ -318,6 +323,8 @@ fn degree_overlay_shard_index(node_id: u64) -> usize { pub(crate) struct DegreeSidecar { data: Mmap, + payload_offset: usize, + payload_len: usize, entry_count: usize, block_count: usize, } @@ -332,16 +339,19 @@ impl DegreeSidecar { ))); } let data = unsafe { Mmap::map(&file)? }; - validate_degree_sidecar(&data)?; - let entry_count = read_u64_at(&data, 16)? as usize; - let block_count = read_u64_at(&data, 24)? as usize; + let (payload_offset, payload_len) = degree_payload_range(path, &data)?; + let payload = &data[payload_offset..payload_offset + payload_len]; + let shape = validate_degree_sidecar_shape(payload)?; Ok(Self { data, - entry_count, - block_count, + payload_offset, + payload_len, + entry_count: shape.entry_count, + block_count: shape.block_count, }) } + #[cfg(test)] pub(crate) fn open_optional(path: &Path) -> Option { Self::open(path).ok() } @@ -355,6 +365,7 @@ impl DegreeSidecar { if self.entry_count == 0 || self.block_count == 0 { return DegreeDelta::ZERO; } + let data = self.payload(); let mut lo = 0usize; let mut hi = self.block_count; @@ -371,23 +382,20 @@ impl DegreeSidecar { return DegreeDelta::ZERO; } let block_index = lo - 1; - let start = self.block_entry_start(block_index); - let end = if block_index + 1 < self.block_count { - self.block_entry_start(block_index + 1) - } else { - self.entry_count - }; + let block_size = DEGREE_DELTA_BLOCK_SIZE as usize; + let start = block_index * block_size; + let end = start.saturating_add(block_size).min(self.entry_count); let mut entry_lo = start; let mut entry_hi = end; while entry_lo < entry_hi { let mid = entry_lo + (entry_hi - entry_lo) / 2; - let mid_node = read_sidecar_entry_node_id(&self.data, self.block_count, mid); + let mid_node = read_sidecar_entry_node_id(data, self.block_count, mid); match mid_node.cmp(&node_id) { std::cmp::Ordering::Less => entry_lo = mid + 1, std::cmp::Ordering::Greater => entry_hi = mid, std::cmp::Ordering::Equal => { - return read_sidecar_entry_delta(&self.data, self.block_count, mid); + return read_sidecar_entry_delta(data, self.block_count, mid); } } } @@ -410,22 +418,49 @@ impl DegreeSidecar { return None; } Some(( - read_sidecar_entry_node_id(&self.data, self.block_count, index), - read_sidecar_entry_delta(&self.data, self.block_count, index), + read_sidecar_entry_node_id(self.payload(), self.block_count, index), + read_sidecar_entry_delta(self.payload(), self.block_count, index), )) } + fn payload(&self) -> &[u8] { + &self.data[self.payload_offset..self.payload_offset + self.payload_len] + } + #[inline] fn block_first_node_id(&self, block_index: usize) -> u64 { let offset = HEADER_SIZE + block_index * BLOCK_INDEX_ENTRY_SIZE; - u64::from_le_bytes(self.data[offset..offset + 8].try_into().unwrap()) + let data = self.payload(); + u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap()) } +} - #[inline] - fn block_entry_start(&self, block_index: usize) -> usize { - let offset = HEADER_SIZE + block_index * BLOCK_INDEX_ENTRY_SIZE + 8; - u64::from_le_bytes(self.data[offset..offset + 8].try_into().unwrap()) as usize +fn degree_payload_range(path: &Path, data: &[u8]) -> Result<(usize, usize), EngineError> { + if data.len() >= COMPONENT_IDENTITY_HEADER_LEN + && data[0..COMPONENT_IDENTITY_HEADER_MAGIC.len()] == COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = decode_identity_header(data)?; + let end = header + .payload_offset + .checked_add(header.payload_len) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "degree sidecar payload range overflows for {}", + path.display() + )) + })?; + if end > data.len() as u64 { + return Err(EngineError::CorruptRecord(format!( + "degree sidecar payload range [{}, {}) exceeds file length {} for {}", + header.payload_offset, + end, + data.len(), + path.display() + ))); + } + return Ok((header.payload_offset as usize, header.payload_len as usize)); } + Ok((0, data.len())) } #[cfg(test)] @@ -455,12 +490,26 @@ pub(crate) fn write_degree_delta_sidecar( write_sorted_degree_delta_sidecar_unchecked(path, &coalesced) } +#[cfg(test)] pub(crate) fn write_sorted_degree_delta_sidecar( path: &Path, entries: &[(u64, DegreeDelta)], ) -> Result<(), EngineError> { validate_sorted_degree_delta_entries(entries)?; - write_sorted_degree_delta_sidecar_unchecked(path, entries) + let mut file = File::create(path)?; + write_sorted_degree_delta_sidecar_payload(&mut file, entries)?; + file.sync_all()?; + Ok(()) +} + +pub(crate) fn write_sorted_degree_delta_sidecar_payload( + writer: &mut impl Write, + entries: &[(u64, DegreeDelta)], +) -> Result<(), EngineError> { + validate_sorted_degree_delta_entries(entries)?; + let bytes = encode_sorted_degree_delta_sidecar_unchecked(entries); + writer.write_all(&bytes)?; + Ok(()) } fn validate_sorted_degree_delta_entries(entries: &[(u64, DegreeDelta)]) -> Result<(), EngineError> { @@ -481,10 +530,19 @@ fn validate_sorted_degree_delta_entries(entries: &[(u64, DegreeDelta)]) -> Resul Ok(()) } +#[cfg(test)] fn write_sorted_degree_delta_sidecar_unchecked( path: &Path, entries: &[(u64, DegreeDelta)], ) -> Result<(), EngineError> { + let bytes = encode_sorted_degree_delta_sidecar_unchecked(entries); + let mut file = File::create(path)?; + file.write_all(&bytes)?; + file.sync_all()?; + Ok(()) +} + +fn encode_sorted_degree_delta_sidecar_unchecked(entries: &[(u64, DegreeDelta)]) -> Vec { let entry_count = entries.len(); let block_size = DEGREE_DELTA_BLOCK_SIZE as usize; let block_count = if entry_count == 0 { @@ -524,17 +582,29 @@ fn write_sorted_degree_delta_sidecar_unchecked( let crc = crc32fast::hash(&bytes); bytes.extend_from_slice(&crc.to_le_bytes()); - - let mut file = File::create(path)?; - file.write_all(&bytes)?; - file.sync_all()?; - Ok(()) + bytes } +#[cfg(test)] pub(crate) fn write_folded_degree_delta_sidecar_from_sidecars( path: &Path, sidecars: &[&DegreeSidecar], ) -> Result<(), EngineError> { + let mut writer = BufWriter::new(File::create(path)?); + write_folded_degree_delta_sidecar_payload_from_sidecars(&mut writer, sidecars)?; + writer.flush()?; + writer.get_ref().sync_all()?; + Ok(()) +} + +pub(crate) fn write_folded_degree_delta_sidecar_payload_from_sidecars( + writer: &mut impl Write, + sidecars: &[&DegreeSidecar], +) -> Result<(), EngineError> { + for sidecar in sidecars { + validate_degree_sidecar(sidecar.payload())?; + } + let mut entry_count = 0usize; let mut block_index = Vec::new(); let block_size = DEGREE_DELTA_BLOCK_SIZE as usize; @@ -546,33 +616,24 @@ pub(crate) fn write_folded_degree_delta_sidecar_from_sidecars( Ok(()) })?; - let mut writer = BufWriter::new(File::create(path)?); let mut hasher = crc32fast::Hasher::new(); - write_hashed(&mut writer, &mut hasher, &DEGREE_DELTA_MAGIC)?; + write_hashed(writer, &mut hasher, &DEGREE_DELTA_MAGIC)?; write_hashed( - &mut writer, + writer, &mut hasher, &DEGREE_DELTA_FORMAT_VERSION.to_le_bytes(), )?; + write_hashed(writer, &mut hasher, &DEGREE_DELTA_BLOCK_SIZE.to_le_bytes())?; + write_hashed(writer, &mut hasher, &(entry_count as u64).to_le_bytes())?; write_hashed( - &mut writer, - &mut hasher, - &DEGREE_DELTA_BLOCK_SIZE.to_le_bytes(), - )?; - write_hashed( - &mut writer, - &mut hasher, - &(entry_count as u64).to_le_bytes(), - )?; - write_hashed( - &mut writer, + writer, &mut hasher, &(block_index.len() as u64).to_le_bytes(), )?; for &(first_node_id, entry_start) in &block_index { - write_hashed(&mut writer, &mut hasher, &first_node_id.to_le_bytes())?; - write_hashed(&mut writer, &mut hasher, &entry_start.to_le_bytes())?; + write_hashed(writer, &mut hasher, &first_node_id.to_le_bytes())?; + write_hashed(writer, &mut hasher, &entry_start.to_le_bytes())?; } let mut emitted = 0usize; @@ -583,7 +644,7 @@ pub(crate) fn write_folded_degree_delta_sidecar_from_sidecars( Some(&(node_id, emitted as u64)) ); } - write_degree_delta_entry_hashed(&mut writer, &mut hasher, node_id, delta)?; + write_degree_delta_entry_hashed(writer, &mut hasher, node_id, delta)?; emitted += 1; Ok(()) })?; @@ -591,8 +652,6 @@ pub(crate) fn write_folded_degree_delta_sidecar_from_sidecars( let crc = hasher.finalize(); writer.write_all(&crc.to_le_bytes())?; - writer.flush()?; - writer.get_ref().sync_all()?; Ok(()) } @@ -725,7 +784,12 @@ impl PartialOrd for DegreeSidecarHeapEntry { } } -fn validate_degree_sidecar(data: &[u8]) -> Result<(), EngineError> { +struct DegreeSidecarShape { + entry_count: usize, + block_count: usize, +} + +fn validate_degree_sidecar_shape(data: &[u8]) -> Result { if data.len() < HEADER_SIZE + CRC_SIZE { return Err(EngineError::CorruptRecord(format!( "degree sidecar length {} is smaller than header", @@ -790,6 +854,17 @@ fn validate_degree_sidecar(data: &[u8]) -> Result<(), EngineError> { ))); } + Ok(DegreeSidecarShape { + entry_count, + block_count, + }) +} + +fn validate_degree_sidecar(data: &[u8]) -> Result<(), EngineError> { + let shape = validate_degree_sidecar_shape(data)?; + let entry_count = shape.entry_count; + let block_count = shape.block_count; + let stored_crc = read_u32_at(data, data.len() - CRC_SIZE)?; let actual_crc = crc32fast::hash(&data[..data.len() - CRC_SIZE]); if stored_crc != actual_crc { @@ -1126,7 +1201,7 @@ mod tests { } #[test] - fn sidecar_crc_validation_disables_optional_open() { + fn sidecar_crc_validation_moves_to_compaction_time() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(DEGREE_DELTA_FILENAME); write_degree_delta_sidecar(&path, &[(1, DegreeDelta::add_valid_edge(1, 2, 1.0))]).unwrap(); @@ -1134,7 +1209,57 @@ mod tests { let last = bytes.len() - 1; bytes[last] ^= 0xff; std::fs::write(&path, bytes).unwrap(); - assert!(DegreeSidecar::open(&path).is_err()); - assert!(DegreeSidecar::open_optional(&path).is_none()); + + let sidecar = DegreeSidecar::open(&path).unwrap(); + assert!(DegreeSidecar::open_optional(&path).is_some()); + assert_eq!(sidecar.lookup(1).out_degree, 1); + + let output_path = dir.path().join("folded_degree_delta.dat"); + let err = + write_folded_degree_delta_sidecar_from_sidecars(&output_path, &[&sidecar]).unwrap_err(); + assert!(matches!( + err, + EngineError::CorruptRecord(message) if message.contains("CRC mismatch") + )); + } + + #[test] + fn sidecar_lookup_derives_block_ranges_after_cheap_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(DEGREE_DELTA_FILENAME); + let mut entries = Vec::new(); + for node_id in 1..=600u64 { + entries.push(( + node_id * 2, + DegreeDelta { + out_degree: node_id as i64, + out_weight_sum: node_id as f64, + ..DegreeDelta::ZERO + }, + )); + } + write_degree_delta_sidecar(&path, &entries).unwrap(); + + let mut bytes = std::fs::read(&path).unwrap(); + let second_block_start_offset = HEADER_SIZE + BLOCK_INDEX_ENTRY_SIZE + 8; + bytes[second_block_start_offset..second_block_start_offset + 8] + .copy_from_slice(&u64::MAX.to_le_bytes()); + let crc_offset = bytes.len() - CRC_SIZE; + let crc = crc32fast::hash(&bytes[..crc_offset]); + bytes[crc_offset..crc_offset + CRC_SIZE].copy_from_slice(&crc.to_le_bytes()); + std::fs::write(&path, bytes).unwrap(); + + let sidecar = DegreeSidecar::open(&path).unwrap(); + assert_eq!(sidecar.lookup(600).out_degree, 300); + assert_eq!(sidecar.lookup(1200).out_degree, 600); + + let output_path = dir.path().join("folded_degree_delta.dat"); + let err = + write_folded_degree_delta_sidecar_from_sidecars(&output_path, &[&sidecar]).unwrap_err(); + assert!(matches!( + err, + EngineError::CorruptRecord(message) + if message.contains("block 1 starts") + )); } } diff --git a/src/dense_hnsw.rs b/src/dense_hnsw.rs index eed0d7c..d3765fa 100644 --- a/src/dense_hnsw.rs +++ b/src/dense_hnsw.rs @@ -3,9 +3,7 @@ use crate::parallel::engine_cpu_install; use crate::types::{DenseMetric, DenseVectorConfig, NodeIdSet}; use std::cmp::Ordering; use std::collections::BinaryHeap; -use std::fs::File; -use std::io::{BufWriter, Write}; -use std::path::Path; +use std::io::Write; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::{Mutex, RwLock}; @@ -214,15 +212,28 @@ impl PartialOrd for MaxCandidate { } } -pub(crate) fn write_dense_hnsw_index_from_points( - seg_dir: &Path, - dense_config: Option<&DenseVectorConfig>, +pub(crate) fn build_dense_hnsw_from_points( points: Vec, + config: &DenseVectorConfig, +) -> Result, EngineError> { + let points = dense_point_inputs_to_loaded_points(points); + if points.is_empty() { + return Ok(None); + } + build_hnsw(points, config).map(Some) +} + +pub(crate) fn write_prebuilt_hnsw_to_writers( + meta_w: &mut impl Write, + graph_w: &mut impl Write, + config: &DenseVectorConfig, + built: &BuiltHnsw, ) -> Result<(), EngineError> { - let Some(config) = dense_config else { - return Ok(()); - }; - let points: Vec = points + write_hnsw_payloads(meta_w, graph_w, config, built) +} + +fn dense_point_inputs_to_loaded_points(points: Vec) -> Vec { + points .into_iter() .map(|point| DensePoint { node_id: point.node_id, @@ -230,23 +241,16 @@ pub(crate) fn write_dense_hnsw_index_from_points( norm: dense_vector_norm(&point.values), values: point.values, }) - .collect(); - write_dense_hnsw_index_from_loaded_points(seg_dir, config, points) + .collect() } -fn write_dense_hnsw_index_from_loaded_points( - seg_dir: &Path, - config: &DenseVectorConfig, - points: Vec, -) -> Result<(), EngineError> { - if points.is_empty() { - return Ok(()); - } - - let built = build_hnsw(points, config)?; - write_hnsw_files(seg_dir, config, &built) +fn hnsw_point_count_usize(header: DenseHnswHeader) -> Result { + usize::try_from(header.point_count).map_err(|_| { + EngineError::CorruptRecord("dense HNSW point count exceeds addressable memory".into()) + }) } +#[cfg(test)] pub(crate) fn validate_dense_hnsw_files( meta: &[u8], graph: &[u8], @@ -275,7 +279,8 @@ pub(crate) fn validate_dense_hnsw_files( "dense HNSW metadata has zero points".into(), )); } - if header.point_count as usize != dense_vector_count { + let point_count = hnsw_point_count_usize(header)?; + if point_count != dense_vector_count { return Err(EngineError::CorruptRecord(format!( "dense HNSW point count {} does not match dense vector count {}", header.point_count, dense_vector_count @@ -315,8 +320,11 @@ pub(crate) fn validate_dense_hnsw_files( } } + let point_meta_bytes = point_count + .checked_mul(DENSE_HNSW_POINT_META_SIZE) + .ok_or_else(|| EngineError::CorruptRecord("dense HNSW metadata size overflow".into()))?; let expected_meta_len = DENSE_HNSW_HEADER_SIZE - .checked_add(header.point_count as usize * DENSE_HNSW_POINT_META_SIZE) + .checked_add(point_meta_bytes) .ok_or_else(|| EngineError::CorruptRecord("dense HNSW metadata size overflow".into()))?; if meta.len() != expected_meta_len { return Err(EngineError::CorruptRecord(format!( @@ -341,7 +349,7 @@ pub(crate) fn validate_dense_hnsw_files( let mut prev_node_id = None; let mut expected_level_offset = 0usize; - for index in 0..header.point_count as usize { + for index in 0..point_count { let point = read_point_meta(meta, index)?; if let Some(prev_node_id) = prev_node_id { if point.node_id <= prev_node_id { @@ -353,7 +361,12 @@ pub(crate) fn validate_dense_hnsw_files( } prev_node_id = Some(point.node_id); - let dense_offset = point.dense_vector_offset as usize; + let dense_offset = usize::try_from(point.dense_vector_offset).map_err(|_| { + EngineError::CorruptRecord(format!( + "dense HNSW point {} vector offset exceeds addressable memory", + index + )) + })?; let dense_end = dense_offset .checked_add(dense_vector_bytes) .ok_or_else(|| EngineError::CorruptRecord("dense vector range overflow".into()))?; @@ -422,6 +435,97 @@ pub(crate) fn validate_dense_hnsw_files( Ok(Some(header)) } +pub(crate) fn validate_dense_hnsw_files_for_open( + meta: &[u8], + graph: &[u8], + _dense_blob_len: usize, + dense_vector_count: usize, + dense_config: Option<&DenseVectorConfig>, +) -> Result, EngineError> { + if meta.is_empty() && graph.is_empty() { + return Ok(None); + } + if meta.is_empty() || graph.is_empty() { + return Err(EngineError::CorruptRecord( + "dense HNSW files must appear together".into(), + )); + } + if dense_vector_count == 0 { + return Err(EngineError::CorruptRecord( + "segment has dense HNSW files but no dense vectors".into(), + )); + } + + let header = read_header(meta)?; + if header.point_count == 0 { + return Err(EngineError::CorruptRecord( + "dense HNSW metadata has zero points".into(), + )); + } + let point_count = hnsw_point_count_usize(header)?; + if point_count != dense_vector_count { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW point count {} does not match dense vector count {}", + header.point_count, dense_vector_count + ))); + } + match dense_config { + Some(config) => { + if header.metric != config.metric { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW metric {:?} does not match configured metric {:?}", + header.metric, config.metric + ))); + } + if header.dimension != config.dimension { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW dimension {} does not match configured dimension {}", + header.dimension, config.dimension + ))); + } + if header.m != config.hnsw.m { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW m {} does not match configured m {}", + header.m, config.hnsw.m + ))); + } + if header.ef_construction != config.hnsw.ef_construction { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW ef_construction {} does not match configured ef_construction {}", + header.ef_construction, config.hnsw.ef_construction + ))); + } + } + None => { + return Err(EngineError::CorruptRecord( + "dense HNSW files require DbOptions::dense_vector to be configured".into(), + )); + } + } + + let point_meta_bytes = point_count + .checked_mul(DENSE_HNSW_POINT_META_SIZE) + .ok_or_else(|| EngineError::CorruptRecord("dense HNSW metadata size overflow".into()))?; + let expected_meta_len = DENSE_HNSW_HEADER_SIZE + .checked_add(point_meta_bytes) + .ok_or_else(|| EngineError::CorruptRecord("dense HNSW metadata size overflow".into()))?; + if meta.len() != expected_meta_len { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW metadata size {} does not match expected {}", + meta.len(), + expected_meta_len + ))); + } + if header.entry_point as u64 >= header.point_count { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW entry point {} out of range for {} points", + header.entry_point, header.point_count + ))); + } + + Ok(Some(header)) +} + #[cfg(test)] pub(crate) fn search_dense_hnsw( meta: &[u8], @@ -444,7 +548,7 @@ pub(crate) fn search_dense_hnsw( ))); } - let point_count = header.point_count as usize; + let point_count = hnsw_point_count_usize(header)?; if point_count == 0 { return Ok(Vec::new()); } @@ -462,7 +566,7 @@ pub(crate) fn search_dense_hnsw_with_points( ef_search: usize, limit: usize, ) -> Result, EngineError> { - let point_count = header.point_count as usize; + let point_count = hnsw_point_count_usize(header)?; if point_count == 0 { return Ok(Vec::new()); } @@ -508,6 +612,12 @@ pub(crate) fn search_dense_hnsw_with_points( level_neighbor_span_from_point(points[entry_point], graph, level)?; for neighbor_idx in 0..neighbor_count { let neighbor = read_u32_at(graph, neighbors_start + neighbor_idx * 4)? as usize; + if neighbor >= point_count { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW neighbor {} out of range for {} points", + neighbor, point_count + ))); + } let neighbor_distance = point_distance( points, dense_blob, @@ -555,6 +665,12 @@ pub(crate) fn search_dense_hnsw_with_points( level_neighbor_span_from_point(points[candidate.point], graph, 0)?; for neighbor_idx in 0..neighbor_count { let neighbor = read_u32_at(graph, neighbors_start + neighbor_idx * 4)? as usize; + if neighbor >= point_count { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW neighbor {} out of range for {} points", + neighbor, point_count + ))); + } if is_visited(&visited, neighbor) { continue; } @@ -620,7 +736,7 @@ fn mark_visited(visited: &mut [u64], point: usize) { visited[word] |= mask; } -struct BuiltHnsw { +pub(crate) struct BuiltHnsw { header: DenseHnswHeader, point_metas: Vec, graph: Vec>>, @@ -821,9 +937,15 @@ fn build_hnsw( max_level: level as u16, }); for level_neighbors in &graph[point_metas.len() - 1] { + let neighbor_bytes = u64::try_from(level_neighbors.len()) + .ok() + .and_then(|len| len.checked_mul(4)) + .ok_or_else(|| { + EngineError::CorruptRecord("dense HNSW graph offset overflow".into()) + })?; level_offset = level_offset .checked_add(4) - .and_then(|offset| offset.checked_add(level_neighbors.len() as u64 * 4)) + .and_then(|offset| offset.checked_add(neighbor_bytes)) .ok_or_else(|| { EngineError::CorruptRecord("dense HNSW graph offset overflow".into()) })?; @@ -1293,7 +1415,7 @@ pub(crate) fn search_dense_hnsw_scoped_with_points( limit: usize, scope_ids: &NodeIdSet, ) -> Result, EngineError> { - let point_count = header.point_count as usize; + let point_count = hnsw_point_count_usize(header)?; if point_count == 0 { return Ok(Vec::new()); } @@ -1533,8 +1655,9 @@ pub(crate) fn load_dense_hnsw_query_points( meta: &[u8], header: DenseHnswHeader, ) -> Result, EngineError> { - let mut points = Vec::with_capacity(header.point_count as usize); - for point_idx in 0..header.point_count as usize { + let point_count = hnsw_point_count_usize(header)?; + let mut points = Vec::with_capacity(point_count); + for point_idx in 0..point_count { let point = read_point_meta(meta, point_idx)?; points.push(DenseQueryPoint { node_id: point.node_id, @@ -1661,11 +1784,20 @@ fn level_neighbor_span_from_point( let neighbors_start = cursor.checked_add(4).ok_or_else(|| { EngineError::CorruptRecord("dense HNSW neighbor header overflow".into()) })?; - let neighbors_end = neighbors_start - .checked_add(neighbor_count * 4) - .ok_or_else(|| { - EngineError::CorruptRecord("dense HNSW neighbor range overflow".into()) - })?; + let neighbor_bytes = neighbor_count.checked_mul(4).ok_or_else(|| { + EngineError::CorruptRecord("dense HNSW neighbor bytes overflow".into()) + })?; + let neighbors_end = neighbors_start.checked_add(neighbor_bytes).ok_or_else(|| { + EngineError::CorruptRecord("dense HNSW neighbor range overflow".into()) + })?; + if neighbors_end > graph.len() { + return Err(EngineError::CorruptRecord(format!( + "dense HNSW neighbor range [{}, {}) exceeds graph length {}", + neighbors_start, + neighbors_end, + graph.len() + ))); + } if current_level == level { return Ok((neighbors_start, neighbor_count)); } @@ -1686,51 +1818,43 @@ impl From for PointMeta { } } -fn write_hnsw_files( - seg_dir: &Path, +fn write_hnsw_payloads( + meta_w: &mut impl Write, + graph_w: &mut impl Write, config: &DenseVectorConfig, built: &BuiltHnsw, ) -> Result<(), EngineError> { - let meta_file = File::create(seg_dir.join(DENSE_HNSW_META_FILENAME))?; - let graph_file = File::create(seg_dir.join(DENSE_HNSW_GRAPH_FILENAME))?; - let mut meta_w = BufWriter::new(meta_file); - let mut graph_w = BufWriter::new(graph_file); - meta_w.write_all(&DENSE_HNSW_MAGIC)?; - write_u32(&mut meta_w, DENSE_HNSW_VERSION)?; - write_u64(&mut meta_w, built.header.point_count)?; - write_u32(&mut meta_w, built.header.entry_point)?; - write_u16(&mut meta_w, built.header.max_level)?; - write_u16(&mut meta_w, built.header.m)?; - write_u16(&mut meta_w, built.header.ef_construction)?; - write_u8(&mut meta_w, metric_to_u8(config.metric))?; - write_u8(&mut meta_w, 0)?; - write_u32(&mut meta_w, config.dimension)?; - write_u32(&mut meta_w, 0)?; + write_u32(meta_w, DENSE_HNSW_VERSION)?; + write_u64(meta_w, built.header.point_count)?; + write_u32(meta_w, built.header.entry_point)?; + write_u16(meta_w, built.header.max_level)?; + write_u16(meta_w, built.header.m)?; + write_u16(meta_w, built.header.ef_construction)?; + write_u8(meta_w, metric_to_u8(config.metric))?; + write_u8(meta_w, 0)?; + write_u32(meta_w, config.dimension)?; + write_u32(meta_w, 0)?; for point in &built.point_metas { - write_u64(&mut meta_w, point.node_id)?; - write_u64(&mut meta_w, point.dense_vector_offset)?; - write_u64(&mut meta_w, point.level_offset)?; - write_u16(&mut meta_w, point.max_level)?; - write_u16(&mut meta_w, 0)?; - write_u32(&mut meta_w, 0)?; + write_u64(meta_w, point.node_id)?; + write_u64(meta_w, point.dense_vector_offset)?; + write_u64(meta_w, point.level_offset)?; + write_u16(meta_w, point.max_level)?; + write_u16(meta_w, 0)?; + write_u32(meta_w, 0)?; } for levels in &built.graph { for neighbors in levels { - write_u16(&mut graph_w, neighbors.len() as u16)?; - write_u16(&mut graph_w, 0)?; + write_u16(graph_w, neighbors.len() as u16)?; + write_u16(graph_w, 0)?; for &neighbor in neighbors { - write_u32(&mut graph_w, neighbor as u32)?; + write_u32(graph_w, neighbor as u32)?; } } } - meta_w.flush()?; - meta_w.get_ref().sync_all()?; - graph_w.flush()?; - graph_w.get_ref().sync_all()?; Ok(()) } @@ -1870,12 +1994,11 @@ fn read_u64_at(data: &[u8], offset: usize) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::segment_writer::{ - write_segment_without_degree_sidecar_for_test as write_segment, - NODE_DENSE_VECTOR_BLOB_FILENAME, - }; + use crate::segment_writer::write_segment_without_degree_sidecar_for_test as write_segment; use crate::types::NodeIdSet; - use crate::types::{DenseMetric, HnswConfig, NodeRecord, WalOp, DEFAULT_DENSE_EF_SEARCH}; + use crate::types::{ + DenseMetric, HnswConfig, NodeLabelSet, NodeRecord, WalOp, DEFAULT_DENSE_EF_SEARCH, + }; use crate::{memtable::Memtable, types::DenseVectorConfig}; use std::collections::BTreeMap; use std::env; @@ -1893,7 +2016,7 @@ mod tests { fn node(id: u64, key: &str, dense: Vec) -> NodeRecord { NodeRecord { id, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: key.to_string(), props: BTreeMap::new(), created_at: 1, @@ -1905,6 +2028,50 @@ mod tests { } } + fn read_component_payload(path: &std::path::Path) -> Vec { + let data = fs::read(path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] + == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + return data[start..end].to_vec(); + } + data + } + + fn read_manifest_component_payload( + seg_dir: &std::path::Path, + kind: crate::segment_components::SegmentComponentKind, + ) -> Vec { + let manifest_bytes = + fs::read(seg_dir.join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME)) + .unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap_or_else(|| panic!("missing component {:?}", kind)); + match &record.handle { + crate::segment_components::ComponentHandleV1::ExternalFile { + relative_path, .. + } => read_component_payload(&seg_dir.join(relative_path)), + crate::segment_components::ComponentHandleV1::PackedRange { offset, len, .. } => { + let core = read_component_payload( + &seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME), + ); + let start = *offset as usize; + let end = start + *len as usize; + core[start..end].to_vec() + } + } + } + fn normalize_vector(values: &mut [f32]) { let norm = values.iter().map(|value| value * value).sum::().sqrt(); if norm > 0.0 { @@ -2091,9 +2258,12 @@ mod tests { write_segment(&seg_dir, 1, &mt, Some(&config)).unwrap(); let build_elapsed = build_started.elapsed(); - let meta = fs::read(seg_dir.join(DENSE_HNSW_META_FILENAME)).unwrap(); - let graph = fs::read(seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - let dense_blob = fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(); + let meta = read_component_payload(&seg_dir.join(DENSE_HNSW_META_FILENAME)); + let graph = read_component_payload(&seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)); + let dense_blob = read_manifest_component_payload( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob, + ); let header = read_header(&meta).unwrap(); let points = load_dense_hnsw_query_points(&meta, header).unwrap(); @@ -2246,9 +2416,12 @@ mod tests { write_segment(&seg_dir, 1, &mt, Some(&config)).unwrap(); ( config, - fs::read(seg_dir.join(DENSE_HNSW_META_FILENAME)).unwrap(), - fs::read(seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)).unwrap(), - fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(), + read_component_payload(&seg_dir.join(DENSE_HNSW_META_FILENAME)), + read_component_payload(&seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)), + read_manifest_component_payload( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob, + ), ) } @@ -2308,6 +2481,29 @@ mod tests { } } + #[test] + fn test_validate_dense_hnsw_rejects_metadata_length_overflow() { + let (config, mut meta, graph, dense_blob) = valid_dense_hnsw_files(); + let too_many_points = usize::MAX / DENSE_HNSW_POINT_META_SIZE + 1; + meta[8..16].copy_from_slice(&u64::try_from(too_many_points).unwrap().to_le_bytes()); + + match validate_dense_hnsw_files( + &meta, + &graph, + dense_blob.len(), + too_many_points, + Some(&config), + ) { + Err(EngineError::CorruptRecord(message)) => { + assert!(message.contains("overflow") || message.contains("addressable")); + } + other => panic!( + "expected dense HNSW metadata overflow error, got {:?}", + other + ), + } + } + #[test] fn test_validate_dense_hnsw_rejects_out_of_range_neighbor() { let (config, meta, mut graph, dense_blob) = valid_dense_hnsw_files(); @@ -2383,9 +2579,12 @@ mod tests { let seg_dir = dir.path().join("seg_0001"); write_segment(&seg_dir, 1, &mt, Some(&config)).unwrap(); - let meta = fs::read(seg_dir.join(DENSE_HNSW_META_FILENAME)).unwrap(); - let graph = fs::read(seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - let dense_blob = fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(); + let meta = read_component_payload(&seg_dir.join(DENSE_HNSW_META_FILENAME)); + let graph = read_component_payload(&seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)); + let dense_blob = read_manifest_component_payload( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob, + ); let mut total_recall = 0.0f32; let query_count = 8; @@ -2458,9 +2657,12 @@ mod tests { write_segment(&seg_dir, 1, &mt, Some(&config)).unwrap(); - let meta = fs::read(seg_dir.join(DENSE_HNSW_META_FILENAME)).unwrap(); - let graph = fs::read(seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - let dense_blob = fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(); + let meta = read_component_payload(&seg_dir.join(DENSE_HNSW_META_FILENAME)); + let graph = read_component_payload(&seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)); + let dense_blob = read_manifest_component_payload( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob, + ); let query = vec![1.0, 0.0]; let ann_hits = search_dense_hnsw(&meta, &graph, &dense_blob, &query, 8, 4).unwrap(); @@ -2500,9 +2702,12 @@ mod tests { write_segment(&seg_dir, 1, &mt, Some(&config)).unwrap(); - let meta = fs::read(seg_dir.join(DENSE_HNSW_META_FILENAME)).unwrap(); - let graph = fs::read(seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - let dense_blob = fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(); + let meta = read_component_payload(&seg_dir.join(DENSE_HNSW_META_FILENAME)); + let graph = read_component_payload(&seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)); + let dense_blob = read_manifest_component_payload( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob, + ); let mut total_recall = 0.0f32; for index in [8usize, 24, 32, 48, 64, 96, 128, 160, 192, 208, 224, 240] { @@ -2670,8 +2875,11 @@ mod tests { ); write_segment(&seg_dir, 1, &mt, Some(&config)).unwrap(); - let meta = fs::read(seg_dir.join(DENSE_HNSW_META_FILENAME)).unwrap(); - let dense_blob = fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(); + let meta = read_component_payload(&seg_dir.join(DENSE_HNSW_META_FILENAME)); + let dense_blob = read_manifest_component_payload( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob, + ); let hits = exact_dense_search_above_cutoff( &meta, @@ -2776,9 +2984,12 @@ mod tests { } write_segment(&seg_dir, 1, &mt, Some(&config)).unwrap(); - let meta = fs::read(seg_dir.join(DENSE_HNSW_META_FILENAME)).unwrap(); - let graph_bytes = fs::read(seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - let dense_blob = fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(); + let meta = read_component_payload(&seg_dir.join(DENSE_HNSW_META_FILENAME)); + let graph_bytes = read_component_payload(&seg_dir.join(DENSE_HNSW_GRAPH_FILENAME)); + let dense_blob = read_manifest_component_payload( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob, + ); let query = vec![0.5f32; 8]; let ann_hits = search_dense_hnsw(&meta, &graph_bytes, &dense_blob, &query, 32, 10).unwrap(); diff --git a/src/edge_metadata.rs b/src/edge_metadata.rs new file mode 100644 index 0000000..a16a179 --- /dev/null +++ b/src/edge_metadata.rs @@ -0,0 +1,161 @@ +use crate::types::EdgeRecord; + +pub(crate) const EDGE_WEIGHT_INDEX_LOGICAL_NAME: &str = "edge_weight_index"; +pub(crate) const EDGE_UPDATED_AT_INDEX_LOGICAL_NAME: &str = "edge_updated_at_index"; +pub(crate) const EDGE_VALID_FROM_INDEX_LOGICAL_NAME: &str = "edge_valid_from_index"; +pub(crate) const EDGE_VALID_TO_INDEX_LOGICAL_NAME: &str = "edge_valid_to_index"; + +pub(crate) const EDGE_WEIGHT_INDEX_ENTRY_SIZE: usize = 16; +pub(crate) const EDGE_I64_METADATA_INDEX_ENTRY_SIZE: usize = 20; + +pub(crate) type EdgeWeightIndexEntry = (u32, u32, u64); +pub(crate) type EdgeI64MetadataIndexEntry = (u32, i64, u64); + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct EdgeMetadataIndexEntries { + pub(crate) weight: Vec, + pub(crate) updated_at: Vec, + pub(crate) valid_from: Vec, + pub(crate) valid_to: Vec, +} + +impl EdgeMetadataIndexEntries { + pub(crate) fn with_capacity(edge_count: usize) -> Self { + Self { + weight: Vec::with_capacity(edge_count), + updated_at: Vec::with_capacity(edge_count), + valid_from: Vec::with_capacity(edge_count), + valid_to: Vec::with_capacity(edge_count), + } + } + + pub(crate) fn push( + &mut self, + label_id: u32, + updated_at: i64, + weight: f32, + valid_from: i64, + valid_to: i64, + edge_id: u64, + ) { + if let Some(weight_key) = encode_edge_weight_key(weight) { + self.weight.push((label_id, weight_key, edge_id)); + } + self.updated_at.push((label_id, updated_at, edge_id)); + self.valid_from.push((label_id, valid_from, edge_id)); + self.valid_to.push((label_id, valid_to, edge_id)); + } + + pub(crate) fn sort_all(&mut self) { + self.weight.sort_unstable(); + self.updated_at.sort_unstable(); + self.valid_from.sort_unstable(); + self.valid_to.sort_unstable(); + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct EdgeMetadataCandidate { + pub(crate) edge_id: u64, + pub(crate) from: u64, + pub(crate) to: u64, + pub(crate) label_id: u32, + pub(crate) updated_at: i64, + pub(crate) weight: f32, + pub(crate) valid_from: i64, + pub(crate) valid_to: i64, +} + +impl EdgeMetadataCandidate { + pub(crate) fn from_edge(edge: &EdgeRecord) -> Self { + Self { + edge_id: edge.id, + from: edge.from, + to: edge.to, + label_id: edge.label_id, + updated_at: edge.updated_at, + weight: edge.weight, + valid_from: edge.valid_from, + valid_to: edge.valid_to, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RangeBoundFlags { + pub(crate) lower: Option, + pub(crate) lower_inclusive: bool, + pub(crate) upper: Option, + pub(crate) upper_inclusive: bool, +} + +impl RangeBoundFlags { + pub(crate) fn inclusive(lower: Option, upper: Option) -> Self { + Self { + lower, + lower_inclusive: true, + upper, + upper_inclusive: true, + } + } +} + +pub(crate) fn encode_edge_weight_key(weight: f32) -> Option { + if weight.is_nan() { + return None; + } + let normalized = if weight == 0.0 { 0.0 } else { weight }; + let bits = normalized.to_bits(); + Some(if bits & (1u32 << 31) != 0 { + !bits + } else { + bits ^ (1u32 << 31) + }) +} + +pub(crate) fn i64_matches_bounds(value: i64, bounds: RangeBoundFlags) -> bool { + if let Some(lower) = bounds.lower { + if bounds.lower_inclusive { + if value < lower { + return false; + } + } else if value <= lower { + return false; + } + } + if let Some(upper) = bounds.upper { + if bounds.upper_inclusive { + if value > upper { + return false; + } + } else if value >= upper { + return false; + } + } + true +} + +pub(crate) fn weight_matches_bounds(weight: f32, bounds: RangeBoundFlags) -> bool { + if weight.is_nan() { + return false; + } + if let Some(lower) = bounds.lower { + if bounds.lower_inclusive { + if weight < lower { + return false; + } + } else if weight <= lower { + return false; + } + } + if let Some(upper) = bounds.upper { + if bounds.upper_inclusive { + if weight > upper { + return false; + } + } else if weight >= upper { + return false; + } + } + true +} diff --git a/src/encoding.rs b/src/encoding.rs index 4236dd3..714e7bc 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -144,6 +144,33 @@ fn read_str(cursor: &mut Cursor<&[u8]>) -> Result { String::from_utf8(buf).map_err(|_| EngineError::CorruptRecord("invalid UTF-8 in string".into())) } +fn read_node_label_set(cursor: &mut Cursor<&[u8]>) -> Result { + let count = read_u8(cursor)? as usize; + if count == 0 { + return Err(EngineError::CorruptRecord( + "node WAL label_count must be at least 1".into(), + )); + } + if count > MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::CorruptRecord(format!( + "node WAL label_count {} exceeds maximum {}", + count, MAX_NODE_LABELS_PER_NODE + ))); + } + + let mut ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + for index in 0..count { + ids[index] = read_u32(cursor)?; + if index > 0 && ids[index - 1] >= ids[index] { + return Err(EngineError::CorruptRecord( + "node WAL label IDs must be sorted ascending and unique".into(), + )); + } + } + NodeLabelSet::from_canonical_ids(&ids[..count]) + .map_err(|err| EngineError::CorruptRecord(format!("invalid node WAL label set: {err}"))) +} + // --- Public API --- /// Encode a WalOp into the provided buffer (clears first, reuses allocation). @@ -154,7 +181,10 @@ pub(crate) fn encode_wal_op_into(op: &WalOp, buf: &mut Vec) -> Result<(), En WalOp::UpsertNode(node) => { write_u8(buf, OpTag::UpsertNode as u8); write_u64(buf, node.id); - write_u32(buf, node.type_id); + write_u8(buf, node.label_ids.len() as u8); + for &label_id in node.label_ids.as_slice() { + write_u32(buf, label_id); + } write_str(buf, &node.key)?; write_i64(buf, node.created_at); write_i64(buf, node.updated_at); @@ -188,7 +218,7 @@ pub(crate) fn encode_wal_op_into(op: &WalOp, buf: &mut Vec) -> Result<(), En write_u64(buf, edge.id); write_u64(buf, edge.from); write_u64(buf, edge.to); - write_u32(buf, edge.type_id); + write_u32(buf, edge.label_id); write_i64(buf, edge.created_at); write_i64(buf, edge.updated_at); write_f32(buf, edge.weight); @@ -209,6 +239,32 @@ pub(crate) fn encode_wal_op_into(op: &WalOp, buf: &mut Vec) -> Result<(), En write_u64(buf, *id); write_i64(buf, *deleted_at); } + WalOp::EnsureNodeLabel { label, label_id } => { + write_u8(buf, OpTag::EnsureNodeLabel as u8); + write_str(buf, label)?; + write_u32(buf, *label_id); + } + WalOp::EnsureEdgeLabel { label, label_id } => { + write_u8(buf, OpTag::EnsureEdgeLabel as u8); + write_str(buf, label)?; + write_u32(buf, *label_id); + } + WalOp::BeginAtomicBatch { + first_seq, + op_count, + } => { + write_u8(buf, OpTag::BeginAtomicBatch as u8); + write_u64(buf, *first_seq); + write_u32(buf, *op_count); + } + WalOp::CommitAtomicBatch { + first_seq, + op_count, + } => { + write_u8(buf, OpTag::CommitAtomicBatch as u8); + write_u64(buf, *first_seq); + write_u32(buf, *op_count); + } } Ok(()) @@ -240,7 +296,7 @@ pub(crate) fn decode_wal_op(data: &[u8]) -> Result { match OpTag::from_u8(op_tag) { Some(OpTag::UpsertNode) => { let id = read_u64(&mut cursor)?; - let type_id = read_u32(&mut cursor)?; + let label_ids = read_node_label_set(&mut cursor)?; let key = read_str(&mut cursor)?; let created_at = read_i64(&mut cursor)?; let updated_at = read_i64(&mut cursor)?; @@ -290,7 +346,7 @@ pub(crate) fn decode_wal_op(data: &[u8]) -> Result { Ok(WalOp::UpsertNode(NodeRecord { id, - type_id, + label_ids, key, props, created_at, @@ -305,7 +361,7 @@ pub(crate) fn decode_wal_op(data: &[u8]) -> Result { let id = read_u64(&mut cursor)?; let from = read_u64(&mut cursor)?; let to = read_u64(&mut cursor)?; - let type_id = read_u32(&mut cursor)?; + let label_id = read_u32(&mut cursor)?; let created_at = read_i64(&mut cursor)?; let updated_at = read_i64(&mut cursor)?; let weight = read_f32(&mut cursor)?; @@ -321,7 +377,7 @@ pub(crate) fn decode_wal_op(data: &[u8]) -> Result { id, from, to, - type_id, + label_id, props, created_at, updated_at, @@ -343,6 +399,36 @@ pub(crate) fn decode_wal_op(data: &[u8]) -> Result { reject_trailing_bytes(&cursor, "delete-edge WAL op")?; Ok(WalOp::DeleteEdge { id, deleted_at }) } + Some(OpTag::EnsureNodeLabel) => { + let label = read_str(&mut cursor)?; + let label_id = read_u32(&mut cursor)?; + reject_trailing_bytes(&cursor, "ensure-node-label WAL op")?; + Ok(WalOp::EnsureNodeLabel { label, label_id }) + } + Some(OpTag::EnsureEdgeLabel) => { + let label = read_str(&mut cursor)?; + let label_id = read_u32(&mut cursor)?; + reject_trailing_bytes(&cursor, "ensure-edge-label WAL op")?; + Ok(WalOp::EnsureEdgeLabel { label, label_id }) + } + Some(OpTag::BeginAtomicBatch) => { + let first_seq = read_u64(&mut cursor)?; + let op_count = read_u32(&mut cursor)?; + reject_trailing_bytes(&cursor, "begin-atomic-batch WAL op")?; + Ok(WalOp::BeginAtomicBatch { + first_seq, + op_count, + }) + } + Some(OpTag::CommitAtomicBatch) => { + let first_seq = read_u64(&mut cursor)?; + let op_count = read_u32(&mut cursor)?; + reject_trailing_bytes(&cursor, "commit-atomic-batch WAL op")?; + Ok(WalOp::CommitAtomicBatch { + first_seq, + op_count, + }) + } None => Err(EngineError::CorruptRecord(format!( "unknown op tag: {}", op_tag @@ -363,7 +449,7 @@ mod tests { let op = WalOp::UpsertNode(NodeRecord { id: 42, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "user:alice".to_string(), props, created_at: 1000000, @@ -375,12 +461,14 @@ mod tests { }); let encoded = encode_wal_op(&op).unwrap(); + assert_eq!(encoded[9], 1); + assert_eq!(u32::from_le_bytes(encoded[10..14].try_into().unwrap()), 1); let decoded = decode_wal_op(&encoded).unwrap(); match decoded { WalOp::UpsertNode(node) => { assert_eq!(node.id, 42); - assert_eq!(node.type_id, 1); + assert_eq!(node.label_ids.as_slice(), &[1]); assert_eq!(node.key, "user:alice"); assert_eq!(node.created_at, 1000000); assert_eq!(node.updated_at, 1000001); @@ -399,7 +487,7 @@ mod tests { fn test_roundtrip_upsert_node_with_vectors() { let op = WalOp::UpsertNode(NodeRecord { id: 7, - type_id: 2, + label_ids: NodeLabelSet::single(2).unwrap(), key: "vector-node".to_string(), props: BTreeMap::new(), created_at: 10, @@ -423,7 +511,39 @@ mod tests { } #[test] - fn test_decode_legacy_upsert_node_without_vector_payload() { + fn test_roundtrip_upsert_node_with_multi_label_sets() { + let cases: &[&[u32]] = &[&[2, 5], &[10, 11, 12, 13, 14, 15, 16, 17, 18, 19]]; + + for &label_ids in cases { + let op = WalOp::UpsertNode(NodeRecord { + id: 100 + label_ids.len() as u64, + label_ids: NodeLabelSet::from_canonical_ids(label_ids).unwrap(), + key: format!("multi-label-{}", label_ids.len()), + props: BTreeMap::new(), + created_at: 10, + updated_at: 11, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }); + + let encoded = encode_wal_op(&op).unwrap(); + assert_eq!(encoded[9], label_ids.len() as u8); + let decoded = decode_wal_op(&encoded).unwrap(); + + match decoded { + WalOp::UpsertNode(node) => { + assert_eq!(node.label_ids.as_slice(), label_ids); + assert_eq!(node.key, format!("multi-label-{}", label_ids.len())); + } + _ => panic!("expected UpsertNode"), + } + } + } + + #[test] + fn test_decode_legacy_upsert_node_without_vector_payload_is_rejected() { let mut props = BTreeMap::new(); props.insert("name".to_string(), PropValue::String("legacy".to_string())); @@ -438,14 +558,42 @@ mod tests { let props_bytes = rmp_serde::to_vec(&props).unwrap(); write_bytes(&mut encoded, &props_bytes).unwrap(); - let decoded = decode_wal_op(&encoded).unwrap(); - match decoded { - WalOp::UpsertNode(node) => { - assert_eq!(node.key, "legacy"); - assert!(node.dense_vector.is_none()); - assert!(node.sparse_vector.is_none()); + let err = decode_wal_op(&encoded).unwrap_err(); + assert!( + matches!( + err, + EngineError::CorruptRecord(_) | EngineError::SerializationError(_) + ), + "unexpected legacy WAL error: {err}" + ); + } + + #[test] + fn test_decode_upsert_node_rejects_invalid_label_sets() { + fn encode_with_labels(labels: &[u32]) -> Vec { + let mut encoded = Vec::new(); + write_u8(&mut encoded, OpTag::UpsertNode as u8); + write_u64(&mut encoded, 42); + write_u8(&mut encoded, labels.len() as u8); + for &label_id in labels { + write_u32(&mut encoded, label_id); } - _ => panic!("expected UpsertNode"), + write_str(&mut encoded, "bad").unwrap(); + write_i64(&mut encoded, 1); + write_i64(&mut encoded, 2); + write_f32(&mut encoded, 1.0); + let props_bytes = rmp_serde::to_vec(&BTreeMap::::new()).unwrap(); + write_bytes(&mut encoded, &props_bytes).unwrap(); + write_u8(&mut encoded, 0); + encoded + } + + for labels in [&[][..], &[2, 1][..], &[1, 1][..], &[0][..]] { + let err = decode_wal_op(&encode_with_labels(labels)).unwrap_err(); + assert!( + err.to_string().contains("node WAL"), + "unexpected error for {labels:?}: {err}" + ); } } @@ -458,7 +606,7 @@ mod tests { id: 100, from: 1, to: 2, - type_id: 10, + label_id: 10, props, created_at: 2000000, updated_at: 2000001, @@ -476,7 +624,7 @@ mod tests { assert_eq!(edge.id, 100); assert_eq!(edge.from, 1); assert_eq!(edge.to, 2); - assert_eq!(edge.type_id, 10); + assert_eq!(edge.label_id, 10); assert_eq!(edge.created_at, 2000000); assert_eq!(edge.updated_at, 2000001); assert!((edge.weight - 1.0).abs() < f32::EPSILON); @@ -527,11 +675,84 @@ mod tests { } } + #[test] + fn test_roundtrip_ensure_node_label() { + let op = WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 7, + }; + let encoded = encode_wal_op(&op).unwrap(); + let decoded = decode_wal_op(&encoded).unwrap(); + + match decoded { + WalOp::EnsureNodeLabel { label, label_id } => { + assert_eq!(label, "Person"); + assert_eq!(label_id, 7); + } + _ => panic!("expected EnsureNodeLabel"), + } + } + + #[test] + fn test_roundtrip_ensure_edge_label() { + let op = WalOp::EnsureEdgeLabel { + label: "KNOWS".to_string(), + label_id: 11, + }; + let encoded = encode_wal_op(&op).unwrap(); + let decoded = decode_wal_op(&encoded).unwrap(); + + match decoded { + WalOp::EnsureEdgeLabel { label, label_id } => { + assert_eq!(label, "KNOWS"); + assert_eq!(label_id, 11); + } + _ => panic!("expected EnsureEdgeLabel"), + } + } + + #[test] + fn test_roundtrip_atomic_batch_markers() { + let begin = WalOp::BeginAtomicBatch { + first_seq: 42, + op_count: 3, + }; + let encoded = encode_wal_op(&begin).unwrap(); + let decoded = decode_wal_op(&encoded).unwrap(); + match decoded { + WalOp::BeginAtomicBatch { + first_seq, + op_count, + } => { + assert_eq!(first_seq, 42); + assert_eq!(op_count, 3); + } + _ => panic!("expected BeginAtomicBatch"), + } + + let commit = WalOp::CommitAtomicBatch { + first_seq: 42, + op_count: 3, + }; + let encoded = encode_wal_op(&commit).unwrap(); + let decoded = decode_wal_op(&encoded).unwrap(); + match decoded { + WalOp::CommitAtomicBatch { + first_seq, + op_count, + } => { + assert_eq!(first_seq, 42); + assert_eq!(op_count, 3); + } + _ => panic!("expected CommitAtomicBatch"), + } + } + #[test] fn test_roundtrip_empty_props() { let op = WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "test".to_string(), props: BTreeMap::new(), created_at: 0, @@ -577,7 +798,7 @@ mod tests { let op = WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "test".to_string(), props, created_at: 0, @@ -649,7 +870,7 @@ mod tests { let long_key = "x".repeat(65536); let op = WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: long_key, props: BTreeMap::new(), created_at: 0, @@ -671,7 +892,7 @@ mod tests { let max_key = "x".repeat(u16::MAX as usize); let op = WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: max_key.clone(), props: BTreeMap::new(), created_at: 0, diff --git a/src/engine/graph_ops.rs b/src/engine/graph_ops.rs index 84213db..dee981b 100644 --- a/src/engine/graph_ops.rs +++ b/src/engine/graph_ops.rs @@ -227,6 +227,179 @@ impl UnionFind { impl ReadView { // --- Degree counts + aggregations (Phase 18a) --- + fn resolve_edge_label_filter_for_graph( + &self, + edge_labels: Option<&[String]>, + ) -> Result { + self.label_catalog + .resolve_edge_label_filter_for_read(edge_labels) + } + + fn resolve_node_label_filter_request_for_graph( + &self, + filter: Option<&NodeLabelFilter>, + ) -> Result { + self.label_catalog.resolve_node_label_filter_request(filter) + } + + fn node_label_filter_is_unconstrained(filter: &ResolvedNodeLabelFilter) -> bool { + matches!(filter, ResolvedNodeLabelFilter::Unconstrained) + } + + fn filter_node_ids_by_resolved_label_filter( + &self, + ids: &[u64], + filter: &ResolvedNodeLabelFilter, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result, EngineError> { + if ids.is_empty() { + return Ok(Vec::new()); + } + if matches!(filter, ResolvedNodeLabelFilter::Empty { .. }) { + return Ok(Vec::new()); + } + + let mut sorted_ids = ids.to_vec(); + sorted_ids.sort_unstable(); + sorted_ids.dedup(); + if Self::node_label_filter_is_unconstrained(filter) && policy_cutoffs.is_none() { + return Ok(sorted_ids); + } + + let mut filtered = Vec::with_capacity(sorted_ids.len()); + for chunk in sorted_ids.chunks(QUERY_VERIFY_CHUNK) { + #[cfg(test)] + self.note_node_visibility_meta_reads(chunk.len()); + let visibility = self.sources().find_node_visibility_meta(chunk)?; + for (&node_id, state) in chunk.iter().zip(visibility.iter()) { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + if policy_cutoffs + .is_some_and(|cutoffs| { + cutoffs.excludes_fields(&meta.label_ids, meta.updated_at, meta.weight) + }) + { + continue; + } + if node_label_filter_matches(filter, &meta.label_ids) { + filtered.push(node_id); + } + } + } + Ok(filtered) + } + + fn collect_unconstrained_node_ids_from_sources(&self) -> Result, EngineError> { + let mut node_set = IdSet::default(); + for node_id in self.memtable.visible_node_ids_at(self.snapshot_seq) { + node_set.insert(node_id); + } + for epoch in &self.immutable_epochs { + for node_id in epoch.memtable.visible_node_ids_at(self.snapshot_seq) { + node_set.insert(node_id); + } + } + for segment in &self.segments { + for &node_id in segment.node_ids()? { + node_set.insert(node_id); + } + } + + let mut node_ids: Vec = node_set.into_iter().collect(); + node_ids.sort_unstable(); + Ok(node_ids) + } + + fn collect_node_ids_for_resolved_label_filter( + &self, + filter: &ResolvedNodeLabelFilter, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result, EngineError> { + match filter { + ResolvedNodeLabelFilter::Empty { .. } => Ok(Vec::new()), + ResolvedNodeLabelFilter::Unconstrained => { + if policy_cutoffs.is_some() { + let node_ids = self.collect_unconstrained_node_ids_from_sources()?; + return self.filter_node_ids_by_resolved_label_filter( + &node_ids, + filter, + policy_cutoffs, + ); + } + + let mut label_ids: HashSet = + self.memtable.visible_node_label_ids(self.snapshot_seq).into_iter().collect(); + for epoch in &self.immutable_epochs { + label_ids.extend(epoch.memtable.visible_node_label_ids(self.snapshot_seq)); + } + for seg in &self.segments { + for label_id in seg.node_label_ids()? { + label_ids.insert(label_id); + } + } + + let mut node_set = IdSet::default(); + for label_id in label_ids { + for node_id in self.nodes_by_label_id(label_id)? { + node_set.insert(node_id); + } + } + let mut node_ids: Vec = node_set.into_iter().collect(); + node_ids.sort_unstable(); + Ok(node_ids) + } + ResolvedNodeLabelFilter::LabelSet { + mode, + label_ids, + .. + } => { + let scan_labels: Vec = match mode { + LabelMatchMode::Any => label_ids.as_slice().to_vec(), + LabelMatchMode::All => { + let driver = self + .node_label_filter_estimate(label_ids, LabelMatchMode::All)? + .driver_label_id + .unwrap_or_else(|| label_ids.as_slice()[0]); + vec![driver] + } + }; + let mut node_ids = Vec::new(); + self.scan_raw_node_label_candidates( + &scan_labels, + None, + QUERY_VERIFY_CHUNK, + |chunk| { + #[cfg(test)] + self.note_node_visibility_meta_reads(chunk.len()); + let visibility = self.sources().find_node_visibility_meta(chunk)?; + for (&node_id, state) in chunk.iter().zip(visibility.iter()) { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + if policy_cutoffs + .is_some_and(|cutoffs| { + cutoffs.excludes_fields( + &meta.label_ids, + meta.updated_at, + meta.weight, + ) + }) + { + continue; + } + if node_label_filter_matches(filter, &meta.label_ids) { + node_ids.push(node_id); + } + } + Ok(ControlFlow::Continue(())) + }, + )?; + Ok(node_ids) + } + } + } + pub(crate) fn degree_delta_sum(&self, node_id: u64) -> Option { let mut sum = self.active_degree_overlay.get(node_id); for epoch in &self.immutable_epochs { @@ -238,10 +411,12 @@ impl ReadView { Some(sum) } - fn degree_fast_path_globally_eligible(&self, options: &DegreeOptions) -> bool { - options.type_filter.is_none() - && options.at_epoch.is_none() - && self.manifest.prune_policies.is_empty() + fn degree_fast_path_globally_eligible_resolved( + &self, + label_filter_ids: Option<&[u32]>, + at_epoch: Option, + ) -> bool { + label_filter_ids.is_none() && at_epoch.is_none() && self.manifest.prune_policies.is_empty() } fn degree_sidecars_available(&self) -> bool { @@ -305,13 +480,13 @@ impl ReadView { /// segments. Deduplicates by edge_id, skips tombstoned edges/nodes, /// applies temporal filtering. No prune policy filtering. /// - /// Used as the fallback for type-filtered, temporal, or policy-filtered + /// Used as the fallback for label-filtered, temporal, or policy-filtered /// queries. fn degree_stats_raw_walk( &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, reference_time: i64, ) -> Result<(u64, f64), EngineError> { let (deleted_nodes, deleted_edges) = self.collect_tombstones(); @@ -319,7 +494,7 @@ impl ReadView { self.degree_stats_raw_walk_inner( node_id, direction, - type_filter, + label_filter_ids, reference_time, &deleted_nodes, &deleted_edges, @@ -334,7 +509,7 @@ impl ReadView { &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, reference_time: i64, deleted_nodes: &IdSet, deleted_edges: &IdSet, @@ -350,7 +525,7 @@ impl ReadView { let _ = self.memtable.for_each_adj_entry_at( node_id, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, _neighbor_id, weight, valid_from, valid_to| { seen_edges.insert(edge_id); @@ -368,7 +543,7 @@ impl ReadView { let _ = epoch.memtable.for_each_adj_entry_at( node_id, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { if !seen_edges.insert(edge_id) { @@ -397,7 +572,7 @@ impl ReadView { let _ = seg.for_each_adj_posting( node_id, direction, - type_filter, + label_filter_ids, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { if !seen_edges.insert(edge_id) { return ControlFlow::Continue(()); @@ -429,19 +604,19 @@ impl ReadView { &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, reference_time: i64, ) -> Result<(u64, f64), EngineError> { // No policies → delegate to walk path. Cache acceleration is handled // at the public API layer (degree/sum_edge_weights/avg_edge_weight); // by the time we reach here, the caller has already determined the - // cache cannot be used (type-filtered or temporal query). + // cache cannot be used (label-filtered or temporal query). if self.manifest.prune_policies.is_empty() { - return self.degree_stats_raw_walk(node_id, direction, type_filter, reference_time); + return self.degree_stats_raw_walk(node_id, direction, label_filter_ids, reference_time); } // Policy path: track per-neighbor-id stats so we can subtract excluded ones. - // This avoids materializing Vec while still respecting policies. + // This avoids materializing Vec while still respecting policies. let mut neighbor_stats: IdMap<(u64, f64)> = IdMap::default(); // neighbor_id → (count, weight_sum) let mut total_count: u64 = 0; let mut total_weight: f64 = 0.0; @@ -465,7 +640,7 @@ impl ReadView { let _ = self.memtable.for_each_adj_entry_at( node_id, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { seen_edges.insert(edge_id); @@ -488,7 +663,7 @@ impl ReadView { let _ = epoch.memtable.for_each_adj_entry_at( node_id, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { if !seen_edges.insert(edge_id) { @@ -519,7 +694,7 @@ impl ReadView { let _ = seg.for_each_adj_posting( node_id, direction, - type_filter, + label_filter_ids, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { if !seen_edges.insert(edge_id) { return ControlFlow::Continue(()); @@ -568,7 +743,20 @@ impl ReadView { options: &DegreeOptions, ) -> Result, EngineError> { let direction = options.direction; - if self.degree_fast_path_globally_eligible(options) { + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => { + return Ok(DegreeQueryOutcome { + value: 0, + routes: DegreeQueryRouteTally::default(), + }) + } + }; + if self.degree_fast_path_globally_eligible_resolved(label_filter_ids.as_deref(), options.at_epoch) + { if let Some((count, _)) = self.degree_fast_path_result(node_id, direction) { return Ok(DegreeQueryOutcome { value: count, @@ -577,9 +765,9 @@ impl ReadView { } } - let type_filter = options.type_filter.as_deref(); let reference_time = options.at_epoch.unwrap_or_else(now_millis); - let (count, _) = self.degree_stats(node_id, direction, type_filter, reference_time)?; + let (count, _) = + self.degree_stats(node_id, direction, label_filter_ids.as_deref(), reference_time)?; Ok(DegreeQueryOutcome { value: count, routes: DegreeQueryRouteTally::walk_path(), @@ -592,7 +780,20 @@ impl ReadView { options: &DegreeOptions, ) -> Result, EngineError> { let direction = options.direction; - if self.degree_fast_path_globally_eligible(options) { + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => { + return Ok(DegreeQueryOutcome { + value: 0.0, + routes: DegreeQueryRouteTally::default(), + }) + } + }; + if self.degree_fast_path_globally_eligible_resolved(label_filter_ids.as_deref(), options.at_epoch) + { if let Some((_, weight_sum)) = self.degree_fast_path_result(node_id, direction) { return Ok(DegreeQueryOutcome { value: weight_sum, @@ -601,9 +802,9 @@ impl ReadView { } } - let type_filter = options.type_filter.as_deref(); let reference_time = options.at_epoch.unwrap_or_else(now_millis); - let (_, weight_sum) = self.degree_stats(node_id, direction, type_filter, reference_time)?; + let (_, weight_sum) = + self.degree_stats(node_id, direction, label_filter_ids.as_deref(), reference_time)?; Ok(DegreeQueryOutcome { value: weight_sum, routes: DegreeQueryRouteTally::walk_path(), @@ -616,7 +817,20 @@ impl ReadView { options: &DegreeOptions, ) -> Result>, EngineError> { let direction = options.direction; - if self.degree_fast_path_globally_eligible(options) { + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => { + return Ok(DegreeQueryOutcome { + value: None, + routes: DegreeQueryRouteTally::default(), + }) + } + }; + if self.degree_fast_path_globally_eligible_resolved(label_filter_ids.as_deref(), options.at_epoch) + { if let Some((count, weight_sum)) = self.degree_fast_path_result(node_id, direction) { return Ok(DegreeQueryOutcome { value: if count == 0 { @@ -629,10 +843,9 @@ impl ReadView { } } - let type_filter = options.type_filter.as_deref(); let reference_time = options.at_epoch.unwrap_or_else(now_millis); let (count, weight_sum) = - self.degree_stats(node_id, direction, type_filter, reference_time)?; + self.degree_stats(node_id, direction, label_filter_ids.as_deref(), reference_time)?; Ok(DegreeQueryOutcome { value: if count == 0 { None @@ -649,7 +862,18 @@ impl ReadView { options: &DegreeOptions, ) -> Result>, EngineError> { let direction = options.direction; - let type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => { + return Ok(DegreeQueryOutcome { + value: NodeIdMap::default(), + routes: DegreeQueryRouteTally::default(), + }) + } + }; if node_ids.is_empty() { return Ok(DegreeQueryOutcome { value: NodeIdMap::default(), @@ -657,7 +881,9 @@ impl ReadView { }); } - if self.degree_fast_path_globally_eligible(options) && self.degree_sidecars_available() { + if self.degree_fast_path_globally_eligible_resolved(label_filter_ids.as_deref(), options.at_epoch) + && self.degree_sidecars_available() + { let sorted_ids: Vec = { let mut ids = node_ids.to_vec(); ids.sort_unstable(); @@ -697,7 +923,8 @@ impl ReadView { let reference_time = options.at_epoch.unwrap_or_else(now_millis); if self.manifest.prune_policies.is_empty() { - let value = self.degrees_raw(node_ids, direction, type_filter, reference_time)?; + let value = + self.degrees_raw(node_ids, direction, label_filter_ids.as_deref(), reference_time)?; let mut routes = DegreeQueryRouteTally::default(); routes.add_walk_paths({ let mut ids = node_ids.to_vec(); @@ -707,6 +934,7 @@ impl ReadView { }); return Ok(DegreeQueryOutcome { value, routes }); } + let label_filter_ids = label_filter_ids.as_deref(); // Policy path: single walk tracking both total counts AND per-neighbor // counts so we can subtract excluded neighbors after one batch policy check. @@ -734,7 +962,7 @@ impl ReadView { let _ = self.memtable.for_each_adj_entry_at( nid, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, _weight, valid_from, valid_to| { seen_edges.insert((nid, edge_id)); @@ -760,7 +988,7 @@ impl ReadView { let _ = epoch.memtable.for_each_adj_entry_at( nid, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((nid, edge_id)) { @@ -795,7 +1023,7 @@ impl ReadView { let _ = seg.for_each_adj_posting_batch( &sorted_ids, direction, - type_filter, + label_filter_ids, &mut |queried_nid, edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((queried_nid, edge_id)) { return ControlFlow::Continue(()); @@ -858,7 +1086,7 @@ impl ReadView { &self, node_ids: &[u64], direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, reference_time: i64, ) -> Result, EngineError> { if node_ids.is_empty() { @@ -888,7 +1116,7 @@ impl ReadView { let _ = self.memtable.for_each_adj_entry_at( nid, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, _neighbor_id, _weight, valid_from, valid_to| { seen_edges.insert((nid, edge_id)); @@ -911,7 +1139,7 @@ impl ReadView { let _ = epoch.memtable.for_each_adj_entry_at( nid, direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((nid, edge_id)) { @@ -942,7 +1170,7 @@ impl ReadView { let _ = seg.for_each_adj_posting_batch( &sorted_ids, direction, - type_filter, + label_filter_ids, &mut |queried_nid, edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((queried_nid, edge_id)) { return ControlFlow::Continue(()); @@ -972,7 +1200,7 @@ impl ReadView { /// /// Algorithm auto-selected from `options.weight_field`: /// - `None` → bidirectional BFS (unweighted, hop count) - /// - `Some("weight")` → bidirectional Dijkstra reading `NeighborEntry.weight` + /// - `Some("weight")` → bidirectional Dijkstra reading `NeighborRecord.weight` /// - `Some("")` → bidirectional Dijkstra reading `edge.props[field]` as f64 /// /// Returns `None` if no path exists within the given constraints. @@ -983,7 +1211,13 @@ impl ReadView { options: &ShortestPathOptions, ) -> Result, EngineError> { let direction = options.direction; - let edge_type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let (edge_label_filter, edge_filter_empty) = match resolved_filter { + LabelFilterResolution::Unconstrained => (None, false), + LabelFilterResolution::Known(label_ids) => (Some(label_ids), false), + LabelFilterResolution::EmptyConstraint => (Some(Vec::new()), true), + }; let weight_field = options.weight_field.as_deref(); let at_epoch = options.at_epoch; let max_depth = options.max_depth; @@ -1000,6 +1234,9 @@ impl ReadView { total_cost: 0.0, })); } + if edge_filter_empty { + return Ok(None); + } let reference_time = at_epoch.unwrap_or_else(now_millis); match weight_field { @@ -1007,7 +1244,7 @@ impl ReadView { from, to, direction, - edge_type_filter, + edge_label_filter.as_deref(), reference_time, max_depth, policy_cutoffs.as_ref(), @@ -1016,7 +1253,7 @@ impl ReadView { from, to, direction, - edge_type_filter, + edge_label_filter.as_deref(), wf, reference_time, max_depth, @@ -1037,7 +1274,13 @@ impl ReadView { options: &IsConnectedOptions, ) -> Result { let direction = options.direction; - let edge_type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let (edge_label_filter, edge_filter_empty) = match resolved_filter { + LabelFilterResolution::Unconstrained => (None, false), + LabelFilterResolution::Known(label_ids) => (Some(label_ids), false), + LabelFilterResolution::EmptyConstraint => (Some(Vec::new()), true), + }; let at_epoch = options.at_epoch; let max_depth = options.max_depth; @@ -1048,12 +1291,15 @@ impl ReadView { if from == to { return Ok(true); } + if edge_filter_empty { + return Ok(false); + } let reference_time = at_epoch.unwrap_or_else(now_millis); self.bfs_is_connected( from, to, direction, - edge_type_filter, + edge_label_filter.as_deref(), reference_time, max_depth, policy_cutoffs.as_ref(), @@ -1062,7 +1308,7 @@ impl ReadView { /// Traverse outward from `start` with deterministic BFS ordering. /// - /// Results are emitted in `(depth ASC, node_id ASC)` order. `node_type_filter` + /// Results are emitted in `(depth ASC, node_id ASC)` order. `emit_node_label_filter` /// applies only to emitted hits; traversal still expands through visible nodes /// that do not match the filter. pub fn traverse( @@ -1070,14 +1316,58 @@ impl ReadView { start: u64, max_depth: u32, options: &TraverseOptions, + ) -> Result { + let min_depth = options.min_depth; + let decay_lambda = options.decay_lambda; + if min_depth > max_depth { + return Err(EngineError::InvalidOperation( + "min_depth must be <= max_depth".to_string(), + )); + } + if let Some(lambda) = decay_lambda { + if !lambda.is_finite() || lambda < 0.0 { + return Err(EngineError::InvalidOperation( + "decay_lambda must be finite and non-negative".to_string(), + )); + } + } + let resolved_edge_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let edge_label_filter = match resolved_edge_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => Some(Vec::new()), + }; + let emit_node_label_filter = + self.resolve_node_label_filter_request_for_graph(options.emit_node_label_filter.as_ref())?; + if emit_node_label_filter.is_empty_constraint() { + return Ok(TraversalPageResult { + items: Vec::new(), + next_cursor: None, + }); + } + + self.traverse_resolved( + start, + max_depth, + options, + edge_label_filter.as_deref(), + &emit_node_label_filter, + ) + } + + fn traverse_resolved( + &self, + start: u64, + max_depth: u32, + options: &TraverseOptions, + edge_label_filter: Option<&[u32]>, + emit_node_label_filter: &ResolvedNodeLabelFilter, ) -> Result { let min_depth = options.min_depth; let direction = options.direction; - let edge_type_filter = options.edge_type_filter.as_deref(); - let node_type_filter = options.node_type_filter.as_deref(); let at_epoch = options.at_epoch; let decay_lambda = options.decay_lambda; - let limit = options.limit; let cursor = options.cursor.as_ref(); if min_depth > max_depth { return Err(EngineError::InvalidOperation( @@ -1092,7 +1382,7 @@ impl ReadView { } } - let limit = limit.unwrap_or(usize::MAX); + let limit = options.limit.unwrap_or(usize::MAX); if limit == 0 { return Ok(TraversalPageResult { items: Vec::new(), @@ -1118,17 +1408,21 @@ impl ReadView { }); } - let node_type_filter: Option> = - node_type_filter.map(|types| types.iter().copied().collect()); + if edge_label_filter.is_some_and(|label_ids| label_ids.is_empty()) { + return self.traverse_without_edges( + start, + options, + &start_node, + emit_node_label_filter, + ); + } let mut hits: Vec = Vec::with_capacity(limit.min(64)); let mut last_emitted_cursor: Option = None; let mut has_more = false; if min_depth == 0 && Self::traversal_after_cursor(cursor, 0, start) - && node_type_filter - .as_ref() - .is_none_or(|types| types.contains(&start_node.type_id)) + && node_label_filter_matches(emit_node_label_filter, &start_node.label_ids) { if hits.len() < limit { hits.push(TraversalHit { @@ -1168,7 +1462,7 @@ impl ReadView { let _ = self.expand_frontier( &frontier, direction, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |source, neighbor, edge_id| { @@ -1201,7 +1495,7 @@ impl ReadView { self.classify_traversal_layer( &mut layer_order, - node_type_filter.as_ref(), + emit_node_label_filter, policy_cutoffs.as_ref(), &mut blocked_hidden, &mut visible_nodes, @@ -1244,6 +1538,35 @@ impl ReadView { }) } + fn traverse_without_edges( + &self, + start: u64, + options: &TraverseOptions, + start_node: &NodeRecord, + emit_node_label_filter: &ResolvedNodeLabelFilter, + ) -> Result { + if options.limit == Some(0) + || options.min_depth > 0 + || !Self::traversal_after_cursor(options.cursor.as_ref(), 0, start) + || !node_label_filter_matches(emit_node_label_filter, &start_node.label_ids) + { + return Ok(TraversalPageResult { + items: Vec::new(), + next_cursor: None, + }); + } + + Ok(TraversalPageResult { + items: vec![TraversalHit { + node_id: start, + depth: 0, + via_edge_id: None, + score: Self::traversal_score(options.decay_lambda, 0), + }], + next_cursor: None, + }) + } + /// Reverse a direction for backward search in bidirectional algorithms. fn reverse_direction(direction: Direction) -> Direction { match direction { @@ -1279,7 +1602,7 @@ impl ReadView { fn classify_traversal_layer( &self, layer_order: &mut [u64], - node_type_filter: Option<&HashSet>, + emit_node_label_filter: &ResolvedNodeLabelFilter, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, blocked_hidden: &mut IdSet, visible_nodes: &mut Vec, @@ -1292,23 +1615,32 @@ impl ReadView { } layer_order.sort_unstable(); - if policy_cutoffs.is_none() && node_type_filter.is_none() { + if policy_cutoffs.is_none() + && Self::node_label_filter_is_unconstrained(emit_node_label_filter) + { visible_nodes.extend(layer_order.iter().copied()); emitted_nodes.extend(layer_order.iter().copied()); return Ok(()); } - let nodes = self.get_nodes_raw(layer_order)?; - for (&node_id, slot) in layer_order.iter().zip(nodes.iter()) { - if let Some(node) = slot { - if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { - blocked_hidden.insert(node_id); - continue; - } - visible_nodes.push(node_id); - if node_type_filter.is_none_or(|types| types.contains(&node.type_id)) { - emitted_nodes.push(node_id); - } + #[cfg(test)] + self.note_node_visibility_meta_reads(layer_order.len()); + let visibility = self.sources().find_node_visibility_meta(layer_order)?; + for (&node_id, state) in layer_order.iter().zip(visibility.iter()) { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + if policy_cutoffs + .is_some_and(|cutoffs| { + cutoffs.excludes_fields(&meta.label_ids, meta.updated_at, meta.weight) + }) + { + blocked_hidden.insert(node_id); + continue; + } + visible_nodes.push(node_id); + if node_label_filter_matches(emit_node_label_filter, &meta.label_ids) { + emitted_nodes.push(node_id); } } Ok(()) @@ -1321,7 +1653,7 @@ impl ReadView { &self, frontier: &[u64], direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, tombstones: &mut TraversalTombstoneView<'_>, on_neighbor: &mut F, @@ -1349,7 +1681,7 @@ impl ReadView { .for_each_adj_entry_at( nid, direction, - edge_type_filter, + edge_label_filter, self.snapshot_seq, &mut |edge_id, neighbor_id, _weight, valid_from, valid_to| { seen_edges.insert((nid, edge_id)); @@ -1373,7 +1705,7 @@ impl ReadView { .for_each_adj_entry_at( nid, direction, - edge_type_filter, + edge_label_filter, self.snapshot_seq, &mut |edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((nid, edge_id)) { @@ -1408,7 +1740,7 @@ impl ReadView { .for_each_adj_posting_batch( &sorted_ids, direction, - edge_type_filter, + edge_label_filter, &mut |queried_nid, edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((queried_nid, edge_id)) { return ControlFlow::Continue(()); @@ -1624,7 +1956,7 @@ impl ReadView { dist: &mut IdMap, settled: &mut IdSet, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, limit_cost: f64, @@ -1648,7 +1980,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, tombstones, @@ -1699,7 +2031,7 @@ impl ReadView { settled: &mut [IdSet], layer_best: &mut IdMap>, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: u32, @@ -1731,7 +2063,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, tombstones, @@ -1783,7 +2115,7 @@ impl ReadView { &self, from: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, best_hops: u32, bwd_depth: &IdMap, @@ -1815,7 +2147,7 @@ impl ReadView { let _ = self.expand_frontier( &frontier, direction, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |source, neighbor, edge_id| { @@ -1897,7 +2229,7 @@ impl ReadView { &self, node: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, @@ -1914,7 +2246,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, tombstones, @@ -1955,7 +2287,7 @@ impl ReadView { &self, fwd_dist: &IdMap, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, best_cost: f64, @@ -1980,7 +2312,7 @@ impl ReadView { let next_steps = self.collect_weighted_successors_from_node( node, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, policy_cutoffs, @@ -2014,7 +2346,7 @@ impl ReadView { &self, fwd_dist: &[IdMap], direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: u32, @@ -2046,7 +2378,7 @@ impl ReadView { let next_steps = self.collect_weighted_successors_from_node( node, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, policy_cutoffs, @@ -2084,7 +2416,7 @@ impl ReadView { node: u64, depth_so_far: u32, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, best_hops: u32, bwd_depth: &IdMap, @@ -2100,7 +2432,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, tombstones, @@ -2126,7 +2458,7 @@ impl ReadView { cache: &mut IdMap>, node: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, best_cost: f64, @@ -2147,7 +2479,7 @@ impl ReadView { self.collect_weighted_successors_from_node( node, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, policy_cutoffs, @@ -2177,7 +2509,7 @@ impl ReadView { node: u64, hops_so_far: u32, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: u32, @@ -2203,7 +2535,7 @@ impl ReadView { self.collect_weighted_successors_from_node( node, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, policy_cutoffs, @@ -2231,14 +2563,14 @@ impl ReadView { /// Bidirectional BFS shortest path (unweighted). /// /// Alternates expanding the smaller frontier. Uses callback-based - /// adjacency iteration to avoid materializing `Vec`. + /// adjacency iteration to avoid materializing `Vec`. #[allow(clippy::too_many_arguments)] fn bfs_shortest_path( &self, from: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, max_depth: Option, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, @@ -2284,7 +2616,7 @@ impl ReadView { let _ = self.expand_frontier( &fwd_frontier, direction, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |source, neighbor, edge_id| { @@ -2345,7 +2677,7 @@ impl ReadView { let _ = self.expand_frontier( &bwd_frontier, bwd_direction, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |source, neighbor, edge_id| { @@ -2411,7 +2743,7 @@ impl ReadView { from: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, max_depth: Option, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, @@ -2452,7 +2784,7 @@ impl ReadView { let _ = self.expand_frontier( &fwd_frontier, direction, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |_source, neighbor, _edge_id| { @@ -2505,7 +2837,7 @@ impl ReadView { let _ = self.expand_frontier( &bwd_frontier, bwd_direction, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |_source, neighbor, _edge_id| { @@ -2624,7 +2956,7 @@ impl ReadView { &self, node_id: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, tombstones: &mut TraversalTombstoneView<'_>, @@ -2642,7 +2974,7 @@ impl ReadView { .for_each_adj_entry_at( node_id, direction, - edge_type_filter, + edge_label_filter, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { scratch.seen_edges.insert(edge_id); @@ -2683,7 +3015,7 @@ impl ReadView { .for_each_adj_entry_at( node_id, direction, - edge_type_filter, + edge_label_filter, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { if !scratch.seen_edges.insert(edge_id) { @@ -2731,7 +3063,7 @@ impl ReadView { .for_each_adj_posting( node_id, direction, - edge_type_filter, + edge_label_filter, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { if !scratch.seen_edges.insert(edge_id) { return ControlFlow::Continue(()); @@ -2786,7 +3118,7 @@ impl ReadView { from: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: Option, @@ -2798,7 +3130,7 @@ impl ReadView { from, to, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, max_hops, @@ -2871,7 +3203,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -2942,7 +3274,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, bwd_direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -3017,7 +3349,7 @@ impl ReadView { from: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: u32, @@ -3103,7 +3435,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -3191,7 +3523,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, bwd_direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -3290,7 +3622,13 @@ impl ReadView { options: &AllShortestPathsOptions, ) -> Result, EngineError> { let direction = options.direction; - let edge_type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let (edge_label_filter, edge_filter_empty) = match resolved_filter { + LabelFilterResolution::Unconstrained => (None, false), + LabelFilterResolution::Known(label_ids) => (Some(label_ids), false), + LabelFilterResolution::EmptyConstraint => (Some(Vec::new()), true), + }; let weight_field = options.weight_field.as_deref(); let at_epoch = options.at_epoch; let max_depth = options.max_depth; @@ -3308,6 +3646,9 @@ impl ReadView { total_cost: 0.0, }]); } + if edge_filter_empty { + return Ok(Vec::new()); + } let reference_time = at_epoch.unwrap_or_else(now_millis); let paths_cap = max_paths.unwrap_or(100); @@ -3316,7 +3657,7 @@ impl ReadView { from, to, direction, - edge_type_filter, + edge_label_filter.as_deref(), reference_time, max_depth, paths_cap, @@ -3326,7 +3667,7 @@ impl ReadView { from, to, direction, - edge_type_filter, + edge_label_filter.as_deref(), wf, reference_time, max_depth, @@ -3344,7 +3685,7 @@ impl ReadView { from: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, max_depth: Option, max_paths: usize, @@ -3417,7 +3758,7 @@ impl ReadView { } else { bwd_direction }, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |_source, neighbor, _edge_id| { @@ -3495,7 +3836,7 @@ impl ReadView { let _ = self.expand_frontier( &bwd_frontier, bwd_direction, - edge_type_filter, + edge_label_filter, reference_time, &mut tombstones, &mut |_source, neighbor, _edge_id| { @@ -3536,7 +3877,7 @@ impl ReadView { let successors = self.build_bfs_shortest_path_successors( from, direction, - edge_type_filter, + edge_label_filter, reference_time, best_hops, &bwd_depth, @@ -3562,7 +3903,7 @@ impl ReadView { to, 0, direction, - edge_type_filter, + edge_label_filter, reference_time, best_hops, &bwd_depth, @@ -3586,7 +3927,7 @@ impl ReadView { from: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: Option, @@ -3599,7 +3940,7 @@ impl ReadView { from, to, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, max_hops, @@ -3668,7 +4009,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -3736,7 +4077,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, bwd_direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -3803,7 +4144,7 @@ impl ReadView { &mut bwd_dist, &mut bwd_settled, bwd_direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, mu, @@ -3817,7 +4158,7 @@ impl ReadView { &mut fwd_dist, &mut fwd_settled, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, mu, @@ -3841,7 +4182,7 @@ impl ReadView { let successors = self.build_weighted_shortest_path_successors( &fwd_dist, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, mu, @@ -3865,7 +4206,7 @@ impl ReadView { from, to, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, mu, @@ -3892,7 +4233,7 @@ impl ReadView { from: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: u32, @@ -3980,7 +4321,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -4070,7 +4411,7 @@ impl ReadView { let _ = self.for_each_search_neighbor( node, bwd_direction, - edge_type_filter, + edge_label_filter, reference_time, policy_cutoffs, &mut tombstones, @@ -4146,7 +4487,7 @@ impl ReadView { &mut bwd_settled, &mut bwd_best, bwd_direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, max_depth, @@ -4162,7 +4503,7 @@ impl ReadView { &mut fwd_settled, &mut fwd_best, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, max_depth, @@ -4186,7 +4527,7 @@ impl ReadView { let successors = self.build_weighted_bounded_shortest_path_successors( &fwd_dist, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, max_depth, @@ -4214,7 +4555,7 @@ impl ReadView { to, 0, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, max_depth, @@ -4301,7 +4642,7 @@ impl ReadView { to: u64, depth_so_far: u32, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, reference_time: i64, best_hops: u32, bwd_depth: &IdMap, @@ -4336,7 +4677,7 @@ impl ReadView { node, depth_so_far, direction, - edge_type_filter, + edge_label_filter, reference_time, best_hops, bwd_depth, @@ -4353,7 +4694,7 @@ impl ReadView { to, depth_so_far + 1, direction, - edge_type_filter, + edge_label_filter, reference_time, best_hops, bwd_depth, @@ -4478,7 +4819,7 @@ impl ReadView { node: u64, to: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, best_cost: f64, @@ -4502,7 +4843,7 @@ impl ReadView { successors_cache, current_node, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, best_cost, @@ -4576,7 +4917,7 @@ impl ReadView { to: u64, hops_so_far: u32, direction: Direction, - edge_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[u32]>, weight_field: &str, reference_time: i64, max_depth: u32, @@ -4602,7 +4943,7 @@ impl ReadView { current_node, hops, direction, - edge_type_filter, + edge_label_filter, weight_field, reference_time, max_depth, @@ -4643,12 +4984,12 @@ impl ReadView { &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, limit: usize, at_epoch: Option, decay_lambda: Option, tombstones: Option<(&IdSet, &IdSet)>, - ) -> Result, EngineError> { + ) -> Result, EngineError> { if let Some(l) = decay_lambda { if l < 0.0 { return Err(EngineError::InvalidOperation( @@ -4673,14 +5014,14 @@ impl ReadView { // Start with memtable results (fetch without limit to allow for temporal filtering) let mut results = self.memtable - .neighbors_at(node_id, direction, type_filter, 0, self.snapshot_seq); + .neighbors_at(node_id, direction, label_filter_ids, 0, self.snapshot_seq); let mut seen_edges: IdSet = results.iter().map(|e| e.edge_id).collect(); // Immutable memtables (newest-first) for epoch in &self.immutable_epochs { let mt_results = epoch .memtable - .neighbors_at(node_id, direction, type_filter, 0, self.snapshot_seq); + .neighbors_at(node_id, direction, label_filter_ids, 0, self.snapshot_seq); for entry in mt_results { if !seen_edges.insert(entry.edge_id) { continue; @@ -4697,7 +5038,7 @@ impl ReadView { // Add segment results for seg in &self.segments { - let seg_results = seg.neighbors(node_id, direction, type_filter, 0)?; + let seg_results = seg.neighbors(node_id, direction, label_filter_ids, 0)?; for entry in seg_results { if !seen_edges.insert(entry.edge_id) { continue; @@ -4766,9 +5107,15 @@ impl ReadView { &self, node_id: u64, options: &NeighborOptions, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let direction = options.direction; - let type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => return Ok(Vec::new()), + }; let limit = options.limit.unwrap_or(0); let at_epoch = options.at_epoch; let decay_lambda = options.decay_lambda; @@ -4778,7 +5125,7 @@ impl ReadView { return self.neighbors_raw( node_id, direction, - type_filter, + label_filter_ids.as_deref(), limit, at_epoch, decay_lambda, @@ -4790,7 +5137,7 @@ impl ReadView { let mut results = self.neighbors_raw( node_id, direction, - type_filter, + label_filter_ids.as_deref(), 0, at_epoch, decay_lambda, @@ -4823,7 +5170,7 @@ impl ReadView { /// per segment instead of O(M log N) individual binary searches. /// /// `node_ids` need not be sorted (sorted internally). Returns a - /// [`NodeIdMap>`] mapping each queried node_id to its + /// [`NodeIdMap>`] mapping each queried node_id to its /// neighbor entries. `NodeIdMap` is a `HashMap` with identity hashing /// optimized for engine-generated numeric IDs. /// @@ -4832,24 +5179,47 @@ impl ReadView { &self, node_ids: &[u64], options: &NeighborOptions, - ) -> Result>, EngineError> { + ) -> Result>, EngineError> { let direction = options.direction; - let type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => return Ok(NodeIdMap::default()), + }; let at_epoch = options.at_epoch; let decay_lambda = options.decay_lambda; + self.neighbors_batch_resolved( + node_ids, + direction, + label_filter_ids.as_deref(), + at_epoch, + decay_lambda, + ) + } + + fn neighbors_batch_resolved( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + at_epoch: Option, + decay_lambda: Option, + ) -> Result>, EngineError> { if self.manifest.prune_policies.is_empty() { return self.neighbors_batch_raw( node_ids, direction, - type_filter, + label_filter_ids, at_epoch, decay_lambda, ); } let mut results = - self.neighbors_batch_raw(node_ids, direction, type_filter, at_epoch, decay_lambda)?; + self.neighbors_batch_raw(node_ids, direction, label_filter_ids, at_epoch, decay_lambda)?; // Collect unique neighbor node IDs for batch policy check (dedup avoids // redundant merge-walks when many queried nodes share neighbors). @@ -4881,10 +5251,10 @@ impl ReadView { &self, node_ids: &[u64], direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, at_epoch: Option, decay_lambda: Option, - ) -> Result>, EngineError> { + ) -> Result>, EngineError> { if node_ids.is_empty() { return Ok(NodeIdMap::default()); } @@ -4914,7 +5284,7 @@ impl ReadView { // Start with memtable results. let mut results = self .memtable - .neighbors_batch_at(&sorted_ids, direction, type_filter, self.snapshot_seq); + .neighbors_batch_at(&sorted_ids, direction, label_filter_ids, self.snapshot_seq); // Track seen edge IDs per node for dedup (memtable/newer segment wins) let mut seen_edges: NodeIdMap = NodeIdMap::default(); @@ -4930,7 +5300,7 @@ impl ReadView { let mt_results = epoch.memtable.neighbors_batch_at( &sorted_ids, direction, - type_filter, + label_filter_ids, self.snapshot_seq, ); for (nid, mt_entries) in mt_results { @@ -4953,7 +5323,7 @@ impl ReadView { // Merge segment results (newest-to-oldest, one cursor walk per segment) for seg in &self.segments { - let seg_results = seg.neighbors_batch(&sorted_ids, direction, type_filter)?; + let seg_results = seg.neighbors_batch(&sorted_ids, direction, label_filter_ids)?; for (nid, seg_entries) in seg_results { let seen = seen_edges.entry(nid).or_default(); let node_entries = results.entry(nid).or_default(); @@ -5023,9 +5393,20 @@ impl ReadView { node_id: u64, options: &NeighborOptions, page: &PageRequest, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let direction = options.direction; - let type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => { + return Ok(PageResult { + items: Vec::new(), + next_cursor: None, + }) + } + }; let at_epoch = options.at_epoch; let decay_lambda = options.decay_lambda; @@ -5049,18 +5430,24 @@ impl ReadView { // Collect sources: memtable (unsorted) + immutable memtables + segments (sorted by edge_id) let memtable_entries = self.memtable - .neighbors_at(node_id, direction, type_filter, 0, self.snapshot_seq); - let mut segment_entries: Vec> = + .neighbors_at(node_id, direction, label_filter_ids.as_deref(), 0, self.snapshot_seq); + let mut segment_entries: Vec> = Vec::with_capacity(self.immutable_epochs.len() + self.segments.len()); for epoch in &self.immutable_epochs { segment_entries.push( epoch .memtable - .neighbors_at(node_id, direction, type_filter, 0, self.snapshot_seq), + .neighbors_at( + node_id, + direction, + label_filter_ids.as_deref(), + 0, + self.snapshot_seq, + ), ); } for seg in &self.segments { - segment_entries.push(seg.neighbors(node_id, direction, type_filter, 0)?); + segment_entries.push(seg.neighbors(node_id, direction, label_filter_ids.as_deref(), 0)?); } if apply_decay { @@ -5251,9 +5638,15 @@ impl ReadView { node_id: u64, k: usize, options: &TopKOptions, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let direction = options.direction; - let type_filter = options.type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let label_filter_ids = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => return Ok(Vec::new()), + }; let scoring = options.scoring.clone(); let at_epoch = options.at_epoch; @@ -5278,7 +5671,7 @@ impl ReadView { // Gather all neighbors from memtable + immutable memtables + segments with dedup + deletion filter let mut all_entries = self.memtable - .neighbors_at(node_id, direction, type_filter, 0, self.snapshot_seq); + .neighbors_at(node_id, direction, label_filter_ids.as_deref(), 0, self.snapshot_seq); // Filter memtable results against deleted sets (M2 fix) all_entries .retain(|e| !deleted_edges.contains(&e.edge_id) && !deleted_nodes.contains(&e.node_id)); @@ -5288,7 +5681,7 @@ impl ReadView { for epoch in &self.immutable_epochs { let mt_results = epoch .memtable - .neighbors_at(node_id, direction, type_filter, 0, self.snapshot_seq); + .neighbors_at(node_id, direction, label_filter_ids.as_deref(), 0, self.snapshot_seq); for entry in mt_results { if !seen_edges.insert(entry.edge_id) { continue; @@ -5304,7 +5697,7 @@ impl ReadView { } for seg in &self.segments { - let seg_results = seg.neighbors(node_id, direction, type_filter, 0)?; + let seg_results = seg.neighbors(node_id, direction, label_filter_ids.as_deref(), 0)?; for entry in seg_results { if !seen_edges.insert(entry.edge_id) { continue; @@ -5338,7 +5731,7 @@ impl ReadView { // Use f64 bits for the min-heap key to preserve i64 timestamp precision // for Recency scoring (f32 can't represent epoch-millis without loss). let mut heap: BinaryHeap> = BinaryHeap::new(); - let mut scored: Vec = Vec::new(); + let mut scored: Vec = Vec::new(); for mut entry in all_entries { if !is_edge_valid_at(entry.valid_from, entry.valid_to, reference_time) { @@ -5380,7 +5773,7 @@ impl ReadView { // Sort descending by score top_indices.sort_by(|a, b| b.0.cmp(&a.0)); - let results: Vec = top_indices + let results: Vec = top_indices .into_iter() .map(|(_, idx)| scored[idx].clone()) .collect(); @@ -5393,11 +5786,11 @@ impl ReadView { /// /// Uses BFS with cycle detection. Edges to already-visited nodes (cross-edges /// and back-edges) are included in the result. The traversal respects - /// direction and edge type filters. + /// direction and edge-label filters. /// /// - `max_depth`: maximum number of hops. 0 returns just the start node. /// - `direction`: which edge direction to follow during traversal. - /// - `edge_type_filter`: only traverse edges of these types. `None` = all. + /// - `edge_label_filter`: only traverse edges with these labels. `None` = all. /// - `at_epoch`: temporal filter. Only include edges valid at this time. /// /// Returns an empty `Subgraph` if the start node does not exist. @@ -5408,7 +5801,21 @@ impl ReadView { options: &SubgraphOptions, ) -> Result { let direction = options.direction; - let edge_type_filter = options.edge_type_filter.as_deref(); + let resolved_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let edge_label_filter = match resolved_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => Some(Vec::new()), + }; + let node_label_filter = + self.resolve_node_label_filter_request_for_graph(options.node_label_filter.as_ref())?; + if node_label_filter.is_empty_constraint() { + return Ok(Subgraph { + nodes: Vec::new(), + edges: Vec::new(), + }); + } let at_epoch = options.at_epoch; // Check start node exists let start_node = match self.get_node(start_node_id)? { @@ -5420,8 +5827,21 @@ impl ReadView { }) } }; + if !node_label_filter_matches(&node_label_filter, &start_node.label_ids) { + return Ok(Subgraph { + nodes: Vec::new(), + edges: Vec::new(), + }); + } + if edge_label_filter.as_deref().is_some_and(|label_ids| label_ids.is_empty()) { + return Ok(Subgraph { + nodes: vec![node_view_from_record(start_node, self.label_catalog.as_ref())?], + edges: Vec::new(), + }); + } let mut visited_nodes: IdSet = IdSet::default(); + let mut blocked_nodes: IdSet = IdSet::default(); let mut collected_edge_ids: IdSet = IdSet::default(); let mut node_records: Vec = Vec::new(); let mut edge_records: Vec = Vec::new(); @@ -5429,40 +5849,68 @@ impl ReadView { visited_nodes.insert(start_node_id); node_records.push(start_node); - if max_depth == 0 { - return Ok(Subgraph { - nodes: node_records, - edges: edge_records, - }); - } - - // BFS level by level, batch-fetch neighbors for entire frontier + // BFS level by level, batch-fetch neighbors for entire frontier. let mut frontier: Vec = vec![start_node_id]; for _depth in 0..max_depth { let mut new_edge_ids: Vec = Vec::new(); let mut new_node_ids: Vec = Vec::new(); + let mut candidate_seen = IdSet::default(); + let mut candidate_edges: Vec<(u64, u64)> = Vec::new(); // Batch adjacency: one cursor walk per segment for the whole frontier - let batch_opts = NeighborOptions { - direction, - type_filter: edge_type_filter.map(|s| s.to_vec()), - limit: None, - at_epoch, - decay_lambda: None, - }; - let all_neighbors = self.neighbors_batch(&frontier, &batch_opts)?; + let all_neighbors = self.neighbors_batch_resolved( + &frontier, + direction, + edge_label_filter.as_deref(), + at_epoch, + None, + )?; for ¤t_node in &frontier { let empty = Vec::new(); let neighbors = all_neighbors.get(¤t_node).unwrap_or(&empty); for entry in neighbors { - if collected_edge_ids.insert(entry.edge_id) { - new_edge_ids.push(entry.edge_id); + if visited_nodes.contains(&entry.node_id) { + if collected_edge_ids.insert(entry.edge_id) { + new_edge_ids.push(entry.edge_id); + } + } else if blocked_nodes.contains(&entry.node_id) { + continue; + } else { + candidate_edges.push((entry.edge_id, entry.node_id)); + if candidate_seen.insert(entry.node_id) { + new_node_ids.push(entry.node_id); + } } - if visited_nodes.insert(entry.node_id) { - new_node_ids.push(entry.node_id); + } + } + + let node_filter_constrained = + !Self::node_label_filter_is_unconstrained(&node_label_filter); + let eligible_new_node_ids = if node_filter_constrained { + self.filter_node_ids_by_resolved_label_filter( + &new_node_ids, + &node_label_filter, + None, + )? + } else { + new_node_ids.clone() + }; + let eligible_new_nodes: IdSet = eligible_new_node_ids.iter().copied().collect(); + for (edge_id, node_id) in candidate_edges { + if eligible_new_nodes.contains(&node_id) && collected_edge_ids.insert(edge_id) { + new_edge_ids.push(edge_id); + } + } + for &node_id in &eligible_new_node_ids { + visited_nodes.insert(node_id); + } + if node_filter_constrained { + for &node_id in &new_node_ids { + if !eligible_new_nodes.contains(&node_id) { + blocked_nodes.insert(node_id); } } } @@ -5473,12 +5921,12 @@ impl ReadView { edge_records.push(edge); } - let fetched_nodes = self.get_nodes(&new_node_ids)?; + let fetched_nodes = self.get_nodes(&eligible_new_node_ids)?; let mut next_frontier: Vec = Vec::new(); for (i, node_opt) in fetched_nodes.into_iter().enumerate() { if let Some(node) = node_opt { node_records.push(node); - next_frontier.push(new_node_ids[i]); + next_frontier.push(eligible_new_node_ids[i]); } } @@ -5488,10 +5936,16 @@ impl ReadView { } } - Ok(Subgraph { - nodes: node_records, - edges: edge_records, - }) + let nodes = node_records + .into_iter() + .map(|node| node_view_from_record(node, self.label_catalog.as_ref())) + .collect::, _>>()?; + let edges = edge_records + .into_iter() + .map(|edge| edge_view_from_record(edge, self.label_catalog.as_ref())) + .collect::, _>>()?; + + Ok(Subgraph { nodes, edges }) } // --- Connected Components (WCC) --- @@ -5505,8 +5959,8 @@ impl ReadView { /// `NodeIdMap` is a `HashMap` with identity hashing optimized for /// engine-generated numeric IDs. /// - /// - `edge_type_filter`: only consider edges of these types. `None` = all. - /// - `node_type_filter`: only include nodes of these types. `None` = all. + /// - `edge_label_filter`: only consider edges of these labels. `None` = all. + /// - `node_label_filter`: only include nodes with these labels. `None` = all. /// - `at_epoch`: reference time for temporal edge visibility. `None` → now. /// /// Isolated nodes (no visible edges after filtering) become singleton @@ -5531,52 +5985,37 @@ impl ReadView { &self, options: &ComponentOptions, ) -> Result, EngineError> { - let edge_type_filter = options.edge_type_filter.as_deref(); - let node_type_filter = options.node_type_filter.as_deref(); - let at_epoch = options.at_epoch; - // 1. Collect all visible node IDs (policy-filtered, type-filtered). - let node_types: Vec = { - let mut types: HashSet = - self.memtable.visible_types(self.snapshot_seq).into_iter().collect(); - for epoch in &self.immutable_epochs { - types.extend(epoch.memtable.visible_types(self.snapshot_seq)); - } - for seg in &self.segments { - for tid in seg.node_type_ids()? { - types.insert(tid); - } - } - if let Some(filter) = node_type_filter { - let allowed: HashSet = filter.iter().copied().collect(); - types.retain(|t| allowed.contains(t)); - } - types.into_iter().collect() + let resolved_edge_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let edge_label_filter = match resolved_edge_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => Some(Vec::new()), }; - - // Collect all visible node IDs per type, then build the set with - // known capacity to avoid repeated re-hashing. - let mut per_type_ids: Vec> = Vec::with_capacity(node_types.len()); - let mut total_count: usize = 0; - for &tid in &node_types { - let ids = self.nodes_by_type(tid)?; - total_count += ids.len(); - per_type_ids.push(ids); + let node_label_filter = + self.resolve_node_label_filter_request_for_graph(options.node_label_filter.as_ref())?; + if node_label_filter.is_empty_constraint() { + return Ok(NodeIdMap::default()); } - if total_count == 0 { + let at_epoch = options.at_epoch; + let policy_cutoffs = self.query_policy_cutoffs(); + // 1. Collect all visible node IDs (policy-filtered, label-filtered). + let node_ids = self.collect_node_ids_for_resolved_label_filter( + &node_label_filter, + policy_cutoffs.as_ref(), + )?; + if node_ids.is_empty() { return Ok(NodeIdMap::default()); } - - let mut node_set: IdSet = - IdSet::with_capacity_and_hasher(total_count, IdBuildHasher::default()); - for ids in &per_type_ids { - for &id in ids { - node_set.insert(id); + let node_set: IdSet = node_ids.iter().copied().collect(); + if edge_label_filter.as_deref().is_some_and(|label_ids| label_ids.is_empty()) { + let mut result = + NodeIdMap::with_capacity_and_hasher(node_ids.len(), IdBuildHasher::default()); + for id in node_ids { + result.insert(id, id); } + return Ok(result); } - drop(per_type_ids); - - let mut node_ids: Vec = node_set.iter().copied().collect(); - node_ids.sort_unstable(); // 2. Initialize union-find with one set per visible node. let mut uf = UnionFind::with_capacity(node_ids.len()); @@ -5587,7 +6026,7 @@ impl ReadView { // 3. Global outgoing adjacency scan: streaming union. // Each directed edge appears exactly once; union(from, to) // captures undirected connectivity. Unlike neighbors_batch(), - // this never materializes Vec. The callback + // this never materializes Vec. The callback // unions endpoints inline during the cursor walk. let reference_time = at_epoch.unwrap_or_else(now_millis); @@ -5600,7 +6039,7 @@ impl ReadView { let _ = self.memtable.for_each_adj_entry_at( nid, Direction::Outgoing, - edge_type_filter, + edge_label_filter.as_deref(), self.snapshot_seq, &mut |edge_id, neighbor_id, _weight, valid_from, valid_to| { seen_edges.insert((nid, edge_id)); @@ -5621,7 +6060,7 @@ impl ReadView { let _ = epoch.memtable.for_each_adj_entry_at( nid, Direction::Outgoing, - edge_type_filter, + edge_label_filter.as_deref(), self.snapshot_seq, &mut |edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((nid, edge_id)) { @@ -5650,7 +6089,7 @@ impl ReadView { let _ = seg.for_each_adj_posting_batch( &node_ids, Direction::Outgoing, - edge_type_filter, + edge_label_filter.as_deref(), &mut |queried_nid, edge_id, neighbor_id, _weight, valid_from, valid_to| { if !seen_edges.insert((queried_nid, edge_id)) { return ControlFlow::Continue(()); @@ -5708,7 +6147,7 @@ impl ReadView { /// /// Returns an empty `Vec` if the node doesn't exist, is deleted, or is /// hidden by prune policy. Returns an empty `Vec` if the node exists but - /// is excluded by `node_type_filter`. + /// is excluded by `node_label_filter`. /// /// # Examples /// @@ -5724,24 +6163,33 @@ impl ReadView { node_id: u64, options: &ComponentOptions, ) -> Result, EngineError> { - let edge_type_filter = options.edge_type_filter.as_deref(); - let node_type_filter = options.node_type_filter.as_deref(); + let resolved_edge_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let edge_label_filter = match resolved_edge_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => Some(Vec::new()), + }; + let node_label_filter = + self.resolve_node_label_filter_request_for_graph(options.node_label_filter.as_ref())?; + if node_label_filter.is_empty_constraint() { + return Ok(Vec::new()); + } let at_epoch = options.at_epoch; + let policy_cutoffs = self.query_policy_cutoffs(); // Check if the start node exists and is visible (tombstones + policies). let start_node = match self.get_node(node_id)? { Some(n) => n, None => return Ok(Vec::new()), }; - // If node_type_filter is set, the start node must pass it. - if let Some(filter) = node_type_filter { - if !filter.contains(&start_node.type_id) { - return Ok(Vec::new()); - } + // If node_label_filter is set, the start node must pass it. + if !node_label_filter_matches(&node_label_filter, &start_node.label_ids) { + return Ok(Vec::new()); + } + if edge_label_filter.as_deref().is_some_and(|label_ids| label_ids.is_empty()) { + return Ok(vec![node_id]); } - - let type_filter_set: Option> = - node_type_filter.map(|f| f.iter().copied().collect()); let mut processed: IdSet = IdSet::with_hasher(IdBuildHasher::default()); processed.insert(node_id); @@ -5753,14 +6201,13 @@ impl ReadView { while !frontier.is_empty() { // Batch-fetch neighbors for all frontier nodes (undirected). - let batch_opts = NeighborOptions { - direction: Direction::Both, - type_filter: edge_type_filter.map(|s| s.to_vec()), - limit: None, - at_epoch, - decay_lambda: None, - }; - let all_neighbors = self.neighbors_batch(&frontier, &batch_opts)?; + let all_neighbors = self.neighbors_batch_resolved( + &frontier, + Direction::Both, + edge_label_filter.as_deref(), + at_epoch, + None, + )?; // Collect newly discovered neighbor IDs. candidate_ids.clear(); @@ -5776,23 +6223,21 @@ impl ReadView { break; } - // If node_type_filter is set, check types of discovered nodes. - // Only nodes passing the filter join the component and frontier. next_frontier.clear(); - if let Some(ref type_set) = type_filter_set { - candidate_ids.sort_unstable(); - let nodes = self.get_nodes_raw(&candidate_ids)?; - for (&cid, slot) in candidate_ids.iter().zip(nodes.iter()) { - if let Some(node) = slot { - if type_set.contains(&node.type_id) { - component.push(cid); - next_frontier.push(cid); - } - } - } - } else { + if Self::node_label_filter_is_unconstrained(&node_label_filter) + && policy_cutoffs.is_none() + { component.extend(&candidate_ids); next_frontier.extend(&candidate_ids); + } else { + // Only nodes passing the filter join the component and frontier. + let filtered_ids = self.filter_node_ids_by_resolved_label_filter( + &candidate_ids, + &node_label_filter, + policy_cutoffs.as_ref(), + )?; + component.extend(&filtered_ids); + next_frontier.extend(&filtered_ids); } std::mem::swap(&mut frontier, &mut next_frontier); diff --git a/src/engine/mod.rs b/src/engine/mod.rs index ac9c6f1..625023c 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -1,31 +1,41 @@ use crate::degree_cache::{DegreeDelta, DegreeOverlayEdit, DegreeOverlaySnapshot}; use crate::dense_hnsw::exact_dense_search_above_cutoff; +use crate::edge_metadata::EdgeMetadataCandidate; use crate::error::EngineError; use crate::manifest::{default_manifest, load_manifest, load_manifest_readonly, write_manifest}; use crate::memtable::{encode_range_prop_value, Memtable}; use crate::planner_stats::{ planner_stats_declaration_fingerprint_for_entry, write_targeted_secondary_index_planner_stats_sidecar, DeclaredIndexRuntimeCoverage, - EstimateConfidence, PlannerEstimateKind, PlannerStatsDirection, PlannerStatsView, - PlannerStatsWriteOutcome, StalePostingRisk, + EstimateConfidence, PlannerEstimateKind, PlannerStatsDeclaredIndexTarget, + PlannerStatsDirection, PlannerStatsView, StalePostingRisk, }; -use crate::segment_reader::{SegmentReader, SegmentTypePosting}; +use crate::segment_components::{ComponentAvailability, SegmentComponentKind}; +use crate::segment_reader::{SegmentAdjPostingCursor, SegmentLabelPosting, SegmentReader}; use crate::segment_writer::{ - node_prop_eq_sidecar_path, node_prop_range_sidecar_path, segment_dir, segment_tmp_dir, - write_indexes_from_metadata_with_secondary_indexes, write_merged_edges_dat, - write_merged_nodes_dat, write_node_prop_eq_sidecar_to_path, - write_node_prop_range_sidecar_to_path, write_segment_with_degree_overlay_and_secondary_indexes, - write_v3_edges_dat, write_v3_nodes_dat, CompactEdgeMeta, CompactNodeMeta, FastMergeCopyInfo, + cleanup_orphan_optional_component_files, create_compaction_core_writer, + finalize_compaction_segment, finish_compaction_core_writer, + is_optional_component_publication_conflict, + maintained_secondary_index_ids_from_segment_manifest, publish_edge_prop_eq_sidecar_component, + publish_edge_prop_range_sidecar_component, publish_node_prop_eq_sidecar_component, + publish_node_prop_range_sidecar_component, remove_secondary_index_component_records, + secondary_index_sidecar_paths_for_entry, segment_dir, segment_tmp_dir, + write_compaction_source_components, write_indexes_from_metadata_with_secondary_indexes, + write_merged_edges_dat, write_merged_nodes_dat, + write_segment_with_degree_overlay_and_secondary_indexes, write_v3_edges_dat, + write_v3_nodes_dat, CompactEdgeMeta, CompactNodeMeta, FastMergeCopyInfo, SecondaryIndexMaintenanceReport, }; use crate::source_list::SourceList; -use crate::sparse_postings::{accumulate_sparse_posting_scores, sparse_dot_score}; +use crate::sparse_postings::sparse_dot_score; use crate::types::*; -use crate::wal::{remove_wal_generation, wal_generation_path, WalReader, WalWriter}; +#[cfg(test)] +use crate::wal::wal_generation_path; +use crate::wal::{remove_wal_generation, truncate_wal_generation_to, WalReader, WalWriter}; use crate::wal_sync::{shutdown_sync_thread, sync_thread_loop, WalSyncState}; use arc_swap::ArcSwap; use std::cmp::Reverse; -use std::collections::{BTreeMap, BinaryHeap, HashMap, HashSet, VecDeque}; +use std::collections::{hash_map::Entry, BTreeMap, BinaryHeap, HashMap, HashSet, VecDeque}; use std::ops::ControlFlow; use std::path::{Path, PathBuf}; @@ -34,8 +44,54 @@ use std::sync::{Arc, Condvar, Mutex, RwLock, Weak}; use std::thread::JoinHandle; use std::time::SystemTime; -type SecondaryIndexCatalog = - HashMap>>; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum SecondaryIndexTargetDiscriminant { + Node, + Edge, +} + +fn secondary_index_target_discriminant( + target: &SecondaryIndexTarget, +) -> SecondaryIndexTargetDiscriminant { + match target { + SecondaryIndexTarget::NodeProperty { .. } => SecondaryIndexTargetDiscriminant::Node, + SecondaryIndexTarget::EdgeProperty { .. } => SecondaryIndexTargetDiscriminant::Edge, + } +} + +fn secondary_index_target_label_id(target: &SecondaryIndexTarget) -> u32 { + match target { + SecondaryIndexTarget::NodeProperty { label_id, .. } => *label_id, + SecondaryIndexTarget::EdgeProperty { label_id, .. } => *label_id, + } +} + +fn secondary_index_target_prop_key(target: &SecondaryIndexTarget) -> &str { + match target { + SecondaryIndexTarget::NodeProperty { prop_key, .. } + | SecondaryIndexTarget::EdgeProperty { prop_key, .. } => prop_key, + } +} + +fn secondary_index_range_domain_rank(domain: SecondaryIndexRangeDomain) -> u8 { + match domain { + SecondaryIndexRangeDomain::Int => 0, + SecondaryIndexRangeDomain::UInt => 1, + SecondaryIndexRangeDomain::Float => 2, + } +} + +fn secondary_index_kind_rank(kind: &SecondaryIndexKind) -> (u8, u8) { + match kind { + SecondaryIndexKind::Equality => (0, 0), + SecondaryIndexKind::Range { domain } => (1, secondary_index_range_domain_rank(*domain)), + } +} + +type SecondaryIndexCatalog = HashMap< + (SecondaryIndexTargetDiscriminant, u32), + HashMap>, +>; type SecondaryIndexEntries = Vec; /// Generic K-way merge across already-sorted sources with early termination @@ -145,7 +201,7 @@ fn merge_sorted_paged( /// K-way merge for u64 ID lists. Thin wrapper around `merge_sorted_paged` /// with identity key and deleted-set skip function. -fn merge_type_ids_paged( +fn merge_record_ids_paged( memtable_ids: Vec, segment_sorted_ids: Vec>, deleted: &NodeIdSet, @@ -194,12 +250,11 @@ fn reconcile_dense_vector_manifest( } fn secondary_index_lookup_key(entry: &SecondaryIndexManifestEntry) -> SecondaryIndexLookupKey { - match &entry.target { - SecondaryIndexTarget::NodeProperty { type_id, prop_key } => SecondaryIndexLookupKey { - type_id: *type_id, - prop_key: prop_key.clone(), - kind: entry.kind.clone(), - }, + SecondaryIndexLookupKey { + discriminant: secondary_index_target_discriminant(&entry.target), + target_label_id: secondary_index_target_label_id(&entry.target), + prop_key: secondary_index_target_prop_key(&entry.target).to_string(), + kind: entry.kind.clone(), } } @@ -224,11 +279,17 @@ fn normalize_secondary_index_manifest(manifest: &mut ManifestState) -> Result "node", + SecondaryIndexTargetDiscriminant::Edge => "edge", + }; return Err(EngineError::ManifestError(format!( - "duplicate range declaration for node property ({}, {})", - type_id, prop_key + "duplicate range declaration for {} property ({}, {})", + target_label, target_label_id, prop_key ))); } } @@ -256,28 +317,31 @@ fn build_secondary_index_catalog( let mut catalog: SecondaryIndexCatalog = HashMap::with_capacity(entries.len()); let mut seen_range_targets = HashSet::new(); for entry in entries { - match &entry.target { - SecondaryIndexTarget::NodeProperty { type_id, prop_key } => { - if matches!(entry.kind, SecondaryIndexKind::Range { .. }) - && !seen_range_targets.insert((*type_id, prop_key.clone())) - { - return Err(EngineError::ManifestError(format!( - "duplicate range declaration loaded from manifest for node property ({}, {})", - type_id, prop_key - ))); - } - let kind_map = catalog - .entry(*type_id) - .or_default() - .entry(prop_key.clone()) - .or_default(); - if kind_map.insert(entry.kind.clone(), entry.clone()).is_some() { - return Err(EngineError::ManifestError(format!( - "duplicate secondary index declaration loaded from manifest: {:?}", - entry.target - ))); - } - } + let disc = secondary_index_target_discriminant(&entry.target); + let target_label_id = secondary_index_target_label_id(&entry.target); + let prop_key = secondary_index_target_prop_key(&entry.target); + if matches!(entry.kind, SecondaryIndexKind::Range { .. }) + && !seen_range_targets.insert((disc, target_label_id, prop_key.to_string())) + { + let target_label = match disc { + SecondaryIndexTargetDiscriminant::Node => "node", + SecondaryIndexTargetDiscriminant::Edge => "edge", + }; + return Err(EngineError::ManifestError(format!( + "duplicate range declaration loaded from manifest for {} property ({}, {})", + target_label, target_label_id, prop_key + ))); + } + let kind_map = catalog + .entry((disc, target_label_id)) + .or_default() + .entry(prop_key.to_string()) + .or_default(); + if kind_map.insert(entry.kind.clone(), entry.clone()).is_some() { + return Err(EngineError::ManifestError(format!( + "duplicate secondary index declaration loaded from manifest: {:?}", + entry.target + ))); } } Ok(catalog) @@ -311,6 +375,587 @@ fn merge_runtime_manifest_counters_from_shared( .max(engine_seq_seen.load(Ordering::Acquire)); } +#[derive(Debug, Clone)] +pub(crate) struct RuntimeLabelCatalog { + pub node_label_to_id: BTreeMap, + pub node_id_to_label: BTreeMap, + node_label_wal_generation: BTreeMap, + pub edge_label_to_id: BTreeMap, + pub edge_id_to_label: BTreeMap, + edge_label_wal_generation: BTreeMap, + pub next_node_label_id: u32, + pub next_edge_label_id: u32, +} + +impl RuntimeLabelCatalog { + fn from_manifest(manifest: &ManifestState) -> Result { + let mut catalog = Self { + node_label_to_id: manifest.node_label_tokens.clone(), + node_id_to_label: BTreeMap::new(), + node_label_wal_generation: BTreeMap::new(), + edge_label_to_id: manifest.edge_label_tokens.clone(), + edge_id_to_label: BTreeMap::new(), + edge_label_wal_generation: BTreeMap::new(), + next_node_label_id: manifest.next_node_label_id, + next_edge_label_id: manifest.next_edge_label_id, + }; + catalog.rebuild_reverse_maps()?; + Ok(catalog) + } + + fn rebuild_reverse_maps(&mut self) -> Result<(), EngineError> { + self.node_id_to_label.clear(); + for (label, &label_id) in &self.node_label_to_id { + if let Some(existing) = self.node_id_to_label.insert(label_id, label.clone()) { + return Err(EngineError::ManifestError(format!( + "node label token conflict: label_id {label_id} is assigned to both '{existing}' and '{label}'" + ))); + } + } + self.edge_id_to_label.clear(); + for (label, &label_id) in &self.edge_label_to_id { + if let Some(existing) = self.edge_id_to_label.insert(label_id, label.clone()) { + return Err(EngineError::ManifestError(format!( + "edge-label token conflict: label_id {label_id} is assigned to both '{existing}' and '{label}'" + ))); + } + } + Ok(()) + } + + fn apply_to_manifest(&self, manifest: &mut ManifestState) { + manifest.label_token_schema_version = LABEL_TOKEN_SCHEMA_VERSION; + manifest.node_label_tokens = self.node_label_to_id.clone(); + manifest.edge_label_tokens = self.edge_label_to_id.clone(); + manifest.next_node_label_id = self.next_node_label_id; + manifest.next_edge_label_id = self.next_edge_label_id; + } + + fn apply_checkpointed_to_manifest( + &self, + manifest: &mut ManifestState, + max_wal_generation: Option, + ) { + manifest.label_token_schema_version = LABEL_TOKEN_SCHEMA_VERSION; + for (label, &label_id) in &self.node_label_to_id { + if manifest.node_label_tokens.get(label) == Some(&label_id) + || self + .node_label_wal_generation + .get(label) + .is_some_and(|generation| { + max_wal_generation + .is_some_and(|max_generation| *generation <= max_generation) + }) + { + manifest.node_label_tokens.insert(label.clone(), label_id); + } + } + for (label, &label_id) in &self.edge_label_to_id { + if manifest.edge_label_tokens.get(label) == Some(&label_id) + || self + .edge_label_wal_generation + .get(label) + .is_some_and(|generation| { + max_wal_generation + .is_some_and(|max_generation| *generation <= max_generation) + }) + { + manifest.edge_label_tokens.insert(label.clone(), label_id); + } + } + manifest.next_node_label_id = manifest.next_node_label_id.max( + manifest + .node_label_tokens + .values() + .copied() + .max() + .unwrap_or(0) + .saturating_add(1), + ); + manifest.next_edge_label_id = manifest.next_edge_label_id.max( + manifest + .edge_label_tokens + .values() + .copied() + .max() + .unwrap_or(0) + .saturating_add(1), + ); + } + + fn reserve_node_label(&self, label: &str) -> Result<(u32, bool), EngineError> { + if let Some(&label_id) = self.node_label_to_id.get(label) { + return Ok((label_id, false)); + } + validate_label_token_name(label)?; + if self.next_node_label_id == u32::MAX { + return Err(EngineError::InvalidOperation( + "node label token ID space exhausted".to_string(), + )); + } + Ok((self.next_node_label_id, true)) + } + + fn reserve_edge_label(&self, label: &str) -> Result<(u32, bool), EngineError> { + if let Some(&label_id) = self.edge_label_to_id.get(label) { + return Ok((label_id, false)); + } + validate_label_token_name(label)?; + if self.next_edge_label_id == u32::MAX { + return Err(EngineError::InvalidOperation( + "edge-label token ID space exhausted".to_string(), + )); + } + Ok((self.next_edge_label_id, true)) + } + + fn apply_node_label( + &mut self, + label: String, + label_id: u32, + wal_generation: Option, + ) -> Result<(), EngineError> { + validate_label_token_name(&label)?; + if label_id == 0 { + return Err(EngineError::InvalidOperation( + "node label token ID 0 is reserved".to_string(), + )); + } + if let Some(existing_id) = self.node_label_to_id.get(&label) { + if *existing_id == label_id { + if let Some(wal_generation) = wal_generation { + self.node_label_wal_generation + .entry(label) + .and_modify(|existing| *existing = (*existing).min(wal_generation)) + .or_insert(wal_generation); + } + return Ok(()); + } + return Err(EngineError::CorruptWal(format!( + "node label token conflict: label '{label}' is assigned to both label_id {existing_id} and {label_id}" + ))); + } + if let Some(existing_label) = self.node_id_to_label.get(&label_id) { + return Err(EngineError::CorruptWal(format!( + "node label token conflict: label_id {label_id} is assigned to both '{existing_label}' and '{label}'" + ))); + } + self.node_label_to_id.insert(label.clone(), label_id); + self.node_id_to_label.insert(label_id, label); + if let Some(wal_generation) = wal_generation { + let stored_label = self + .node_id_to_label + .get(&label_id) + .expect("node label reverse map was just inserted") + .clone(); + self.node_label_wal_generation + .insert(stored_label, wal_generation); + } + self.next_node_label_id = self.next_node_label_id.max(label_id.saturating_add(1)); + Ok(()) + } + + fn apply_edge_label( + &mut self, + label: String, + label_id: u32, + wal_generation: Option, + ) -> Result<(), EngineError> { + validate_label_token_name(&label)?; + if label_id == 0 { + return Err(EngineError::InvalidOperation( + "edge-label token ID 0 is reserved".to_string(), + )); + } + if let Some(existing_id) = self.edge_label_to_id.get(&label) { + if *existing_id == label_id { + if let Some(wal_generation) = wal_generation { + self.edge_label_wal_generation + .entry(label) + .and_modify(|existing| *existing = (*existing).min(wal_generation)) + .or_insert(wal_generation); + } + return Ok(()); + } + return Err(EngineError::CorruptWal(format!( + "edge-label token conflict: edge label '{label}' is assigned to both label_id {existing_id} and {label_id}" + ))); + } + if let Some(existing_label) = self.edge_id_to_label.get(&label_id) { + return Err(EngineError::CorruptWal(format!( + "edge-label token conflict: label_id {label_id} is assigned to both '{existing_label}' and '{label}'" + ))); + } + self.edge_label_to_id.insert(label.clone(), label_id); + self.edge_id_to_label.insert(label_id, label); + if let Some(wal_generation) = wal_generation { + let stored_label = self + .edge_id_to_label + .get(&label_id) + .expect("edge-label reverse map was just inserted") + .clone(); + self.edge_label_wal_generation + .insert(stored_label, wal_generation); + } + self.next_edge_label_id = self.next_edge_label_id.max(label_id.saturating_add(1)); + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct ReadLabelCatalogSnapshot { + node_label_to_id: HashMap, + node_id_to_label: ReadLabelNameLookup, + edge_label_to_id: HashMap, + edge_id_to_label: ReadLabelNameLookup, +} + +#[derive(Debug, Clone)] +enum ReadLabelNameLookup { + Dense(Box<[Option>]>), + Sparse(HashMap>), +} + +impl ReadLabelNameLookup { + const MAX_DENSE_LEN: usize = 1_000_000; + + fn from_runtime_map(names: &BTreeMap) -> Self { + let Some(max_label_id) = names.keys().next_back().copied() else { + return Self::Dense(Vec::new().into_boxed_slice()); + }; + let dense_len = usize::try_from(max_label_id) + .ok() + .and_then(|max| max.checked_add(1)); + let dense_threshold = names.len().saturating_mul(8).saturating_add(64); + + if let Some(len) = + dense_len.filter(|&len| len <= Self::MAX_DENSE_LEN && len <= dense_threshold) + { + let mut dense = vec![None; len]; + for (&label_id, name) in names { + dense[label_id as usize] = Some(Arc::::from(name.as_str())); + } + return Self::Dense(dense.into_boxed_slice()); + } + + Self::Sparse( + names + .iter() + .map(|(&label_id, name)| (label_id, Arc::::from(name.as_str()))) + .collect(), + ) + } + + fn get(&self, label_id: u32) -> Option<&str> { + match self { + Self::Dense(names) => usize::try_from(label_id) + .ok() + .and_then(|idx| names.get(idx)) + .and_then(Option::as_deref), + Self::Sparse(names) => names.get(&label_id).map(AsRef::as_ref), + } + } +} + +trait LabelCatalogLookup { + fn node_label(&self, label_id: u32) -> Option<&str>; + fn edge_label(&self, label_id: u32) -> Option<&str>; +} + +impl LabelCatalogLookup for RuntimeLabelCatalog { + fn node_label(&self, label_id: u32) -> Option<&str> { + self.node_id_to_label.get(&label_id).map(String::as_str) + } + + fn edge_label(&self, label_id: u32) -> Option<&str> { + self.edge_id_to_label.get(&label_id).map(String::as_str) + } +} + +impl LabelCatalogLookup for ReadLabelCatalogSnapshot { + fn node_label(&self, label_id: u32) -> Option<&str> { + self.node_id_to_label.get(label_id) + } + + fn edge_label(&self, label_id: u32) -> Option<&str> { + self.edge_id_to_label.get(label_id) + } +} + +impl ReadLabelCatalogSnapshot { + fn from_runtime(catalog: &RuntimeLabelCatalog) -> Self { + Self { + node_label_to_id: catalog.node_label_to_id.clone().into_iter().collect(), + node_id_to_label: ReadLabelNameLookup::from_runtime_map(&catalog.node_id_to_label), + edge_label_to_id: catalog.edge_label_to_id.clone().into_iter().collect(), + edge_id_to_label: ReadLabelNameLookup::from_runtime_map(&catalog.edge_id_to_label), + } + } + + fn resolve_node_label_for_read(&self, label: &str) -> Result, EngineError> { + validate_label_token_name(label)?; + Ok(self.node_label_to_id.get(label).copied()) + } + + fn resolve_edge_label_for_read(&self, label: &str) -> Result, EngineError> { + validate_label_token_name(label)?; + Ok(self.edge_label_to_id.get(label).copied()) + } + + fn resolve_edge_label_filter( + &self, + edge_labels: Option<&[String]>, + ) -> Result<(LabelFilterResolution, Vec), EngineError> { + let Some(edge_labels) = edge_labels else { + return Ok((LabelFilterResolution::Unconstrained, Vec::new())); + }; + if edge_labels.is_empty() { + return Ok((LabelFilterResolution::Unconstrained, Vec::new())); + } + + let mut known = Vec::new(); + let mut warnings = Vec::new(); + for label in edge_labels { + match self.resolve_edge_label_for_read(label)? { + Some(label_id) => known.push(label_id), + None => push_query_warning(&mut warnings, QueryPlanWarning::UnknownEdgeLabel), + } + } + + if known.is_empty() { + return Ok((LabelFilterResolution::EmptyConstraint, warnings)); + } + known.sort_unstable(); + known.dedup(); + Ok((LabelFilterResolution::Known(known), warnings)) + } + + fn resolve_edge_label_filter_for_read( + &self, + edge_labels: Option<&[String]>, + ) -> Result { + Ok(self.resolve_edge_label_filter(edge_labels)?.0) + } + + #[allow(dead_code)] + fn resolve_node_label_filter_request( + &self, + filter: Option<&NodeLabelFilter>, + ) -> Result { + let Some(filter) = filter else { + return Ok(ResolvedNodeLabelFilter::Unconstrained); + }; + validate_node_label_filter(filter)?; + + let mut known = Vec::with_capacity(filter.labels.len()); + let mut unknown_label_count = 0usize; + for label in &filter.labels { + match self.resolve_node_label_for_read(label)? { + Some(label_id) => known.push(label_id), + None => unknown_label_count += 1, + } + } + + if known.is_empty() { + return Ok(ResolvedNodeLabelFilter::empty( + filter.mode, + unknown_label_count, + )); + } + if filter.mode == LabelMatchMode::All && unknown_label_count > 0 { + return Ok(ResolvedNodeLabelFilter::empty( + filter.mode, + unknown_label_count, + )); + } + Ok(ResolvedNodeLabelFilter::known( + filter.mode, + NodeLabelSet::from_label_ids(known)?, + unknown_label_count, + )) + } +} + +fn node_view_from_record( + record: NodeRecord, + catalog: &ReadLabelCatalogSnapshot, +) -> Result { + let labels = match record.label_ids.as_slice() { + &[label_id] => vec![catalog + .node_label(label_id) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "node record {} references missing node label_id {}", + record.id, label_id + )) + })? + .to_string()], + label_ids => { + let mut labels = Vec::with_capacity(label_ids.len()); + for &label_id in label_ids { + labels.push( + catalog + .node_label(label_id) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "node record {} references missing node label_id {}", + record.id, label_id + )) + })? + .to_string(), + ); + } + labels + } + }; + + Ok(NodeView { + id: record.id, + labels, + key: record.key, + props: record.props, + created_at: record.created_at, + updated_at: record.updated_at, + weight: record.weight, + dense_vector: record.dense_vector, + sparse_vector: record.sparse_vector, + }) +} + +fn node_view_from_record_with_resolved_label( + record: NodeRecord, + expected_label_id: u32, + catalog: &ReadLabelCatalogSnapshot, +) -> Result { + if !record.label_ids.contains(expected_label_id) { + return Err(EngineError::InvalidOperation(format!( + "node record {} resolved by label_id {} but found {:?}", + record.id, expected_label_id, record.label_ids + ))); + } + + node_view_from_record(record, catalog) +} + +fn edge_view_from_record( + record: EdgeRecord, + catalog: &ReadLabelCatalogSnapshot, +) -> Result { + let label = catalog + .edge_label(record.label_id) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "edge record {} references missing edge-label label_id {}", + record.id, record.label_id + )) + })? + .to_string(); + + Ok(EdgeView { + id: record.id, + from: record.from, + to: record.to, + label, + props: record.props, + created_at: record.created_at, + updated_at: record.updated_at, + weight: record.weight, + valid_from: record.valid_from, + valid_to: record.valid_to, + }) +} + +fn edge_view_from_record_with_resolved_label( + record: EdgeRecord, + expected_label_id: u32, + label: String, +) -> Result { + if record.label_id != expected_label_id { + return Err(EngineError::InvalidOperation(format!( + "edge record {} resolved by edge label '{}' expected label_id {} but found {}", + record.id, label, expected_label_id, record.label_id + ))); + } + + Ok(EdgeView { + id: record.id, + from: record.from, + to: record.to, + label, + props: record.props, + created_at: record.created_at, + updated_at: record.updated_at, + weight: record.weight, + valid_from: record.valid_from, + valid_to: record.valid_to, + }) +} + +fn neighbor_entry_from_record( + record: NeighborRecord, + catalog: &ReadLabelCatalogSnapshot, +) -> Result { + let label = catalog + .edge_label(record.edge_label_id) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "neighbor edge {} references missing edge-label label_id {}", + record.edge_id, record.edge_label_id + )) + })? + .to_string(); + + Ok(NeighborEntry { + node_id: record.node_id, + edge_id: record.edge_id, + label, + weight: record.weight, + valid_from: record.valid_from, + valid_to: record.valid_to, + }) +} + +fn resolve_node_label_for_read( + catalog: &RuntimeLabelCatalog, + label: &str, +) -> Result, EngineError> { + validate_label_token_name(label)?; + Ok(catalog.node_label_to_id.get(label).copied()) +} + +fn resolve_edge_label_for_read( + catalog: &RuntimeLabelCatalog, + label: &str, +) -> Result, EngineError> { + validate_label_token_name(label)?; + Ok(catalog.edge_label_to_id.get(label).copied()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum LabelFilterResolution { + Unconstrained, + Known(Vec), + EmptyConstraint, +} + +fn merge_runtime_label_catalog_into_manifest( + manifest: &mut ManifestState, + label_catalog: &Arc>, +) { + label_catalog.read().unwrap().apply_to_manifest(manifest); +} + +fn merge_checkpointed_label_catalog_into_manifest( + manifest: &mut ManifestState, + label_catalog: &Arc>, + max_wal_generation: Option, +) { + label_catalog + .read() + .unwrap() + .apply_checkpointed_to_manifest(manifest, max_wal_generation); +} + #[allow(clippy::too_many_arguments)] fn update_secondary_index_manifest_runtime( db_dir: &Path, @@ -320,6 +965,8 @@ fn update_secondary_index_manifest_runtime( next_node_id_seen: &AtomicU64, next_edge_id_seen: &AtomicU64, engine_seq_seen: &AtomicU64, + label_catalog: Option<&Arc>>, + max_token_checkpoint_wal_generation: Option, mutate: impl FnOnce(&mut ManifestState) -> Result<(), EngineError>, ) -> Result<(), EngineError> { let _guard = manifest_write_lock.lock().unwrap(); @@ -332,6 +979,13 @@ fn update_secondary_index_manifest_runtime( next_edge_id_seen, engine_seq_seen, ); + if let Some(label_catalog) = label_catalog { + merge_checkpointed_label_catalog_into_manifest( + &mut manifest, + label_catalog, + max_token_checkpoint_wal_generation, + ); + } write_manifest(db_dir, &manifest)?; sync_secondary_index_runtime_state(catalog_lock, entries_lock, &manifest.secondary_indexes)?; Ok(()) @@ -344,6 +998,29 @@ fn is_not_found_io_error(error: &EngineError) -> bool { ) } +fn manifestless_database_artifacts(path: &Path) -> Result, EngineError> { + let mut artifacts = Vec::new(); + if path.join("data.wal").exists() { + artifacts.push("data.wal".to_string()); + } + if path.join("segments").exists() { + artifacts.push("segments/".to_string()); + } + for entry in std::fs::read_dir(path)? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if name.starts_with("wal_") && name.ends_with(".wal") { + artifacts.push(name.to_string()); + } + } + artifacts.sort(); + artifacts.dedup(); + Ok(artifacts) +} + fn apply_secondary_index_failure_report( manifest: &mut ManifestState, report: &SecondaryIndexMaintenanceReport, @@ -374,22 +1051,6 @@ fn apply_secondary_index_failure_report( } } -fn equality_index_ids_snapshot(entries: &[SecondaryIndexManifestEntry]) -> NodeIdSet { - entries - .iter() - .filter(|entry| matches!(entry.kind, SecondaryIndexKind::Equality)) - .map(|entry| entry.index_id) - .collect() -} - -fn range_index_ids_snapshot(entries: &[SecondaryIndexManifestEntry]) -> NodeIdSet { - entries - .iter() - .filter(|entry| matches!(entry.kind, SecondaryIndexKind::Range { .. })) - .map(|entry| entry.index_id) - .collect() -} - fn reconcile_background_output_equality_declarations( manifest: &mut ManifestState, maintained_equality_index_ids: &NodeIdSet, @@ -459,6 +1120,7 @@ fn mark_secondary_index_failed( next_node_id_seen: &AtomicU64, next_edge_id_seen: &AtomicU64, engine_seq_seen: &AtomicU64, + label_catalog: &Arc>, index_id: u64, error: &EngineError, ) { @@ -471,6 +1133,8 @@ fn mark_secondary_index_failed( next_node_id_seen, next_edge_id_seen, engine_seq_seen, + Some(label_catalog), + None, |manifest| { if let Some(entry) = manifest .secondary_indexes @@ -487,66 +1151,94 @@ fn mark_secondary_index_failed( fn build_secondary_eq_groups_for_segment( segment: &SegmentReader, - type_id: u32, + target_label_id: u32, prop_key: &str, ) -> Result>, EngineError> { - let target_key_hash = hash_prop_key(prop_key); - let legacy_hashes = segment.raw_node_prop_hashes_mmap(); let mut groups: BTreeMap> = BTreeMap::new(); for index in 0..segment.node_meta_count() as usize { - let ( - node_id, - data_offset, - _data_len, - node_type_id, - _updated_at, - _weight, - _key_len, - prop_hash_offset, - prop_hash_count, - _last_write_seq, - ) = segment.node_meta_at(index)?; - if node_type_id != type_id { + let meta = segment.node_meta_at(index)?; + if !meta.label_ids.contains(target_label_id) { continue; } - let mut value_hash = None; - if !legacy_hashes.is_empty() && prop_hash_count > 0 { - let base = prop_hash_offset as usize; - for pair_index in 0..prop_hash_count as usize { - let pair_off = base + pair_index * 16; - let pair_end = pair_off + 16; - if pair_end > legacy_hashes.len() { - return Err(EngineError::CorruptRecord(format!( - "node {} prop hash pair at offset {} exceeds source length {}", - node_id, - pair_off, - legacy_hashes.len() - ))); - } - let key_hash = - u64::from_le_bytes(legacy_hashes[pair_off..pair_off + 8].try_into().unwrap()); - if key_hash != target_key_hash { - continue; - } - value_hash = Some(u64::from_le_bytes( - legacy_hashes[pair_off + 8..pair_off + 16] - .try_into() - .unwrap(), - )); - break; - } + if let Some(value_hash) = segment + .node_property_value_at_offset(meta.node_id, meta.data_offset, prop_key)? + .map(|value| hash_prop_value(&value)) + { + groups.entry(value_hash).or_default().push(meta.node_id); } + } - if value_hash.is_none() { - value_hash = segment - .node_property_value_at_offset(node_id, data_offset, prop_key)? - .map(|value| hash_prop_value(&value)); + for ids in groups.values_mut() { + ids.sort_unstable(); + ids.dedup(); + } + Ok(groups) +} + +fn build_secondary_range_entries_for_segment( + segment: &SegmentReader, + target_label_id: u32, + prop_key: &str, + domain: SecondaryIndexRangeDomain, +) -> Result, EngineError> { + let mut entries = Vec::new(); + + for index in 0..segment.node_meta_count() as usize { + let meta = segment.node_meta_at(index)?; + if !meta.label_ids.contains(target_label_id) { + continue; + } + + let Some(value) = + segment.node_property_value_at_offset(meta.node_id, meta.data_offset, prop_key)? + else { + continue; + }; + let Some(encoded_value) = encode_range_prop_value(domain, &value) else { + continue; + }; + entries.push((encoded_value, meta.node_id)); + } + + entries.sort_unstable(); + entries.dedup(); + Ok(entries) +} + +fn build_edge_secondary_eq_groups_for_segment( + segment: &SegmentReader, + label_id: u32, + prop_key: &str, +) -> Result>, EngineError> { + let mut groups: BTreeMap> = BTreeMap::new(); + + for index in 0..segment.edge_meta_count() as usize { + let ( + edge_id, + data_offset, + _data_len, + _from, + _to, + edge_label_id, + _updated_at, + _weight, + _valid_from, + _valid_to, + _last_write_seq, + ) = segment.edge_meta_at(index)?; + if edge_label_id != label_id { + continue; } - if let Some(value_hash) = value_hash { - groups.entry(value_hash).or_default().push(node_id); + if let Some(value) = + segment.edge_property_value_at_offset(edge_id, data_offset, prop_key)? + { + groups + .entry(hash_prop_value(&value)) + .or_default() + .push(edge_id); } } @@ -557,39 +1249,40 @@ fn build_secondary_eq_groups_for_segment( Ok(groups) } -fn build_secondary_range_entries_for_segment( +fn build_edge_secondary_range_entries_for_segment( segment: &SegmentReader, - type_id: u32, + label_id: u32, prop_key: &str, domain: SecondaryIndexRangeDomain, ) -> Result, EngineError> { let mut entries = Vec::new(); - for index in 0..segment.node_meta_count() as usize { + for index in 0..segment.edge_meta_count() as usize { let ( - node_id, + edge_id, data_offset, _data_len, - node_type_id, + _from, + _to, + edge_label_id, _updated_at, _weight, - _key_len, - _prop_hash_offset, - _prop_hash_count, + _valid_from, + _valid_to, _last_write_seq, - ) = segment.node_meta_at(index)?; - if node_type_id != type_id { + ) = segment.edge_meta_at(index)?; + if edge_label_id != label_id { continue; } - let Some(value) = segment.node_property_value_at_offset(node_id, data_offset, prop_key)? + let Some(value) = segment.edge_property_value_at_offset(edge_id, data_offset, prop_key)? else { continue; }; let Some(encoded_value) = encode_range_prop_value(domain, &value) else { continue; }; - entries.push((encoded_value, node_id)); + entries.push((encoded_value, edge_id)); } entries.sort_unstable(); @@ -599,48 +1292,45 @@ fn build_secondary_range_entries_for_segment( fn install_secondary_eq_sidecar( seg_dir: &Path, - index_id: u64, + entry: &SecondaryIndexManifestEntry, groups: &BTreeMap>, ) -> Result<(), EngineError> { - let index_dir = seg_dir.join("secondary_indexes"); - match std::fs::create_dir(&index_dir) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error.into()), - } - let final_path = node_prop_eq_sidecar_path(seg_dir, index_id); - let tmp_path = index_dir.join(format!(".node_prop_eq_{}.tmp", index_id)); - write_node_prop_eq_sidecar_to_path(&tmp_path, groups)?; - std::fs::rename(&tmp_path, &final_path)?; - fsync_dir(&index_dir)?; - Ok(()) + publish_node_prop_eq_sidecar_component(seg_dir, entry, groups) } fn install_secondary_range_sidecar( seg_dir: &Path, - index_id: u64, + entry: &SecondaryIndexManifestEntry, entries: &[(u64, u64)], ) -> Result<(), EngineError> { - let index_dir = seg_dir.join("secondary_indexes"); - match std::fs::create_dir(&index_dir) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error.into()), - } - let final_path = node_prop_range_sidecar_path(seg_dir, index_id); - let tmp_path = index_dir.join(format!(".node_prop_range_{}.tmp", index_id)); - write_node_prop_range_sidecar_to_path(&tmp_path, entries)?; - std::fs::rename(&tmp_path, &final_path)?; - fsync_dir(&index_dir)?; - Ok(()) + publish_node_prop_range_sidecar_component(seg_dir, entry, entries) +} + +fn install_edge_secondary_eq_sidecar( + seg_dir: &Path, + entry: &SecondaryIndexManifestEntry, + groups: &BTreeMap>, +) -> Result<(), EngineError> { + publish_edge_prop_eq_sidecar_component(seg_dir, entry, groups) +} + +fn install_edge_secondary_range_sidecar( + seg_dir: &Path, + entry: &SecondaryIndexManifestEntry, + entries: &[(u64, u64)], +) -> Result<(), EngineError> { + publish_edge_prop_range_sidecar_component(seg_dir, entry, entries) } #[derive(Clone)] struct SecondaryEqBuildSnapshot { dense_config: Option, - type_id: u32, + target: SecondaryIndexTargetDiscriminant, + target_label_id: u32, prop_key: String, segment_ids: Vec, + segment_infos: Vec, + secondary_indexes: Vec, } enum SecondaryEqCoverageStatus { @@ -660,10 +1350,13 @@ enum SecondaryEqFinalizeOutcome { #[derive(Clone)] struct SecondaryRangeBuildSnapshot { dense_config: Option, - type_id: u32, + target: SecondaryIndexTargetDiscriminant, + target_label_id: u32, prop_key: String, domain: SecondaryIndexRangeDomain, segment_ids: Vec, + segment_infos: Vec, + secondary_indexes: Vec, } enum SecondaryRangeCoverageStatus { @@ -680,11 +1373,27 @@ enum SecondaryRangeFinalizeOutcome { Inactive, } +fn segment_info_for_id(segment_infos: &[SegmentInfo], segment_id: u64) -> Option<&SegmentInfo> { + segment_infos + .iter() + .find(|segment| segment.id == segment_id) +} + +fn planner_stats_target_from_discriminant( + target: SecondaryIndexTargetDiscriminant, +) -> PlannerStatsDeclaredIndexTarget { + match target { + SecondaryIndexTargetDiscriminant::Node => PlannerStatsDeclaredIndexTarget::NodeProperty, + SecondaryIndexTargetDiscriminant::Edge => PlannerStatsDeclaredIndexTarget::EdgeProperty, + } +} + #[derive(Clone, Debug)] struct SecondaryIndexReadyApplied { index_id: u64, + target: SecondaryIndexTarget, kind: SecondaryIndexKind, - type_id: u32, + target_label_id: u32, prop_key: String, declaration_fingerprint: u64, snapshot_segment_ids: Vec, @@ -698,12 +1407,14 @@ impl SecondaryIndexReadyApplied { if entry.state != SecondaryIndexState::Ready { return None; } - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; + let target_label_id = secondary_index_target_label_id(&entry.target); + let prop_key = secondary_index_target_prop_key(&entry.target); Some(Self { index_id: entry.index_id, + target: entry.target.clone(), kind: entry.kind.clone(), - type_id: *type_id, - prop_key: prop_key.clone(), + target_label_id, + prop_key: prop_key.to_string(), declaration_fingerprint: planner_stats_declaration_fingerprint_for_entry(entry), snapshot_segment_ids, }) @@ -712,13 +1423,15 @@ impl SecondaryIndexReadyApplied { fn matches_entry(&self, entry: &SecondaryIndexManifestEntry) -> bool { if entry.state != SecondaryIndexState::Ready || entry.index_id != self.index_id + || entry.target != self.target || entry.kind != self.kind { return false; } - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; - *type_id == self.type_id - && prop_key == &self.prop_key + let target_label_id = secondary_index_target_label_id(&entry.target); + let prop_key = secondary_index_target_prop_key(&entry.target); + target_label_id == self.target_label_id + && prop_key == self.prop_key && planner_stats_declaration_fingerprint_for_entry(entry) == self.declaration_fingerprint } @@ -746,14 +1459,21 @@ fn load_secondary_eq_build_snapshot( return Ok(None); } - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = entry.target; + let target = secondary_index_target_discriminant(&entry.target); + let target_label_id = secondary_index_target_label_id(&entry.target); + let prop_key = secondary_index_target_prop_key(&entry.target).to_string(); let mut segment_ids: Vec = manifest.segments.iter().map(|segment| segment.id).collect(); segment_ids.sort_unstable(); + let mut segment_infos = manifest.segments.clone(); + segment_infos.sort_by_key(|segment| segment.id); Ok(Some(SecondaryEqBuildSnapshot { dense_config: manifest.dense_vector.clone(), - type_id, + target, + target_label_id, prop_key, segment_ids, + segment_infos, + secondary_indexes: manifest.secondary_indexes.clone(), })) } @@ -772,23 +1492,89 @@ fn build_secondary_eq_sidecars_for_snapshot( if !seg_path.exists() { continue; } + let Some(seg_info) = segment_info_for_id(&snapshot.segment_infos, segment_id) else { + continue; + }; - let segment = - match SegmentReader::open(&seg_path, segment_id, snapshot.dense_config.as_ref()) { - Ok(segment) => segment, - Err(error) if is_not_found_io_error(&error) => continue, - Err(error) => return Err(error), - }; + let segment = match SegmentReader::open_with_info( + &seg_path, + seg_info, + snapshot.dense_config.as_ref(), + &snapshot.secondary_indexes, + ) { + Ok(segment) => segment, + Err(error) if is_not_found_io_error(&error) => continue, + Err(error) => return Err(error), + }; + + if snapshot.target == SecondaryIndexTargetDiscriminant::Edge { + match segment.validate_secondary_eq_sidecar_for_target( + index_id, + PlannerStatsDeclaredIndexTarget::EdgeProperty, + ) { + Ok(true) => continue, + Ok(false) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; + let groups = build_edge_secondary_eq_groups_for_segment( + &segment, + snapshot.target_label_id, + &snapshot.prop_key, + )?; + match install_edge_secondary_eq_sidecar(&seg_path, entry, &groups) { + Ok(()) => {} + Err(error) if is_not_found_io_error(&error) => {} + Err(error) => return Err(error), + } + } + Err(error) if is_not_found_io_error(&error) => {} + Err(_) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; + let groups = build_edge_secondary_eq_groups_for_segment( + &segment, + snapshot.target_label_id, + &snapshot.prop_key, + )?; + match install_edge_secondary_eq_sidecar(&seg_path, entry, &groups) { + Ok(()) => {} + Err(error) if is_not_found_io_error(&error) => {} + Err(error) => return Err(error), + } + } + } + continue; + } - match segment.validate_secondary_eq_sidecar(index_id) { + match segment.validate_secondary_eq_sidecar_for_target( + index_id, + planner_stats_target_from_discriminant(snapshot.target), + ) { Ok(true) => continue, Ok(false) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; let groups = build_secondary_eq_groups_for_segment( &segment, - snapshot.type_id, + snapshot.target_label_id, &snapshot.prop_key, )?; - match install_secondary_eq_sidecar(&seg_path, index_id, &groups) { + match install_secondary_eq_sidecar(&seg_path, entry, &groups) { Ok(()) => {} Err(error) if is_not_found_io_error(&error) => {} Err(error) => return Err(error), @@ -796,12 +1582,19 @@ fn build_secondary_eq_sidecars_for_snapshot( } Err(error) if is_not_found_io_error(&error) => {} Err(_) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; let groups = build_secondary_eq_groups_for_segment( &segment, - snapshot.type_id, + snapshot.target_label_id, &snapshot.prop_key, )?; - match install_secondary_eq_sidecar(&seg_path, index_id, &groups) { + match install_secondary_eq_sidecar(&seg_path, entry, &groups) { Ok(()) => {} Err(error) if is_not_found_io_error(&error) => {} Err(error) => return Err(error), @@ -831,18 +1624,29 @@ fn validate_secondary_eq_snapshot_coverage( all_present = false; continue; } + let Some(seg_info) = segment_info_for_id(&snapshot.segment_infos, segment_id) else { + all_present = false; + continue; + }; - let segment = - match SegmentReader::open(&seg_path, segment_id, snapshot.dense_config.as_ref()) { - Ok(segment) => segment, - Err(error) if is_not_found_io_error(&error) => { - all_present = false; - continue; - } - Err(error) => return Err(error), - }; + let segment = match SegmentReader::open_with_info( + &seg_path, + seg_info, + snapshot.dense_config.as_ref(), + &snapshot.secondary_indexes, + ) { + Ok(segment) => segment, + Err(error) if is_not_found_io_error(&error) => { + all_present = false; + continue; + } + Err(error) => return Err(error), + }; - match segment.validate_secondary_eq_sidecar(index_id) { + match segment.validate_secondary_eq_sidecar_for_target( + index_id, + planner_stats_target_from_discriminant(snapshot.target), + ) { Ok(true) => {} Ok(false) => { all_present = false; @@ -869,6 +1673,7 @@ fn finalize_secondary_eq_build_snapshot( next_node_id_seen: &AtomicU64, next_edge_id_seen: &AtomicU64, engine_seq_seen: &AtomicU64, + label_catalog: &Arc>, index_id: u64, snapshot: &SecondaryEqBuildSnapshot, coverage: &SecondaryEqCoverageStatus, @@ -882,6 +1687,8 @@ fn finalize_secondary_eq_build_snapshot( next_node_id_seen, next_edge_id_seen, engine_seq_seen, + Some(label_catalog), + None, |manifest| { let Some(entry_pos) = manifest .secondary_indexes @@ -961,15 +1768,22 @@ fn load_secondary_range_build_snapshot( let SecondaryIndexKind::Range { domain } = entry.kind else { return Ok(None); }; - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = entry.target; + let target = secondary_index_target_discriminant(&entry.target); + let target_label_id = secondary_index_target_label_id(&entry.target); + let prop_key = secondary_index_target_prop_key(&entry.target).to_string(); let mut segment_ids: Vec = manifest.segments.iter().map(|segment| segment.id).collect(); segment_ids.sort_unstable(); + let mut segment_infos = manifest.segments.clone(); + segment_infos.sort_by_key(|segment| segment.id); Ok(Some(SecondaryRangeBuildSnapshot { dense_config: manifest.dense_vector.clone(), - type_id, + target, + target_label_id, prop_key, domain, segment_ids, + segment_infos, + secondary_indexes: manifest.secondary_indexes.clone(), })) } @@ -988,24 +1802,92 @@ fn build_secondary_range_sidecars_for_snapshot( if !seg_path.exists() { continue; } + let Some(seg_info) = segment_info_for_id(&snapshot.segment_infos, segment_id) else { + continue; + }; - let segment = - match SegmentReader::open(&seg_path, segment_id, snapshot.dense_config.as_ref()) { - Ok(segment) => segment, - Err(error) if is_not_found_io_error(&error) => continue, - Err(error) => return Err(error), - }; + let segment = match SegmentReader::open_with_info( + &seg_path, + seg_info, + snapshot.dense_config.as_ref(), + &snapshot.secondary_indexes, + ) { + Ok(segment) => segment, + Err(error) if is_not_found_io_error(&error) => continue, + Err(error) => return Err(error), + }; + + if snapshot.target == SecondaryIndexTargetDiscriminant::Edge { + match segment.validate_secondary_range_sidecar_for_target( + index_id, + PlannerStatsDeclaredIndexTarget::EdgeProperty, + ) { + Ok(true) => continue, + Ok(false) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; + let entries = build_edge_secondary_range_entries_for_segment( + &segment, + snapshot.target_label_id, + &snapshot.prop_key, + snapshot.domain, + )?; + match install_edge_secondary_range_sidecar(&seg_path, entry, &entries) { + Ok(()) => {} + Err(error) if is_not_found_io_error(&error) => {} + Err(error) => return Err(error), + } + } + Err(error) if is_not_found_io_error(&error) => {} + Err(_) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; + let entries = build_edge_secondary_range_entries_for_segment( + &segment, + snapshot.target_label_id, + &snapshot.prop_key, + snapshot.domain, + )?; + match install_edge_secondary_range_sidecar(&seg_path, entry, &entries) { + Ok(()) => {} + Err(error) if is_not_found_io_error(&error) => {} + Err(error) => return Err(error), + } + } + } + continue; + } - match segment.validate_secondary_range_sidecar(index_id) { + match segment.validate_secondary_range_sidecar_for_target( + index_id, + planner_stats_target_from_discriminant(snapshot.target), + ) { Ok(true) => continue, Ok(false) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; let entries = build_secondary_range_entries_for_segment( &segment, - snapshot.type_id, + snapshot.target_label_id, &snapshot.prop_key, snapshot.domain, )?; - match install_secondary_range_sidecar(&seg_path, index_id, &entries) { + match install_secondary_range_sidecar(&seg_path, entry, &entries) { Ok(()) => {} Err(error) if is_not_found_io_error(&error) => {} Err(error) => return Err(error), @@ -1013,13 +1895,20 @@ fn build_secondary_range_sidecars_for_snapshot( } Err(error) if is_not_found_io_error(&error) => {} Err(_) => { + let Some(entry) = snapshot + .secondary_indexes + .iter() + .find(|entry| entry.index_id == index_id) + else { + continue; + }; let entries = build_secondary_range_entries_for_segment( &segment, - snapshot.type_id, + snapshot.target_label_id, &snapshot.prop_key, snapshot.domain, )?; - match install_secondary_range_sidecar(&seg_path, index_id, &entries) { + match install_secondary_range_sidecar(&seg_path, entry, &entries) { Ok(()) => {} Err(error) if is_not_found_io_error(&error) => {} Err(error) => return Err(error), @@ -1049,18 +1938,29 @@ fn validate_secondary_range_snapshot_coverage( all_present = false; continue; } + let Some(seg_info) = segment_info_for_id(&snapshot.segment_infos, segment_id) else { + all_present = false; + continue; + }; - let segment = - match SegmentReader::open(&seg_path, segment_id, snapshot.dense_config.as_ref()) { - Ok(segment) => segment, - Err(error) if is_not_found_io_error(&error) => { - all_present = false; - continue; - } - Err(error) => return Err(error), - }; + let segment = match SegmentReader::open_with_info( + &seg_path, + seg_info, + snapshot.dense_config.as_ref(), + &snapshot.secondary_indexes, + ) { + Ok(segment) => segment, + Err(error) if is_not_found_io_error(&error) => { + all_present = false; + continue; + } + Err(error) => return Err(error), + }; - match segment.validate_secondary_range_sidecar(index_id) { + match segment.validate_secondary_range_sidecar_for_target( + index_id, + planner_stats_target_from_discriminant(snapshot.target), + ) { Ok(true) => {} Ok(false) => { all_present = false; @@ -1087,6 +1987,7 @@ fn finalize_secondary_range_build_snapshot( next_node_id_seen: &AtomicU64, next_edge_id_seen: &AtomicU64, engine_seq_seen: &AtomicU64, + label_catalog: &Arc>, index_id: u64, snapshot: &SecondaryRangeBuildSnapshot, coverage: &SecondaryRangeCoverageStatus, @@ -1100,6 +2001,8 @@ fn finalize_secondary_range_build_snapshot( next_node_id_seen, next_edge_id_seen, engine_seq_seen, + Some(label_catalog), + None, |manifest| { let Some(entry_pos) = manifest .secondary_indexes @@ -1165,6 +2068,7 @@ fn process_secondary_index_build( next_node_id_seen: &AtomicU64, next_edge_id_seen: &AtomicU64, engine_seq_seen: &AtomicU64, + label_catalog: &Arc>, #[cfg(test)] build_pause: &Arc>>, index_id: u64, cancel: &AtomicBool, @@ -1204,6 +2108,7 @@ fn process_secondary_index_build( next_node_id_seen, next_edge_id_seen, engine_seq_seen, + label_catalog, index_id, &snapshot, &coverage, @@ -1232,6 +2137,7 @@ fn process_secondary_index_build( next_node_id_seen, next_edge_id_seen, engine_seq_seen, + label_catalog, index_id, &snapshot, &coverage, @@ -1253,6 +2159,8 @@ struct TargetedStatsRefreshSnapshot { target_entry: SecondaryIndexManifestEntry, ready_indexes: Vec, segment_ids: Vec, + segment_infos: Vec, + secondary_indexes: Vec, } fn load_targeted_stats_refresh_snapshot( @@ -1286,6 +2194,13 @@ fn load_targeted_stats_refresh_snapshot( if segment_ids.is_empty() { return Ok(None); } + let mut segment_infos: Vec<_> = manifest + .segments + .iter() + .filter(|segment| snapshot_segment_ids.contains(&segment.id)) + .cloned() + .collect(); + segment_infos.sort_by_key(|segment| segment.id); let mut ready_indexes: Vec<_> = manifest .secondary_indexes @@ -1299,6 +2214,8 @@ fn load_targeted_stats_refresh_snapshot( target_entry, ready_indexes, segment_ids, + segment_infos, + secondary_indexes: manifest.secondary_indexes.clone(), })) } @@ -1323,10 +2240,16 @@ fn target_secondary_sidecar_is_valid( segment: &SegmentReader, ready: &SecondaryIndexReadyApplied, ) -> Result { + let target = match &ready.target { + SecondaryIndexTarget::NodeProperty { .. } => PlannerStatsDeclaredIndexTarget::NodeProperty, + SecondaryIndexTarget::EdgeProperty { .. } => PlannerStatsDeclaredIndexTarget::EdgeProperty, + }; match ready.kind { - SecondaryIndexKind::Equality => segment.validate_secondary_eq_sidecar(ready.index_id), + SecondaryIndexKind::Equality => { + segment.validate_secondary_eq_sidecar_for_target(ready.index_id, target) + } SecondaryIndexKind::Range { .. } => { - segment.validate_secondary_range_sidecar(ready.index_id) + segment.validate_secondary_range_sidecar_for_target(ready.index_id, target) } } } @@ -1337,6 +2260,8 @@ fn refresh_ready_secondary_index_planner_stats( ready: &SecondaryIndexReadyApplied, cancel: &AtomicBool, ) -> Vec<(u64, Arc)> { + const TARGETED_STATS_REFRESH_MAX_ATTEMPTS: usize = 4; + let Some(initial_snapshot) = load_targeted_stats_refresh_snapshot(db_dir, manifest_write_lock, ready) .ok() @@ -1351,46 +2276,106 @@ fn refresh_ready_secondary_index_planner_stats( break; } let seg_dir = segment_dir(db_dir, segment_id); - let segment = - match SegmentReader::open(&seg_dir, segment_id, initial_snapshot.dense_config.as_ref()) - { - Ok(segment) => segment, - Err(error) if is_not_found_io_error(&error) => continue, - Err(_) => continue, - }; + let Some(initial_seg_info) = + segment_info_for_id(&initial_snapshot.segment_infos, segment_id) + else { + continue; + }; + let segment = match SegmentReader::open_with_info( + &seg_dir, + initial_seg_info, + initial_snapshot.dense_config.as_ref(), + &initial_snapshot.secondary_indexes, + ) { + Ok(segment) => segment, + Err(error) if is_not_found_io_error(&error) => continue, + Err(_) => continue, + }; match target_secondary_sidecar_is_valid(&segment, ready) { Ok(true) => {} Ok(false) | Err(_) => continue, } - let Some(snapshot) = targeted_refresh_snapshot_contains_segment( - db_dir, - manifest_write_lock, - ready, - segment_id, - ) - .ok() - .flatten() else { + let mut latest_snapshot = None; + for _ in 0..TARGETED_STATS_REFRESH_MAX_ATTEMPTS { + if cancel.load(Ordering::Relaxed) { + break; + } + let Some(snapshot) = targeted_refresh_snapshot_contains_segment( + db_dir, + manifest_write_lock, + ready, + segment_id, + ) + .ok() + .flatten() else { + latest_snapshot = None; + break; + }; + let Some(seg_info) = segment_info_for_id(&snapshot.segment_infos, segment_id) else { + latest_snapshot = None; + break; + }; + let latest_segment = match SegmentReader::open_with_info( + &seg_dir, + seg_info, + snapshot.dense_config.as_ref(), + &snapshot.secondary_indexes, + ) { + Ok(segment) => segment, + Err(error) if is_not_found_io_error(&error) => { + latest_snapshot = None; + break; + } + Err(_) => { + latest_snapshot = None; + break; + } + }; + match target_secondary_sidecar_is_valid(&latest_segment, ready) { + Ok(true) => {} + Ok(false) | Err(_) => { + latest_snapshot = None; + break; + } + } + match write_targeted_secondary_index_planner_stats_sidecar( + &seg_dir, + &latest_segment, + &snapshot.target_entry, + &snapshot.ready_indexes, + ) { + Ok(_) => { + latest_snapshot = Some(snapshot); + break; + } + Err(error) if is_optional_component_publication_conflict(&error) => { + latest_snapshot = Some(snapshot); + continue; + } + Err(_) => { + latest_snapshot = Some(snapshot); + break; + } + } + } + let Some(snapshot) = latest_snapshot else { + continue; + }; + let Some(seg_info) = segment_info_for_id(&snapshot.segment_infos, segment_id) else { continue; }; - match write_targeted_secondary_index_planner_stats_sidecar( + + let refreshed_reader = match SegmentReader::open_with_info( &seg_dir, - &segment, - &snapshot.target_entry, - &snapshot.ready_indexes, + seg_info, + snapshot.dense_config.as_ref(), + &snapshot.secondary_indexes, ) { - Ok(PlannerStatsWriteOutcome::Written) => {} - Ok(PlannerStatsWriteOutcome::SkippedOversize) - | Ok(PlannerStatsWriteOutcome::SkippedTargetUnavailable) - | Err(_) => continue, - } - - let refreshed_reader = - match SegmentReader::open(&seg_dir, segment_id, snapshot.dense_config.as_ref()) { - Ok(reader) => reader, - Err(error) if is_not_found_io_error(&error) => continue, - Err(_) => continue, - }; + Ok(reader) => reader, + Err(error) if is_not_found_io_error(&error) => continue, + Err(_) => continue, + }; for entry in &snapshot.ready_indexes { refreshed_reader.warm_declared_index_runtime_coverage(entry); } @@ -1404,7 +2389,7 @@ fn refresh_ready_secondary_index_planner_stats( fn process_secondary_index_drop_cleanup( db_dir: &Path, - index_id: u64, + entry: &SecondaryIndexManifestEntry, cancel: &AtomicBool, ) -> Result<(), EngineError> { let manifest = load_manifest_readonly(db_dir)? @@ -1415,10 +2400,18 @@ fn process_secondary_index_drop_cleanup( } let seg_dir = segment_dir(db_dir, segment_info.id); - for sidecar_path in [ - node_prop_eq_sidecar_path(&seg_dir, index_id), - node_prop_range_sidecar_path(&seg_dir, index_id), - ] { + let mut sidecar_paths = match remove_secondary_index_component_records(&seg_dir, entry) { + Ok(paths) => paths, + Err(error) if is_not_found_io_error(&error) => Vec::new(), + Err(error) => return Err(error), + }; + sidecar_paths.extend(secondary_index_sidecar_paths_for_entry(&seg_dir, entry)); + + let mut seen_paths = HashSet::new(); + for sidecar_path in sidecar_paths { + if !seen_paths.insert(sidecar_path.clone()) { + continue; + } if sidecar_path.exists() { let _ = std::fs::remove_file(&sidecar_path); if let Some(parent) = sidecar_path.parent() { @@ -1426,6 +2419,7 @@ fn process_secondary_index_drop_cleanup( } } } + cleanup_orphan_optional_component_files(&seg_dir); } Ok(()) } @@ -1442,6 +2436,7 @@ fn bg_secondary_index_worker( next_node_id_seen: Arc, next_edge_id_seen: Arc, engine_seq_seen: Arc, + label_catalog: Arc>, #[cfg(test)] build_pause: Arc>>, ) { while let Ok(job) = rx.recv() { @@ -1458,6 +2453,7 @@ fn bg_secondary_index_worker( next_node_id_seen.as_ref(), next_edge_id_seen.as_ref(), engine_seq_seen.as_ref(), + &label_catalog, #[cfg(test)] &build_pause, index_id, @@ -1473,6 +2469,7 @@ fn bg_secondary_index_worker( next_node_id_seen.as_ref(), next_edge_id_seen.as_ref(), engine_seq_seen.as_ref(), + &label_catalog, index_id, &error, ); @@ -1498,8 +2495,8 @@ fn bg_secondary_index_worker( } } } - SecondaryIndexJob::DropCleanup { index_id } => { - let _ = process_secondary_index_drop_cleanup(&db_dir, index_id, &cancel); + SecondaryIndexJob::DropCleanup { entry } => { + let _ = process_secondary_index_drop_cleanup(&db_dir, &entry, &cancel); } SecondaryIndexJob::Shutdown => break, } @@ -1548,6 +2545,11 @@ fn normalize_wal_op_for_write( normalized.sparse_vector = sparse_vector; Ok(WalOp::UpsertNode(normalized)) } + WalOp::BeginAtomicBatch { .. } | WalOp::CommitAtomicBatch { .. } => { + Err(EngineError::InvalidOperation( + "WAL atomic batch markers are not write operations".into(), + )) + } _ => Ok(op.clone()), } } @@ -1567,6 +2569,46 @@ fn normalize_wal_op_for_replay( }) } +fn validate_or_apply_replayed_label_token_op( + catalog: &mut RuntimeLabelCatalog, + op: &WalOp, + wal_generation_id: u64, +) -> Result<(), EngineError> { + match op { + WalOp::EnsureNodeLabel { label, label_id } => { + catalog.apply_node_label(label.clone(), *label_id, Some(wal_generation_id)) + } + WalOp::EnsureEdgeLabel { label, label_id } => { + catalog.apply_edge_label(label.clone(), *label_id, Some(wal_generation_id)) + } + WalOp::UpsertNode(node) => { + for &label_id in node.label_ids.as_slice() { + if !catalog.node_id_to_label.contains_key(&label_id) { + return Err(EngineError::CorruptWal(format!( + "node record {} references missing node label label_id {}", + node.id, label_id + ))); + } + } + Ok(()) + } + WalOp::UpsertEdge(edge) => { + if catalog.edge_id_to_label.contains_key(&edge.label_id) { + Ok(()) + } else { + Err(EngineError::CorruptWal(format!( + "edge record {} references missing edge-label label_id {}", + edge.id, edge.label_id + ))) + } + } + WalOp::DeleteNode { .. } | WalOp::DeleteEdge { .. } => Ok(()), + WalOp::BeginAtomicBatch { .. } | WalOp::CommitAtomicBatch { .. } => Err( + EngineError::CorruptWal("WAL atomic batch marker reached normal replay apply".into()), + ), + } +} + /// Returns true if an edge is valid (not expired, not future) at the given reference time. /// Same predicate used by `neighbors()`. Extracted to prevent drift. #[inline] @@ -1580,15 +2622,15 @@ fn is_edge_valid_at(valid_from: i64, valid_to: i64, reference_time: i64) -> bool /// Core prune-policy match: does a single (precomputed) policy match the given fields? /// AND within policy: all set criteria must match. fn matches_prune_cutoff( - type_id: u32, + label_ids: &NodeLabelSet, updated_at: i64, weight: f32, policy_age_cutoff: Option, policy_max_weight: Option, - policy_type_id: Option, + policy_label_id: Option, ) -> bool { - if let Some(tid) = policy_type_id { - if type_id != tid { + if let Some(label_id) = policy_label_id { + if !label_ids.contains(label_id) { return false; } } @@ -1605,18 +2647,73 @@ fn matches_prune_cutoff( true } +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ResolvedPrunePolicy { + max_age_ms: Option, + max_weight: Option, + label_id: Option, +} + +fn public_prune_policy_from_resolved( + policy: &ResolvedPrunePolicy, + catalog: &RuntimeLabelCatalog, +) -> Result { + let label = policy + .label_id + .map(|label_id| { + catalog + .node_id_to_label + .get(&label_id) + .cloned() + .ok_or_else(|| { + EngineError::ManifestError(format!( + "prune policy references missing node label_id {label_id}" + )) + }) + }) + .transpose()?; + Ok(PrunePolicy { + max_age_ms: policy.max_age_ms, + max_weight: policy.max_weight, + label, + }) +} + +fn resolve_manifest_prune_policy( + policy: &PrunePolicy, + catalog: &RuntimeLabelCatalog, +) -> Result { + let label_id = policy + .label + .as_deref() + .map(|label| { + validate_label_token_name(label)?; + catalog.node_label_to_id.get(label).copied().ok_or_else(|| { + EngineError::ManifestError(format!( + "prune policy references missing node label '{label}'" + )) + }) + }) + .transpose()?; + Ok(ResolvedPrunePolicy { + max_age_ms: policy.max_age_ms, + max_weight: policy.max_weight, + label_id, + }) +} + struct PrecomputedPruneCutoffs { - /// (age_cutoff, max_weight, type_id) per policy. + /// (age_cutoff, max_weight, label_id) per policy. policies: Vec<(Option, Option, Option)>, } impl PrecomputedPruneCutoffs { - fn from_policies(policies: &BTreeMap, now: i64) -> Self { + fn from_policies(policies: &BTreeMap, now: i64) -> Self { let policies = policies .values() .map(|p| { let age_cutoff = p.max_age_ms.map(|age| now - age); - (age_cutoff, p.max_weight, p.type_id) + (age_cutoff, p.max_weight, p.label_id) }) .collect(); Self { policies } @@ -1624,18 +2721,18 @@ impl PrecomputedPruneCutoffs { /// Returns true if the node matches ANY registered policy (should be excluded). fn excludes(&self, node: &NodeRecord) -> bool { - self.excludes_fields(node.type_id, node.updated_at, node.weight) + self.excludes_fields(&node.label_ids, node.updated_at, node.weight) } - fn excludes_fields(&self, node_type_id: u32, updated_at: i64, weight: f32) -> bool { - for &(age_cutoff, max_weight, policy_type_id) in &self.policies { + fn excludes_fields(&self, label_ids: &NodeLabelSet, updated_at: i64, weight: f32) -> bool { + for &(age_cutoff, max_weight, policy_label_id) in &self.policies { if matches_prune_cutoff( - node_type_id, + label_ids, updated_at, weight, age_cutoff, max_weight, - policy_type_id, + policy_label_id, ) { return true; } @@ -1690,7 +2787,7 @@ fn is_cache_bypass_edge(valid_from: i64, valid_to: i64, created_at: i64) -> bool #[derive(Clone)] struct ReadManifestState { - prune_policies: BTreeMap, + prune_policies: BTreeMap, dense_vector: Option, } @@ -1726,15 +2823,31 @@ pub(crate) struct QueryPlanningProbeSnapshot { #[cfg(test)] #[derive(Default)] struct QueryExecutionCounters { + node_record_hydration_reads: AtomicUsize, + node_visibility_meta_reads: AtomicUsize, + edge_record_hydration_reads: AtomicUsize, + edge_record_hydration_calls: AtomicUsize, equality_materialization_record_reads: AtomicUsize, final_verifier_record_reads: AtomicUsize, + edge_full_scan_pages: AtomicUsize, + endpoint_adjacency_candidates: AtomicUsize, + pattern_edge_pending_entries: AtomicUsize, + public_edge_query_calls: AtomicUsize, } #[cfg(test)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct QueryExecutionCounterSnapshot { + pub node_record_hydration_reads: usize, + pub node_visibility_meta_reads: usize, + pub edge_record_hydration_reads: usize, + pub edge_record_hydration_calls: usize, pub equality_materialization_record_reads: usize, pub final_verifier_record_reads: usize, + pub edge_full_scan_pages: usize, + pub endpoint_adjacency_candidates: usize, + pub pattern_edge_pending_entries: usize, + pub public_edge_query_calls: usize, } /// Published read-visible snapshot for CP1 point/dedup reads. @@ -1743,12 +2856,14 @@ pub(crate) struct ReadView { sources: Arc, snapshot_seq: u64, active_degree_overlay: Arc, + label_catalog: Arc, } pub(crate) type ReadViewImmutableEpoch = ImmutableEpoch; struct PublishedReadState { view: Arc, + label_catalog: Arc, edge_uniqueness: bool, #[cfg(test)] engine_seq: u64, @@ -1760,6 +2875,7 @@ struct PublishedReadState { enum PublishImpact { NoPublish, SnapshotOnly, + SnapshotWithLabelCatalog, RebuildSources, } @@ -1997,15 +3113,29 @@ pub(crate) struct DegreeQueryOutcome { } enum CoreWriteRequest { + EnsureNodeLabel { + label: String, + }, + EnsureEdgeLabel { + label: String, + }, UpsertNode { - type_id: u32, + labels: Vec, key: String, options: UpsertNodeOptions, }, + AddNodeLabel { + id: u64, + label: String, + }, + RemoveNodeLabel { + id: u64, + label: String, + }, UpsertEdge { from: u64, to: u64, - type_id: u32, + label: String, options: UpsertEdgeOptions, }, BatchUpsertNodes { @@ -2049,12 +3179,22 @@ enum CoreWriteRequest { name: String, }, EnsureNodePropertyIndex { - type_id: u32, + label: String, prop_key: String, kind: SecondaryIndexKind, }, DropNodePropertyIndex { - type_id: u32, + label: String, + prop_key: String, + kind: SecondaryIndexKind, + }, + EnsureEdgePropertyIndex { + label: String, + prop_key: String, + kind: SecondaryIndexKind, + }, + DropEdgePropertyIndex { + label: String, prop_key: String, kind: SecondaryIndexKind, }, @@ -2069,6 +3209,7 @@ enum CoreWriteRequest { } enum CoreWriteReply { + U32(u32), U64(u64), VecU64(Vec), Unit, @@ -2078,6 +3219,7 @@ enum CoreWriteReply { PruneResult(PruneResult), Bool(bool), NodePropertyIndexInfo(NodePropertyIndexInfo), + EdgePropertyIndexInfo(EdgePropertyIndexInfo), OptionSegmentInfo(Option), OptionCompactionStats(Option), } @@ -2087,6 +3229,7 @@ struct CoreWritePlan { reply: CoreWriteReply, auto_flush: bool, track_ids: bool, + label_catalog_changed: bool, } struct QueuedCoreWrite { @@ -2423,7 +3566,7 @@ impl DbRuntime { core: &mut EngineCore, impact: PublishImpact, _apply_test_pause: bool, - ) { + ) -> Result<(), EngineError> { #[cfg(test)] match impact { PublishImpact::NoPublish => { @@ -2431,7 +3574,7 @@ impl DbRuntime { .skipped .fetch_add(1, Ordering::Relaxed); } - PublishImpact::SnapshotOnly => { + PublishImpact::SnapshotOnly | PublishImpact::SnapshotWithLabelCatalog => { self.publish_counters .snapshot_only .fetch_add(1, Ordering::Relaxed); @@ -2448,7 +3591,7 @@ impl DbRuntime { if impact == PublishImpact::NoPublish { core.retry_deferred_segment_cleanup(); - return; + return Ok(()); } #[cfg(test)] @@ -2460,14 +3603,22 @@ impl DbRuntime { } if impact == PublishImpact::RebuildSources { - core.rebuild_published_read_sources(); + core.rebuild_published_read_sources()?; } - let published = Arc::new(core.published_read_state()); + let label_catalog = match impact { + PublishImpact::SnapshotOnly => Arc::clone(&self.published.load_full().label_catalog), + PublishImpact::SnapshotWithLabelCatalog | PublishImpact::RebuildSources => { + core.read_label_catalog_snapshot() + } + PublishImpact::NoPublish => unreachable!("NoPublish returned before publish rebuild"), + }; + let published = Arc::new(core.published_read_state_with_catalog(label_catalog)); // Publish before releasing the core mutex so later writers cannot overtake // this committed snapshot and install an older view afterward. self.published.store(published); core.retry_deferred_segment_cleanup(); + Ok(()) } fn with_core_ref( @@ -2491,8 +3642,9 @@ impl DbRuntime { let result = f(core); - self.publish_locked(core, PublishImpact::RebuildSources, true); + let publish_result = self.publish_locked(core, PublishImpact::RebuildSources, true); drop(core_guard); + publish_result?; result } @@ -2502,7 +3654,7 @@ impl DbRuntime { return; }; core.manifest.secondary_indexes = core.secondary_index_entries_snapshot(); - self.publish_locked(core, PublishImpact::RebuildSources, true); + let _ = self.publish_locked(core, PublishImpact::RebuildSources, true); } fn republish_secondary_index_state_and_refreshed_stats_if_open( @@ -2523,11 +3675,19 @@ impl DbRuntime { .any(|entry| ready.matches_entry(entry)); if ready_still_current { for (segment_id, reader) in refreshed_readers { - if !core + let Some(root_segment) = core .manifest .segments .iter() - .any(|segment| segment.id == segment_id) + .find(|segment| segment.id == segment_id) + else { + continue; + }; + if root_segment.segment_data_id != reader.segment_data_id() { + continue; + } + if root_segment.node_count != reader.node_count() + || root_segment.edge_count != reader.edge_count() { continue; } @@ -2543,13 +3703,16 @@ impl DbRuntime { }; if core.segments[position].node_count() == reader.node_count() && core.segments[position].edge_count() == reader.edge_count() + && core.segments[position].segment_data_id() == reader.segment_data_id() + && core.segments[position].component_manifest_generation() + <= reader.component_manifest_generation() { core.segments[position] = reader; } } } - self.publish_locked(core, PublishImpact::RebuildSources, true); + let _ = self.publish_locked(core, PublishImpact::RebuildSources, true); } fn start_coordinator(self: &Arc) { @@ -2713,6 +3876,8 @@ impl DbRuntime { let uses_write_backpressure = matches!( &command.request, CoreWriteRequest::UpsertNode { .. } + | CoreWriteRequest::AddNodeLabel { .. } + | CoreWriteRequest::RemoveNodeLabel { .. } | CoreWriteRequest::UpsertEdge { .. } | CoreWriteRequest::BatchUpsertNodes { .. } | CoreWriteRequest::BatchUpsertEdges { .. } @@ -2731,12 +3896,18 @@ impl DbRuntime { match backpressure_result { Ok(BackpressureFlushAction::Ready) => {} Ok(BackpressureFlushAction::Wait) => { - self.publish_locked(core, publish_impact, true); + if let Err(err) = self.publish_locked(core, publish_impact, true) { + drop(core_guard); + return QueuedWriteProgress::Complete { + command, + result: Err(err), + }; + } drop(core_guard); return QueuedWriteProgress::WaitForLifecycle { command }; } Err(err) => { - self.publish_locked(core, publish_impact, true); + let _ = self.publish_locked(core, publish_impact, true); drop(core_guard); return QueuedWriteProgress::Complete { command, @@ -2758,18 +3929,34 @@ impl DbRuntime { Err(err) => (Err(err), PublishImpact::NoPublish), }, CoreWriteRequest::EnsureNodePropertyIndex { - type_id, + label, prop_key, kind, - } => match core.ensure_node_property_index(*type_id, prop_key, kind.clone()) { + } => match core.ensure_node_property_index(label, prop_key, kind.clone()) { Ok((info, impact)) => (Ok(CoreWriteReply::NodePropertyIndexInfo(info)), impact), Err(err) => (Err(err), PublishImpact::NoPublish), }, CoreWriteRequest::DropNodePropertyIndex { - type_id, + label, prop_key, kind, - } => match core.drop_node_property_index(*type_id, prop_key, kind.clone()) { + } => match core.drop_node_property_index(label, prop_key, kind.clone()) { + Ok((removed, impact)) => (Ok(CoreWriteReply::Bool(removed)), impact), + Err(err) => (Err(err), PublishImpact::NoPublish), + }, + CoreWriteRequest::EnsureEdgePropertyIndex { + label, + prop_key, + kind, + } => match core.ensure_edge_property_index(label, prop_key, kind.clone()) { + Ok((info, impact)) => (Ok(CoreWriteReply::EdgePropertyIndexInfo(info)), impact), + Err(err) => (Err(err), PublishImpact::NoPublish), + }, + CoreWriteRequest::DropEdgePropertyIndex { + label, + prop_key, + kind, + } => match core.drop_edge_property_index(label, prop_key, kind.clone()) { Ok((removed, impact)) => (Ok(CoreWriteReply::Bool(removed)), impact), Err(err) => (Err(err), PublishImpact::NoPublish), }, @@ -2792,8 +3979,13 @@ impl DbRuntime { .unwrap_or_else(|err| (Err(err), PublishImpact::NoPublish)), }; publish_impact = publish_impact.combine(request_publish_impact); - self.publish_locked(core, publish_impact, true); + let publish_result = self.publish_locked(core, publish_impact, true); drop(core_guard); + let result = match (result, publish_result) { + (Err(err), _) => Err(err), + (Ok(reply), Ok(())) => Ok(reply), + (Ok(_), Err(err)) => Err(err), + }; QueuedWriteProgress::Complete { command, result } } @@ -2827,7 +4019,7 @@ impl DbRuntime { if core.immutable_epochs.is_empty() { if mutated { - self.publish_locked(core, PublishImpact::RebuildSources, true); + self.publish_locked(core, PublishImpact::RebuildSources, true)?; } let result = core.current_flush_pipeline_error().map_or(Ok(None), Err); drop(core_guard); @@ -2837,14 +4029,14 @@ impl DbRuntime { core.ensure_bg_flush_worker(); if let Err(error) = core.enqueue_all_non_in_flight() { if mutated { - self.publish_locked(core, PublishImpact::RebuildSources, true); + self.publish_locked(core, PublishImpact::RebuildSources, true)?; } drop(core_guard); return Err(error); } target_epoch = core.immutable_epochs.first().map(|epoch| epoch.epoch_id); if mutated { - self.publish_locked(core, PublishImpact::RebuildSources, true); + self.publish_locked(core, PublishImpact::RebuildSources, true)?; } } else if !core .immutable_epochs @@ -2875,7 +4067,7 @@ impl DbRuntime { if let Some(previous) = core.ingest_saved_compact_after_n_flushes.take() { core.compact_after_n_flushes = previous; } - self.publish_locked(core, PublishImpact::NoPublish, true); + self.publish_locked(core, PublishImpact::NoPublish, true)?; drop(core_guard); Ok(()) } @@ -2913,7 +4105,7 @@ impl DbRuntime { } else { PublishImpact::NoPublish }; - self.publish_locked(core, publish_impact, true); + self.publish_locked(core, publish_impact, true)?; drop(core_guard); return result; } @@ -2943,7 +4135,7 @@ impl DbRuntime { } else { PublishImpact::NoPublish }); - self.publish_locked(core, publish_impact, false); + let _ = self.publish_locked(core, publish_impact, false); drop(core_guard); if progressed { @@ -3058,7 +4250,7 @@ impl DbRuntime { let err = core .current_flush_pipeline_error() .expect("flush pipeline error must be present"); - self.publish_locked(core, PublishImpact::NoPublish, true); + self.publish_locked(core, PublishImpact::NoPublish, true)?; drop(core_guard); return Err(err); } @@ -3190,6 +4382,7 @@ impl ReadView { sources: Arc, snapshot_seq: u64, active_degree_overlay: Arc, + label_catalog: Arc, ) -> Self { debug_assert!( sources.declared_index_runtime_coverage.entry_count() @@ -3202,6 +4395,7 @@ impl ReadView { sources, snapshot_seq, active_degree_overlay, + label_catalog, } } @@ -3228,6 +4422,30 @@ impl ReadView { .fetch_add(1, Ordering::Relaxed); } + #[cfg(test)] + fn note_node_record_hydration_reads(&self, count: usize) { + self.query_execution_counters + .node_record_hydration_reads + .fetch_add(count, Ordering::Relaxed); + } + + #[cfg(test)] + fn note_node_visibility_meta_reads(&self, count: usize) { + self.query_execution_counters + .node_visibility_meta_reads + .fetch_add(count, Ordering::Relaxed); + } + + #[cfg(test)] + fn note_edge_record_hydration_reads(&self, count: usize) { + self.query_execution_counters + .edge_record_hydration_reads + .fetch_add(count, Ordering::Relaxed); + self.query_execution_counters + .edge_record_hydration_calls + .fetch_add(1, Ordering::Relaxed); + } + #[cfg(test)] fn note_equality_materialization_record_reads(&self, count: usize) { self.query_execution_counters @@ -3242,14 +4460,48 @@ impl ReadView { .fetch_add(count, Ordering::Relaxed); } + #[cfg(test)] + fn note_edge_full_scan_page(&self) { + self.query_execution_counters + .edge_full_scan_pages + .fetch_add(1, Ordering::Relaxed); + } + + #[cfg(test)] + fn note_endpoint_adjacency_candidates(&self, count: usize) { + self.query_execution_counters + .endpoint_adjacency_candidates + .fetch_add(count, Ordering::Relaxed); + } + + #[cfg(test)] + fn note_pattern_edge_pending_entry(&self) { + self.query_execution_counters + .pattern_edge_pending_entries + .fetch_add(1, Ordering::Relaxed); + } + fn node_property_index_entry( &self, - type_id: u32, + label_id: u32, + prop_key: &str, + kind: &SecondaryIndexKind, + ) -> Option { + self.secondary_index_catalog + .get(&(SecondaryIndexTargetDiscriminant::Node, label_id))? + .get(prop_key)? + .get(kind) + .cloned() + } + + fn edge_property_index_entry( + &self, + label_id: u32, prop_key: &str, kind: &SecondaryIndexKind, ) -> Option { self.secondary_index_catalog - .get(&type_id)? + .get(&(SecondaryIndexTargetDiscriminant::Edge, label_id))? .get(prop_key)? .get(kind) .cloned() @@ -3274,16 +4526,20 @@ impl ReadView { self.sources().find_edge(id) } - fn get_node_by_key_raw( + fn get_node_by_label_key_raw( &self, - type_id: u32, + label_id: u32, key: &str, ) -> Result, EngineError> { - self.sources().find_node_by_key(type_id, key) + self.sources().find_node_by_label_key(label_id, key) } - fn get_node_by_key(&self, type_id: u32, key: &str) -> Result, EngineError> { - let node = match self.get_node_by_key_raw(type_id, key)? { + fn get_node_by_label_key( + &self, + label_id: u32, + key: &str, + ) -> Result, EngineError> { + let node = match self.get_node_by_label_key_raw(label_id, key)? { Some(node) => node, None => return Ok(None), }; @@ -3297,12 +4553,14 @@ impl ReadView { &self, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result, EngineError> { - self.sources().find_edge_by_triple(from, to, type_id) + self.sources().find_edge_by_triple(from, to, label_id) } fn get_nodes_raw(&self, ids: &[u64]) -> Result>, EngineError> { + #[cfg(test)] + self.note_node_record_hydration_reads(ids.len()); self.sources().find_nodes(ids) } @@ -3322,18 +4580,18 @@ impl ReadView { Ok(results) } - fn get_nodes_by_keys_raw( + fn get_nodes_by_label_keys_raw( &self, keys: &[(u32, &str)], ) -> Result>, EngineError> { - self.sources().find_nodes_by_keys(keys) + self.sources().find_nodes_by_label_keys(keys) } - fn get_nodes_by_keys( + fn get_nodes_by_label_keys( &self, keys: &[(u32, &str)], ) -> Result>, EngineError> { - let mut results = self.get_nodes_by_keys_raw(keys)?; + let mut results = self.get_nodes_by_label_keys_raw(keys)?; if !self.manifest.prune_policies.is_empty() { let cutoffs = PrecomputedPruneCutoffs::from_policies(&self.manifest.prune_policies, now_millis()); @@ -3349,6 +4607,8 @@ impl ReadView { } fn get_edges(&self, ids: &[u64]) -> Result>, EngineError> { + #[cfg(test)] + self.note_edge_record_hydration_reads(ids.len()); self.sources().find_edges(ids) } } @@ -3367,28 +4627,28 @@ impl EngineCore { self.sources().find_edge(id) } - fn get_node_by_key_raw( + fn get_node_by_label_key_raw( &self, - type_id: u32, + label_id: u32, key: &str, ) -> Result, EngineError> { - self.sources().find_node_by_key(type_id, key) + self.sources().find_node_by_label_key(label_id, key) } - fn get_nodes_by_keys_raw( + fn get_nodes_by_label_keys_raw( &self, keys: &[(u32, &str)], ) -> Result>, EngineError> { - self.sources().find_nodes_by_keys(keys) + self.sources().find_nodes_by_label_keys(keys) } fn get_edge_by_triple( &self, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result, EngineError> { - self.sources().find_edge_by_triple(from, to, type_id) + self.sources().find_edge_by_triple(from, to, label_id) } fn get_nodes_raw(&self, ids: &[u64]) -> Result>, EngineError> { @@ -3399,8 +4659,8 @@ impl EngineCore { self.read_view().get_edges(ids) } - fn nodes_by_type_raw(&self, type_id: u32) -> Result, EngineError> { - self.read_view().nodes_by_type_raw(type_id) + fn nodes_by_label_id_raw(&self, label_id: u32) -> Result, EngineError> { + self.read_view().nodes_by_label_id_raw(label_id) } fn collect_tombstones( @@ -3417,7 +4677,7 @@ impl EngineCore { &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, limit: usize, at_epoch: Option, decay_lambda: Option, @@ -3425,11 +4685,11 @@ impl EngineCore { &HashSet, &HashSet, )>, - ) -> Result, EngineError> { + ) -> Result, EngineError> { self.read_view().neighbors_raw( node_id, direction, - type_filter, + label_filter_ids, limit, at_epoch, decay_lambda, @@ -3477,6 +4737,8 @@ impl EngineCore { &self.next_node_id_seen, &self.next_edge_id_seen, &self.engine_seq_seen, + Some(&self.label_catalog), + self.checkpointable_wal_generation(), |manifest| { if let Some(entry) = manifest .secondary_indexes @@ -3541,6 +4803,8 @@ impl EngineCore { &self.next_node_id_seen, &self.next_edge_id_seen, &self.engine_seq_seen, + Some(&self.label_catalog), + self.checkpointable_wal_generation(), |manifest| { if let Some(entry) = manifest .secondary_indexes @@ -3595,16 +4859,135 @@ impl DatabaseEngine { self.runtime.close(true) } - pub fn upsert_node( + pub fn ensure_node_label(&self, label: &str) -> Result { + match self + .runtime + .submit_core_write(CoreWriteRequest::EnsureNodeLabel { + label: label.to_string(), + })? { + CoreWriteReply::U32(label_id) => Ok(label_id), + _ => unreachable!("ensure_node_label must return a label id"), + } + } + + pub fn ensure_edge_label(&self, label: &str) -> Result { + match self + .runtime + .submit_core_write(CoreWriteRequest::EnsureEdgeLabel { + label: label.to_string(), + })? { + CoreWriteReply::U32(label_id) => Ok(label_id), + _ => unreachable!("ensure_edge_label must return a label id"), + } + } + + pub fn get_node_label_id(&self, label: &str) -> Result, EngineError> { + validate_label_token_name(label)?; + self.with_core_ref(|core| { + Ok(core + .label_catalog + .read() + .unwrap() + .node_label_to_id + .get(label) + .copied()) + }) + } + + pub fn get_edge_label_id(&self, label: &str) -> Result, EngineError> { + validate_label_token_name(label)?; + self.with_core_ref(|core| { + Ok(core + .label_catalog + .read() + .unwrap() + .edge_label_to_id + .get(label) + .copied()) + }) + } + + pub fn get_node_label(&self, label_id: u32) -> Result, EngineError> { + self.with_core_ref(|core| { + Ok(core + .label_catalog + .read() + .unwrap() + .node_id_to_label + .get(&label_id) + .cloned()) + }) + } + + pub fn get_edge_label(&self, label_id: u32) -> Result, EngineError> { + self.with_core_ref(|core| { + Ok(core + .label_catalog + .read() + .unwrap() + .edge_id_to_label + .get(&label_id) + .cloned()) + }) + } + + pub fn list_node_labels(&self) -> Result, EngineError> { + self.with_core_ref(|core| { + let mut labels: Vec = core + .label_catalog + .read() + .unwrap() + .node_label_to_id + .iter() + .map(|(label, &label_id)| NodeLabelInfo { + label: label.clone(), + label_id, + }) + .collect(); + labels.sort_by(|a, b| { + a.label_id + .cmp(&b.label_id) + .then_with(|| a.label.cmp(&b.label)) + }); + Ok(labels) + }) + } + + pub fn list_edge_labels(&self) -> Result, EngineError> { + self.with_core_ref(|core| { + let mut edge_labels: Vec = core + .label_catalog + .read() + .unwrap() + .edge_label_to_id + .iter() + .map(|(label, &label_id)| EdgeLabelInfo { + label: label.clone(), + label_id, + }) + .collect(); + edge_labels.sort_by(|a, b| { + a.label_id + .cmp(&b.label_id) + .then_with(|| a.label.cmp(&b.label)) + }); + Ok(edge_labels) + }) + } + + pub fn upsert_node( &self, - type_id: u32, + labels: L, key: &str, options: UpsertNodeOptions, - ) -> Result { + ) -> Result + where + L: IntoNodeLabels, + { match self .runtime .submit_core_write(CoreWriteRequest::UpsertNode { - type_id, + labels: labels.into_node_labels(), key: key.to_string(), options, })? { @@ -3617,7 +5000,7 @@ impl DatabaseEngine { &self, from: u64, to: u64, - type_id: u32, + label: &str, options: UpsertEdgeOptions, ) -> Result { match self @@ -3625,7 +5008,7 @@ impl DatabaseEngine { .submit_core_write(CoreWriteRequest::UpsertEdge { from, to, - type_id, + label: label.to_string(), options, })? { CoreWriteReply::U64(id) => Ok(id), @@ -3633,23 +5016,45 @@ impl DatabaseEngine { } } - pub fn batch_upsert_nodes(&self, inputs: &[NodeInput]) -> Result, EngineError> { + pub fn add_node_label(&self, id: u64, label: &str) -> Result { + match self + .runtime + .submit_core_write(CoreWriteRequest::AddNodeLabel { + id, + label: label.to_string(), + })? { + CoreWriteReply::Bool(changed) => Ok(changed), + _ => unreachable!("add_node_label must return changed bool"), + } + } + + pub fn remove_node_label(&self, id: u64, label: &str) -> Result { match self .runtime - .submit_core_write(CoreWriteRequest::BatchUpsertNodes { - inputs: inputs.to_vec(), + .submit_core_write(CoreWriteRequest::RemoveNodeLabel { + id, + label: label.to_string(), })? { + CoreWriteReply::Bool(changed) => Ok(changed), + _ => unreachable!("remove_node_label must return changed bool"), + } + } + + pub fn batch_upsert_nodes(&self, inputs: Vec) -> Result, EngineError> { + match self + .runtime + .submit_core_write(CoreWriteRequest::BatchUpsertNodes { inputs })? + { CoreWriteReply::VecU64(ids) => Ok(ids), _ => unreachable!("batch_upsert_nodes must return node ids"), } } - pub fn batch_upsert_edges(&self, inputs: &[EdgeInput]) -> Result, EngineError> { + pub fn batch_upsert_edges(&self, inputs: Vec) -> Result, EngineError> { match self .runtime - .submit_core_write(CoreWriteRequest::BatchUpsertEdges { - inputs: inputs.to_vec(), - })? { + .submit_core_write(CoreWriteRequest::BatchUpsertEdges { inputs })? + { CoreWriteReply::VecU64(ids) => Ok(ids), _ => unreachable!("batch_upsert_edges must return edge ids"), } @@ -3675,26 +5080,24 @@ impl DatabaseEngine { } } - pub fn invalidate_edge( - &self, - id: u64, - valid_to: i64, - ) -> Result, EngineError> { - match self + pub fn invalidate_edge(&self, id: u64, valid_to: i64) -> Result, EngineError> { + let edge = match self .runtime .submit_core_write(CoreWriteRequest::InvalidateEdge { id, valid_to })? { - CoreWriteReply::OptionEdge(edge) => Ok(edge), + CoreWriteReply::OptionEdge(edge) => edge, _ => unreachable!("invalidate_edge must return an optional edge"), - } + }; + let (_guard, published) = self.runtime.published_snapshot()?; + edge.map(|edge| edge_view_from_record(edge, published.label_catalog.as_ref())) + .transpose() } - pub fn graph_patch(&self, patch: &GraphPatch) -> Result { + pub fn graph_patch(&self, patch: GraphPatch) -> Result { match self .runtime - .submit_core_write(CoreWriteRequest::GraphPatch { - patch: patch.clone(), - })? { + .submit_core_write(CoreWriteRequest::GraphPatch { patch })? + { CoreWriteReply::PatchResult(result) => Ok(result), _ => unreachable!("graph_patch must return patch results"), } @@ -3732,20 +5135,20 @@ impl DatabaseEngine { } } - pub fn list_prune_policies(&self) -> Result, EngineError> { - self.with_core_ref(|core| Ok(core.list_prune_policies())) + pub fn list_prune_policies(&self) -> Result, EngineError> { + self.with_core_ref(|core| core.list_prune_policies()) } pub fn ensure_node_property_index( &self, - type_id: u32, + label: &str, prop_key: &str, kind: SecondaryIndexKind, ) -> Result { match self .runtime .submit_core_write(CoreWriteRequest::EnsureNodePropertyIndex { - type_id, + label: label.to_string(), prop_key: prop_key.to_string(), kind, })? { @@ -3756,14 +5159,14 @@ impl DatabaseEngine { pub fn drop_node_property_index( &self, - type_id: u32, + label: &str, prop_key: &str, kind: SecondaryIndexKind, ) -> Result { match self .runtime .submit_core_write(CoreWriteRequest::DropNodePropertyIndex { - type_id, + label: label.to_string(), prop_key: prop_key.to_string(), kind, })? { @@ -3774,67 +5177,213 @@ impl DatabaseEngine { pub fn list_node_property_indexes(&self) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - let mut indexes: Vec = published + let catalog = published.label_catalog.as_ref(); + let mut entries: Vec<&SecondaryIndexManifestEntry> = published .view .secondary_index_entries .iter() - .map(EngineCore::node_property_index_info) + .filter(|e| matches!(&e.target, SecondaryIndexTarget::NodeProperty { .. })) .collect(); - indexes.sort_unstable_by(|left, right| { - left.type_id - .cmp(&right.type_id) - .then_with(|| left.prop_key.cmp(&right.prop_key)) - .then_with(|| format!("{:?}", left.kind).cmp(&format!("{:?}", right.kind))) + entries.sort_unstable_by(|left, right| { + secondary_index_target_label_id(&left.target) + .cmp(&secondary_index_target_label_id(&right.target)) + .then_with(|| { + secondary_index_target_prop_key(&left.target) + .cmp(secondary_index_target_prop_key(&right.target)) + }) + .then_with(|| { + secondary_index_kind_rank(&left.kind) + .cmp(&secondary_index_kind_rank(&right.kind)) + }) .then_with(|| left.index_id.cmp(&right.index_id)) }); - Ok(indexes) + entries + .into_iter() + .map(|entry| EngineCore::node_property_index_info(entry, catalog)) + .collect() + } + + pub fn ensure_edge_property_index( + &self, + label: &str, + prop_key: &str, + kind: SecondaryIndexKind, + ) -> Result { + match self + .runtime + .submit_core_write(CoreWriteRequest::EnsureEdgePropertyIndex { + label: label.to_string(), + prop_key: prop_key.to_string(), + kind, + })? { + CoreWriteReply::EdgePropertyIndexInfo(info) => Ok(info), + _ => unreachable!("ensure_edge_property_index must return index info"), + } + } + + pub fn drop_edge_property_index( + &self, + label: &str, + prop_key: &str, + kind: SecondaryIndexKind, + ) -> Result { + match self + .runtime + .submit_core_write(CoreWriteRequest::DropEdgePropertyIndex { + label: label.to_string(), + prop_key: prop_key.to_string(), + kind, + })? { + CoreWriteReply::Bool(dropped) => Ok(dropped), + _ => unreachable!("drop_edge_property_index must return bool"), + } } - pub fn get_node(&self, id: u64) -> Result, EngineError> { + pub fn list_edge_property_indexes(&self) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_node(id) + let catalog = published.label_catalog.as_ref(); + let mut entries: Vec<&SecondaryIndexManifestEntry> = published + .view + .secondary_index_entries + .iter() + .filter(|e| matches!(&e.target, SecondaryIndexTarget::EdgeProperty { .. })) + .collect(); + entries.sort_unstable_by(|left, right| { + secondary_index_target_label_id(&left.target) + .cmp(&secondary_index_target_label_id(&right.target)) + .then_with(|| { + secondary_index_target_prop_key(&left.target) + .cmp(secondary_index_target_prop_key(&right.target)) + }) + .then_with(|| { + secondary_index_kind_rank(&left.kind) + .cmp(&secondary_index_kind_rank(&right.kind)) + }) + .then_with(|| left.index_id.cmp(&right.index_id)) + }); + entries + .into_iter() + .map(|entry| EngineCore::edge_property_index_info(entry, catalog)) + .collect() } - pub fn get_edge(&self, id: u64) -> Result, EngineError> { + pub fn get_node(&self, id: u64) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_edge(id) + let node = published.view.get_node(id)?; + node.map(|node| node_view_from_record(node, published.label_catalog.as_ref())) + .transpose() } - pub fn get_node_by_key( - &self, - type_id: u32, - key: &str, - ) -> Result, EngineError> { + pub fn get_edge(&self, id: u64) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_node_by_key(type_id, key) + let edge = published.view.get_edge(id)?; + edge.map(|edge| edge_view_from_record(edge, published.label_catalog.as_ref())) + .transpose() + } + + pub fn get_node_by_key(&self, label: &str, key: &str) -> Result, EngineError> { + let (_guard, published) = self.runtime.published_snapshot()?; + validate_label_token_name(label)?; + let Some(label_id) = published.label_catalog.resolve_node_label_for_read(label)? else { + return Ok(None); + }; + let node = published.view.get_node_by_label_key(label_id, key)?; + node.map(|node| { + node_view_from_record_with_resolved_label( + node, + label_id, + published.label_catalog.as_ref(), + ) + }) + .transpose() } pub fn get_edge_by_triple( &self, from: u64, to: u64, - type_id: u32, - ) -> Result, EngineError> { + label: &str, + ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_edge_by_triple(from, to, type_id) + validate_label_token_name(label)?; + let Some(label_id) = published.label_catalog.resolve_edge_label_for_read(label)? else { + return Ok(None); + }; + let edge = published.view.get_edge_by_triple(from, to, label_id)?; + edge.map(|edge| { + edge_view_from_record_with_resolved_label(edge, label_id, label.to_string()) + }) + .transpose() } - pub fn get_nodes(&self, ids: &[u64]) -> Result>, EngineError> { + pub fn get_nodes(&self, ids: &[u64]) -> Result>, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_nodes(ids) + let nodes = published.view.get_nodes(ids)?; + nodes + .into_iter() + .map(|node| { + node.map(|node| node_view_from_record(node, published.label_catalog.as_ref())) + .transpose() + }) + .collect() } pub fn get_nodes_by_keys( &self, - keys: &[(u32, &str)], - ) -> Result>, EngineError> { + keys: &[NodeKeyQuery], + ) -> Result>, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_nodes_by_keys(keys) + let mut label_ids: BTreeMap<&str, Option> = BTreeMap::new(); + for query in keys { + validate_label_token_name(&query.label)?; + let label = query.label.as_str(); + if !label_ids.contains_key(label) { + let label_id = published.label_catalog.resolve_node_label_for_read(label)?; + label_ids.insert(label, label_id); + } + } + + let mut resolved = Vec::new(); + let mut positions = Vec::new(); + for (idx, query) in keys.iter().enumerate() { + if let Some(label_id) = label_ids + .get(query.label.as_str()) + .copied() + .expect("label was validated and resolved in first pass") + { + positions.push((idx, label_id)); + resolved.push((label_id, query.key.as_str())); + } + } + let mut output = vec![None; keys.len()]; + if resolved.is_empty() { + return Ok(output); + } + let nodes = published.view.get_nodes_by_label_keys(&resolved)?; + for ((idx, label_id), node) in positions.into_iter().zip(nodes) { + output[idx] = node + .map(|node| { + node_view_from_record_with_resolved_label( + node, + label_id, + published.label_catalog.as_ref(), + ) + }) + .transpose()?; + } + Ok(output) } - pub fn get_edges(&self, ids: &[u64]) -> Result>, EngineError> { + pub fn get_edges(&self, ids: &[u64]) -> Result>, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_edges(ids) + let edges = published.view.get_edges(ids)?; + edges + .into_iter() + .map(|edge| { + edge.map(|edge| edge_view_from_record(edge, published.label_catalog.as_ref())) + .transpose() + }) + .collect() } pub fn vector_search( @@ -3845,82 +5394,255 @@ impl DatabaseEngine { published.view.vector_search(request) } - pub fn nodes_by_type(&self, type_id: u32) -> Result, EngineError> { + pub fn edges_by_label(&self, label: &str) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.nodes_by_type(type_id) + let label_id = published.label_catalog.resolve_edge_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(Vec::new()); + }; + published.view.edges_by_label_id(label_id) } - pub fn edges_by_type(&self, type_id: u32) -> Result, EngineError> { + pub fn get_edges_by_label(&self, label: &str) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.edges_by_type(type_id) + let label_id = published.label_catalog.resolve_edge_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(Vec::new()); + }; + let label = label.to_string(); + published + .view + .get_edges_by_label_id(label_id)? + .into_iter() + .map(|edge| edge_view_from_record_with_resolved_label(edge, label_id, label.clone())) + .collect() } - pub fn get_nodes_by_type(&self, type_id: u32) -> Result, EngineError> { - let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_nodes_by_type(type_id) + pub fn nodes_by_labels(&self, labels: L) -> Result, EngineError> + where + L: IntoNodeLabels, + { + let labels = labels.into_node_labels(); + validate_public_node_label_list(labels.iter().map(String::as_str))?; + if let [label] = labels.as_slice() { + let (_guard, published) = self.runtime.published_snapshot()?; + let Some(label_id) = published.label_catalog.resolve_node_label_for_read(label)? else { + return Ok(Vec::new()); + }; + return published.view.nodes_by_label_id(label_id); + } + Ok(self + .query_node_ids(&NodeQuery { + label_filter: Some(NodeLabelFilter { + labels, + mode: LabelMatchMode::All, + }), + ..Default::default() + })? + .items) } - pub fn get_edges_by_type(&self, type_id: u32) -> Result, EngineError> { - let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_edges_by_type(type_id) + pub fn get_nodes_by_labels(&self, labels: L) -> Result, EngineError> + where + L: IntoNodeLabels, + { + let labels = labels.into_node_labels(); + validate_public_node_label_list(labels.iter().map(String::as_str))?; + if let [label] = labels.as_slice() { + let (_guard, published) = self.runtime.published_snapshot()?; + let Some(label_id) = published.label_catalog.resolve_node_label_for_read(label)? else { + return Ok(Vec::new()); + }; + return published + .view + .get_nodes_by_label_id(label_id)? + .into_iter() + .map(|node| { + node_view_from_record_with_resolved_label( + node, + label_id, + published.label_catalog.as_ref(), + ) + }) + .collect(); + } + Ok(self + .query_nodes(&NodeQuery { + label_filter: Some(NodeLabelFilter { + labels, + mode: LabelMatchMode::All, + }), + ..Default::default() + })? + .items) } - pub fn count_nodes_by_type(&self, type_id: u32) -> Result { + pub fn count_nodes_by_labels(&self, labels: L) -> Result + where + L: IntoNodeLabels, + { + let labels = labels.into_node_labels(); let (_guard, published) = self.runtime.published_snapshot()?; - published.view.count_nodes_by_type(type_id) + let filter = NodeLabelFilter { + labels, + mode: LabelMatchMode::All, + }; + let resolved = published + .label_catalog + .resolve_node_label_filter_request(Some(&filter))?; + published + .view + .count_nodes_by_resolved_label_filter(&resolved) } - pub fn count_edges_by_type(&self, type_id: u32) -> Result { + pub fn count_edges_by_label(&self, label: &str) -> Result { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.count_edges_by_type(type_id) + let label_id = published.label_catalog.resolve_edge_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(0); + }; + published.view.count_edges_by_label_id(label_id) } - pub fn nodes_by_type_paged( + pub fn nodes_by_labels_paged( &self, - type_id: u32, + labels: L, page: &PageRequest, - ) -> Result, EngineError> { - let (_guard, published) = self.runtime.published_snapshot()?; - published.view.nodes_by_type_paged(type_id, page) + ) -> Result, EngineError> + where + L: IntoNodeLabels, + { + let labels = labels.into_node_labels(); + validate_public_node_label_list(labels.iter().map(String::as_str))?; + if let [label] = labels.as_slice() { + let (_guard, published) = self.runtime.published_snapshot()?; + let Some(label_id) = published.label_catalog.resolve_node_label_for_read(label)? else { + return Ok(PageResult { + items: Vec::new(), + next_cursor: None, + }); + }; + return published.view.nodes_by_label_id_paged(label_id, page); + } + let result = self.query_node_ids(&NodeQuery { + label_filter: Some(NodeLabelFilter { + labels, + mode: LabelMatchMode::All, + }), + page: page.clone(), + ..Default::default() + })?; + Ok(PageResult { + items: result.items, + next_cursor: result.next_cursor, + }) } - pub fn edges_by_type_paged( + pub fn get_nodes_by_labels_paged( &self, - type_id: u32, + labels: L, page: &PageRequest, - ) -> Result, EngineError> { - let (_guard, published) = self.runtime.published_snapshot()?; - published.view.edges_by_type_paged(type_id, page) + ) -> Result, EngineError> + where + L: IntoNodeLabels, + { + let labels = labels.into_node_labels(); + validate_public_node_label_list(labels.iter().map(String::as_str))?; + if let [label] = labels.as_slice() { + let (_guard, published) = self.runtime.published_snapshot()?; + let Some(label_id) = published.label_catalog.resolve_node_label_for_read(label)? else { + return Ok(PageResult { + items: Vec::new(), + next_cursor: None, + }); + }; + let page = published.view.get_nodes_by_label_id_paged(label_id, page)?; + let items = page + .items + .into_iter() + .map(|node| { + node_view_from_record_with_resolved_label( + node, + label_id, + published.label_catalog.as_ref(), + ) + }) + .collect::, _>>()?; + return Ok(PageResult { + items, + next_cursor: page.next_cursor, + }); + } + let result = self.query_nodes(&NodeQuery { + label_filter: Some(NodeLabelFilter { + labels, + mode: LabelMatchMode::All, + }), + page: page.clone(), + ..Default::default() + })?; + Ok(PageResult { + items: result.items, + next_cursor: result.next_cursor, + }) } - pub fn get_nodes_by_type_paged( + pub fn edges_by_label_paged( &self, - type_id: u32, + label: &str, page: &PageRequest, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_nodes_by_type_paged(type_id, page) + let label_id = published.label_catalog.resolve_edge_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(PageResult { + items: Vec::new(), + next_cursor: None, + }); + }; + published.view.edges_by_label_id_paged(label_id, page) } - pub fn get_edges_by_type_paged( + pub fn get_edges_by_label_paged( &self, - type_id: u32, + label: &str, page: &PageRequest, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.get_edges_by_type_paged(type_id, page) + let label_id = published.label_catalog.resolve_edge_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(PageResult { + items: Vec::new(), + next_cursor: None, + }); + }; + let label = label.to_string(); + let page = published.view.get_edges_by_label_id_paged(label_id, page)?; + let items = page + .items + .into_iter() + .map(|edge| edge_view_from_record_with_resolved_label(edge, label_id, label.clone())) + .collect::, _>>()?; + Ok(PageResult { + items, + next_cursor: page.next_cursor, + }) } pub fn find_nodes( &self, - type_id: u32, + label: &str, prop_key: &str, prop_value: &PropValue, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; + let label_id = published.label_catalog.resolve_node_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(Vec::new()); + }; let outcome = published .view - .find_nodes_outcome(type_id, prop_key, prop_value)?; + .find_nodes_outcome(label_id, prop_key, prop_value)?; self.runtime.record_property_query_route(outcome.route); if let Some(followup) = outcome.followup { self.runtime.enqueue_secondary_index_read_followup(followup); @@ -3930,15 +5652,22 @@ impl DatabaseEngine { pub fn find_nodes_paged( &self, - type_id: u32, + label: &str, prop_key: &str, prop_value: &PropValue, page: &PageRequest, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; + let label_id = published.label_catalog.resolve_node_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(PageResult { + items: Vec::new(), + next_cursor: None, + }); + }; let outcome = published .view - .find_nodes_paged_outcome(type_id, prop_key, prop_value, page)?; + .find_nodes_paged_outcome(label_id, prop_key, prop_value, page)?; self.runtime.record_property_query_route(outcome.route); if let Some(followup) = outcome.followup { self.runtime.enqueue_secondary_index_read_followup(followup); @@ -3948,14 +5677,19 @@ impl DatabaseEngine { pub fn find_nodes_range( &self, - type_id: u32, + label: &str, prop_key: &str, lower: Option<&PropertyRangeBound>, upper: Option<&PropertyRangeBound>, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; + let label_id = published.label_catalog.resolve_node_label_for_read(label)?; + let Some(label_id) = label_id else { + ReadView::validate_property_range_bounds(lower, upper, None)?; + return Ok(Vec::new()); + }; let outcome = published.view.find_nodes_range_paged_outcome( - type_id, + label_id, prop_key, lower, upper, @@ -3970,16 +5704,24 @@ impl DatabaseEngine { pub fn find_nodes_range_paged( &self, - type_id: u32, + label: &str, prop_key: &str, lower: Option<&PropertyRangeBound>, upper: Option<&PropertyRangeBound>, page: &PropertyRangePageRequest, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; + let label_id = published.label_catalog.resolve_node_label_for_read(label)?; + let Some(label_id) = label_id else { + ReadView::validate_property_range_bounds(lower, upper, page.after.as_ref())?; + return Ok(PropertyRangePageResult { + items: Vec::new(), + next_cursor: None, + }); + }; let outcome = published .view - .find_nodes_range_paged_outcome(type_id, prop_key, lower, upper, page)?; + .find_nodes_range_paged_outcome(label_id, prop_key, lower, upper, page)?; self.runtime.record_property_query_route(outcome.route); if let Some(followup) = outcome.followup { self.runtime.enqueue_secondary_index_read_followup(followup); @@ -3989,27 +5731,38 @@ impl DatabaseEngine { pub fn find_nodes_by_time_range( &self, - type_id: u32, + label: &str, from_ms: i64, to_ms: i64, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; + let label_id = published.label_catalog.resolve_node_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(Vec::new()); + }; published .view - .find_nodes_by_time_range(type_id, from_ms, to_ms) + .find_nodes_by_time_range(label_id, from_ms, to_ms) } pub fn find_nodes_by_time_range_paged( &self, - type_id: u32, + label: &str, from_ms: i64, to_ms: i64, page: &PageRequest, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; + let label_id = published.label_catalog.resolve_node_label_for_read(label)?; + let Some(label_id) = label_id else { + return Ok(PageResult { + items: Vec::new(), + next_cursor: None, + }); + }; published .view - .find_nodes_by_time_range_paged(type_id, from_ms, to_ms, page) + .find_nodes_by_time_range_paged(label_id, from_ms, to_ms, page) } pub fn personalized_pagerank( @@ -4115,7 +5868,11 @@ impl DatabaseEngine { options: &NeighborOptions, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.neighbors(node_id, options) + let entries = published.view.neighbors(node_id, options)?; + entries + .into_iter() + .map(|entry| neighbor_entry_from_record(entry, published.label_catalog.as_ref())) + .collect() } pub fn neighbors_batch( @@ -4124,7 +5881,17 @@ impl DatabaseEngine { options: &NeighborOptions, ) -> Result>, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.neighbors_batch(node_ids, options) + let batch = published.view.neighbors_batch(node_ids, options)?; + let mut output = + NodeIdMap::with_capacity_and_hasher(batch.len(), NodeIdBuildHasher::default()); + for (node_id, entries) in batch { + let entries = entries + .into_iter() + .map(|entry| neighbor_entry_from_record(entry, published.label_catalog.as_ref())) + .collect::, _>>()?; + output.insert(node_id, entries); + } + Ok(output) } pub fn neighbors_paged( @@ -4134,7 +5901,16 @@ impl DatabaseEngine { page: &PageRequest, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.neighbors_paged(node_id, options, page) + let page = published.view.neighbors_paged(node_id, options, page)?; + let items = page + .items + .into_iter() + .map(|entry| neighbor_entry_from_record(entry, published.label_catalog.as_ref())) + .collect::, _>>()?; + Ok(PageResult { + items, + next_cursor: page.next_cursor, + }) } pub fn top_k_neighbors( @@ -4144,7 +5920,11 @@ impl DatabaseEngine { options: &TopKOptions, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; - published.view.top_k_neighbors(node_id, k, options) + let entries = published.view.top_k_neighbors(node_id, k, options)?; + entries + .into_iter() + .map(|entry| neighbor_entry_from_record(entry, published.label_catalog.as_ref())) + .collect() } pub fn extract_subgraph( @@ -4230,6 +6010,11 @@ impl DatabaseEngine { self.with_core_ref(|core| Ok(core.stats())) } + pub fn scrub(&self) -> Result { + let manifest = self.manifest()?; + crate::scrub::scrub_database(self.path(), &manifest) + } + #[cfg(test)] pub(crate) fn write_op(&self, op: &WalOp) -> Result<(), EngineError> { match self @@ -4256,6 +6041,11 @@ impl DatabaseEngine { self.runtime.path() } + /// Return a raw manifest snapshot for diagnostics. + /// + /// This is an explicit introspection exception: the returned manifest may + /// contain internal numeric token IDs and storage metadata. Ordinary public + /// graph APIs use node labels and edge-label names instead. pub fn manifest(&self) -> Result { self.with_core_ref(|core| { let mut manifest = core.manifest.clone(); @@ -4265,6 +6055,7 @@ impl DatabaseEngine { &core.next_edge_id_seen, &core.engine_seq_seen, ); + merge_runtime_label_catalog_into_manifest(&mut manifest, &core.label_catalog); Ok(manifest) }) } @@ -4325,13 +6116,13 @@ impl DatabaseEngine { pub(crate) fn find_existing_node( &self, - type_id: u32, + label_id: u32, key: &str, ) -> Result, EngineError> { let (_guard, published) = self.runtime.published_snapshot()?; Ok(published .view - .get_node_by_key_raw(type_id, key)? + .get_node_by_label_key_raw(label_id, key)? .map(|node| (node.id, node.created_at))) } @@ -4392,6 +6183,11 @@ impl DatabaseEngine { Arc::clone(&self.published_state().view) } + #[cfg(test)] + fn published_label_catalog_snapshot_for_test(&self) -> Arc { + Arc::clone(&self.published_state().label_catalog) + } + pub(crate) fn planner_stats_view_for_test(&self) -> Arc { Arc::clone(&self.published_state().view.planner_stats) } @@ -4414,8 +6210,23 @@ impl DatabaseEngine { .as_mut() .ok_or_else(|| EngineError::InvalidOperation("database is closed".into()))?; let seg_path = segment_dir(&core.db_dir, segment_id); - let reader = - SegmentReader::open(&seg_path, segment_id, core.manifest.dense_vector.as_ref())?; + let seg_info = core + .manifest + .segments + .iter() + .find(|segment| segment.id == segment_id) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "segment {} is not present in the root manifest", + segment_id + )) + })?; + let reader = SegmentReader::open_with_info( + &seg_path, + seg_info, + core.manifest.dense_vector.as_ref(), + &core.manifest.secondary_indexes, + )?; core.warm_declared_index_runtime_coverage_for_reader(&reader); let Some(position) = core .segments @@ -4429,7 +6240,7 @@ impl DatabaseEngine { }; core.segments[position] = Arc::new(reader); self.runtime - .publish_locked(core, PublishImpact::RebuildSources, false); + .publish_locked(core, PublishImpact::RebuildSources, false)?; Ok(()) } @@ -4535,6 +6346,26 @@ impl DatabaseEngine { ) -> QueryExecutionCounterSnapshot { let published = self.published_state(); QueryExecutionCounterSnapshot { + node_record_hydration_reads: published + .view + .query_execution_counters + .node_record_hydration_reads + .load(Ordering::Relaxed), + node_visibility_meta_reads: published + .view + .query_execution_counters + .node_visibility_meta_reads + .load(Ordering::Relaxed), + edge_record_hydration_reads: published + .view + .query_execution_counters + .edge_record_hydration_reads + .load(Ordering::Relaxed), + edge_record_hydration_calls: published + .view + .query_execution_counters + .edge_record_hydration_calls + .load(Ordering::Relaxed), equality_materialization_record_reads: published .view .query_execution_counters @@ -4545,12 +6376,52 @@ impl DatabaseEngine { .query_execution_counters .final_verifier_record_reads .load(Ordering::Relaxed), + edge_full_scan_pages: published + .view + .query_execution_counters + .edge_full_scan_pages + .load(Ordering::Relaxed), + endpoint_adjacency_candidates: published + .view + .query_execution_counters + .endpoint_adjacency_candidates + .load(Ordering::Relaxed), + pattern_edge_pending_entries: published + .view + .query_execution_counters + .pattern_edge_pending_entries + .load(Ordering::Relaxed), + public_edge_query_calls: published + .view + .query_execution_counters + .public_edge_query_calls + .load(Ordering::Relaxed), } } #[cfg(test)] pub(crate) fn reset_query_execution_counters_for_test(&self) { let published = self.published_state(); + published + .view + .query_execution_counters + .node_record_hydration_reads + .store(0, Ordering::Relaxed); + published + .view + .query_execution_counters + .node_visibility_meta_reads + .store(0, Ordering::Relaxed); + published + .view + .query_execution_counters + .edge_record_hydration_reads + .store(0, Ordering::Relaxed); + published + .view + .query_execution_counters + .edge_record_hydration_calls + .store(0, Ordering::Relaxed); published .view .query_execution_counters @@ -4561,6 +6432,26 @@ impl DatabaseEngine { .query_execution_counters .final_verifier_record_reads .store(0, Ordering::Relaxed); + published + .view + .query_execution_counters + .edge_full_scan_pages + .store(0, Ordering::Relaxed); + published + .view + .query_execution_counters + .endpoint_adjacency_candidates + .store(0, Ordering::Relaxed); + published + .view + .query_execution_counters + .pattern_edge_pending_entries + .store(0, Ordering::Relaxed); + published + .view + .query_execution_counters + .public_edge_query_calls + .store(0, Ordering::Relaxed); } pub(crate) fn set_flush_pause( @@ -4796,7 +6687,7 @@ struct EngineCore { next_node_id: u64, /// Running edge ID counter. Monotonically increasing. next_edge_id: u64, - /// Whether to enforce edge uniqueness on (from, to, type_id). + /// Whether to enforce edge uniqueness on (from, to, label_id). edge_uniqueness: bool, /// Memtable size threshold for auto-flush (bytes). 0 = manual only. flush_threshold: usize, @@ -4832,6 +6723,9 @@ struct EngineCore { next_edge_id_seen: Arc, /// Monotonic shared view of the latest durable engine_seq. engine_seq_seen: Arc, + /// Shared runtime node-label/edge-label catalog. Manifest writers merge this + /// before checkpointing so token WAL generations are not retired early. + label_catalog: Arc>, /// Serialize all manifest writes across engine, flush publisher, and compaction. manifest_write_lock: Arc>, /// Frozen memtable epochs awaiting or undergoing flush, newest-first. @@ -4846,7 +6740,7 @@ struct EngineCore { active_wal_generation_id: u64, /// Handle for the persistent background flush worker thread. bg_flush: Option, - /// Runtime declaration catalog keyed by `(type_id, prop_key, kind)`. + /// Runtime declaration catalog keyed by `(target_label_id, prop_key, kind)`. secondary_index_catalog: Arc>, /// Runtime declaration entries kept in sync with background state changes. secondary_index_entries: Arc>, @@ -4951,7 +6845,7 @@ struct BgCompactResult { reader: SegmentReader, old_seg_dirs: Vec, stats: CompactionStats, - input_segment_ids: NodeIdSet, + input_segment_snapshots: Vec, maintained_equality_index_ids: NodeIdSet, maintained_range_index_ids: NodeIdSet, secondary_index_report: SecondaryIndexMaintenanceReport, @@ -4977,14 +6871,15 @@ struct BgFlushHandle { #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct SecondaryIndexLookupKey { - type_id: u32, + discriminant: SecondaryIndexTargetDiscriminant, + target_label_id: u32, prop_key: String, kind: SecondaryIndexKind, } enum SecondaryIndexJob { Build { index_id: u64 }, - DropCleanup { index_id: u64 }, + DropCleanup { entry: SecondaryIndexManifestEntry }, Shutdown, } @@ -5070,6 +6965,7 @@ struct BuiltFlushResult { dense_config: Option, maintained_equality_index_ids: NodeIdSet, maintained_range_index_ids: NodeIdSet, + secondary_indexes: Vec, } /// Cheap foreground-only adoption payload. No disk I/O remains at this stage. @@ -5199,6 +7095,15 @@ impl EngineCore { // Load or create manifest let loaded_manifest = load_manifest(path)?; let created_manifest = loaded_manifest.is_none(); + if created_manifest { + let artifacts = manifestless_database_artifacts(path)?; + if !artifacts.is_empty() { + return Err(EngineError::ManifestError(format!( + "database artifacts exist without a manifest label-token schema: {}; old numeric-only databases are not migrated", + artifacts.join(", ") + ))); + } + } let mut manifest = match loaded_manifest { Some(m) => m, None => default_manifest(), @@ -5209,20 +7114,6 @@ impl EngineCore { write_manifest(path, &manifest)?; }; - // --- WAL generation migration --- - // If this is an old manifest (next_wal_generation_id == 0) and data.wal - // exists, migrate to WAL generation format by renaming data.wal → wal_0.wal. - let legacy_wal_path = path.join("data.wal"); - let gen0_path = wal_generation_path(path, 0); - if manifest.next_wal_generation_id == 0 - && manifest.active_wal_generation_id == 0 - && manifest.pending_flush_epochs.is_empty() - && legacy_wal_path.exists() - && !gen0_path.exists() - { - std::fs::rename(&legacy_wal_path, &gen0_path)?; - } - // Ensure next_wal_generation_id is at least active + 1 and above any // pending epoch generation IDs. let mut max_gen = manifest.active_wal_generation_id; @@ -5244,8 +7135,12 @@ impl EngineCore { for seg_info in manifest.segments.iter().rev() { let seg_path = segment_dir(path, seg_info.id); if seg_path.exists() { - let reader = - SegmentReader::open(&seg_path, seg_info.id, manifest.dense_vector.as_ref())?; + let reader = SegmentReader::open_with_info( + &seg_path, + seg_info, + manifest.dense_vector.as_ref(), + &manifest.secondary_indexes, + )?; segments.push(Arc::new(reader)); } else if manifest.pending_flush_epochs.iter().any(|e| { e.state == FlushEpochState::PublishedPendingRetire @@ -5292,6 +7187,8 @@ impl EngineCore { write_manifest(path, &manifest)?; } + let mut runtime_label_catalog = RuntimeLabelCatalog::from_manifest(&manifest)?; + // --- Replay WAL generations --- // Frozen epochs are replayed into separate immutable memtables so their // WAL files and manifest entries are preserved. Published degree overlays @@ -5310,10 +7207,11 @@ impl EngineCore { let mut immutable_bytes_on_open: usize = 0; for &(epoch_id, wal_gen_id) in &frozen_epochs { - let (frozen_mt, degree_overlay) = replay_wal_generation_to_memtable_and_overlay( + let (frozen_mt, degree_overlay, _) = replay_wal_generation_to_memtable_and_overlay( path, wal_gen_id, manifest.dense_vector.as_ref(), + &mut runtime_label_catalog, &mut engine_seq, &immutable_epochs_on_open, &segments, @@ -5334,14 +7232,22 @@ impl EngineCore { // Replay active WAL generation into the active memtable and overlay. // Use the persisted engine_seq from each WAL record (V3 format). - let (memtable, active_degree_overlay) = replay_wal_generation_to_memtable_and_overlay( + let (memtable, active_degree_overlay, active_wal_durable_len) = + replay_wal_generation_to_memtable_and_overlay( + path, + manifest.active_wal_generation_id, + manifest.dense_vector.as_ref(), + &mut runtime_label_catalog, + &mut engine_seq, + &immutable_epochs_on_open, + &segments, + )?; + truncate_wal_generation_to( path, manifest.active_wal_generation_id, - manifest.dense_vector.as_ref(), - &mut engine_seq, - &immutable_epochs_on_open, - &segments, + active_wal_durable_len, )?; + runtime_label_catalog.apply_to_manifest(&mut manifest); // Compute next IDs from active memtable + immutable epochs + manifest. let mut max_node_id = manifest @@ -5369,6 +7275,7 @@ impl EngineCore { // write and manifest update, or between bg compact output and apply). // Safe to delete. The manifest is the source of truth. cleanup_orphan_segments(path, &manifest); + cleanup_orphan_optional_refresh_files(path, &manifest); cleanup_orphan_wal_files(path, &manifest); // Open WAL writer for the active generation @@ -5430,6 +7337,7 @@ impl EngineCore { let next_node_id_seen = Arc::new(AtomicU64::new(next_node_id)); let next_edge_id_seen = Arc::new(AtomicU64::new(next_edge_id)); let engine_seq_seen = Arc::new(AtomicU64::new(engine_seq)); + let label_catalog = Arc::new(RwLock::new(runtime_label_catalog)); let manifest_write_lock = Arc::new(Mutex::new(())); let mut engine = EngineCore { db_dir: path.to_path_buf(), @@ -5458,6 +7366,7 @@ impl EngineCore { next_node_id_seen, next_edge_id_seen, engine_seq_seen, + label_catalog, manifest_write_lock, immutable_epochs: immutable_epochs_on_open, immutable_bytes_total: immutable_bytes_on_open, @@ -5490,7 +7399,7 @@ impl EngineCore { engine.rebuild_secondary_index_catalog()?; engine.seed_secondary_indexes_from_manifest()?; engine.warm_declared_index_runtime_coverage_for_current_readers(); - engine.rebuild_published_read_sources(); + engine.rebuild_published_read_sources()?; Ok(engine) } @@ -5556,7 +7465,7 @@ impl EngineCore { } } let active_wal_generation_id = self.active_wal_generation_id; - self.with_runtime_manifest_write(|manifest| { + self.with_synced_runtime_manifest_write(|manifest| { manifest.active_wal_generation_id = active_wal_generation_id; Ok(()) }) @@ -5566,11 +7475,29 @@ impl EngineCore { &self.memtable } - fn build_read_manifest_state(&self) -> ReadManifestState { - ReadManifestState { - prune_policies: self.manifest.prune_policies.clone(), + fn build_read_manifest_state(&self) -> Result { + let catalog = self.label_catalog.read().unwrap(); + let prune_policies = self + .manifest + .prune_policies + .iter() + .map(|(name, policy)| { + resolve_manifest_prune_policy(policy, &catalog).map(|policy| (name.clone(), policy)) + }) + .collect::, _>>()?; + Ok(ReadManifestState { + prune_policies, dense_vector: self.manifest.dense_vector.clone(), - } + }) + } + + fn resolved_manifest_prune_policies(&self) -> Result, EngineError> { + let catalog = self.label_catalog.read().unwrap(); + self.manifest + .prune_policies + .values() + .map(|policy| resolve_manifest_prune_policy(policy, &catalog)) + .collect() } fn warm_declared_index_runtime_coverage_for_reader(&self, reader: &SegmentReader) { @@ -5589,7 +7516,10 @@ impl EngineCore { } } - fn build_published_read_sources(&self) -> Arc { + fn build_published_read_sources( + &self, + generation: u64, + ) -> Result, EngineError> { #[cfg(test)] self.published_read_source_builds .fetch_add(1, Ordering::Relaxed); @@ -5602,13 +7532,13 @@ impl EngineCore { &secondary_index_entries, )); let planner_stats = Arc::new(PlannerStatsView::build_from_readers( - self.published_read_sources_generation, + generation, &self.segments, &secondary_index_entries, declared_index_runtime_coverage.as_ref(), )); - Arc::new(PublishedReadSources { - manifest: self.build_read_manifest_state(), + Ok(Arc::new(PublishedReadSources { + manifest: self.build_read_manifest_state()?, memtable: Arc::clone(&self.memtable), immutable_epochs: self.immutable_epochs.clone(), segments: self.segments.iter().map(Arc::clone).collect(), @@ -5620,13 +7550,15 @@ impl EngineCore { planning_probe_counters: QueryPlanningProbeCounters::default(), #[cfg(test)] query_execution_counters: QueryExecutionCounters::default(), - }) + })) } - fn rebuild_published_read_sources(&mut self) { - self.published_read_sources_generation = - self.published_read_sources_generation.saturating_add(1); - self.published_read_sources = Some(self.build_published_read_sources()); + fn rebuild_published_read_sources(&mut self) -> Result<(), EngineError> { + let generation = self.published_read_sources_generation.saturating_add(1); + let sources = self.build_published_read_sources(generation)?; + self.published_read_sources_generation = generation; + self.published_read_sources = Some(sources); + Ok(()) } fn current_published_read_sources(&self) -> Arc { @@ -5660,16 +7592,26 @@ impl EngineCore { self.current_published_read_sources(), self.engine_seq, Arc::clone(&self.active_degree_overlay), + self.read_label_catalog_snapshot(), ) } fn published_read_state(&self) -> PublishedReadState { + self.published_read_state_with_catalog(self.read_label_catalog_snapshot()) + } + + fn published_read_state_with_catalog( + &self, + label_catalog: Arc, + ) -> PublishedReadState { PublishedReadState { view: Arc::new(ReadView::from_published_sources( self.current_published_read_sources(), self.engine_seq, Arc::clone(&self.active_degree_overlay), + Arc::clone(&label_catalog), )), + label_catalog, edge_uniqueness: self.edge_uniqueness, #[cfg(test)] engine_seq: self.engine_seq, @@ -5678,6 +7620,11 @@ impl EngineCore { } } + fn read_label_catalog_snapshot(&self) -> Arc { + let catalog = self.label_catalog.read().unwrap(); + Arc::new(ReadLabelCatalogSnapshot::from_runtime(&catalog)) + } + fn secondary_index_entries_snapshot(&self) -> SecondaryIndexEntries { self.secondary_index_entries.read().unwrap().clone() } @@ -5699,18 +7646,70 @@ impl EngineCore { for segment in &self.segments { let validation = match entry.kind { - SecondaryIndexKind::Equality => { - segment.validate_secondary_eq_sidecar(entry.index_id) - } - SecondaryIndexKind::Range { .. } => { - segment.validate_secondary_range_sidecar(entry.index_id) - } + SecondaryIndexKind::Equality => segment + .secondary_eq_sidecar_lightweight_available_for_target( + entry.index_id, + match &entry.target { + SecondaryIndexTarget::NodeProperty { .. } => { + PlannerStatsDeclaredIndexTarget::NodeProperty + } + SecondaryIndexTarget::EdgeProperty { .. } => { + PlannerStatsDeclaredIndexTarget::EdgeProperty + } + }, + ), + SecondaryIndexKind::Range { .. } => segment + .secondary_range_sidecar_lightweight_available_for_target( + entry.index_id, + match &entry.target { + SecondaryIndexTarget::NodeProperty { .. } => { + PlannerStatsDeclaredIndexTarget::NodeProperty + } + SecondaryIndexTarget::EdgeProperty { .. } => { + PlannerStatsDeclaredIndexTarget::EdgeProperty + } + }, + ), }; match validation { Ok(true) => continue, Ok(false) => { - entry.state = SecondaryIndexState::Building; - entry.last_error = None; + let is_edge = + matches!(&entry.target, SecondaryIndexTarget::EdgeProperty { .. }); + let kind = match (&entry.kind, is_edge) { + (SecondaryIndexKind::Equality, false) => { + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + } + } + (SecondaryIndexKind::Range { .. }, false) => { + SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + } + } + (SecondaryIndexKind::Equality, true) => { + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + } + } + (SecondaryIndexKind::Range { .. }, true) => { + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: entry.index_id, + } + } + }; + match segment.optional_component_availability(kind) { + ComponentAvailability::Missing | ComponentAvailability::Available => { + entry.state = SecondaryIndexState::Building; + entry.last_error = None; + } + ComponentAvailability::Incompatible { reason } + | ComponentAvailability::CorruptIdentity { reason } + | ComponentAvailability::Unsupported { reason } => { + entry.state = SecondaryIndexState::Failed; + entry.last_error = Some(reason); + } + } dirty = true; break; } @@ -5725,7 +7724,15 @@ impl EngineCore { } if dirty { - write_manifest(&self.db_dir, &self.manifest)?; + let new_manifest = { + let _guard = self.manifest_write_lock.lock().unwrap(); + let mut manifest = self.load_current_manifest_for_write()?; + manifest.secondary_indexes = self.manifest.secondary_indexes.clone(); + self.merge_checkpointed_runtime_manifest_state(&mut manifest); + write_manifest(&self.db_dir, &manifest)?; + manifest + }; + self.manifest = new_manifest; } Ok(()) } @@ -5786,6 +7793,7 @@ impl EngineCore { let next_node_id_seen = Arc::clone(&self.next_node_id_seen); let next_edge_id_seen = Arc::clone(&self.next_edge_id_seen); let engine_seq_seen = Arc::clone(&self.engine_seq_seen); + let label_catalog = Arc::clone(&self.label_catalog); #[cfg(test)] let build_pause = Arc::clone(&self.secondary_index_build_pause); let handle = std::thread::spawn(move || { @@ -5800,6 +7808,7 @@ impl EngineCore { next_node_id_seen, next_edge_id_seen, engine_seq_seen, + label_catalog, #[cfg(test)] build_pause, ) @@ -5874,6 +7883,10 @@ impl EngineCore { .fetch_max(self.engine_seq, Ordering::Release); } + fn checkpointable_wal_generation(&self) -> Option { + self.active_wal_generation_id.checked_sub(1) + } + fn merge_runtime_manifest_counters(&self, manifest: &mut ManifestState) { merge_runtime_manifest_counters_from_shared( manifest, @@ -5883,6 +7896,20 @@ impl EngineCore { ); } + fn merge_checkpointed_runtime_manifest_state(&self, manifest: &mut ManifestState) { + self.merge_runtime_manifest_counters(manifest); + merge_checkpointed_label_catalog_into_manifest( + manifest, + &self.label_catalog, + self.checkpointable_wal_generation(), + ); + } + + fn merge_synced_runtime_manifest_state(&self, manifest: &mut ManifestState) { + self.merge_runtime_manifest_counters(manifest); + merge_runtime_label_catalog_into_manifest(manifest, &self.label_catalog); + } + fn load_current_manifest_for_write(&self) -> Result { let mut manifest = load_manifest_readonly(&self.db_dir)? .ok_or_else(|| EngineError::ManifestError("manifest missing".into()))?; @@ -5892,6 +7919,24 @@ impl EngineCore { manifest.active_wal_generation_id = manifest .active_wal_generation_id .max(self.manifest.active_wal_generation_id); + merge_checkpointed_label_catalog_into_manifest( + &mut manifest, + &self.label_catalog, + self.checkpointable_wal_generation(), + ); + Ok(manifest) + } + + fn load_current_manifest_for_synced_write(&self) -> Result { + let mut manifest = load_manifest_readonly(&self.db_dir)? + .ok_or_else(|| EngineError::ManifestError("manifest missing".into()))?; + manifest.next_wal_generation_id = manifest + .next_wal_generation_id + .max(self.manifest.next_wal_generation_id); + manifest.active_wal_generation_id = manifest + .active_wal_generation_id + .max(self.manifest.active_wal_generation_id); + merge_runtime_label_catalog_into_manifest(&mut manifest, &self.label_catalog); Ok(manifest) } @@ -5902,7 +7947,20 @@ impl EngineCore { let _guard = self.manifest_write_lock.lock().unwrap(); let mut manifest = self.load_current_manifest_for_write()?; let result = mutate(&mut manifest)?; - self.merge_runtime_manifest_counters(&mut manifest); + self.merge_checkpointed_runtime_manifest_state(&mut manifest); + write_manifest(&self.db_dir, &manifest)?; + self.manifest = manifest; + Ok(result) + } + + fn with_synced_runtime_manifest_write( + &mut self, + mutate: impl FnOnce(&mut ManifestState) -> Result, + ) -> Result { + let _guard = self.manifest_write_lock.lock().unwrap(); + let mut manifest = self.load_current_manifest_for_synced_write()?; + let result = mutate(&mut manifest)?; + self.merge_synced_runtime_manifest_state(&mut manifest); write_manifest(&self.db_dir, &manifest)?; self.manifest = manifest; Ok(result) @@ -5910,6 +7968,69 @@ impl EngineCore { // --- Write path helpers --- + fn apply_label_token_op_after_wal_append(&mut self, op: &WalOp) -> Result<(), EngineError> { + match op { + WalOp::EnsureNodeLabel { label, label_id } => { + { + let mut catalog = self.label_catalog.write().unwrap(); + catalog.apply_node_label( + label.clone(), + *label_id, + Some(self.active_wal_generation_id), + )?; + catalog.apply_to_manifest(&mut self.manifest); + } + Ok(()) + } + WalOp::EnsureEdgeLabel { label, label_id } => { + { + let mut catalog = self.label_catalog.write().unwrap(); + catalog.apply_edge_label( + label.clone(), + *label_id, + Some(self.active_wal_generation_id), + )?; + catalog.apply_to_manifest(&mut self.manifest); + } + Ok(()) + } + WalOp::UpsertNode(node) => { + let catalog = self.label_catalog.read().unwrap(); + for &label_id in node.label_ids.as_slice() { + if !catalog.node_id_to_label.contains_key(&label_id) { + return Err(EngineError::InvalidOperation(format!( + "node label_id {} does not exist in the node label catalog", + label_id + ))); + } + } + Ok(()) + } + WalOp::UpsertEdge(edge) => { + if self + .label_catalog + .read() + .unwrap() + .edge_id_to_label + .contains_key(&edge.label_id) + { + Ok(()) + } else { + Err(EngineError::InvalidOperation(format!( + "edge-label label_id {} does not exist in the edge-label catalog", + edge.label_id + ))) + } + } + WalOp::DeleteNode { .. } | WalOp::DeleteEdge { .. } => Ok(()), + WalOp::BeginAtomicBatch { .. } | WalOp::CommitAtomicBatch { .. } => { + Err(EngineError::InvalidOperation( + "WAL atomic batch markers cannot be applied as normal write ops".into(), + )) + } + } + } + /// Metadata-only logical edge lookup for degree-cache maintenance. /// Checks memtable first, then segments newest-to-oldest, respecting /// tombstones. Avoids full property decode on the write path. @@ -5998,7 +8119,12 @@ impl EngineCore { Self::collect_degree_delta_for_old_edge(deltas, old); } } - WalOp::DeleteNode { .. } | WalOp::UpsertNode(_) => {} + WalOp::DeleteNode { .. } + | WalOp::UpsertNode(_) + | WalOp::EnsureNodeLabel { .. } + | WalOp::EnsureEdgeLabel { .. } + | WalOp::BeginAtomicBatch { .. } + | WalOp::CommitAtomicBatch { .. } => {} } } @@ -6017,7 +8143,12 @@ impl EngineCore { match op { WalOp::UpsertEdge(edge) => Some(edge.id), WalOp::DeleteEdge { id, .. } => Some(*id), - WalOp::UpsertNode(_) | WalOp::DeleteNode { .. } => None, + WalOp::UpsertNode(_) + | WalOp::DeleteNode { .. } + | WalOp::EnsureNodeLabel { .. } + | WalOp::EnsureEdgeLabel { .. } + | WalOp::BeginAtomicBatch { .. } + | WalOp::CommitAtomicBatch { .. } => None, } } @@ -6032,7 +8163,13 @@ impl EngineCore { valid_from: edge.valid_from, valid_to: edge.valid_to, }), - WalOp::DeleteEdge { .. } | WalOp::DeleteNode { .. } | WalOp::UpsertNode(_) => None, + WalOp::DeleteEdge { .. } + | WalOp::DeleteNode { .. } + | WalOp::UpsertNode(_) + | WalOp::EnsureNodeLabel { .. } + | WalOp::EnsureEdgeLabel { .. } + | WalOp::BeginAtomicBatch { .. } + | WalOp::CommitAtomicBatch { .. } => None, } } @@ -6071,6 +8208,7 @@ impl EngineCore { let old_edge = edge_id .and_then(|id| edge_states.get(&id).copied().flatten()) .map(OldEdgeInfo::from_core); + self.apply_label_token_op_after_wal_append(op)?; self.active_memtable().apply_op(op, *seq); Self::collect_degree_delta_for_op(op, old_edge, &mut degree_deltas); if let Some(edge_id) = edge_id { @@ -6092,6 +8230,7 @@ impl EngineCore { .map(OldEdgeInfo::from_core); self.wal_append(|w| w.append(op, seq))?; self.engine_seq = seq; + self.apply_label_token_op_after_wal_append(op)?; self.active_memtable().apply_op(op, seq); let mut degree_deltas: NodeIdMap = NodeIdMap::default(); Self::collect_degree_delta_for_op(op, old_edge, &mut degree_deltas); @@ -6217,7 +8356,7 @@ impl EngineCore { // 2. Allocate new WAL generation let old_wal_gen = self.active_wal_generation_id; let epoch_id = old_wal_gen; - let new_wal_gen = self.with_runtime_manifest_write(|manifest| { + let new_wal_gen = self.with_synced_runtime_manifest_write(|manifest| { let new_wal_gen = manifest.next_wal_generation_id; manifest.next_wal_generation_id = new_wal_gen + 1; manifest.pending_flush_epochs.push(FlushEpochMeta { @@ -6429,6 +8568,7 @@ impl EngineCore { let next_node_id_seen = Arc::clone(&self.next_node_id_seen); let next_edge_id_seen = Arc::clone(&self.next_edge_id_seen); let engine_seq_seen = Arc::clone(&self.engine_seq_seen); + let label_catalog = Arc::clone(&self.label_catalog); let publish_runtime = self.runtime.clone(); #[cfg(test)] let publish_pause = Arc::clone(&self.flush_publish_pause); @@ -6443,6 +8583,7 @@ impl EngineCore { next_node_id_seen, next_edge_id_seen, engine_seq_seen, + label_catalog, publish_cancel, publish_events_ready, publish_runtime, @@ -6840,12 +8981,21 @@ impl EngineCore { return Ok(()); } - // Snapshot current segment IDs and paths for the background thread. - let input_segments: Vec<(u64, PathBuf)> = self + // Snapshot current root SegmentInfo and paths for the background thread. + let input_segments: Vec<(SegmentInfo, PathBuf)> = self .segments .iter() - .map(|s| (s.segment_id, segment_dir(&self.db_dir, s.segment_id))) + .filter_map(|s| { + segment_info_for_id(&self.manifest.segments, s.segment_id) + .cloned() + .map(|info| (info, segment_dir(&self.db_dir, s.segment_id))) + }) .collect(); + if input_segments.len() != self.segments.len() { + return Err(EngineError::ManifestError( + "background compaction snapshot missing root segment info".into(), + )); + } // Allocate the output segment ID on the main thread. let seg_id = self.next_segment_id; self.next_segment_id += 1; @@ -6854,8 +9004,7 @@ impl EngineCore { self.flush_count_since_last_compact = 0; let db_dir = self.db_dir.clone(); - let prune_policies: Vec = - self.manifest.prune_policies.values().cloned().collect(); + let prune_policies = self.resolved_manifest_prune_policies()?; let dense_vector = self.manifest.dense_vector.clone(); let secondary_indexes = self.secondary_index_entries_snapshot(); let cancel = Arc::new(AtomicBool::new(false)); @@ -6953,18 +9102,27 @@ impl EngineCore { } }; - let live_seg_ids: NodeIdSet = manifest.segments.iter().map(|s| s.id).collect(); - for input_id in &result.input_segment_ids { - if !live_seg_ids.contains(input_id) { + for input_info in &result.input_segment_snapshots { + let Some(live_info) = segment_info_for_id(&manifest.segments, input_info.id) else { + let output_dir = segment_dir(&self.db_dir, result.stats.output_segment_id); + let _ = std::fs::remove_dir_all(output_dir); + return None; + }; + if live_info.segment_data_id != input_info.segment_data_id { let output_dir = segment_dir(&self.db_dir, result.stats.output_segment_id); let _ = std::fs::remove_dir_all(output_dir); return None; } } + let input_segment_ids: NodeIdSet = result + .input_segment_snapshots + .iter() + .map(|segment| segment.id) + .collect(); manifest .segments - .retain(|s| !result.input_segment_ids.contains(&s.id)); + .retain(|s| !input_segment_ids.contains(&s.id)); manifest.segments.push(result.seg_info.clone()); apply_secondary_index_failure_report(&mut manifest, &result.secondary_index_report); let rebuild_equality_index_ids = reconcile_background_output_equality_declarations( @@ -6975,7 +9133,7 @@ impl EngineCore { &mut manifest, &result.maintained_range_index_ids, ); - self.merge_runtime_manifest_counters(&mut manifest); + self.merge_checkpointed_runtime_manifest_state(&mut manifest); if let Err(e) = write_manifest(&self.db_dir, &manifest) { eprintln!("Background compaction: manifest write failed: {}", e); @@ -6999,8 +9157,13 @@ impl EngineCore { } // Remove input segments, keep any new segments added by flushes during // background compaction (they have different IDs). + let input_segment_ids: NodeIdSet = result + .input_segment_snapshots + .iter() + .map(|segment| segment.id) + .collect(); self.segments - .retain(|s| !result.input_segment_ids.contains(&s.segment_id)); + .retain(|s| !input_segment_ids.contains(&s.segment_id)); // Compacted segment is oldest; push to end (segments are newest-first). self.warm_declared_index_runtime_coverage_for_reader(&result.reader); self.segments.push(Arc::new(result.reader)); @@ -7073,7 +9236,6 @@ impl EngineCore { if self.segments.len() < 2 { return Ok(None); } - self.compacting = true; let result = self.compact_with_progress_inner(&mut callback); self.compacting = false; @@ -7105,7 +9267,7 @@ impl EngineCore { let total_input_edges: u64 = self.segments.iter().map(|s| s.edge_count()).sum(); let has_tombstones = self.segments.iter().any(|s| s.has_tombstones()); - let policies: Vec = self.manifest.prune_policies.values().cloned().collect(); + let policies = self.resolved_manifest_prune_policies()?; let secondary_indexes = self.secondary_index_entries_snapshot(); let degree_sidecar_expected = policies.is_empty() && self.segments.iter().all(|s| s.degree_delta_available()); @@ -7165,19 +9327,30 @@ impl EngineCore { let _ = std::fs::remove_dir_all(&tmp_dir); return Err(e.into()); } + if let Some(parent) = final_dir.parent() { + if let Err(e) = fsync_dir(parent) { + self.next_segment_id -= 1; + let _ = std::fs::remove_dir_all(&final_dir); + return Err(e); + } + } // Open new segment reader BEFORE modifying any state (M3 fix) - let new_reader = - match SegmentReader::open(&final_dir, seg_id, self.manifest.dense_vector.as_ref()) { - Ok(r) => r, - Err(e) => { - // Output segment exists on disk but we can't read it. - // Clean up orphan directory and release the segment ID. - self.next_segment_id -= 1; - let _ = std::fs::remove_dir_all(&final_dir); - return Err(e); - } - }; + let new_reader = match SegmentReader::open_with_info( + &final_dir, + &seg_info, + self.manifest.dense_vector.as_ref(), + &secondary_indexes, + ) { + Ok(r) => r, + Err(e) => { + // Output segment exists on disk but we can't read it. + // Clean up orphan directory and release the segment ID. + self.next_segment_id -= 1; + let _ = std::fs::remove_dir_all(&final_dir); + return Err(e); + } + }; if degree_sidecar_expected && !new_reader.degree_delta_available() { self.next_segment_id -= 1; let _ = std::fs::remove_dir_all(&final_dir); @@ -7200,7 +9373,7 @@ impl EngineCore { manifest.segments.retain(|s| !old_seg_ids.contains(&s.id)); manifest.segments.push(seg_info.clone()); apply_secondary_index_failure_report(&mut manifest, &secondary_index_report); - self.merge_runtime_manifest_counters(&mut manifest); + self.merge_checkpointed_runtime_manifest_state(&mut manifest); write_manifest(&self.db_dir, &manifest)?; manifest }; @@ -7329,7 +9502,7 @@ impl EngineCore { seg_id: u64, callback: &mut F, has_tombstones: bool, - prune_policies: &[PrunePolicy], + prune_policies: &[ResolvedPrunePolicy], secondary_indexes: &[SecondaryIndexManifestEntry], input_segment_count: usize, total_input_nodes: u64, @@ -7506,6 +9679,7 @@ impl EngineCore { self.update_next_edge_id_seen(); } } + WalOp::EnsureNodeLabel { .. } | WalOp::EnsureEdgeLabel { .. } => {} _ => {} } } @@ -7585,51 +9759,230 @@ fn get_edge_core_from_sources( return Ok(None); } } - for seg in segments { - if seg.is_edge_deleted(id) { - return Ok(None); - } - if let Some((from, to, created_at, updated_at, weight, valid_from, valid_to)) = - seg.get_edge_core(id)? - { - return Ok(Some(EdgeCore { - from, - to, - created_at, - updated_at, - weight, - valid_from, - valid_to, - })); + for seg in segments { + if seg.is_edge_deleted(id) { + return Ok(None); + } + if let Some((from, to, created_at, updated_at, weight, valid_from, valid_to)) = + seg.get_edge_core(id)? + { + return Ok(Some(EdgeCore { + from, + to, + created_at, + updated_at, + weight, + valid_from, + valid_to, + })); + } + } + Ok(None) +} + +struct ReplayAtomicBatch { + first_seq: u64, + op_count: u32, + ops: Vec<(u64, WalOp)>, +} + +impl ReplayAtomicBatch { + fn new(first_seq: u64, op_count: u32) -> Option { + if first_seq == 0 || op_count < 2 { + return None; + } + Some(Self { + first_seq, + op_count, + ops: Vec::new(), + }) + } + + fn push(&mut self, seq: u64, op: WalOp) -> bool { + let Ok(op_count) = usize::try_from(self.op_count) else { + return false; + }; + if self.ops.len() >= op_count { + return false; + } + self.ops.push((seq, op)); + true + } + + fn matches_commit(&self, first_seq: u64, op_count: u32) -> bool { + if self.first_seq != first_seq || self.op_count != op_count { + return false; + } + if self.ops.len() != op_count as usize { + return false; + } + self.ops.iter().enumerate().all(|(idx, (seq, _))| { + self.first_seq + .checked_add(idx as u64) + .is_some_and(|expected| *seq == expected) + }) + } +} + +fn replay_apply_normalized_op( + memtable: &Memtable, + degree_deltas: &mut NodeIdMap, + engine_seq: &mut u64, + seq: u64, + op: &WalOp, + old_edge: Option, +) { + *engine_seq = (*engine_seq).max(seq); + memtable.apply_op(op, seq); + EngineCore::collect_degree_delta_for_op(op, old_edge, degree_deltas); +} + +fn capture_replay_batch_edge_states( + memtable: &Memtable, + immutable_epochs: &[ImmutableEpoch], + segments: &[Arc], + ops: &[(u64, WalOp)], +) -> Result>, EngineError> { + let mut states: NodeIdMap> = NodeIdMap::default(); + for (_, op) in ops { + let Some(edge_id) = EngineCore::edge_id_for_degree_op(op) else { + continue; + }; + if states.contains_key(&edge_id) { + continue; + } + states.insert( + edge_id, + get_edge_core_from_sources(memtable, immutable_epochs, segments, edge_id)?, + ); + } + Ok(states) +} + +#[allow(clippy::too_many_arguments)] +fn replay_apply_committed_atomic_batch( + batch: ReplayAtomicBatch, + memtable: &Memtable, + degree_deltas: &mut NodeIdMap, + dense_config: Option<&DenseVectorConfig>, + label_catalog: &mut RuntimeLabelCatalog, + engine_seq: &mut u64, + wal_generation_id: u64, + immutable_epochs: &[ImmutableEpoch], + segments: &[Arc], +) -> Result<(), EngineError> { + let normalized_ops: Vec<(u64, WalOp)> = batch + .ops + .into_iter() + .map(|(seq, op)| normalize_wal_op_for_replay(dense_config, op).map(|op| (seq, op))) + .collect::>()?; + + let mut staged_catalog = label_catalog.clone(); + for (_, op) in &normalized_ops { + validate_or_apply_replayed_label_token_op(&mut staged_catalog, op, wal_generation_id)?; + } + + let mut edge_states = + capture_replay_batch_edge_states(memtable, immutable_epochs, segments, &normalized_ops)?; + *label_catalog = staged_catalog; + + for (seq, op) in normalized_ops { + let edge_id = EngineCore::edge_id_for_degree_op(&op); + let old_edge = edge_id + .and_then(|id| edge_states.get(&id).copied().flatten()) + .map(OldEdgeInfo::from_core); + replay_apply_normalized_op(memtable, degree_deltas, engine_seq, seq, &op, old_edge); + if let Some(edge_id) = edge_id { + edge_states.insert(edge_id, EngineCore::edge_core_after_op(&op)); } } - Ok(None) + + Ok(()) } fn replay_wal_generation_to_memtable_and_overlay( db_dir: &Path, wal_generation_id: u64, dense_config: Option<&DenseVectorConfig>, + label_catalog: &mut RuntimeLabelCatalog, engine_seq: &mut u64, immutable_epochs: &[ImmutableEpoch], segments: &[Arc], -) -> Result<(Memtable, Arc), EngineError> { +) -> Result<(Memtable, Arc, u64), EngineError> { let memtable = Memtable::new(); let mut degree_deltas: NodeIdMap = NodeIdMap::default(); + let mut open_batch: Option = None; + let read_result = WalReader::read_generation_recoverable(db_dir, wal_generation_id)?; + let durable_len = read_result.durable_len; - for (seq, op) in WalReader::read_generation(db_dir, wal_generation_id)? { - let op = normalize_wal_op_for_replay(dense_config, op)?; - let old_edge = EngineCore::edge_id_for_degree_op(&op) - .map(|id| get_edge_core_from_sources(&memtable, immutable_epochs, segments, id)) - .transpose()? - .flatten() - .map(OldEdgeInfo::from_core); - *engine_seq = (*engine_seq).max(seq); - memtable.apply_op(&op, seq); - EngineCore::collect_degree_delta_for_op(&op, old_edge, &mut degree_deltas); + for (seq, op) in read_result.records { + match op { + WalOp::BeginAtomicBatch { + first_seq, + op_count, + } => { + if open_batch.is_some() { + break; + } + let Some(batch) = ReplayAtomicBatch::new(first_seq, op_count) else { + break; + }; + open_batch = Some(batch); + } + WalOp::CommitAtomicBatch { + first_seq, + op_count, + } => { + let Some(batch) = open_batch.take() else { + break; + }; + if !batch.matches_commit(first_seq, op_count) { + break; + } + replay_apply_committed_atomic_batch( + batch, + &memtable, + &mut degree_deltas, + dense_config, + label_catalog, + engine_seq, + wal_generation_id, + immutable_epochs, + segments, + )?; + } + op => { + if let Some(batch) = open_batch.as_mut() { + if !batch.push(seq, op) { + break; + } + continue; + } + + let op = normalize_wal_op_for_replay(dense_config, op)?; + validate_or_apply_replayed_label_token_op(label_catalog, &op, wal_generation_id)?; + let old_edge = EngineCore::edge_id_for_degree_op(&op) + .map(|id| get_edge_core_from_sources(&memtable, immutable_epochs, segments, id)) + .transpose()? + .flatten() + .map(OldEdgeInfo::from_core); + replay_apply_normalized_op( + &memtable, + &mut degree_deltas, + engine_seq, + seq, + &op, + old_edge, + ); + } + } } - Ok((memtable, DegreeOverlaySnapshot::from_flat(degree_deltas))) + Ok(( + memtable, + DegreeOverlaySnapshot::from_flat(degree_deltas), + durable_len, + )) } impl Drop for EngineCore { @@ -7730,6 +10083,12 @@ fn cleanup_orphan_segments(db_dir: &Path, manifest: &ManifestState) { } } +fn cleanup_orphan_optional_refresh_files(db_dir: &Path, manifest: &ManifestState) { + for segment in &manifest.segments { + cleanup_orphan_optional_component_files(&segment_dir(db_dir, segment.id)); + } +} + /// Remove WAL generation files on disk that are not referenced by the manifest. /// Orphan WAL files can appear when a crash occurs after WAL retirement completes /// on disk but before the manifest is updated, or from other interrupted sequences. @@ -7969,8 +10328,6 @@ fn bg_flush_build_worker( } let current_secondary_indexes = secondary_index_entries.read().unwrap().clone(); - let maintained_equality_index_ids = equality_index_ids_snapshot(¤t_secondary_indexes); - let maintained_range_index_ids = range_index_ids_snapshot(¤t_secondary_indexes); let needs_reseed = current_secondary_indexes.iter().any(|entry| { !work .frozen @@ -8017,6 +10374,32 @@ fn bg_flush_build_worker( cancel.store(true, Ordering::Relaxed); break; } else { + let maintained_index_ids = + match maintained_secondary_index_ids_from_segment_manifest( + &work.final_dir, + ¤t_secondary_indexes, + ) { + Ok(ids) => ids, + Err(e) => { + let _ = std::fs::remove_dir_all(&work.final_dir); + send_bg_flush_event( + &event_tx, + &events_ready, + &runtime, + BgFlushEvent::Failed(FlushPipelineError { + epoch_id: work.epoch_id, + wal_generation_id: work.wal_gen_id, + stage: FlushPipelineStage::Build, + message: format!( + "segment maintained index scan failed: {}", + e + ), + }), + ); + cancel.store(true, Ordering::Relaxed); + break; + } + }; BuiltFlushResult { epoch_id: work.epoch_id, wal_gen_to_retire: work.wal_gen_id, @@ -8024,8 +10407,10 @@ fn bg_flush_build_worker( seg_id: work.seg_id, final_dir: work.final_dir, dense_config: work.dense_config, - maintained_equality_index_ids, - maintained_range_index_ids, + maintained_equality_index_ids: maintained_index_ids + .equality_index_ids, + maintained_range_index_ids: maintained_index_ids.range_index_ids, + secondary_indexes: current_secondary_indexes, } } } else { @@ -8099,6 +10484,7 @@ fn bg_flush_publish_worker( next_node_id_seen: Arc, next_edge_id_seen: Arc, engine_seq_seen: Arc, + label_catalog: Arc>, cancel: Arc, events_ready: Arc, runtime: Option>, @@ -8109,10 +10495,11 @@ fn bg_flush_publish_worker( break; } - let reader = match SegmentReader::open( + let reader = match SegmentReader::open_with_info( &result.final_dir, - result.seg_id, + &result.seg_info, result.dense_config.as_ref(), + &result.secondary_indexes, ) { Ok(reader) => reader, Err(e) => { @@ -8191,6 +10578,11 @@ fn bg_flush_publish_worker( manifest.next_engine_seq = manifest .next_engine_seq .max(engine_seq_seen.load(Ordering::Acquire)); + merge_checkpointed_label_catalog_into_manifest( + &mut manifest, + &label_catalog, + Some(result.wal_gen_to_retire), + ); let rebuild_equality_index_ids = reconcile_background_output_equality_declarations( &mut manifest, &result.maintained_equality_index_ids, @@ -8257,16 +10649,14 @@ fn bg_flush_publish_worker( fn bg_compact_worker( db_dir: PathBuf, seg_id: u64, - input_segments: Vec<(u64, PathBuf)>, - prune_policies: Vec, + input_segments: Vec<(SegmentInfo, PathBuf)>, + prune_policies: Vec, dense_vector: Option, secondary_indexes: SecondaryIndexEntries, cancel: &AtomicBool, #[cfg(test)] compact_pause: &Arc>>, ) -> Result { let compact_start = std::time::Instant::now(); - let maintained_equality_index_ids = equality_index_ids_snapshot(&secondary_indexes); - let maintained_range_index_ids = range_index_ids_snapshot(&secondary_indexes); #[cfg(test)] if let Some(hook) = compact_pause.lock().unwrap().take() { @@ -8277,11 +10667,12 @@ fn bg_compact_worker( // Re-open input segments (independent mmap handles, safe to use concurrently // with the main thread's readers of the same files). let mut segments = Vec::with_capacity(input_segments.len()); - for (id, path) in &input_segments { - segments.push(Arc::new(SegmentReader::open( + for (info, path) in &input_segments { + segments.push(Arc::new(SegmentReader::open_with_info( path, - *id, + info, dense_vector.as_ref(), + &secondary_indexes, )?)); } @@ -8341,9 +10732,30 @@ fn bg_compact_worker( let _ = std::fs::remove_dir_all(&tmp_dir); return Err(e.into()); } + if let Some(parent) = final_dir.parent() { + if let Err(e) = fsync_dir(parent) { + let _ = std::fs::remove_dir_all(&final_dir); + return Err(e); + } + } + let maintained_index_ids = match maintained_secondary_index_ids_from_segment_manifest( + &final_dir, + &secondary_indexes, + ) { + Ok(ids) => ids, + Err(error) => { + let _ = std::fs::remove_dir_all(&final_dir); + return Err(error); + } + }; // Open the output segment reader (will be sent back to the main thread). - let reader = match SegmentReader::open(&final_dir, seg_id, dense_vector.as_ref()) { + let reader = match SegmentReader::open_with_info( + &final_dir, + &seg_info, + dense_vector.as_ref(), + &secondary_indexes, + ) { Ok(r) => r, Err(e) => { let _ = std::fs::remove_dir_all(&final_dir); @@ -8361,10 +10773,13 @@ fn bg_compact_worker( reader.warm_declared_index_runtime_coverage(entry); } - let input_segment_ids: NodeIdSet = input_segments.iter().map(|(id, _)| *id).collect(); + let input_segment_snapshots: Vec = input_segments + .iter() + .map(|(info, _)| info.clone()) + .collect(); let old_seg_dirs: Vec = input_segments .iter() - .map(|(id, _)| segment_dir(&db_dir, *id)) + .map(|(info, _)| segment_dir(&db_dir, info.id)) .collect(); let stats = CompactionStats { @@ -8384,9 +10799,9 @@ fn bg_compact_worker( reader, old_seg_dirs, stats, - input_segment_ids, - maintained_equality_index_ids, - maintained_range_index_ids, + input_segment_snapshots, + maintained_equality_index_ids: maintained_index_ids.equality_index_ids, + maintained_range_index_ids: maintained_index_ids.range_index_ids, secondary_index_report, }) } @@ -8400,7 +10815,7 @@ struct NodeWinner { seg_idx: usize, data_offset: u64, data_len: u32, - type_id: u32, + label_ids: NodeLabelSet, updated_at: i64, weight: f32, key_len: u16, @@ -8418,7 +10833,7 @@ struct EdgeWinner { data_len: u32, from: u64, to: u64, - type_id: u32, + label_id: u32, updated_at: i64, weight: f32, valid_from: i64, @@ -8438,21 +10853,21 @@ struct V3Plan { /// OR across policies (any match → pruned), AND within each policy. /// Uses the shared `matches_prune_cutoff` helper (same logic as read-time filtering). fn matches_any_prune_policy_meta( - type_id: u32, + label_ids: &NodeLabelSet, updated_at: i64, weight: f32, - policies: &[PrunePolicy], + policies: &[ResolvedPrunePolicy], now: i64, ) -> bool { for policy in policies { let age_cutoff = policy.max_age_ms.map(|age| now - age); if matches_prune_cutoff( - type_id, + label_ids, updated_at, weight, age_cutoff, policy.max_weight, - policy.type_id, + policy.label_id, ) { return true; } @@ -8467,7 +10882,7 @@ fn matches_any_prune_policy_meta( /// fields without decoding full records. fn v3_plan_winners( segments: &[Arc], - prune_policies: &[PrunePolicy], + prune_policies: &[ResolvedPrunePolicy], deleted_nodes: &NodeIdSet, deleted_edges: &NodeIdSet, ) -> Result { @@ -8482,52 +10897,47 @@ fn v3_plan_winners( for (seg_idx, seg) in segments.iter().enumerate() { let count = seg.node_meta_count() as usize; for i in 0..count { - let ( - node_id, - data_offset, - data_len, - type_id, - updated_at, - weight, - key_len, - _prop_hash_offset, - _prop_hash_count, - last_write_seq, - ) = seg.node_meta_at(i)?; + let meta = seg.node_meta_at(i)?; let (dense_vector_offset, dense_vector_len, sparse_vector_offset, sparse_vector_len) = seg.node_vector_meta_at(i)?; - if seen_nodes.contains(&node_id) { + if seen_nodes.contains(&meta.node_id) { continue; // Already have a newer version } - seen_nodes.insert(node_id); + seen_nodes.insert(meta.node_id); - if deleted_nodes.contains(&node_id) { + if deleted_nodes.contains(&meta.node_id) { continue; // Tombstoned } if has_policies - && matches_any_prune_policy_meta(type_id, updated_at, weight, prune_policies, now) + && matches_any_prune_policy_meta( + &meta.label_ids, + meta.updated_at, + meta.weight, + prune_policies, + now, + ) { - pruned_node_ids.insert(node_id); + pruned_node_ids.insert(meta.node_id); continue; } node_winners.insert( - node_id, + meta.node_id, NodeWinner { seg_idx, - data_offset, - data_len, - type_id, - updated_at, - weight, - key_len, + data_offset: meta.data_offset, + data_len: meta.data_len, + label_ids: meta.label_ids, + updated_at: meta.updated_at, + weight: meta.weight, + key_len: meta.key_len, dense_vector_offset, dense_vector_len, sparse_vector_offset, sparse_vector_len, - last_write_seq, + last_write_seq: meta.last_write_seq, }, ); } @@ -8549,7 +10959,7 @@ fn v3_plan_winners( data_len, from, to, - type_id, + label_id, updated_at, weight, valid_from, @@ -8586,7 +10996,7 @@ fn v3_plan_winners( data_len, from, to, - type_id, + label_id, updated_at, weight, valid_from, @@ -8630,9 +11040,13 @@ fn v3_build_output( .map(|(&id, w)| (id, w.seg_idx, w.data_offset, w.data_len)) .collect(); + let mut core_writer = create_compaction_core_writer(tmp_dir, seg_id)?; + // Raw-copy winning records to output data files - let node_data = write_v3_nodes_dat(tmp_dir, segments, &node_winner_list)?; - let edge_data = write_v3_edges_dat(tmp_dir, segments, &edge_winner_list)?; + let (node_record, node_data) = + write_v3_nodes_dat(&mut core_writer, segments, &node_winner_list)?; + let (edge_record, edge_data) = + write_v3_edges_dat(&mut core_writer, segments, &edge_winner_list)?; // Build CompactNodeMeta/CompactEdgeMeta by zipping planner winners with output offsets. // Both are sorted by ID (BTreeMap iteration + write order), so a linear zip replaces @@ -8666,7 +11080,7 @@ fn v3_build_output( node_id, new_data_offset, data_len, - type_id: w.type_id, + label_ids: w.label_ids, updated_at: w.updated_at, weight: w.weight, key_len: w.key_len, @@ -8696,38 +11110,49 @@ fn v3_build_output( data_len, from: w.from, to: w.to, - type_id: w.type_id, + label_id: w.label_id, updated_at: w.updated_at, weight: w.weight, valid_from: w.valid_from, valid_to: w.valid_to, + src_seg_idx: w.seg_idx, + src_data_offset: w.data_offset, last_write_seq: w.last_write_seq, }); } + let (source_groups, dense_points) = write_compaction_source_components( + seg_id, + &mut core_writer, + segments, + node_record, + edge_record, + &node_metas, + &edge_metas, + )?; + // Build all secondary indexes and sidecars from metadata - let secondary_index_report = write_indexes_from_metadata_with_secondary_indexes( + let component_output = write_indexes_from_metadata_with_secondary_indexes( seg_id, tmp_dir, + &mut core_writer, segments, &node_metas, &edge_metas, dense_config, + dense_points, write_degree_sidecar, secondary_indexes, + source_groups, )?; + let mut records = component_output.records; + records.extend(finish_compaction_core_writer(core_writer)?); let node_count = plan.node_winners.len() as u64; let edge_count = plan.edge_winners.len() as u64; + let seg_info = finalize_compaction_segment(tmp_dir, seg_id, node_count, edge_count, records)?; - Ok(( - SegmentInfo { - id: seg_id, - node_count, - edge_count, - }, - secondary_index_report, - )) + Ok((seg_info, component_output.report)) } fn collect_fast_merge_node_metas( @@ -8738,51 +11163,45 @@ fn collect_fast_merge_node_metas( for (seg_idx, seg) in segments.iter().enumerate() { let info = ©_info[seg_idx]; for i in 0..seg.node_meta_count() as usize { - let ( - node_id, - data_offset, - data_len, - type_id, - updated_at, - weight, - key_len, - _prop_hash_offset, - _prop_hash_count, - last_write_seq, - ) = seg.node_meta_at(i)?; + let meta = seg.node_meta_at(i)?; let (dense_vector_offset, dense_vector_len, sparse_vector_offset, sparse_vector_len) = seg.node_vector_meta_at(i)?; - let rebased_offset = - info.new_data_base - .checked_add(data_offset.checked_sub(info.orig_data_start).ok_or_else( - || { + let rebased_offset = info + .new_data_base + .checked_add( + meta.data_offset + .checked_sub(info.orig_data_start) + .ok_or_else(|| { EngineError::CorruptRecord(format!( "segment {} node {} data offset {} precedes data section {}", - seg.segment_id, node_id, data_offset, info.orig_data_start + seg.segment_id, + meta.node_id, + meta.data_offset, + info.orig_data_start )) - }, - )?) - .ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} node {} merged offset overflow", - seg.segment_id, node_id - )) - })?; + })?, + ) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} node {} merged offset overflow", + seg.segment_id, meta.node_id + )) + })?; metas.push(CompactNodeMeta { - node_id, + node_id: meta.node_id, new_data_offset: rebased_offset, - data_len, - type_id, - updated_at, - weight, - key_len, + data_len: meta.data_len, + label_ids: meta.label_ids, + updated_at: meta.updated_at, + weight: meta.weight, + key_len: meta.key_len, dense_vector_offset, dense_vector_len, sparse_vector_offset, sparse_vector_len, src_seg_idx: seg_idx, - src_data_offset: data_offset, - last_write_seq, + src_data_offset: meta.data_offset, + last_write_seq: meta.last_write_seq, }); } } @@ -8812,7 +11231,7 @@ fn collect_fast_merge_edge_metas( data_len, from, to, - type_id, + label_id, updated_at, weight, valid_from, @@ -8841,11 +11260,13 @@ fn collect_fast_merge_edge_metas( data_len, from, to, - type_id, + label_id, updated_at, weight, valid_from, valid_to, + src_seg_idx: seg_idx, + src_data_offset: data_offset, last_write_seq, }); } @@ -8871,30 +11292,46 @@ fn build_fast_merge_output( ) -> Result<(SegmentInfo, SecondaryIndexMaintenanceReport), EngineError> { std::fs::create_dir_all(tmp_dir)?; - let node_copy_info = write_merged_nodes_dat(tmp_dir, segments)?; - let edge_copy_info = write_merged_edges_dat(tmp_dir, segments)?; + let mut core_writer = create_compaction_core_writer(tmp_dir, seg_id)?; + let (node_record, node_copy_info) = write_merged_nodes_dat(&mut core_writer, segments)?; + let (edge_record, edge_copy_info) = write_merged_edges_dat(&mut core_writer, segments)?; let node_metas = collect_fast_merge_node_metas(segments, &node_copy_info)?; let edge_metas = collect_fast_merge_edge_metas(segments, &edge_copy_info)?; - let secondary_index_report = write_indexes_from_metadata_with_secondary_indexes( + let (source_groups, dense_points) = write_compaction_source_components( + seg_id, + &mut core_writer, + segments, + node_record, + edge_record, + &node_metas, + &edge_metas, + )?; + + let component_output = write_indexes_from_metadata_with_secondary_indexes( seg_id, tmp_dir, + &mut core_writer, segments, &node_metas, &edge_metas, dense_config, + dense_points, true, secondary_indexes, + source_groups, + )?; + let mut records = component_output.records; + records.extend(finish_compaction_core_writer(core_writer)?); + let seg_info = finalize_compaction_segment( + tmp_dir, + seg_id, + node_metas.len() as u64, + edge_metas.len() as u64, + records, )?; - Ok(( - SegmentInfo { - id: seg_id, - node_count: node_metas.len() as u64, - edge_count: edge_metas.len() as u64, - }, - secondary_index_report, - )) + Ok((seg_info, component_output.report)) } /// V3 background merge: metadata-only planning + raw binary copy. @@ -8931,7 +11368,7 @@ fn bg_standard_merge( tmp_dir: &Path, seg_id: u64, has_tombstones: bool, - prune_policies: &[PrunePolicy], + prune_policies: &[ResolvedPrunePolicy], dense_config: Option<&DenseVectorConfig>, secondary_indexes: &[SecondaryIndexManifestEntry], cancel: &AtomicBool, @@ -8993,12 +11430,193 @@ mod tests { use super::*; use tempfile::TempDir; + fn internal_node_record( + engine: &DatabaseEngine, + id: u64, + ) -> Result, EngineError> { + let (_guard, published) = engine.runtime.published_snapshot()?; + published.view.get_node(id) + } + + fn internal_edge_record( + engine: &DatabaseEngine, + id: u64, + ) -> Result, EngineError> { + let (_guard, published) = engine.runtime.published_snapshot()?; + published.view.get_edge(id) + } + + fn internal_edge_records( + engine: &DatabaseEngine, + ids: &[u64], + ) -> Result>, EngineError> { + let (_guard, published) = engine.runtime.published_snapshot()?; + published.view.get_edges(ids) + } + + fn seed_internal_wal_op_tokens( + engine: &DatabaseEngine, + ops: &[WalOp], + ) -> Result<(), EngineError> { + let node_label_for_label_id = |label_id| match label_id { + 1 => "Person".to_string(), + 2 => "Company".to_string(), + 3 => "Article".to_string(), + 4 => "Topic".to_string(), + 5 => "City".to_string(), + 6 => "Project".to_string(), + 7 => "Account".to_string(), + 8 => "Team".to_string(), + 9 => "User".to_string(), + 10 => "Document".to_string(), + 20 => "Group".to_string(), + 90 => "Metric".to_string(), + 99 => "MissingLabel".to_string(), + 110 => "SearchNode110".to_string(), + 117 => "SearchNode117".to_string(), + 120 => "SearchNode120".to_string(), + 831 => "SpecialNode831".to_string(), + 999 => "SpecialNode999".to_string(), + 1024 => "SpecialNode1024".to_string(), + _ => format!("NodeLabel{label_id}"), + }; + let edge_label_for_label_id = |label_id| match label_id { + 1 => "RELATES_TO".to_string(), + 2 => "WORKS_AT".to_string(), + 3 => "LIKES".to_string(), + 4 => "MENTIONS".to_string(), + 5 => "OWNS".to_string(), + 6 => "FOLLOWS".to_string(), + 7 => "FRIENDS_WITH".to_string(), + 8 => "COLLABORATES_WITH".to_string(), + 9 => "RELATED_TO".to_string(), + 10 => "KNOWS".to_string(), + 11 => "BLOCKS".to_string(), + 12 => "DEPENDS_ON".to_string(), + 13 => "ASSIGNED_TO".to_string(), + 14 => "REVIEWED_BY".to_string(), + 15 => "PUBLISHED_BY".to_string(), + 16 => "TAGGED_WITH".to_string(), + 20 => "REPORTS_TO".to_string(), + 30 => "RATES".to_string(), + 40 => "REFERENCES".to_string(), + 99 => "MISSING_EDGE_LABEL".to_string(), + 831 => "SPECIAL_EDGE_831".to_string(), + 999 => "SPECIAL_EDGE_999".to_string(), + 1024 => "SPECIAL_EDGE_1024".to_string(), + _ => format!("EDGE_LABEL_{label_id}"), + }; + engine.with_core_mut(|core| { + let mut dirty = false; + { + let mut catalog = core.label_catalog.write().unwrap(); + for op in ops { + match op { + WalOp::UpsertNode(node) => { + for &label_id in node.label_ids.as_slice() { + let label = node_label_for_label_id(label_id); + if !catalog.node_id_to_label.contains_key(&label_id) { + catalog.apply_node_label(label, label_id, None)?; + dirty = true; + } + } + } + WalOp::UpsertEdge(edge) => { + let label = edge_label_for_label_id(edge.label_id); + if !catalog.edge_id_to_label.contains_key(&edge.label_id) { + catalog.apply_edge_label(label, edge.label_id, None)?; + dirty = true; + } + } + WalOp::EnsureNodeLabel { .. } + | WalOp::EnsureEdgeLabel { .. } + | WalOp::DeleteNode { .. } + | WalOp::DeleteEdge { .. } + | WalOp::BeginAtomicBatch { .. } + | WalOp::CommitAtomicBatch { .. } => {} + } + } + if dirty { + catalog.apply_to_manifest(&mut core.manifest); + } + } + if dirty { + write_manifest(&core.db_dir, &core.manifest)?; + } + Ok(()) + }) + } + + fn seed_internal_node_labels( + engine: &DatabaseEngine, + label_ids: &[u32], + ) -> Result<(), EngineError> { + let node_label_for_label_id = |label_id| match label_id { + 1 => "Person".to_string(), + 2 => "Company".to_string(), + 3 => "Article".to_string(), + 4 => "Topic".to_string(), + 5 => "City".to_string(), + 6 => "Project".to_string(), + 7 => "Account".to_string(), + 8 => "Team".to_string(), + 9 => "User".to_string(), + 10 => "Document".to_string(), + 20 => "Group".to_string(), + 90 => "Metric".to_string(), + 99 => "MissingLabel".to_string(), + 110 => "SearchNode110".to_string(), + 117 => "SearchNode117".to_string(), + 120 => "SearchNode120".to_string(), + 831 => "SpecialNode831".to_string(), + 999 => "SpecialNode999".to_string(), + 1024 => "SpecialNode1024".to_string(), + _ => format!("NodeLabel{label_id}"), + }; + engine.with_core_mut(|core| { + let mut dirty = false; + { + let mut catalog = core.label_catalog.write().unwrap(); + for &label_id in label_ids { + if !catalog.node_id_to_label.contains_key(&label_id) { + catalog.apply_node_label( + node_label_for_label_id(label_id), + label_id, + None, + )?; + dirty = true; + } + } + if dirty { + catalog.apply_to_manifest(&mut core.manifest); + } + } + if dirty { + write_manifest(&core.db_dir, &core.manifest)?; + } + Ok(()) + }) + } + + fn write_internal_wal_op(engine: &DatabaseEngine, op: &WalOp) -> Result<(), EngineError> { + seed_internal_wal_op_tokens(engine, std::slice::from_ref(op))?; + engine.write_op(op) + } + + fn write_internal_wal_op_batch( + engine: &DatabaseEngine, + ops: &[WalOp], + ) -> Result<(), EngineError> { + seed_internal_wal_op_tokens(engine, ops)?; + engine.write_op_batch(ops) + } + fn make_node(id: u64, key: &str) -> NodeRecord { let mut props = BTreeMap::new(); props.insert("name".to_string(), PropValue::String(key.to_string())); NodeRecord { id, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: key.to_string(), props, created_at: 1000 * id as i64, @@ -9015,7 +11633,7 @@ mod tests { id, from, to, - type_id: 10, + label_id: 10, props: BTreeMap::new(), created_at: 2000 * id as i64, updated_at: 2000 * id as i64 + 1, @@ -9026,6 +11644,233 @@ mod tests { } } + fn is_atomic_batch_marker(op: &WalOp) -> bool { + matches!( + op, + WalOp::BeginAtomicBatch { .. } | WalOp::CommitAtomicBatch { .. } + ) + } + + fn marker_free_wal_records(records: &[(u64, WalOp)]) -> Vec<(u64, WalOp)> { + records + .iter() + .filter(|(_, op)| !is_atomic_batch_marker(op)) + .cloned() + .collect() + } + + fn assert_no_atomic_batch_markers(records: &[(u64, WalOp)]) { + assert!( + records.iter().all(|(_, op)| !is_atomic_batch_marker(op)), + "expected marker-free WAL records, got {records:?}" + ); + } + + fn split_top_level_args(call_args: &str) -> Vec<&str> { + let mut args = Vec::new(); + let mut start = 0usize; + let mut paren_depth = 0i32; + let mut bracket_depth = 0i32; + let mut brace_depth = 0i32; + let mut in_string = false; + let mut escaped = false; + + for (idx, ch) in call_args.char_indices() { + if in_string { + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + in_string = false; + } + continue; + } + + match ch { + '"' => in_string = true, + '(' => paren_depth += 1, + ')' => paren_depth -= 1, + '[' => bracket_depth += 1, + ']' => bracket_depth -= 1, + '{' => brace_depth += 1, + '}' => brace_depth -= 1, + ',' if paren_depth == 0 && bracket_depth == 0 && brace_depth == 0 => { + args.push(call_args[start..idx].trim()); + start = idx + ch.len_utf8(); + } + _ => {} + } + } + args.push(call_args[start..].trim()); + args + } + + fn call_args_after<'a>(source: &'a str, needle: &str) -> Vec<&'a str> { + let mut calls = Vec::new(); + let mut offset = 0usize; + while let Some(pos) = source[offset..].find(needle) { + let needle_start = offset + pos; + let after_needle = needle_start + needle.len(); + let Some(open_rel) = source[after_needle..].find('(') else { + break; + }; + let open = after_needle + open_rel; + if !source[after_needle..open].trim().is_empty() { + offset = after_needle; + continue; + } + + let mut depth = 0i32; + let mut in_string = false; + let mut escaped = false; + for (rel, ch) in source[open..].char_indices() { + if in_string { + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + in_string = false; + } + continue; + } + + match ch { + '"' => in_string = true, + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + let close = open + rel; + calls.push(&source[open + 1..close]); + offset = close + ch.len_utf8(); + break; + } + } + _ => {} + } + } + + if offset <= needle_start { + offset = after_needle; + } + } + calls + } + + fn arg_starts_with_number(arg: &str) -> bool { + arg.trim_start() + .chars() + .next() + .is_some_and(|ch| ch.is_ascii_digit()) + } + + #[test] + fn engine_tests_do_not_use_legacy_numeric_public_adapter_patterns() { + let adapter_forbidden = [ + concat!("struct DatabaseEngine", "(super::DatabaseEngine)"), + concat!("legacy", "_node_label"), + concat!("legacy", "_edge", "_", "type"), + concat!("seed", "_legacy_numeric_test_label_tokens"), + concat!("test_node", "_label("), + concat!("test_edge", "_", "type("), + concat!("test_node", "_label_names("), + concat!("test_edge", "_", "type_names("), + concat!("test_node", "_label_option("), + concat!("test_node", "_key_query("), + concat!("test_node", "_key_queries("), + ]; + let included_test_forbidden = [ + adapter_forbidden.as_slice(), + &[ + concat!("node_type", "_filter:"), + concat!("get_nodes_by_keys", "(&[("), + ], + ] + .concat(); + + let engine_mod_path = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/engine/mod.rs"); + let test_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/engine/tests"); + let files = std::fs::read_dir(&test_dir).unwrap(); + let mut violations = Vec::new(); + + let engine_mod_source = std::fs::read_to_string(&engine_mod_path).unwrap(); + let engine_mod_display = engine_mod_path + .strip_prefix(env!("CARGO_MANIFEST_DIR")) + .unwrap_or(&engine_mod_path); + for forbidden in adapter_forbidden { + if engine_mod_source.contains(forbidden) { + violations.push(format!( + "{} contains `{}`", + engine_mod_display.display(), + forbidden + )); + } + } + + for file in files { + let path = file.unwrap().path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("rs") { + continue; + } + let source = std::fs::read_to_string(&path).unwrap(); + let display = path + .strip_prefix(env!("CARGO_MANIFEST_DIR")) + .unwrap_or(&path); + + for forbidden in &included_test_forbidden { + if source.contains(*forbidden) { + violations.push(format!("{} contains `{}`", display.display(), forbidden)); + } + } + + for (method, arg_index) in [ + ("upsert_node", 0usize), + ("ensure_node_property_index", 0), + ("ensure_edge_property_index", 0), + ("get_node_by_key", 0), + ("upsert_edge", 2), + ("get_edge_by_triple", 2), + ] { + for call_args in call_args_after(&source, method) { + let args = split_top_level_args(call_args); + if args + .get(arg_index) + .is_some_and(|arg| arg_starts_with_number(arg)) + { + violations.push(format!( + "{} has numeric public `{}` argument in `{}`", + display.display(), + method, + call_args.lines().next().unwrap_or(call_args).trim() + )); + } + } + } + + for (line_no, line) in source.lines().enumerate() { + if line.contains("label_filter_ids:") && !line.contains("edge_label_filter:") { + violations.push(format!( + "{}:{} contains legacy `{}`", + display.display(), + line_no + 1, + line.trim() + )); + } + } + } + + assert!( + violations.is_empty(), + "legacy numeric public API patterns remain:\n{}", + violations.join("\n") + ); + } + + include!("tests/label_catalog.rs"); + include!("tests/wal_atomic.rs"); include!("tests/lifecycle.rs"); include!("tests/txn.rs"); include!("tests/write.rs"); diff --git a/src/engine/query.rs b/src/engine/query.rs index 13a401a..29f33de 100644 --- a/src/engine/query.rs +++ b/src/engine/query.rs @@ -25,6 +25,44 @@ impl DatabaseEngine { published.view.explain_node_query(query) } + pub fn query_edge_ids( + &self, + query: &EdgeQuery, + ) -> Result { + let (_guard, published) = self.runtime.published_snapshot()?; + #[cfg(test)] + published + .view + .query_execution_counters + .public_edge_query_calls + .fetch_add(1, Ordering::Relaxed); + let outcome = published.view.query_edge_ids_outcome(query)?; + for followup in outcome.followups { + self.runtime.enqueue_secondary_index_read_followup(followup); + } + Ok(outcome.value) + } + + pub fn query_edges(&self, query: &EdgeQuery) -> Result { + let (_guard, published) = self.runtime.published_snapshot()?; + #[cfg(test)] + published + .view + .query_execution_counters + .public_edge_query_calls + .fetch_add(1, Ordering::Relaxed); + let outcome = published.view.query_edges_outcome(query)?; + for followup in outcome.followups { + self.runtime.enqueue_secondary_index_read_followup(followup); + } + Ok(outcome.value) + } + + pub fn explain_edge_query(&self, query: &EdgeQuery) -> Result { + let (_guard, published) = self.runtime.published_snapshot()?; + published.view.explain_edge_query(query) + } + pub fn query_pattern( &self, query: &GraphPatternQuery, diff --git a/src/engine/query_exec.rs b/src/engine/query_exec.rs index 0bccb0d..9611e8f 100644 --- a/src/engine/query_exec.rs +++ b/src/engine/query_exec.rs @@ -1,6 +1,7 @@ const QUERY_VERIFY_CHUNK: usize = 256; const PROPERTY_IN_LINEAR_VERIFY_THRESHOLD: usize = 16; const PATTERN_FRONTIER_BUDGET: usize = 65_536; +const EDGE_INTERSECTION_TINY_SET: usize = 64; struct QueryExecutionOutcome { value: T, @@ -13,6 +14,12 @@ struct VerifiedNodePage { next_cursor: Option, } +struct VerifiedEdgePage { + ids: Vec, + edges: Vec, + next_cursor: Option, +} + enum CandidateMaterializationResult { Ready { ids: Vec, @@ -34,6 +41,70 @@ enum FullScanNodeSource<'a> { Segment(&'a SegmentReader), } +enum FullScanEdgeSource<'a> { + Memtable { + memtable: &'a Memtable, + snapshot_seq: u64, + next_after: Option, + }, + Segment { + segment: &'a SegmentReader, + next: usize, + }, +} + +enum LabelEdgeSource<'a> { + Memtable { + memtable: &'a Memtable, + snapshot_seq: u64, + label_id: u32, + next_after: Option, + }, + Segment { + segment: &'a SegmentReader, + posting: SegmentLabelPosting, + next: usize, + }, +} + +enum EndpointEdgeSource<'a> { + Memtable(MemtableEndpointEdgeSource<'a>), + Segment(SegmentEndpointEdgeSource<'a>), +} + +#[derive(Clone, Copy)] +enum MemtableEndpointDirection { + Outgoing, + Incoming, +} + +struct MemtableEndpointCursor<'a> { + memtable: &'a Memtable, + node_id: u64, + direction: MemtableEndpointDirection, + label_filter_ids: Option<&'a [u32]>, + snapshot_seq: u64, + next_after: Option, +} + +struct MemtableEndpointEdgeSource<'a> { + cursors: Vec>, + heap: BinaryHeap>, + last_seen: Option, +} + +struct SegmentEndpointEdgeSource<'a> { + segment: &'a SegmentReader, + cursors: Vec, + heap: BinaryHeap>, + last_seen: Option, +} + +#[derive(Default)] +struct EdgeEndpointVisibilityCache { + visible: NodeIdMap, +} + impl FullScanNodeSource<'_> { fn get_id(&self, index: usize) -> Result, EngineError> { match self { @@ -56,6 +127,367 @@ impl FullScanNodeSource<'_> { } } +impl<'a> FullScanEdgeSource<'a> { + fn memtable( + memtable: &'a Memtable, + snapshot_seq: u64, + after: Option, + ) -> FullScanEdgeSource<'a> { + FullScanEdgeSource::Memtable { + memtable, + snapshot_seq, + next_after: after, + } + } + + fn segment( + segment: &'a SegmentReader, + after: Option, + ) -> Result, EngineError> { + let next = match after { + Some(after) => { + let mut lo = 0usize; + let mut hi = segment.edge_meta_count() as usize; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let (edge_id, ..) = segment.edge_meta_at(mid)?; + if edge_id <= after { + lo = mid + 1; + } else { + hi = mid; + } + } + lo + } + None => 0, + }; + Ok(FullScanEdgeSource::Segment { segment, next }) + } + + fn next_id(&mut self) -> Result, EngineError> { + match self { + FullScanEdgeSource::Memtable { + memtable, + snapshot_seq, + next_after, + } => { + let edge_id = memtable.next_visible_edge_id_after(*snapshot_seq, *next_after); + if let Some(edge_id) = edge_id { + *next_after = Some(edge_id); + } + Ok(edge_id) + } + FullScanEdgeSource::Segment { segment, next } => { + if *next >= segment.edge_meta_count() as usize { + return Ok(None); + } + let (edge_id, ..) = segment.edge_meta_at(*next)?; + *next += 1; + Ok(Some(edge_id)) + } + } + } +} + +impl<'a> LabelEdgeSource<'a> { + fn memtable( + memtable: &'a Memtable, + snapshot_seq: u64, + label_id: u32, + after: Option, + ) -> LabelEdgeSource<'a> { + LabelEdgeSource::Memtable { + memtable, + snapshot_seq, + label_id, + next_after: after, + } + } + + fn segment( + segment: &'a SegmentReader, + posting: SegmentLabelPosting, + after: Option, + ) -> Result, EngineError> { + let next = match after { + Some(after) => segment.edge_label_id_lower_bound_posting(posting, after)?, + None => 0, + }; + Ok(LabelEdgeSource::Segment { + segment, + posting, + next, + }) + } + + fn next_id(&mut self) -> Result, EngineError> { + match self { + LabelEdgeSource::Memtable { + memtable, + snapshot_seq, + label_id, + next_after, + } => { + let edge_id = + memtable.next_visible_edge_by_label_id_after(*label_id, *snapshot_seq, *next_after); + if let Some(edge_id) = edge_id { + *next_after = Some(edge_id); + } + Ok(edge_id) + } + LabelEdgeSource::Segment { + segment, + posting, + next, + } => { + let edge_id = segment.edge_label_id_at_posting(*posting, *next)?; + if edge_id.is_some() { + *next += 1; + } + Ok(edge_id) + } + } + } +} + +impl<'a> EndpointEdgeSource<'a> { + fn memtable( + memtable: &'a Memtable, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&'a [u32]>, + snapshot_seq: u64, + after: Option, + ) -> Self { + Self::Memtable(MemtableEndpointEdgeSource::new( + memtable, + node_ids, + direction, + label_filter_ids, + snapshot_seq, + after, + )) + } + + fn segment( + segment: &'a SegmentReader, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + after: Option, + ) -> Result { + Ok(Self::Segment(SegmentEndpointEdgeSource::new( + segment, + node_ids, + direction, + label_filter_ids, + after, + )?)) + } + + fn next_id(&mut self) -> Result, EngineError> { + match self { + Self::Memtable(source) => Ok(source.next_id()), + Self::Segment(source) => source.next_id(), + } + } +} + +impl<'a> MemtableEndpointEdgeSource<'a> { + fn new( + memtable: &'a Memtable, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&'a [u32]>, + snapshot_seq: u64, + after: Option, + ) -> Self { + let mut cursors = Vec::new(); + for &node_id in node_ids { + match direction { + Direction::Outgoing => cursors.push(MemtableEndpointCursor { + memtable, + node_id, + direction: MemtableEndpointDirection::Outgoing, + label_filter_ids, + snapshot_seq, + next_after: after, + }), + Direction::Incoming => cursors.push(MemtableEndpointCursor { + memtable, + node_id, + direction: MemtableEndpointDirection::Incoming, + label_filter_ids, + snapshot_seq, + next_after: after, + }), + Direction::Both => { + cursors.push(MemtableEndpointCursor { + memtable, + node_id, + direction: MemtableEndpointDirection::Outgoing, + label_filter_ids, + snapshot_seq, + next_after: after, + }); + cursors.push(MemtableEndpointCursor { + memtable, + node_id, + direction: MemtableEndpointDirection::Incoming, + label_filter_ids, + snapshot_seq, + next_after: after, + }); + } + } + } + + let mut source = Self { + cursors, + heap: BinaryHeap::new(), + last_seen: None, + }; + for cursor_index in 0..source.cursors.len() { + if let Some(edge_id) = source.next_cursor_id(cursor_index) { + source.heap.push(Reverse((edge_id, cursor_index))); + } + } + source + } + + fn next_cursor_id(&mut self, cursor_index: usize) -> Option { + let cursor = &mut self.cursors[cursor_index]; + let edge_id = match cursor.direction { + MemtableEndpointDirection::Outgoing => cursor.memtable.next_visible_edge_from_endpoint_after( + cursor.node_id, + cursor.label_filter_ids, + cursor.snapshot_seq, + cursor.next_after, + ), + MemtableEndpointDirection::Incoming => cursor.memtable.next_visible_edge_to_endpoint_after( + cursor.node_id, + cursor.label_filter_ids, + cursor.snapshot_seq, + cursor.next_after, + ), + }; + if let Some(edge_id) = edge_id { + cursor.next_after = Some(edge_id); + } + edge_id + } + + fn next_id(&mut self) -> Option { + while let Some(Reverse((edge_id, cursor_index))) = self.heap.pop() { + if let Some(next_id) = self.next_cursor_id(cursor_index) { + self.heap.push(Reverse((next_id, cursor_index))); + } + if self.last_seen == Some(edge_id) { + continue; + } + self.last_seen = Some(edge_id); + return Some(edge_id); + } + None + } +} + +impl<'a> SegmentEndpointEdgeSource<'a> { + fn new( + segment: &'a SegmentReader, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + after: Option, + ) -> Result { + let mut cursors = segment.endpoint_adj_posting_cursors(node_ids, direction, label_filter_ids)?; + let mut heap = BinaryHeap::new(); + for (cursor_index, cursor) in cursors.iter_mut().enumerate() { + while let Some(edge_id) = segment.next_adj_posting_edge_id(cursor)? { + if after.is_none_or(|after| edge_id > after) { + heap.push(Reverse((edge_id, cursor_index))); + break; + } + } + } + Ok(Self { + segment, + cursors, + heap, + last_seen: None, + }) + } + + fn next_id(&mut self) -> Result, EngineError> { + while let Some(Reverse((edge_id, cursor_index))) = self.heap.pop() { + if let Some(next_id) = + self.segment + .next_adj_posting_edge_id(&mut self.cursors[cursor_index])? + { + self.heap.push(Reverse((next_id, cursor_index))); + } + if self.last_seen == Some(edge_id) { + continue; + } + self.last_seen = Some(edge_id); + return Ok(Some(edge_id)); + } + Ok(None) + } +} + +impl EdgeEndpointVisibilityCache { + fn ensure_endpoint_ids( + &mut self, + sources: &SourceList<'_>, + endpoint_ids: &[u64], + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result<(), EngineError> { + let mut missing = Vec::new(); + for &endpoint_id in endpoint_ids { + if !self.visible.contains_key(&endpoint_id) { + missing.push(endpoint_id); + } + } + if missing.is_empty() { + return Ok(()); + } + + missing.sort_unstable(); + missing.dedup(); + let states = sources.find_node_visibility_meta(&missing)?; + for (&endpoint_id, state) in missing.iter().zip(states.iter()) { + let visible = match state { + NodeVisibilityState::Live(meta) => policy_cutoffs.is_none_or(|cutoffs| { + !cutoffs.excludes_fields(&meta.label_ids, meta.updated_at, meta.weight) + }), + NodeVisibilityState::Deleted | NodeVisibilityState::Missing => false, + }; + self.visible.insert(endpoint_id, visible); + } + Ok(()) + } + + fn ensure_edge_endpoints( + &mut self, + sources: &SourceList<'_>, + metas: &[EdgeMetadataCandidate], + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result<(), EngineError> { + let mut endpoint_ids = Vec::with_capacity(metas.len().saturating_mul(2)); + for meta in metas { + endpoint_ids.push(meta.from); + endpoint_ids.push(meta.to); + } + self.ensure_endpoint_ids(sources, &endpoint_ids, policy_cutoffs) + } + + fn edge_endpoints_visible(&self, meta: EdgeMetadataCandidate) -> bool { + self.visible.get(&meta.from).copied().unwrap_or(false) + && self.visible.get(&meta.to).copied().unwrap_or(false) + } +} + fn first_candidate_after(candidate_ids: &[u64], after: Option) -> usize { let Some(after) = after else { return 0; @@ -78,6 +510,37 @@ fn page_verify_target(limit: usize) -> usize { } } +fn edge_plan_is_filter_source(plan: &EdgePhysicalPlan) -> bool { + match plan { + EdgePhysicalPlan::Source(source) => matches!( + source.kind, + EdgeQueryCandidateSourceKind::EdgeWeightIndex + | EdgeQueryCandidateSourceKind::EdgeUpdatedAtIndex + | EdgeQueryCandidateSourceKind::EdgeValidFromIndex + | EdgeQueryCandidateSourceKind::EdgeValidToIndex + | EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex + | EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex + | EdgeQueryCandidateSourceKind::EdgeMetadataScan + ), + EdgePhysicalPlan::Intersect(inputs) | EdgePhysicalPlan::Union(inputs) => { + inputs.iter().all(edge_plan_is_filter_source) + } + EdgePhysicalPlan::Empty => false, + } +} + +fn edge_materialization_uses_limited_probe(materialization: &EdgeCandidateMaterialization) -> bool { + matches!( + materialization, + EdgeCandidateMaterialization::EdgeWeightIndex { .. } + | EdgeCandidateMaterialization::EdgeUpdatedAtIndex { .. } + | EdgeCandidateMaterialization::EdgeValidFromIndex { .. } + | EdgeCandidateMaterialization::EdgeValidToIndex { .. } + | EdgeCandidateMaterialization::EdgePropertyEqualityIndex { .. } + | EdgeCandidateMaterialization::EdgePropertyRangeIndex { .. } + ) +} + fn finalize_verified_page( mut ids: Vec, mut nodes: Vec, @@ -100,6 +563,28 @@ fn finalize_verified_page( } } +fn finalize_verified_edge_page( + mut ids: Vec, + mut edges: Vec, + limit: usize, +) -> VerifiedEdgePage { + let next_cursor = if limit > 0 && ids.len() > limit { + ids.truncate(limit); + if !edges.is_empty() { + edges.truncate(limit); + } + ids.last().copied() + } else { + None + }; + + VerifiedEdgePage { + ids, + edges, + next_cursor, + } +} + fn intersect_sorted_unique(left: &[u64], right: &[u64]) -> Vec { let mut intersection = Vec::with_capacity(left.len().min(right.len())); let mut left_index = 0; @@ -238,8 +723,88 @@ fn node_filter_matches(filter: &NormalizedNodeFilter, node: &NodeRecord) -> bool } } +fn node_filter_visibility_meta_matches( + filter: &NormalizedNodeFilter, + meta: &NodeVisibilityMeta, +) -> Option { + match filter { + NormalizedNodeFilter::AlwaysTrue => Some(true), + NormalizedNodeFilter::AlwaysFalse => Some(false), + NormalizedNodeFilter::UpdatedAtRange { lower_ms, upper_ms } => { + Some(meta.updated_at >= *lower_ms && meta.updated_at <= *upper_ms) + } + NormalizedNodeFilter::And(children) => { + for child in children { + match node_filter_visibility_meta_matches(child, meta) { + Some(true) => {} + Some(false) => return Some(false), + None => return None, + } + } + Some(true) + } + NormalizedNodeFilter::Or(children) => { + let mut needs_record = false; + for child in children { + match node_filter_visibility_meta_matches(child, meta) { + Some(true) => return Some(true), + Some(false) => {} + None => needs_record = true, + } + } + if needs_record { + None + } else { + Some(false) + } + } + NormalizedNodeFilter::Not(child) => { + node_filter_visibility_meta_matches(child, meta).map(|matched| !matched) + } + NormalizedNodeFilter::PropertyEquals { .. } + | NormalizedNodeFilter::PropertyIn { .. } + | NormalizedNodeFilter::PropertyRange { .. } + | NormalizedNodeFilter::PropertyExists { .. } + | NormalizedNodeFilter::PropertyMissing { .. } => None, + } +} + +fn node_filter_visibility_meta_compatible(filter: &NormalizedNodeFilter) -> bool { + match filter { + NormalizedNodeFilter::AlwaysTrue + | NormalizedNodeFilter::AlwaysFalse + | NormalizedNodeFilter::UpdatedAtRange { .. } => true, + NormalizedNodeFilter::And(children) | NormalizedNodeFilter::Or(children) => { + children.iter().all(node_filter_visibility_meta_compatible) + } + NormalizedNodeFilter::Not(child) => node_filter_visibility_meta_compatible(child), + NormalizedNodeFilter::PropertyEquals { .. } + | NormalizedNodeFilter::PropertyIn { .. } + | NormalizedNodeFilter::PropertyRange { .. } + | NormalizedNodeFilter::PropertyExists { .. } + | NormalizedNodeFilter::PropertyMissing { .. } => false, + } +} + +fn node_label_filter_matches(filter: &ResolvedNodeLabelFilter, labels: &NodeLabelSet) -> bool { + match filter { + ResolvedNodeLabelFilter::Unconstrained => true, + ResolvedNodeLabelFilter::Empty { .. } => false, + ResolvedNodeLabelFilter::LabelSet { + mode: LabelMatchMode::Any, + label_ids, + .. + } => label_ids.as_slice().iter().any(|&label_id| labels.contains(label_id)), + ResolvedNodeLabelFilter::LabelSet { + mode: LabelMatchMode::All, + label_ids, + .. + } => label_ids.as_slice().iter().all(|&label_id| labels.contains(label_id)), + } +} + fn query_node_matches(query: &NormalizedNodeQuery, node: &NodeRecord) -> bool { - if query.type_id.is_some_and(|type_id| node.type_id != type_id) { + if !node_label_filter_matches(&query.label_filter, &node.label_ids) { return false; } if !query.ids.is_empty() && query.ids.binary_search(&node.id).is_err() { @@ -252,93 +817,524 @@ fn query_node_matches(query: &NormalizedNodeQuery, node: &NodeRecord) -> bool { node_filter_matches(&query.filter, node) } -#[derive(Clone)] -struct PatternExecutionState { - nodes: Vec>, - edges: Vec>, -} - -#[derive(Clone, Copy)] -struct PatternEdgeContext { - source_id: u64, - target_index: usize, - target_id: Option, - direction: Direction, +fn query_node_visibility_meta_matches( + query: &NormalizedNodeQuery, + node_id: u64, + meta: &NodeVisibilityMeta, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, +) -> bool { + if !node_label_filter_matches(&query.label_filter, &meta.label_ids) { + return false; + } + if !query.ids.is_empty() && query.ids.binary_search(&node_id).is_err() { + return false; + } + if policy_cutoffs.is_some_and(|cutoffs| { + cutoffs.excludes_fields(&meta.label_ids, meta.updated_at, meta.weight) + }) { + return false; + } + if !node_filter_visibility_meta_matches(&query.filter, meta).unwrap_or(false) { + return false; + } + true } -struct PatternUnnamedAccumulator { - seen_edges: NodeIdSet, - best_by_target: NodeIdMap, - best_bound_entry: Option, - exceeded: bool, +#[derive(Clone, Copy, Debug)] +struct EdgeMetadataForQuery { + id: u64, + from: u64, + to: u64, + label_id: u32, + updated_at: i64, + weight: f32, + valid_from: i64, + valid_to: i64, } -impl PatternUnnamedAccumulator { - fn new() -> Self { +impl From<&EdgeRecord> for EdgeMetadataForQuery { + fn from(edge: &EdgeRecord) -> Self { Self { - seen_edges: NodeIdSet::default(), - best_by_target: NodeIdMap::default(), - best_bound_entry: None, - exceeded: false, + id: edge.id, + from: edge.from, + to: edge.to, + label_id: edge.label_id, + updated_at: edge.updated_at, + weight: edge.weight, + valid_from: edge.valid_from, + valid_to: edge.valid_to, } } +} - fn is_done(&self) -> bool { - self.best_bound_entry.is_some() || self.exceeded +impl From for EdgeMetadataForQuery { + fn from(meta: crate::edge_metadata::EdgeMetadataCandidate) -> Self { + Self { + id: meta.edge_id, + from: meta.from, + to: meta.to, + label_id: meta.label_id, + updated_at: meta.updated_at, + weight: meta.weight, + valid_from: meta.valid_from, + valid_to: meta.valid_to, + } } } -#[derive(Clone)] -struct PatternPendingEntry { - state_index: usize, - entry: NeighborEntry, +fn i64_range_matches(value: i64, lower: i64, upper: i64) -> bool { + value >= lower && value <= upper } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -struct PatternMatchSortKey { - anchor_node_id: u64, - node_ids: Vec, - edge_ids: Vec, +fn edge_weight_range_matches(value: f32, lower: Option, upper: Option) -> bool { + if value.is_nan() { + return false; + } + if lower.is_some_and(|lower| value < lower) { + return false; + } + if upper.is_some_and(|upper| value > upper) { + return false; + } + true } -fn reverse_pattern_direction(direction: Direction) -> Direction { - match direction { - Direction::Outgoing => Direction::Incoming, - Direction::Incoming => Direction::Outgoing, - Direction::Both => Direction::Both, +fn edge_filter_requires_hydration(filter: &NormalizedEdgeFilter) -> bool { + match filter { + NormalizedEdgeFilter::PropertyEquals { .. } + | NormalizedEdgeFilter::PropertyIn { .. } + | NormalizedEdgeFilter::PropertyRange { .. } + | NormalizedEdgeFilter::PropertyExists { .. } + | NormalizedEdgeFilter::PropertyMissing { .. } => true, + NormalizedEdgeFilter::And(children) | NormalizedEdgeFilter::Or(children) => { + children.iter().any(edge_filter_requires_hydration) + } + NormalizedEdgeFilter::Not(child) => edge_filter_requires_hydration(child), + _ => false, } } -fn edge_post_filter_matches( - predicates: &[EdgePostFilterPredicate], - edge: &EdgeRecord, -) -> bool { - predicates.iter().all(|predicate| match predicate { - EdgePostFilterPredicate::PropertyEquals { key, value } => { - edge.props.get(key).is_some_and(|candidate| candidate == value) +fn edge_filter_needs_full_metadata(filter: &NormalizedEdgeFilter) -> bool { + match filter { + NormalizedEdgeFilter::UpdatedAtRange { .. } => true, + NormalizedEdgeFilter::And(children) | NormalizedEdgeFilter::Or(children) => { + children.iter().any(edge_filter_needs_full_metadata) } - EdgePostFilterPredicate::PropertyRange { key, lower, upper } => edge - .props - .get(key) - .and_then(|value| range_value_within_bounds(value, lower.as_ref(), upper.as_ref())) - == Some(true), - }) + NormalizedEdgeFilter::Not(child) => edge_filter_needs_full_metadata(child), + _ => false, + } } -fn pattern_distinct_node_binding_ok( - state: &PatternExecutionState, - target_index: usize, - candidate_id: u64, -) -> bool { - state - .nodes - .iter() - .enumerate() - .all(|(index, bound)| index == target_index || *bound != Some(candidate_id)) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PatternEdgeFilterMode { + AlwaysTrue, + PostingMetadataOnly, + BatchMetadataNeeded, + PropertyVerifier, } -fn pattern_match_from_state( - query: &NormalizedGraphPatternQuery, +fn pattern_edge_filter_mode(filter: &NormalizedEdgeFilter) -> PatternEdgeFilterMode { + if filter.is_always_true() { + PatternEdgeFilterMode::AlwaysTrue + } else if edge_filter_requires_hydration(filter) { + PatternEdgeFilterMode::PropertyVerifier + } else if edge_filter_needs_full_metadata(filter) { + PatternEdgeFilterMode::BatchMetadataNeeded + } else { + PatternEdgeFilterMode::PostingMetadataOnly + } +} + +#[derive(Clone, Copy, Debug)] +struct EdgePostingMetadataForQuery { + weight: f32, + valid_from: i64, + valid_to: i64, +} + +fn edge_filter_posting_metadata_outcome( + filter: &NormalizedEdgeFilter, + meta: &EdgePostingMetadataForQuery, +) -> Option { + match filter { + NormalizedEdgeFilter::AlwaysTrue => Some(true), + NormalizedEdgeFilter::AlwaysFalse => Some(false), + NormalizedEdgeFilter::PropertyEquals { .. } + | NormalizedEdgeFilter::PropertyIn { .. } + | NormalizedEdgeFilter::PropertyRange { .. } + | NormalizedEdgeFilter::PropertyExists { .. } + | NormalizedEdgeFilter::PropertyMissing { .. } + | NormalizedEdgeFilter::UpdatedAtRange { .. } => None, + NormalizedEdgeFilter::WeightRange { lower, upper } => { + Some(edge_weight_range_matches(meta.weight, *lower, *upper)) + } + NormalizedEdgeFilter::ValidAt { epoch_ms } => { + Some(meta.valid_from <= *epoch_ms && *epoch_ms < meta.valid_to) + } + NormalizedEdgeFilter::ValidFromRange { lower_ms, upper_ms } => { + Some(i64_range_matches(meta.valid_from, *lower_ms, *upper_ms)) + } + NormalizedEdgeFilter::ValidToRange { lower_ms, upper_ms } => { + Some(i64_range_matches(meta.valid_to, *lower_ms, *upper_ms)) + } + NormalizedEdgeFilter::And(children) => { + let mut unknown = false; + for child in children { + match edge_filter_posting_metadata_outcome(child, meta) { + Some(false) => return Some(false), + Some(true) => {} + None => unknown = true, + } + } + if unknown { None } else { Some(true) } + } + NormalizedEdgeFilter::Or(children) => { + let mut unknown = false; + for child in children { + match edge_filter_posting_metadata_outcome(child, meta) { + Some(true) => return Some(true), + Some(false) => {} + None => unknown = true, + } + } + if unknown { None } else { Some(false) } + } + NormalizedEdgeFilter::Not(child) => { + edge_filter_posting_metadata_outcome(child, meta).map(|matched| !matched) + } + } +} + +fn edge_filter_posting_metadata_maybe_matches( + filter: &NormalizedEdgeFilter, + meta: &EdgePostingMetadataForQuery, +) -> bool { + edge_filter_posting_metadata_outcome(filter, meta).unwrap_or(true) +} + +fn edge_filter_metadata_outcome( + filter: &NormalizedEdgeFilter, + meta: &EdgeMetadataForQuery, +) -> Option { + match filter { + NormalizedEdgeFilter::AlwaysTrue => Some(true), + NormalizedEdgeFilter::AlwaysFalse => Some(false), + NormalizedEdgeFilter::PropertyEquals { .. } + | NormalizedEdgeFilter::PropertyIn { .. } + | NormalizedEdgeFilter::PropertyRange { .. } + | NormalizedEdgeFilter::PropertyExists { .. } + | NormalizedEdgeFilter::PropertyMissing { .. } => None, + NormalizedEdgeFilter::WeightRange { lower, upper } => { + Some(edge_weight_range_matches(meta.weight, *lower, *upper)) + } + NormalizedEdgeFilter::UpdatedAtRange { lower_ms, upper_ms } => { + Some(i64_range_matches(meta.updated_at, *lower_ms, *upper_ms)) + } + NormalizedEdgeFilter::ValidAt { epoch_ms } => { + Some(meta.valid_from <= *epoch_ms && *epoch_ms < meta.valid_to) + } + NormalizedEdgeFilter::ValidFromRange { lower_ms, upper_ms } => { + Some(i64_range_matches(meta.valid_from, *lower_ms, *upper_ms)) + } + NormalizedEdgeFilter::ValidToRange { lower_ms, upper_ms } => { + Some(i64_range_matches(meta.valid_to, *lower_ms, *upper_ms)) + } + NormalizedEdgeFilter::And(children) => { + let mut unknown = false; + for child in children { + match edge_filter_metadata_outcome(child, meta) { + Some(false) => return Some(false), + Some(true) => {} + None => unknown = true, + } + } + if unknown { None } else { Some(true) } + } + NormalizedEdgeFilter::Or(children) => { + let mut unknown = false; + for child in children { + match edge_filter_metadata_outcome(child, meta) { + Some(true) => return Some(true), + Some(false) => {} + None => unknown = true, + } + } + if unknown { None } else { Some(false) } + } + NormalizedEdgeFilter::Not(child) => { + edge_filter_metadata_outcome(child, meta).map(|matched| !matched) + } + } +} + +#[cfg(test)] +fn edge_filter_metadata_maybe_matches( + filter: &NormalizedEdgeFilter, + meta: &EdgeMetadataForQuery, +) -> bool { + edge_filter_metadata_outcome(filter, meta).unwrap_or(true) +} + +#[cfg(test)] +fn edge_filter_matches(filter: &NormalizedEdgeFilter, edge: &EdgeRecord) -> bool { + match filter { + NormalizedEdgeFilter::AlwaysTrue => true, + NormalizedEdgeFilter::AlwaysFalse => false, + NormalizedEdgeFilter::PropertyEquals { key, value } => edge + .props + .get(key) + .is_some_and(|candidate| prop_values_equal_for_filter(candidate, value)), + NormalizedEdgeFilter::PropertyIn { + key, + values, + value_keys, + } => edge + .props + .get(key) + .is_some_and(|candidate| property_in_filter_matches(candidate, values, value_keys)), + NormalizedEdgeFilter::PropertyRange { key, lower, upper } => edge + .props + .get(key) + .and_then(|value| range_value_within_bounds(value, lower.as_ref(), upper.as_ref())) + == Some(true), + NormalizedEdgeFilter::PropertyExists { key } => edge.props.contains_key(key), + NormalizedEdgeFilter::PropertyMissing { key } => !edge.props.contains_key(key), + NormalizedEdgeFilter::WeightRange { lower, upper } => { + edge_weight_range_matches(edge.weight, *lower, *upper) + } + NormalizedEdgeFilter::UpdatedAtRange { lower_ms, upper_ms } => { + i64_range_matches(edge.updated_at, *lower_ms, *upper_ms) + } + NormalizedEdgeFilter::ValidAt { epoch_ms } => { + edge.valid_from <= *epoch_ms && *epoch_ms < edge.valid_to + } + NormalizedEdgeFilter::ValidFromRange { lower_ms, upper_ms } => { + i64_range_matches(edge.valid_from, *lower_ms, *upper_ms) + } + NormalizedEdgeFilter::ValidToRange { lower_ms, upper_ms } => { + i64_range_matches(edge.valid_to, *lower_ms, *upper_ms) + } + NormalizedEdgeFilter::And(children) => { + children.iter().all(|child| edge_filter_matches(child, edge)) + } + NormalizedEdgeFilter::Or(children) => { + children.iter().any(|child| edge_filter_matches(child, edge)) + } + NormalizedEdgeFilter::Not(child) => !edge_filter_matches(child, edge), + } +} + +fn collect_edge_filter_property_keys(filter: &NormalizedEdgeFilter, keys: &mut Vec) { + match filter { + NormalizedEdgeFilter::PropertyEquals { key, .. } + | NormalizedEdgeFilter::PropertyIn { key, .. } + | NormalizedEdgeFilter::PropertyRange { key, .. } + | NormalizedEdgeFilter::PropertyExists { key } + | NormalizedEdgeFilter::PropertyMissing { key } => { + if !keys.iter().any(|existing| existing == key) { + keys.push(key.clone()); + } + } + NormalizedEdgeFilter::And(children) | NormalizedEdgeFilter::Or(children) => { + for child in children { + collect_edge_filter_property_keys(child, keys); + } + } + NormalizedEdgeFilter::Not(child) => collect_edge_filter_property_keys(child, keys), + NormalizedEdgeFilter::AlwaysTrue + | NormalizedEdgeFilter::AlwaysFalse + | NormalizedEdgeFilter::WeightRange { .. } + | NormalizedEdgeFilter::UpdatedAtRange { .. } + | NormalizedEdgeFilter::ValidAt { .. } + | NormalizedEdgeFilter::ValidFromRange { .. } + | NormalizedEdgeFilter::ValidToRange { .. } => {} + } +} + +fn edge_filter_projected_matches( + filter: &NormalizedEdgeFilter, + meta: &EdgeMetadataForQuery, + props: &BTreeMap, +) -> bool { + match filter { + NormalizedEdgeFilter::AlwaysTrue => true, + NormalizedEdgeFilter::AlwaysFalse => false, + NormalizedEdgeFilter::PropertyEquals { key, value } => props + .get(key) + .is_some_and(|candidate| prop_values_equal_for_filter(candidate, value)), + NormalizedEdgeFilter::PropertyIn { + key, + values, + value_keys, + } => props + .get(key) + .is_some_and(|candidate| property_in_filter_matches(candidate, values, value_keys)), + NormalizedEdgeFilter::PropertyRange { key, lower, upper } => props + .get(key) + .and_then(|value| range_value_within_bounds(value, lower.as_ref(), upper.as_ref())) + == Some(true), + NormalizedEdgeFilter::PropertyExists { key } => props.contains_key(key), + NormalizedEdgeFilter::PropertyMissing { key } => !props.contains_key(key), + NormalizedEdgeFilter::WeightRange { lower, upper } => { + edge_weight_range_matches(meta.weight, *lower, *upper) + } + NormalizedEdgeFilter::UpdatedAtRange { lower_ms, upper_ms } => { + i64_range_matches(meta.updated_at, *lower_ms, *upper_ms) + } + NormalizedEdgeFilter::ValidAt { epoch_ms } => { + meta.valid_from <= *epoch_ms && *epoch_ms < meta.valid_to + } + NormalizedEdgeFilter::ValidFromRange { lower_ms, upper_ms } => { + i64_range_matches(meta.valid_from, *lower_ms, *upper_ms) + } + NormalizedEdgeFilter::ValidToRange { lower_ms, upper_ms } => { + i64_range_matches(meta.valid_to, *lower_ms, *upper_ms) + } + NormalizedEdgeFilter::And(children) => children + .iter() + .all(|child| edge_filter_projected_matches(child, meta, props)), + NormalizedEdgeFilter::Or(children) => children + .iter() + .any(|child| edge_filter_projected_matches(child, meta, props)), + NormalizedEdgeFilter::Not(child) => !edge_filter_projected_matches(child, meta, props), + } +} + +#[cfg(test)] +fn edge_query_metadata_matches( + query: &NormalizedEdgeQuery, + meta: &EdgeMetadataForQuery, +) -> bool { + if !edge_query_metadata_constraints_match(query, meta) { + return false; + } + + edge_filter_metadata_maybe_matches(&query.filter, meta) +} + +fn edge_query_metadata_constraints_match( + query: &NormalizedEdgeQuery, + meta: &EdgeMetadataForQuery, +) -> bool { + if query.label_id.is_some_and(|label_id| meta.label_id != label_id) { + return false; + } + if !query.ids.is_empty() && query.ids.binary_search(&meta.id).is_err() { + return false; + } + if !query.from_ids.is_empty() && query.from_ids.binary_search(&meta.from).is_err() { + return false; + } + if !query.to_ids.is_empty() && query.to_ids.binary_search(&meta.to).is_err() { + return false; + } + if !query.endpoint_ids.is_empty() + && query.endpoint_ids.binary_search(&meta.from).is_err() + && query.endpoint_ids.binary_search(&meta.to).is_err() + { + return false; + } + + true +} + +#[cfg(test)] +fn edge_query_matches(query: &NormalizedEdgeQuery, edge: &EdgeRecord) -> bool { + let meta = EdgeMetadataForQuery::from(edge); + if !edge_query_metadata_matches(query, &meta) { + return false; + } + edge_filter_matches(&query.filter, edge) +} + +#[derive(Clone)] +struct PatternExecutionState { + nodes: Vec>, + edges: Vec>, +} + +#[derive(Clone, Copy)] +struct VerifiedPatternEdgeAnchor { + meta: EdgeMetadataCandidate, +} + +enum PatternAnchorFrontier { + Ready { + states: Vec, + followups: Vec, + expansion_order: Vec, + sort_anchor_alias: String, + }, + TooBroad { + followups: Vec, + }, +} + +#[derive(Clone, Copy)] +struct PatternEdgeContext { + source_id: u64, + target_index: usize, + target_id: Option, + direction: Direction, +} + +struct PatternUnnamedAccumulator { + seen_edges: NodeIdSet, + best_by_target: NodeIdMap, + best_bound_entry: Option, + exceeded: bool, +} + +impl PatternUnnamedAccumulator { + fn new() -> Self { + Self { + seen_edges: NodeIdSet::default(), + best_by_target: NodeIdMap::default(), + best_bound_entry: None, + exceeded: false, + } + } + + fn is_done(&self) -> bool { + self.best_bound_entry.is_some() || self.exceeded + } +} + +#[derive(Clone)] +struct PatternPendingEntry { + state_index: usize, + entry: NeighborRecord, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct PatternMatchSortKey { + anchor_node_id: u64, + node_ids: Vec, + edge_ids: Vec, +} + +fn reverse_pattern_direction(direction: Direction) -> Direction { + match direction { + Direction::Outgoing => Direction::Incoming, + Direction::Incoming => Direction::Outgoing, + Direction::Both => Direction::Both, + } +} + +fn pattern_distinct_node_binding_ok( + state: &PatternExecutionState, + target_index: usize, + candidate_id: u64, +) -> bool { + state + .nodes + .iter() + .enumerate() + .all(|(index, bound)| index == target_index || *bound != Some(candidate_id)) +} + +fn pattern_match_from_state( + query: &NormalizedGraphPatternQuery, state: &PatternExecutionState, ) -> QueryMatch { let mut nodes = BTreeMap::new(); @@ -400,16 +1396,17 @@ fn insert_bounded_pattern_match( fn record_pattern_unnamed_entry( state: &PatternExecutionState, context: &PatternEdgeContext, + edge_filter: &NormalizedEdgeFilter, reference_time: i64, deleted_nodes: &NodeIdSet, deleted_edges: &NodeIdSet, seen_edges: &mut NodeIdSet, - best_by_target: &mut NodeIdMap, - best_bound_entry: &mut Option, + best_by_target: &mut NodeIdMap, + best_bound_entry: &mut Option, exceeded: &mut bool, edge_id: u64, neighbor_id: u64, - edge_type_id: u32, + edge_label_id: u32, weight: f32, valid_from: i64, valid_to: i64, @@ -423,14 +1420,22 @@ fn record_pattern_unnamed_entry( if !is_edge_valid_at(valid_from, valid_to, reference_time) { return ControlFlow::Continue(()); } + let posting_meta = EdgePostingMetadataForQuery { + weight, + valid_from, + valid_to, + }; + if !edge_filter_posting_metadata_maybe_matches(edge_filter, &posting_meta) { + return ControlFlow::Continue(()); + } if let Some(target_id) = context.target_id { if neighbor_id != target_id { return ControlFlow::Continue(()); } - *best_bound_entry = Some(NeighborEntry { + *best_bound_entry = Some(NeighborRecord { node_id: neighbor_id, edge_id, - edge_type_id, + edge_label_id, weight, valid_from, valid_to, @@ -440,10 +1445,10 @@ fn record_pattern_unnamed_entry( if !pattern_distinct_node_binding_ok(state, context.target_index, neighbor_id) { return ControlFlow::Continue(()); } - let entry = NeighborEntry { + let entry = NeighborRecord { node_id: neighbor_id, edge_id, - edge_type_id, + edge_label_id, weight, valid_from, valid_to, @@ -463,40 +1468,6 @@ fn record_pattern_unnamed_entry( } impl ReadView { - fn query_node_page_from_type_id_index( - &self, - query: &NormalizedNodeQuery, - type_id: u32, - ) -> Result { - let limit = page_limit(&query.page); - let target = page_verify_target(limit); - let mut ids = Vec::with_capacity(if limit > 0 { limit } else { 0 }); - - if query.ids.is_empty() { - self.scan_type_ids_unfiltered(type_id, query.page.after, |node_id| { - ids.push(node_id); - if ids.len() >= target { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - })?; - return Ok(finalize_verified_page(ids, Vec::new(), limit)); - } - - self.scan_type_ids_unfiltered(type_id, query.page.after, |node_id| { - if query.ids.binary_search(&node_id).is_ok() { - ids.push(node_id); - if ids.len() >= target { - return ControlFlow::Break(()); - } - } - ControlFlow::Continue(()) - })?; - - Ok(finalize_verified_page(ids, Vec::new(), limit)) - } - fn query_node_page_from_candidates( &self, candidate_ids: &[u64], @@ -504,6 +1475,15 @@ impl ReadView { hydrate: bool, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, ) -> Result { + if query.filter.is_always_true() && query.keys.is_empty() { + return self.query_node_page_from_metadata_candidates( + candidate_ids, + query, + hydrate, + policy_cutoffs, + ); + } + let limit = page_limit(&query.page); let target = page_verify_target(limit); let start = first_candidate_after(candidate_ids, query.page.after); @@ -537,41 +1517,182 @@ impl ReadView { Ok(finalize_verified_page(ids, nodes, limit)) } - fn query_node_page_from_type_scan( + fn query_node_page_from_metadata_candidates( &self, + candidate_ids: &[u64], query: &NormalizedNodeQuery, hydrate: bool, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, ) -> Result { - let type_id = query - .type_id - .expect("normalized type-scan query must have type_id"); let limit = page_limit(&query.page); let target = page_verify_target(limit); - let chunk_limit = match query.page.limit { - Some(limit) if limit > 0 => limit.saturating_add(1).saturating_mul(4).max(limit + 1), - _ => QUERY_VERIFY_CHUNK, - }; + let start = first_candidate_after(candidate_ids, query.page.after); let mut ids = Vec::with_capacity(if limit > 0 { limit } else { 0 }); - let mut nodes = Vec::with_capacity(if hydrate && limit > 0 { limit } else { 0 }); - - if !hydrate - && policy_cutoffs.is_none() - && query.filter.is_always_true() - && query.keys.is_empty() - { - return self.query_node_page_from_type_id_index(query, type_id); - } - self.scan_nodes_by_type_filtered( - type_id, - query.page.after, - chunk_limit, - policy_cutoffs, - |node_id, node| { - if query_node_matches(query, node) { - ids.push(node_id); - if hydrate { + for chunk in candidate_ids[start..].chunks(QUERY_VERIFY_CHUNK) { + #[cfg(test)] + self.note_node_visibility_meta_reads(chunk.len()); + let visibility = self.sources().find_node_visibility_meta(chunk)?; + for (&node_id, state) in chunk.iter().zip(visibility.iter()) { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + if !query_node_visibility_meta_matches(query, node_id, meta, policy_cutoffs) { + continue; + } + ids.push(node_id); + if ids.len() >= target { + let mut page = finalize_verified_page(ids, Vec::new(), limit); + if hydrate { + let nodes = self.get_nodes_raw(&page.ids)?; + page.nodes = nodes.into_iter().flatten().collect(); + } + return Ok(page); + } + } + } + + let mut page = finalize_verified_page(ids, Vec::new(), limit); + if hydrate { + let nodes = self.get_nodes_raw(&page.ids)?; + page.nodes = nodes.into_iter().flatten().collect(); + } + Ok(page) + } + + fn query_node_page_from_label_scan( + &self, + query: &NormalizedNodeQuery, + label_ids: &[u32], + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + let limit = page_limit(&query.page); + let target = page_verify_target(limit); + let chunk_limit = match query.page.limit { + Some(limit) if limit > 0 => limit.saturating_add(1).saturating_mul(4).max(limit + 1), + _ => QUERY_VERIFY_CHUNK, + }; + let metadata_only = query.filter.is_always_true() && query.keys.is_empty(); + let mut ids = Vec::with_capacity(if limit > 0 { limit } else { 0 }); + let mut nodes = Vec::with_capacity(if hydrate && limit > 0 { limit } else { 0 }); + + self.scan_raw_node_label_candidates( + label_ids, + query.page.after, + chunk_limit, + |chunk| { + if metadata_only { + #[cfg(test)] + self.note_node_visibility_meta_reads(chunk.len()); + let visibility = self.sources().find_node_visibility_meta(chunk)?; + for (&node_id, state) in chunk.iter().zip(visibility.iter()) { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + if !query_node_visibility_meta_matches( + query, + node_id, + meta, + policy_cutoffs, + ) { + continue; + } + ids.push(node_id); + if ids.len() >= target { + return Ok(ControlFlow::Break(())); + } + } + return Ok(ControlFlow::Continue(())); + } + + #[cfg(test)] + self.note_final_verifier_record_reads(chunk.len()); + let chunk_nodes = self.get_nodes_raw(chunk)?; + for (&node_id, node) in chunk.iter().zip(chunk_nodes.into_iter()) { + let Some(node) = node.as_ref() else { + continue; + }; + if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { + continue; + } + if !query_node_matches(query, node) { + continue; + } + ids.push(node_id); + if hydrate { + nodes.push(node.clone()); + } + if ids.len() >= target { + return Ok(ControlFlow::Break(())); + } + } + Ok(ControlFlow::Continue(())) + }, + )?; + + if metadata_only { + let mut page = finalize_verified_page(ids, Vec::new(), limit); + if hydrate { + let nodes = self.get_nodes_raw(&page.ids)?; + page.nodes = nodes.into_iter().flatten().collect(); + } + Ok(page) + } else { + Ok(finalize_verified_page(ids, nodes, limit)) + } + } + + fn query_node_page_from_single_label_scan( + &self, + query: &NormalizedNodeQuery, + single_label_id: u32, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + let limit = page_limit(&query.page); + let target = page_verify_target(limit); + let chunk_limit = match query.page.limit { + Some(limit) if limit > 0 => limit.saturating_add(1).saturating_mul(4).max(limit + 1), + _ => QUERY_VERIFY_CHUNK, + }; + let mut ids = Vec::with_capacity(if limit > 0 { limit } else { 0 }); + let mut nodes = Vec::with_capacity(if hydrate && limit > 0 { limit } else { 0 }); + + if !hydrate + && policy_cutoffs.is_none() + && query.filter.is_always_true() + && query.keys.is_empty() + && query.ids.is_empty() + && single_resolved_label_id(&query.label_filter) == Some(single_label_id) + { + let id_page = + self.nodes_by_single_label_id_paged_unfiltered(single_label_id, &query.page)?; + return Ok(VerifiedNodePage { + ids: id_page.items, + nodes: Vec::new(), + next_cursor: id_page.next_cursor, + }); + } + + if query.filter.is_always_true() && query.keys.is_empty() { + return self.query_node_page_from_label_scan( + query, + &[single_label_id], + hydrate, + policy_cutoffs, + ); + } + + self.scan_nodes_by_single_label_id_filtered( + single_label_id, + query.page.after, + chunk_limit, + policy_cutoffs, + |node_id, node| { + if query_node_matches(query, node) { + ids.push(node_id); + if hydrate { nodes.push(node.clone()); } if ids.len() >= target { @@ -783,12 +1904,12 @@ impl ReadView { } } NodeCandidateMaterialization::TimestampIndex { - type_id, + label_id, lower_ms, upper_ms, } => { let ids = self.timestamp_candidate_ids( - *type_id, + *label_id, *lower_ms, *upper_ms, eager_cap + 1, @@ -804,8 +1925,9 @@ impl ReadView { }) } } - NodeCandidateMaterialization::NodeTypeIndex - | NodeCandidateMaterialization::FallbackTypeScan + NodeCandidateMaterialization::NodeLabelAny { .. } + | NodeCandidateMaterialization::NodeLabelIndex { .. } + | NodeCandidateMaterialization::FallbackNodeLabelScan { .. } | NodeCandidateMaterialization::FallbackFullNodeScan => { Ok(CandidateMaterializationResult::TooBroad { followups: Vec::new(), @@ -901,10 +2023,31 @@ impl ReadView { policy_cutoffs: Option<&PrecomputedPruneCutoffs>, ) -> Result<(VerifiedNodePage, Vec), EngineError> { match source.kind { - NodeQueryCandidateSourceKind::NodeTypeIndex - | NodeQueryCandidateSourceKind::FallbackTypeScan => { + NodeQueryCandidateSourceKind::NodeLabelIndex + | NodeQueryCandidateSourceKind::FallbackNodeLabelScan => { + let label_id = match &source.materialization { + NodeCandidateMaterialization::NodeLabelIndex { label_id } + | NodeCandidateMaterialization::FallbackNodeLabelScan { label_id } => *label_id, + NodeCandidateMaterialization::NodeLabelAny { label_ids } => { + return Ok(( + self.query_node_page_from_label_scan( + query, + label_ids.as_slice(), + hydrate, + policy_cutoffs, + )?, + Vec::new(), + )); + } + _ => unreachable!("node label source must carry label materialization"), + }; Ok(( - self.query_node_page_from_type_scan(query, hydrate, policy_cutoffs)?, + self.query_node_page_from_single_label_scan( + query, + label_id, + hydrate, + policy_cutoffs, + )?, Vec::new(), )) } @@ -952,7 +2095,7 @@ impl ReadView { let mut plans = self.legal_universe_plans(query, true)?; if plans.is_empty() { return Err(EngineError::InvalidOperation( - "node query requires type_id, ids, keys, or allow_full_scan".into(), + "node query requires label_filter, ids, keys, or allow_full_scan".into(), )); } self.sort_physical_plans_by_selectivity(&mut plans); @@ -1048,108 +2191,1211 @@ impl ReadView { let policy_cutoffs = self.query_policy_cutoffs(); let (page, followups) = self.query_node_page_planned(&normalized, &planned, true, policy_cutoffs.as_ref())?; + let items = page + .nodes + .into_iter() + .map(|node| node_view_from_record(node, self.label_catalog.as_ref())) + .collect::, _>>()?; let value = QueryNodesResult { - items: page.nodes, + items, next_cursor: page.next_cursor, }; Ok(QueryExecutionOutcome { value, followups }) } - fn pattern_edge_context( - edge: &NormalizedEdgePattern, - state: &PatternExecutionState, - ) -> Result { - let from_bound = state.nodes[edge.from_index].is_some(); - let to_bound = state.nodes[edge.to_index].is_some(); - let (source_index, target_index, direction) = if from_bound { - (edge.from_index, edge.to_index, edge.direction) - } else if to_bound { - ( - edge.to_index, - edge.from_index, - reverse_pattern_direction(edge.direction), - ) + fn ready_edge_equality_candidate_ids_raw_limited( + &self, + index_id: u64, + value_hashes: &[u64], + raw_posting_cap: usize, + ) -> Result<(Option>, Option), EngineError> { + match self.sources().edge_ids_by_secondary_eq_hashes_limited_read( + index_id, + value_hashes, + raw_posting_cap, + ) { + Ok(crate::source_list::LimitedEdgeIndexRead::Ready(ids)) => Ok((Some(ids), None)), + Ok(crate::source_list::LimitedEdgeIndexRead::TooBroad) => Ok((None, None)), + Ok(crate::source_list::LimitedEdgeIndexRead::MissingSidecar) => { + Ok((None, self.equality_sidecar_failure_followup(index_id, None))) + } + Err(error) => Ok(( + None, + self.equality_sidecar_failure_followup(index_id, Some(error)), + )), + } + } + + fn materialize_edge_candidate_source( + &self, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + source: &PlannedEdgeCandidateSource, + ) -> Result { + let cap = cap_context.source_cap(source.kind, query.page.limit, source.estimate); + if !matches!( + source.materialization, + EdgeCandidateMaterialization::Precomputed(_) + | EdgeCandidateMaterialization::FallbackFullEdgeScan + ) && !edge_materialization_uses_limited_probe(&source.materialization) + && source + .estimate + .known_upper_bound() + .is_some_and(|count| count > cap as u64) + { + return Ok(CandidateMaterializationResult::TooBroad { + followups: Vec::new(), + }); + } + + if matches!( + source.materialization, + EdgeCandidateMaterialization::FallbackFullEdgeScan + ) { + return Ok(CandidateMaterializationResult::TooBroad { + followups: Vec::new(), + }); + } + + let sources = self.sources(); + let ids = match &source.materialization { + EdgeCandidateMaterialization::Precomputed(ids) => { + return Ok(CandidateMaterializationResult::Ready { + ids: ids.clone(), + followups: Vec::new(), + }); + } + EdgeCandidateMaterialization::EdgeLabelIndex { label_id } => sources.edge_ids_by_label_id(*label_id), + EdgeCandidateMaterialization::EdgeTripleIndex { from, to, label_id } => { + sources.edge_ids_by_triple(*from, *to, *label_id) + } + EdgeCandidateMaterialization::FromEndpointAdjacency { + node_ids, + label_filter_ids, + } => self.edge_ids_by_endpoint_sources( + node_ids, + Direction::Outgoing, + label_filter_ids.as_deref(), + cap.saturating_add(1), + ), + EdgeCandidateMaterialization::ToEndpointAdjacency { + node_ids, + label_filter_ids, + } => self.edge_ids_by_endpoint_sources( + node_ids, + Direction::Incoming, + label_filter_ids.as_deref(), + cap.saturating_add(1), + ), + EdgeCandidateMaterialization::AnyEndpointAdjacency { + node_ids, + label_filter_ids, + } => self.edge_ids_by_endpoint_sources( + node_ids, + Direction::Both, + label_filter_ids.as_deref(), + cap.saturating_add(1), + ), + EdgeCandidateMaterialization::EdgeWeightIndex { label_id, bounds } => { + sources.edge_ids_by_weight_range_limited( + *label_id, + *bounds, + cap.saturating_add(1), + ) + } + EdgeCandidateMaterialization::EdgeUpdatedAtIndex { label_id, bounds } => { + sources.edge_ids_by_updated_at_range_limited( + *label_id, + *bounds, + cap.saturating_add(1), + ) + } + EdgeCandidateMaterialization::EdgeValidFromIndex { label_id, bounds } => { + sources.edge_ids_by_valid_from_range_limited( + *label_id, + *bounds, + cap.saturating_add(1), + ) + } + EdgeCandidateMaterialization::EdgeValidToIndex { label_id, bounds } => { + sources.edge_ids_by_valid_to_range_limited( + *label_id, + *bounds, + cap.saturating_add(1), + ) + } + EdgeCandidateMaterialization::EdgePropertyEqualityIndex { + index_id, + label_id, + prop_key, + value, + value_hashes, + } => { + let _ = (label_id, prop_key, value); + let (ids, followup) = self.ready_edge_equality_candidate_ids_raw_limited( + *index_id, + value_hashes, + cap.saturating_add(1), + )?; + let followups = materialization_followups(followup); + let Some(ids) = ids else { + return Ok(CandidateMaterializationResult::TooBroad { followups }); + }; + if ids.len() > cap { + return Ok(CandidateMaterializationResult::TooBroad { followups }); + } + return Ok(CandidateMaterializationResult::Ready { ids, followups }); + } + EdgeCandidateMaterialization::EdgePropertyRangeIndex { + index_id, + label_id, + prop_key, + domain, + lower, + upper, + } => { + let _ = (label_id, prop_key); + let (ids, followup) = self.ready_edge_range_candidate_ids( + *index_id, + *domain, + lower.as_ref(), + upper.as_ref(), + cap.saturating_add(1), + )?; + let followups = materialization_followups(followup); + let Some(ids) = ids else { + return Ok(CandidateMaterializationResult::TooBroad { followups }); + }; + if ids.len() > cap { + return Ok(CandidateMaterializationResult::TooBroad { followups }); + } + return Ok(CandidateMaterializationResult::Ready { ids, followups }); + } + EdgeCandidateMaterialization::FallbackFullEdgeScan => unreachable!("handled above"), + }?; + if ids.len() > cap { + Ok(CandidateMaterializationResult::TooBroad { + followups: Vec::new(), + }) } else { - return Err(EngineError::InvalidOperation( - "pattern expansion encountered an unbound edge".into(), - )); - }; + Ok(CandidateMaterializationResult::Ready { + ids, + followups: Vec::new(), + }) + } + } - Ok(PatternEdgeContext { - source_id: state.nodes[source_index] - .expect("pattern expansion source alias must already be bound"), - target_index, - target_id: state.nodes[target_index], - direction, - }) + fn edge_ids_by_endpoint_sources( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + limit: usize, + ) -> Result, EngineError> { + self.sources() + .edge_ids_by_endpoints_limited(node_ids, direction, label_filter_ids, limit) } - fn pattern_verified_target_ids_by_index( + fn materialize_edge_physical_plan( &self, - query: &NormalizedGraphPatternQuery, - mut candidate_ids_by_index: Vec>, - policy_cutoffs: Option<&PrecomputedPruneCutoffs>, - ) -> Result, EngineError> { - let mut verified_by_index = vec![NodeIdSet::default(); query.nodes.len()]; - for (target_index, candidate_ids) in candidate_ids_by_index.iter_mut().enumerate() { - if candidate_ids.is_empty() { - continue; + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + plan: &EdgePhysicalPlan, + ) -> Result { + match plan { + EdgePhysicalPlan::Empty => Ok(CandidateMaterializationResult::Ready { + ids: Vec::new(), + followups: Vec::new(), + }), + EdgePhysicalPlan::Source(source) => { + self.materialize_edge_candidate_source(query, cap_context, source) } - candidate_ids.sort_unstable(); - candidate_ids.dedup(); - let nodes = self.get_nodes_raw(candidate_ids)?; - for (&node_id, node) in candidate_ids.iter().zip(nodes.iter()) { - if let Some(node) = node.as_ref() { - if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { + EdgePhysicalPlan::Intersect(inputs) => { + let mut sets = Vec::with_capacity(inputs.len()); + let mut followups = Vec::new(); + for input in inputs { + if !sets.is_empty() + && edge_plan_is_filter_source(input) + && sets + .iter() + .map(Vec::len) + .min() + .is_some_and(|len| len <= EDGE_INTERSECTION_TINY_SET) + { continue; } - if query_node_matches(&query.nodes[target_index].query, node) { - verified_by_index[target_index].insert(node_id); + match self.materialize_edge_physical_plan(query, cap_context, input)? { + CandidateMaterializationResult::Ready { + ids, + followups: mut input_followups, + } => { + followups.append(&mut input_followups); + if ids.is_empty() { + return Ok(CandidateMaterializationResult::Ready { + ids: Vec::new(), + followups, + }); + } + sets.push(ids); + } + CandidateMaterializationResult::TooBroad { + followups: mut input_followups, + } => { + followups.append(&mut input_followups); + if !sets.is_empty() { + continue; + } + } } } + if sets.is_empty() { + Ok(CandidateMaterializationResult::TooBroad { followups }) + } else { + Ok(CandidateMaterializationResult::Ready { + ids: intersect_candidate_sets(&sets), + followups, + }) + } } - } - Ok(verified_by_index) - } - - #[allow(clippy::too_many_arguments)] - fn flush_pattern_pending_entries( - &self, - query: &NormalizedGraphPatternQuery, - edge_index: usize, - edge: &NormalizedEdgePattern, - states: &[PatternExecutionState], - contexts: &[PatternEdgeContext], - pending: &mut Vec, - policy_cutoffs: Option<&PrecomputedPruneCutoffs>, - unnamed_emitted_targets: &mut [NodeIdSet], - unnamed_bound_satisfied: &mut [bool], - on_next: &mut impl FnMut(PatternExecutionState) -> Result, EngineError>, - ) -> Result, EngineError> { - if pending.is_empty() { - return Ok(ControlFlow::Continue(())); - } + EdgePhysicalPlan::Union(inputs) => { + let mut sets = Vec::with_capacity(inputs.len()); + let mut followups = Vec::new(); + let mut total_len = 0usize; + let cap = cap_context.union_total_cap(query.page.limit, plan.estimate()); + for input in inputs { + match self.materialize_edge_physical_plan(query, cap_context, input)? { + CandidateMaterializationResult::Ready { + ids, + followups: mut input_followups, + } => { + total_len = total_len.saturating_add(ids.len()); + followups.append(&mut input_followups); + if total_len > cap { + return Ok(CandidateMaterializationResult::TooBroad { followups }); + } + sets.push(ids); + } + CandidateMaterializationResult::TooBroad { + followups: mut input_followups, + } => { + followups.append(&mut input_followups); + return Ok(CandidateMaterializationResult::TooBroad { followups }); + } + } + } + let ids = union_candidate_sets(&sets); + if ids.len() > cap { + Ok(CandidateMaterializationResult::TooBroad { followups }) + } else { + Ok(CandidateMaterializationResult::Ready { ids, followups }) + } + } + } + } + + fn full_scan_edge_sources( + &self, + start_after: Option, + ) -> Result>, EngineError> { + let mut sources = Vec::with_capacity(1 + self.immutable_epochs.len() + self.segments.len()); + sources.push(FullScanEdgeSource::memtable( + &self.memtable, + self.snapshot_seq, + start_after, + )); + for epoch in &self.immutable_epochs { + sources.push(FullScanEdgeSource::memtable( + &epoch.memtable, + self.snapshot_seq, + start_after, + )); + } + for segment in &self.segments { + sources.push(FullScanEdgeSource::segment(segment.as_ref(), start_after)?); + } + Ok(sources) + } + + fn label_edge_sources( + &self, + label_id: u32, + start_after: Option, + ) -> Result>, EngineError> { + let mut sources = Vec::with_capacity(1 + self.immutable_epochs.len() + self.segments.len()); + sources.push(LabelEdgeSource::memtable( + &self.memtable, + self.snapshot_seq, + label_id, + start_after, + )); + for epoch in &self.immutable_epochs { + sources.push(LabelEdgeSource::memtable( + &epoch.memtable, + self.snapshot_seq, + label_id, + start_after, + )); + } + for segment in &self.segments { + if let Some(posting) = segment.edge_label_posting(label_id)? { + sources.push(LabelEdgeSource::segment( + segment.as_ref(), + posting, + start_after, + )?); + } + } + Ok(sources) + } + + fn endpoint_edge_sources<'a>( + &'a self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&'a [u32]>, + start_after: Option, + ) -> Result>, EngineError> { + if node_ids.is_empty() { + return Ok(Vec::new()); + } + let mut sorted_node_ids = node_ids.to_vec(); + sorted_node_ids.sort_unstable(); + sorted_node_ids.dedup(); + + let mut sources = Vec::with_capacity(1 + self.immutable_epochs.len() + self.segments.len()); + sources.push(EndpointEdgeSource::memtable( + &self.memtable, + &sorted_node_ids, + direction, + label_filter_ids, + self.snapshot_seq, + start_after, + )); + for epoch in &self.immutable_epochs { + sources.push(EndpointEdgeSource::memtable( + &epoch.memtable, + &sorted_node_ids, + direction, + label_filter_ids, + self.snapshot_seq, + start_after, + )); + } + for segment in &self.segments { + sources.push(EndpointEdgeSource::segment( + segment.as_ref(), + &sorted_node_ids, + direction, + label_filter_ids, + start_after, + )?); + } + Ok(sources) + } + + fn scan_full_edge_id_chunks( + &self, + start_after: Option, + chunk_limit: usize, + mut visitor: F, + ) -> Result<(), EngineError> + where + F: FnMut(&[u64]) -> Result, EngineError>, + { + let mut sources = self.full_scan_edge_sources(start_after)?; + let mut heap = BinaryHeap::new(); + for (source_index, source) in sources.iter_mut().enumerate() { + if let Some(edge_id) = source.next_id()? { + heap.push(Reverse((edge_id, source_index))); + } + } + + let chunk_limit = chunk_limit.max(1); + let mut chunk = Vec::with_capacity(chunk_limit); + let mut last_seen = None; + while let Some(Reverse((edge_id, source_index))) = heap.pop() { + if let Some(next_id) = sources[source_index].next_id()? { + heap.push(Reverse((next_id, source_index))); + } + + if last_seen == Some(edge_id) { + continue; + } + last_seen = Some(edge_id); + chunk.push(edge_id); + if chunk.len() >= chunk_limit { + if visitor(&chunk)?.is_break() { + return Ok(()); + } + chunk.clear(); + } + } + + if !chunk.is_empty() { + let _ = visitor(&chunk)?; + } + Ok(()) + } + + fn scan_label_edge_id_chunks( + &self, + label_id: u32, + start_after: Option, + chunk_limit: usize, + mut visitor: F, + ) -> Result<(), EngineError> + where + F: FnMut(&[u64]) -> Result, EngineError>, + { + let mut sources = self.label_edge_sources(label_id, start_after)?; + let mut heap = BinaryHeap::new(); + for (source_index, source) in sources.iter_mut().enumerate() { + if let Some(edge_id) = source.next_id()? { + heap.push(Reverse((edge_id, source_index))); + } + } + + let chunk_limit = chunk_limit.max(1); + let mut chunk = Vec::with_capacity(chunk_limit); + let mut last_seen = None; + while let Some(Reverse((edge_id, source_index))) = heap.pop() { + if let Some(next_id) = sources[source_index].next_id()? { + heap.push(Reverse((next_id, source_index))); + } + + if last_seen == Some(edge_id) { + continue; + } + last_seen = Some(edge_id); + chunk.push(edge_id); + if chunk.len() >= chunk_limit { + if visitor(&chunk)?.is_break() { + return Ok(()); + } + chunk.clear(); + } + } + + if !chunk.is_empty() { + let _ = visitor(&chunk)?; + } + Ok(()) + } + + fn scan_endpoint_edge_id_chunks( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + start_after: Option, + chunk_limit: usize, + mut visitor: F, + ) -> Result<(), EngineError> + where + F: FnMut(&[u64]) -> Result, EngineError>, + { + let mut sources = + self.endpoint_edge_sources(node_ids, direction, label_filter_ids, start_after)?; + let mut heap = BinaryHeap::new(); + for (source_index, source) in sources.iter_mut().enumerate() { + if let Some(edge_id) = source.next_id()? { + heap.push(Reverse((edge_id, source_index))); + } + } + + let chunk_limit = chunk_limit.max(1); + let mut chunk = Vec::with_capacity(chunk_limit); + let mut last_seen = None; + while let Some(Reverse((edge_id, source_index))) = heap.pop() { + if let Some(next_id) = sources[source_index].next_id()? { + heap.push(Reverse((next_id, source_index))); + } + + if last_seen == Some(edge_id) { + continue; + } + last_seen = Some(edge_id); + #[cfg(test)] + self.note_endpoint_adjacency_candidates(1); + chunk.push(edge_id); + if chunk.len() >= chunk_limit { + if visitor(&chunk)?.is_break() { + return Ok(()); + } + chunk.clear(); + } + } + + if !chunk.is_empty() { + let _ = visitor(&chunk)?; + } + Ok(()) + } + + fn populate_verified_edge_records( + &self, + page: &mut VerifiedEdgePage, + hydrated_records: &mut NodeIdMap, + ) -> Result<(), EngineError> { + if page.ids.is_empty() { + return Ok(()); + } + + let mut slots = vec![None; page.ids.len()]; + let mut missing_positions = Vec::new(); + let mut missing_ids = Vec::new(); + for (index, &edge_id) in page.ids.iter().enumerate() { + if let Some(edge) = hydrated_records.remove(&edge_id) { + slots[index] = Some(edge); + } else { + missing_positions.push(index); + missing_ids.push(edge_id); + } + } + + if !missing_ids.is_empty() { + let records = self.get_edges(&missing_ids)?; + for (index, record) in missing_positions.into_iter().zip(records.into_iter()) { + slots[index] = record; + } + } + + page.edges = slots.into_iter().flatten().collect(); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn verify_edge_candidate_chunk( + &self, + chunk: &[u64], + query: &NormalizedEdgeQuery, + _hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + endpoint_cache: &mut EdgeEndpointVisibilityCache, + ids: &mut Vec, + _hydrated_records: &mut NodeIdMap, + target: usize, + ) -> Result, EngineError> { + let metadata = self.sources().find_edge_metadata(chunk)?; + let metas = metadata.iter().filter_map(|meta| *meta).collect::>(); + { + let sources = self.sources(); + endpoint_cache.ensure_edge_endpoints(&sources, &metas, policy_cutoffs)?; + } + + let mut decisions = Vec::new(); + let mut property_candidate_ids = Vec::new(); + let mut property_keys = Vec::new(); + collect_edge_filter_property_keys(&query.filter, &mut property_keys); + + for meta in metas { + if !endpoint_cache.edge_endpoints_visible(meta) { + continue; + } + let query_meta = EdgeMetadataForQuery::from(meta); + if !edge_query_metadata_constraints_match(query, &query_meta) { + continue; + } + match edge_filter_metadata_outcome(&query.filter, &query_meta) { + Some(false) => continue, + Some(true) => { + decisions.push((meta.edge_id, query_meta, false)); + } + None => { + decisions.push((meta.edge_id, query_meta, true)); + property_candidate_ids.push(meta.edge_id); + } + } + } + + let mut property_matches = NodeIdSet::default(); + if !property_candidate_ids.is_empty() { + let mut metadata_by_property_candidate = NodeIdMap::with_capacity_and_hasher( + property_candidate_ids.len(), + Default::default(), + ); + for (edge_id, query_meta, needs_properties) in &decisions { + if *needs_properties { + metadata_by_property_candidate.insert(*edge_id, *query_meta); + } + } + let projected = self + .sources() + .find_edge_properties(&property_candidate_ids, &property_keys)?; + for (&edge_id, props) in property_candidate_ids.iter().zip(projected.iter()) { + let Some(props) = props else { + continue; + }; + let Some(query_meta) = metadata_by_property_candidate.get(&edge_id) else { + continue; + }; + if edge_filter_projected_matches(&query.filter, query_meta, props) { + property_matches.insert(edge_id); + } + } + } + + for (edge_id, _, needs_properties) in decisions { + if needs_properties && !property_matches.contains(&edge_id) { + continue; + } + ids.push(edge_id); + if ids.len() >= target { + return Ok(ControlFlow::Break(())); + } + } + + Ok(ControlFlow::Continue(())) + } + + fn query_edge_page_from_candidates( + &self, + candidate_ids: &[u64], + query: &NormalizedEdgeQuery, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + let limit = page_limit(&query.page); + let target = page_verify_target(limit); + let mut ids = Vec::new(); + let mut hydrated_records = NodeIdMap::default(); + let mut endpoint_cache = EdgeEndpointVisibilityCache::default(); + let start = first_candidate_after(candidate_ids, query.page.after); + let mut cursor = start; + + while cursor < candidate_ids.len() && ids.len() < target { + let end = (cursor + QUERY_VERIFY_CHUNK).min(candidate_ids.len()); + let chunk = &candidate_ids[cursor..end]; + if self + .verify_edge_candidate_chunk( + chunk, + query, + hydrate, + policy_cutoffs, + &mut endpoint_cache, + &mut ids, + &mut hydrated_records, + target, + )? + .is_break() + { + break; + } + + cursor = end; + } + + let mut page = finalize_verified_edge_page(ids, Vec::new(), limit); + if hydrate { + self.populate_verified_edge_records(&mut page, &mut hydrated_records)?; + } + Ok(page) + } + + fn query_edge_page_from_label_scan( + &self, + label_id: u32, + query: &NormalizedEdgeQuery, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + let limit = page_limit(&query.page); + let target = page_verify_target(limit); + let chunk_limit = match query.page.limit { + Some(limit) if limit > 0 => limit.saturating_add(1).saturating_mul(4).max(limit + 1), + _ => QUERY_VERIFY_CHUNK, + }; + let mut ids = Vec::new(); + let mut hydrated_records = NodeIdMap::default(); + let mut endpoint_cache = EdgeEndpointVisibilityCache::default(); + + self.scan_label_edge_id_chunks(label_id, query.page.after, chunk_limit, |chunk| { + self.verify_edge_candidate_chunk( + chunk, + query, + hydrate, + policy_cutoffs, + &mut endpoint_cache, + &mut ids, + &mut hydrated_records, + target, + ) + })?; + + let mut page = finalize_verified_edge_page(ids, Vec::new(), limit); + if hydrate { + self.populate_verified_edge_records(&mut page, &mut hydrated_records)?; + } + Ok(page) + } + + fn query_edge_page_from_endpoint_scan( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + query: &NormalizedEdgeQuery, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + let limit = page_limit(&query.page); + let target = page_verify_target(limit); + let chunk_limit = match query.page.limit { + Some(limit) if limit > 0 => limit.saturating_add(1).saturating_mul(4).max(limit + 1), + _ => QUERY_VERIFY_CHUNK, + }; + let mut ids = Vec::new(); + let mut hydrated_records = NodeIdMap::default(); + let mut endpoint_cache = EdgeEndpointVisibilityCache::default(); + + self.scan_endpoint_edge_id_chunks( + node_ids, + direction, + label_filter_ids, + query.page.after, + chunk_limit, + |chunk| { + self.verify_edge_candidate_chunk( + chunk, + query, + hydrate, + policy_cutoffs, + &mut endpoint_cache, + &mut ids, + &mut hydrated_records, + target, + ) + }, + )?; + + let mut page = finalize_verified_edge_page(ids, Vec::new(), limit); + if hydrate { + self.populate_verified_edge_records(&mut page, &mut hydrated_records)?; + } + Ok(page) + } + + fn query_edge_page_from_full_scan( + &self, + query: &NormalizedEdgeQuery, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + #[cfg(test)] + self.note_edge_full_scan_page(); + + let limit = page_limit(&query.page); + let target = page_verify_target(limit); + let chunk_limit = match query.page.limit { + Some(limit) if limit > 0 => limit.saturating_add(1).saturating_mul(4).max(limit + 1), + _ => QUERY_VERIFY_CHUNK, + }; + let mut ids = Vec::new(); + let mut hydrated_records = NodeIdMap::default(); + let mut endpoint_cache = EdgeEndpointVisibilityCache::default(); + + self.scan_full_edge_id_chunks(query.page.after, chunk_limit, |chunk| { + self.verify_edge_candidate_chunk( + chunk, + query, + hydrate, + policy_cutoffs, + &mut endpoint_cache, + &mut ids, + &mut hydrated_records, + target, + ) + })?; + + let mut page = finalize_verified_edge_page(ids, Vec::new(), limit); + if hydrate { + self.populate_verified_edge_records(&mut page, &mut hydrated_records)?; + } + Ok(page) + } + + fn query_edge_page_from_source_driver( + &self, + source: &PlannedEdgeCandidateSource, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result<(VerifiedEdgePage, Vec), EngineError> { + match &source.materialization { + EdgeCandidateMaterialization::EdgeLabelIndex { label_id } => { + Ok(( + self.query_edge_page_from_label_scan(*label_id, query, hydrate, policy_cutoffs)?, + Vec::new(), + )) + } + EdgeCandidateMaterialization::FromEndpointAdjacency { + node_ids, + label_filter_ids, + } => Ok(( + self.query_edge_page_from_endpoint_scan( + node_ids, + Direction::Outgoing, + label_filter_ids.as_deref(), + query, + hydrate, + policy_cutoffs, + )?, + Vec::new(), + )), + EdgeCandidateMaterialization::ToEndpointAdjacency { + node_ids, + label_filter_ids, + } => Ok(( + self.query_edge_page_from_endpoint_scan( + node_ids, + Direction::Incoming, + label_filter_ids.as_deref(), + query, + hydrate, + policy_cutoffs, + )?, + Vec::new(), + )), + EdgeCandidateMaterialization::AnyEndpointAdjacency { + node_ids, + label_filter_ids, + } => Ok(( + self.query_edge_page_from_endpoint_scan( + node_ids, + Direction::Both, + label_filter_ids.as_deref(), + query, + hydrate, + policy_cutoffs, + )?, + Vec::new(), + )), + EdgeCandidateMaterialization::FallbackFullEdgeScan => { + Ok(( + self.query_edge_page_from_full_scan(query, hydrate, policy_cutoffs)?, + Vec::new(), + )) + } + _ => match self.materialize_edge_candidate_source(query, cap_context, source)? { + CandidateMaterializationResult::Ready { ids, followups } => Ok(( + self.query_edge_page_from_candidates(&ids, query, hydrate, policy_cutoffs)?, + followups, + )), + CandidateMaterializationResult::TooBroad { + followups: mut materialization_followups, + } => { + let (page, mut fallback_followups) = self.query_edge_page_from_legal_universe( + query, + cap_context, + hydrate, + policy_cutoffs, + )?; + materialization_followups.append(&mut fallback_followups); + Ok((page, materialization_followups)) + } + }, + } + } + + fn query_edge_page_from_legal_universe( + &self, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result<(VerifiedEdgePage, Vec), EngineError> { + let mut sources = self.edge_legal_universe_sources(query); + if sources.is_empty() { + return Err(EngineError::InvalidOperation( + "edge query requires label, ids, from_ids, to_ids, endpoint_ids, or allow_full_scan" + .into(), + )); + } + sources.sort_by_key(|source| EdgePhysicalPlan::source(source.clone()).plan_cost()); + let source = sources + .first() + .expect("legal edge universe sources must be non-empty"); + self.query_edge_page_from_source_driver( + source, + query, + cap_context, + hydrate, + policy_cutoffs, + ) + } + + fn query_edge_page_planned( + &self, + query: &NormalizedEdgeQuery, + planned: PlannedEdgeQuery, + hydrate: bool, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result<(VerifiedEdgePage, Vec), EngineError> { + let PlannedEdgeQuery { + driver, + cap_context, + warnings: _, + mut followups, + } = planned; + + if let EdgePhysicalPlan::Source(source) = &driver { + let (page, mut source_followups) = self.query_edge_page_from_source_driver( + source, + query, + cap_context, + hydrate, + policy_cutoffs, + )?; + followups.append(&mut source_followups); + return Ok((page, followups)); + } + + match self.materialize_edge_physical_plan(query, cap_context, &driver)? { + CandidateMaterializationResult::Ready { + ids, + followups: mut materialization_followups, + } => { + let page = self.query_edge_page_from_candidates( + &ids, + query, + hydrate, + policy_cutoffs, + )?; + followups.append(&mut materialization_followups); + Ok((page, followups)) + } + CandidateMaterializationResult::TooBroad { + followups: mut materialization_followups, + } => { + let (page, mut fallback_followups) = self.query_edge_page_from_legal_universe( + query, + cap_context, + hydrate, + policy_cutoffs, + )?; + followups.append(&mut materialization_followups); + followups.append(&mut fallback_followups); + Ok((page, followups)) + } + } + } + + fn query_edge_ids_outcome( + &self, + query: &EdgeQuery, + ) -> Result, EngineError> { + let normalized = self.normalize_edge_query(query)?; + let planned = self.plan_normalized_edge_query(&normalized)?; + let policy_cutoffs = self.query_policy_cutoffs(); + let (page, followups) = + self.query_edge_page_planned(&normalized, planned, false, policy_cutoffs.as_ref())?; + Ok(QueryExecutionOutcome { + value: QueryEdgeIdsResult { + edge_ids: page.ids, + next_cursor: page.next_cursor, + }, + followups, + }) + } + + fn query_edges_outcome( + &self, + query: &EdgeQuery, + ) -> Result, EngineError> { + let normalized = self.normalize_edge_query(query)?; + let planned = self.plan_normalized_edge_query(&normalized)?; + let policy_cutoffs = self.query_policy_cutoffs(); + let (page, followups) = + self.query_edge_page_planned(&normalized, planned, true, policy_cutoffs.as_ref())?; + let edges = page + .edges + .into_iter() + .map(|edge| edge_view_from_record(edge, self.label_catalog.as_ref())) + .collect::, _>>()?; + Ok(QueryExecutionOutcome { + value: QueryEdgesResult { + edges, + next_cursor: page.next_cursor, + }, + followups, + }) + } + + fn pattern_edge_context( + edge: &NormalizedEdgePattern, + state: &PatternExecutionState, + ) -> Result { + let from_bound = state.nodes[edge.from_index].is_some(); + let to_bound = state.nodes[edge.to_index].is_some(); + let (source_index, target_index, direction) = if from_bound { + (edge.from_index, edge.to_index, edge.direction) + } else if to_bound { + ( + edge.to_index, + edge.from_index, + reverse_pattern_direction(edge.direction), + ) + } else { + return Err(EngineError::InvalidOperation( + "pattern expansion encountered an unbound edge".into(), + )); + }; + + Ok(PatternEdgeContext { + source_id: state.nodes[source_index] + .expect("pattern expansion source alias must already be bound"), + target_index, + target_id: state.nodes[target_index], + direction, + }) + } + + fn pattern_verified_target_ids_by_index( + &self, + query: &NormalizedGraphPatternQuery, + mut candidate_ids_by_index: Vec>, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result, EngineError> { + let mut verified_by_index = vec![NodeIdSet::default(); query.nodes.len()]; + for (target_index, candidate_ids) in candidate_ids_by_index.iter_mut().enumerate() { + if candidate_ids.is_empty() { + continue; + } + candidate_ids.sort_unstable(); + candidate_ids.dedup(); + let target_query = &query.nodes[target_index].query; + if target_query.keys.is_empty() + && node_filter_visibility_meta_compatible(&target_query.filter) + { + #[cfg(test)] + self.note_node_visibility_meta_reads(candidate_ids.len()); + let visibility = self.sources().find_node_visibility_meta(candidate_ids)?; + for (&node_id, state) in candidate_ids.iter().zip(visibility.iter()) { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + if query_node_visibility_meta_matches( + target_query, + node_id, + meta, + policy_cutoffs, + ) { + verified_by_index[target_index].insert(node_id); + } + } + } else { + let nodes = self.get_nodes_raw(candidate_ids)?; + for (&node_id, node) in candidate_ids.iter().zip(nodes.iter()) { + let Some(node) = node.as_ref() else { + continue; + }; + if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { + continue; + } + if query_node_matches(target_query, node) { + verified_by_index[target_index].insert(node_id); + } + } + } + } + Ok(verified_by_index) + } + + #[allow(clippy::too_many_arguments)] + fn flush_pattern_pending_entries( + &self, + query: &NormalizedGraphPatternQuery, + edge_index: usize, + edge: &NormalizedEdgePattern, + states: &[PatternExecutionState], + contexts: &[PatternEdgeContext], + pending: &mut Vec, + edge_filter_match_cache: &mut NodeIdMap, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + unnamed_emitted_targets: &mut [NodeIdSet], + unnamed_bound_satisfied: &mut [bool], + on_next: &mut impl FnMut(PatternExecutionState) -> Result, EngineError>, + ) -> Result, EngineError> { + if pending.is_empty() { + return Ok(ControlFlow::Continue(())); + } + + let mode = pattern_edge_filter_mode(&edge.filter); + let matching_edges = match mode { + PatternEdgeFilterMode::AlwaysTrue | PatternEdgeFilterMode::PostingMetadataOnly => None, + PatternEdgeFilterMode::BatchMetadataNeeded + | PatternEdgeFilterMode::PropertyVerifier => { + let mut edge_ids: Vec = pending + .iter() + .map(|pending| pending.entry.edge_id) + .collect(); + edge_ids.sort_unstable(); + edge_ids.dedup(); + let mut matching_edges = NodeIdSet::default(); + let mut unresolved_edge_ids = Vec::new(); + for edge_id in edge_ids { + match edge_filter_match_cache.get(&edge_id).copied() { + Some(true) => { + matching_edges.insert(edge_id); + } + Some(false) => {} + None => unresolved_edge_ids.push(edge_id), + } + } - let matching_edges = if edge.property_predicates.is_empty() { - None - } else { - let mut edge_ids: Vec = pending - .iter() - .map(|pending| pending.entry.edge_id) - .collect(); - edge_ids.sort_unstable(); - edge_ids.dedup(); - let edges = self.get_edges(&edge_ids)?; - let mut matching_edges = NodeIdSet::default(); - for (&edge_id, edge_record) in edge_ids.iter().zip(edges.iter()) { - if edge_record - .as_ref() - .is_some_and(|edge_record| edge_post_filter_matches(&edge.property_predicates, edge_record)) - { - matching_edges.insert(edge_id); + let metadata = self.sources().find_edge_metadata(&unresolved_edge_ids)?; + let requires_properties = mode == PatternEdgeFilterMode::PropertyVerifier; + let mut property_candidate_ids = Vec::new(); + let mut metadata_by_property_candidate = + NodeIdMap::with_capacity_and_hasher(unresolved_edge_ids.len(), Default::default()); + for (&edge_id, meta) in unresolved_edge_ids.iter().zip(metadata.iter()) { + let Some(meta) = meta else { + edge_filter_match_cache.insert(edge_id, false); + continue; + }; + let query_meta = EdgeMetadataForQuery::from(*meta); + match edge_filter_metadata_outcome(&edge.filter, &query_meta) { + Some(false) => { + edge_filter_match_cache.insert(edge_id, false); + } + Some(true) => { + edge_filter_match_cache.insert(edge_id, true); + matching_edges.insert(edge_id); + } + None if requires_properties => { + property_candidate_ids.push(edge_id); + metadata_by_property_candidate.insert(edge_id, query_meta); + } + None => { + edge_filter_match_cache.insert(edge_id, true); + matching_edges.insert(edge_id); + } + } + } + + if requires_properties && !property_candidate_ids.is_empty() { + let mut property_keys = Vec::new(); + collect_edge_filter_property_keys(&edge.filter, &mut property_keys); + let projected = self + .sources() + .find_edge_properties(&property_candidate_ids, &property_keys)?; + for (&edge_id, props) in property_candidate_ids.iter().zip(projected.iter()) { + let matched = props.as_ref().is_some_and(|props| { + metadata_by_property_candidate + .get(&edge_id) + .is_some_and(|query_meta| { + edge_filter_projected_matches(&edge.filter, query_meta, props) + }) + }); + edge_filter_match_cache.insert(edge_id, matched); + if matched { + matching_edges.insert(edge_id); + } + } } + Some(matching_edges) } - Some(matching_edges) }; let mut candidate_ids_by_index = vec![Vec::new(); query.nodes.len()]; @@ -1223,6 +3469,7 @@ impl ReadView { contexts: &[PatternEdgeContext], seen_edges: &mut [NodeIdSet], pending: &mut Vec, + edge_filter_match_cache: &mut NodeIdMap, unnamed_emitted_targets: &mut [NodeIdSet], unnamed_bound_satisfied: &mut [bool], policy_cutoffs: Option<&PrecomputedPruneCutoffs>, @@ -1230,7 +3477,7 @@ impl ReadView { state_index: usize, edge_id: u64, neighbor_id: u64, - edge_type_id: u32, + edge_label_id: u32, weight: f32, valid_from: i64, valid_to: i64, @@ -1250,6 +3497,14 @@ impl ReadView { if !is_edge_valid_at(valid_from, valid_to, reference_time) { return Ok(ControlFlow::Continue(())); } + let posting_meta = EdgePostingMetadataForQuery { + weight, + valid_from, + valid_to, + }; + if !edge_filter_posting_metadata_maybe_matches(&edge.filter, &posting_meta) { + return Ok(ControlFlow::Continue(())); + } let state = &states[state_index]; let context = contexts[state_index]; @@ -1261,20 +3516,21 @@ impl ReadView { return Ok(ControlFlow::Continue(())); } + #[cfg(test)] + self.note_pattern_edge_pending_entry(); pending.push(PatternPendingEntry { state_index, - entry: NeighborEntry { + entry: NeighborRecord { node_id: neighbor_id, edge_id, - edge_type_id, + edge_label_id, weight, valid_from, valid_to, }, }); - let flush_now = pending.len() >= QUERY_VERIFY_CHUNK - || (edge.alias.is_none() && context.target_id.is_some()); + let flush_now = pending.len() >= QUERY_VERIFY_CHUNK; if flush_now { return self.flush_pattern_pending_entries( query, @@ -1283,6 +3539,7 @@ impl ReadView { states, contexts, pending, + edge_filter_match_cache, policy_cutoffs, unnamed_emitted_targets, unnamed_bound_satisfied, @@ -1304,7 +3561,7 @@ impl ReadView { ) -> Result<(), EngineError> { let edge = &query.edges[edge_index]; let (deleted_nodes, deleted_edges) = self.collect_tombstones(); - let type_filter = edge.type_filter.as_deref(); + let label_filter_ids = edge.label_filter_ids.as_deref(); let mut contexts = Vec::with_capacity(states.len()); for state in &states { @@ -1322,12 +3579,13 @@ impl ReadView { let flow = self.memtable.for_each_adj_entry_at( context.source_id, context.direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { record_pattern_unnamed_entry( state, &context, + &edge.filter, reference_time, &deleted_nodes, &deleted_edges, @@ -1364,12 +3622,13 @@ impl ReadView { let flow = epoch.memtable.for_each_adj_entry_at( context.source_id, context.direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { record_pattern_unnamed_entry( state, &context, + &edge.filter, reference_time, &deleted_nodes, &deleted_edges, @@ -1424,7 +3683,7 @@ impl ReadView { let flow = segment.for_each_adj_posting_batch( &source_ids, direction, - type_filter, + label_filter_ids, &mut |queried_node_id, edge_id, neighbor_id, weight, valid_from, valid_to| { let Some(state_indices) = context_indices_by_source.get(&queried_node_id) else { @@ -1440,6 +3699,7 @@ impl ReadView { let flow = record_pattern_unnamed_entry( &states[state_index], &context, + &edge.filter, reference_time, &deleted_nodes, &deleted_edges, @@ -1511,7 +3771,7 @@ impl ReadView { .zip(contexts.into_iter()) .zip(accumulators.into_iter()) { - let mut entries: Vec = + let mut entries: Vec = if let Some(entry) = accumulator.best_bound_entry { vec![entry] } else { @@ -1549,7 +3809,7 @@ impl ReadView { ) -> Result<(), EngineError> { let edge = &query.edges[edge_index]; let (deleted_nodes, deleted_edges) = self.collect_tombstones(); - let type_filter = edge.type_filter.as_deref(); + let label_filter_ids = edge.label_filter_ids.as_deref(); let mut contexts = Vec::with_capacity(states.len()); for state in &states { contexts.push(Self::pattern_edge_context(edge, state)?); @@ -1558,6 +3818,7 @@ impl ReadView { let mut seen_edges: Vec = states.iter().map(|_| NodeIdSet::default()).collect(); let mut pending = Vec::with_capacity(QUERY_VERIFY_CHUNK); + let mut edge_filter_match_cache = NodeIdMap::default(); let mut unnamed_emitted_targets: Vec = states.iter().map(|_| NodeIdSet::default()).collect(); let mut unnamed_bound_satisfied = vec![false; states.len()]; @@ -1571,7 +3832,7 @@ impl ReadView { let flow = self.memtable.for_each_adj_entry_at( context.source_id, context.direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { let was_satisfied = unnamed_bound_satisfied[state_index]; @@ -1583,6 +3844,7 @@ impl ReadView { &contexts, &mut seen_edges, &mut pending, + &mut edge_filter_match_cache, &mut unnamed_emitted_targets, &mut unnamed_bound_satisfied, policy_cutoffs, @@ -1622,6 +3884,29 @@ impl ReadView { if let Some(error) = stream_error { return Err(error); } + if edge.alias.is_none() && context.target_id.is_some() { + if self + .flush_pattern_pending_entries( + query, + edge_index, + edge, + &states, + &contexts, + &mut pending, + &mut edge_filter_match_cache, + policy_cutoffs, + &mut unnamed_emitted_targets, + &mut unnamed_bound_satisfied, + &mut on_next, + )? + .is_break() + { + return Ok(()); + } + if !flow.is_break() && unnamed_bound_satisfied[state_index] { + satisfied_this_walk = true; + } + } if flow.is_break() && !satisfied_this_walk { return Ok(()); } @@ -1637,7 +3922,7 @@ impl ReadView { let flow = epoch.memtable.for_each_adj_entry_at( context.source_id, context.direction, - type_filter, + label_filter_ids, self.snapshot_seq, &mut |edge_id, neighbor_id, weight, valid_from, valid_to| { let was_satisfied = unnamed_bound_satisfied[state_index]; @@ -1649,6 +3934,7 @@ impl ReadView { &contexts, &mut seen_edges, &mut pending, + &mut edge_filter_match_cache, &mut unnamed_emitted_targets, &mut unnamed_bound_satisfied, policy_cutoffs, @@ -1688,6 +3974,29 @@ impl ReadView { if let Some(error) = stream_error { return Err(error); } + if edge.alias.is_none() && context.target_id.is_some() { + if self + .flush_pattern_pending_entries( + query, + edge_index, + edge, + &states, + &contexts, + &mut pending, + &mut edge_filter_match_cache, + policy_cutoffs, + &mut unnamed_emitted_targets, + &mut unnamed_bound_satisfied, + &mut on_next, + )? + .is_break() + { + return Ok(()); + } + if !flow.is_break() && unnamed_bound_satisfied[state_index] { + satisfied_this_walk = true; + } + } if flow.is_break() && !satisfied_this_walk { return Ok(()); } @@ -1735,7 +4044,7 @@ impl ReadView { let flow = segment.for_each_adj_posting_batch( &source_ids, direction, - type_filter, + label_filter_ids, &mut |queried_node_id, edge_id, neighbor_id, weight, valid_from, valid_to| { let Some(state_indices) = context_indices_by_source.get(&queried_node_id) else { @@ -1754,6 +4063,7 @@ impl ReadView { &contexts, &mut seen_edges, &mut pending, + &mut edge_filter_match_cache, &mut unnamed_emitted_targets, &mut unnamed_bound_satisfied, policy_cutoffs, @@ -1795,9 +4105,41 @@ impl ReadView { if let Some(error) = stream_error { return Err(error); } + if stop_when_target_bound { + if self + .flush_pattern_pending_entries( + query, + edge_index, + edge, + &states, + &contexts, + &mut pending, + &mut edge_filter_match_cache, + policy_cutoffs, + &mut unnamed_emitted_targets, + &mut unnamed_bound_satisfied, + &mut on_next, + )? + .is_break() + { + return Ok(()); + } + remaining_target_bound = contexts + .iter() + .enumerate() + .filter(|(state_index, context)| { + context.direction == direction + && context.target_id.is_some() + && !unnamed_bound_satisfied[*state_index] + }) + .count(); + } if flow.is_break() { return Ok(()); } + if stop_when_target_bound && remaining_target_bound == 0 { + continue; + } } } @@ -1809,6 +4151,7 @@ impl ReadView { &states, &contexts, &mut pending, + &mut edge_filter_match_cache, policy_cutoffs, &mut unnamed_emitted_targets, &mut unnamed_bound_satisfied, @@ -1836,7 +4179,13 @@ impl ReadView { } let edge = &query.edges[edge_index]; - if edge.alias.is_none() && edge.property_predicates.is_empty() { + let mode = pattern_edge_filter_mode(&edge.filter); + if edge.alias.is_none() + && matches!( + mode, + PatternEdgeFilterMode::AlwaysTrue | PatternEdgeFilterMode::PostingMetadataOnly + ) + { return self.execute_pattern_unnamed_constraint_frontier_for_each( query, edge_index, @@ -1919,50 +4268,377 @@ impl ReadView { ) } - fn query_pattern_planned( + #[allow(clippy::too_many_arguments)] + fn verify_pattern_edge_anchor_candidate_chunk( + &self, + candidate_ids: &[u64], + edge: &NormalizedEdgePattern, + reference_time: i64, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + endpoint_cache: &mut EdgeEndpointVisibilityCache, + property_keys: &[String], + verified: &mut Vec, + ) -> Result<(), EngineError> { + let metadata = self.sources().find_edge_metadata(candidate_ids)?; + let mut metas = Vec::new(); + for meta in metadata.into_iter().flatten() { + if edge + .label_filter_ids + .as_ref() + .is_some_and(|label_ids| label_ids.binary_search(&meta.label_id).is_err()) + { + continue; + } + if !is_edge_valid_at(meta.valid_from, meta.valid_to, reference_time) { + continue; + } + metas.push(meta); + } + + { + let sources = self.sources(); + endpoint_cache.ensure_edge_endpoints(&sources, &metas, policy_cutoffs)?; + } + + let mut decisions = Vec::new(); + let mut property_candidate_ids = Vec::new(); + + for meta in metas { + if !endpoint_cache.edge_endpoints_visible(meta) { + continue; + } + let query_meta = EdgeMetadataForQuery::from(meta); + match edge_filter_metadata_outcome(&edge.filter, &query_meta) { + Some(false) => continue, + Some(true) => decisions.push((meta, false)), + None => { + property_candidate_ids.push(meta.edge_id); + decisions.push((meta, true)); + } + } + } + + let mut property_matches = NodeIdSet::default(); + if !property_candidate_ids.is_empty() { + let mut metadata_by_edge_id = + NodeIdMap::with_capacity_and_hasher(property_candidate_ids.len(), Default::default()); + for (meta, needs_properties) in &decisions { + if *needs_properties { + metadata_by_edge_id.insert(meta.edge_id, EdgeMetadataForQuery::from(*meta)); + } + } + let projected = self + .sources() + .find_edge_properties(&property_candidate_ids, property_keys)?; + for (&edge_id, props) in property_candidate_ids.iter().zip(projected.iter()) { + let Some(props) = props else { + continue; + }; + let Some(query_meta) = metadata_by_edge_id.get(&edge_id) else { + continue; + }; + if edge_filter_projected_matches(&edge.filter, query_meta, props) { + property_matches.insert(edge_id); + } + } + } + + for (meta, needs_properties) in decisions { + if needs_properties && !property_matches.contains(&meta.edge_id) { + continue; + } + verified.push(VerifiedPatternEdgeAnchor { meta }); + } + Ok(()) + } + + fn verify_pattern_edge_anchor_candidates( + &self, + candidate_ids: &[u64], + edge: &NormalizedEdgePattern, + reference_time: i64, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result, EngineError> { + let mut verified = Vec::new(); + let mut endpoint_cache = EdgeEndpointVisibilityCache::default(); + let mut property_keys = Vec::new(); + collect_edge_filter_property_keys(&edge.filter, &mut property_keys); + + for chunk in candidate_ids.chunks(QUERY_VERIFY_CHUNK) { + self.verify_pattern_edge_anchor_candidate_chunk( + chunk, + edge, + reference_time, + policy_cutoffs, + &mut endpoint_cache, + &property_keys, + &mut verified, + )?; + } + Ok(verified) + } + + fn pattern_edge_anchor_orientation_bindings( + edge: &NormalizedEdgePattern, + meta: EdgeMetadataCandidate, + ) -> Vec<(u64, u64)> { + match edge.direction { + Direction::Outgoing => vec![(meta.from, meta.to)], + Direction::Incoming => vec![(meta.to, meta.from)], + Direction::Both if meta.from == meta.to => vec![(meta.from, meta.to)], + Direction::Both => vec![(meta.from, meta.to), (meta.to, meta.from)], + } + } + + fn pattern_edge_anchor_states_from_verified( &self, query: &NormalizedGraphPatternQuery, - planned: &PlannedPatternQuery, - ) -> Result, EngineError> { - match query.order { - PatternOrder::AnchorThenAliasesAsc => {} + edge_index: usize, + verified_edges: &[VerifiedPatternEdgeAnchor], + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result, EngineError> { + let edge = &query.edges[edge_index]; + let mut provisional = Vec::new(); + let mut candidate_ids_by_index = vec![Vec::new(); query.nodes.len()]; + let mut seen = HashSet::new(); + + for verified in verified_edges { + for (from_id, to_id) in Self::pattern_edge_anchor_orientation_bindings(edge, verified.meta) { + let mut state = PatternExecutionState { + nodes: vec![None; query.nodes.len()], + edges: vec![None; query.edges.len()], + }; + if edge.from_index == edge.to_index { + if from_id != to_id { + continue; + } + state.nodes[edge.from_index] = Some(from_id); + candidate_ids_by_index[edge.from_index].push(from_id); + } else { + if from_id == to_id { + continue; + } + state.nodes[edge.from_index] = Some(from_id); + state.nodes[edge.to_index] = Some(to_id); + candidate_ids_by_index[edge.from_index].push(from_id); + candidate_ids_by_index[edge.to_index].push(to_id); + } + state.edges[edge_index] = Some(verified.meta.edge_id); + + let mut edge_key = state.edges.clone(); + if edge.alias.is_none() { + edge_key[edge_index] = None; + } + if seen.insert((state.nodes.clone(), edge_key)) { + provisional.push(state); + if provisional.len() > PATTERN_FRONTIER_BUDGET { + return Err(EngineError::InvalidOperation(format!( + "pattern intermediate frontier exceeded {PATTERN_FRONTIER_BUDGET} states" + ))); + } + } + } } - let anchor = &query.nodes[planned.anchor_index]; - let policy_cutoffs = self.query_policy_cutoffs(); - let pattern_reference_time = query.at_epoch.unwrap_or_else(now_millis); + let verified_targets = self.pattern_verified_target_ids_by_index( + query, + candidate_ids_by_index, + policy_cutoffs, + )?; + provisional.retain(|state| { + [edge.from_index, edge.to_index].into_iter().all(|node_index| { + state.nodes[node_index].is_some_and(|node_id| { + verified_targets[node_index].contains(&node_id) + }) + }) + }); + Ok(provisional) + } + + fn pattern_node_anchor_frontier( + &self, + query: &NormalizedGraphPatternQuery, + node_index: usize, + anchor_plan: &PlannedNodeQuery, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + let anchor = &query.nodes[node_index]; let mut anchor_query = anchor.query.clone(); anchor_query.page.limit = Some(PATTERN_FRONTIER_BUDGET); - let (anchor_page, followups) = self.query_node_page_planned( - &anchor_query, - &planned.anchor_plan, - false, - policy_cutoffs.as_ref(), - )?; + let (anchor_page, followups) = + self.query_node_page_planned(&anchor_query, anchor_plan, false, policy_cutoffs)?; if anchor_page.next_cursor.is_some() { return Err(EngineError::InvalidOperation(format!( "pattern intermediate frontier exceeded {PATTERN_FRONTIER_BUDGET} states" ))); } - let anchor_ids = anchor_page.ids; - let mut frontier = Vec::with_capacity(anchor_ids.len()); - for anchor_id in anchor_ids { + let mut states = Vec::with_capacity(anchor_page.ids.len()); + for anchor_id in anchor_page.ids { let mut state = PatternExecutionState { nodes: vec![None; query.nodes.len()], edges: vec![None; query.edges.len()], }; - state.nodes[planned.anchor_index] = Some(anchor_id); - frontier.push(state); - if frontier.len() > PATTERN_FRONTIER_BUDGET { + state.nodes[node_index] = Some(anchor_id); + states.push(state); + if states.len() > PATTERN_FRONTIER_BUDGET { return Err(EngineError::InvalidOperation(format!( "pattern intermediate frontier exceeded {PATTERN_FRONTIER_BUDGET} states" ))); } } + Ok(PatternAnchorFrontier::Ready { + states, + followups, + expansion_order: Vec::new(), + sort_anchor_alias: String::new(), + }) + } + + #[allow(clippy::too_many_arguments)] + fn pattern_edge_anchor_frontier( + &self, + query: &NormalizedGraphPatternQuery, + edge_index: usize, + edge_query: &NormalizedEdgeQuery, + edge_plan: PlannedEdgeQuery, + edge_fallback_plans: Vec, + reference_time: i64, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + let mut deferred_followups = Vec::new(); + for plan in std::iter::once(edge_plan).chain(edge_fallback_plans.into_iter()) { + let PlannedEdgeQuery { + driver, + cap_context, + warnings: _, + mut followups, + } = plan; + let materialized = self.materialize_edge_physical_plan(edge_query, cap_context, &driver)?; + let candidate_ids = match materialized { + CandidateMaterializationResult::Ready { + ids, + followups: mut materialization_followups, + } => { + followups.append(&mut materialization_followups); + ids + } + CandidateMaterializationResult::TooBroad { + followups: mut materialization_followups, + } => { + followups.append(&mut materialization_followups); + deferred_followups.append(&mut followups); + continue; + } + }; + let verified = self.verify_pattern_edge_anchor_candidates( + &candidate_ids, + &query.edges[edge_index], + reference_time, + policy_cutoffs, + )?; + let states = self.pattern_edge_anchor_states_from_verified( + query, + edge_index, + &verified, + policy_cutoffs, + )?; + deferred_followups.append(&mut followups); + return Ok(PatternAnchorFrontier::Ready { + states, + followups: deferred_followups, + expansion_order: Vec::new(), + sort_anchor_alias: String::new(), + }); + } + Ok(PatternAnchorFrontier::TooBroad { + followups: deferred_followups, + }) + } + + fn pattern_anchor_frontier( + &self, + query: &NormalizedGraphPatternQuery, + anchor: PatternAnchorPlan, + reference_time: i64, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + ) -> Result { + match anchor { + PatternAnchorPlan::Node { + node_index, + anchor_plan, + expansion_order, + sort_anchor_alias, + .. + } => match self.pattern_node_anchor_frontier(query, node_index, &anchor_plan, policy_cutoffs)? { + PatternAnchorFrontier::Ready { states, followups, .. } => { + Ok(PatternAnchorFrontier::Ready { + states, + followups, + expansion_order, + sort_anchor_alias, + }) + } + PatternAnchorFrontier::TooBroad { followups } => { + Ok(PatternAnchorFrontier::TooBroad { followups }) + } + }, + PatternAnchorPlan::Edge { + edge_index, + edge_query, + edge_plan, + edge_fallback_plans, + expansion_order, + sort_anchor_alias, + from_index, + to_index, + orientation_policy, + .. + } => { + let edge = &query.edges[edge_index]; + debug_assert_eq!(from_index, edge.from_index); + debug_assert_eq!(to_index, edge.to_index); + debug_assert_eq!( + orientation_policy, + EdgeAnchorOrientationPolicy::from_direction(edge.direction) + ); + match self.pattern_edge_anchor_frontier( + query, + edge_index, + &edge_query, + edge_plan, + edge_fallback_plans, + reference_time, + policy_cutoffs, + )? { + PatternAnchorFrontier::Ready { states, followups, .. } => { + Ok(PatternAnchorFrontier::Ready { + states, + followups, + expansion_order, + sort_anchor_alias, + }) + } + PatternAnchorFrontier::TooBroad { followups } => { + Ok(PatternAnchorFrontier::TooBroad { followups }) + } + } + } + } + } - let sort_anchor_alias = &planned.sort_anchor_alias; + #[allow(clippy::too_many_arguments)] + fn finish_pattern_from_frontier( + &self, + query: &NormalizedGraphPatternQuery, + expansion_order: &[usize], + mut frontier: Vec, + sort_anchor_alias: &str, + reference_time: i64, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + followups: Vec, + ) -> Result, EngineError> { let mut matches = Vec::with_capacity(query.limit.saturating_add(1)); - let Some((&final_edge_index, prefix_order)) = planned.expansion_order.split_last() else { + let Some((&final_edge_index, prefix_order)) = expansion_order.split_last() else { for state in frontier { insert_bounded_pattern_match( &mut matches, @@ -1985,8 +4661,8 @@ impl ReadView { query, edge_index, frontier, - pattern_reference_time, - policy_cutoffs.as_ref(), + reference_time, + policy_cutoffs, )?; if frontier.is_empty() { break; @@ -2000,8 +4676,8 @@ impl ReadView { frontier, &mut matches, sort_anchor_alias, - pattern_reference_time, - policy_cutoffs.as_ref(), + reference_time, + policy_cutoffs, )?; } @@ -2014,12 +4690,59 @@ impl ReadView { }) } + fn query_pattern_planned( + &self, + query: &NormalizedGraphPatternQuery, + planned: PlannedPatternQuery, + ) -> Result, EngineError> { + match query.order { + PatternOrder::AnchorThenAliasesAsc => {} + } + + let policy_cutoffs = self.query_policy_cutoffs(); + let pattern_reference_time = query.at_epoch.unwrap_or_else(now_millis); + let mut deferred_followups = Vec::new(); + for anchor in std::iter::once(planned.anchor).chain(planned.fallback_anchors.into_iter()) { + match self.pattern_anchor_frontier( + query, + anchor, + pattern_reference_time, + policy_cutoffs.as_ref(), + )? { + PatternAnchorFrontier::Ready { + states, + mut followups, + expansion_order, + sort_anchor_alias, + } => { + deferred_followups.append(&mut followups); + return self.finish_pattern_from_frontier( + query, + &expansion_order, + states, + &sort_anchor_alias, + pattern_reference_time, + policy_cutoffs.as_ref(), + deferred_followups, + ); + } + PatternAnchorFrontier::TooBroad { mut followups } => { + deferred_followups.append(&mut followups); + } + } + } + + Err(EngineError::InvalidOperation( + "pattern edge anchor source exceeded candidate cap".into(), + )) + } + fn query_pattern_outcome( &self, query: &GraphPatternQuery, ) -> Result, EngineError> { let normalized = self.normalize_pattern_query(query)?; let planned = self.plan_normalized_pattern_query(&normalized)?; - self.query_pattern_planned(&normalized, &planned) + self.query_pattern_planned(&normalized, planned) } } diff --git a/src/engine/query_ir.rs b/src/engine/query_ir.rs index b25eef1..5a6b374 100644 --- a/src/engine/query_ir.rs +++ b/src/engine/query_ir.rs @@ -2,11 +2,11 @@ enum NodeQueryCandidateSourceKind { ExplicitIds, KeyLookup, - NodeTypeIndex, + NodeLabelIndex, PropertyEqualityIndex, PropertyRangeIndex, TimestampIndex, - FallbackTypeScan, + FallbackNodeLabelScan, FallbackFullNodeScan, } @@ -45,12 +45,75 @@ enum NormalizedNodeFilter { #[derive(Clone, Debug)] struct NormalizedNodeQuery { - type_id: Option, + single_label_id: Option, + label_filter: ResolvedNodeLabelFilter, ids: Vec, keys: Vec, filter: NormalizedNodeFilter, allow_full_scan: bool, page: PageRequest, + warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +enum NormalizedEdgeFilter { + AlwaysTrue, + AlwaysFalse, + PropertyEquals { + key: String, + value: PropValue, + }, + PropertyIn { + key: String, + values: Vec, + value_keys: Vec>, + }, + PropertyRange { + key: String, + lower: Option, + upper: Option, + }, + PropertyExists { + key: String, + }, + PropertyMissing { + key: String, + }, + WeightRange { + lower: Option, + upper: Option, + }, + UpdatedAtRange { + lower_ms: i64, + upper_ms: i64, + }, + ValidAt { + epoch_ms: i64, + }, + ValidFromRange { + lower_ms: i64, + upper_ms: i64, + }, + ValidToRange { + lower_ms: i64, + upper_ms: i64, + }, + And(Vec), + Or(Vec), + Not(Box), +} + +#[derive(Clone, Debug)] +struct NormalizedEdgeQuery { + label_id: Option, + ids: Vec, + from_ids: Vec, + to_ids: Vec, + endpoint_ids: Vec, + filter: NormalizedEdgeFilter, + allow_full_scan: bool, + page: PageRequest, + warnings: Vec, } #[derive(Clone, Debug)] @@ -65,8 +128,8 @@ struct NormalizedEdgePattern { from_index: usize, to_index: usize, direction: Direction, - type_filter: Option>, - property_predicates: Vec, + label_filter_ids: Option>, + filter: NormalizedEdgeFilter, } #[derive(Clone, Debug)] @@ -76,6 +139,7 @@ struct NormalizedGraphPatternQuery { at_epoch: Option, limit: usize, order: PatternOrder, + warnings: Vec, } fn prop_value_canonical_bytes(value: &PropValue) -> Vec { @@ -164,6 +228,100 @@ impl NormalizedNodeFilter { } } +impl NormalizedEdgeFilter { + fn is_always_true(&self) -> bool { + matches!(self, Self::AlwaysTrue) + } + + fn is_always_false(&self) -> bool { + matches!(self, Self::AlwaysFalse) + } + + fn structural_key(&self) -> Vec { + let mut key = Vec::new(); + match self { + Self::AlwaysTrue => key.push(0), + Self::AlwaysFalse => key.push(1), + Self::PropertyEquals { key: prop_key, value } => { + key.push(2); + push_len_prefixed_bytes(&mut key, prop_key.as_bytes()); + push_len_prefixed_bytes(&mut key, &prop_value_canonical_bytes(value)); + } + Self::PropertyIn { + key: prop_key, + value_keys, + .. + } => { + key.push(3); + push_len_prefixed_bytes(&mut key, prop_key.as_bytes()); + for value_key in value_keys { + push_len_prefixed_bytes(&mut key, value_key); + } + } + Self::PropertyRange { + key: prop_key, + lower, + upper, + } => { + key.push(4); + push_len_prefixed_bytes(&mut key, prop_key.as_bytes()); + key.extend_from_slice(format!("{lower:?}:{upper:?}").as_bytes()); + } + Self::PropertyExists { key: prop_key } => { + key.push(5); + push_len_prefixed_bytes(&mut key, prop_key.as_bytes()); + } + Self::PropertyMissing { key: prop_key } => { + key.push(6); + push_len_prefixed_bytes(&mut key, prop_key.as_bytes()); + } + Self::WeightRange { lower, upper } => { + key.push(7); + key.extend_from_slice(&lower.map(f32::to_bits).unwrap_or(0).to_be_bytes()); + key.extend_from_slice(&upper.map(f32::to_bits).unwrap_or(0).to_be_bytes()); + key.push(lower.is_some() as u8); + key.push(upper.is_some() as u8); + } + Self::UpdatedAtRange { lower_ms, upper_ms } => { + key.push(8); + key.extend_from_slice(&lower_ms.to_be_bytes()); + key.extend_from_slice(&upper_ms.to_be_bytes()); + } + Self::ValidAt { epoch_ms } => { + key.push(9); + key.extend_from_slice(&epoch_ms.to_be_bytes()); + } + Self::ValidFromRange { lower_ms, upper_ms } => { + key.push(10); + key.extend_from_slice(&lower_ms.to_be_bytes()); + key.extend_from_slice(&upper_ms.to_be_bytes()); + } + Self::ValidToRange { lower_ms, upper_ms } => { + key.push(11); + key.extend_from_slice(&lower_ms.to_be_bytes()); + key.extend_from_slice(&upper_ms.to_be_bytes()); + } + Self::And(children) => { + key.push(12); + for child in children { + push_len_prefixed_bytes(&mut key, &child.structural_key()); + } + } + Self::Or(children) => { + key.push(13); + for child in children { + push_len_prefixed_bytes(&mut key, &child.structural_key()); + } + } + Self::Not(child) => { + key.push(14); + push_len_prefixed_bytes(&mut key, &child.structural_key()); + } + } + key + } +} + fn require_non_empty_filter_key(key: &str, context: &str) -> Result<(), EngineError> { if key.is_empty() { Err(EngineError::InvalidOperation(format!( @@ -362,12 +520,388 @@ fn normalize_optional_node_filter( filter.map(normalize_node_filter_expr).unwrap_or(Ok(NormalizedNodeFilter::AlwaysTrue)) } +fn edge_filter_children_sorted_dedup(mut children: Vec) -> Vec { + children.sort_by_key(NormalizedEdgeFilter::structural_key); + children.dedup_by(|left, right| left.structural_key() == right.structural_key()); + children +} + +fn normalize_normalized_edge_and_filter( + mut flattened: Vec, +) -> Result { + let mut eq_by_key: HashMap = HashMap::new(); + let mut exists_keys = HashSet::new(); + let mut missing_keys = HashSet::new(); + for child in &flattened { + match child { + NormalizedEdgeFilter::PropertyEquals { key, value } => { + if let Some(existing) = eq_by_key.get(key) { + if !prop_values_equal_for_filter(existing, value) { + return Ok(NormalizedEdgeFilter::AlwaysFalse); + } + } else { + eq_by_key.insert(key.clone(), value.clone()); + } + exists_keys.insert(key.clone()); + } + NormalizedEdgeFilter::PropertyIn { key, .. } + | NormalizedEdgeFilter::PropertyRange { key, .. } + | NormalizedEdgeFilter::PropertyExists { key } => { + exists_keys.insert(key.clone()); + } + NormalizedEdgeFilter::PropertyMissing { key } => { + missing_keys.insert(key.clone()); + } + _ => {} + } + } + + if exists_keys.iter().any(|key| missing_keys.contains(key)) { + return Ok(NormalizedEdgeFilter::AlwaysFalse); + } + + flattened.retain(|child| match child { + NormalizedEdgeFilter::PropertyExists { key } => !eq_by_key.contains_key(key), + _ => true, + }); + + let flattened = edge_filter_children_sorted_dedup(flattened); + Ok(match flattened.len() { + 0 => NormalizedEdgeFilter::AlwaysTrue, + 1 => flattened.into_iter().next().unwrap(), + _ => NormalizedEdgeFilter::And(flattened), + }) +} + +fn normalize_edge_and_filter( + children: &[EdgeFilterExpr], +) -> Result { + if children.is_empty() { + return Err(EngineError::InvalidOperation( + "edge and filters must contain at least one child".into(), + )); + } + + let mut flattened = Vec::new(); + for child in children { + match normalize_edge_filter_expr(child)? { + NormalizedEdgeFilter::AlwaysFalse => return Ok(NormalizedEdgeFilter::AlwaysFalse), + NormalizedEdgeFilter::AlwaysTrue => {} + NormalizedEdgeFilter::And(grandchildren) => flattened.extend(grandchildren), + normalized => flattened.push(normalized), + } + } + + normalize_normalized_edge_and_filter(flattened) +} + +fn normalize_edge_or_filter(children: &[EdgeFilterExpr]) -> Result { + if children.is_empty() { + return Err(EngineError::InvalidOperation( + "edge or filters must contain at least one child".into(), + )); + } + + let mut flattened = Vec::new(); + for child in children { + match normalize_edge_filter_expr(child)? { + NormalizedEdgeFilter::AlwaysTrue => return Ok(NormalizedEdgeFilter::AlwaysTrue), + NormalizedEdgeFilter::AlwaysFalse => {} + NormalizedEdgeFilter::Or(grandchildren) => flattened.extend(grandchildren), + normalized => flattened.push(normalized), + } + } + + let flattened = edge_filter_children_sorted_dedup(flattened); + Ok(match flattened.len() { + 0 => NormalizedEdgeFilter::AlwaysFalse, + 1 => flattened.into_iter().next().unwrap(), + _ => NormalizedEdgeFilter::Or(flattened), + }) +} + +fn normalize_i64_range( + lower: Option, + upper: Option, + context: &str, +) -> Result, EngineError> { + if lower.is_none() && upper.is_none() { + return Err(EngineError::InvalidOperation(format!( + "{context} range filters require at least one bound" + ))); + } + let lower = lower.unwrap_or(i64::MIN); + let upper = upper.unwrap_or(i64::MAX); + if lower > upper { + Ok(None) + } else { + Ok(Some((lower, upper))) + } +} + +fn normalize_weight_range( + lower: Option, + upper: Option, +) -> Result { + if lower.is_none() && upper.is_none() { + return Err(EngineError::InvalidOperation( + "edge weight range filters require at least one bound".into(), + )); + } + if lower.is_some_and(f32::is_nan) || upper.is_some_and(f32::is_nan) { + return Err(EngineError::InvalidOperation( + "edge weight range bounds must not be NaN".into(), + )); + } + if let (Some(lower), Some(upper)) = (lower, upper) { + if lower > upper { + return Ok(NormalizedEdgeFilter::AlwaysFalse); + } + } + Ok(NormalizedEdgeFilter::WeightRange { lower, upper }) +} + +fn normalize_edge_filter_expr(expr: &EdgeFilterExpr) -> Result { + match expr { + EdgeFilterExpr::PropertyEquals { key, value } => { + require_non_empty_filter_key(key, "edge property equals filter")?; + Ok(NormalizedEdgeFilter::PropertyEquals { + key: key.clone(), + value: value.clone(), + }) + } + EdgeFilterExpr::PropertyIn { key, values } => { + require_non_empty_filter_key(key, "edge property in filter")?; + if values.is_empty() { + return Ok(NormalizedEdgeFilter::AlwaysFalse); + } + let mut keyed_values: Vec<(Vec, PropValue)> = values + .iter() + .map(|value| (prop_value_canonical_bytes(value), value.clone())) + .collect(); + keyed_values.sort_by(|left, right| left.0.cmp(&right.0)); + keyed_values.dedup_by(|left, right| left.0 == right.0); + if keyed_values.len() == 1 { + let (_, value) = keyed_values.into_iter().next().unwrap(); + return Ok(NormalizedEdgeFilter::PropertyEquals { + key: key.clone(), + value, + }); + } + let (value_keys, deduped_values): (Vec>, Vec) = + keyed_values.into_iter().unzip(); + Ok(NormalizedEdgeFilter::PropertyIn { + key: key.clone(), + values: deduped_values, + value_keys, + }) + } + EdgeFilterExpr::PropertyRange { key, lower, upper } => { + require_non_empty_filter_key(key, "edge property range filter")?; + ReadView::validate_property_range_bounds(lower.as_ref(), upper.as_ref(), None)?; + Ok(NormalizedEdgeFilter::PropertyRange { + key: key.clone(), + lower: lower.clone(), + upper: upper.clone(), + }) + } + EdgeFilterExpr::PropertyExists { key } => { + require_non_empty_filter_key(key, "edge property exists filter")?; + Ok(NormalizedEdgeFilter::PropertyExists { key: key.clone() }) + } + EdgeFilterExpr::PropertyMissing { key } => { + require_non_empty_filter_key(key, "edge property missing filter")?; + Ok(NormalizedEdgeFilter::PropertyMissing { key: key.clone() }) + } + EdgeFilterExpr::WeightRange { lower, upper } => normalize_weight_range(*lower, *upper), + EdgeFilterExpr::UpdatedAtRange { lower_ms, upper_ms } => { + let Some((lower_ms, upper_ms)) = + normalize_i64_range(*lower_ms, *upper_ms, "edge updated-at")? + else { + return Ok(NormalizedEdgeFilter::AlwaysFalse); + }; + Ok(NormalizedEdgeFilter::UpdatedAtRange { lower_ms, upper_ms }) + } + EdgeFilterExpr::ValidAt { epoch_ms } => { + Ok(NormalizedEdgeFilter::ValidAt { epoch_ms: *epoch_ms }) + } + EdgeFilterExpr::ValidFromRange { lower_ms, upper_ms } => { + let Some((lower_ms, upper_ms)) = + normalize_i64_range(*lower_ms, *upper_ms, "edge valid-from")? + else { + return Ok(NormalizedEdgeFilter::AlwaysFalse); + }; + Ok(NormalizedEdgeFilter::ValidFromRange { lower_ms, upper_ms }) + } + EdgeFilterExpr::ValidToRange { lower_ms, upper_ms } => { + let Some((lower_ms, upper_ms)) = + normalize_i64_range(*lower_ms, *upper_ms, "edge valid-to")? + else { + return Ok(NormalizedEdgeFilter::AlwaysFalse); + }; + Ok(NormalizedEdgeFilter::ValidToRange { lower_ms, upper_ms }) + } + EdgeFilterExpr::And(children) => normalize_edge_and_filter(children), + EdgeFilterExpr::Or(children) => normalize_edge_or_filter(children), + EdgeFilterExpr::Not(child) => match normalize_edge_filter_expr(child)? { + NormalizedEdgeFilter::AlwaysTrue => Ok(NormalizedEdgeFilter::AlwaysFalse), + NormalizedEdgeFilter::AlwaysFalse => Ok(NormalizedEdgeFilter::AlwaysTrue), + NormalizedEdgeFilter::PropertyExists { key } => { + Ok(NormalizedEdgeFilter::PropertyMissing { key }) + } + NormalizedEdgeFilter::PropertyMissing { key } => { + Ok(NormalizedEdgeFilter::PropertyExists { key }) + } + NormalizedEdgeFilter::Not(grandchild) => Ok(*grandchild), + normalized => Ok(NormalizedEdgeFilter::Not(Box::new(normalized))), + }, + } +} + +fn normalize_optional_edge_filter( + filter: Option<&EdgeFilterExpr>, +) -> Result { + filter.map(normalize_edge_filter_expr).unwrap_or(Ok(NormalizedEdgeFilter::AlwaysTrue)) +} + +fn sorted_dedup_u64(mut values: Vec) -> Vec { + values.sort_unstable(); + values.dedup(); + values +} + +fn push_query_warning(warnings: &mut Vec, warning: QueryPlanWarning) { + if !warnings.contains(&warning) { + warnings.push(warning); + } +} + +fn unknown_node_label_count(filter: &ResolvedNodeLabelFilter) -> usize { + match filter { + ResolvedNodeLabelFilter::Unconstrained => 0, + ResolvedNodeLabelFilter::Empty { + unknown_label_count, + .. + } + | ResolvedNodeLabelFilter::LabelSet { + unknown_label_count, + .. + } => *unknown_label_count, + } +} + +fn single_resolved_label_id(filter: &ResolvedNodeLabelFilter) -> Option { + match filter { + ResolvedNodeLabelFilter::LabelSet { label_ids, .. } if label_ids.len() == 1 => { + Some(label_ids.single_label_id()) + } + _ => None, + } +} + +fn normalized_query_has_label_anchor(query: &NormalizedNodeQuery) -> bool { + matches!( + query.label_filter, + ResolvedNodeLabelFilter::LabelSet { .. } | ResolvedNodeLabelFilter::Empty { .. } + ) +} + impl ReadView { + fn resolve_node_query_label_filter( + &self, + label_filter: Option<&NodeLabelFilter>, + ) -> Result<(ResolvedNodeLabelFilter, Option, Vec), EngineError> { + let resolved = self + .label_catalog + .resolve_node_label_filter_request(label_filter)?; + let mut warnings = Vec::new(); + if unknown_node_label_count(&resolved) > 0 { + push_query_warning(&mut warnings, QueryPlanWarning::UnknownNodeLabel); + } + let single_label_id = single_resolved_label_id(&resolved); + Ok((resolved, single_label_id, warnings)) + } + + fn resolve_node_pattern_label_filter( + &self, + pattern: &NodePattern, + ) -> Result<(ResolvedNodeLabelFilter, Option, Vec), EngineError> { + let resolved = self + .label_catalog + .resolve_node_label_filter_request(pattern.label_filter.as_ref())?; + let mut warnings = Vec::new(); + if unknown_node_label_count(&resolved) > 0 { + push_query_warning(&mut warnings, QueryPlanWarning::UnknownNodeLabel); + } + let single_label_id = single_resolved_label_id(&resolved); + Ok((resolved, single_label_id, warnings)) + } + + fn normalize_edge_query_with_anchor_requirement( + &self, + query: &EdgeQuery, + require_anchor: bool, + ) -> Result { + let mut warnings = Vec::new(); + let mut label_id = None; + let mut filter = normalize_optional_edge_filter(query.filter.as_ref())?; + if let Some(label) = query.label.as_deref() { + match self.label_catalog.resolve_edge_label_for_read(label)? { + Some(resolved) => label_id = Some(resolved), + None => { + filter = NormalizedEdgeFilter::AlwaysFalse; + push_query_warning(&mut warnings, QueryPlanWarning::UnknownEdgeLabel); + } + } + } + let ids = sorted_dedup_u64(query.ids.clone()); + let from_ids = sorted_dedup_u64(query.from_ids.clone()); + let to_ids = sorted_dedup_u64(query.to_ids.clone()); + let endpoint_ids = sorted_dedup_u64(query.endpoint_ids.clone()); + + if require_anchor + && label_id.is_none() + && ids.is_empty() + && from_ids.is_empty() + && to_ids.is_empty() + && endpoint_ids.is_empty() + && !query.allow_full_scan + && !filter.is_always_false() + { + return Err(EngineError::InvalidOperation( + "edge query requires label, ids, from_ids, to_ids, endpoint_ids, or allow_full_scan".into(), + )); + } + + Ok(NormalizedEdgeQuery { + label_id, + ids, + from_ids, + to_ids, + endpoint_ids, + filter, + allow_full_scan: query.allow_full_scan, + page: query.page.clone(), + warnings, + }) + } + + fn normalize_edge_query(&self, query: &EdgeQuery) -> Result { + self.normalize_edge_query_with_anchor_requirement(query, true) + } + fn normalize_node_query_with_anchor_requirement( &self, query: &NodeQuery, require_anchor: bool, ) -> Result { + let (label_filter, single_label_id, warnings) = self.resolve_node_query_label_filter( + query.label_filter.as_ref(), + )?; + let mut filter = normalize_optional_node_filter(query.filter.as_ref())?; + if label_filter.is_empty_constraint() { + filter = NormalizedNodeFilter::AlwaysFalse; + } let mut ids = query.ids.clone(); ids.sort_unstable(); ids.dedup(); @@ -376,33 +910,48 @@ impl ReadView { keys.sort(); keys.dedup(); - if !keys.is_empty() && query.type_id.is_none() { - return Err(EngineError::InvalidOperation( - "node query keys require type_id".into(), - )); + if !keys.is_empty() { + match label_filter { + ResolvedNodeLabelFilter::LabelSet { label_ids, .. } if label_ids.len() == 1 => {} + ResolvedNodeLabelFilter::Empty { .. } => {} + ResolvedNodeLabelFilter::Unconstrained => { + return Err(EngineError::InvalidOperation( + "node query keys require exactly one resolved label".into(), + )); + } + ResolvedNodeLabelFilter::LabelSet { .. } => { + return Err(EngineError::InvalidOperation( + "node query keys require exactly one resolved label".into(), + )); + } + } } - let filter = normalize_optional_node_filter(query.filter.as_ref())?; - + let has_label_anchor = matches!( + label_filter, + ResolvedNodeLabelFilter::LabelSet { .. } | ResolvedNodeLabelFilter::Empty { .. } + ); if require_anchor && ids.is_empty() && keys.is_empty() - && query.type_id.is_none() + && !has_label_anchor && !query.allow_full_scan && !filter.is_always_false() { return Err(EngineError::InvalidOperation( - "node query requires type_id, ids, keys, or allow_full_scan".into(), + "node query requires label_filter, ids, keys, or allow_full_scan".into(), )); } Ok(NormalizedNodeQuery { - type_id: query.type_id, + single_label_id, + label_filter, ids, keys, filter, allow_full_scan: query.allow_full_scan, page: query.page.clone(), + warnings, }) } @@ -420,38 +969,47 @@ impl ReadView { )); } - let query = NodeQuery { - type_id: pattern.type_id, - ids: pattern.ids.clone(), - keys: pattern.keys.clone(), - filter: pattern.filter.clone(), - page: PageRequest::default(), - order: NodeQueryOrder::NodeIdAsc, - allow_full_scan: false, - }; - Ok(NormalizedNodePattern { - alias: pattern.alias.clone(), - query: self.normalize_node_query_with_anchor_requirement(&query, false)?, - }) - } - - fn normalize_edge_post_filter_predicates( - &self, - predicates: &[EdgePostFilterPredicate], - ) -> Result, EngineError> { - let mut normalized = Vec::with_capacity(predicates.len()); - for predicate in predicates { - match predicate { - EdgePostFilterPredicate::PropertyEquals { .. } => { - normalized.push(predicate.clone()); + let (label_filter, single_label_id, warnings) = + self.resolve_node_pattern_label_filter(pattern)?; + let mut filter = normalize_optional_node_filter(pattern.filter.as_ref())?; + if label_filter.is_empty_constraint() { + filter = NormalizedNodeFilter::AlwaysFalse; + } + if !pattern.keys.is_empty() { + match label_filter { + ResolvedNodeLabelFilter::LabelSet { label_ids, .. } if label_ids.len() == 1 => {} + ResolvedNodeLabelFilter::Empty { .. } => {} + ResolvedNodeLabelFilter::Unconstrained => { + return Err(EngineError::InvalidOperation( + "node pattern keys require exactly one resolved label".into(), + )); } - EdgePostFilterPredicate::PropertyRange { lower, upper, .. } => { - Self::validate_property_range_bounds(lower.as_ref(), upper.as_ref(), None)?; - normalized.push(predicate.clone()); + ResolvedNodeLabelFilter::LabelSet { .. } => { + return Err(EngineError::InvalidOperation( + "node pattern keys require exactly one resolved label".into(), + )); } } } - Ok(normalized) + let mut ids = pattern.ids.clone(); + ids.sort_unstable(); + ids.dedup(); + let mut keys = pattern.keys.clone(); + keys.sort(); + keys.dedup(); + Ok(NormalizedNodePattern { + alias: pattern.alias.clone(), + query: NormalizedNodeQuery { + single_label_id, + label_filter, + ids, + keys, + filter, + allow_full_scan: false, + page: PageRequest::default(), + warnings, + }, + }) } fn normalize_pattern_query( @@ -474,10 +1032,15 @@ impl ReadView { )); } + let mut query_warnings = Vec::new(); + let mut nodes = Vec::with_capacity(query.nodes.len()); let mut alias_to_index = HashMap::with_capacity(query.nodes.len()); for pattern in &query.nodes { let normalized = self.normalize_node_pattern(pattern)?; + for warning in &normalized.query.warnings { + push_query_warning(&mut query_warnings, *warning); + } if alias_to_index .insert(normalized.alias.clone(), nodes.len()) .is_some() @@ -527,25 +1090,30 @@ impl ReadView { adjacency[from_index].push(to_index); adjacency[to_index].push(from_index); - let mut type_filter = pattern.type_filter.clone(); - if let Some(filter) = type_filter.as_mut() { - filter.sort_unstable(); - filter.dedup(); - if filter.is_empty() { - return Err(EngineError::InvalidOperation( - "edge pattern type_filter must not be empty".into(), - )); - } + let (label_filter_ids, mut warnings) = + match self + .label_catalog + .resolve_edge_label_filter(Some(&pattern.label_filter))? + { + (LabelFilterResolution::Unconstrained, warnings) => (None, warnings), + (LabelFilterResolution::Known(label_ids), warnings) => (Some(label_ids), warnings), + (LabelFilterResolution::EmptyConstraint, warnings) => { + (Some(Vec::new()), warnings) + } + }; + for warning in warnings.drain(..) { + push_query_warning(&mut query_warnings, warning); } + let filter = normalize_optional_edge_filter(pattern.filter.as_ref())?; + edges.push(NormalizedEdgePattern { alias: pattern.alias.clone(), from_index, to_index, direction: pattern.direction, - type_filter, - property_predicates: self - .normalize_edge_post_filter_predicates(&pattern.property_predicates)?, + label_filter_ids, + filter, }); } @@ -578,6 +1146,7 @@ impl ReadView { at_epoch: query.at_epoch, limit: query.limit, order: query.order, + warnings: query_warnings, }) } } diff --git a/src/engine/query_plan.rs b/src/engine/query_plan.rs index be0c822..0acc78f 100644 --- a/src/engine/query_plan.rs +++ b/src/engine/query_plan.rs @@ -10,6 +10,7 @@ const FANOUT_HUB_HIGH_RATIO: u64 = 8; const FANOUT_HUB_MEDIUM_RATIO: u64 = 4; const FANOUT_CONFIDENCE_DOWNGRADE_STEP: u8 = 1; +#[derive(Clone)] struct PlannedNodeQuery { driver: NodePhysicalPlan, cap_context: QueryCapContext, @@ -17,14 +18,51 @@ struct PlannedNodeQuery { } struct PlannedPatternQuery { - anchor_index: usize, - anchor_alias: String, - sort_anchor_alias: String, - anchor_plan: PlannedNodeQuery, - expansion_order: Vec, + anchor: PatternAnchorPlan, + fallback_anchors: Vec, warnings: Vec, } +struct PlannedPatternEdgeAnchorSource { + edge_plan: PlannedEdgeQuery, + edge_fallback_plans: Vec, +} + +type PatternEdgeLabeledBranchPlan = ( + EdgePhysicalPlan, + Vec, + Vec, +); + +enum PatternAnchorPlan { + Node { + node_index: usize, + anchor_alias: String, + sort_anchor_alias: String, + anchor_plan: PlannedNodeQuery, + expansion_order: Vec, + }, + Edge { + edge_index: usize, + edge_alias: Option, + from_index: usize, + to_index: usize, + sort_anchor_alias: String, + edge_query: Box, + edge_plan: PlannedEdgeQuery, + edge_fallback_plans: Vec, + expansion_order: Vec, + orientation_policy: EdgeAnchorOrientationPolicy, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum EdgeAnchorOrientationPolicy { + Outgoing, + Incoming, + Both, +} + #[derive(Clone)] enum NodePhysicalPlan { Empty, @@ -62,12 +100,179 @@ struct PlannedNodeCandidateSource { materialization: NodeCandidateMaterialization, } +struct PlannedEdgeQuery { + driver: EdgePhysicalPlan, + cap_context: EdgeQueryCapContext, + warnings: Vec, + followups: Vec, +} + +#[derive(Clone, Copy)] +struct EdgeMetadataSidecarAvailability { + weight: bool, + updated_at: bool, + valid_from: bool, + valid_to: bool, +} + +#[derive(Clone)] +enum EdgePhysicalPlan { + Empty, + Source(PlannedEdgeCandidateSource), + Intersect(Vec), + Union(Vec), +} + +#[derive(Clone)] +struct PlannedEdgeCandidateSource { + kind: EdgeQueryCandidateSourceKind, + canonical_key: String, + estimate: PlannerEstimate, + materialization: EdgeCandidateMaterialization, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum EdgeQueryCandidateSourceKind { + ExplicitEdgeIds, + EdgeLabelIndex, + EdgeTripleIndex, + FromEndpointAdjacency, + ToEndpointAdjacency, + AnyEndpointAdjacency, + EdgeWeightIndex, + EdgeUpdatedAtIndex, + EdgeValidFromIndex, + EdgeValidToIndex, + EdgeMetadataScan, + EdgePropertyEqualityIndex, + EdgePropertyRangeIndex, + FallbackFullEdgeScan, +} + +impl EdgeQueryCandidateSourceKind { + fn plan_node(self) -> QueryPlanNode { + match self { + Self::ExplicitEdgeIds => QueryPlanNode::ExplicitEdgeIds, + Self::EdgeLabelIndex => QueryPlanNode::EdgeLabelIndex, + Self::EdgeTripleIndex => QueryPlanNode::EdgeTripleIndex, + Self::FromEndpointAdjacency + | Self::ToEndpointAdjacency + | Self::AnyEndpointAdjacency => QueryPlanNode::EdgeEndpointAdjacency, + Self::EdgeWeightIndex => QueryPlanNode::EdgeWeightIndex, + Self::EdgeUpdatedAtIndex => QueryPlanNode::EdgeUpdatedAtIndex, + Self::EdgeValidFromIndex | Self::EdgeValidToIndex => QueryPlanNode::EdgeValidityIndex, + Self::EdgeMetadataScan => QueryPlanNode::EdgeMetadataScan, + Self::EdgePropertyEqualityIndex => QueryPlanNode::EdgePropertyEqualityIndex, + Self::EdgePropertyRangeIndex => QueryPlanNode::EdgePropertyRangeIndex, + Self::FallbackFullEdgeScan => QueryPlanNode::FallbackFullEdgeScan, + } + } + + fn source_rank(self) -> usize { + match self { + Self::ExplicitEdgeIds => 0, + Self::EdgeTripleIndex => 1, + Self::FromEndpointAdjacency | Self::ToEndpointAdjacency => 2, + Self::AnyEndpointAdjacency => 3, + Self::EdgeWeightIndex + | Self::EdgeUpdatedAtIndex + | Self::EdgeValidFromIndex + | Self::EdgeValidToIndex + | Self::EdgePropertyEqualityIndex + | Self::EdgePropertyRangeIndex => 4, + Self::EdgeLabelIndex => 5, + Self::EdgeMetadataScan => 6, + Self::FallbackFullEdgeScan => 7, + } + } +} + +#[derive(Clone)] +enum EdgeCandidateMaterialization { + Precomputed(Vec), + EdgeLabelIndex { + label_id: u32, + }, + EdgeTripleIndex { + from: u64, + to: u64, + label_id: u32, + }, + FromEndpointAdjacency { + node_ids: Vec, + label_filter_ids: Option>, + }, + ToEndpointAdjacency { + node_ids: Vec, + label_filter_ids: Option>, + }, + AnyEndpointAdjacency { + node_ids: Vec, + label_filter_ids: Option>, + }, + EdgeWeightIndex { + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + }, + EdgeUpdatedAtIndex { + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + }, + EdgeValidFromIndex { + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + }, + EdgeValidToIndex { + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + }, + EdgePropertyEqualityIndex { + index_id: u64, + label_id: u32, + prop_key: String, + value: PropValue, + value_hashes: Vec, + }, + EdgePropertyRangeIndex { + index_id: u64, + label_id: u32, + prop_key: String, + domain: SecondaryIndexRangeDomain, + lower: Option, + upper: Option, + }, + FallbackFullEdgeScan, +} + struct CandidateProbe { source: Option, warning: Option, followup: Option, } +struct EdgeCandidateProbe { + source: Option, + warning: Option, + followup: Option, +} + +#[allow(clippy::large_enum_variant)] +enum EdgeBooleanPlanClassification { + AlwaysFalse, + VerifyOnly, + Bounded { + plan: EdgePhysicalPlan, + estimate: PlannerEstimate, + structural_key: Vec, + complete: bool, + }, +} + +struct EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification, + has_verify_only: bool, +} + #[derive(Clone, Debug, PartialEq)] struct InProbeValue { value: PropValue, @@ -91,6 +296,11 @@ struct QueryCapContext { cheapest_legal_universe: Option, } +#[derive(Clone, Copy, Debug, Default)] +struct EdgeQueryCapContext { + cheapest_legal_universe: Option, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum FanoutCoverage { Complete, @@ -196,11 +406,23 @@ struct PlannerEstimate { current_posting_bound: bool, } +#[allow(dead_code)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct NodeLabelMembershipEstimate { + estimate: PlannerEstimate, + driver_label_id: Option, +} + #[derive(Clone)] enum NodeCandidateMaterialization { Precomputed(Vec), KeyLookup, - NodeTypeIndex, + NodeLabelIndex { + label_id: u32, + }, + NodeLabelAny { + label_ids: NodeLabelSet, + }, PropertyEqualityIndex { index_id: u64, key: String, @@ -213,11 +435,13 @@ enum NodeCandidateMaterialization { upper: Option, }, TimestampIndex { - type_id: u32, + label_id: u32, lower_ms: i64, upper_ms: i64, }, - FallbackTypeScan, + FallbackNodeLabelScan { + label_id: u32, + }, FallbackFullNodeScan, } @@ -248,6 +472,8 @@ fn plan_warning_rank(warning: QueryPlanWarning) -> usize { QueryPlanWarning::VerifyOnlyFilter => 10, QueryPlanWarning::BooleanBranchFallback => 11, QueryPlanWarning::PlanningProbeBudgetExceeded => 12, + QueryPlanWarning::UnknownNodeLabel => 13, + QueryPlanWarning::UnknownEdgeLabel => 14, } } @@ -268,11 +494,31 @@ fn explicit_anchor_universe_count(query: &NormalizedNodeQuery) -> Option { count } +fn node_index_candidate_labels(query: &NormalizedNodeQuery) -> Option { + if let Some(single_label_id) = query.single_label_id { + return NodeLabelSet::single(single_label_id).ok(); + } + + match query.label_filter { + ResolvedNodeLabelFilter::LabelSet { + mode: LabelMatchMode::All, + label_ids, + .. + } => Some(label_ids), + ResolvedNodeLabelFilter::Unconstrained + | ResolvedNodeLabelFilter::Empty { .. } + | ResolvedNodeLabelFilter::LabelSet { + mode: LabelMatchMode::Any, + .. + } => None, + } +} + fn should_skip_filter_planning_for_explicit_anchor(query: &NormalizedNodeQuery) -> bool { let Some(count) = explicit_anchor_universe_count(query) else { return false; }; - query.type_id.is_none() || count <= TINY_EXPLICIT_ANCHOR_MAX + node_index_candidate_labels(query).is_none() || count <= TINY_EXPLICIT_ANCHOR_MAX } fn filter_has_intrinsic_verify_only(filter: &NormalizedNodeFilter) -> bool { @@ -360,6 +606,14 @@ fn weaker_confidence(left: EstimateConfidence, right: EstimateConfidence) -> Est } } +fn higher_stale_posting_risk(left: StalePostingRisk, right: StalePostingRisk) -> StalePostingRisk { + if left.rank() >= right.rank() { + left + } else { + right + } +} + fn higher_hub_risk(left: FanoutHubRisk, right: FanoutHubRisk) -> FanoutHubRisk { if left.rank() >= right.rank() { left @@ -560,6 +814,67 @@ fn adaptive_union_total_cap( .min(crate::planner_stats::PLANNER_STATS_HARD_CANDIDATE_CAP) } +fn adaptive_edge_candidate_cap( + source_kind: EdgeQueryCandidateSourceKind, + query_limit: Option, + cheapest_legal_universe: Option, + source_estimate: PlannerEstimate, +) -> usize { + let default_cap = crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP; + let hard_cap = crate::planner_stats::PLANNER_STATS_HARD_CANDIDATE_CAP; + let Some(source_count) = source_estimate.known_upper_bound() else { + return default_cap.min(hard_cap); + }; + + let legal_count = cheapest_legal_universe.and_then(PlannerEstimate::known_upper_bound); + if legal_count.is_some_and(|legal_count| source_count >= legal_count) { + return default_cap.min(hard_cap); + } + + let high_confidence = matches!( + source_estimate.confidence, + EstimateConfidence::Exact | EstimateConfidence::High + ) && !matches!(source_estimate.stale_risk, StalePostingRisk::High); + + let mut cap = default_cap.min(hard_cap); + if high_confidence { + if let Some(limit) = query_limit.filter(|limit| *limit > 0) { + let proof_cap = limit + .saturating_add(1) + .saturating_mul(64) + .max(default_cap); + cap = cap.max(proof_cap.min(hard_cap)); + } + if matches!( + source_kind, + EdgeQueryCandidateSourceKind::EdgeWeightIndex + | EdgeQueryCandidateSourceKind::EdgeUpdatedAtIndex + | EdgeQueryCandidateSourceKind::EdgeValidFromIndex + | EdgeQueryCandidateSourceKind::EdgeValidToIndex + | EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex + | EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex + ) { + cap = cap.max((source_count.min(hard_cap as u64)) as usize); + } + } + cap.min(hard_cap) +} + +fn adaptive_edge_union_total_cap( + query_limit: Option, + cheapest_legal_universe: Option, + union_estimate: PlannerEstimate, +) -> usize { + adaptive_edge_candidate_cap( + EdgeQueryCandidateSourceKind::EdgeWeightIndex, + query_limit, + cheapest_legal_universe, + union_estimate, + ) + .saturating_mul(2) + .min(crate::planner_stats::PLANNER_STATS_HARD_CANDIDATE_CAP) +} + impl QueryCapContext { fn source_cap( self, @@ -601,6 +916,51 @@ impl QueryCapContext { } } +impl EdgeQueryCapContext { + fn source_cap( + self, + source_kind: EdgeQueryCandidateSourceKind, + query_limit: Option, + source_estimate: PlannerEstimate, + ) -> usize { + adaptive_edge_candidate_cap( + source_kind, + query_limit, + self.cheapest_legal_universe, + source_estimate, + ) + } + + fn source_estimate_exceeds_cap( + self, + source_kind: EdgeQueryCandidateSourceKind, + query_limit: Option, + source_estimate: PlannerEstimate, + ) -> bool { + let Some(count) = source_estimate.known_upper_bound() else { + return true; + }; + count > self.source_cap(source_kind, query_limit, source_estimate) as u64 + } + + fn union_total_cap( + self, + query_limit: Option, + union_estimate: PlannerEstimate, + ) -> usize { + adaptive_edge_union_total_cap( + query_limit, + self.cheapest_legal_universe, + union_estimate, + ) + } + + fn cheapest_legal_count(self) -> Option { + self.cheapest_legal_universe + .and_then(PlannerEstimate::known_upper_bound) + } +} + fn cap_warning_for_source(kind: NodeQueryCandidateSourceKind) -> QueryPlanWarning { match kind { NodeQueryCandidateSourceKind::PropertyRangeIndex => { @@ -611,18 +971,69 @@ fn cap_warning_for_source(kind: NodeQueryCandidateSourceKind) -> QueryPlanWarnin } } +fn edge_cap_warning_for_source(kind: EdgeQueryCandidateSourceKind) -> QueryPlanWarning { + match kind { + EdgeQueryCandidateSourceKind::EdgeWeightIndex + | EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex => { + QueryPlanWarning::RangeCandidateCapExceeded + } + EdgeQueryCandidateSourceKind::EdgeUpdatedAtIndex + | EdgeQueryCandidateSourceKind::EdgeValidFromIndex + | EdgeQueryCandidateSourceKind::EdgeValidToIndex => { + QueryPlanWarning::TimestampCandidateCapExceeded + } + _ => QueryPlanWarning::CandidateCapExceeded, + } +} + +fn memtable_secondary_eq_edge_count_for_filter( + memtable: &Memtable, + index_id: u64, + prop_key: &str, + prop_value: &PropValue, + snapshot_seq: u64, +) -> usize { + let Some(probe_values) = signed_zero_equality_probe_values(prop_value) else { + return memtable.secondary_eq_edge_count_at(index_id, prop_key, prop_value, snapshot_seq); + }; + + probe_values + .iter() + .map(|probe_value| { + memtable.secondary_eq_edge_count_at(index_id, prop_key, probe_value, snapshot_seq) + }) + .sum() +} + +fn segment_edge_secondary_eq_posting_count_for_filter( + segment: &SegmentReader, + index_id: u64, + prop_value: &PropValue, +) -> Result, EngineError> { + let mut count = 0usize; + for value_hash in equality_probe_value_hashes(prop_value) { + let Some(probe_count) = + segment.edge_secondary_eq_posting_count_if_present(index_id, value_hash)? + else { + return Ok(None); + }; + count = count.saturating_add(probe_count); + } + Ok(Some(count)) +} + impl NodeQueryCandidateSourceKind { fn plan_node(self) -> QueryPlanNode { match self { NodeQueryCandidateSourceKind::ExplicitIds => QueryPlanNode::ExplicitIds, NodeQueryCandidateSourceKind::KeyLookup => QueryPlanNode::KeyLookup, - NodeQueryCandidateSourceKind::NodeTypeIndex => QueryPlanNode::NodeTypeIndex, + NodeQueryCandidateSourceKind::NodeLabelIndex => QueryPlanNode::NodeLabelIndex, NodeQueryCandidateSourceKind::PropertyEqualityIndex => { QueryPlanNode::PropertyEqualityIndex } NodeQueryCandidateSourceKind::PropertyRangeIndex => QueryPlanNode::PropertyRangeIndex, NodeQueryCandidateSourceKind::TimestampIndex => QueryPlanNode::TimestampIndex, - NodeQueryCandidateSourceKind::FallbackTypeScan => QueryPlanNode::FallbackTypeScan, + NodeQueryCandidateSourceKind::FallbackNodeLabelScan => QueryPlanNode::FallbackNodeLabelScan, NodeQueryCandidateSourceKind::FallbackFullNodeScan => QueryPlanNode::FallbackFullNodeScan, } } @@ -634,8 +1045,8 @@ impl NodeQueryCandidateSourceKind { NodeQueryCandidateSourceKind::PropertyEqualityIndex => 2, NodeQueryCandidateSourceKind::PropertyRangeIndex => 3, NodeQueryCandidateSourceKind::TimestampIndex => 4, - NodeQueryCandidateSourceKind::NodeTypeIndex => 5, - NodeQueryCandidateSourceKind::FallbackTypeScan => 6, + NodeQueryCandidateSourceKind::NodeLabelIndex => 5, + NodeQueryCandidateSourceKind::FallbackNodeLabelScan => 6, NodeQueryCandidateSourceKind::FallbackFullNodeScan => 7, } } @@ -662,21 +1073,30 @@ impl PlannedNodeCandidateSource { } } - fn node_type_index(type_id: u32, estimate: PlannerEstimate) -> Self { + fn node_label_index(label_id: u32, estimate: PlannerEstimate) -> Self { + Self { + kind: NodeQueryCandidateSourceKind::NodeLabelIndex, + canonical_key: format!("label:{label_id}"), + estimate, + materialization: NodeCandidateMaterialization::NodeLabelIndex { label_id }, + } + } + + fn node_label_any_index(label_ids: NodeLabelSet, estimate: PlannerEstimate) -> Self { Self { - kind: NodeQueryCandidateSourceKind::NodeTypeIndex, - canonical_key: format!("type:{type_id}"), + kind: NodeQueryCandidateSourceKind::NodeLabelIndex, + canonical_key: format!("label_any:{:?}", label_ids.as_slice()), estimate, - materialization: NodeCandidateMaterialization::NodeTypeIndex, + materialization: NodeCandidateMaterialization::NodeLabelAny { label_ids }, } } - fn fallback_type_scan(type_id: u32, estimate: PlannerEstimate) -> Self { + fn fallback_node_label_scan(label_id: u32, estimate: PlannerEstimate) -> Self { Self { - kind: NodeQueryCandidateSourceKind::FallbackTypeScan, - canonical_key: format!("fallback_type:{type_id}"), + kind: NodeQueryCandidateSourceKind::FallbackNodeLabelScan, + canonical_key: format!("fallback_label:{label_id}"), estimate, - materialization: NodeCandidateMaterialization::FallbackTypeScan, + materialization: NodeCandidateMaterialization::FallbackNodeLabelScan { label_id }, } } @@ -690,14 +1110,14 @@ impl PlannedNodeCandidateSource { } fn property_equality_index( - type_id: u32, + label_id: u32, index_id: u64, key: &str, value: &PropValue, estimate: PlannerEstimate, ) -> Self { Self::property_equality_index_with_hash( - type_id, + label_id, index_id, key, value, @@ -707,7 +1127,7 @@ impl PlannedNodeCandidateSource { } fn property_equality_index_with_hash( - type_id: u32, + label_id: u32, index_id: u64, key: &str, value: &PropValue, @@ -716,7 +1136,7 @@ impl PlannedNodeCandidateSource { ) -> Self { Self { kind: NodeQueryCandidateSourceKind::PropertyEqualityIndex, - canonical_key: format!("eq:{type_id}:{key}:{value_hash}"), + canonical_key: format!("eq:{label_id}:{key}:{value_hash}"), estimate, materialization: NodeCandidateMaterialization::PropertyEqualityIndex { index_id, @@ -748,17 +1168,17 @@ impl PlannedNodeCandidateSource { } fn timestamp_index( - type_id: u32, + label_id: u32, lower_ms: i64, upper_ms: i64, estimate: PlannerEstimate, ) -> Self { Self { kind: NodeQueryCandidateSourceKind::TimestampIndex, - canonical_key: format!("time:{type_id}:{lower_ms}:{upper_ms}"), + canonical_key: format!("time:{label_id}:{lower_ms}:{upper_ms}"), estimate, materialization: NodeCandidateMaterialization::TimestampIndex { - type_id, + label_id, lower_ms, upper_ms, }, @@ -766,7 +1186,10 @@ impl PlannedNodeCandidateSource { } fn plan_node(&self) -> QueryPlanNode { - self.kind.plan_node() + match self.materialization { + NodeCandidateMaterialization::NodeLabelAny { .. } => QueryPlanNode::NodeLabelAnyIndex, + _ => self.kind.plan_node(), + } } fn broad_skip_warnable(&self) -> bool { @@ -779,46 +1202,304 @@ impl PlannedNodeCandidateSource { } } -impl NodePhysicalPlan { - fn source(source: PlannedNodeCandidateSource) -> Self { - Self::Source(source) +impl PlannedEdgeCandidateSource { + fn with_ids(kind: EdgeQueryCandidateSourceKind, canonical_key: String, ids: Vec) -> Self { + let ids = normalize_candidate_ids(ids); + let estimate = PlannerEstimate::exact_cheap(ids.len() as u64); + Self { + kind, + canonical_key, + estimate, + materialization: EdgeCandidateMaterialization::Precomputed(ids), + } } - fn intersect(inputs: Vec) -> Self { - let mut flattened = Vec::new(); - for input in inputs { - match input { - NodePhysicalPlan::Empty => return NodePhysicalPlan::Empty, - NodePhysicalPlan::Intersect(children) => flattened.extend(children), - plan => flattened.push(plan), - } - } - match flattened.len() { - 0 => NodePhysicalPlan::Empty, - 1 => flattened.into_iter().next().unwrap(), - _ => NodePhysicalPlan::Intersect(flattened), + fn edge_label_index(label_id: u32, estimate: PlannerEstimate) -> Self { + Self { + kind: EdgeQueryCandidateSourceKind::EdgeLabelIndex, + canonical_key: format!("label:{label_id}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgeLabelIndex { label_id }, } } - fn union(inputs: Vec) -> Self { - let mut flattened = Vec::new(); - for input in inputs { - match input { - NodePhysicalPlan::Empty => {} - NodePhysicalPlan::Union(children) => flattened.extend(children), - plan => flattened.push(plan), - } - } - match flattened.len() { - 0 => NodePhysicalPlan::Empty, - 1 => flattened.into_iter().next().unwrap(), - _ => NodePhysicalPlan::Union(flattened), + fn edge_triple_index(from: u64, to: u64, label_id: u32) -> Self { + Self { + kind: EdgeQueryCandidateSourceKind::EdgeTripleIndex, + canonical_key: format!("edge_triple:{from}:{to}:{label_id}"), + estimate: PlannerEstimate::unknown(), + materialization: EdgeCandidateMaterialization::EdgeTripleIndex { from, to, label_id }, } } - fn plan_node(&self) -> QueryPlanNode { - match self { - NodePhysicalPlan::Empty => QueryPlanNode::EmptyResult, + fn endpoint_adjacency( + kind: EdgeQueryCandidateSourceKind, + node_ids: Vec, + label_filter_ids: Option>, + estimate: PlannerEstimate, + ) -> Self { + let canonical_key = format!("{kind:?}:{node_ids:?}:{label_filter_ids:?}"); + let materialization = match kind { + EdgeQueryCandidateSourceKind::FromEndpointAdjacency => { + EdgeCandidateMaterialization::FromEndpointAdjacency { + node_ids, + label_filter_ids, + } + } + EdgeQueryCandidateSourceKind::ToEndpointAdjacency => { + EdgeCandidateMaterialization::ToEndpointAdjacency { + node_ids, + label_filter_ids, + } + } + EdgeQueryCandidateSourceKind::AnyEndpointAdjacency => { + EdgeCandidateMaterialization::AnyEndpointAdjacency { + node_ids, + label_filter_ids, + } + } + _ => unreachable!("endpoint source kind required"), + }; + Self { + kind, + canonical_key, + estimate, + materialization, + } + } + + fn edge_weight_index( + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + indexed: bool, + estimate: PlannerEstimate, + ) -> Self { + Self { + kind: if indexed { + EdgeQueryCandidateSourceKind::EdgeWeightIndex + } else { + EdgeQueryCandidateSourceKind::EdgeMetadataScan + }, + canonical_key: format!("edge_weight:{label_id:?}:{bounds:?}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgeWeightIndex { label_id, bounds }, + } + } + + fn edge_updated_at_index( + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + indexed: bool, + estimate: PlannerEstimate, + ) -> Self { + Self { + kind: if indexed { + EdgeQueryCandidateSourceKind::EdgeUpdatedAtIndex + } else { + EdgeQueryCandidateSourceKind::EdgeMetadataScan + }, + canonical_key: format!("edge_updated_at:{label_id:?}:{bounds:?}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgeUpdatedAtIndex { label_id, bounds }, + } + } + + fn edge_valid_from_index( + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + indexed: bool, + estimate: PlannerEstimate, + ) -> Self { + Self { + kind: if indexed { + EdgeQueryCandidateSourceKind::EdgeValidFromIndex + } else { + EdgeQueryCandidateSourceKind::EdgeMetadataScan + }, + canonical_key: format!("edge_valid_from:{label_id:?}:{bounds:?}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgeValidFromIndex { label_id, bounds }, + } + } + + fn edge_valid_to_index( + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + indexed: bool, + estimate: PlannerEstimate, + ) -> Self { + Self { + kind: if indexed { + EdgeQueryCandidateSourceKind::EdgeValidToIndex + } else { + EdgeQueryCandidateSourceKind::EdgeMetadataScan + }, + canonical_key: format!("edge_valid_to:{label_id:?}:{bounds:?}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgeValidToIndex { label_id, bounds }, + } + } + + fn edge_property_equality_index( + label_id: u32, + index_id: u64, + prop_key: &str, + value: &PropValue, + estimate: PlannerEstimate, + ) -> Self { + let value_hashes = equality_probe_value_hashes(value); + Self { + kind: EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex, + canonical_key: format!("edge_prop_eq:{label_id}:{prop_key}:{value_hashes:?}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgePropertyEqualityIndex { + index_id, + label_id, + prop_key: prop_key.to_string(), + value: value.clone(), + value_hashes, + }, + } + } + + fn edge_property_equality_index_with_hash( + label_id: u32, + index_id: u64, + prop_key: &str, + value: &PropValue, + value_hash: u64, + estimate: PlannerEstimate, + ) -> Self { + Self { + kind: EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex, + canonical_key: format!("edge_prop_eq:{label_id}:{prop_key}:{value_hash}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgePropertyEqualityIndex { + index_id, + label_id, + prop_key: prop_key.to_string(), + value: value.clone(), + value_hashes: vec![value_hash], + }, + } + } + + fn edge_property_range_index( + label_id: u32, + index_id: u64, + prop_key: &str, + domain: SecondaryIndexRangeDomain, + lower: Option<&PropertyRangeBound>, + upper: Option<&PropertyRangeBound>, + estimate: PlannerEstimate, + ) -> Self { + Self { + kind: EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex, + canonical_key: format!("edge_prop_range:{label_id}:{index_id}:{prop_key}:{lower:?}:{upper:?}"), + estimate, + materialization: EdgeCandidateMaterialization::EdgePropertyRangeIndex { + index_id, + label_id, + prop_key: prop_key.to_string(), + domain, + lower: lower.cloned(), + upper: upper.cloned(), + }, + } + } + + fn fallback_full_scan(estimate: PlannerEstimate) -> Self { + Self { + kind: EdgeQueryCandidateSourceKind::FallbackFullEdgeScan, + canonical_key: "fallback_full_edge_scan".to_string(), + estimate, + materialization: EdgeCandidateMaterialization::FallbackFullEdgeScan, + } + } + + fn plan_node(&self) -> QueryPlanNode { + self.kind.plan_node() + } + + fn broad_skip_warnable(&self) -> bool { + matches!( + self.kind, + EdgeQueryCandidateSourceKind::EdgeLabelIndex + | EdgeQueryCandidateSourceKind::FromEndpointAdjacency + | EdgeQueryCandidateSourceKind::ToEndpointAdjacency + | EdgeQueryCandidateSourceKind::AnyEndpointAdjacency + | EdgeQueryCandidateSourceKind::EdgeWeightIndex + | EdgeQueryCandidateSourceKind::EdgeUpdatedAtIndex + | EdgeQueryCandidateSourceKind::EdgeValidFromIndex + | EdgeQueryCandidateSourceKind::EdgeValidToIndex + | EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex + | EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex + | EdgeQueryCandidateSourceKind::EdgeMetadataScan + ) + } + + fn estimated_work(&self) -> u64 { + let count = self.estimate.known_upper_bound().unwrap_or(u64::MAX); + let (setup, candidate_weight) = match self.kind { + EdgeQueryCandidateSourceKind::ExplicitEdgeIds => (1u64, 1u64), + EdgeQueryCandidateSourceKind::EdgeTripleIndex => (2, 1), + EdgeQueryCandidateSourceKind::FromEndpointAdjacency + | EdgeQueryCandidateSourceKind::ToEndpointAdjacency + | EdgeQueryCandidateSourceKind::AnyEndpointAdjacency => (8, 2), + EdgeQueryCandidateSourceKind::EdgeWeightIndex + | EdgeQueryCandidateSourceKind::EdgeUpdatedAtIndex + | EdgeQueryCandidateSourceKind::EdgeValidFromIndex + | EdgeQueryCandidateSourceKind::EdgeValidToIndex + | EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex => (12, 2), + EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex => (8, 2), + EdgeQueryCandidateSourceKind::EdgeLabelIndex + | EdgeQueryCandidateSourceKind::EdgeMetadataScan => (24, 3), + EdgeQueryCandidateSourceKind::FallbackFullEdgeScan => (64, 4), + }; + setup.saturating_add(count.saturating_mul(candidate_weight)) + } +} + +impl NodePhysicalPlan { + fn source(source: PlannedNodeCandidateSource) -> Self { + Self::Source(source) + } + + fn intersect(inputs: Vec) -> Self { + let mut flattened = Vec::new(); + for input in inputs { + match input { + NodePhysicalPlan::Empty => return NodePhysicalPlan::Empty, + NodePhysicalPlan::Intersect(children) => flattened.extend(children), + plan => flattened.push(plan), + } + } + match flattened.len() { + 0 => NodePhysicalPlan::Empty, + 1 => flattened.into_iter().next().unwrap(), + _ => NodePhysicalPlan::Intersect(flattened), + } + } + + fn union(inputs: Vec) -> Self { + let mut flattened = Vec::new(); + for input in inputs { + match input { + NodePhysicalPlan::Empty => {} + NodePhysicalPlan::Union(children) => flattened.extend(children), + plan => flattened.push(plan), + } + } + match flattened.len() { + 0 => NodePhysicalPlan::Empty, + 1 => flattened.into_iter().next().unwrap(), + _ => NodePhysicalPlan::Union(flattened), + } + } + + fn plan_node(&self) -> QueryPlanNode { + match self { + NodePhysicalPlan::Empty => QueryPlanNode::EmptyResult, NodePhysicalPlan::Source(source) => source.plan_node(), NodePhysicalPlan::Intersect(inputs) => QueryPlanNode::Intersect { inputs: inputs.iter().map(NodePhysicalPlan::plan_node).collect(), @@ -939,95 +1620,358 @@ impl NodePhysicalPlan { } } } -} - -impl Ord for PlanCost { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.estimated_work - .cmp(&other.estimated_work) - .then_with(|| { - self.estimated_candidates - .unwrap_or(u64::MAX) - .cmp(&other.estimated_candidates.unwrap_or(u64::MAX)) - }) - .then_with(|| self.estimate_kind_rank.cmp(&other.estimate_kind_rank)) - .then_with(|| self.confidence_rank.cmp(&other.confidence_rank)) - .then_with(|| self.stale_risk_rank.cmp(&other.stale_risk_rank)) - .then_with(|| self.materialization_rank.cmp(&other.materialization_rank)) - .then_with(|| self.source_rank.cmp(&other.source_rank)) - .then_with(|| self.canonical_key.cmp(&other.canonical_key)) - } -} - -impl PartialOrd for PlanCost { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} -impl PlanMaterializationClass { - fn rank(self) -> u8 { + fn uses_label_postings(&self) -> bool { match self { - PlanMaterializationClass::Empty => 0, - PlanMaterializationClass::Precomputed => 1, - PlanMaterializationClass::KeyLookup => 2, - PlanMaterializationClass::EagerIndex => 3, - PlanMaterializationClass::Compound => 4, - PlanMaterializationClass::StreamingLegalUniverse => 5, + NodePhysicalPlan::Source(source) => matches!( + source.materialization, + NodeCandidateMaterialization::NodeLabelIndex { .. } + | NodeCandidateMaterialization::NodeLabelAny { .. } + | NodeCandidateMaterialization::FallbackNodeLabelScan { .. } + ), + NodePhysicalPlan::Intersect(inputs) | NodePhysicalPlan::Union(inputs) => { + inputs.iter().any(NodePhysicalPlan::uses_label_postings) + } + NodePhysicalPlan::Empty => false, } } -} -impl NodeCandidateMaterialization { - fn materialization_class(&self) -> PlanMaterializationClass { + fn uses_label_any_union(&self) -> bool { match self { - NodeCandidateMaterialization::Precomputed(_) => PlanMaterializationClass::Precomputed, - NodeCandidateMaterialization::KeyLookup => PlanMaterializationClass::KeyLookup, - NodeCandidateMaterialization::PropertyEqualityIndex { .. } - | NodeCandidateMaterialization::PropertyRangeIndex { .. } - | NodeCandidateMaterialization::TimestampIndex { .. } => { - PlanMaterializationClass::EagerIndex - } - NodeCandidateMaterialization::NodeTypeIndex - | NodeCandidateMaterialization::FallbackTypeScan - | NodeCandidateMaterialization::FallbackFullNodeScan => { - PlanMaterializationClass::StreamingLegalUniverse + NodePhysicalPlan::Source(source) => matches!( + source.materialization, + NodeCandidateMaterialization::NodeLabelAny { .. } + ), + NodePhysicalPlan::Intersect(inputs) | NodePhysicalPlan::Union(inputs) => { + inputs.iter().any(NodePhysicalPlan::uses_label_any_union) } + NodePhysicalPlan::Empty => false, } } } -impl PlannerEstimate { - fn exact_cheap(count: u64) -> Self { - Self { - count: Some(count), - kind: PlannerEstimateKind::ExactCheap, - confidence: EstimateConfidence::Exact, - stale_risk: StalePostingRisk::Low, - proves_empty: count == 0, - current_posting_bound: false, +impl EdgePhysicalPlan { + fn source(source: PlannedEdgeCandidateSource) -> Self { + Self::Source(source) + } + + fn intersect(inputs: Vec) -> Self { + let mut flattened = Vec::new(); + for input in inputs { + match input { + EdgePhysicalPlan::Empty => return EdgePhysicalPlan::Empty, + EdgePhysicalPlan::Intersect(children) => flattened.extend(children), + plan => flattened.push(plan), + } + } + match flattened.len() { + 0 => EdgePhysicalPlan::Empty, + 1 => flattened.into_iter().next().unwrap(), + _ => EdgePhysicalPlan::Intersect(flattened), } } - fn stats_exact(count: u64) -> Self { - Self { - count: Some(count), - kind: PlannerEstimateKind::StatsExact, - confidence: EstimateConfidence::Exact, - stale_risk: StalePostingRisk::Low, - proves_empty: false, - current_posting_bound: false, + fn union(inputs: Vec) -> Self { + let mut flattened = Vec::new(); + for input in inputs { + match input { + EdgePhysicalPlan::Empty => {} + EdgePhysicalPlan::Union(children) => flattened.extend(children), + plan => flattened.push(plan), + } + } + match flattened.len() { + 0 => EdgePhysicalPlan::Empty, + 1 => flattened.into_iter().next().unwrap(), + _ => EdgePhysicalPlan::Union(flattened), } } - fn stats_estimated(count: u64, confidence: EstimateConfidence, stale_risk: StalePostingRisk) -> Self { - Self { - count: Some(count), - kind: PlannerEstimateKind::StatsEstimated, - confidence, - stale_risk, - proves_empty: false, - current_posting_bound: false, + fn plan_node(&self) -> QueryPlanNode { + match self { + EdgePhysicalPlan::Empty => QueryPlanNode::EmptyResult, + EdgePhysicalPlan::Source(source) => source.plan_node(), + EdgePhysicalPlan::Intersect(inputs) => QueryPlanNode::Intersect { + inputs: inputs.iter().map(EdgePhysicalPlan::plan_node).collect(), + }, + EdgePhysicalPlan::Union(inputs) => QueryPlanNode::Union { + inputs: inputs.iter().map(EdgePhysicalPlan::plan_node).collect(), + }, + } + } + + fn estimate(&self) -> PlannerEstimate { + match self { + EdgePhysicalPlan::Empty => PlannerEstimate::exact_cheap(0), + EdgePhysicalPlan::Source(source) => source.estimate, + EdgePhysicalPlan::Intersect(inputs) => inputs + .iter() + .map(EdgePhysicalPlan::estimate) + .filter_map(PlannerEstimate::known_upper_bound) + .min() + .map(PlannerEstimate::upper_bound) + .unwrap_or_else(PlannerEstimate::unknown), + EdgePhysicalPlan::Union(inputs) => { + let mut total = 0u64; + for input in inputs { + let Some(count) = input.estimate().known_upper_bound() else { + return PlannerEstimate::unknown(); + }; + total = total.saturating_add(count); + } + PlannerEstimate::upper_bound(total) + } + } + } + + fn cap_source_kind(&self) -> EdgeQueryCandidateSourceKind { + match self { + EdgePhysicalPlan::Empty => EdgeQueryCandidateSourceKind::ExplicitEdgeIds, + EdgePhysicalPlan::Source(source) => source.kind, + EdgePhysicalPlan::Intersect(inputs) => inputs + .iter() + .min_by(|left, right| left.plan_cost().cmp(&right.plan_cost())) + .map(EdgePhysicalPlan::cap_source_kind) + .unwrap_or(EdgeQueryCandidateSourceKind::EdgeMetadataScan), + EdgePhysicalPlan::Union(inputs) => { + let mut kinds = inputs.iter().map(EdgePhysicalPlan::cap_source_kind); + let Some(first) = kinds.next() else { + return EdgeQueryCandidateSourceKind::EdgeMetadataScan; + }; + if kinds.all(|kind| kind == first) { + first + } else { + EdgeQueryCandidateSourceKind::EdgeMetadataScan + } + } + } + } + + fn materialization_cap( + &self, + cap_context: EdgeQueryCapContext, + query_limit: Option, + ) -> usize { + let estimate = self.estimate(); + match self { + EdgePhysicalPlan::Union(_) => cap_context.union_total_cap(query_limit, estimate), + _ => cap_context.source_cap(self.cap_source_kind(), query_limit, estimate), + } + } + + fn estimate_exceeds_cap( + &self, + cap_context: EdgeQueryCapContext, + query_limit: Option, + ) -> bool { + let estimate = self.estimate(); + let Some(count) = estimate.known_upper_bound() else { + return true; + }; + count > self.materialization_cap(cap_context, query_limit) as u64 + } + + fn canonical_key(&self) -> String { + match self { + EdgePhysicalPlan::Empty => "edge_empty".to_string(), + EdgePhysicalPlan::Source(source) => source.canonical_key.clone(), + EdgePhysicalPlan::Intersect(inputs) => { + let mut key = String::from("edge_and:"); + for input in inputs { + key.push_str(&input.canonical_key()); + key.push('|'); + } + key + } + EdgePhysicalPlan::Union(inputs) => { + let mut key = String::from("edge_or:"); + for input in inputs { + key.push_str(&input.canonical_key()); + key.push('|'); + } + key + } + } + } + + fn source_rank(&self) -> usize { + match self { + EdgePhysicalPlan::Empty => 0, + EdgePhysicalPlan::Source(source) => source.kind.source_rank(), + EdgePhysicalPlan::Intersect(inputs) => inputs + .iter() + .map(EdgePhysicalPlan::source_rank) + .min() + .unwrap_or(usize::MAX), + EdgePhysicalPlan::Union(_) => 4, + } + } + + fn materialization_class(&self) -> PlanMaterializationClass { + match self { + EdgePhysicalPlan::Empty => PlanMaterializationClass::Empty, + EdgePhysicalPlan::Source(source) => source.materialization.materialization_class(), + EdgePhysicalPlan::Intersect(_) | EdgePhysicalPlan::Union(_) => { + PlanMaterializationClass::Compound + } + } + } + + fn plan_cost(&self) -> PlanCost { + let estimate = self.estimate(); + let estimated_candidates = estimate.known_upper_bound(); + let base_work = match self { + EdgePhysicalPlan::Empty => 0, + EdgePhysicalPlan::Source(source) => source.estimated_work(), + EdgePhysicalPlan::Intersect(inputs) | EdgePhysicalPlan::Union(inputs) => { + inputs.iter().fold(16u64, |total, input| { + total.saturating_add( + input + .estimate() + .known_upper_bound() + .unwrap_or(u64::MAX / 4) + .saturating_mul(2), + ) + }) + } + }; + PlanCost { + estimated_work: estimate.apply_cost_penalties(base_work), + estimated_candidates, + estimate_kind_rank: estimate.kind.rank(), + confidence_rank: estimate.confidence.rank(), + stale_risk_rank: estimate.stale_risk.rank(), + materialization_rank: self.materialization_class().rank(), + source_rank: self.source_rank(), + canonical_key: self.canonical_key(), + } + } + + fn broad_skip_warnable(&self) -> bool { + match self { + EdgePhysicalPlan::Empty => false, + EdgePhysicalPlan::Source(source) => source.broad_skip_warnable(), + EdgePhysicalPlan::Intersect(inputs) | EdgePhysicalPlan::Union(inputs) => { + inputs.iter().any(EdgePhysicalPlan::broad_skip_warnable) + } + } + } +} + +impl Ord for PlanCost { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.estimated_work + .cmp(&other.estimated_work) + .then_with(|| { + self.estimated_candidates + .unwrap_or(u64::MAX) + .cmp(&other.estimated_candidates.unwrap_or(u64::MAX)) + }) + .then_with(|| self.estimate_kind_rank.cmp(&other.estimate_kind_rank)) + .then_with(|| self.confidence_rank.cmp(&other.confidence_rank)) + .then_with(|| self.stale_risk_rank.cmp(&other.stale_risk_rank)) + .then_with(|| self.materialization_rank.cmp(&other.materialization_rank)) + .then_with(|| self.source_rank.cmp(&other.source_rank)) + .then_with(|| self.canonical_key.cmp(&other.canonical_key)) + } +} + +impl PartialOrd for PlanCost { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PlanMaterializationClass { + fn rank(self) -> u8 { + match self { + PlanMaterializationClass::Empty => 0, + PlanMaterializationClass::Precomputed => 1, + PlanMaterializationClass::KeyLookup => 2, + PlanMaterializationClass::EagerIndex => 3, + PlanMaterializationClass::Compound => 4, + PlanMaterializationClass::StreamingLegalUniverse => 5, + } + } +} + +impl NodeCandidateMaterialization { + fn materialization_class(&self) -> PlanMaterializationClass { + match self { + NodeCandidateMaterialization::Precomputed(_) => PlanMaterializationClass::Precomputed, + NodeCandidateMaterialization::KeyLookup => PlanMaterializationClass::KeyLookup, + NodeCandidateMaterialization::PropertyEqualityIndex { .. } + | NodeCandidateMaterialization::PropertyRangeIndex { .. } + | NodeCandidateMaterialization::TimestampIndex { .. } => { + PlanMaterializationClass::EagerIndex + } + NodeCandidateMaterialization::NodeLabelIndex { .. } + | NodeCandidateMaterialization::NodeLabelAny { .. } + | NodeCandidateMaterialization::FallbackNodeLabelScan { .. } + | NodeCandidateMaterialization::FallbackFullNodeScan => { + PlanMaterializationClass::StreamingLegalUniverse + } + } + } +} + +impl EdgeCandidateMaterialization { + fn materialization_class(&self) -> PlanMaterializationClass { + match self { + EdgeCandidateMaterialization::Precomputed(_) => PlanMaterializationClass::Precomputed, + EdgeCandidateMaterialization::EdgeTripleIndex { .. } => { + PlanMaterializationClass::KeyLookup + } + EdgeCandidateMaterialization::EdgeWeightIndex { .. } + | EdgeCandidateMaterialization::EdgeUpdatedAtIndex { .. } + | EdgeCandidateMaterialization::EdgeValidFromIndex { .. } + | EdgeCandidateMaterialization::EdgeValidToIndex { .. } + | EdgeCandidateMaterialization::EdgePropertyEqualityIndex { .. } + | EdgeCandidateMaterialization::EdgePropertyRangeIndex { .. } => { + PlanMaterializationClass::EagerIndex + } + EdgeCandidateMaterialization::EdgeLabelIndex { .. } + | EdgeCandidateMaterialization::FromEndpointAdjacency { .. } + | EdgeCandidateMaterialization::ToEndpointAdjacency { .. } + | EdgeCandidateMaterialization::AnyEndpointAdjacency { .. } + | EdgeCandidateMaterialization::FallbackFullEdgeScan => { + PlanMaterializationClass::StreamingLegalUniverse + } + } + } +} + +impl PlannerEstimate { + fn exact_cheap(count: u64) -> Self { + Self { + count: Some(count), + kind: PlannerEstimateKind::ExactCheap, + confidence: EstimateConfidence::Exact, + stale_risk: StalePostingRisk::Low, + proves_empty: count == 0, + current_posting_bound: false, + } + } + + fn stats_exact(count: u64) -> Self { + Self { + count: Some(count), + kind: PlannerEstimateKind::StatsExact, + confidence: EstimateConfidence::Exact, + stale_risk: StalePostingRisk::Low, + proves_empty: false, + current_posting_bound: false, + } + } + + fn stats_estimated(count: u64, confidence: EstimateConfidence, stale_risk: StalePostingRisk) -> Self { + Self { + count: Some(count), + kind: PlannerEstimateKind::StatsEstimated, + confidence, + stale_risk, + proves_empty: false, + current_posting_bound: false, } } @@ -1042,6 +1986,32 @@ impl PlannerEstimate { } } + fn upper_bound_with_confidence(count: u64, confidence: EstimateConfidence) -> Self { + Self { + count: Some(count), + kind: PlannerEstimateKind::UpperBound, + confidence, + stale_risk: StalePostingRisk::Unknown, + proves_empty: false, + current_posting_bound: false, + } + } + + fn upper_bound_with_quality( + count: u64, + confidence: EstimateConfidence, + stale_risk: StalePostingRisk, + ) -> Self { + Self { + count: Some(count), + kind: PlannerEstimateKind::UpperBound, + confidence, + stale_risk, + proves_empty: false, + current_posting_bound: false, + } + } + fn unknown() -> Self { Self { count: None, @@ -1087,8 +2057,8 @@ impl PlannedNodeCandidateSource { NodeQueryCandidateSourceKind::PropertyEqualityIndex => (8, 2), NodeQueryCandidateSourceKind::PropertyRangeIndex | NodeQueryCandidateSourceKind::TimestampIndex => (12, 2), - NodeQueryCandidateSourceKind::NodeTypeIndex - | NodeQueryCandidateSourceKind::FallbackTypeScan => (24, 3), + NodeQueryCandidateSourceKind::NodeLabelIndex + | NodeQueryCandidateSourceKind::FallbackNodeLabelScan => (24, 3), NodeQueryCandidateSourceKind::FallbackFullNodeScan => (64, 4), }; setup.saturating_add(count.saturating_mul(candidate_weight)) @@ -1100,7 +2070,7 @@ impl PlannedNodeQuery { self.driver.estimate().known_upper_bound() } - fn explain_plan(&self) -> QueryPlan { + fn explain_plan(&self, public_inputs: QueryPlanPublicInputs) -> QueryPlan { QueryPlan { kind: QueryPlanKind::NodeQuery, root: QueryPlanNode::VerifyNodeFilter { @@ -1108,53 +2078,323 @@ impl PlannedNodeQuery { }, estimated_candidates: self.estimated_candidate_count(), warnings: self.warnings.clone(), + notes: Vec::new(), + public_inputs, } } } -impl PlannedPatternQuery { - fn explain_plan(&self) -> QueryPlan { - let anchor_input = QueryPlanNode::VerifyNodeFilter { - input: Box::new(self.anchor_plan.driver.plan_node()), - }; - let pattern = QueryPlanNode::PatternExpand { - anchor_alias: self.anchor_alias.clone(), - input: Box::new(anchor_input), - }; - let root = if self - .warnings - .contains(&QueryPlanWarning::EdgePropertyPostFilter) - { - QueryPlanNode::VerifyEdgePredicates { - input: Box::new(pattern), - } - } else { - pattern - }; +impl PlannedEdgeQuery { + fn estimated_candidate_count(&self) -> Option { + self.driver.estimate().known_upper_bound() + } + + fn explain_plan(&self, public_inputs: QueryPlanPublicInputs) -> QueryPlan { + let input = self.driver.plan_node(); QueryPlan { - kind: QueryPlanKind::PatternQuery, - root, - estimated_candidates: self.anchor_plan.estimated_candidate_count(), + kind: QueryPlanKind::EdgeQuery, + root: QueryPlanNode::VerifyEdgeFilter { + input: Box::new(input), + }, + estimated_candidates: self.estimated_candidate_count(), warnings: self.warnings.clone(), + notes: Vec::new(), + public_inputs, } } } -impl ReadView { - fn key_lookup_candidate_ids( - &self, - query: &NormalizedNodeQuery, - ) -> Result, EngineError> { - let type_id = query - .type_id - .expect("normalized key query must have type_id"); +impl EdgeAnchorOrientationPolicy { + fn from_direction(direction: Direction) -> Self { + match direction { + Direction::Outgoing => Self::Outgoing, + Direction::Incoming => Self::Incoming, + Direction::Both => Self::Both, + } + } +} + +impl PatternAnchorPlan { + #[cfg(test)] + fn sort_anchor_alias(&self) -> &str { + match self { + PatternAnchorPlan::Node { + sort_anchor_alias, .. + } + | PatternAnchorPlan::Edge { + sort_anchor_alias, .. + } => sort_anchor_alias, + } + } + + fn expansion_order(&self) -> &[usize] { + match self { + PatternAnchorPlan::Node { + expansion_order, .. + } + | PatternAnchorPlan::Edge { + expansion_order, .. + } => expansion_order, + } + } + + fn estimated_candidate_count(&self) -> Option { + match self { + PatternAnchorPlan::Node { anchor_plan, .. } => { + anchor_plan.estimated_candidate_count() + } + PatternAnchorPlan::Edge { edge_plan, .. } => edge_plan.estimated_candidate_count(), + } + } + + fn explain_anchor_input(&self) -> QueryPlanNode { + match self { + PatternAnchorPlan::Node { + anchor_alias, + anchor_plan, + .. + } => { + let anchor_input = QueryPlanNode::VerifyNodeFilter { + input: Box::new(anchor_plan.driver.plan_node()), + }; + QueryPlanNode::PatternExpand { + anchor_alias: anchor_alias.clone(), + input: Box::new(anchor_input), + } + } + PatternAnchorPlan::Edge { + edge_alias, + edge_plan, + sort_anchor_alias, + .. + } => { + let edge_anchor = QueryPlanNode::PatternEdgeAnchor { + edge_alias: edge_alias.clone(), + input: Box::new(QueryPlanNode::VerifyEdgeFilter { + input: Box::new(edge_plan.driver.plan_node()), + }), + }; + QueryPlanNode::PatternExpand { + anchor_alias: sort_anchor_alias.clone(), + input: Box::new(edge_anchor), + } + } + } + } +} + +impl PlannedPatternQuery { + fn explain_plan(&self, public_inputs: QueryPlanPublicInputs) -> QueryPlan { + let pattern = self.anchor.explain_anchor_input(); + let root = if self + .warnings + .contains(&QueryPlanWarning::EdgePropertyPostFilter) + { + QueryPlanNode::VerifyEdgePredicates { + input: Box::new(pattern), + } + } else { + pattern + }; + QueryPlan { + kind: QueryPlanKind::PatternQuery, + root, + estimated_candidates: self.anchor.estimated_candidate_count(), + warnings: self.warnings.clone(), + notes: Vec::new(), + public_inputs, + } + } +} + +impl ReadView { + fn public_inputs_for_node_query( + &self, + query: &NodeQuery, + ) -> Result { + let mut public_inputs = QueryPlanPublicInputs::default(); + if let Some(filter) = query.label_filter.as_ref() { + for label in &filter.labels { + let known = self + .label_catalog + .resolve_node_label_for_read(label)? + .is_some(); + public_inputs.node_labels.push(QueryPlanPublicName { + alias: None, + name: label.clone(), + known, + mode: Some(filter.mode), + }); + } + } + Ok(public_inputs) + } + + fn public_inputs_for_edge_query( + &self, + query: &EdgeQuery, + ) -> Result { + let mut public_inputs = QueryPlanPublicInputs::default(); + if let Some(label) = query.label.as_ref() { + let known = self + .label_catalog + .resolve_edge_label_for_read(label)? + .is_some(); + public_inputs.edge_labels.push(QueryPlanPublicName { + alias: None, + name: label.clone(), + known, + mode: None, + }); + } + Ok(public_inputs) + } + + fn public_inputs_for_pattern_query( + &self, + query: &GraphPatternQuery, + ) -> Result { + let mut public_inputs = QueryPlanPublicInputs::default(); + + for pattern in &query.nodes { + if let Some(filter) = pattern.label_filter.as_ref() { + for label in &filter.labels { + let known = self + .label_catalog + .resolve_node_label_for_read(label)? + .is_some(); + public_inputs.node_labels.push(QueryPlanPublicName { + alias: Some(pattern.alias.clone()), + name: label.clone(), + known, + mode: Some(filter.mode), + }); + } + } + } + + for pattern in &query.edges { + for label in &pattern.label_filter { + let known = self + .label_catalog + .resolve_edge_label_for_read(label)? + .is_some(); + public_inputs.edge_labels.push(QueryPlanPublicName { + alias: pattern.alias.clone(), + name: label.clone(), + known, + mode: None, + }); + } + } + + Ok(public_inputs) + } + + fn add_node_label_filter_notes( + notes: &mut Vec, + filter: &ResolvedNodeLabelFilter, + source_plan: Option<&NodePhysicalPlan>, + ) { + let ResolvedNodeLabelFilter::LabelSet { + mode, label_ids, .. + } = filter + else { + return; + }; + let uses_label_postings = source_plan.is_some_and(NodePhysicalPlan::uses_label_postings); + let uses_label_any_union = source_plan.is_some_and(NodePhysicalPlan::uses_label_any_union); + match mode { + LabelMatchMode::Any if label_ids.len() > 1 => { + if uses_label_any_union { + notes.push(QueryPlanNote::NodeLabelAnyDedupeBeforePagination); + } + notes.push(QueryPlanNote::NodeLabelAnyFinalVerification); + if uses_label_postings { + notes.push(QueryPlanNote::StaleNodeLabelMembershipVerification); + } + } + LabelMatchMode::All if label_ids.len() > 1 => { + notes.push(QueryPlanNote::NodeLabelAllSupersetVerification); + if uses_label_postings { + notes.push(QueryPlanNote::StaleNodeLabelMembershipVerification); + } + } + _ => { + if uses_label_postings { + notes.push(QueryPlanNote::StaleNodeLabelMembershipVerification); + } + } + } + } + + fn node_query_explain_notes( + query: &NormalizedNodeQuery, + driver: &NodePhysicalPlan, + ) -> Vec { + let mut notes = Vec::new(); + Self::add_node_label_filter_notes(&mut notes, &query.label_filter, Some(driver)); + notes.sort_by_key(|note| match note { + QueryPlanNote::NodeLabelAnyDedupeBeforePagination => 0, + QueryPlanNote::NodeLabelAnyFinalVerification => 1, + QueryPlanNote::NodeLabelAllSupersetVerification => 2, + QueryPlanNote::StaleNodeLabelMembershipVerification => 3, + }); + notes.dedup(); + notes + } + + fn pattern_anchor_source_plan_for_node( + anchor: &PatternAnchorPlan, + target_node_index: usize, + ) -> Option<&NodePhysicalPlan> { + match anchor { + PatternAnchorPlan::Node { + node_index, + anchor_plan, + .. + } if *node_index == target_node_index => Some(&anchor_plan.driver), + _ => None, + } + } + + fn pattern_query_explain_notes( + query: &NormalizedGraphPatternQuery, + planned: &PlannedPatternQuery, + ) -> Vec { + let mut notes = Vec::new(); + for (node_index, node) in query.nodes.iter().enumerate() { + let source_plan = Self::pattern_anchor_source_plan_for_node(&planned.anchor, node_index); + Self::add_node_label_filter_notes( + &mut notes, + &node.query.label_filter, + source_plan, + ); + } + notes.sort_by_key(|note| match note { + QueryPlanNote::NodeLabelAnyDedupeBeforePagination => 0, + QueryPlanNote::NodeLabelAnyFinalVerification => 1, + QueryPlanNote::NodeLabelAllSupersetVerification => 2, + QueryPlanNote::StaleNodeLabelMembershipVerification => 3, + }); + notes.dedup(); + notes + } + + fn key_lookup_candidate_ids( + &self, + query: &NormalizedNodeQuery, + ) -> Result, EngineError> { + let label_id = query + .single_label_id + .expect("normalized key query must have single_label_id"); let key_refs: Vec<(u32, &str)> = query .keys .iter() - .map(|key| (type_id, key.as_str())) + .map(|key| (label_id, key.as_str())) .collect(); let ids = self - .get_nodes_by_keys_raw(&key_refs)? + .get_nodes_by_label_keys_raw(&key_refs)? .into_iter() .flatten() .map(|node| node.id) @@ -1166,12 +2406,12 @@ impl ReadView { &self, query: &NormalizedNodeQuery, cap_context: QueryCapContext, - type_id: u32, + label_id: u32, key: &str, value: &PropValue, ) -> Result { let Some(entry) = - self.node_property_index_entry(type_id, key, &SecondaryIndexKind::Equality) + self.node_property_index_entry(label_id, key, &SecondaryIndexKind::Equality) else { return Ok(CandidateProbe { source: None, @@ -1210,7 +2450,7 @@ impl ReadView { Ok(CandidateProbe { source: Some(PlannedNodeCandidateSource::property_equality_index( - type_id, + label_id, entry.index_id, key, value, @@ -1226,7 +2466,7 @@ impl ReadView { &self, query: &NormalizedNodeQuery, cap_context: QueryCapContext, - type_id: u32, + label_id: u32, key: &str, lower: Option<&PropertyRangeBound>, upper: Option<&PropertyRangeBound>, @@ -1234,7 +2474,7 @@ impl ReadView { ) -> Result { let domain = Self::validate_property_range_bounds(lower, upper, None)?; let Some(entry) = self.node_property_index_entry( - type_id, + label_id, key, &SecondaryIndexKind::Range { domain }, ) else { @@ -1457,24 +2697,24 @@ impl ReadView { &self, query: &NormalizedNodeQuery, cap_context: QueryCapContext, - type_id: u32, + label_id: u32, lower_ms: i64, upper_ms: i64, budget: &mut BooleanPlanningBudget, ) -> Result { if let Some(stats_estimate) = - self.planner_stats.timestamp_estimate(type_id, lower_ms, upper_ms) + self.planner_stats.timestamp_estimate(label_id, lower_ms, upper_ms) { let mut count = self .memtable - .visible_nodes_by_time_range(type_id, lower_ms, upper_ms, self.snapshot_seq) + .visible_nodes_by_time_range(label_id, lower_ms, upper_ms, self.snapshot_seq) .len() as u64; for epoch in &self.immutable_epochs { count = count.saturating_add( epoch .memtable .visible_nodes_by_time_range( - type_id, + label_id, lower_ms, upper_ms, self.snapshot_seq, @@ -1491,7 +2731,7 @@ impl ReadView { .filter(|segment| { !self .planner_stats - .timestamp_covers_segment(type_id, segment.segment_id) + .timestamp_covers_segment(label_id, segment.segment_id) }) .map(|segment| segment.as_ref()) .collect(); @@ -1516,7 +2756,7 @@ impl ReadView { break; } let flow = segment.for_each_node_by_time_range( - type_id, + label_id, lower_ms, upper_ms, |_| { @@ -1571,7 +2811,7 @@ impl ReadView { return Ok(CandidateProbe { source: Some(PlannedNodeCandidateSource::timestamp_index( - type_id, lower_ms, upper_ms, estimate, + label_id, lower_ms, upper_ms, estimate, )), warning: None, followup: None, @@ -1589,7 +2829,7 @@ impl ReadView { }); } let ids = self.timestamp_candidate_ids( - type_id, + label_id, lower_ms, upper_ms, probe_limit + 1, @@ -1598,7 +2838,7 @@ impl ReadView { budget.consume_probe_ids(ids.len()); Ok(CandidateProbe { source: Some(PlannedNodeCandidateSource::timestamp_index( - type_id, + label_id, lower_ms, upper_ms, PlannerEstimate::exact_cheap(ids.len() as u64), @@ -1665,35 +2905,115 @@ impl ReadView { PlannerEstimate::upper_bound(count) } - fn node_type_estimate(&self, type_id: u32) -> Result { + fn node_label_estimate(&self, label_id: u32) -> Result { let mut count = self .memtable - .visible_nodes_by_type_count(type_id, self.snapshot_seq) as u64; + .visible_nodes_by_label_id_count(label_id, self.snapshot_seq) as u64; if self.active_memtable_only_exact_estimates() { return Ok(PlannerEstimate::exact_cheap(count)); } for epoch in &self.immutable_epochs { count += epoch .memtable - .visible_nodes_by_type_count(type_id, self.snapshot_seq) as u64; + .visible_nodes_by_label_id_count(label_id, self.snapshot_seq) as u64; } - count = count.saturating_add(self.planner_stats.type_node_count(type_id)); - let mut used_fallback = self.planner_stats.type_coverage.has_uncovered(); + count = count.saturating_add(self.planner_stats.node_label_count(label_id)); + let mut used_fallback = self.planner_stats.node_label_coverage.has_uncovered(); for segment in &self.segments { - if self.planner_stats.type_coverage.covers(segment.segment_id) { + if self.planner_stats.node_label_coverage.covers(segment.segment_id) { continue; } used_fallback = true; - count = count.saturating_add(segment.node_type_posting_count(type_id)? as u64); + count = count.saturating_add(segment.node_label_posting_count(label_id)? as u64); } Ok(self.planner_stats_estimate_from_rollup( count, - self.planner_stats.type_coverage.covered_count() > 0, + self.planner_stats.node_label_coverage.covered_count() > 0, used_fallback, true, )) } + #[allow(dead_code)] + fn node_label_filter_estimate( + &self, + label_ids: &NodeLabelSet, + mode: LabelMatchMode, + ) -> Result { + if label_ids.len() == 1 { + let label_id = label_ids.single_label_id(); + return Ok(NodeLabelMembershipEstimate { + estimate: self.node_label_estimate(label_id)?, + driver_label_id: Some(label_id), + }); + } + + match mode { + LabelMatchMode::Any => { + let mut count = 0u64; + let mut confidence = EstimateConfidence::Exact; + let mut stale_risk = StalePostingRisk::Low; + for &label_id in label_ids.as_slice() { + let estimate = self.node_label_estimate(label_id)?; + let Some(label_count) = estimate.known_upper_bound() else { + return Ok(NodeLabelMembershipEstimate { + estimate: PlannerEstimate::unknown(), + driver_label_id: None, + }); + }; + count = count.saturating_add(label_count); + confidence = weaker_confidence(confidence, estimate.confidence); + stale_risk = higher_stale_posting_risk(stale_risk, estimate.stale_risk); + } + Ok(NodeLabelMembershipEstimate { + estimate: PlannerEstimate::upper_bound_with_quality( + count, + confidence, + stale_risk, + ), + driver_label_id: None, + }) + } + LabelMatchMode::All => { + let mut best: Option<(u64, u32)> = None; + let mut confidence = EstimateConfidence::Exact; + let mut stale_risk = StalePostingRisk::Low; + for &label_id in label_ids.as_slice() { + let estimate = self.node_label_estimate(label_id)?; + confidence = weaker_confidence(confidence, estimate.confidence); + stale_risk = higher_stale_posting_risk(stale_risk, estimate.stale_risk); + let Some(label_count) = estimate.known_upper_bound() else { + continue; + }; + let better = match best { + Some((best_count, best_label_id)) => { + label_count < best_count + || (label_count == best_count && label_id < best_label_id) + } + None => true, + }; + if better { + best = Some((label_count, label_id)); + } + } + let Some((count, driver_label_id)) = best else { + return Ok(NodeLabelMembershipEstimate { + estimate: PlannerEstimate::unknown(), + driver_label_id: None, + }); + }; + Ok(NodeLabelMembershipEstimate { + estimate: PlannerEstimate::upper_bound_with_quality( + count, + confidence, + stale_risk, + ), + driver_label_id: Some(driver_label_id), + }) + } + } + } + fn full_scan_estimate(&self) -> PlannerEstimate { let mut count = self.memtable.visible_node_count_at(self.snapshot_seq) as u64; if self.active_memtable_only_exact_estimates() { @@ -1714,15 +3034,1285 @@ impl ReadView { { continue; } - used_fallback = true; - count = count.saturating_add(segment.node_count()); + used_fallback = true; + count = count.saturating_add(segment.node_count()); + } + self.planner_stats_estimate_from_rollup( + count, + self.planner_stats.full_rollup.coverage.covered_count() > 0, + used_fallback, + true, + ) + } + + fn edge_full_scan_estimate(&self) -> PlannerEstimate { + let mut count = self.memtable.edge_count() as u64; + for epoch in &self.immutable_epochs { + count = count.saturating_add(epoch.memtable.edge_count() as u64); + } + for segment in &self.segments { + count = count.saturating_add(segment.edge_count()); + } + PlannerEstimate::upper_bound(count) + } + + fn edge_label_estimate(&self, label_id: u32) -> PlannerEstimate { + let mut count = + self.memtable.visible_edges_by_label_id_count(label_id, self.snapshot_seq) as u64; + for epoch in &self.immutable_epochs { + count = count.saturating_add( + epoch + .memtable + .visible_edges_by_label_id_count(label_id, self.snapshot_seq) + as u64, + ); + } + for segment in &self.segments { + let Ok(segment_count) = segment.edge_label_posting_count(label_id) else { + return self.edge_full_scan_estimate(); + }; + count = count.saturating_add(segment_count as u64); + } + PlannerEstimate::upper_bound(count) + } + + fn edge_metadata_source_estimate(&self, label_id: Option) -> PlannerEstimate { + label_id + .map(|label_id| self.edge_label_estimate(label_id)) + .unwrap_or_else(|| self.edge_full_scan_estimate()) + } + + fn edge_weight_range_estimate( + &self, + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + indexed: bool, + ) -> PlannerEstimate { + if !indexed { + return self.edge_metadata_source_estimate(label_id); + } + let mut count = 0u64; + let add_memtable = |memtable: &Memtable| { + let mut local = 0u64; + let _ = memtable.for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::weight_matches_bounds(meta.weight, bounds) + { + local = local.saturating_add(1); + } + ControlFlow::Continue(()) + }); + local + }; + count = count.saturating_add(add_memtable(&self.memtable)); + for epoch in &self.immutable_epochs { + count = count.saturating_add(add_memtable(&epoch.memtable)); + } + for segment in &self.segments { + let Some(segment_count) = segment.edge_weight_range_count(label_id, bounds) else { + return self.edge_metadata_source_estimate(label_id); + }; + count = count.saturating_add(segment_count as u64); + } + PlannerEstimate::upper_bound(count) + } + + fn edge_i64_metadata_range_estimate( + &self, + label_id: Option, + bounds: crate::edge_metadata::RangeBoundFlags, + indexed: bool, + memtable_value: impl Fn(EdgeMetadataCandidate) -> i64, + segment_count: impl Fn(&SegmentReader) -> Option, + ) -> PlannerEstimate { + if !indexed { + return self.edge_metadata_source_estimate(label_id); + } + let mut count = 0u64; + let add_memtable = |memtable: &Memtable| { + let mut local = 0u64; + let _ = memtable.for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(memtable_value(meta), bounds) + { + local = local.saturating_add(1); + } + ControlFlow::Continue(()) + }); + local + }; + count = count.saturating_add(add_memtable(&self.memtable)); + for epoch in &self.immutable_epochs { + count = count.saturating_add(add_memtable(&epoch.memtable)); + } + for segment in &self.segments { + let Some(segment_count) = segment_count(segment) else { + return self.edge_metadata_source_estimate(label_id); + }; + count = count.saturating_add(segment_count as u64); + } + PlannerEstimate::upper_bound(count) + } + + fn edge_endpoint_estimate( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + ) -> PlannerEstimate { + if node_ids.is_empty() { + return PlannerEstimate::exact_cheap(0); + } + let mut sorted_node_ids = node_ids.to_vec(); + sorted_node_ids.sort_unstable(); + sorted_node_ids.dedup(); + + let mut count = 0u64; + let mut confidence = EstimateConfidence::Medium; + let mut add_memtable_estimate = + |estimate: crate::memtable::MemtableEndpointCountEstimate| { + count = count.saturating_add(estimate.count as u64); + if !estimate.exact { + confidence = weaker_confidence(confidence, EstimateConfidence::Low); + } + }; + for &node_id in &sorted_node_ids { + match direction { + Direction::Outgoing => { + add_memtable_estimate( + self.memtable + .visible_edges_from_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + for epoch in &self.immutable_epochs { + add_memtable_estimate( + epoch.memtable.visible_edges_from_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + } + } + Direction::Incoming => { + add_memtable_estimate( + self.memtable + .visible_edges_to_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + for epoch in &self.immutable_epochs { + add_memtable_estimate( + epoch.memtable.visible_edges_to_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + } + } + Direction::Both => { + add_memtable_estimate( + self.memtable + .visible_edges_from_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + add_memtable_estimate( + self.memtable + .visible_edges_to_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + for epoch in &self.immutable_epochs { + add_memtable_estimate( + epoch.memtable.visible_edges_from_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + add_memtable_estimate( + epoch.memtable.visible_edges_to_endpoint_count_estimate( + node_id, + label_filter_ids, + self.snapshot_seq, + ), + ); + } + } + } + } + + for segment in &self.segments { + match segment.endpoint_adj_posting_count(&sorted_node_ids, direction, label_filter_ids) { + Ok(segment_count) => count = count.saturating_add(segment_count as u64), + Err(_) => return self.edge_full_scan_estimate(), + } + } + + PlannerEstimate::upper_bound_with_confidence(count, confidence) + } + + fn edge_metadata_sidecar_availability(&self) -> EdgeMetadataSidecarAvailability { + let mut has_nonempty_segment = false; + let mut weight = true; + let mut updated_at = true; + let mut valid_from = true; + let mut valid_to = true; + + for segment in &self.segments { + if segment.edge_count() == 0 { + continue; + } + has_nonempty_segment = true; + weight &= segment.edge_weight_index_available(); + updated_at &= segment.edge_updated_at_index_available(); + valid_from &= segment.edge_valid_from_index_available(); + valid_to &= segment.edge_valid_to_index_available(); + } + + EdgeMetadataSidecarAvailability { + weight: has_nonempty_segment && weight, + updated_at: has_nonempty_segment && updated_at, + valid_from: has_nonempty_segment && valid_from, + valid_to: has_nonempty_segment && valid_to, + } + } + + fn edge_metadata_filter_candidate_plan( + &self, + filter: &NormalizedEdgeFilter, + label_id: Option, + availability: EdgeMetadataSidecarAvailability, + ) -> Option { + match filter { + NormalizedEdgeFilter::AlwaysTrue => None, + NormalizedEdgeFilter::AlwaysFalse => Some(EdgePhysicalPlan::Empty), + NormalizedEdgeFilter::WeightRange { lower, upper } => { + let bounds = crate::edge_metadata::RangeBoundFlags::inclusive(*lower, *upper); + Some(EdgePhysicalPlan::source(PlannedEdgeCandidateSource::edge_weight_index( + label_id, + bounds, + availability.weight, + self.edge_weight_range_estimate(label_id, bounds, availability.weight), + ))) + } + NormalizedEdgeFilter::UpdatedAtRange { lower_ms, upper_ms } => { + let bounds = + crate::edge_metadata::RangeBoundFlags::inclusive(Some(*lower_ms), Some(*upper_ms)); + Some(EdgePhysicalPlan::source(PlannedEdgeCandidateSource::edge_updated_at_index( + label_id, + bounds, + availability.updated_at, + self.edge_i64_metadata_range_estimate( + label_id, + bounds, + availability.updated_at, + |meta| meta.updated_at, + |segment| segment.edge_updated_at_range_count(label_id, bounds), + ), + ))) + } + NormalizedEdgeFilter::ValidFromRange { lower_ms, upper_ms } => { + let bounds = + crate::edge_metadata::RangeBoundFlags::inclusive(Some(*lower_ms), Some(*upper_ms)); + Some(EdgePhysicalPlan::source(PlannedEdgeCandidateSource::edge_valid_from_index( + label_id, + bounds, + availability.valid_from, + self.edge_i64_metadata_range_estimate( + label_id, + bounds, + availability.valid_from, + |meta| meta.valid_from, + |segment| segment.edge_valid_from_range_count(label_id, bounds), + ), + ))) + } + NormalizedEdgeFilter::ValidToRange { lower_ms, upper_ms } => { + let bounds = + crate::edge_metadata::RangeBoundFlags::inclusive(Some(*lower_ms), Some(*upper_ms)); + Some(EdgePhysicalPlan::source(PlannedEdgeCandidateSource::edge_valid_to_index( + label_id, + bounds, + availability.valid_to, + self.edge_i64_metadata_range_estimate( + label_id, + bounds, + availability.valid_to, + |meta| meta.valid_to, + |segment| segment.edge_valid_to_range_count(label_id, bounds), + ), + ))) + } + NormalizedEdgeFilter::ValidAt { epoch_ms } => { + let valid_from_bounds = + crate::edge_metadata::RangeBoundFlags::inclusive(None, Some(*epoch_ms)); + let valid_from = EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::edge_valid_from_index( + label_id, + valid_from_bounds, + availability.valid_from, + self.edge_i64_metadata_range_estimate( + label_id, + valid_from_bounds, + availability.valid_from, + |meta| meta.valid_from, + |segment| { + segment.edge_valid_from_range_count(label_id, valid_from_bounds) + }, + ), + ), + ); + let valid_to_bounds = crate::edge_metadata::RangeBoundFlags { + lower: Some(*epoch_ms), + lower_inclusive: false, + upper: None, + upper_inclusive: true, + }; + let valid_to = EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::edge_valid_to_index( + label_id, + valid_to_bounds, + availability.valid_to, + self.edge_i64_metadata_range_estimate( + label_id, + valid_to_bounds, + availability.valid_to, + |meta| meta.valid_to, + |segment| segment.edge_valid_to_range_count(label_id, valid_to_bounds), + ), + ), + ); + Some(EdgePhysicalPlan::intersect(vec![valid_from, valid_to])) + } + NormalizedEdgeFilter::And(children) => { + let plans = children + .iter() + .filter_map(|child| { + self.edge_metadata_filter_candidate_plan(child, label_id, availability) + }) + .collect::>(); + (!plans.is_empty()).then(|| EdgePhysicalPlan::intersect(plans)) + } + NormalizedEdgeFilter::Or(children) => { + let mut plans = Vec::with_capacity(children.len()); + for child in children { + let plan = + self.edge_metadata_filter_candidate_plan(child, label_id, availability)?; + plans.push(plan); + } + Some(EdgePhysicalPlan::union(plans)) + } + NormalizedEdgeFilter::Not(_) + | NormalizedEdgeFilter::PropertyEquals { .. } + | NormalizedEdgeFilter::PropertyIn { .. } + | NormalizedEdgeFilter::PropertyRange { .. } + | NormalizedEdgeFilter::PropertyExists { .. } + | NormalizedEdgeFilter::PropertyMissing { .. } => None, + } + } + + fn edge_equality_candidate_estimate( + &self, + index_id: u64, + key: &str, + value: &PropValue, + ) -> Result<(Option, Option), EngineError> { + let mut count = memtable_secondary_eq_edge_count_for_filter( + &self.memtable, + index_id, + key, + value, + self.snapshot_seq, + ) as u64; + if self.active_memtable_only_exact_estimates() { + return Ok((Some(PlannerEstimate::exact_cheap(count)), None)); + } + for epoch in &self.immutable_epochs { + count = count.saturating_add(memtable_secondary_eq_edge_count_for_filter( + &epoch.memtable, + index_id, + key, + value, + self.snapshot_seq, + ) as u64); + } + + let value_hashes = equality_probe_value_hashes(value); + let mut used_stats = false; + let mut used_fallback = false; + let mut stats_values_exact = true; + for segment in &self.segments { + if let Some(segment_estimate) = self.planner_stats.equality_segment_estimate( + index_id, + segment.segment_id, + &value_hashes, + ) { + used_stats = true; + stats_values_exact &= segment_estimate.exact; + count = count.saturating_add(segment_estimate.count); + continue; + } + used_fallback = true; + match segment_edge_secondary_eq_posting_count_for_filter(segment, index_id, value) { + Ok(Some(posting_count)) => count = count.saturating_add(posting_count as u64), + Ok(None) => { + return Ok((None, self.equality_sidecar_failure_followup(index_id, None))); + } + Err(error) => { + return Ok(( + None, + self.equality_sidecar_failure_followup(index_id, Some(error)), + )); + } + } + } + + let mut estimate = + self.planner_stats_estimate_from_rollup(count, used_stats, used_fallback, stats_values_exact); + if !used_stats { + estimate = estimate.with_current_posting_bound(); + } + Ok((Some(estimate), None)) + } + + fn edge_equality_candidate_probe( + &self, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + label_id: u32, + key: &str, + value: &PropValue, + ) -> Result { + let Some(entry) = + self.edge_property_index_entry(label_id, key, &SecondaryIndexKind::Equality) + else { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup: None, + }); + }; + if entry.state != SecondaryIndexState::Ready { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup: None, + }); + } + + let (estimate, followup) = + self.edge_equality_candidate_estimate(entry.index_id, key, value)?; + let Some(estimate) = estimate else { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup, + }); + }; + if cap_context.source_estimate_exceeds_cap( + EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex, + query.page.limit, + estimate, + ) { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::CandidateCapExceeded), + followup, + }); + } + + Ok(EdgeCandidateProbe { + source: Some(PlannedEdgeCandidateSource::edge_property_equality_index( + label_id, + entry.index_id, + key, + value, + estimate, + )), + warning: None, + followup, + }) + } + + fn ready_edge_range_candidate_ids( + &self, + index_id: u64, + domain: SecondaryIndexRangeDomain, + lower: Option<&PropertyRangeBound>, + upper: Option<&PropertyRangeBound>, + max_ids: usize, + ) -> Result<(Option>, Option), EngineError> { + let lower_encoded = Self::encode_property_range_bound(domain, lower); + let upper_encoded = Self::encode_property_range_bound(domain, upper); + match self.sources().edge_ids_by_secondary_range_index_limited( + index_id, + lower_encoded, + upper_encoded, + max_ids, + ) { + Ok(Some(ids)) => Ok((Some(ids), None)), + Ok(None) => Ok((None, self.range_sidecar_failure_followup(index_id, None))), + Err(error) => Ok((None, self.range_sidecar_failure_followup(index_id, Some(error)))), + } + } + + #[allow(clippy::too_many_arguments)] + fn edge_range_candidate_probe( + &self, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + label_id: u32, + key: &str, + lower: Option<&PropertyRangeBound>, + upper: Option<&PropertyRangeBound>, + budget: &mut BooleanPlanningBudget, + ) -> Result { + let domain = Self::validate_property_range_bounds(lower, upper, None)?; + let Some(entry) = self.edge_property_index_entry( + label_id, + key, + &SecondaryIndexKind::Range { domain }, + ) else { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup: None, + }); + }; + if entry.state != SecondaryIndexState::Ready { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup: None, + }); + } + + let lower_encoded = Self::encode_property_range_bound(domain, lower); + let upper_encoded = Self::encode_property_range_bound(domain, upper); + if let Some(stats_estimate) = + self.planner_stats + .range_index_estimate(entry.index_id, lower_encoded, upper_encoded) + { + let mut count = self + .memtable + .visible_secondary_range_entries( + entry.index_id, + lower_encoded, + upper_encoded, + None, + self.snapshot_seq, + ) + .len() as u64; + for epoch in &self.immutable_epochs { + count = count.saturating_add( + epoch + .memtable + .visible_secondary_range_entries( + entry.index_id, + lower_encoded, + upper_encoded, + None, + self.snapshot_seq, + ) + .len() as u64, + ); + } + count = count.saturating_add(stats_estimate.count); + + let mut used_fallback = false; + let coverage = &self + .planner_stats + .range_index_rollups + .get(&entry.index_id) + .expect("range estimate must have matching rollup") + .coverage; + let uncovered_segments: Vec<&SegmentReader> = self + .segments + .iter() + .filter(|segment| !coverage.covers(segment.segment_id)) + .map(|segment| segment.as_ref()) + .collect(); + if !uncovered_segments.is_empty() { + used_fallback = true; + let probe_limit = budget.probe_limit(); + if probe_limit == 0 { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::PlanningProbeBudgetExceeded), + followup: None, + }); + } + + #[cfg(test)] + self.note_range_planning_probe(); + + let mut fallback_ids = 0usize; + let total_read_limit = probe_limit.saturating_add(1); + for segment in uncovered_segments { + let remaining_read_limit = total_read_limit.saturating_sub(fallback_ids); + if remaining_read_limit == 0 { + break; + } + match segment.find_edges_by_secondary_range_index_if_present_limited( + entry.index_id, + lower_encoded, + upper_encoded, + None, + Some(remaining_read_limit), + ) { + Ok(Some(ids)) => { + fallback_ids = fallback_ids.saturating_add(ids.len()); + if fallback_ids > probe_limit { + break; + } + } + Ok(None) => { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup: self.range_sidecar_failure_followup(entry.index_id, None), + }); + } + Err(error) => { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup: self + .range_sidecar_failure_followup(entry.index_id, Some(error)), + }); + } + } + } + budget.consume_probe_ids(fallback_ids); + if fallback_ids > probe_limit { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(if probe_limit < QUERY_RANGE_CANDIDATE_CAP { + QueryPlanWarning::PlanningProbeBudgetExceeded + } else { + QueryPlanWarning::RangeCandidateCapExceeded + }), + followup: None, + }); + } + count = count.saturating_add(fallback_ids as u64); + } + + let estimate = if self.active_memtable_only_exact_estimates() { + PlannerEstimate::exact_cheap(count) + } else { + self.planner_stats_estimate_from_rollup( + count, + true, + used_fallback, + stats_estimate.exact, + ) + }; + if cap_context.source_estimate_exceeds_cap( + EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex, + query.page.limit, + estimate, + ) { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::RangeCandidateCapExceeded), + followup: None, + }); + } + + return Ok(EdgeCandidateProbe { + source: Some(PlannedEdgeCandidateSource::edge_property_range_index( + label_id, + entry.index_id, + key, + domain, + lower, + upper, + estimate, + )), + warning: None, + followup: None, + }); + } + + #[cfg(test)] + self.note_range_planning_probe(); + let probe_limit = budget.probe_limit(); + if probe_limit == 0 { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::PlanningProbeBudgetExceeded), + followup: None, + }); + } + let (ids, followup) = self.ready_edge_range_candidate_ids( + entry.index_id, + domain, + lower, + upper, + probe_limit + 1, + )?; + let estimate = match ids { + Some(ids) if ids.len() <= probe_limit => { + budget.consume_probe_ids(ids.len()); + PlannerEstimate::exact_cheap(ids.len() as u64) + } + Some(ids) => { + budget.consume_probe_ids(ids.len().min(probe_limit + 1)); + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(if probe_limit < QUERY_RANGE_CANDIDATE_CAP { + QueryPlanWarning::PlanningProbeBudgetExceeded + } else { + QueryPlanWarning::RangeCandidateCapExceeded + }), + followup, + }); + } + None => { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::MissingReadyIndex), + followup, + }); + } + }; + + if cap_context.source_estimate_exceeds_cap( + EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex, + query.page.limit, + estimate, + ) { + return Ok(EdgeCandidateProbe { + source: None, + warning: Some(QueryPlanWarning::RangeCandidateCapExceeded), + followup: None, + }); + } + + Ok(EdgeCandidateProbe { + source: Some(PlannedEdgeCandidateSource::edge_property_range_index( + label_id, + entry.index_id, + key, + domain, + lower, + upper, + estimate, + )), + warning: None, + followup: None, + }) + } + + fn classification_from_edge_probe( + &self, + probe: EdgeCandidateProbe, + structural_key: Vec, + warnings: &mut Vec, + followups: &mut Vec, + ) -> EdgeBooleanPlanResult { + if let Some(warning) = probe.warning { + add_plan_warning(warnings, warning); + } + if let Some(followup) = probe.followup { + followups.push(followup); + } + match probe.source { + Some(source) if source.estimate.proves_empty() => EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::AlwaysFalse, + has_verify_only: false, + }, + Some(source) => EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::Bounded { + estimate: source.estimate, + structural_key, + complete: true, + plan: EdgePhysicalPlan::source(source), + }, + has_verify_only: false, + }, + None => EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }, + } + } + + fn sort_edge_physical_plans_by_selectivity(&self, plans: &mut [EdgePhysicalPlan]) { + plans.sort_by_key(|plan| plan.plan_cost()); + } + + fn select_bounded_edge_and_plans( + &self, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + mut plans: Vec, + warnings: &mut Vec, + ) -> (Vec, bool) { + self.sort_edge_physical_plans_by_selectivity(&mut plans); + let Some(first) = plans.first() else { + return (Vec::new(), false); + }; + let smallest_cost = first.plan_cost(); + let mut selected = Vec::new(); + let mut skipped_to_verifier = false; + + for plan in plans { + if selected.is_empty() { + selected.push(plan); + continue; + } + let plan_cost = plan.plan_cost(); + let estimate = plan.estimate(); + let cap = plan.materialization_cap(cap_context, query.page.limit); + let within_input_cap = estimate + .known_upper_bound() + .is_some_and(|count| count <= cap as u64); + let include = within_input_cap + && plan_cost.estimated_work + <= smallest_cost + .estimated_work + .saturating_mul(QUERY_BROAD_SOURCE_FACTOR); + if include { + selected.push(plan); + } else { + skipped_to_verifier = true; + if plan.estimate().known_upper_bound().is_some() && plan.broad_skip_warnable() { + add_plan_warning(warnings, QueryPlanWarning::IndexSkippedAsBroad); + } + } + } + + (selected, skipped_to_verifier) + } + + #[allow(clippy::too_many_arguments)] + fn plan_edge_property_in_filter( + &self, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + key: &str, + values: &[PropValue], + structural_key: Vec, + warnings: &mut Vec, + followups: &mut Vec, + ) -> Result { + let Some(label_id) = query.label_id else { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + if values.len() == 1 { + let probe = + self.edge_equality_candidate_probe(query, cap_context, label_id, key, &values[0])?; + return Ok(self.classification_from_edge_probe( + probe, + structural_key, + warnings, + followups, + )); + } + + let unique_values = unique_in_probe_values(values); + if unique_values.len() > MAX_BOOLEAN_UNION_INPUTS { + add_plan_warning(warnings, QueryPlanWarning::PlanningProbeBudgetExceeded); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + + let Some(entry) = + self.edge_property_index_entry(label_id, key, &SecondaryIndexKind::Equality) + else { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + if entry.state != SecondaryIndexState::Ready { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + + let mut plans = Vec::new(); + let mut estimated_total = 0u64; + for probe in &unique_values { + let (estimate, followup) = + self.edge_equality_candidate_estimate(entry.index_id, key, &probe.value)?; + if let Some(followup) = followup { + followups.push(followup); + } + let Some(estimate) = estimate else { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + let Some(count) = estimate.known_upper_bound() else { + add_plan_warning(warnings, QueryPlanWarning::IndexSkippedAsBroad); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + if cap_context.source_estimate_exceeds_cap( + EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex, + query.page.limit, + estimate, + ) { + add_plan_warning( + warnings, + edge_cap_warning_for_source( + EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex, + ), + ); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + estimated_total = estimated_total.saturating_add(count); + let union_estimate = PlannerEstimate::upper_bound(estimated_total); + let union_cap = cap_context.union_total_cap(query.page.limit, union_estimate); + if estimated_total > union_cap as u64 { + add_plan_warning(warnings, QueryPlanWarning::PlanningProbeBudgetExceeded); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + if count == 0 && estimate.proves_empty() { + continue; + } + plans.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::edge_property_equality_index_with_hash( + label_id, + entry.index_id, + key, + &probe.value, + probe.value_hash, + estimate, + ), + )); + } + + if plans.is_empty() { + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::AlwaysFalse, + has_verify_only: false, + }); + } + + if cap_context + .cheapest_legal_count() + .is_some_and(|legal| legal <= estimated_total) + { + add_plan_warning(warnings, QueryPlanWarning::IndexSkippedAsBroad); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + + Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::Bounded { + plan: EdgePhysicalPlan::union(plans), + estimate: PlannerEstimate::upper_bound(estimated_total), + structural_key, + complete: true, + }, + has_verify_only: false, + }) + } + + #[allow(clippy::too_many_arguments)] + fn plan_edge_filter_subtree( + &self, + query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + filter: &NormalizedEdgeFilter, + availability: EdgeMetadataSidecarAvailability, + budget: &mut BooleanPlanningBudget, + warnings: &mut Vec, + followups: &mut Vec, + ) -> Result { + let structural_key = filter.structural_key(); + match filter { + NormalizedEdgeFilter::AlwaysFalse => Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::AlwaysFalse, + has_verify_only: false, + }), + NormalizedEdgeFilter::AlwaysTrue => Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: false, + }), + NormalizedEdgeFilter::PropertyEquals { key, value } => { + let Some(label_id) = query.label_id else { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + let probe = + self.edge_equality_candidate_probe(query, cap_context, label_id, key, value)?; + Ok(self.classification_from_edge_probe( + probe, + structural_key, + warnings, + followups, + )) + } + NormalizedEdgeFilter::PropertyIn { key, values, .. } => self + .plan_edge_property_in_filter( + query, + cap_context, + key, + values, + structural_key, + warnings, + followups, + ), + NormalizedEdgeFilter::PropertyRange { key, lower, upper } => { + let Some(label_id) = query.label_id else { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + let probe = self.edge_range_candidate_probe( + query, + cap_context, + label_id, + key, + lower.as_ref(), + upper.as_ref(), + budget, + )?; + Ok(self.classification_from_edge_probe( + probe, + structural_key, + warnings, + followups, + )) + } + NormalizedEdgeFilter::PropertyExists { .. } + | NormalizedEdgeFilter::PropertyMissing { .. } + | NormalizedEdgeFilter::Not(_) => Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }), + NormalizedEdgeFilter::WeightRange { .. } + | NormalizedEdgeFilter::UpdatedAtRange { .. } + | NormalizedEdgeFilter::ValidAt { .. } + | NormalizedEdgeFilter::ValidFromRange { .. } + | NormalizedEdgeFilter::ValidToRange { .. } => { + match self.edge_metadata_filter_candidate_plan(filter, query.label_id, availability) + { + Some(EdgePhysicalPlan::Empty) => Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::AlwaysFalse, + has_verify_only: false, + }), + Some(plan) => Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::Bounded { + estimate: plan.estimate(), + structural_key, + complete: true, + plan, + }, + has_verify_only: false, + }), + None => Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: false, + }), + } + } + NormalizedEdgeFilter::And(children) => { + let mut plans = Vec::new(); + let mut has_verify_only = false; + for child in children { + let planned = self.plan_edge_filter_subtree( + query, + cap_context, + child, + availability, + budget, + warnings, + followups, + )?; + has_verify_only |= planned.has_verify_only; + match planned.classification { + EdgeBooleanPlanClassification::AlwaysFalse => { + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::AlwaysFalse, + has_verify_only, + }); + } + EdgeBooleanPlanClassification::VerifyOnly => {} + EdgeBooleanPlanClassification::Bounded { plan, complete, .. } + if complete => + { + plans.push(plan) + } + EdgeBooleanPlanClassification::Bounded { .. } => { + has_verify_only = true; + } + } + } + + let (selected, skipped_to_verifier) = + self.select_bounded_edge_and_plans(query, cap_context, plans, warnings); + has_verify_only |= skipped_to_verifier; + if selected.is_empty() { + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only, + }); + } + let plan = EdgePhysicalPlan::intersect(selected); + Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::Bounded { + estimate: plan.estimate(), + structural_key, + complete: true, + plan, + }, + has_verify_only, + }) + } + NormalizedEdgeFilter::Or(children) => { + if children.len() > MAX_BOOLEAN_UNION_INPUTS { + add_plan_warning(warnings, QueryPlanWarning::PlanningProbeBudgetExceeded); + add_plan_warning(warnings, QueryPlanWarning::BooleanBranchFallback); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + + let mut plan_entries = Vec::new(); + let mut estimated_total = 0u64; + let mut has_verify_only = false; + for child in children { + let planned = self.plan_edge_filter_subtree( + query, + cap_context, + child, + availability, + budget, + warnings, + followups, + )?; + has_verify_only |= planned.has_verify_only; + match planned.classification { + EdgeBooleanPlanClassification::AlwaysFalse => {} + EdgeBooleanPlanClassification::VerifyOnly => { + add_plan_warning(warnings, QueryPlanWarning::BooleanBranchFallback); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + EdgeBooleanPlanClassification::Bounded { + plan, + estimate, + structural_key, + complete, + } if complete => { + let Some(count) = estimate.known_upper_bound() else { + add_plan_warning(warnings, QueryPlanWarning::IndexSkippedAsBroad); + add_plan_warning(warnings, QueryPlanWarning::BooleanBranchFallback); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + estimated_total = estimated_total.saturating_add(count); + let union_estimate = PlannerEstimate::upper_bound(estimated_total); + let union_cap = + cap_context.union_total_cap(query.page.limit, union_estimate); + if estimated_total > union_cap as u64 { + add_plan_warning( + warnings, + QueryPlanWarning::PlanningProbeBudgetExceeded, + ); + add_plan_warning(warnings, QueryPlanWarning::BooleanBranchFallback); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + plan_entries.push((structural_key, plan)); + } + EdgeBooleanPlanClassification::Bounded { .. } => { + add_plan_warning(warnings, QueryPlanWarning::BooleanBranchFallback); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + } + } + + if plan_entries.is_empty() { + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::AlwaysFalse, + has_verify_only, + }); + } + if cap_context + .cheapest_legal_count() + .is_some_and(|legal| legal <= estimated_total) + { + add_plan_warning(warnings, QueryPlanWarning::IndexSkippedAsBroad); + add_plan_warning(warnings, QueryPlanWarning::BooleanBranchFallback); + return Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + } + + plan_entries.sort_by(|left, right| left.0.cmp(&right.0)); + let plan = EdgePhysicalPlan::union( + plan_entries + .into_iter() + .map(|(_, plan)| plan) + .collect(), + ); + Ok(EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::Bounded { + estimate: plan.estimate(), + structural_key, + complete: true, + plan, + }, + has_verify_only, + }) + } } - self.planner_stats_estimate_from_rollup( - count, - self.planner_stats.full_rollup.coverage.covered_count() > 0, - used_fallback, - true, - ) } fn equality_candidate_estimate( @@ -1829,14 +4419,51 @@ impl ReadView { PlannedNodeCandidateSource::key_lookup(query.keys.len()), )); } - if let Some(type_id) = query.type_id { - let estimate = self.node_type_estimate(type_id)?; - let source = if filter_driver { - PlannedNodeCandidateSource::fallback_type_scan(type_id, estimate) - } else { - PlannedNodeCandidateSource::node_type_index(type_id, estimate) - }; - plans.push(NodePhysicalPlan::source(source)); + if let ResolvedNodeLabelFilter::LabelSet { + mode, label_ids, .. + } = query.label_filter + { + let membership_estimate = self.node_label_filter_estimate(&label_ids, mode)?; + match (mode, label_ids.as_slice()) { + (_, [single_label_id]) => { + let source = if filter_driver { + PlannedNodeCandidateSource::fallback_node_label_scan( + *single_label_id, + membership_estimate.estimate, + ) + } else { + PlannedNodeCandidateSource::node_label_index( + *single_label_id, + membership_estimate.estimate, + ) + }; + plans.push(NodePhysicalPlan::source(source)); + } + (LabelMatchMode::Any, _) => { + plans.push(NodePhysicalPlan::source( + PlannedNodeCandidateSource::node_label_any_index( + label_ids, + membership_estimate.estimate, + ), + )); + } + (LabelMatchMode::All, _) => { + if let Some(driver_label_id) = membership_estimate.driver_label_id { + let source = if filter_driver { + PlannedNodeCandidateSource::fallback_node_label_scan( + driver_label_id, + membership_estimate.estimate, + ) + } else { + PlannedNodeCandidateSource::node_label_index( + driver_label_id, + membership_estimate.estimate, + ) + }; + plans.push(NodePhysicalPlan::source(source)); + } + } + } } if query.allow_full_scan { plans.push(NodePhysicalPlan::source( @@ -1928,26 +4555,103 @@ impl ReadView { } } + fn candidate_probe_warning_precedence(warning: QueryPlanWarning) -> usize { + match warning { + QueryPlanWarning::PlanningProbeBudgetExceeded => 0, + QueryPlanWarning::CandidateCapExceeded + | QueryPlanWarning::RangeCandidateCapExceeded + | QueryPlanWarning::TimestampCandidateCapExceeded + | QueryPlanWarning::IndexSkippedAsBroad => 1, + QueryPlanWarning::MissingReadyIndex => 2, + _ => 3, + } + } + + fn remember_candidate_probe_warning( + current: &mut Option<(QueryPlanWarning, Option)>, + warning: QueryPlanWarning, + followup: Option, + ) { + let replace = current.as_ref().is_none_or(|(current_warning, _)| { + Self::candidate_probe_warning_precedence(warning) + < Self::candidate_probe_warning_precedence(*current_warning) + }); + if replace { + *current = Some((warning, followup)); + } + } + + fn best_candidate_probe_for_labels( + &self, + label_ids: NodeLabelSet, + mut probe_label: impl FnMut(u32) -> Result, + ) -> Result { + let labels = label_ids.as_slice(); + if let [label_id] = labels { + return probe_label(*label_id); + } + + let mut best_source: Option<( + PlanCost, + PlannedNodeCandidateSource, + Option, + )> = None; + let mut best_warning = None; + + for &label_id in labels { + let probe = probe_label(label_id)?; + if let Some(source) = probe.source { + let cost = NodePhysicalPlan::source(source.clone()).plan_cost(); + if best_source + .as_ref() + .is_none_or(|(best_cost, _, _)| cost < *best_cost) + { + best_source = Some((cost, source, probe.followup)); + } + continue; + } + + if let Some(warning) = probe.warning { + Self::remember_candidate_probe_warning( + &mut best_warning, + warning, + probe.followup, + ); + } + } + + if let Some((_, source, followup)) = best_source { + return Ok(CandidateProbe { + source: Some(source), + warning: None, + followup, + }); + } + + let (warning, followup) = + best_warning.unwrap_or((QueryPlanWarning::MissingReadyIndex, None)); + Ok(CandidateProbe { + source: None, + warning: Some(warning), + followup, + }) + } + #[allow(clippy::too_many_arguments)] - fn plan_property_in_filter( + fn plan_property_in_filter_for_label( &self, query: &NormalizedNodeQuery, cap_context: QueryCapContext, + label_id: u32, key: &str, values: &[PropValue], structural_key: Vec, warnings: &mut Vec, followups: &mut Vec, ) -> Result { - let Some(type_id) = query.type_id else { - add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); - return Ok(BooleanPlanResult { - classification: BooleanPlanClassification::VerifyOnly, - has_verify_only: true, - }); - }; if values.len() == 1 { - let probe = self.equality_candidate_probe(query, cap_context, type_id, key, &values[0])?; + let probe = + self.equality_candidate_probe(query, cap_context, label_id, key, &values[0])?; return Ok(self.classification_from_probe(probe, structural_key, warnings, followups)); } let unique_values = unique_in_probe_values(values); @@ -1960,7 +4664,7 @@ impl ReadView { } let Some(entry) = - self.node_property_index_entry(type_id, key, &SecondaryIndexKind::Equality) + self.node_property_index_entry(label_id, key, &SecondaryIndexKind::Equality) else { add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); return Ok(BooleanPlanResult { @@ -2024,7 +4728,7 @@ impl ReadView { } plans.push(NodePhysicalPlan::source( PlannedNodeCandidateSource::property_equality_index_with_hash( - type_id, + label_id, entry.index_id, key, &probe.value, @@ -2063,6 +4767,109 @@ impl ReadView { }) } + #[allow(clippy::too_many_arguments)] + fn plan_property_in_filter( + &self, + query: &NormalizedNodeQuery, + cap_context: QueryCapContext, + key: &str, + values: &[PropValue], + structural_key: Vec, + warnings: &mut Vec, + followups: &mut Vec, + ) -> Result { + let Some(label_ids) = node_index_candidate_labels(query) else { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + return Ok(BooleanPlanResult { + classification: BooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }); + }; + let labels = label_ids.as_slice(); + if let [label_id] = labels { + return self.plan_property_in_filter_for_label( + query, + cap_context, + *label_id, + key, + values, + structural_key, + warnings, + followups, + ); + } + + let mut best_plan: Option<( + PlanCost, + BooleanPlanResult, + Vec, + Vec, + )> = None; + let mut fallback_warnings = Vec::new(); + let mut fallback_followups = Vec::new(); + + for &label_id in labels { + let mut label_warnings = Vec::new(); + let mut label_followups = Vec::new(); + let planned = self.plan_property_in_filter_for_label( + query, + cap_context, + label_id, + key, + values, + structural_key.clone(), + &mut label_warnings, + &mut label_followups, + )?; + + match &planned.classification { + BooleanPlanClassification::AlwaysFalse => { + for warning in label_warnings { + add_plan_warning(warnings, warning); + } + followups.append(&mut label_followups); + return Ok(planned); + } + BooleanPlanClassification::Bounded { plan, .. } => { + let cost = plan.plan_cost(); + if best_plan + .as_ref() + .is_none_or(|(best_cost, _, _, _)| cost < *best_cost) + { + best_plan = Some((cost, planned, label_warnings, label_followups)); + } + } + BooleanPlanClassification::VerifyOnly => { + for warning in label_warnings { + add_plan_warning(&mut fallback_warnings, warning); + } + fallback_followups.append(&mut label_followups); + } + } + } + + if let Some((_, planned, selected_warnings, mut selected_followups)) = best_plan { + for warning in selected_warnings { + add_plan_warning(warnings, warning); + } + followups.append(&mut selected_followups); + return Ok(planned); + } + + if fallback_warnings.is_empty() { + add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); + } else { + for warning in fallback_warnings { + add_plan_warning(warnings, warning); + } + } + followups.append(&mut fallback_followups); + Ok(BooleanPlanResult { + classification: BooleanPlanClassification::VerifyOnly, + has_verify_only: true, + }) + } + fn plan_filter_subtree( &self, query: &NormalizedNodeQuery, @@ -2083,14 +4890,16 @@ impl ReadView { has_verify_only: false, }), NormalizedNodeFilter::PropertyEquals { key, value } => { - let Some(type_id) = query.type_id else { + let Some(label_ids) = node_index_candidate_labels(query) else { add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); return Ok(BooleanPlanResult { classification: BooleanPlanClassification::VerifyOnly, has_verify_only: true, }); }; - let probe = self.equality_candidate_probe(query, cap_context, type_id, key, value)?; + let probe = self.best_candidate_probe_for_labels(label_ids, |label_id| { + self.equality_candidate_probe(query, cap_context, label_id, key, value) + })?; Ok(self.classification_from_probe(probe, structural_key, warnings, followups)) } NormalizedNodeFilter::PropertyIn { key, values, .. } => self @@ -2104,40 +4913,44 @@ impl ReadView { followups, ), NormalizedNodeFilter::PropertyRange { key, lower, upper } => { - let Some(type_id) = query.type_id else { + let Some(label_ids) = node_index_candidate_labels(query) else { add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); return Ok(BooleanPlanResult { classification: BooleanPlanClassification::VerifyOnly, has_verify_only: true, }); }; - let probe = self.range_candidate_probe( - query, - cap_context, - type_id, - key, - lower.as_ref(), - upper.as_ref(), - budget, - )?; + let probe = self.best_candidate_probe_for_labels(label_ids, |label_id| { + self.range_candidate_probe( + query, + cap_context, + label_id, + key, + lower.as_ref(), + upper.as_ref(), + budget, + ) + })?; Ok(self.classification_from_probe(probe, structural_key, warnings, followups)) } NormalizedNodeFilter::UpdatedAtRange { lower_ms, upper_ms } => { - let Some(type_id) = query.type_id else { + let Some(label_ids) = node_index_candidate_labels(query) else { add_plan_warning(warnings, QueryPlanWarning::MissingReadyIndex); return Ok(BooleanPlanResult { classification: BooleanPlanClassification::VerifyOnly, has_verify_only: true, }); }; - let probe = self.timestamp_candidate_probe( - query, - cap_context, - type_id, - *lower_ms, - *upper_ms, - budget, - )?; + let probe = self.best_candidate_probe_for_labels(label_ids, |label_id| { + self.timestamp_candidate_probe( + query, + cap_context, + label_id, + *lower_ms, + *upper_ms, + budget, + ) + })?; Ok(self.classification_from_probe(probe, structural_key, warnings, followups)) } NormalizedNodeFilter::PropertyExists { .. } @@ -2313,7 +5126,7 @@ impl ReadView { &self, query: &NormalizedNodeQuery, ) -> Result { - let mut warnings = Vec::new(); + let mut warnings = query.warnings.clone(); if query.filter.is_always_false() { finalize_plan_warnings(&mut warnings); return Ok(PlannedNodeQuery { @@ -2373,7 +5186,7 @@ impl ReadView { if driver_candidates.is_empty() { return Err(EngineError::InvalidOperation( - "node query requires type_id, ids, keys, or allow_full_scan".into(), + "node query requires label_filter, ids, keys, or allow_full_scan".into(), )); } @@ -2383,53 +5196,333 @@ impl ReadView { .cloned() .expect("driver candidates must be non-empty"); - if let Some(bounded_plan) = bounded_filter_plan.as_ref() { - if bounded_plan.canonical_key() != driver.canonical_key() { - if let (Some(selected), Some(bounded)) = ( - driver.estimate().known_upper_bound(), - bounded_plan.estimate().known_upper_bound(), - ) { - if bounded > selected.saturating_mul(QUERY_BROAD_SOURCE_FACTOR) - && bounded_plan.broad_skip_warnable() - { - add_plan_warning(&mut warnings, QueryPlanWarning::IndexSkippedAsBroad); - } + if let Some(bounded_plan) = bounded_filter_plan.as_ref() { + if bounded_plan.canonical_key() != driver.canonical_key() { + if let (Some(selected), Some(bounded)) = ( + driver.estimate().known_upper_bound(), + bounded_plan.estimate().known_upper_bound(), + ) { + if bounded > selected.saturating_mul(QUERY_BROAD_SOURCE_FACTOR) + && bounded_plan.broad_skip_warnable() + { + add_plan_warning(&mut warnings, QueryPlanWarning::IndexSkippedAsBroad); + } + } + } + } + + match &driver { + NodePhysicalPlan::Source(source) + if source.kind == NodeQueryCandidateSourceKind::FallbackNodeLabelScan => + { + add_plan_warning(&mut warnings, QueryPlanWarning::UsingFallbackScan); + } + NodePhysicalPlan::Source(source) + if source.kind == NodeQueryCandidateSourceKind::FallbackFullNodeScan => + { + add_plan_warning(&mut warnings, QueryPlanWarning::FullScanExplicitlyAllowed); + } + _ => {} + } + + finalize_plan_warnings(&mut warnings); + Ok(PlannedNodeQuery { + driver, + cap_context, + warnings, + }) + } + + fn explain_node_query(&self, query: &NodeQuery) -> Result { + let normalized = self.normalize_node_query(query)?; + let public_inputs = self.public_inputs_for_node_query(query)?; + let planned = self.plan_normalized_node_query(&normalized)?; + let mut plan = planned.explain_plan(public_inputs); + plan.notes = Self::node_query_explain_notes(&normalized, &planned.driver); + Ok(plan) + } + + fn edge_legal_universe_sources( + &self, + query: &NormalizedEdgeQuery, + ) -> Vec { + let mut sources = Vec::new(); + if !query.ids.is_empty() { + sources.push(PlannedEdgeCandidateSource::with_ids( + EdgeQueryCandidateSourceKind::ExplicitEdgeIds, + "edge_ids".to_string(), + query.ids.clone(), + )); + } + + let label_filter_ids = query.label_id.map(|label_id| vec![label_id]); + if let Some(label_id) = query.label_id { + sources.push(PlannedEdgeCandidateSource::edge_label_index( + label_id, + self.edge_label_estimate(label_id), + )); + } + if !query.from_ids.is_empty() { + let estimate = self.edge_endpoint_estimate( + &query.from_ids, + Direction::Outgoing, + label_filter_ids.as_deref(), + ); + sources.push(PlannedEdgeCandidateSource::endpoint_adjacency( + EdgeQueryCandidateSourceKind::FromEndpointAdjacency, + query.from_ids.clone(), + label_filter_ids.clone(), + estimate, + )); + } + if !query.to_ids.is_empty() { + let estimate = + self.edge_endpoint_estimate(&query.to_ids, Direction::Incoming, label_filter_ids.as_deref()); + sources.push(PlannedEdgeCandidateSource::endpoint_adjacency( + EdgeQueryCandidateSourceKind::ToEndpointAdjacency, + query.to_ids.clone(), + label_filter_ids.clone(), + estimate, + )); + } + if !query.endpoint_ids.is_empty() { + let estimate = + self.edge_endpoint_estimate(&query.endpoint_ids, Direction::Both, label_filter_ids.as_deref()); + sources.push(PlannedEdgeCandidateSource::endpoint_adjacency( + EdgeQueryCandidateSourceKind::AnyEndpointAdjacency, + query.endpoint_ids.clone(), + label_filter_ids, + estimate, + )); + } + if query.allow_full_scan { + sources.push(PlannedEdgeCandidateSource::fallback_full_scan( + self.edge_full_scan_estimate(), + )); + } + sources + } + + fn edge_query_cap_context(&self, query: &NormalizedEdgeQuery) -> EdgeQueryCapContext { + let cheapest_legal_universe = self + .edge_legal_universe_sources(query) + .into_iter() + .map(|source| source.estimate) + .filter_map(|estimate| estimate.known_upper_bound().map(|count| (count, estimate))) + .min_by_key(|(count, _)| *count) + .map(|(_, estimate)| estimate); + EdgeQueryCapContext { + cheapest_legal_universe, + } + } + + fn plan_normalized_edge_query( + &self, + query: &NormalizedEdgeQuery, + ) -> Result { + let mut warnings = query.warnings.clone(); + let cap_context = self.edge_query_cap_context(query); + if query.filter.is_always_false() { + return Ok(PlannedEdgeQuery { + driver: EdgePhysicalPlan::Empty, + cap_context, + warnings, + followups: Vec::new(), + }); + } + + let mut inputs = Vec::new(); + if !query.ids.is_empty() { + inputs.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::with_ids( + EdgeQueryCandidateSourceKind::ExplicitEdgeIds, + "edge_ids".to_string(), + query.ids.clone(), + ), + )); + } + + let triple_source_used = if let (Some(label_id), [from], [to]) = + (query.label_id, query.from_ids.as_slice(), query.to_ids.as_slice()) + { + inputs.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::edge_triple_index(*from, *to, label_id), + )); + true + } else { + false + }; + + let label_filter_ids = query.label_id.map(|label_id| vec![label_id]); + if !triple_source_used { + if let Some(label_id) = query.label_id { + inputs.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::edge_label_index( + label_id, + self.edge_label_estimate(label_id), + ), + )); + } + if !query.from_ids.is_empty() { + let estimate = self.edge_endpoint_estimate( + &query.from_ids, + Direction::Outgoing, + label_filter_ids.as_deref(), + ); + inputs.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::endpoint_adjacency( + EdgeQueryCandidateSourceKind::FromEndpointAdjacency, + query.from_ids.clone(), + label_filter_ids.clone(), + estimate, + ), + )); + } + if !query.to_ids.is_empty() { + let estimate = self.edge_endpoint_estimate( + &query.to_ids, + Direction::Incoming, + label_filter_ids.as_deref(), + ); + inputs.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::endpoint_adjacency( + EdgeQueryCandidateSourceKind::ToEndpointAdjacency, + query.to_ids.clone(), + label_filter_ids.clone(), + estimate, + ), + )); + } + } + if !query.endpoint_ids.is_empty() { + let estimate = self.edge_endpoint_estimate( + &query.endpoint_ids, + Direction::Both, + label_filter_ids.as_deref(), + ); + inputs.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::endpoint_adjacency( + EdgeQueryCandidateSourceKind::AnyEndpointAdjacency, + query.endpoint_ids.clone(), + label_filter_ids, + estimate, + ), + )); + } + + let has_filter = !query.filter.is_always_true(); + let mut budget = BooleanPlanningBudget::new(); + let mut filter_followups = Vec::new(); + let filter_plan = if has_filter { + self.plan_edge_filter_subtree( + query, + cap_context, + &query.filter, + self.edge_metadata_sidecar_availability(), + &mut budget, + &mut warnings, + &mut filter_followups, + )? + } else { + EdgeBooleanPlanResult { + classification: EdgeBooleanPlanClassification::VerifyOnly, + has_verify_only: false, + } + }; + if matches!( + &filter_plan.classification, + EdgeBooleanPlanClassification::AlwaysFalse + ) { + finalize_plan_warnings(&mut warnings); + return Ok(PlannedEdgeQuery { + driver: EdgePhysicalPlan::Empty, + cap_context, + warnings, + followups: filter_followups, + }); + } + if let EdgeBooleanPlanClassification::Bounded { plan, .. } = &filter_plan.classification { + inputs.push(plan.clone()); + } + if has_filter && filter_plan.has_verify_only && edge_filter_requires_hydration(&query.filter) { + add_plan_warning(&mut warnings, QueryPlanWarning::EdgePropertyPostFilter); + add_plan_warning(&mut warnings, QueryPlanWarning::VerifyOnlyFilter); + } + + if inputs.is_empty() { + if query.allow_full_scan { + add_plan_warning(&mut warnings, QueryPlanWarning::FullScanExplicitlyAllowed); + inputs.push(EdgePhysicalPlan::source( + PlannedEdgeCandidateSource::fallback_full_scan( + self.edge_full_scan_estimate(), + ), + )); + } else { + return Err(EngineError::InvalidOperation( + "edge query requires label, ids, from_ids, to_ids, endpoint_ids, or allow_full_scan".into(), + )); + } + } else if query.allow_full_scan + && query.label_id.is_none() + && query.ids.is_empty() + && query.from_ids.is_empty() + && query.to_ids.is_empty() + && query.endpoint_ids.is_empty() + { + add_plan_warning(&mut warnings, QueryPlanWarning::FullScanExplicitlyAllowed); + } + + for input in &inputs { + if let EdgePhysicalPlan::Source(source) = input { + if source.kind == EdgeQueryCandidateSourceKind::EdgeMetadataScan { + add_plan_warning(&mut warnings, QueryPlanWarning::UsingFallbackScan); + } + if source.broad_skip_warnable() + && cap_context.source_estimate_exceeds_cap( + source.kind, + query.page.limit, + source.estimate, + ) + { + add_plan_warning(&mut warnings, edge_cap_warning_for_source(source.kind)); } } } - match &driver { - NodePhysicalPlan::Source(source) - if source.kind == NodeQueryCandidateSourceKind::FallbackTypeScan => - { - add_plan_warning(&mut warnings, QueryPlanWarning::UsingFallbackScan); - } - NodePhysicalPlan::Source(source) - if source.kind == NodeQueryCandidateSourceKind::FallbackFullNodeScan => - { - add_plan_warning(&mut warnings, QueryPlanWarning::FullScanExplicitlyAllowed); + inputs.sort_by_key(EdgePhysicalPlan::plan_cost); + if let Some(driver) = inputs.first() { + if let Some(driver_count) = driver.estimate().known_upper_bound() { + for skipped in inputs.iter().skip(1) { + if let Some(skipped_count) = skipped.estimate().known_upper_bound() { + if skipped_count > driver_count.saturating_mul(QUERY_BROAD_SOURCE_FACTOR) + && skipped.broad_skip_warnable() + { + add_plan_warning(&mut warnings, QueryPlanWarning::IndexSkippedAsBroad); + } + } + } } - _ => {} } - finalize_plan_warnings(&mut warnings); - Ok(PlannedNodeQuery { - driver, + Ok(PlannedEdgeQuery { + driver: EdgePhysicalPlan::intersect(inputs), cap_context, warnings, + followups: filter_followups, }) } - fn explain_node_query(&self, query: &NodeQuery) -> Result { - let normalized = self.normalize_node_query(query)?; - let planned = self.plan_normalized_node_query(&normalized)?; - Ok(planned.explain_plan()) + fn explain_edge_query(&self, query: &EdgeQuery) -> Result { + let normalized = self.normalize_edge_query(query)?; + let public_inputs = self.public_inputs_for_edge_query(query)?; + let planned = self.plan_normalized_edge_query(&normalized)?; + Ok(planned.explain_plan(public_inputs)) } fn normalized_node_pattern_has_initial_universe(pattern: &NormalizedNodePattern) -> bool { !pattern.query.ids.is_empty() || !pattern.query.keys.is_empty() - || pattern.query.type_id.is_some() + || normalized_query_has_label_anchor(&pattern.query) || pattern.query.filter.is_always_false() } @@ -2441,7 +5534,7 @@ impl ReadView { return Some(pattern.query.ids.clone()); } if pattern.query.keys.is_empty() - || pattern.query.type_id.is_none() + || pattern.query.single_label_id.is_none() || pattern.query.keys.len() as u64 > TINY_EXPLICIT_ANCHOR_MAX { return None; @@ -2473,7 +5566,7 @@ impl ReadView { fn fanout_rollup_estimate( &self, direction: PlannerStatsDirection, - edge_type_id: Option, + edge_label_id: Option, coverage: FanoutCoverage, confidence: EstimateConfidence, canonical_key: String, @@ -2482,7 +5575,7 @@ impl ReadView { let rollup = self .planner_stats .adjacency_rollups - .get(&(direction, edge_type_id))?; + .get(&(direction, edge_label_id))?; let coverage = if self.has_unrolled_memtable_edges() || rollup.coverage.has_uncovered() { FanoutCoverage::GlobalFallback } else { @@ -2504,14 +5597,14 @@ impl ReadView { fn single_direction_fanout_estimate( &self, direction: PlannerStatsDirection, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, known_source_ids: Option<&[u64]>, ) -> FanoutEstimate { let direction_key = match direction { PlannerStatsDirection::Outgoing => "out", PlannerStatsDirection::Incoming => "in", }; - let Some(types) = type_filter else { + let Some(label_ids) = label_filter_ids else { return self .fanout_rollup_estimate( direction, @@ -2524,14 +5617,14 @@ impl ReadView { .unwrap_or_else(|| FanoutEstimate::unknown(format!("{direction_key}:global"))); }; - if types.len() == 1 { - let type_id = types[0]; + if label_ids.len() == 1 { + let label_id = label_ids[0]; if let Some(estimate) = self.fanout_rollup_estimate( direction, - Some(type_id), + Some(label_id), FanoutCoverage::Complete, EstimateConfidence::High, - format!("{direction_key}:type:{type_id}"), + format!("{direction_key}:label:{label_id}"), known_source_ids, ) { return estimate; @@ -2543,7 +5636,7 @@ impl ReadView { .get(&(direction, None)) .is_some_and(|rollup| !rollup.coverage.has_uncovered()) { - return FanoutEstimate::zero(format!("{direction_key}:type:{type_id}:absent")); + return FanoutEstimate::zero(format!("{direction_key}:label:{label_id}:absent")); } return self .fanout_rollup_estimate( @@ -2551,22 +5644,22 @@ impl ReadView { None, FanoutCoverage::GlobalFallback, EstimateConfidence::Low, - format!("{direction_key}:global_fallback:{type_id}"), + format!("{direction_key}:global_fallback:{label_id}"), known_source_ids, ) .unwrap_or_else(|| { - FanoutEstimate::unknown(format!("{direction_key}:missing:{type_id}")) + FanoutEstimate::unknown(format!("{direction_key}:missing:{label_id}")) }); } let mut combined: Option = None; - for type_id in types { + for label_id in label_ids { let Some(estimate) = self.fanout_rollup_estimate( direction, - Some(*type_id), + Some(*label_id), FanoutCoverage::Complete, EstimateConfidence::High, - format!("{direction_key}:type:{type_id}"), + format!("{direction_key}:label:{label_id}"), known_source_ids, ) else { return self @@ -2575,17 +5668,17 @@ impl ReadView { None, FanoutCoverage::GlobalFallback, EstimateConfidence::Low, - format!("{direction_key}:global_fallback:{types:?}"), + format!("{direction_key}:global_fallback:{label_ids:?}"), known_source_ids, ) .unwrap_or_else(|| { - FanoutEstimate::unknown(format!("{direction_key}:missing:{types:?}")) + FanoutEstimate::unknown(format!("{direction_key}:missing:{label_ids:?}")) }); }; combined = Some(match combined { Some(current) => current.combine_sum( estimate, - format!("{direction_key}:types:{types:?}"), + format!("{direction_key}:labels:{label_ids:?}"), ), None => estimate, }); @@ -2597,7 +5690,7 @@ impl ReadView { fn pattern_edge_fanout_estimate( &self, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, known_source_ids: Option<&[u64]>, query: &NormalizedGraphPatternQuery, edge: &NormalizedEdgePattern, @@ -2606,13 +5699,13 @@ impl ReadView { for planner_direction in Self::planner_direction(direction) { let estimate = self.single_direction_fanout_estimate( *planner_direction, - type_filter, + label_filter_ids, known_source_ids, ); combined = Some(match combined { Some(current) => current.combine_sum( estimate, - format!("{direction:?}:{:?}", type_filter.unwrap_or(&[])), + format!("{direction:?}:{:?}", label_filter_ids.unwrap_or(&[])), ), None => estimate, }); @@ -2626,7 +5719,7 @@ impl ReadView { if !self.manifest.prune_policies.is_empty() { downgrade_steps = downgrade_steps.saturating_add(1); } - if !edge.property_predicates.is_empty() { + if edge_filter_requires_hydration(&edge.filter) { downgrade_steps = downgrade_steps.saturating_add(1); } if downgrade_steps > 0 { @@ -2694,6 +5787,7 @@ impl ReadView { Ok(Some(planned)) } + #[allow(dead_code)] fn planned_pattern_expansion_order_legacy( &self, query: &NormalizedGraphPatternQuery, @@ -2749,18 +5843,42 @@ impl ReadView { ) -> (Vec, PatternPlanCost) { let mut bound = vec![false; query.nodes.len()]; bound[anchor_index] = true; - let mut visited_edges = vec![false; query.edges.len()]; - let mut order = Vec::with_capacity(query.edges.len()); - let anchor_cost = anchor_plan.driver.plan_cost(); + let visited_edges = vec![false; query.edges.len()]; + self.planned_pattern_expansion_order_from_frontier( + query, + bound, + visited_edges, + anchor_plan.driver.plan_cost(), + anchor_plan.estimated_candidate_count().unwrap_or(1).max(1), + known_node_ids, + target_selectivities, + format!("{}:{anchor_index}", query.nodes[anchor_index].alias), + ) + } + + #[allow(clippy::too_many_arguments)] + fn planned_pattern_expansion_order_from_frontier( + &self, + query: &NormalizedGraphPatternQuery, + mut bound: Vec, + mut visited_edges: Vec, + anchor_cost: PlanCost, + initial_frontier: u64, + known_node_ids: &[Option>], + target_selectivities: &[Option<(u64, u64)>], + canonical_key: String, + ) -> (Vec, PatternPlanCost) { + let target_order_len = visited_edges.iter().filter(|visited| !**visited).count(); + let mut order = Vec::with_capacity(target_order_len); let mut total_work = anchor_cost.estimated_work; - let mut frontier = anchor_plan.estimated_candidate_count().unwrap_or(1).max(1); + let mut frontier = initial_frontier.max(1); let mut fanout_complete = true; let mut confidence = EstimateConfidence::Exact; let mut stale_risk = StalePostingRisk::Low; let mut frontier_capped = frontier > PATTERN_FRONTIER_BUDGET as u64; frontier = frontier.min(PATTERN_FRONTIER_BUDGET as u64 + 1); - while order.len() < query.edges.len() { + while order.len() < target_order_len { let mut choices = Vec::new(); for (edge_index, edge) in query.edges.iter().enumerate() { if visited_edges[edge_index] { @@ -2783,7 +5901,7 @@ impl ReadView { }; let fanout = self.pattern_edge_fanout_estimate( direction, - edge.type_filter.as_deref(), + edge.label_filter_ids.as_deref(), known_node_ids[source_index].as_deref(), query, edge, @@ -2796,7 +5914,7 @@ impl ReadView { }; let mut estimated_expansion = Self::apply_pattern_target_selectivity(raw_expansion, target_selectivity); - if !edge.property_predicates.is_empty() { + if edge_filter_requires_hydration(&edge.filter) { estimated_expansion = estimated_expansion.saturating_add(raw_expansion); } let next_frontier = if bound[target_index] { @@ -2874,9 +5992,7 @@ impl ReadView { frontier = next_frontier.min(PATTERN_FRONTIER_BUDGET as u64 + 1); } - if order.len() != query.edges.len() { - let legacy = self.planned_pattern_expansion_order_legacy(query, anchor_index); - order = legacy; + if order.len() != target_order_len { fanout_complete = false; total_work = anchor_cost.estimated_work; } @@ -2889,11 +6005,413 @@ impl ReadView { confidence_rank: confidence.rank(), stale_risk_rank: stale_risk.rank(), frontier_capped, - canonical_key: format!("{}:{anchor_index}", query.nodes[anchor_index].alias), + canonical_key, }; (order, cost) } + fn pattern_edge_anchor_query( + label_id: Option, + filter: NormalizedEdgeFilter, + ) -> NormalizedEdgeQuery { + NormalizedEdgeQuery { + label_id, + ids: Vec::new(), + from_ids: Vec::new(), + to_ids: Vec::new(), + endpoint_ids: Vec::new(), + filter, + allow_full_scan: false, + page: PageRequest { + limit: Some(PATTERN_FRONTIER_BUDGET), + after: None, + }, + warnings: Vec::new(), + } + } + + fn pattern_edge_anchor_cap_context(&self, label_id: Option) -> EdgeQueryCapContext { + EdgeQueryCapContext { + cheapest_legal_universe: Some( + label_id + .map(|label_id| self.edge_label_estimate(label_id)) + .unwrap_or_else(|| self.edge_full_scan_estimate()), + ), + } + } + + fn edge_plan_exceeds_pattern_anchor_cap( + &self, + edge_query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + plan: &EdgePhysicalPlan, + ) -> bool { + if matches!(plan, EdgePhysicalPlan::Empty) { + return false; + } + plan.estimate_exceeds_cap(cap_context, edge_query.page.limit) + } + + fn bounded_edge_label_anchor_plan( + &self, + edge_query: &NormalizedEdgeQuery, + cap_context: EdgeQueryCapContext, + label_id: u32, + ) -> Option { + let source = PlannedEdgeCandidateSource::edge_label_index( + label_id, + self.edge_label_estimate(label_id), + ); + if cap_context.source_estimate_exceeds_cap( + source.kind, + edge_query.page.limit, + source.estimate, + ) { + return None; + } + Some(EdgePhysicalPlan::source(source)) + } + + fn pattern_edge_anchor_cap_context_for_label_filter( + &self, + label_filter_ids: Option<&[u32]>, + ) -> EdgeQueryCapContext { + match label_filter_ids { + Some([label_id]) => self.pattern_edge_anchor_cap_context(Some(*label_id)), + Some(label_ids) => { + let count = label_ids.iter().fold(0u64, |total, label_id| { + total.saturating_add( + self.edge_label_estimate(*label_id) + .known_upper_bound() + .unwrap_or(u64::MAX / 4), + ) + }); + EdgeQueryCapContext { + cheapest_legal_universe: Some(PlannerEstimate::upper_bound(count)), + } + } + None => self.pattern_edge_anchor_cap_context(None), + } + } + + fn pattern_edge_bounded_label_fallback_plan( + &self, + edge: &NormalizedEdgePattern, + ) -> Option { + let label_ids = edge.label_filter_ids.as_deref()?; + let edge_query = Self::pattern_edge_anchor_query(None, edge.filter.clone()); + let cap_context = + self.pattern_edge_anchor_cap_context_for_label_filter(edge.label_filter_ids.as_deref()); + let driver = match label_ids { + [label_id] => { + let branch_query = Self::pattern_edge_anchor_query(Some(*label_id), edge.filter.clone()); + let branch_cap_context = self.pattern_edge_anchor_cap_context(Some(*label_id)); + self.bounded_edge_label_anchor_plan(&branch_query, branch_cap_context, *label_id)? + } + [] => return None, + label_ids => { + let mut branches = Vec::with_capacity(label_ids.len()); + for &label_id in label_ids { + let branch_query = + Self::pattern_edge_anchor_query(Some(label_id), edge.filter.clone()); + let branch_cap_context = self.pattern_edge_anchor_cap_context(Some(label_id)); + branches.push(self.bounded_edge_label_anchor_plan( + &branch_query, + branch_cap_context, + label_id, + )?); + } + EdgePhysicalPlan::union(branches) + } + }; + if self.edge_plan_exceeds_pattern_anchor_cap(&edge_query, cap_context, &driver) { + return None; + } + Some(PlannedEdgeQuery { + driver, + cap_context, + warnings: Vec::new(), + followups: Vec::new(), + }) + } + + fn pattern_edge_bounded_metadata_fallback_plan( + &self, + edge: &NormalizedEdgePattern, + availability: EdgeMetadataSidecarAvailability, + ) -> Option { + let edge_query = Self::pattern_edge_anchor_query(None, edge.filter.clone()); + let cap_context = + self.pattern_edge_anchor_cap_context_for_label_filter(edge.label_filter_ids.as_deref()); + let driver = match edge.label_filter_ids.as_deref() { + Some([label_id]) => { + self.edge_metadata_filter_candidate_plan(&edge.filter, Some(*label_id), availability)? + } + Some([]) => return None, + Some(label_ids) => { + let mut branches = Vec::with_capacity(label_ids.len()); + for &label_id in label_ids { + branches.push(self.edge_metadata_filter_candidate_plan( + &edge.filter, + Some(label_id), + availability, + )?); + } + EdgePhysicalPlan::union(branches) + } + None => self.edge_metadata_filter_candidate_plan(&edge.filter, None, availability)?, + }; + if self.edge_plan_exceeds_pattern_anchor_cap(&edge_query, cap_context, &driver) { + return None; + } + Some(PlannedEdgeQuery { + driver, + cap_context, + warnings: Vec::new(), + followups: Vec::new(), + }) + } + + fn pattern_edge_anchor_local_fallback_plans( + &self, + edge: &NormalizedEdgePattern, + availability: EdgeMetadataSidecarAvailability, + primary: &EdgePhysicalPlan, + ) -> Vec { + let primary_key = primary.canonical_key(); + let mut keys = vec![primary_key]; + let mut fallbacks = Vec::new(); + for candidate in [ + self.pattern_edge_bounded_metadata_fallback_plan(edge, availability), + self.pattern_edge_bounded_label_fallback_plan(edge), + ] + .into_iter() + .flatten() + { + let key = candidate.driver.canonical_key(); + if !keys.iter().any(|existing| existing == &key) { + keys.push(key); + fallbacks.push(candidate); + } + } + fallbacks.sort_by(|left, right| left.driver.plan_cost().cmp(&right.driver.plan_cost())); + fallbacks + } + + fn plan_pattern_edge_anchor_for_labeled_branch( + &self, + edge: &NormalizedEdgePattern, + label_id: u32, + availability: EdgeMetadataSidecarAvailability, + ) -> Result, EngineError> + { + let edge_query = Self::pattern_edge_anchor_query(Some(label_id), edge.filter.clone()); + let cap_context = self.pattern_edge_anchor_cap_context(Some(label_id)); + let mut warnings = Vec::new(); + let mut followups = Vec::new(); + + let plan = if edge.filter.is_always_false() { + Some(EdgePhysicalPlan::Empty) + } else if edge.filter.is_always_true() { + self.bounded_edge_label_anchor_plan(&edge_query, cap_context, label_id) + } else { + let mut budget = BooleanPlanningBudget::new(); + let planned = self.plan_edge_filter_subtree( + &edge_query, + cap_context, + &edge.filter, + availability, + &mut budget, + &mut warnings, + &mut followups, + )?; + match planned.classification { + EdgeBooleanPlanClassification::AlwaysFalse => Some(EdgePhysicalPlan::Empty), + EdgeBooleanPlanClassification::Bounded { plan, complete, .. } if complete => { + if planned.has_verify_only { + add_plan_warning(&mut warnings, QueryPlanWarning::VerifyOnlyFilter); + if edge_filter_requires_hydration(&edge.filter) { + add_plan_warning(&mut warnings, QueryPlanWarning::EdgePropertyPostFilter); + } + } + Some(plan) + } + _ => { + add_plan_warning(&mut warnings, QueryPlanWarning::VerifyOnlyFilter); + if edge_filter_requires_hydration(&edge.filter) { + add_plan_warning(&mut warnings, QueryPlanWarning::EdgePropertyPostFilter); + } + self.bounded_edge_label_anchor_plan(&edge_query, cap_context, label_id) + } + } + }; + + let Some(plan) = plan else { + add_plan_warning(&mut warnings, QueryPlanWarning::IndexSkippedAsBroad); + finalize_plan_warnings(&mut warnings); + return Ok(None); + }; + if self.edge_plan_exceeds_pattern_anchor_cap(&edge_query, cap_context, &plan) { + add_plan_warning(&mut warnings, QueryPlanWarning::IndexSkippedAsBroad); + finalize_plan_warnings(&mut warnings); + return Ok(None); + } + + finalize_plan_warnings(&mut warnings); + Ok(Some((plan, warnings, followups))) + } + + fn plan_pattern_edge_anchor_labelless( + &self, + edge: &NormalizedEdgePattern, + availability: EdgeMetadataSidecarAvailability, + ) -> Option<(EdgePhysicalPlan, Vec)> { + let edge_query = Self::pattern_edge_anchor_query(None, edge.filter.clone()); + let cap_context = self.pattern_edge_anchor_cap_context(None); + let plan = self.edge_metadata_filter_candidate_plan(&edge.filter, None, availability)?; + if self.edge_plan_exceeds_pattern_anchor_cap(&edge_query, cap_context, &plan) { + return None; + } + let mut warnings = Vec::new(); + if edge_filter_requires_hydration(&edge.filter) { + add_plan_warning(&mut warnings, QueryPlanWarning::EdgePropertyPostFilter); + add_plan_warning(&mut warnings, QueryPlanWarning::VerifyOnlyFilter); + } + finalize_plan_warnings(&mut warnings); + Some((plan, warnings)) + } + + fn plan_pattern_edge_anchor_source( + &self, + edge: &NormalizedEdgePattern, + ) -> Result, EngineError> { + let availability = self.edge_metadata_sidecar_availability(); + let mut warnings = Vec::new(); + let mut followups = Vec::new(); + let edge_query = Self::pattern_edge_anchor_query(None, edge.filter.clone()); + let cap_context = + self.pattern_edge_anchor_cap_context_for_label_filter(edge.label_filter_ids.as_deref()); + + let driver = match edge.label_filter_ids.as_deref() { + Some([]) => EdgePhysicalPlan::Empty, + Some([label_id]) => { + let Some((plan, mut branch_warnings, mut branch_followups)) = + self.plan_pattern_edge_anchor_for_labeled_branch(edge, *label_id, availability)? + else { + return Ok(None); + }; + warnings.append(&mut branch_warnings); + followups.append(&mut branch_followups); + plan + } + Some(label_ids) => { + let mut branches = Vec::with_capacity(label_ids.len()); + for &label_id in label_ids { + let Some((plan, mut branch_warnings, mut branch_followups)) = + self.plan_pattern_edge_anchor_for_labeled_branch(edge, label_id, availability)? + else { + return Ok(None); + }; + warnings.append(&mut branch_warnings); + followups.append(&mut branch_followups); + branches.push(plan); + } + EdgePhysicalPlan::union(branches) + } + None => { + let Some((plan, mut plan_warnings)) = + self.plan_pattern_edge_anchor_labelless(edge, availability) + else { + return Ok(None); + }; + warnings.append(&mut plan_warnings); + plan + } + }; + + if matches!(driver, EdgePhysicalPlan::Source(ref source) + if source.kind == EdgeQueryCandidateSourceKind::FallbackFullEdgeScan) + { + return Ok(None); + } + if self.edge_plan_exceeds_pattern_anchor_cap(&edge_query, cap_context, &driver) { + return Ok(None); + } + + finalize_plan_warnings(&mut warnings); + let edge_plan = PlannedEdgeQuery { + driver, + cap_context, + warnings, + followups, + }; + let edge_fallback_plans = + self.pattern_edge_anchor_local_fallback_plans(edge, availability, &edge_plan.driver); + Ok(Some(PlannedPatternEdgeAnchorSource { + edge_plan, + edge_fallback_plans, + })) + } + + #[allow(clippy::too_many_arguments)] + fn planned_pattern_edge_anchor( + &self, + query: &NormalizedGraphPatternQuery, + edge_index: usize, + edge_plan: PlannedEdgeQuery, + edge_fallback_plans: Vec, + known_node_ids: &[Option>], + target_selectivities: &[Option<(u64, u64)>], + sort_anchor_alias: String, + ) -> (PatternAnchorPlan, PatternPlanCost) { + let edge = &query.edges[edge_index]; + let mut bound = vec![false; query.nodes.len()]; + bound[edge.from_index] = true; + bound[edge.to_index] = true; + let mut visited_edges = vec![false; query.edges.len()]; + visited_edges[edge_index] = true; + + let mut anchor_cost = edge_plan.driver.plan_cost(); + let candidate_count = edge_plan.estimated_candidate_count().unwrap_or(1).max(1); + let endpoint_verify_count = if edge.from_index == edge.to_index { 1 } else { 2 }; + anchor_cost.estimated_work = anchor_cost + .estimated_work + .saturating_add(candidate_count.saturating_mul(endpoint_verify_count)); + let orientation_multiplier = match edge.direction { + Direction::Both if edge.from_index != edge.to_index => 2, + _ => 1, + }; + let initial_frontier = candidate_count.saturating_mul(orientation_multiplier); + let (expansion_order, cost) = self.planned_pattern_expansion_order_from_frontier( + query, + bound, + visited_edges, + anchor_cost, + initial_frontier, + known_node_ids, + target_selectivities, + format!( + "edge:{}:{}:{}", + edge.alias.as_deref().unwrap_or(""), + query.nodes[edge.from_index].alias, + query.nodes[edge.to_index].alias + ), + ); + let anchor = PatternAnchorPlan::Edge { + edge_index, + edge_alias: edge.alias.clone(), + from_index: edge.from_index, + to_index: edge.to_index, + sort_anchor_alias, + edge_query: Box::new(Self::pattern_edge_anchor_query(None, edge.filter.clone())), + edge_plan, + edge_fallback_plans, + expansion_order, + orientation_policy: EdgeAnchorOrientationPolicy::from_direction(edge.direction), + }; + (anchor, cost) + } + fn plan_normalized_pattern_query( &self, query: &NormalizedGraphPatternQuery, @@ -2908,18 +6426,24 @@ impl ReadView { .plan_normalized_node_pattern_anchor(pattern)? .expect("AlwaysFalse pattern must be an anchorable empty universe"); let mut warnings = anchor_plan.warnings.clone(); + for warning in &query.warnings { + add_plan_warning(&mut warnings, *warning); + } finalize_plan_warnings(&mut warnings); return Ok(PlannedPatternQuery { - anchor_index: node_index, - anchor_alias: pattern.alias.clone(), - sort_anchor_alias: pattern.alias.clone(), - expansion_order: Vec::new(), + anchor: PatternAnchorPlan::Node { + node_index, + anchor_alias: pattern.alias.clone(), + sort_anchor_alias: pattern.alias.clone(), + expansion_order: Vec::new(), + anchor_plan, + }, + fallback_anchors: Vec::new(), warnings, - anchor_plan, }); } - let mut anchor_candidates = Vec::new(); + let mut node_candidates = Vec::new(); let known_node_ids: Vec<_> = query .nodes .iter() @@ -2943,7 +6467,7 @@ impl ReadView { &known_node_ids, &target_selectivities, ); - anchor_candidates.push(( + node_candidates.push(( pattern_cost, pattern.alias.as_str(), sort_count, @@ -2954,13 +6478,7 @@ impl ReadView { )); } - anchor_candidates.sort_by(|left, right| { - left.0 - .cmp(&right.0) - .then_with(|| left.1.cmp(right.1)) - }); - - let sort_anchor_alias = anchor_candidates + let sort_anchor_alias = node_candidates .iter() .min_by(|left, right| { left.2 @@ -2968,46 +6486,137 @@ impl ReadView { .then_with(|| left.3.cmp(&right.3)) .then_with(|| left.1.cmp(right.1)) }) - .map(|candidate| candidate.1.to_string()); + .map(|candidate| candidate.1.to_string()) + .unwrap_or_else(|| { + query + .nodes + .iter() + .map(|node| node.alias.as_str()) + .min() + .unwrap_or("") + .to_string() + }); - let Some((_, _, _, _, anchor_index, expansion_order, anchor_plan)) = - anchor_candidates.into_iter().next() - else { + let mut anchor_candidates = Vec::new(); + for (pattern_cost, alias, _, _, node_index, expansion_order, anchor_plan) in node_candidates + { + anchor_candidates.push(( + pattern_cost, + alias.to_string(), + PatternAnchorPlan::Node { + node_index, + anchor_alias: query.nodes[node_index].alias.clone(), + sort_anchor_alias: sort_anchor_alias.clone(), + anchor_plan, + expansion_order, + }, + )); + } + + let mut edge_planning_warnings = Vec::new(); + for (edge_index, edge) in query.edges.iter().enumerate() { + let Some(edge_anchor_source) = self.plan_pattern_edge_anchor_source(edge)? else { + if edge_filter_requires_hydration(&edge.filter) { + add_plan_warning( + &mut edge_planning_warnings, + QueryPlanWarning::EdgePropertyPostFilter, + ); + add_plan_warning( + &mut edge_planning_warnings, + QueryPlanWarning::VerifyOnlyFilter, + ); + } + continue; + }; + for warning in &edge_anchor_source.edge_plan.warnings { + add_plan_warning(&mut edge_planning_warnings, *warning); + } + let (anchor, pattern_cost) = self.planned_pattern_edge_anchor( + query, + edge_index, + edge_anchor_source.edge_plan, + edge_anchor_source.edge_fallback_plans, + &known_node_ids, + &target_selectivities, + sort_anchor_alias.clone(), + ); + anchor_candidates.push(( + pattern_cost, + format!( + "edge:{}:{}:{}", + edge.alias.as_deref().unwrap_or(""), + query.nodes[edge.from_index].alias, + query.nodes[edge.to_index].alias + ), + anchor, + )); + } + + anchor_candidates.sort_by(|left, right| { + left.0 + .cmp(&right.0) + .then_with(|| left.1.cmp(&right.1)) + }); + + let mut anchor_candidates = anchor_candidates.into_iter(); + let Some((_, _, anchor)) = anchor_candidates.next() else { return Err(EngineError::InvalidOperation( - "pattern query requires an anchorable node pattern".into(), + "pattern query requires an anchorable node pattern or edge pattern".into(), )); }; + let fallback_anchors = anchor_candidates + .map(|(_, _, anchor)| anchor) + .collect::>(); - if expansion_order.len() != query.edges.len() { + let expected_expansions = match &anchor { + PatternAnchorPlan::Node { .. } => query.edges.len(), + PatternAnchorPlan::Edge { .. } => query.edges.len().saturating_sub(1), + }; + if anchor.expansion_order().len() != expected_expansions { return Err(EngineError::InvalidOperation( "pattern query must be one connected component".into(), )); } - let mut warnings = anchor_plan.warnings.clone(); - if query - .edges - .iter() - .any(|edge| !edge.property_predicates.is_empty()) - { - add_plan_warning(&mut warnings, QueryPlanWarning::EdgePropertyPostFilter); + let anchored_edge_index = match &anchor { + PatternAnchorPlan::Edge { edge_index, .. } => Some(*edge_index), + PatternAnchorPlan::Node { .. } => None, + }; + let mut warnings = match &anchor { + PatternAnchorPlan::Node { anchor_plan, .. } => anchor_plan.warnings.clone(), + PatternAnchorPlan::Edge { edge_plan, .. } => edge_plan.warnings.clone(), + }; + for warning in &query.warnings { + add_plan_warning(&mut warnings, *warning); + } + for warning in edge_planning_warnings { + add_plan_warning(&mut warnings, warning); + } + for (edge_index, edge) in query.edges.iter().enumerate() { + if Some(edge_index) == anchored_edge_index { + continue; + } + if edge_filter_requires_hydration(&edge.filter) { + add_plan_warning(&mut warnings, QueryPlanWarning::EdgePropertyPostFilter); + add_plan_warning(&mut warnings, QueryPlanWarning::VerifyOnlyFilter); + } } finalize_plan_warnings(&mut warnings); Ok(PlannedPatternQuery { - anchor_index, - anchor_alias: query.nodes[anchor_index].alias.clone(), - sort_anchor_alias: sort_anchor_alias.unwrap_or_else(|| query.nodes[anchor_index].alias.clone()), - expansion_order, + anchor, + fallback_anchors, warnings, - anchor_plan, }) } fn explain_pattern_query(&self, query: &GraphPatternQuery) -> Result { let normalized = self.normalize_pattern_query(query)?; + let public_inputs = self.public_inputs_for_pattern_query(query)?; let planned = self.plan_normalized_pattern_query(&normalized)?; - Ok(planned.explain_plan()) + let mut plan = planned.explain_plan(public_inputs); + plan.notes = Self::pattern_query_explain_notes(&normalized, &planned); + Ok(plan) } } @@ -3035,6 +6644,15 @@ mod query_plan_unit_tests { } } + fn edge_source_plan(kind: EdgeQueryCandidateSourceKind, count: u64) -> EdgePhysicalPlan { + EdgePhysicalPlan::source(PlannedEdgeCandidateSource { + kind, + canonical_key: format!("test-edge-source:{kind:?}:{count}"), + estimate: PlannerEstimate::stats_exact(count), + materialization: EdgeCandidateMaterialization::Precomputed(Vec::new()), + }) + } + #[test] fn plan_cost_lower_work_beats_source_rank() { let lower_work_worse_rank = plan_cost( @@ -3155,6 +6773,48 @@ mod query_plan_unit_tests { assert!(!PlannerEstimate::unknown().proves_empty()); } + #[test] + fn edge_plan_cap_uses_source_kind_for_broad_label_and_metadata_sources() { + let cap_context = EdgeQueryCapContext { + cheapest_legal_universe: Some(PlannerEstimate::upper_bound(100_000)), + }; + let broad_count = QUERY_RANGE_CANDIDATE_CAP as u64 + 1; + + let label_plan = edge_source_plan(EdgeQueryCandidateSourceKind::EdgeLabelIndex, broad_count); + assert!(label_plan.estimate_exceeds_cap(cap_context, Some(16))); + + let metadata_plan = + edge_source_plan(EdgeQueryCandidateSourceKind::EdgeMetadataScan, broad_count); + assert!(metadata_plan.estimate_exceeds_cap(cap_context, Some(16))); + + let equality_plan = + edge_source_plan(EdgeQueryCandidateSourceKind::EdgePropertyEqualityIndex, broad_count); + assert!(!equality_plan.estimate_exceeds_cap(cap_context, Some(16))); + } + + #[test] + fn edge_plan_cap_allows_selective_property_range_and_metadata_sources() { + let cap_context = EdgeQueryCapContext { + cheapest_legal_universe: Some(PlannerEstimate::upper_bound(100_000)), + }; + + let selective_metadata = + edge_source_plan(EdgeQueryCandidateSourceKind::EdgeMetadataScan, 128); + assert!(!selective_metadata.estimate_exceeds_cap(cap_context, Some(16))); + + let range_count = QUERY_RANGE_CANDIDATE_CAP as u64 + 256; + let range_plan = + edge_source_plan(EdgeQueryCandidateSourceKind::EdgePropertyRangeIndex, range_count); + assert!(!range_plan.estimate_exceeds_cap(cap_context, Some(16))); + + let union_count = QUERY_RANGE_CANDIDATE_CAP as u64 * 2 + 1; + let broad_union = EdgePhysicalPlan::union(vec![ + edge_source_plan(EdgeQueryCandidateSourceKind::EdgeMetadataScan, union_count / 2), + edge_source_plan(EdgeQueryCandidateSourceKind::EdgeMetadataScan, union_count.div_ceil(2)), + ]); + assert!(broad_union.estimate_exceeds_cap(cap_context, Some(16))); + } + #[test] fn unique_in_probe_values_preserves_distinct_values_with_same_hash() { let values = vec![ diff --git a/src/engine/read.rs b/src/engine/read.rs index 4b2adbc..7006e63 100644 --- a/src/engine/read.rs +++ b/src/engine/read.rs @@ -24,20 +24,51 @@ impl PartialOrd for SparseTopKEntry { } } -enum TypeScanNodeSource<'a> { +struct ResolvedVectorSearchScope { + start_node_id: u64, + max_depth: u32, + direction: Direction, + edge_label_filter: Option>, + at_epoch: Option, +} + +struct ResolvedVectorSearchRequest<'a> { + mode: VectorSearchMode, + dense_query: Option<&'a DenseVector>, + sparse_query: Option<&'a SparseVector>, + k: usize, + label_filter: ResolvedNodeLabelFilter, + ef_search: Option, + scope: Option, + dense_weight: Option, + sparse_weight: Option, + fusion_mode: Option, +} + +fn finish_verified_id_page(mut items: Vec, limit: usize) -> PageResult { + let next_cursor = if limit > 0 && items.len() > limit { + items.truncate(limit); + items.last().copied() + } else { + None + }; + PageResult { items, next_cursor } +} + +enum NodeLabelScanSource<'a> { Owned(Vec), Segment { segment: &'a SegmentReader, - posting: SegmentTypePosting, + posting: SegmentLabelPosting, }, } -impl TypeScanNodeSource<'_> { +impl NodeLabelScanSource<'_> { fn get_id(&self, index: usize) -> Result, EngineError> { match self { - TypeScanNodeSource::Owned(ids) => Ok(ids.get(index).copied()), - TypeScanNodeSource::Segment { segment, posting } => { - segment.node_type_id_at_posting(*posting, index) + NodeLabelScanSource::Owned(ids) => Ok(ids.get(index).copied()), + NodeLabelScanSource::Segment { segment, posting } => { + segment.node_id_at_label_posting(*posting, index) } } } @@ -47,12 +78,12 @@ impl TypeScanNodeSource<'_> { return Ok(0); }; match self { - TypeScanNodeSource::Owned(ids) => match ids.binary_search(&after) { + NodeLabelScanSource::Owned(ids) => match ids.binary_search(&after) { Ok(index) => Ok(index + 1), Err(index) => Ok(index), }, - TypeScanNodeSource::Segment { segment, posting } => { - segment.node_type_id_lower_bound_posting(*posting, after) + NodeLabelScanSource::Segment { segment, posting } => { + segment.node_label_posting_lower_bound(*posting, after) } } } @@ -532,7 +563,7 @@ impl ReadView { } /// Batch-compute the set of node IDs that should be excluded by prune policies. - /// Uses the batched merge-walk (`get_nodes_raw`) instead of N individual lookups. + /// Uses latest-source visibility metadata instead of hydrating full records. /// Returns an empty set when no policies are registered (zero overhead). fn policy_excluded_node_ids(&self, node_ids: &[u64]) -> Result { if self.manifest.prune_policies.is_empty() || node_ids.is_empty() { @@ -540,12 +571,12 @@ impl ReadView { } let cutoffs = PrecomputedPruneCutoffs::from_policies(&self.manifest.prune_policies, now_millis()); - let records = self.get_nodes_raw(node_ids)?; + let visibility = self.sources().find_node_visibility_meta(node_ids)?; let mut excluded = NodeIdSet::default(); - for (i, slot) in records.iter().enumerate() { - if let Some(ref node) = slot { - if cutoffs.excludes(node) { - excluded.insert(node_ids[i]); + for (&node_id, state) in node_ids.iter().zip(visibility.iter()) { + if let NodeVisibilityState::Live(meta) = state { + if cutoffs.excludes_fields(&meta.label_ids, meta.updated_at, meta.weight) { + excluded.insert(node_id); } } } @@ -567,14 +598,14 @@ impl ReadView { fn ready_equality_node_matches( node: Option<&NodeRecord>, - type_id: u32, + label_id: u32, prop_key: &str, prop_value: &PropValue, ) -> bool { let Some(node) = node else { return false; }; - if node.type_id != type_id + if !node.label_ids.contains(label_id) || !node .props .get(prop_key) @@ -597,11 +628,13 @@ impl ReadView { return Ok(NodeIdSet::default()); } - let records = self.get_nodes_raw(node_ids)?; + let visibility = self.sources().find_node_visibility_meta(node_ids)?; let mut visible = NodeIdSet::with_capacity_and_hasher(node_ids.len(), Default::default()); - for (&node_id, slot) in node_ids.iter().zip(records.iter()) { - if let Some(node) = slot { - if policy_cutoffs.is_none_or(|cutoffs| !cutoffs.excludes(node)) { + for (&node_id, state) in node_ids.iter().zip(visibility.iter()) { + if let NodeVisibilityState::Live(meta) = state { + if policy_cutoffs + .is_none_or(|cutoffs| !cutoffs.excludes_fields(&meta.label_ids, meta.updated_at, meta.weight)) + { visible.insert(node_id); } } @@ -657,7 +690,7 @@ impl ReadView { &self, merged_ids: &[u64], memtable_verified: &NodeIdSet, - type_id: u32, + label_id: u32, prop_key: &str, prop_value: &PropValue, ) -> Result { @@ -677,12 +710,11 @@ impl ReadView { return Ok(visible); } + let segment_candidates = + self.filter_node_ids_by_current_label(segment_candidates, label_id)?; let batch_results = self.get_nodes_raw(&segment_candidates)?; - for (id, node) in segment_candidates - .into_iter() - .zip(batch_results.into_iter()) - { - if Self::ready_equality_node_matches(node.as_ref(), type_id, prop_key, prop_value) { + for (id, node) in segment_candidates.into_iter().zip(batch_results.into_iter()) { + if Self::ready_equality_node_matches(node.as_ref(), label_id, prop_key, prop_value) { visible.insert(id); } } @@ -708,7 +740,7 @@ impl ReadView { prop_value, self.snapshot_seq, ); - let mut memtable_verified: NodeIdSet = memtable_ids.iter().copied().collect(); + let memtable_verified: NodeIdSet = memtable_ids.iter().copied().collect(); let mut deleted_above = self.memtable.collect_deleted_nodes_at(self.snapshot_seq); let mut segment_ids: Vec> = Vec::with_capacity(self.immutable_epochs.len() + self.segments.len()); @@ -724,7 +756,6 @@ impl ReadView { .into_iter() .filter(|id| !deleted_above.contains(id)) .collect(); - memtable_verified.extend(ids.iter().copied()); segment_ids.push(ids); deleted_above.extend(epoch.memtable.collect_deleted_nodes_at(self.snapshot_seq)); } @@ -967,7 +998,7 @@ impl ReadView { let deleted = NodeIdSet::default(); let page = PageRequest::default(); - let merged = merge_type_ids_paged(memtable_ids, segment_ids, &deleted, &page); + let merged = merge_record_ids_paged(memtable_ids, segment_ids, &deleted, &page); return Ok((Some(merged.items), followup)); }; @@ -1148,7 +1179,7 @@ impl ReadView { limit: Some(limit), after: None, }; - let merged = merge_type_ids_paged(memtable_ids, segment_ids, &deleted, &page); + let merged = merge_record_ids_paged(memtable_ids, segment_ids, &deleted, &page); Ok((Some(merged.items), followup)) } @@ -1230,40 +1261,103 @@ impl ReadView { Some(SecondaryIndexReadFollowup::RangeSidecarFailure { index_id, error }) } - fn nodes_by_type_paged_unfiltered( + fn nodes_by_single_label_id_paged_unfiltered( &self, - type_id: u32, + single_label_id: u32, page: &PageRequest, ) -> Result, EngineError> { - let deleted = self.sources().collect_deleted_nodes(); - let memtable_ids = self.memtable.visible_nodes_by_type(type_id, self.snapshot_seq); - let mut segment_ids: Vec> = - Vec::with_capacity(self.immutable_epochs.len() + self.segments.len()); + if self.immutable_epochs.is_empty() && self.segments.is_empty() { + let deleted = NodeIdSet::default(); + return Ok(merge_record_ids_paged( + self.memtable + .visible_nodes_by_label_id(single_label_id, self.snapshot_seq), + Vec::new(), + &deleted, + page, + )); + } + + let limit = page.limit.unwrap_or(0); + let target = page_verify_target(limit); + let chunk_limit = match page.limit { + Some(limit) if limit > 0 => limit.saturating_add(1).saturating_mul(4).max(limit + 1), + _ => QUERY_VERIFY_CHUNK, + }; + let mut collected = Vec::with_capacity(if limit > 0 { limit } else { 0 }); + + self.scan_raw_node_label_candidates(&[single_label_id], page.after, chunk_limit, |chunk| { + #[cfg(test)] + self.note_node_visibility_meta_reads(chunk.len()); + let visibility = self.sources().find_node_visibility_meta(chunk)?; + for (&node_id, state) in chunk.iter().zip(visibility.iter()) { + if matches!( + state, + NodeVisibilityState::Live(meta) if meta.label_ids.contains(single_label_id) + ) { + collected.push(node_id); + if collected.len() >= target { + return Ok(ControlFlow::Break(())); + } + } + } + Ok(ControlFlow::Continue(())) + })?; + + let next_cursor = if limit > 0 && collected.len() > limit { + collected.truncate(limit); + collected.last().copied() + } else { + None + }; + Ok(PageResult { + items: collected, + next_cursor, + }) + } + + fn single_label_node_sources( + &self, + single_label_id: u32, + ) -> Result>, EngineError> { + let mut sources = Vec::with_capacity(1 + self.immutable_epochs.len() + self.segments.len()); + sources.push(NodeLabelScanSource::Owned( + self.memtable + .visible_nodes_by_label_id(single_label_id, self.snapshot_seq), + )); for epoch in &self.immutable_epochs { - segment_ids.push(epoch.memtable.visible_nodes_by_type(type_id, self.snapshot_seq)); + sources.push(NodeLabelScanSource::Owned( + epoch + .memtable + .visible_nodes_by_label_id(single_label_id, self.snapshot_seq), + )); } - for seg in &self.segments { - segment_ids.push(seg.nodes_by_type(type_id)?); + for segment in &self.segments { + if let Some(posting) = segment.node_label_posting(single_label_id)? { + sources.push(NodeLabelScanSource::Segment { + segment: segment.as_ref(), + posting, + }); + } } - Ok(merge_type_ids_paged( - memtable_ids, - segment_ids, - &deleted, - page, - )) + Ok(sources) } - fn scan_type_ids_unfiltered( + fn scan_raw_node_label_candidates( &self, - type_id: u32, + label_ids: &[u32], start_after: Option, + chunk_limit: usize, mut visitor: F, ) -> Result<(), EngineError> where - F: FnMut(u64) -> ControlFlow<()>, + F: FnMut(&[u64]) -> Result, EngineError>, { - let deleted = self.sources().collect_deleted_nodes(); - let sources = self.type_scan_node_sources(type_id)?; + let chunk_limit = chunk_limit.max(1); + let mut sources = Vec::new(); + for &label_id in label_ids { + sources.extend(self.single_label_node_sources(label_id)?); + } + let mut heap = BinaryHeap::new(); for (source_index, source) in sources.iter().enumerate() { let start = source.seek_after(start_after)?; @@ -1272,6 +1366,7 @@ impl ReadView { } } + let mut chunk = Vec::with_capacity(chunk_limit); let mut last_seen = None; while let Some(Reverse((node_id, source_index, offset))) = heap.pop() { let next_offset = offset + 1; @@ -1283,47 +1378,24 @@ impl ReadView { continue; } last_seen = Some(node_id); - if deleted.contains(&node_id) { - continue; - } - if visitor(node_id).is_break() { - return Ok(()); + chunk.push(node_id); + if chunk.len() >= chunk_limit { + if visitor(&chunk)?.is_break() { + return Ok(()); + } + chunk.clear(); } } - Ok(()) - } - - fn type_scan_node_sources( - &self, - type_id: u32, - ) -> Result>, EngineError> { - let mut sources = Vec::with_capacity(1 + self.immutable_epochs.len() + self.segments.len()); - sources.push(TypeScanNodeSource::Owned( - self.memtable - .visible_nodes_by_type(type_id, self.snapshot_seq), - )); - for epoch in &self.immutable_epochs { - sources.push(TypeScanNodeSource::Owned( - epoch - .memtable - .visible_nodes_by_type(type_id, self.snapshot_seq), - )); - } - for segment in &self.segments { - if let Some(posting) = segment.node_type_posting(type_id)? { - sources.push(TypeScanNodeSource::Segment { - segment: segment.as_ref(), - posting, - }); - } + if !chunk.is_empty() { + let _ = visitor(&chunk)?; } - Ok(sources) + Ok(()) } - fn scan_nodes_by_type_filtered( + fn scan_nodes_by_single_label_id_filtered( &self, - type_id: u32, + single_label_id: u32, start_after: Option, chunk_limit: usize, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, @@ -1333,7 +1405,7 @@ impl ReadView { F: FnMut(u64, &NodeRecord) -> Result, EngineError>, { let chunk_limit = chunk_limit.max(1); - let sources = self.type_scan_node_sources(type_id)?; + let sources = self.single_label_node_sources(single_label_id)?; let mut heap = BinaryHeap::new(); for (source_index, source) in sources.iter().enumerate() { let start = source.seek_after(start_after)?; @@ -1357,7 +1429,12 @@ impl ReadView { chunk.push(node_id); if chunk.len() >= chunk_limit { if self - .visit_type_scan_chunk(type_id, &chunk, policy_cutoffs, &mut visitor)? + .visit_single_label_scan_chunk( + single_label_id, + &chunk, + policy_cutoffs, + &mut visitor, + )? .is_break() { return Ok(()); @@ -1367,14 +1444,19 @@ impl ReadView { } if !chunk.is_empty() { - let _ = self.visit_type_scan_chunk(type_id, &chunk, policy_cutoffs, &mut visitor)?; + let _ = self.visit_single_label_scan_chunk( + single_label_id, + &chunk, + policy_cutoffs, + &mut visitor, + )?; } Ok(()) } - fn visit_type_scan_chunk( + fn visit_single_label_scan_chunk( &self, - type_id: u32, + single_label_id: u32, chunk: &[u64], policy_cutoffs: Option<&PrecomputedPruneCutoffs>, visitor: &mut F, @@ -1387,7 +1469,7 @@ impl ReadView { let Some(node) = node.as_ref() else { continue; }; - if node.type_id != type_id { + if !node.label_ids.contains(single_label_id) { continue; } if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { @@ -1484,14 +1566,14 @@ impl ReadView { fn find_nodes_scan_fallback( &self, - type_id: u32, + label_id: u32, prop_key: &str, prop_value: &PropValue, ) -> Result, EngineError> { let policy_cutoffs = self.query_policy_cutoffs(); let mut results = Vec::new(); - self.scan_nodes_by_type_filtered( - type_id, + self.scan_nodes_by_single_label_id_filtered( + label_id, None, 256, policy_cutoffs.as_ref(), @@ -1511,7 +1593,7 @@ impl ReadView { fn find_nodes_paged_scan_fallback( &self, - type_id: u32, + label_id: u32, prop_key: &str, prop_value: &PropValue, page: &PageRequest, @@ -1525,8 +1607,8 @@ impl ReadView { if limit == 0 { let mut items = Vec::new(); - self.scan_nodes_by_type_filtered( - type_id, + self.scan_nodes_by_single_label_id_filtered( + label_id, page.after, chunk_limit, policy_cutoffs.as_ref(), @@ -1547,10 +1629,10 @@ impl ReadView { }); } + let target = page_verify_target(limit); let mut items = Vec::with_capacity(limit); - let mut next_cursor = None; - self.scan_nodes_by_type_filtered( - type_id, + self.scan_nodes_by_single_label_id_filtered( + label_id, page.after, chunk_limit, policy_cutoffs.as_ref(), @@ -1561,26 +1643,25 @@ impl ReadView { .is_some_and(|value| value == prop_value) { items.push(node_id); - if items.len() >= limit { - next_cursor = Some(node_id); + if items.len() >= target { return Ok(ControlFlow::Break(())); } } Ok(ControlFlow::Continue(())) }, )?; - Ok(PageResult { items, next_cursor }) + Ok(finish_verified_id_page(items, limit)) } fn find_nodes_ready_equality_index( &self, - type_id: u32, + label_id: u32, index_id: u64, prop_key: &str, prop_value: &PropValue, ) -> Result<(Option>, Option), EngineError> { let (page, followup) = self.find_nodes_paged_ready_equality_index( - type_id, + label_id, index_id, prop_key, prop_value, @@ -1591,7 +1672,7 @@ impl ReadView { fn find_nodes_paged_ready_equality_index( &self, - type_id: u32, + label_id: u32, index_id: u64, prop_key: &str, prop_value: &PropValue, @@ -1615,11 +1696,11 @@ impl ReadView { limit: None, after: page.after, }; - let merged = merge_type_ids_paged(memtable_ids, segment_ids, &deleted, &all_page); + let merged = merge_record_ids_paged(memtable_ids, segment_ids, &deleted, &all_page); let visible = self.ready_equality_verified_node_ids( &merged.items, &memtable_verified, - type_id, + label_id, prop_key, prop_value, )?; @@ -1637,7 +1718,11 @@ impl ReadView { followup, )) } else { - let chunk_limit = limit.saturating_mul(4).max(limit); + let target = page_verify_target(limit); + let chunk_limit = limit + .saturating_add(1) + .saturating_mul(4) + .max(limit.saturating_add(1)); let mut collected = Vec::with_capacity(limit); let mut cursor = page.after; @@ -1646,7 +1731,7 @@ impl ReadView { limit: Some(chunk_limit), after: cursor, }; - let merged = merge_type_ids_paged( + let merged = merge_record_ids_paged( memtable_ids.clone(), segment_ids.clone(), &deleted, @@ -1665,7 +1750,7 @@ impl ReadView { let visible = self.ready_equality_verified_node_ids( &merged.items, &memtable_verified, - type_id, + label_id, prop_key, prop_value, )?; @@ -1673,12 +1758,10 @@ impl ReadView { for id in merged.items { if visible.contains(&id) && !excluded.contains(&id) { collected.push(id); - if collected.len() >= limit { + if collected.len() >= target { + let page = finish_verified_id_page(collected, limit); return Ok(( - Some(PageResult { - items: collected, - next_cursor: Some(id), - }), + Some(page), followup, )); } @@ -1701,7 +1784,7 @@ impl ReadView { fn ready_range_node_value( node: Option<&NodeRecord>, - type_id: u32, + label_id: u32, prop_key: &str, domain: SecondaryIndexRangeDomain, lower: Option<&PropertyRangeBound>, @@ -1709,7 +1792,7 @@ impl ReadView { policy_cutoffs: Option<&PrecomputedPruneCutoffs>, ) -> Option { let node = node?; - if node.type_id != type_id + if !node.label_ids.contains(label_id) || policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { return None; @@ -1949,7 +2032,7 @@ impl ReadView { #[allow(clippy::too_many_arguments)] fn find_nodes_paged_ready_range_index( &self, - type_id: u32, + label_id: u32, index_id: u64, prop_key: &str, domain: SecondaryIndexRangeDomain, @@ -2049,14 +2132,25 @@ impl ReadView { } let node_ids: Vec = pending.iter().map(|&(_, node_id)| node_id).collect(); + let current_label_ids = self.filter_node_ids_by_current_label(node_ids, label_id)?; + let current_label_ids: NodeIdSet = current_label_ids.into_iter().collect(); + let node_ids: Vec = pending + .iter() + .filter_map(|&(_, node_id)| current_label_ids.contains(&node_id).then_some(node_id)) + .collect(); let hydrated = self.get_nodes_raw(&node_ids)?; - for ((_, node_id), node) in pending.iter().zip(hydrated.into_iter()) { + let mut hydrated = hydrated.into_iter(); + for (_, node_id) in pending + .iter() + .filter(|(_, node_id)| current_label_ids.contains(node_id)) + { if visible.len() >= target_visible { break; } + let node = hydrated.next().flatten(); let Some(value) = Self::ready_range_node_value( node.as_ref(), - type_id, + label_id, prop_key, domain, lower, @@ -2139,7 +2233,7 @@ impl ReadView { #[allow(clippy::too_many_arguments)] fn collect_property_range_scan_matches( &self, - type_id: u32, + label_id: u32, prop_key: &str, domain: SecondaryIndexRangeDomain, lower: Option<&PropertyRangeBound>, @@ -2151,8 +2245,8 @@ impl ReadView { let mut matches = Vec::new(); let mut bounded: Option> = max_results.map(|_| BinaryHeap::new()); - self.scan_nodes_by_type_filtered( - type_id, + self.scan_nodes_by_single_label_id_filtered( + label_id, None, 256, policy_cutoffs.as_ref(), @@ -2203,7 +2297,7 @@ impl ReadView { candidate_ids: &NodeIdSet, query: &[f32], metric: DenseMetric, - type_filter: Option<&[u32]>, + label_filter: &ResolvedNodeLabelFilter, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, ) -> Result, EngineError> { if candidate_ids.is_empty() { @@ -2220,7 +2314,7 @@ impl ReadView { self.snapshot_seq, &mut remaining, &mut |node_id, node| { - if type_filter.is_some_and(|types| !types.contains(&node.type_id)) { + if !node_label_filter_matches(label_filter, &node.label_ids) { return; } if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { @@ -2253,7 +2347,7 @@ impl ReadView { self.snapshot_seq, &mut next_remaining_imm, &mut |node_id, node| { - if type_filter.is_some_and(|types| !types.contains(&node.type_id)) { + if !node_label_filter_matches(label_filter, &node.label_ids) { return; } if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { @@ -2286,10 +2380,10 @@ impl ReadView { query, metric, query_norm, - |type_id, updated_at, weight| { - type_filter.is_none_or(|types| types.contains(&type_id)) + |label_ids, updated_at, weight| { + node_label_filter_matches(label_filter, &label_ids) && policy_cutoffs.is_none_or(|cutoffs| { - !cutoffs.excludes_fields(type_id, updated_at, weight) + !cutoffs.excludes_fields(&label_ids, updated_at, weight) }) }, &mut hits, @@ -2316,6 +2410,62 @@ impl ReadView { }); } + fn hidden_node_ids_before_segment( + &self, + segment_index: usize, + initial_hidden_ids: &NodeIdSet, + ) -> Result { + let mut hidden_ids = initial_hidden_ids.clone(); + for segment in self.segments.iter().take(segment_index) { + hidden_ids.extend(segment.deleted_node_id_iter()); + hidden_ids.extend(segment.node_ids()?.iter().copied()); + } + Ok(hidden_ids) + } + + #[allow(clippy::too_many_arguments)] + fn score_exact_dense_segment( + &self, + segment_index: usize, + query: &[f32], + metric: DenseMetric, + query_norm: Option, + scope_ids: Option<&NodeIdSet>, + initial_hidden_ids: &NodeIdSet, + label_filter: &ResolvedNodeLabelFilter, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + candidate_ids: &mut NodeIdSet, + hits: &mut Vec, + ) -> Result<(), EngineError> { + let segment = &self.segments[segment_index]; + if segment.dense_vector_count() == 0 { + return Ok(()); + } + + let hidden_ids = self.hidden_node_ids_before_segment(segment_index, initial_hidden_ids)?; + let mut segment_hits = Vec::new(); + segment.exact_dense_vector_search( + query, + metric, + query_norm, + scope_ids, + &hidden_ids, + |label_ids, updated_at, weight| { + node_label_filter_matches(label_filter, &label_ids) + && policy_cutoffs.is_none_or(|cutoffs| { + !cutoffs.excludes_fields(&label_ids, updated_at, weight) + }) + }, + &mut segment_hits, + )?; + for hit in segment_hits { + if candidate_ids.insert(hit.node_id) { + hits.push(hit); + } + } + Ok(()) + } + fn push_sparse_top_k( heap: &mut BinaryHeap, k: usize, @@ -2360,13 +2510,13 @@ impl ReadView { scores: NodeIdMap, hidden_ids: &mut NodeIdSet, scope_ids: Option<&NodeIdSet>, - type_filter: Option<&[u32]>, + label_filter: &ResolvedNodeLabelFilter, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, top_hits: &mut BinaryHeap, k: usize, has_older_segments: bool, candidates: &mut Vec<(u64, f32)>, - meta_results: &mut Vec>, + meta_results: &mut Vec>, remaining: &mut Vec<(usize, u64)>, ) -> Result<(), EngineError> { // Early exit: no scores at all from this segment. @@ -2425,14 +2575,14 @@ impl ReadView { segment.get_node_meta_batch(remaining, meta_results)?; for (index, &(node_id, score)) in candidates.iter().enumerate() { - let Some((type_id, updated_at, weight)) = meta_results[index] else { + let Some((label_ids, updated_at, weight)) = meta_results[index] else { continue; }; - if type_filter.is_some_and(|types| !types.contains(&type_id)) { + if !node_label_filter_matches(label_filter, &label_ids) { continue; } if policy_cutoffs - .is_some_and(|cutoffs| cutoffs.excludes_fields(type_id, updated_at, weight)) + .is_some_and(|cutoffs| cutoffs.excludes_fields(&label_ids, updated_at, weight)) { continue; } @@ -2445,10 +2595,54 @@ impl ReadView { Ok(()) } + fn hide_segment_nodes_for_older_segments( + segment: &SegmentReader, + hidden_ids: &mut NodeIdSet, + ) -> Result<(), EngineError> { + hidden_ids.extend(segment.deleted_node_id_iter()); + hidden_ids.extend(segment.node_ids()?.iter().copied()); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn sparse_reduce_exact_segment_scores( + segment: &SegmentReader, + query: &[(u32, f32)], + hidden_ids: &mut NodeIdSet, + scope_ids: Option<&NodeIdSet>, + label_filter: &ResolvedNodeLabelFilter, + policy_cutoffs: Option<&PrecomputedPruneCutoffs>, + top_hits: &mut BinaryHeap, + k: usize, + has_older_segments: bool, + exact_hits: &mut Vec<(u64, f32)>, + ) -> Result<(), EngineError> { + exact_hits.clear(); + segment.exact_sparse_vector_scores( + query, + scope_ids, + hidden_ids, + |label_ids, updated_at, weight| { + node_label_filter_matches(label_filter, &label_ids) + && policy_cutoffs.is_none_or(|cutoffs| { + !cutoffs.excludes_fields(&label_ids, updated_at, weight) + }) + }, + exact_hits, + )?; + for &(node_id, score) in exact_hits.iter() { + Self::push_sparse_top_k(top_hits, k, node_id, score); + } + if has_older_segments { + Self::hide_segment_nodes_for_older_segments(segment, hidden_ids)?; + } + Ok(()) + } + #[allow(clippy::too_many_arguments)] // Dense tail scan needs all context inline for hot-path efficiency fn exact_dense_tail_candidates( &self, - segments: &[(&SegmentReader, usize)], + segments: &[(usize, &SegmentReader, usize)], exhausted: &[bool], query: &[f32], metric: DenseMetric, @@ -2460,7 +2654,7 @@ impl ReadView { .iter() .zip(exhausted.iter()) .filter(|(_, ex)| !**ex) - .map(|((seg, _), _)| *seg) + .map(|((_, seg, _), _)| *seg) .collect(); if work_items.is_empty() { @@ -2534,18 +2728,28 @@ impl ReadView { /// Resolve a `VectorSearchScope` into a set of reachable node IDs using /// the existing traversal substrate. The start node (depth 0) is included. - fn resolve_scope_ids(&self, scope: &VectorSearchScope) -> Result { + fn resolve_scope_ids(&self, scope: &ResolvedVectorSearchScope) -> Result { let traverse_opts = TraverseOptions { min_depth: 0, direction: scope.direction, - edge_type_filter: scope.edge_type_filter.clone(), - node_type_filter: None, + edge_label_filter: None, + emit_node_label_filter: None, at_epoch: scope.at_epoch, decay_lambda: None, limit: None, cursor: None, }; - let result = self.traverse(scope.start_node_id, scope.max_depth, &traverse_opts)?; + let result = if scope.edge_label_filter.is_none() { + self.traverse(scope.start_node_id, scope.max_depth, &traverse_opts)? + } else { + self.traverse_resolved( + scope.start_node_id, + scope.max_depth, + &traverse_opts, + scope.edge_label_filter.as_deref(), + &ResolvedNodeLabelFilter::Unconstrained, + )? + }; let mut ids = NodeIdSet::with_capacity_and_hasher(result.items.len(), NodeIdBuildHasher::default()); for hit in &result.items { @@ -2556,7 +2760,7 @@ impl ReadView { fn vector_search_dense( &self, - request: &VectorSearchRequest, + request: &ResolvedVectorSearchRequest<'_>, ) -> Result, EngineError> { let scope_ids = match &request.scope { Some(scope) => Some(self.resolve_scope_ids(scope)?), @@ -2567,11 +2771,11 @@ impl ReadView { fn vector_search_dense_with_scope( &self, - request: &VectorSearchRequest, + request: &ResolvedVectorSearchRequest<'_>, k: usize, scope_ids: Option<&NodeIdSet>, ) -> Result, EngineError> { - let Some(query) = request.dense_query.as_ref() else { + let Some(query) = request.dense_query else { return Err(EngineError::InvalidOperation( "vector_search(mode=\"dense\") requires dense_query".into(), )); @@ -2593,16 +2797,17 @@ impl ReadView { } } - let type_filter = request.type_filter.as_deref(); + let label_filter = &request.label_filter; let policy_cutoffs = self.query_policy_cutoffs(); - let searchable_segments: Vec<(&SegmentReader, usize)> = self + let searchable_segments: Vec<(usize, &SegmentReader, usize)> = self .segments .iter() - .filter_map(|segment| { + .enumerate() + .filter_map(|(segment_index, segment)| { segment .dense_hnsw_header() - .map(|header| (segment.as_ref(), header.point_count as usize)) + .map(|header| (segment_index, segment.as_ref(), header.point_count as usize)) }) .collect(); @@ -2628,16 +2833,19 @@ impl ReadView { ControlFlow::Continue(()) }); } - let total_dense_points: usize = - searchable_segments.iter().map(|(_, pc)| *pc).sum::() - + active_dense_points - + immutable_dense_points; + let total_dense_points: usize = self + .segments + .iter() + .map(|segment| segment.dense_vector_count()) + .sum::() + + active_dense_points + + immutable_dense_points; if scope.len() <= total_dense_points / 20 || scope.len() <= 2048 { let mut hits = self.score_dense_candidate_ids( scope, query, config.metric, - type_filter, + label_filter, policy_cutoffs.as_ref(), )?; hits.truncate(k); @@ -2650,12 +2858,16 @@ impl ReadView { let mut active_node_ids = NodeIdSet::with_capacity_and_hasher(0, NodeIdBuildHasher::default()); let mut candidate_ids = NodeIdSet::with_capacity_and_hasher(0, NodeIdBuildHasher::default()); + let mut initial_hidden_ids = + NodeIdSet::with_capacity_and_hasher(0, NodeIdBuildHasher::default()); + initial_hidden_ids.extend(active_deleted.iter().copied()); let _ = self .memtable .for_each_visible_node_at(self.snapshot_seq, &mut |node| { active_node_ids.insert(node.id); + initial_hidden_ids.insert(node.id); if node.dense_vector.is_some() - && type_filter.is_none_or(|types| types.contains(&node.type_id)) + && node_label_filter_matches(label_filter, &node.label_ids) && scope_ids.is_none_or(|scope| scope.contains(&node.id)) { candidate_ids.insert(node.id); @@ -2666,13 +2878,15 @@ impl ReadView { // Note: candidate_ids is a NodeIdSet, so duplicates across immutable memtables // are harmless; score_dense_candidate_ids does a proper newest-first lookup. for epoch in &self.immutable_epochs { + initial_hidden_ids.extend(epoch.memtable.collect_deleted_nodes_at(self.snapshot_seq)); let _ = epoch .memtable .for_each_visible_node_at(self.snapshot_seq, &mut |node| { + initial_hidden_ids.insert(node.id); if node.dense_vector.is_some() && !active_deleted.contains(&node.id) && !active_node_ids.contains(&node.id) - && type_filter.is_none_or(|types| types.contains(&node.type_id)) + && node_label_filter_matches(label_filter, &node.label_ids) && scope_ids.is_none_or(|scope| scope.contains(&node.id)) { candidate_ids.insert(node.id); @@ -2681,17 +2895,48 @@ impl ReadView { }); } - if candidate_ids.is_empty() && searchable_segments.is_empty() { - return Ok(Vec::new()); - } - let mut hits = self.score_dense_candidate_ids( &candidate_ids, query, config.metric, - type_filter, + label_filter, policy_cutoffs.as_ref(), )?; + let query_norm = crate::dense_hnsw::dense_query_norm(config.metric, query); + let mut segment_hidden_ids = initial_hidden_ids.clone(); + for (segment_index, segment) in self.segments.iter().enumerate() { + if segment.dense_vector_count() > 0 && segment.dense_hnsw_header().is_none() { + let mut segment_hits = Vec::new(); + segment.exact_dense_vector_search( + query, + config.metric, + query_norm, + scope_ids, + &segment_hidden_ids, + |label_ids, updated_at, weight| { + node_label_filter_matches(label_filter, &label_ids) + && policy_cutoffs.as_ref().is_none_or(|cutoffs| { + !cutoffs.excludes_fields(&label_ids, updated_at, weight) + }) + }, + &mut segment_hits, + )?; + for hit in segment_hits { + if candidate_ids.insert(hit.node_id) { + hits.push(hit); + } + } + } + if segment_index + 1 < self.segments.len() { + Self::hide_segment_nodes_for_older_segments(segment, &mut segment_hidden_ids)?; + } + } + Self::sort_vector_hits(&mut hits); + + if candidate_ids.is_empty() && searchable_segments.is_empty() { + hits.truncate(k); + return Ok(hits); + } let mut fetch_limit = request .ef_search @@ -2704,21 +2949,18 @@ impl ReadView { let mut segment_exhausted = Vec::with_capacity(searchable_segments.len()); let mut new_candidate_ids = NodeIdSet::with_capacity_and_hasher(0, NodeIdBuildHasher::default()); + let mut direct_exact_hits_added = false; // Threshold on total searchable segments (not just non-trivial ones) // because the serial path must still push into segment_exhausted for limit==0 entries. if searchable_segments.len() <= 1 { - for (segment, point_count) in &searchable_segments { + for (segment_index, segment, point_count) in &searchable_segments { let limit = fetch_limit.min(*point_count); if limit == 0 { segment_exhausted.push(true); continue; } let is_exhausted = limit >= *point_count; - if !is_exhausted { - exhausted_segments = false; - } - segment_exhausted.push(is_exhausted); let ef_search = request .ef_search .unwrap_or(fetch_limit) @@ -2726,13 +2968,37 @@ impl ReadView { .min(*point_count); let segment_hits = if let Some(scope) = scope_ids { - segment.search_dense_hnsw_scoped(query, ef_search, limit, scope)? + segment.search_dense_hnsw_scoped(query, ef_search, limit, scope) } else { - segment.search_dense_hnsw(query, ef_search, limit)? + segment.search_dense_hnsw(query, ef_search, limit) }; - for (node_id, _) in segment_hits { - if candidate_ids.insert(node_id) { - new_candidate_ids.insert(node_id); + match segment_hits { + Ok(segment_hits) => { + if !is_exhausted { + exhausted_segments = false; + } + segment_exhausted.push(is_exhausted); + for (node_id, _) in segment_hits { + if candidate_ids.insert(node_id) { + new_candidate_ids.insert(node_id); + } + } + } + Err(_) => { + segment_exhausted.push(true); + self.score_exact_dense_segment( + *segment_index, + query, + config.metric, + query_norm, + scope_ids, + &initial_hidden_ids, + label_filter, + policy_cutoffs.as_ref(), + &mut candidate_ids, + &mut hits, + )?; + direct_exact_hits_added = true; } } } @@ -2740,26 +3006,27 @@ impl ReadView { // Multi-segment parallel path: collect per-segment results then reduce. let user_ef = request.ef_search; #[allow(clippy::type_complexity)] - let per_segment_results: Vec< + let per_segment_results: Vec<( + usize, Result<(Vec<(u64, f32)>, bool), EngineError>, - > = crate::parallel::engine_cpu_install(|| { + )> = crate::parallel::engine_cpu_install(|| { use rayon::prelude::*; searchable_segments .par_iter() - .map(|(segment, point_count)| { + .map(|(segment_index, segment, point_count)| { let limit = fetch_limit.min(*point_count); if limit == 0 { - return Ok((Vec::new(), true)); + return (*segment_index, Ok((Vec::new(), true))); } let is_exhausted = limit >= *point_count; let ef_search = user_ef.unwrap_or(fetch_limit).max(limit).min(*point_count); - let hits = if let Some(scope) = scope_ids { - segment.search_dense_hnsw_scoped(query, ef_search, limit, scope)? + let result = if let Some(scope) = scope_ids { + segment.search_dense_hnsw_scoped(query, ef_search, limit, scope) } else { - segment.search_dense_hnsw(query, ef_search, limit)? + segment.search_dense_hnsw(query, ef_search, limit) }; - Ok((hits, is_exhausted)) + (*segment_index, result.map(|hits| (hits, is_exhausted))) }) .collect() }); @@ -2767,19 +3034,38 @@ impl ReadView { // Serial reduce: merge per-segment results preserving input order. let total_hits: usize = per_segment_results .iter() - .filter_map(|r| r.as_ref().ok()) + .filter_map(|(_, r)| r.as_ref().ok()) .map(|(hits, _)| hits.len()) .sum(); new_candidate_ids.reserve(total_hits); - for result in per_segment_results { - let (hits, is_exhausted) = result?; - if !is_exhausted { - exhausted_segments = false; - } - segment_exhausted.push(is_exhausted); - for (node_id, _) in hits { - if candidate_ids.insert(node_id) { - new_candidate_ids.insert(node_id); + for (segment_index, result) in per_segment_results { + match result { + Ok((hits, is_exhausted)) => { + if !is_exhausted { + exhausted_segments = false; + } + segment_exhausted.push(is_exhausted); + for (node_id, _) in hits { + if candidate_ids.insert(node_id) { + new_candidate_ids.insert(node_id); + } + } + } + Err(_) => { + segment_exhausted.push(true); + self.score_exact_dense_segment( + segment_index, + query, + config.metric, + query_norm, + scope_ids, + &initial_hidden_ids, + label_filter, + policy_cutoffs.as_ref(), + &mut candidate_ids, + &mut hits, + )?; + direct_exact_hits_added = true; } } } @@ -2790,9 +3076,11 @@ impl ReadView { &new_candidate_ids, query, config.metric, - type_filter, + label_filter, policy_cutoffs.as_ref(), )?); + } + if !new_candidate_ids.is_empty() || direct_exact_hits_added { Self::sort_vector_hits(&mut hits); } @@ -2812,7 +3100,7 @@ impl ReadView { &tail_candidate_ids, query, config.metric, - type_filter, + label_filter, policy_cutoffs.as_ref(), )?); Self::sort_vector_hits(&mut hits); @@ -2837,11 +3125,11 @@ impl ReadView { fn vector_search_sparse_with_scope( &self, - request: &VectorSearchRequest, + request: &ResolvedVectorSearchRequest<'_>, k: usize, scope_ids: Option<&NodeIdSet>, ) -> Result, EngineError> { - let Some(query) = request.sparse_query.as_ref() else { + let Some(query) = request.sparse_query else { return Err(EngineError::InvalidOperation( "vector_search(mode=\"sparse\") requires sparse_query".into(), )); @@ -2854,7 +3142,7 @@ impl ReadView { return Ok(Vec::new()); }; - let type_filter = request.type_filter.as_deref(); + let label_filter = &request.label_filter; let policy_cutoffs = self.query_policy_cutoffs(); // Small-scope fast path: score scope nodes directly instead of @@ -2865,7 +3153,7 @@ impl ReadView { k, &query, scope, - type_filter, + label_filter, policy_cutoffs.as_ref(), ); } @@ -2885,7 +3173,7 @@ impl ReadView { if scope_ids.is_some_and(|scope| !scope.contains(&node.id)) { return ControlFlow::Continue(()); } - if type_filter.is_some_and(|types| !types.contains(&node.type_id)) { + if !node_label_filter_matches(label_filter, &node.label_ids) { return ControlFlow::Continue(()); } if policy_cutoffs @@ -2913,7 +3201,7 @@ impl ReadView { if scope_ids.is_some_and(|scope| !scope.contains(&node.id)) { return ControlFlow::Continue(()); } - if type_filter.is_some_and(|types| !types.contains(&node.type_id)) { + if !node_label_filter_matches(label_filter, &node.label_ids) { return ControlFlow::Continue(()); } if policy_cutoffs @@ -2937,39 +3225,71 @@ impl ReadView { let sparse_segment_count = self .segments .iter() - .filter(|seg| !seg.raw_sparse_posting_index_mmap().is_empty()) + .filter(|seg| seg.sparse_postings_available()) .count(); + let has_sparse_fallback_segments = self + .segments + .iter() + .any(|seg| seg.sparse_vector_count() > 0 && !seg.sparse_postings_available()); // Reusable buffers for the reduce loop — allocated once, cleared per iteration. let mut candidates: Vec<(u64, f32)> = Vec::new(); - let mut meta_results: Vec> = Vec::new(); + let mut meta_results: Vec> = Vec::new(); let mut remaining: Vec<(usize, u64)> = Vec::new(); + let mut exact_hits: Vec<(u64, f32)> = Vec::new(); - if sparse_segment_count <= 1 { - // Single-segment / no-sparse fast path: skip rayon overhead. + if sparse_segment_count <= 1 || has_sparse_fallback_segments { + // Single-segment / fallback path: keep newest-to-oldest visibility + // state local so unavailable or failing postings can exact-scan + // their segment from vector source truth. for (segment_index, segment) in self.segments.iter().enumerate() { let has_older_segments = segment_index + 1 < self.segments.len(); - let mut scores = NodeIdMap::default(); - accumulate_sparse_posting_scores( - segment.raw_sparse_posting_index_mmap(), - segment.raw_sparse_postings_mmap(), - &query, - &mut scores, - )?; - Self::sparse_reduce_segment_scores( - segment, - scores, - &mut hidden_ids, - scope_ids, - type_filter, - policy_cutoffs.as_ref(), - &mut top_hits, - k, - has_older_segments, - &mut candidates, - &mut meta_results, - &mut remaining, - )?; + if segment.sparse_postings_available() { + let mut scores = NodeIdMap::default(); + match segment.accumulate_sparse_posting_scores(&query, &mut scores) { + Ok(()) => Self::sparse_reduce_segment_scores( + segment, + scores, + &mut hidden_ids, + scope_ids, + label_filter, + policy_cutoffs.as_ref(), + &mut top_hits, + k, + has_older_segments, + &mut candidates, + &mut meta_results, + &mut remaining, + )?, + Err(_) => Self::sparse_reduce_exact_segment_scores( + segment, + &query, + &mut hidden_ids, + scope_ids, + label_filter, + policy_cutoffs.as_ref(), + &mut top_hits, + k, + has_older_segments, + &mut exact_hits, + )?, + } + } else if segment.sparse_vector_count() > 0 { + Self::sparse_reduce_exact_segment_scores( + segment, + &query, + &mut hidden_ids, + scope_ids, + label_filter, + policy_cutoffs.as_ref(), + &mut top_hits, + k, + has_older_segments, + &mut exact_hits, + )?; + } else if has_older_segments { + Self::hide_segment_nodes_for_older_segments(segment, &mut hidden_ids)?; + } } } else { // Multi-segment parallel path: parallel score, serial reduce. @@ -2984,12 +3304,7 @@ impl ReadView { cap, NodeIdBuildHasher::default(), ); - accumulate_sparse_posting_scores( - segment.raw_sparse_posting_index_mmap(), - segment.raw_sparse_postings_mmap(), - &query, - &mut scores, - )?; + segment.accumulate_sparse_posting_scores(&query, &mut scores)?; Ok(scores) }) .collect() @@ -3000,21 +3315,34 @@ impl ReadView { self.segments.iter().zip(per_segment_scores).enumerate() { let has_older_segments = segment_index + 1 < self.segments.len(); - let scores = scores_result?; - Self::sparse_reduce_segment_scores( - segment, - scores, - &mut hidden_ids, - scope_ids, - type_filter, - policy_cutoffs.as_ref(), - &mut top_hits, - k, - has_older_segments, - &mut candidates, - &mut meta_results, - &mut remaining, - )?; + match scores_result { + Ok(scores) => Self::sparse_reduce_segment_scores( + segment, + scores, + &mut hidden_ids, + scope_ids, + label_filter, + policy_cutoffs.as_ref(), + &mut top_hits, + k, + has_older_segments, + &mut candidates, + &mut meta_results, + &mut remaining, + )?, + Err(_) => Self::sparse_reduce_exact_segment_scores( + segment, + &query, + &mut hidden_ids, + scope_ids, + label_filter, + policy_cutoffs.as_ref(), + &mut top_hits, + k, + has_older_segments, + &mut exact_hits, + )?, + } } } @@ -3028,7 +3356,7 @@ impl ReadView { k: usize, query: &[(u32, f32)], scope_ids: &NodeIdSet, - type_filter: Option<&[u32]>, + label_filter: &ResolvedNodeLabelFilter, policy_cutoffs: Option<&PrecomputedPruneCutoffs>, ) -> Result, EngineError> { let mut top_hits = BinaryHeap::with_capacity(k); @@ -3044,7 +3372,7 @@ impl ReadView { self.snapshot_seq, &mut remaining, &mut |node_id, node| { - if type_filter.is_some_and(|types| !types.contains(&node.type_id)) { + if !node_label_filter_matches(label_filter, &node.label_ids) { return; } if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { @@ -3070,7 +3398,7 @@ impl ReadView { self.snapshot_seq, &mut next, &mut |node_id, node| { - if type_filter.is_some_and(|types| !types.contains(&node.type_id)) { + if !node_label_filter_matches(label_filter, &node.label_ids) { return; } if policy_cutoffs.is_some_and(|cutoffs| cutoffs.excludes(node)) { @@ -3105,10 +3433,10 @@ impl ReadView { segment.score_sparse_candidates_sorted( &remaining, query, - |type_id, updated_at, weight| { - type_filter.is_none_or(|types| types.contains(&type_id)) + |label_ids, updated_at, weight| { + node_label_filter_matches(label_filter, &label_ids) && policy_cutoffs.is_none_or(|cutoffs| { - !cutoffs.excludes_fields(type_id, updated_at, weight) + !cutoffs.excludes_fields(&label_ids, updated_at, weight) }) }, &mut segment_hits, @@ -3128,7 +3456,7 @@ impl ReadView { /// results using the selected fusion mode. fn vector_search_hybrid( &self, - request: &VectorSearchRequest, + request: &ResolvedVectorSearchRequest<'_>, ) -> Result, EngineError> { let has_dense = request.dense_query.is_some(); let has_sparse = request.sparse_query.is_some(); @@ -3202,6 +3530,99 @@ impl ReadView { Ok(fused) } + fn validate_vector_search_dense_shape( + &self, + query: Option<&DenseVector>, + k: usize, + ef_search: Option, + ) -> Result<(), EngineError> { + let Some(query) = query else { + return Err(EngineError::InvalidOperation( + "vector_search(mode=\"dense\") requires dense_query".into(), + )); + }; + if k == 0 { + return Ok(()); + } + + let Some(config) = self.manifest.dense_vector.as_ref() else { + return Ok(()); + }; + validate_dense_vector(query, config)?; + + if ef_search == Some(0) { + return Err(EngineError::InvalidOperation( + "vector_search ef_search must be > 0".into(), + )); + } + Ok(()) + } + + fn validate_vector_search_sparse_shape( + &self, + query: Option<&SparseVector>, + k: usize, + ) -> Result<(), EngineError> { + let Some(query) = query else { + return Err(EngineError::InvalidOperation( + "vector_search(mode=\"sparse\") requires sparse_query".into(), + )); + }; + if k == 0 { + return Ok(()); + } + let _ = canonicalize_sparse_vector(query)?; + Ok(()) + } + + fn validate_vector_search_shape( + &self, + request: &VectorSearchRequest, + ) -> Result<(), EngineError> { + match request.mode { + VectorSearchMode::Dense => self.validate_vector_search_dense_shape( + request.dense_query.as_ref(), + request.k, + request.ef_search, + ), + VectorSearchMode::Sparse => { + self.validate_vector_search_sparse_shape(request.sparse_query.as_ref(), request.k) + } + VectorSearchMode::Hybrid => { + let has_dense = request.dense_query.is_some(); + let has_sparse = request.sparse_query.is_some(); + + if !has_dense && !has_sparse { + return Err(EngineError::InvalidOperation( + "vector_search(mode=\"hybrid\") requires at least one of dense_query or sparse_query".into(), + )); + } + + if has_dense && !has_sparse { + return self.validate_vector_search_dense_shape( + request.dense_query.as_ref(), + request.k, + request.ef_search, + ); + } + if has_sparse && !has_dense { + return self + .validate_vector_search_sparse_shape(request.sparse_query.as_ref(), request.k); + } + if request.k == 0 { + return Ok(()); + } + + self.validate_vector_search_dense_shape( + request.dense_query.as_ref(), + request.k, + request.ef_search, + )?; + self.validate_vector_search_sparse_shape(request.sparse_query.as_ref(), request.k) + } + } + } + /// Search node vectors and return scored node IDs. /// /// Supports `mode="dense"`, `mode="sparse"`, and `mode="hybrid"`. @@ -3210,58 +3631,146 @@ impl ReadView { &self, request: &VectorSearchRequest, ) -> Result, EngineError> { - match request.mode { - VectorSearchMode::Dense => self.vector_search_dense(request), + let label_filter = self + .label_catalog + .resolve_node_label_filter_request(request.label_filter.as_ref())?; + let scope = request + .scope + .as_ref() + .map(|scope| { + let edge_label_filter = match self + .label_catalog + .resolve_edge_label_filter_for_read(scope.edge_label_filter.as_deref())? + { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => Some(Vec::new()), + }; + Ok::(ResolvedVectorSearchScope { + start_node_id: scope.start_node_id, + max_depth: scope.max_depth, + direction: scope.direction, + edge_label_filter, + at_epoch: scope.at_epoch, + }) + }) + .transpose()?; + if label_filter.is_empty_constraint() { + self.validate_vector_search_shape(request)?; + return Ok(Vec::new()); + }; + let resolved = ResolvedVectorSearchRequest { + mode: request.mode, + dense_query: request.dense_query.as_ref(), + sparse_query: request.sparse_query.as_ref(), + k: request.k, + label_filter, + ef_search: request.ef_search, + scope, + dense_weight: request.dense_weight, + sparse_weight: request.sparse_weight, + fusion_mode: request.fusion_mode, + }; + + match resolved.mode { + VectorSearchMode::Dense => self.vector_search_dense(&resolved), VectorSearchMode::Sparse => { - let scope_ids = match &request.scope { + let scope_ids = match &resolved.scope { Some(scope) => Some(self.resolve_scope_ids(scope)?), None => None, }; - self.vector_search_sparse_with_scope(request, request.k, scope_ids.as_ref()) + self.vector_search_sparse_with_scope(&resolved, resolved.k, scope_ids.as_ref()) } - VectorSearchMode::Hybrid => self.vector_search_hybrid(request), + VectorSearchMode::Hybrid => self.vector_search_hybrid(&resolved), } } // --- Secondary index queries --- - /// Return all live node IDs with the given type_id (raw, unfiltered). - /// Used internally by collect_prune_targets. - fn nodes_by_type_raw(&self, type_id: u32) -> Result, EngineError> { - let deleted = self.sources().collect_deleted_nodes(); + fn filter_node_ids_by_current_label( + &self, + mut ids: Vec, + label_id: u32, + ) -> Result, EngineError> { + ids.sort_unstable(); + ids.dedup(); + if ids.is_empty() { + return Ok(ids); + } + if self.immutable_epochs.is_empty() && self.segments.is_empty() { + return Ok(ids); + } - let mut seen = NodeIdSet::default(); - let mut results = Vec::new(); + let visibility = self.sources().find_node_visibility_meta(&ids)?; + let mut filtered = Vec::with_capacity(ids.len()); + for (node_id, state) in ids.into_iter().zip(visibility.into_iter()) { + if matches!( + state, + NodeVisibilityState::Live(meta) if meta.label_ids.contains(label_id) + ) { + filtered.push(node_id); + } + } + Ok(filtered) + } - for id in self.memtable.visible_nodes_by_type(type_id, self.snapshot_seq) { - if !deleted.contains(&id) && seen.insert(id) { - results.push(id); + fn filter_node_ids_by_current_label_and_time( + &self, + mut ids: Vec, + label_id: u32, + from_ms: i64, + to_ms: i64, + ) -> Result, EngineError> { + ids.sort_unstable(); + ids.dedup(); + if ids.is_empty() { + return Ok(ids); + } + + let visibility = self.sources().find_node_visibility_meta(&ids)?; + let mut filtered = Vec::with_capacity(ids.len()); + for (node_id, state) in ids.into_iter().zip(visibility.into_iter()) { + if matches!( + state, + NodeVisibilityState::Live(meta) + if meta.label_ids.contains(label_id) + && meta.updated_at >= from_ms + && meta.updated_at <= to_ms + ) { + filtered.push(node_id); } } + Ok(filtered) + } + + /// Return all live node IDs with the given label_id (raw, unfiltered). + /// Used internally by collect_prune_targets. + fn nodes_by_label_id_raw(&self, label_id: u32) -> Result, EngineError> { + let mut candidates = Vec::new(); + + for id in self.memtable.visible_nodes_by_label_id(label_id, self.snapshot_seq) { + candidates.push(id); + } for epoch in &self.immutable_epochs { - for id in epoch.memtable.visible_nodes_by_type(type_id, self.snapshot_seq) { - if !deleted.contains(&id) && seen.insert(id) { - results.push(id); - } + for id in epoch.memtable.visible_nodes_by_label_id(label_id, self.snapshot_seq) { + candidates.push(id); } } for seg in &self.segments { - for id in seg.nodes_by_type(type_id)? { - if !deleted.contains(&id) && seen.insert(id) { - results.push(id); - } + for id in seg.nodes_by_label_id(label_id)? { + candidates.push(id); } } - Ok(results) + self.filter_node_ids_by_current_label(candidates, label_id) } - /// Return all live node IDs with the given type_id, merged across + /// Return all live node IDs with the given label_id, merged across /// memtable and all segments. Excludes tombstoned and policy-excluded nodes. - pub fn nodes_by_type(&self, type_id: u32) -> Result, EngineError> { - let mut results = self.nodes_by_type_raw(type_id)?; + pub fn nodes_by_label_id(&self, label_id: u32) -> Result, EngineError> { + let mut results = self.nodes_by_label_id_raw(label_id)?; // Policy filtering: batch-fetch nodes and exclude matches. let excluded = self.policy_excluded_node_ids(&results)?; @@ -3272,22 +3781,22 @@ impl ReadView { Ok(results) } - /// Return all live edge IDs with the given type_id, merged across + /// Return all live edge IDs with the given label_id, merged across /// memtable and all segments. Excludes tombstoned edges. - pub fn edges_by_type(&self, type_id: u32) -> Result, EngineError> { + pub fn edges_by_label_id(&self, label_id: u32) -> Result, EngineError> { let deleted = self.sources().collect_deleted_edges(); let mut seen = NodeIdSet::default(); let mut results = Vec::new(); - for id in self.memtable.visible_edges_by_type(type_id, self.snapshot_seq) { + for id in self.memtable.visible_edges_by_label_id(label_id, self.snapshot_seq) { if !deleted.contains(&id) && seen.insert(id) { results.push(id); } } for epoch in &self.immutable_epochs { - for id in epoch.memtable.visible_edges_by_type(type_id, self.snapshot_seq) { + for id in epoch.memtable.visible_edges_by_label_id(label_id, self.snapshot_seq) { if !deleted.contains(&id) && seen.insert(id) { results.push(id); } @@ -3295,7 +3804,7 @@ impl ReadView { } for seg in &self.segments { - for id in seg.edges_by_type(type_id)? { + for id in seg.edges_by_label_id(label_id)? { if !deleted.contains(&id) && seen.insert(id) { results.push(id); } @@ -3305,57 +3814,114 @@ impl ReadView { Ok(results) } - /// Return all live node records with the given type_id, hydrated from + /// Return all live node records with the given label_id, hydrated from /// memtable and segments. Excludes tombstoned and policy-excluded nodes. /// - /// Uses `nodes_by_type()` for the ID list (already policy-filtered), then + /// Uses `nodes_by_label_id()` for the ID list (already policy-filtered), then /// `get_nodes_raw()` for batch hydration (one merge-walk per segment, /// not N individual lookups). Single policy pass, no redundant filtering. - pub fn get_nodes_by_type(&self, type_id: u32) -> Result, EngineError> { - let ids = self.nodes_by_type(type_id)?; + pub fn get_nodes_by_label_id(&self, label_id: u32) -> Result, EngineError> { + let ids = self.nodes_by_label_id(label_id)?; let results = self.get_nodes_raw(&ids)?; Ok(results.into_iter().flatten().collect()) } - /// Return all live edge records with the given type_id, hydrated from + /// Return all live edge records with the given label_id, hydrated from /// memtable and segments. Excludes tombstoned edges. /// - /// Uses `edges_by_type()` for the ID list, then `get_edges()` for batch + /// Uses `edges_by_label_id()` for the ID list, then `get_edges()` for batch /// hydration (one merge-walk per segment, not N individual lookups). - pub fn get_edges_by_type(&self, type_id: u32) -> Result, EngineError> { - let ids = self.edges_by_type(type_id)?; + pub fn get_edges_by_label_id(&self, label_id: u32) -> Result, EngineError> { + let ids = self.edges_by_label_id(label_id)?; let results = self.get_edges(&ids)?; Ok(results.into_iter().flatten().collect()) } - /// Return the count of live nodes with the given type_id without hydrating - /// records. Excludes tombstoned and policy-excluded nodes. - pub fn count_nodes_by_type(&self, type_id: u32) -> Result { - Ok(self.nodes_by_type(type_id)?.len() as u64) + fn count_nodes_by_resolved_label_filter( + &self, + filter: &ResolvedNodeLabelFilter, + ) -> Result { + let ResolvedNodeLabelFilter::LabelSet { + mode, + label_ids, + .. + } = filter + else { + return match filter { + ResolvedNodeLabelFilter::Empty { .. } => Ok(0), + ResolvedNodeLabelFilter::Unconstrained => { + let policy_cutoffs = self.query_policy_cutoffs(); + Ok(self + .collect_node_ids_for_resolved_label_filter( + filter, + policy_cutoffs.as_ref(), + )? + .len() as u64) + } + ResolvedNodeLabelFilter::LabelSet { .. } => unreachable!(), + }; + }; + + let scan_labels: Vec = match mode { + LabelMatchMode::Any => label_ids.as_slice().to_vec(), + LabelMatchMode::All => { + if label_ids.len() == 1 { + label_ids.as_slice().to_vec() + } else { + let driver = self + .node_label_filter_estimate(label_ids, LabelMatchMode::All)? + .driver_label_id + .unwrap_or_else(|| label_ids.as_slice()[0]); + vec![driver] + } + } + }; + let policy_cutoffs = self.query_policy_cutoffs(); + let mut count = 0u64; + self.scan_raw_node_label_candidates(&scan_labels, None, QUERY_VERIFY_CHUNK, |chunk| { + #[cfg(test)] + self.note_node_visibility_meta_reads(chunk.len()); + let visibility = self.sources().find_node_visibility_meta(chunk)?; + for state in visibility { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + if policy_cutoffs.as_ref().is_some_and(|cutoffs| { + cutoffs.excludes_fields(&meta.label_ids, meta.updated_at, meta.weight) + }) { + continue; + } + if node_label_filter_matches(filter, &meta.label_ids) { + count += 1; + } + } + Ok(ControlFlow::Continue(())) + })?; + Ok(count) } - /// Return the count of live edges with the given type_id without hydrating + /// Return the count of live edges with the given label_id without hydrating /// records. Excludes tombstoned edges (edges are not subject to prune policies). - pub fn count_edges_by_type(&self, type_id: u32) -> Result { - Ok(self.edges_by_type(type_id)?.len() as u64) + pub fn count_edges_by_label_id(&self, label_id: u32) -> Result { + Ok(self.edges_by_label_id(label_id)?.len() as u64) } - // --- Paginated type-index queries --- + // --- Paginated label-index queries --- - /// Paginated version of `nodes_by_type`. Returns a page of node IDs sorted + /// Paginated version of `nodes_by_label_id`. Returns a page of node IDs sorted /// by ID, with cursor-based pagination. Pass `PageRequest::default()` to get - /// all results (equivalent to `nodes_by_type`). + /// all results (equivalent to `nodes_by_label_id`). /// /// Uses K-way merge across already-sorted sources with early termination: /// O(cursor_position + limit) instead of O(N log N) when no prune policies /// are active. With policies, still saves the sort via merge, then applies /// policy filtering and cursor on the sorted result. - pub fn nodes_by_type_paged( + pub fn nodes_by_label_id_paged( &self, - type_id: u32, + label_id: u32, page: &PageRequest, ) -> Result, EngineError> { - let unfiltered = self.nodes_by_type_paged_unfiltered(type_id, page)?; + let unfiltered = self.nodes_by_single_label_id_paged_unfiltered(label_id, page)?; if self.manifest.prune_policies.is_empty() { // Fast path: merge with early termination, no policy filtering needed Ok(unfiltered) @@ -3381,7 +3947,8 @@ impl ReadView { limit: Some(chunk_limit), after: cursor, }; - let chunk = self.nodes_by_type_paged_unfiltered(type_id, &chunk_page)?; + let chunk = + self.nodes_by_single_label_id_paged_unfiltered(label_id, &chunk_page)?; if chunk.items.is_empty() { return Ok(PageResult { items: collected, @@ -3414,28 +3981,28 @@ impl ReadView { } } - /// Paginated version of `edges_by_type`. Returns a page of edge IDs sorted + /// Paginated version of `edges_by_label_id`. Returns a page of edge IDs sorted /// by ID, with cursor-based pagination. Uses K-way merge with early /// termination (edges are not subject to prune policies). - pub fn edges_by_type_paged( + pub fn edges_by_label_id_paged( &self, - type_id: u32, + label_id: u32, page: &PageRequest, ) -> Result, EngineError> { let deleted = self.sources().collect_deleted_edges(); // Collect sources - let memtable_ids = self.memtable.visible_edges_by_type(type_id, self.snapshot_seq); + let memtable_ids = self.memtable.visible_edges_by_label_id(label_id, self.snapshot_seq); let mut segment_ids: Vec> = Vec::with_capacity(self.immutable_epochs.len() + self.segments.len()); for epoch in &self.immutable_epochs { - segment_ids.push(epoch.memtable.visible_edges_by_type(type_id, self.snapshot_seq)); + segment_ids.push(epoch.memtable.visible_edges_by_label_id(label_id, self.snapshot_seq)); } for seg in &self.segments { - segment_ids.push(seg.edges_by_type(type_id)?); + segment_ids.push(seg.edges_by_label_id(label_id)?); } - Ok(merge_type_ids_paged( + Ok(merge_record_ids_paged( memtable_ids, segment_ids, &deleted, @@ -3443,16 +4010,16 @@ impl ReadView { )) } - /// Paginated version of `get_nodes_by_type`. Returns a page of hydrated node + /// Paginated version of `get_nodes_by_label_id`. Returns a page of hydrated node /// records. Only hydrates records in the requested page (not all then slice). /// In rare cases (data inconsistency), the page may contain fewer items than /// `limit` even when `next_cursor` is `Some`. - pub fn get_nodes_by_type_paged( + pub fn get_nodes_by_label_id_paged( &self, - type_id: u32, + label_id: u32, page: &PageRequest, ) -> Result, EngineError> { - let id_page = self.nodes_by_type_paged(type_id, page)?; + let id_page = self.nodes_by_label_id_paged(label_id, page)?; let hydrated = self.get_nodes_raw(&id_page.items)?; let items: Vec = hydrated.into_iter().flatten().collect(); Ok(PageResult { @@ -3461,16 +4028,16 @@ impl ReadView { }) } - /// Paginated version of `get_edges_by_type`. Returns a page of hydrated edge + /// Paginated version of `get_edges_by_label_id`. Returns a page of hydrated edge /// records. Only hydrates records in the requested page (not all then slice). /// In rare cases (data inconsistency), the page may contain fewer items than /// `limit` even when `next_cursor` is `Some`. - pub fn get_edges_by_type_paged( + pub fn get_edges_by_label_id_paged( &self, - type_id: u32, + label_id: u32, page: &PageRequest, ) -> Result, EngineError> { - let id_page = self.edges_by_type_paged(type_id, page)?; + let id_page = self.edges_by_label_id_paged(label_id, page)?; let hydrated = self.get_edges(&id_page.items)?; let items: Vec = hydrated.into_iter().flatten().collect(); Ok(PageResult { @@ -3481,7 +4048,7 @@ impl ReadView { fn timestamp_candidate_ids( &self, - type_id: u32, + label_id: u32, from_ms: i64, to_ms: i64, max_ids: usize, @@ -3493,7 +4060,7 @@ impl ReadView { let mut ids = Vec::with_capacity(max_ids.min(4096)); let mut seen = NodeIdSet::default(); let flow = self.memtable.for_each_visible_node_by_time_range_at( - type_id, + label_id, from_ms, to_ms, self.snapshot_seq, @@ -3508,7 +4075,7 @@ impl ReadView { for epoch in &self.immutable_epochs { let flow = epoch.memtable.for_each_visible_node_by_time_range_at( - type_id, + label_id, from_ms, to_ms, self.snapshot_seq, @@ -3523,7 +4090,7 @@ impl ReadView { } for seg in &self.segments { - let flow = seg.for_each_node_by_time_range(type_id, from_ms, to_ms, |node_id| { + let flow = seg.for_each_node_by_time_range(label_id, from_ms, to_ms, |node_id| { push_unique_candidate_id_limited(&mut ids, &mut seen, node_id, max_ids) })?; if flow.is_break() { @@ -3532,21 +4099,20 @@ impl ReadView { } } - ids.sort_unstable(); - Ok(ids) + self.filter_node_ids_by_current_label_and_time(ids, label_id, from_ms, to_ms) } fn find_nodes_outcome( &self, - type_id: u32, + label_id: u32, prop_key: &str, prop_value: &PropValue, ) -> Result>, EngineError> { let ready_entry = - self.node_property_index_entry(type_id, prop_key, &SecondaryIndexKind::Equality); + self.node_property_index_entry(label_id, prop_key, &SecondaryIndexKind::Equality); if let Some(entry) = ready_entry.filter(|entry| entry.state == SecondaryIndexState::Ready) { let (results, followup) = - self.find_nodes_ready_equality_index(type_id, entry.index_id, prop_key, prop_value)?; + self.find_nodes_ready_equality_index(label_id, entry.index_id, prop_key, prop_value)?; if let Some(results) = results { Ok(PropertyQueryOutcome { @@ -3556,14 +4122,14 @@ impl ReadView { }) } else { Ok(PropertyQueryOutcome { - value: self.find_nodes_scan_fallback(type_id, prop_key, prop_value)?, + value: self.find_nodes_scan_fallback(label_id, prop_key, prop_value)?, route: PropertyQueryRouteKind::EqualityScanFallback, followup, }) } } else { Ok(PropertyQueryOutcome { - value: self.find_nodes_scan_fallback(type_id, prop_key, prop_value)?, + value: self.find_nodes_scan_fallback(label_id, prop_key, prop_value)?, route: PropertyQueryRouteKind::EqualityScanFallback, followup: None, }) @@ -3572,16 +4138,16 @@ impl ReadView { fn find_nodes_paged_outcome( &self, - type_id: u32, + label_id: u32, prop_key: &str, prop_value: &PropValue, page: &PageRequest, ) -> Result>, EngineError> { let ready_entry = - self.node_property_index_entry(type_id, prop_key, &SecondaryIndexKind::Equality); + self.node_property_index_entry(label_id, prop_key, &SecondaryIndexKind::Equality); if let Some(entry) = ready_entry.filter(|entry| entry.state == SecondaryIndexState::Ready) { let (result, followup) = self.find_nodes_paged_ready_equality_index( - type_id, + label_id, entry.index_id, prop_key, prop_value, @@ -3596,7 +4162,7 @@ impl ReadView { } else { Ok(PropertyQueryOutcome { value: self.find_nodes_paged_scan_fallback( - type_id, prop_key, prop_value, page, + label_id, prop_key, prop_value, page, )?, route: PropertyQueryRouteKind::EqualityScanFallback, followup, @@ -3604,7 +4170,7 @@ impl ReadView { } } else { Ok(PropertyQueryOutcome { - value: self.find_nodes_paged_scan_fallback(type_id, prop_key, prop_value, page)?, + value: self.find_nodes_paged_scan_fallback(label_id, prop_key, prop_value, page)?, route: PropertyQueryRouteKind::EqualityScanFallback, followup: None, }) @@ -3613,7 +4179,7 @@ impl ReadView { fn find_nodes_range_paged_outcome( &self, - type_id: u32, + label_id: u32, prop_key: &str, lower: Option<&PropertyRangeBound>, upper: Option<&PropertyRangeBound>, @@ -3621,14 +4187,14 @@ impl ReadView { ) -> Result>, EngineError> { let domain = Self::validate_property_range_bounds(lower, upper, page.after.as_ref())?; let ready_entry = self.node_property_index_entry( - type_id, + label_id, prop_key, &SecondaryIndexKind::Range { domain }, ); let mut followup = None; if let Some(entry) = ready_entry.filter(|entry| entry.state == SecondaryIndexState::Ready) { let (result, ready_followup) = self.find_nodes_paged_ready_range_index( - type_id, + label_id, entry.index_id, prop_key, domain, @@ -3649,7 +4215,7 @@ impl ReadView { let value = match page.limit { Some(limit) if limit > 0 => { let matches = self.collect_property_range_scan_matches( - type_id, + label_id, prop_key, domain, lower, @@ -3672,7 +4238,7 @@ impl ReadView { } _ => { let matches = self.collect_property_range_scan_matches( - type_id, + label_id, prop_key, domain, lower, @@ -3696,12 +4262,12 @@ impl ReadView { // --- Timestamp range queries --- - /// Find node IDs of a given type updated within a time range [from_ms, to_ms] (inclusive). + /// Find node IDs of a given label updated within a time range [from_ms, to_ms] (inclusive). /// Merges across memtable + segments with deduplication. /// Excludes tombstoned and policy-pruned nodes. pub fn find_nodes_by_time_range( &self, - type_id: u32, + label_id: u32, from_ms: i64, to_ms: i64, ) -> Result, EngineError> { @@ -3710,7 +4276,7 @@ impl ReadView { after: None, }; Ok(self - .find_nodes_by_time_range_paged(type_id, from_ms, to_ms, &page)? + .find_nodes_by_time_range_paged(label_id, from_ms, to_ms, &page)? .items) } @@ -3721,29 +4287,27 @@ impl ReadView { /// segment, sort results by node_id). O(log N) seek per source + O(results) scan. pub fn find_nodes_by_time_range_paged( &self, - type_id: u32, + label_id: u32, from_ms: i64, to_ms: i64, page: &PageRequest, ) -> Result, EngineError> { let deleted = self.sources().collect_deleted_nodes(); - - // Collect sources: memtable + immutable memtables + segments let memtable_ids = self.memtable - .visible_nodes_by_time_range(type_id, from_ms, to_ms, self.snapshot_seq); + .visible_nodes_by_time_range(label_id, from_ms, to_ms, self.snapshot_seq); let mut segment_ids: Vec> = Vec::with_capacity(self.immutable_epochs.len() + self.segments.len()); for epoch in &self.immutable_epochs { segment_ids.push(epoch.memtable.visible_nodes_by_time_range( - type_id, + label_id, from_ms, to_ms, self.snapshot_seq, )); } for seg in &self.segments { - segment_ids.push(seg.nodes_by_time_range(type_id, from_ms, to_ms)?); + segment_ids.push(seg.nodes_by_time_range(label_id, from_ms, to_ms)?); } let limit = page.limit.unwrap_or(0); @@ -3752,87 +4316,73 @@ impl ReadView { limit: None, after: page.after, }; - let all = merge_type_ids_paged(memtable_ids, segment_ids, &deleted, &all_page); - let nodes = self.get_nodes_raw(&all.items)?; - let mut items: Vec = Vec::with_capacity(all.items.len()); - for (id, node) in all.items.iter().zip(nodes.iter()) { - if let Some(n) = node { - if n.updated_at >= from_ms && n.updated_at <= to_ms { - items.push(*id); - } - } - } - + let merged = merge_record_ids_paged(memtable_ids, segment_ids, &deleted, &all_page); + let mut items = + self.filter_node_ids_by_current_label_and_time(merged.items, label_id, from_ms, to_ms)?; if !self.manifest.prune_policies.is_empty() { let excluded = self.policy_excluded_node_ids(&items)?; if !excluded.is_empty() { items.retain(|id| !excluded.contains(id)); } } - - Ok(PageResult { + return Ok(PageResult { items, next_cursor: None, - }) - } else { - let chunk_limit = limit.saturating_mul(4).max(limit); - let mut collected = Vec::with_capacity(limit); - let mut cursor = page.after; - - loop { - let chunk_page = PageRequest { - limit: Some(chunk_limit), - after: cursor, - }; - let chunk = merge_type_ids_paged( - memtable_ids.clone(), - segment_ids.clone(), - &deleted, - &chunk_page, - ); - if chunk.items.is_empty() { - return Ok(PageResult { - items: collected, - next_cursor: None, - }); - } - - let nodes = self.get_nodes_raw(&chunk.items)?; - let mut visible: NodeIdSet = - NodeIdSet::with_capacity_and_hasher(chunk.items.len(), Default::default()); - for (id, node) in chunk.items.iter().zip(nodes.iter()) { - if let Some(n) = node { - if n.updated_at >= from_ms && n.updated_at <= to_ms { - visible.insert(*id); - } - } - } + }); + } - let excluded = if self.manifest.prune_policies.is_empty() { - NodeIdSet::default() - } else { - self.policy_excluded_node_ids(&chunk.items)? - }; + let target = page_verify_target(limit); + let chunk_limit = limit + .saturating_add(1) + .saturating_mul(4) + .max(limit.saturating_add(1)); + let mut collected = Vec::with_capacity(limit); + let mut cursor = page.after; + loop { + let chunk_page = PageRequest { + limit: Some(chunk_limit), + after: cursor, + }; + let chunk = merge_record_ids_paged( + memtable_ids.clone(), + segment_ids.clone(), + &deleted, + &chunk_page, + ); + if chunk.items.is_empty() { + return Ok(PageResult { + items: collected, + next_cursor: None, + }); + } - for id in chunk.items { - if visible.contains(&id) && !excluded.contains(&id) { - collected.push(id); - if collected.len() >= limit { - return Ok(PageResult { - items: collected, - next_cursor: Some(id), - }); - } + let visible = self.filter_node_ids_by_current_label_and_time( + chunk.items.clone(), + label_id, + from_ms, + to_ms, + )?; + let excluded = if self.manifest.prune_policies.is_empty() { + NodeIdSet::default() + } else { + self.policy_excluded_node_ids(&visible)? + }; + let visible: NodeIdSet = visible.into_iter().collect(); + for id in chunk.items.iter().copied() { + if visible.contains(&id) && !excluded.contains(&id) { + collected.push(id); + if collected.len() >= target { + return Ok(finish_verified_id_page(collected, limit)); } - cursor = Some(id); } + cursor = Some(id); + } - if chunk.next_cursor.is_none() { - return Ok(PageResult { - items: collected, - next_cursor: None, - }); - } + if chunk.next_cursor.is_none() { + return Ok(PageResult { + items: collected, + next_cursor: None, + }); } } } @@ -3890,7 +4440,13 @@ impl ReadView { "approx_residual_tolerance must be > 0.0".into(), )); } - let edge_filter = options.edge_type_filter.as_deref(); + let resolved_edge_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let edge_filter = match resolved_edge_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => Some(Vec::new()), + }; // Deduplicate seeds and filter to live nodes only. // Without this, deleted/non-existent seeds become dangling nodes @@ -3918,13 +4474,6 @@ impl ReadView { let teleport = (1.0 - damping) / num_seeds; if options.algorithm == PprAlgorithm::ApproxForwardPush { - let batch_opts = NeighborOptions { - direction: Direction::Outgoing, - type_filter: edge_filter.map(|s| s.to_vec()), - limit: None, - at_epoch: None, - decay_lambda: None, - }; let tolerance = options.approx_residual_tolerance; let mut reserve: NodeIdMap = NodeIdMap::with_capacity_and_hasher(seeds.len() * 16, Default::default()); @@ -3953,7 +4502,13 @@ impl ReadView { .collect(); if !uncached.is_empty() { - let all_neighbors = self.neighbors_batch(&uncached, &batch_opts)?; + let all_neighbors = self.neighbors_batch_resolved( + &uncached, + Direction::Outgoing, + edge_filter.as_deref(), + None, + None, + )?; for &node_id in &uncached { let raw_neighbors: Vec<(u64, f32)> = all_neighbors .get(&node_id) @@ -4083,14 +4638,13 @@ impl ReadView { NodeIdMap::with_capacity_and_hasher(seeds.len() * 16, Default::default()); while !wave.is_empty() { - let batch_opts = NeighborOptions { - direction: Direction::Outgoing, - type_filter: edge_filter.map(|s| s.to_vec()), - limit: None, - at_epoch: None, - decay_lambda: None, - }; - let all_neighbors = self.neighbors_batch(&wave, &batch_opts)?; + let all_neighbors = self.neighbors_batch_resolved( + &wave, + Direction::Outgoing, + edge_filter.as_deref(), + None, + None, + )?; let mut next_wave: Vec = Vec::new(); for &node_id in &wave { @@ -4221,8 +4775,8 @@ impl ReadView { /// Export the graph's adjacency structure for external community detection. /// - /// Returns all live node IDs and edges (from, to, type_id, weight), - /// filtered by optional node/edge type filters. Respects tombstones + /// Returns all live node IDs, export-local edge-label names, and edges, + /// filtered by optional node-label and edge-label filters. Respects tombstones /// and prune policies. Each edge is emitted once (outgoing direction only). /// /// Edges are only included if both endpoints are in the exported node set, @@ -4231,53 +4785,85 @@ impl ReadView { &self, options: &ExportOptions, ) -> Result { - // Collect all node type IDs from memtable + immutable memtables + segments - let node_types: Vec = { - let mut types: HashSet = - self.memtable.visible_types(self.snapshot_seq).into_iter().collect(); - for epoch in &self.immutable_epochs { - types.extend(epoch.memtable.visible_types(self.snapshot_seq)); - } - for seg in &self.segments { - for tid in seg.node_type_ids()? { - types.insert(tid); - } - } - // Apply node type filter - if let Some(ref filter) = options.node_type_filter { - let allowed: HashSet = filter.iter().copied().collect(); - types.retain(|t| allowed.contains(t)); - } - types.into_iter().collect() + let resolved_node_filter = + self.resolve_node_label_filter_request_for_graph(options.node_label_filter.as_ref())?; + let resolved_edge_filter = + self.resolve_edge_label_filter_for_graph(options.edge_label_filter.as_deref())?; + let edge_label_filter = match resolved_edge_filter { + LabelFilterResolution::Unconstrained => None, + LabelFilterResolution::Known(label_ids) => Some(label_ids), + LabelFilterResolution::EmptyConstraint => Some(Vec::new()), }; + if resolved_node_filter.is_empty_constraint() { + return Ok(AdjacencyExport { + node_ids: Vec::new(), + node_labels: Vec::new(), + node_label_indexes: Vec::new(), + edge_labels: Vec::new(), + edges: Vec::new(), + }); + } - // Collect all live node IDs (policy-filtered) - let mut node_set: NodeIdSet = NodeIdSet::default(); - for &tid in &node_types { - for id in self.nodes_by_type(tid)? { - node_set.insert(id); + let policy_cutoffs = self.query_policy_cutoffs(); + let node_ids = self.collect_node_ids_for_resolved_label_filter( + &resolved_node_filter, + policy_cutoffs.as_ref(), + )?; + let node_set: NodeIdSet = node_ids.iter().copied().collect(); + + let node_visibility = self.sources().find_node_visibility_meta(&node_ids)?; + let mut node_label_sets = Vec::with_capacity(node_ids.len()); + let mut node_label_index_by_label_id: BTreeMap = BTreeMap::new(); + for (&node_id, state) in node_ids.iter().zip(node_visibility.iter()) { + let NodeVisibilityState::Live(meta) = state else { + return Err(EngineError::InvalidOperation(format!( + "export node {node_id} is not live in latest visibility metadata" + ))); + }; + node_label_sets.push(meta.label_ids); + for &label_id in meta.label_ids.as_slice() { + node_label_index_by_label_id.entry(label_id).or_insert(0); } } - let node_ids: Vec = { - let mut ids: Vec = node_set.iter().copied().collect(); - ids.sort_unstable(); - ids - }; - let edge_filter_slice = options.edge_type_filter.as_deref(); + let mut node_labels = Vec::with_capacity(node_label_index_by_label_id.len()); + for (&label_id, index_slot) in node_label_index_by_label_id.iter_mut() { + let label = self + .label_catalog + .node_label(label_id) + .map(str::to_string) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "export node label side table references missing node label_id {label_id}" + )) + })?; + *index_slot = node_labels.len() as u32; + node_labels.push(label); + } + let node_label_indexes: Vec> = node_label_sets + .iter() + .map(|label_ids| { + label_ids + .as_slice() + .iter() + .map(|label_id| node_label_index_by_label_id[label_id]) + .collect() + }) + .collect(); // Batch-fetch all outgoing neighbors in one cursor walk per segment // instead of O(N) individual binary searches. - let batch_opts = NeighborOptions { - direction: Direction::Outgoing, - type_filter: edge_filter_slice.map(|s| s.to_vec()), - limit: None, - at_epoch: None, - decay_lambda: None, - }; - let all_neighbors = self.neighbors_batch(&node_ids, &batch_opts)?; + let all_neighbors = self.neighbors_batch_resolved( + &node_ids, + Direction::Outgoing, + edge_label_filter.as_deref(), + None, + None, + )?; - let mut edges: Vec<(u64, u64, u32, f32)> = Vec::new(); + let mut edge_label_indexes: BTreeMap = BTreeMap::new(); + let mut edge_labels: Vec = Vec::new(); + let mut edges: Vec = Vec::new(); for &from_id in &node_ids { if let Some(neighbors) = all_neighbors.get(&from_id) { for entry in neighbors { @@ -4285,16 +4871,41 @@ impl ReadView { if !node_set.contains(&entry.node_id) { continue; } - let weight = if options.include_weights { - entry.weight - } else { - 0.0 + let edge_label_index = match edge_label_indexes.get(&entry.edge_label_id) { + Some(index) => *index, + None => { + let label = self + .label_catalog + .edge_label(entry.edge_label_id) + .map(str::to_string) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "export edge {} references missing edge-label label_id {}", + entry.edge_id, entry.edge_label_id + )) + })?; + let index = edge_labels.len() as u32; + edge_labels.push(label); + edge_label_indexes.insert(entry.edge_label_id, index); + index + } }; - edges.push((from_id, entry.node_id, entry.edge_type_id, weight)); + edges.push(ExportEdge { + from: from_id, + to: entry.node_id, + edge_label_index, + weight: options.include_weights.then_some(entry.weight), + }); } } } - Ok(AdjacencyExport { node_ids, edges }) + Ok(AdjacencyExport { + node_ids, + node_labels, + node_label_indexes, + edge_labels, + edges, + }) } } diff --git a/src/engine/tests/graph_ops.rs b/src/engine/tests/graph_ops.rs index e43949d..c14a2d8 100644 --- a/src/engine/tests/graph_ops.rs +++ b/src/engine/tests/graph_ops.rs @@ -1,5 +1,16 @@ // Graph algorithm tests: degree, shortest path, BFS, Dijkstra, all_shortest_paths. +fn graph_filter_names(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_string()).collect() +} + +fn graph_node_label_filter(names: &[&str], mode: LabelMatchMode) -> NodeLabelFilter { + NodeLabelFilter { + labels: graph_filter_names(names), + mode, + } +} + // --- Phase 18a: Degree counts + aggregations --- #[test] @@ -7,12 +18,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 2); @@ -31,15 +42,15 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // a→b, b→a, a→c - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 2); // a→b, a→c @@ -50,35 +61,35 @@ } #[test] - fn test_degree_type_filter() { + fn test_degree_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 20, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); assert_eq!( - db.degree(a, &DegreeOptions { direction: Direction::Outgoing, type_filter: Some(vec![10]), ..Default::default() }) + db.degree(a, &DegreeOptions { direction: Direction::Outgoing, edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }) .unwrap(), 1 ); assert_eq!( - db.degree(a, &DegreeOptions { direction: Direction::Outgoing, type_filter: Some(vec![20]), ..Default::default() }) + db.degree(a, &DegreeOptions { direction: Direction::Outgoing, edge_label_filter: Some(vec!["REPORTS_TO".to_string()]), ..Default::default() }) .unwrap(), 1 ); assert_eq!( - db.degree(a, &DegreeOptions { direction: Direction::Outgoing, type_filter: Some(vec![10, 20]), ..Default::default() }) + db.degree(a, &DegreeOptions { direction: Direction::Outgoing, edge_label_filter: Some(vec!["KNOWS".to_string(), "REPORTS_TO".to_string()]), ..Default::default() }) .unwrap(), 2 ); assert_eq!( - db.degree(a, &DegreeOptions { direction: Direction::Outgoing, type_filter: Some(vec![99]), ..Default::default() }) + db.degree(a, &DegreeOptions { direction: Direction::Outgoing, edge_label_filter: Some(vec!["MISSING_EDGE_LABEL".to_string()]), ..Default::default() }) .unwrap(), 0 ); @@ -91,8 +102,8 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, a, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); // Self-loop: appears in both adj_out and adj_in but must count once @@ -109,14 +120,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, a, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // self-loop - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); // outgoing - db.upsert_edge(c, a, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(c, a, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); // incoming assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 2); // self-loop + a→b @@ -142,10 +153,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let e = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 1); @@ -160,9 +171,9 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 1); @@ -177,12 +188,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -207,14 +218,14 @@ }; let db = DatabaseEngine::open(&dir.path().join("db"), &opts).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); // Re-upsert same edge (same from/to/type → same edge_id with uniqueness) - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 1); @@ -234,9 +245,9 @@ { let db = open_imm(&db_path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.5, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.5, ..Default::default() }) .unwrap(); db.flush().unwrap(); db.close().unwrap(); @@ -253,12 +264,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.5, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.5, ..Default::default() }) .unwrap(); let sum = db @@ -281,13 +292,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); // Crosses memtable + segment @@ -304,12 +315,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }) .unwrap(); let avg = db @@ -339,25 +350,30 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 20, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(d, a, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(d, a, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, d, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }) + db.upsert_edge(a, d, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }) .unwrap(); for dir_val in [Direction::Outgoing, Direction::Incoming, Direction::Both] { - for tf in [None, Some(vec![10u32]), Some(vec![20]), Some(vec![10, 20])] { + for tf in [ + None, + Some(vec!["KNOWS"]), + Some(vec!["REPORTS_TO"]), + Some(vec!["KNOWS", "REPORTS_TO"]), + ] { let tf_ref = tf.as_deref(); - let deg = db.degree(a, &DegreeOptions { direction: dir_val, type_filter: tf_ref.map(|s| s.to_vec()), ..Default::default() }).unwrap(); - let nbrs = db.neighbors(a, &NeighborOptions { direction: dir_val, type_filter: tf_ref.map(|s| s.to_vec()), ..Default::default() }).unwrap(); + let deg = db.degree(a, &DegreeOptions { direction: dir_val, edge_label_filter: tf_ref.map(graph_filter_names), ..Default::default() }).unwrap(); + let nbrs = db.neighbors(a, &NeighborOptions { direction: dir_val, edge_label_filter: tf_ref.map(graph_filter_names), ..Default::default() }).unwrap(); assert_eq!( deg, nbrs.len() as u64, @@ -379,12 +395,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.5, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.5, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.5, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.5, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -405,12 +421,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "hub", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "keep", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "prune_me", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "hub", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "keep", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "prune_me", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); // Before policy: degree=2, sum=5.0 @@ -426,7 +442,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -455,13 +471,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -480,8 +496,8 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, a, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -499,14 +515,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let degs = db @@ -525,27 +541,32 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 20, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(d, a, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(d, a, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let ids = [a, b, c, d]; for dir_val in [Direction::Outgoing, Direction::Incoming, Direction::Both] { - for tf in [None, Some(vec![10u32]), Some(vec![20]), Some(vec![10, 20])] { + for tf in [ + None, + Some(vec!["KNOWS"]), + Some(vec!["REPORTS_TO"]), + Some(vec!["KNOWS", "REPORTS_TO"]), + ] { let tf_ref = tf.as_deref(); - let batch = db.degrees(&ids, &DegreeOptions { direction: dir_val, type_filter: tf_ref.map(|s| s.to_vec()), ..Default::default() }).unwrap(); + let batch = db.degrees(&ids, &DegreeOptions { direction: dir_val, edge_label_filter: tf_ref.map(graph_filter_names), ..Default::default() }).unwrap(); for &nid in &ids { - let individual = db.degree(nid, &DegreeOptions { direction: dir_val, type_filter: tf_ref.map(|s| s.to_vec()), ..Default::default() }).unwrap(); + let individual = db.degree(nid, &DegreeOptions { direction: dir_val, edge_label_filter: tf_ref.map(graph_filter_names), ..Default::default() }).unwrap(); let batch_val = batch.get(&nid).copied().unwrap_or(0); assert_eq!( batch_val, individual, @@ -564,16 +585,16 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // a has edges in segment1, segment2; b has edge in memtable @@ -600,12 +621,12 @@ }; let db = DatabaseEngine::open(&dir.path().join("db"), &opts).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); let degs = db.degrees(&[a], &DegreeOptions::default()).unwrap(); @@ -619,13 +640,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); db.delete_edge(e1).unwrap(); @@ -641,11 +662,11 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, a, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -670,12 +691,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Pass unsorted, with duplicates @@ -694,13 +715,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -720,12 +741,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "hub", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "keep", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "prune_me", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "hub", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "keep", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "prune_me", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Before policy @@ -737,7 +758,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -761,20 +782,20 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, c, 20, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(b, c, "REPORTS_TO", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(d, a, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }) + db.upsert_edge(d, a, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); let ids = [a, b, c, d]; @@ -802,10 +823,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Edge with valid_to = 1 (expired since epoch start) - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 5.0, valid_from: None, valid_to: Some(1), ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 5.0, valid_from: None, valid_to: Some(1), ..Default::default() }) .unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 0); @@ -828,11 +849,11 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Edge with valid_from far in the future let future = now_millis() + 100_000_000; - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 5.0, valid_from: Some(future), valid_to: None, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 5.0, valid_from: Some(future), valid_to: None, ..Default::default() }) .unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 0); @@ -855,13 +876,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 7.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 7.0, ..Default::default() }) .unwrap(); // Invalidate e1 @@ -887,13 +908,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // a→b: valid, a→c: expired - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(1), ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(1), ..Default::default() }) .unwrap(); let degs = db.degrees(&[a], &DegreeOptions::default()).unwrap(); @@ -907,10 +928,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let future = now_millis() + 100_000_000; - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(future), valid_to: None, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(future), valid_to: None, ..Default::default() }) .unwrap(); let degs = db.degrees(&[a], &DegreeOptions::default()).unwrap(); @@ -924,10 +945,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.invalidate_edge(e1, 1).unwrap(); @@ -943,13 +964,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // a→b: valid, a→c: expired - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, valid_from: None, valid_to: Some(1), ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, valid_from: None, valid_to: Some(1), ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -974,14 +995,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // a→b: valid 1000..5000 - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(1000), valid_to: Some(5000), ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(1000), valid_to: Some(5000), ..Default::default() }) .unwrap(); // a→c: valid 3000..8000 - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, valid_from: Some(3000), valid_to: Some(8000), ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, valid_from: Some(3000), valid_to: Some(8000), ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -1052,10 +1073,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -1097,10 +1118,10 @@ let mut nodes = Vec::new(); for i in 0..5u64 { let key = format!("chain_{}", i); - nodes.push(db.upsert_node(1, &key, UpsertNodeOptions::default()).unwrap()); + nodes.push(db.upsert_node("Person", &key, UpsertNodeOptions::default()).unwrap()); } for i in 0..4 { - db.upsert_edge(nodes[i], nodes[i + 1], 10, UpsertEdgeOptions::default()) + db.upsert_edge(nodes[i], nodes[i + 1], "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } nodes @@ -1113,17 +1134,17 @@ /// \ / /// D fn build_diamond(db: &mut DatabaseEngine) -> (u64, u64, u64, u64) { - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); (a, b, c, d) } @@ -1132,10 +1153,10 @@ fn test_shortest_path_direct_neighbors() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let e = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let path = db @@ -1174,8 +1195,8 @@ fn test_shortest_path_no_path_disconnected() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // No edge between a and b let path = db @@ -1190,7 +1211,7 @@ fn test_shortest_path_same_node() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); let path = db .shortest_path(a, a, &ShortestPathOptions::default()) @@ -1228,9 +1249,9 @@ fn test_shortest_path_directed_no_reverse() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // a -> b exists, but b -> a in Outgoing direction does not @@ -1272,34 +1293,34 @@ } #[test] - fn test_shortest_path_edge_type_filter() { + fn test_shortest_path_edge_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); - // Direct path a->c with type 20 - db.upsert_edge(a, c, 20, UpsertEdgeOptions::default()) + // Direct path a->c with label 20 + db.upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - // Indirect path a->b->c with type 10 - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + // Indirect path a->b->c with label 10 + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - // Filter to type 10 only. Must go through b. + // Filter to label 10 only. Must go through b. let path = db - .shortest_path(a, c, &ShortestPathOptions { type_filter: Some(vec![10]), ..Default::default() }) + .shortest_path(a, c, &ShortestPathOptions { edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }) .unwrap(); assert!(path.is_some()); let p = path.unwrap(); assert_eq!(p.total_cost, 2.0); assert_eq!(p.nodes, vec![a, b, c]); - // Filter to type 20. Direct path. + // Filter to label 20. Direct path. let path = db - .shortest_path(a, c, &ShortestPathOptions { type_filter: Some(vec![20]), ..Default::default() }) + .shortest_path(a, c, &ShortestPathOptions { edge_label_filter: Some(vec!["REPORTS_TO".to_string()]), ..Default::default() }) .unwrap(); assert!(path.is_some()); let p = path.unwrap(); @@ -1335,15 +1356,15 @@ fn test_shortest_path_temporal_filtering() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Edge a->b valid from 100 to 200 - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(200), ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(200), ..Default::default() }) .unwrap(); // Edge b->c valid from 100 to 300 - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(300), ..Default::default() }) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(300), ..Default::default() }) .unwrap(); // At time 150: both edges valid, path exists @@ -1392,7 +1413,7 @@ fn test_is_connected_same_node() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); assert!(db .is_connected(a, a, &IsConnectedOptions::default()) @@ -1405,8 +1426,8 @@ fn test_is_connected_disconnected() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); assert!(!db .is_connected(a, b, &IsConnectedOptions::default()) @@ -1458,21 +1479,21 @@ let db = open_imm(&dir.path().join("db")); // First segment: a -> b -> c - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); // Second segment: c -> d -> e - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - let e = db.upsert_node(1, "e", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + let e = db.upsert_node("Person", "e", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(d, e, 10, UpsertEdgeOptions::default()) + db.upsert_edge(d, e, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -1493,14 +1514,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -1520,17 +1541,17 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Direct path a->c and indirect a->b->c let direct = db - .upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Delete direct edge @@ -1552,19 +1573,19 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); // a->b->c and a->d->c - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(d, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(d, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -1588,20 +1609,20 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); let cheap_ab = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let cheap_bc = db - .upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, d, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + db.upsert_edge(a, d, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); - db.upsert_edge(d, c, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + db.upsert_edge(d, c, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -1651,7 +1672,7 @@ db.ingest_mode().unwrap(); for i in 0..10 { - db.upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); } @@ -1660,7 +1681,7 @@ // All data survives for i in 0..10 { - assert!(db.get_node_by_key(1, &format!("n{}", i)).unwrap().is_some()); + assert!(db.get_node_by_key("Person", &format!("n{}", i)).unwrap().is_some()); } let stats = db.end_ingest().unwrap().unwrap(); @@ -1669,7 +1690,7 @@ // All data still intact after compaction for i in 0..10 { - assert!(db.get_node_by_key(1, &format!("n{}", i)).unwrap().is_some()); + assert!(db.get_node_by_key("Person", &format!("n{}", i)).unwrap().is_some()); } db.close().unwrap(); @@ -1723,30 +1744,30 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let end = db.upsert_node(1, "end", UpsertNodeOptions::default()).unwrap(); - let bridge = db.upsert_node(1, "bridge", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let end = db.upsert_node("Person", "end", UpsertNodeOptions::default()).unwrap(); + let bridge = db.upsert_node("Person", "bridge", UpsertNodeOptions::default()).unwrap(); // 100 dead-end nodes from start for i in 0..100 { let key = format!("dead_s_{}", i); - let n = db.upsert_node(1, &key, UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, n, 10, UpsertEdgeOptions::default()) + let n = db.upsert_node("Person", &key, UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(start, n, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } // 100 dead-end nodes from end (incoming) for i in 0..100 { let key = format!("dead_e_{}", i); - let n = db.upsert_node(1, &key, UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(n, end, 10, UpsertEdgeOptions::default()) + let n = db.upsert_node("Person", &key, UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(n, end, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } // The actual path: start -> bridge -> end - db.upsert_edge(start, bridge, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, bridge, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(bridge, end, 10, UpsertEdgeOptions::default()) + db.upsert_edge(bridge, end, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let path = db @@ -1764,14 +1785,14 @@ fn test_shortest_path_incoming_direction() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Edges: a -> b -> c - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // From c to a following Incoming edges (reverse traversal) @@ -1793,19 +1814,19 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); // Cycle: a -> b -> c -> a, with d only reachable from c - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Should find path despite cycle @@ -1824,7 +1845,7 @@ fn test_shortest_path_nonexistent_node() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); // Node 999999 was never created let path = db @@ -1849,18 +1870,18 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let hidden = db.upsert_node(1, "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); - let visible = db.upsert_node(1, "visible", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let hidden = db.upsert_node("Person", "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); + let visible = db.upsert_node("Person", "visible", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, hidden, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, hidden, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(hidden, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(hidden, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, visible, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, visible, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(visible, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(visible, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.set_prune_policy( @@ -1868,7 +1889,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -1904,28 +1925,80 @@ db.close().unwrap(); } + #[test] + fn test_path_apis_exclude_pruned_multi_label_nodes() { + let dir = TempDir::new().unwrap(); + let db = open_imm(&dir.path().join("db")); + + let a = db + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let hidden = db + .upsert_node( + &["Person", "Hidden"], + "hidden", + UpsertNodeOptions { + weight: 0.2, + ..Default::default() + }, + ) + .unwrap(); + let c = db + .upsert_node("Person", "c", UpsertNodeOptions::default()) + .unwrap(); + + db.upsert_edge(a, hidden, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.upsert_edge(hidden, c, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + db.set_prune_policy( + "hide-low-hidden", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("Hidden".to_string()), + }, + ) + .unwrap(); + + assert!(db + .shortest_path(a, c, &ShortestPathOptions::default()) + .unwrap() + .is_none()); + assert!(!db + .is_connected(a, c, &IsConnectedOptions::default()) + .unwrap()); + assert!(db + .all_shortest_paths(a, c, &AllShortestPathsOptions::default()) + .unwrap() + .is_empty()); + + db.close().unwrap(); + } + #[test] fn test_weighted_path_apis_respect_prune_policy_visibility() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let hidden = db.upsert_node(1, "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); - let visible = db.upsert_node(1, "visible", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let hidden = db.upsert_node("Person", "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); + let visible = db.upsert_node("Person", "visible", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); let mut cheap = BTreeMap::new(); cheap.insert("cost".to_string(), PropValue::Float(1.0)); let mut expensive = BTreeMap::new(); expensive.insert("cost".to_string(), PropValue::Float(10.0)); - db.upsert_edge(a, hidden, 10, UpsertEdgeOptions { props: cheap.clone(), ..Default::default() }) + db.upsert_edge(a, hidden, "KNOWS", UpsertEdgeOptions { props: cheap.clone(), ..Default::default() }) .unwrap(); - db.upsert_edge(hidden, c, 10, UpsertEdgeOptions { props: cheap, ..Default::default() }) + db.upsert_edge(hidden, c, "KNOWS", UpsertEdgeOptions { props: cheap, ..Default::default() }) .unwrap(); - db.upsert_edge(a, visible, 10, UpsertEdgeOptions { props: expensive.clone(), weight: 10.0, ..Default::default() }) + db.upsert_edge(a, visible, "KNOWS", UpsertEdgeOptions { props: expensive.clone(), weight: 10.0, ..Default::default() }) .unwrap(); - db.upsert_edge(visible, c, 10, UpsertEdgeOptions { props: expensive, weight: 10.0, ..Default::default() }) + db.upsert_edge(visible, c, "KNOWS", UpsertEdgeOptions { props: expensive, weight: 10.0, ..Default::default() }) .unwrap(); db.set_prune_policy( @@ -1933,7 +2006,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -1970,15 +2043,15 @@ // BFS shortest: A->B (1 hop). Dijkstra shortest: A->C->B (cost 5) let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 10.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 10.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); let _ec = db - .upsert_edge(c, b, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + .upsert_edge(c, b, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); let path = db @@ -1998,21 +2071,21 @@ // Use a custom property "cost" on edges let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); let mut props_ab = BTreeMap::new(); props_ab.insert("cost".to_string(), PropValue::Float(100.0)); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { props: props_ab, weight: 1.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { props: props_ab, weight: 1.0, ..Default::default() }).unwrap(); let mut props_ac = BTreeMap::new(); props_ac.insert("cost".to_string(), PropValue::Float(1.0)); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { props: props_ac, weight: 1.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { props: props_ac, weight: 1.0, ..Default::default() }).unwrap(); let mut props_cb = BTreeMap::new(); props_cb.insert("cost".to_string(), PropValue::Float(2.0)); - db.upsert_edge(c, b, 10, UpsertEdgeOptions { props: props_cb, weight: 1.0, ..Default::default() }).unwrap(); + db.upsert_edge(c, b, "KNOWS", UpsertEdgeOptions { props: props_cb, weight: 1.0, ..Default::default() }).unwrap(); let path = db .shortest_path(a, b, &ShortestPathOptions { weight_field: Some("cost".to_string()), ..Default::default() }) @@ -2031,14 +2104,14 @@ // max_cost=4 should exclude both, returning None let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(c, b, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }) + db.upsert_edge(c, b, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }) .unwrap(); // max_cost=4: nothing reachable @@ -2061,9 +2134,9 @@ fn test_dijkstra_negative_weight_error() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: -1.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: -1.0, ..Default::default() }) .unwrap(); let result = db.shortest_path(a, b, &ShortestPathOptions { weight_field: Some("weight".to_string()), ..Default::default() }); @@ -2080,21 +2153,21 @@ // The !w.is_finite() guard must catch it. let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let mut props = BTreeMap::new(); props.insert("w".to_string(), PropValue::Float(f64::NAN)); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { props, weight: 1.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { props, weight: 1.0, ..Default::default() }).unwrap(); let result = db.shortest_path(a, b, &ShortestPathOptions { weight_field: Some("w".to_string()), ..Default::default() }); assert!(result.is_err()); // Also test Infinity - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); let mut props2 = BTreeMap::new(); props2.insert("w".to_string(), PropValue::Float(f64::INFINITY)); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { props: props2, weight: 1.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { props: props2, weight: 1.0, ..Default::default() }).unwrap(); let result = db.shortest_path(a, c, &ShortestPathOptions { weight_field: Some("w".to_string()), ..Default::default() }); assert!(result.is_err()); @@ -2111,15 +2184,15 @@ // but the total path is 3 hops. Must be rejected. let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // max_depth=2: 3-hop path should be rejected @@ -2144,12 +2217,12 @@ // A->B (weight 0), B->C (weight 0) let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); let path = db @@ -2166,7 +2239,7 @@ fn test_dijkstra_same_node() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); let path = db .shortest_path(a, a, &ShortestPathOptions { weight_field: Some("weight".to_string()), ..Default::default() }) @@ -2193,19 +2266,19 @@ // side meets first. let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, d, 10, UpsertEdgeOptions { weight: 10.0, ..Default::default() }) + db.upsert_edge(a, d, "KNOWS", UpsertEdgeOptions { weight: 10.0, ..Default::default() }) .unwrap(); let path = db @@ -2224,8 +2297,8 @@ fn test_dijkstra_no_path() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // No edges let path = db @@ -2240,12 +2313,12 @@ fn test_dijkstra_after_flush() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -2263,13 +2336,13 @@ fn test_dijkstra_after_compact() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -2290,21 +2363,21 @@ // A->D->E->C (3 hops, cost 1+1+1=3) let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - let e = db.upsert_node(1, "e", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + let e = db.upsert_node("Person", "e", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 100.0, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 100.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 100.0, ..Default::default() }) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 100.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(d, e, 10, UpsertEdgeOptions::default()) + db.upsert_edge(d, e, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(e, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(e, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // BFS: 2 hops (A->B->C) @@ -2330,15 +2403,15 @@ // A->B->C->D, all weight 1 let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // max_depth=1: can only reach B from A side @@ -2400,7 +2473,7 @@ fn test_all_shortest_paths_bfs_same_node() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); let paths = db .all_shortest_paths(a, a, &AllShortestPathsOptions::default()) @@ -2416,8 +2489,8 @@ fn test_all_shortest_paths_bfs_no_path() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let paths = db .all_shortest_paths(a, b, &AllShortestPathsOptions::default()) @@ -2432,17 +2505,17 @@ // Diamond with equal weights: A->B(w=3)->D and A->C(w=3)->D, both cost 6 let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); let paths = db @@ -2462,17 +2535,17 @@ fn test_all_shortest_paths_dijkstra_max_paths() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let paths = db @@ -2487,8 +2560,8 @@ fn test_all_shortest_paths_dijkstra_no_path() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let paths = db .all_shortest_paths(a, b, &AllShortestPathsOptions { weight_field: Some("weight".to_string()), ..Default::default() }) @@ -2502,9 +2575,9 @@ fn test_all_shortest_paths_negative_weight_error() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: -5.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: -5.0, ..Default::default() }) .unwrap(); let result = db.all_shortest_paths(a, b, &AllShortestPathsOptions { weight_field: Some("weight".to_string()), ..Default::default() }); @@ -2532,18 +2605,18 @@ fn test_all_shortest_paths_bfs_after_compact() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -2564,12 +2637,12 @@ // Use PropValue::Int for the weight field let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let mut props = BTreeMap::new(); props.insert("distance".to_string(), PropValue::Int(7)); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { props, weight: 1.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { props, weight: 1.0, ..Default::default() }).unwrap(); let path = db .shortest_path(a, b, &ShortestPathOptions { weight_field: Some("distance".to_string()), ..Default::default() }) @@ -2603,20 +2676,20 @@ // With max_depth=2: must find the 2-hop path S→A→T, not return None. let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let s = db.upsert_node(1, "s", UpsertNodeOptions::default()).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let t = db.upsert_node(1, "t", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(s, a, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }) + let s = db.upsert_node("Person", "s", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let t = db.upsert_node("Person", "t", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(s, a, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, t, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, t, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(s, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(s, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, t, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(c, t, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); // Without max_depth @@ -2647,23 +2720,23 @@ // still recover the shorter-hop equal-cost route. let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let s = db.upsert_node(1, "s", UpsertNodeOptions::default()).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let v = db.upsert_node(1, "v", UpsertNodeOptions::default()).unwrap(); - let x = db.upsert_node(1, "x", UpsertNodeOptions::default()).unwrap(); - let t = db.upsert_node(1, "t", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(s, a, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + let s = db.upsert_node("Person", "s", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let v = db.upsert_node("Person", "v", UpsertNodeOptions::default()).unwrap(); + let x = db.upsert_node("Person", "x", UpsertNodeOptions::default()).unwrap(); + let t = db.upsert_node("Person", "t", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(s, a, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, v, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, v, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(s, x, 10, UpsertEdgeOptions::default()) + db.upsert_edge(s, x, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(x, v, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + db.upsert_edge(x, v, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); - db.upsert_edge(v, t, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + db.upsert_edge(v, t, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); let path = db @@ -2682,20 +2755,20 @@ // valid path instead of filtering the global optimum down to nothing. let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let s = db.upsert_node(1, "s", UpsertNodeOptions::default()).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let t = db.upsert_node(1, "t", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(s, a, 10, UpsertEdgeOptions::default()) + let s = db.upsert_node("Person", "s", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let t = db.upsert_node("Person", "t", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(s, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, t, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, t, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(s, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(s, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(c, t, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(c, t, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); let path = db @@ -2715,14 +2788,14 @@ // Must not stack-overflow; should return path(s) with cost 1 let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let t = db.upsert_node(1, "t", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let t = db.upsert_node("Person", "t", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, a, 10, UpsertEdgeOptions { weight: 0.0, ..Default::default() }) + db.upsert_edge(b, a, "KNOWS", UpsertEdgeOptions { weight: 0.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, t, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, t, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let paths = db @@ -2746,20 +2819,20 @@ // max_depth=3: both paths let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let s = db.upsert_node(1, "s", UpsertNodeOptions::default()).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let t = db.upsert_node(1, "t", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(s, a, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }) + let s = db.upsert_node("Person", "s", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let t = db.upsert_node("Person", "t", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(s, a, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, t, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, t, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(s, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(s, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, t, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(c, t, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); // max_depth=2: only the 2-hop path @@ -2786,20 +2859,20 @@ fn test_all_shortest_paths_dijkstra_max_depth_uses_best_constrained_cost() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let s = db.upsert_node(1, "s", UpsertNodeOptions::default()).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let t = db.upsert_node(1, "t", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(s, a, 10, UpsertEdgeOptions::default()) + let s = db.upsert_node("Person", "s", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let t = db.upsert_node("Person", "t", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(s, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, t, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, t, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(s, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(s, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(c, t, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(c, t, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); let paths = db @@ -2816,17 +2889,17 @@ fn test_all_shortest_paths_dijkstra_after_flush_weighted() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -2848,17 +2921,17 @@ let (a, d) = { let db = open_imm(&db_path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -2903,12 +2976,12 @@ // A->B->C, query with Direction::Both from C to A should find path let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); let path = db @@ -2926,16 +2999,16 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); // Bridge edge in memtable only - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let path = db @@ -2961,15 +3034,15 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); let mut edge_ids = Vec::new(); for i in 0..8 { let neighbor = engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); let eid = engine - .upsert_edge(center, neighbor, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, neighbor, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); edge_ids.push(eid); } @@ -3017,16 +3090,16 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); // Create 4 edges, flush to segment for i in 0..4 { let n = engine - .upsert_node(1, &format!("seg{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("seg{}", i), UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(center, n, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, n, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3034,10 +3107,10 @@ // Create 4 more in memtable for i in 0..4 { let n = engine - .upsert_node(1, &format!("mem{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("mem{}", i), UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(center, n, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, n, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } @@ -3070,20 +3143,20 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); - let n1 = engine.upsert_node(1, "n1", UpsertNodeOptions::default()).unwrap(); - let n2 = engine.upsert_node(1, "n2", UpsertNodeOptions::default()).unwrap(); - let n3 = engine.upsert_node(1, "n3", UpsertNodeOptions::default()).unwrap(); + let n1 = engine.upsert_node("Person", "n1", UpsertNodeOptions::default()).unwrap(); + let n2 = engine.upsert_node("Person", "n2", UpsertNodeOptions::default()).unwrap(); + let n3 = engine.upsert_node("Person", "n3", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(center, n1, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, n1, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let e2 = engine - .upsert_edge(center, n2, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, n2, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(center, n3, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, n3, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.delete_edge(e2).unwrap(); @@ -3105,18 +3178,18 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); - let n1 = engine.upsert_node(1, "n1", UpsertNodeOptions::default()).unwrap(); - let n2 = engine.upsert_node(1, "n2", UpsertNodeOptions::default()).unwrap(); + let n1 = engine.upsert_node("Person", "n1", UpsertNodeOptions::default()).unwrap(); + let n2 = engine.upsert_node("Person", "n2", UpsertNodeOptions::default()).unwrap(); // Edge valid from 100 to 200 engine - .upsert_edge(center, n1, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(200), ..Default::default() }) + .upsert_edge(center, n1, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(200), ..Default::default() }) .unwrap(); // Edge valid from 150 to 300 engine - .upsert_edge(center, n2, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(150), valid_to: Some(300), ..Default::default() }) + .upsert_edge(center, n2, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(150), valid_to: Some(300), ..Default::default() }) .unwrap(); // At epoch 175, both valid @@ -3147,14 +3220,14 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); for i in 0..15 { let n = engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(center, n, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, n, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } @@ -3194,14 +3267,14 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); - let n_past = engine.upsert_node(1, "past", UpsertNodeOptions::default()).unwrap(); + let n_past = engine.upsert_node("Person", "past", UpsertNodeOptions::default()).unwrap(); let n_current = engine - .upsert_node(1, "current", UpsertNodeOptions::default()) + .upsert_node("Person", "current", UpsertNodeOptions::default()) .unwrap(); let n_future = engine - .upsert_node(1, "future", UpsertNodeOptions::default()) + .upsert_node("Person", "future", UpsertNodeOptions::default()) .unwrap(); let now = SystemTime::now() @@ -3212,19 +3285,19 @@ // Expired edge (valid_to in the past) engine .upsert_edge( - center, n_past, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(now - 2000), valid_to: Some(now - 1000), ..Default::default() }, + center, n_past, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(now - 2000), valid_to: Some(now - 1000), ..Default::default() }, ) .unwrap(); // Currently valid edge engine .upsert_edge( - center, n_current, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(now - 1000), valid_to: Some(now + 100_000), ..Default::default() }, + center, n_current, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(now - 1000), valid_to: Some(now + 100_000), ..Default::default() }, ) .unwrap(); // Future edge (valid_from in the future) engine .upsert_edge( - center, n_future, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(now + 50_000), valid_to: Some(now + 100_000), ..Default::default() }, + center, n_future, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(now + 50_000), valid_to: Some(now + 100_000), ..Default::default() }, ) .unwrap(); @@ -3259,7 +3332,7 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); let epoch = 500_000i64; let mut valid_node_ids = Vec::new(); @@ -3267,13 +3340,13 @@ // Create 10 edges, alternating valid/invalid at epoch=500000 for i in 0..10u64 { let n = engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); if i % 2 == 0 { // Valid: valid_from=100000, valid_to=900000 engine .upsert_edge( - center, n, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(100_000), valid_to: Some(900_000), ..Default::default() }, + center, n, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(100_000), valid_to: Some(900_000), ..Default::default() }, ) .unwrap(); valid_node_ids.push(n); @@ -3281,7 +3354,7 @@ // Invalid at epoch: valid_from=600000, valid_to=900000 engine .upsert_edge( - center, n, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(600_000), valid_to: Some(900_000), ..Default::default() }, + center, n, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(600_000), valid_to: Some(900_000), ..Default::default() }, ) .unwrap(); } @@ -3321,17 +3394,17 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let center = engine - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); let mut visible_neighbors = Vec::new(); for i in 0..17u64 { let weight = if i < 12 { 0.1 } else { 1.0 }; let node_id = engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions { weight, ..Default::default() }) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions { weight, ..Default::default() }) .unwrap(); engine - .upsert_edge(center, node_id, 10, UpsertEdgeOptions::default()) + .upsert_edge(center, node_id, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); if weight > 0.5 { visible_neighbors.push(node_id); @@ -3344,7 +3417,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -3375,18 +3448,18 @@ } #[test] - fn test_nodes_by_type_paged_policy_cursor_correctness() { - // Verify cursor is pushed down in policy-filtered nodes_by_type_paged. + fn test_nodes_by_labels_paged_policy_cursor_correctness() { + // Verify cursor is pushed down in policy-filtered nodes_by_labels_paged. // Page 2 should not re-return page 1 items. let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // Create 10 nodes of type 1 + // Create 10 Person-labeled nodes. let mut all_ids = Vec::new(); for i in 0..10 { let id = engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); all_ids.push(id); } @@ -3399,7 +3472,7 @@ PrunePolicy { max_age_ms: Some(999_999_999), max_weight: None, - type_id: Some(999), // non-matching type, excludes nothing + label: Some("SpecialNode999".to_string()), // non-matching type, excludes nothing }, ) .unwrap(); @@ -3409,8 +3482,7 @@ let mut cursor: Option = None; loop { let page = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: cursor, @@ -3452,7 +3524,7 @@ }), ); let id = engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions { props, ..Default::default() }) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions { props, ..Default::default() }) .unwrap(); if i % 3 == 0 { matching_ids.push(id); @@ -3465,7 +3537,7 @@ loop { let page = engine .find_nodes_paged( - 1, + "Person", "color", &PropValue::String("red".to_string()), &PageRequest { @@ -3502,7 +3574,7 @@ let mut props = BTreeMap::new(); props.insert("tag".to_string(), PropValue::String("yes".to_string())); let id = engine - .upsert_node(1, &format!("pre{}", i), UpsertNodeOptions { props, ..Default::default() }) + .upsert_node("Person", &format!("pre{}", i), UpsertNodeOptions { props, ..Default::default() }) .unwrap(); all_matching.push(id); } @@ -3511,7 +3583,7 @@ let mut props = BTreeMap::new(); props.insert("tag".to_string(), PropValue::String("yes".to_string())); let id = engine - .upsert_node(1, &format!("post{}", i), UpsertNodeOptions { props, ..Default::default() }) + .upsert_node("Person", &format!("post{}", i), UpsertNodeOptions { props, ..Default::default() }) .unwrap(); all_matching.push(id); } @@ -3522,7 +3594,7 @@ loop { let page = engine .find_nodes_paged( - 1, + "Person", "tag", &PropValue::String("yes".to_string()), &PageRequest { @@ -3553,7 +3625,7 @@ let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String("red".to_string())); - let n1 = engine.upsert_node(1, "n1", UpsertNodeOptions { props, ..Default::default() }).unwrap(); + let n1 = engine.upsert_node("Person", "n1", UpsertNodeOptions { props, ..Default::default() }).unwrap(); // Flush: n1 with color=red goes to segment engine.flush().unwrap(); @@ -3561,11 +3633,11 @@ // Update n1 to color=blue in memtable let mut props2 = BTreeMap::new(); props2.insert("color".to_string(), PropValue::String("blue".to_string())); - engine.upsert_node(1, "n1", UpsertNodeOptions { props: props2, ..Default::default() }).unwrap(); + engine.upsert_node("Person", "n1", UpsertNodeOptions { props: props2, ..Default::default() }).unwrap(); // find_nodes for color=red must NOT return n1 (stale segment match) let red = engine - .find_nodes(1, "color", &PropValue::String("red".to_string())) + .find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap(); assert!( !red.contains(&n1), @@ -3574,14 +3646,14 @@ // find_nodes for color=blue SHOULD return n1 let blue = engine - .find_nodes(1, "color", &PropValue::String("blue".to_string())) + .find_nodes("Person", "color", &PropValue::String("blue".to_string())) .unwrap(); assert!(blue.contains(&n1)); // Parity: find_nodes_paged must agree let red_paged = engine .find_nodes_paged( - 1, + "Person", "color", &PropValue::String("red".to_string()), &PageRequest { @@ -3594,7 +3666,7 @@ let blue_paged = engine .find_nodes_paged( - 1, + "Person", "color", &PropValue::String("blue".to_string()), &PageRequest { @@ -3615,7 +3687,7 @@ let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String("red".to_string())); - let n1 = engine.upsert_node(1, "n1", UpsertNodeOptions { props, ..Default::default() }).unwrap(); + let n1 = engine.upsert_node("Person", "n1", UpsertNodeOptions { props, ..Default::default() }).unwrap(); // Flush: S1 has n1 with color=red engine.flush().unwrap(); @@ -3623,12 +3695,12 @@ // Update and flush again: S2 has n1 with color=blue let mut props2 = BTreeMap::new(); props2.insert("color".to_string(), PropValue::String("blue".to_string())); - engine.upsert_node(1, "n1", UpsertNodeOptions { props: props2, ..Default::default() }).unwrap(); + engine.upsert_node("Person", "n1", UpsertNodeOptions { props: props2, ..Default::default() }).unwrap(); engine.flush().unwrap(); // find_nodes for color=red must NOT return n1 let red = engine - .find_nodes(1, "color", &PropValue::String("red".to_string())) + .find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap(); assert!( !red.contains(&n1), @@ -3636,7 +3708,7 @@ ); let blue = engine - .find_nodes(1, "color", &PropValue::String("blue".to_string())) + .find_nodes("Person", "color", &PropValue::String("blue".to_string())) .unwrap(); assert!(blue.contains(&n1)); } @@ -3648,24 +3720,24 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let depth1_low = db.upsert_node(1, "depth1-low", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let depth1_low = db.upsert_node("Person", "depth1-low", UpsertNodeOptions::default()).unwrap(); let depth1_high = db - .upsert_node(1, "depth1-high", UpsertNodeOptions::default()) + .upsert_node("Person", "depth1-high", UpsertNodeOptions::default()) .unwrap(); - let depth2_low = db.upsert_node(1, "depth2-low", UpsertNodeOptions::default()).unwrap(); - let depth2_mid = db.upsert_node(1, "depth2-mid", UpsertNodeOptions::default()).unwrap(); - let depth2_high = db.upsert_node(1, "depth2-high", UpsertNodeOptions::default()).unwrap(); + let depth2_low = db.upsert_node("Person", "depth2-low", UpsertNodeOptions::default()).unwrap(); + let depth2_mid = db.upsert_node("Person", "depth2-mid", UpsertNodeOptions::default()).unwrap(); + let depth2_high = db.upsert_node("Person", "depth2-high", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, depth1_high, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, depth1_high, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, depth1_low, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, depth1_low, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(depth1_high, depth2_high, 10, UpsertEdgeOptions::default()) + db.upsert_edge(depth1_high, depth2_high, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(depth1_high, depth2_mid, 10, UpsertEdgeOptions::default()) + db.upsert_edge(depth1_high, depth2_mid, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(depth1_low, depth2_low, 10, UpsertEdgeOptions::default()) + db.upsert_edge(depth1_low, depth2_low, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let result = db @@ -3697,15 +3769,15 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let depth2_only = db @@ -3745,7 +3817,7 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); let page = db .traverse(start, 3, &TraverseOptions { min_depth: 0, limit: Some(1), ..Default::default() }) .unwrap(); @@ -3762,26 +3834,26 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let middle = db.upsert_node(2, "middle", UpsertNodeOptions::default()).unwrap(); - let hidden = db.upsert_node(2, "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let middle = db.upsert_node("Company", "middle", UpsertNodeOptions::default()).unwrap(); + let hidden = db.upsert_node("Company", "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); - db.upsert_edge(start, middle, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, middle, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(middle, hidden, 10, UpsertEdgeOptions::default()) + db.upsert_edge(middle, hidden, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.set_prune_policy( "low_weight", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let page = db - .traverse(start, 2, &TraverseOptions { node_type_filter: Some(vec![2]), limit: Some(1), ..Default::default() }) + .traverse(start, 2, &TraverseOptions { emit_node_label_filter: Some(graph_node_label_filter(&["Company"], LabelMatchMode::Any)), limit: Some(1), ..Default::default() }) .unwrap(); assert_eq!(page.items.len(), 1); @@ -3791,23 +3863,104 @@ db.close().unwrap(); } + #[test] + fn test_traverse_emit_label_filter_supports_single_any_all_multi_label() { + let dir = TempDir::new().unwrap(); + let db = open_imm(&dir.path().join("db")); + + let start = db + .upsert_node("Person", "start", UpsertNodeOptions::default()) + .unwrap(); + let bridge = db + .upsert_node("Company", "bridge", UpsertNodeOptions::default()) + .unwrap(); + let article = db + .upsert_node(&["Article", "Featured"], "article", UpsertNodeOptions::default()) + .unwrap(); + let draft = db + .upsert_node("Article", "draft", UpsertNodeOptions::default()) + .unwrap(); + + db.upsert_edge(start, bridge, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.upsert_edge(bridge, article, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.upsert_edge(bridge, draft, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let single = db + .traverse( + start, + 2, + &TraverseOptions { + emit_node_label_filter: Some(graph_node_label_filter( + &["Article"], + LabelMatchMode::Any, + )), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + single.items.iter().map(|hit| hit.node_id).collect::>(), + vec![article, draft] + ); + + let any = db + .traverse( + start, + 2, + &TraverseOptions { + emit_node_label_filter: Some(graph_node_label_filter( + &["Company", "Featured"], + LabelMatchMode::Any, + )), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + any.items.iter().map(|hit| hit.node_id).collect::>(), + vec![bridge, article] + ); + + let all = db + .traverse( + start, + 2, + &TraverseOptions { + emit_node_label_filter: Some(graph_node_label_filter( + &["Article", "Featured"], + LabelMatchMode::All, + )), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(all.items.len(), 1); + assert_eq!(all.items[0].node_id, article); + assert_eq!(all.items[0].depth, 2); + + db.close().unwrap(); + } + #[test] fn test_traverse_cycle_safe_and_unique() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let result = db @@ -3828,16 +3981,16 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let p1 = db @@ -3864,16 +4017,16 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let p1 = db @@ -3914,13 +4067,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let page = db @@ -3943,10 +4096,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let page = db @@ -3962,24 +4115,24 @@ } #[test] - fn test_traverse_edge_filter_and_node_type_filter_is_emission_only() { + fn test_traverse_edge_filter_and_node_label_filter_is_emission_only() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let middle = db.upsert_node(2, "middle", UpsertNodeOptions::default()).unwrap(); - let target = db.upsert_node(3, "target", UpsertNodeOptions::default()).unwrap(); - let wrong_edge = db.upsert_node(3, "wrong-edge", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let middle = db.upsert_node("Company", "middle", UpsertNodeOptions::default()).unwrap(); + let target = db.upsert_node("Article", "target", UpsertNodeOptions::default()).unwrap(); + let wrong_edge = db.upsert_node("Article", "wrong-edge", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, middle, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, middle, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(middle, target, 10, UpsertEdgeOptions::default()) + db.upsert_edge(middle, target, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, wrong_edge, 20, UpsertEdgeOptions::default()) + db.upsert_edge(start, wrong_edge, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); let result = db - .traverse(start, 2, &TraverseOptions { edge_type_filter: Some(vec![10]), node_type_filter: Some(vec![3]), ..Default::default() }) + .traverse(start, 2, &TraverseOptions { edge_label_filter: Some(vec!["KNOWS".to_string()]), emit_node_label_filter: Some(graph_node_label_filter(&["Article"], LabelMatchMode::Any)), ..Default::default() }) .unwrap(); let pairs: Vec<(u64, u32)> = result.items.iter().map(|hit| (hit.node_id, hit.depth)).collect(); @@ -3993,16 +4146,16 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let incoming = db @@ -4032,30 +4185,30 @@ } #[test] - fn test_traverse_pagination_with_node_type_filter_uses_filtered_path() { + fn test_traverse_pagination_with_node_label_filter_uses_filtered_path() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let mid_a = db.upsert_node(2, "mid-a", UpsertNodeOptions::default()).unwrap(); - let mid_b = db.upsert_node(2, "mid-b", UpsertNodeOptions::default()).unwrap(); - let hit_a = db.upsert_node(3, "hit-a", UpsertNodeOptions::default()).unwrap(); - let hit_b = db.upsert_node(3, "hit-b", UpsertNodeOptions::default()).unwrap(); - let skip = db.upsert_node(4, "skip", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let mid_a = db.upsert_node("Company", "mid-a", UpsertNodeOptions::default()).unwrap(); + let mid_b = db.upsert_node("Company", "mid-b", UpsertNodeOptions::default()).unwrap(); + let hit_a = db.upsert_node("Article", "hit-a", UpsertNodeOptions::default()).unwrap(); + let hit_b = db.upsert_node("Article", "hit-b", UpsertNodeOptions::default()).unwrap(); + let skip = db.upsert_node("Topic", "skip", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, mid_a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, mid_a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, mid_b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, mid_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(mid_a, hit_a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(mid_a, hit_a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(mid_a, skip, 10, UpsertEdgeOptions::default()) + db.upsert_edge(mid_a, skip, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(mid_b, hit_b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(mid_b, hit_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let p1 = db - .traverse(start, 2, &TraverseOptions { node_type_filter: Some(vec![3]), limit: Some(1), ..Default::default() }) + .traverse(start, 2, &TraverseOptions { emit_node_label_filter: Some(graph_node_label_filter(&["Article"], LabelMatchMode::Any)), limit: Some(1), ..Default::default() }) .unwrap(); assert_eq!( p1.items @@ -4073,7 +4226,7 @@ ); let p2 = db - .traverse(start, 2, &TraverseOptions { node_type_filter: Some(vec![3]), limit: Some(1), cursor: p1.next_cursor.clone(), ..Default::default() }) + .traverse(start, 2, &TraverseOptions { emit_node_label_filter: Some(graph_node_label_filter(&["Article"], LabelMatchMode::Any)), limit: Some(1), cursor: p1.next_cursor.clone(), ..Default::default() }) .unwrap(); assert_eq!( p2.items @@ -4092,26 +4245,26 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let hidden = db.upsert_node(1, "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let hidden = db.upsert_node("Person", "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); let hidden_target = db - .upsert_node(1, "hidden-target", UpsertNodeOptions::default()) + .upsert_node("Person", "hidden-target", UpsertNodeOptions::default()) .unwrap(); - let deleted = db.upsert_node(1, "deleted", UpsertNodeOptions::default()).unwrap(); - let future = db.upsert_node(1, "future", UpsertNodeOptions::default()).unwrap(); - let visible = db.upsert_node(1, "visible", UpsertNodeOptions::default()).unwrap(); + let deleted = db.upsert_node("Person", "deleted", UpsertNodeOptions::default()).unwrap(); + let future = db.upsert_node("Person", "future", UpsertNodeOptions::default()).unwrap(); + let visible = db.upsert_node("Person", "visible", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, hidden, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }) + db.upsert_edge(start, hidden, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }) .unwrap(); db.upsert_edge( - hidden, hidden_target, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }, + hidden, hidden_target, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }, ) .unwrap(); - db.upsert_edge(start, deleted, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }) + db.upsert_edge(start, deleted, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }) .unwrap(); - db.upsert_edge(start, future, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(5_000), valid_to: Some(6_000), ..Default::default() }) + db.upsert_edge(start, future, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(5_000), valid_to: Some(6_000), ..Default::default() }) .unwrap(); - db.upsert_edge(start, visible, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }) + db.upsert_edge(start, visible, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(i64::MAX), ..Default::default() }) .unwrap(); db.delete_node(deleted).unwrap(); db.set_prune_policy( @@ -4119,7 +4272,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -4139,16 +4292,16 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let keep_a = db.upsert_node(1, "keep-a", UpsertNodeOptions::default()).unwrap(); - let hidden = db.upsert_node(1, "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); - let keep_b = db.upsert_node(1, "keep-b", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let keep_a = db.upsert_node("Person", "keep-a", UpsertNodeOptions::default()).unwrap(); + let hidden = db.upsert_node("Person", "hidden", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); + let keep_b = db.upsert_node("Person", "keep-b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, keep_a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, keep_a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, hidden, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, hidden, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, keep_b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, keep_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.set_prune_policy( @@ -4156,7 +4309,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -4199,16 +4352,16 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let hidden_start = db.upsert_node(1, "hidden-start", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); - let next = db.upsert_node(1, "next", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(hidden_start, next, 10, UpsertEdgeOptions::default()) + let hidden_start = db.upsert_node("Person", "hidden-start", UpsertNodeOptions { weight: 0.2, ..Default::default() }).unwrap(); + let next = db.upsert_node("Person", "next", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(hidden_start, next, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.set_prune_policy( "low_weight", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -4234,16 +4387,16 @@ let db_path = dir.path().join("db"); let db = open_imm(&db_path); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(start, d, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let baseline = db @@ -4277,18 +4430,18 @@ let db_path = dir.path().join("db"); let db = open_imm(&db_path); - let start = db.upsert_node(1, "start", UpsertNodeOptions::default()).unwrap(); - let middle = db.upsert_node(1, "middle", UpsertNodeOptions::default()).unwrap(); - let target = db.upsert_node(1, "target", UpsertNodeOptions::default()).unwrap(); + let start = db.upsert_node("Person", "start", UpsertNodeOptions::default()).unwrap(); + let middle = db.upsert_node("Person", "middle", UpsertNodeOptions::default()).unwrap(); + let target = db.upsert_node("Person", "target", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(start, middle, 10, UpsertEdgeOptions::default()) + db.upsert_edge(start, middle, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let older_edge = db - .upsert_edge(middle, target, 10, UpsertEdgeOptions::default()) + .upsert_edge(middle, target, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); let newer_edge = db - .upsert_edge(middle, target, 11, UpsertEdgeOptions::default()) + .upsert_edge(middle, target, "BLOCKS", UpsertEdgeOptions::default()) .unwrap(); assert!(older_edge < newer_edge); @@ -4308,8 +4461,8 @@ engine: &DatabaseEngine, start: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, - node_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[&str]>, + node_label_filter: Option<&[&str]>, at_epoch: Option, decay_lambda: Option, limit: Option, @@ -4319,8 +4472,9 @@ .traverse(start, 2, &TraverseOptions { min_depth: 2, direction, - edge_type_filter: edge_type_filter.map(|s| s.to_vec()), - node_type_filter: node_type_filter.map(|s| s.to_vec()), + edge_label_filter: edge_label_filter.map(graph_filter_names), + emit_node_label_filter: node_label_filter + .map(|labels| graph_node_label_filter(labels, LabelMatchMode::Any)), at_epoch, decay_lambda, limit, @@ -4335,15 +4489,15 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - let e = engine.upsert_node(1, "e", UpsertNodeOptions::default()).unwrap(); - engine.upsert_edge(a, b, 1, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, c, 1, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, d, 1, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, e, 1, UpsertEdgeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + let e = engine.upsert_node("Person", "e", UpsertNodeOptions::default()).unwrap(); + engine.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, d, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, e, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); let p1 = traverse_depth_two_page( &engine, @@ -4390,17 +4544,17 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); for i in 0..5 { let b = engine - .upsert_node(1, &format!("b{i}"), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{i}"), UpsertNodeOptions::default()) .unwrap(); - engine.upsert_edge(a, b, 1, UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); for j in 0..2 { let c = engine - .upsert_node(1, &format!("c{i}_{j}"), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{i}_{j}"), UpsertNodeOptions::default()) .unwrap(); - engine.upsert_edge(b, c, 1, UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); } } @@ -4445,11 +4599,11 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - engine.upsert_edge(a, b, 1, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, c, 1, UpsertEdgeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + engine.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); let page = traverse_depth_two_page( &engine, @@ -4475,22 +4629,22 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(10, "c", UpsertNodeOptions::default()).unwrap(); - let d = engine.upsert_node(10, "d", UpsertNodeOptions::default()).unwrap(); - let e = engine.upsert_node(20, "e", UpsertNodeOptions::default()).unwrap(); - engine.upsert_edge(a, b, 1, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, c, 1, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, d, 2, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, e, 1, UpsertEdgeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Document", "c", UpsertNodeOptions::default()).unwrap(); + let d = engine.upsert_node("Document", "d", UpsertNodeOptions::default()).unwrap(); + let e = engine.upsert_node("Group", "e", UpsertNodeOptions::default()).unwrap(); + engine.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, d, "WORKS_AT", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, e, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); let hits = traverse_depth_two_page( &engine, a, Direction::Outgoing, - Some(&[1]), - Some(&[10]), + Some(&["RELATES_TO"]), + Some(&["Document"]), None, None, None, @@ -4506,15 +4660,15 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - engine.upsert_edge(a, b, 1, UpsertEdgeOptions::default()).unwrap(); - engine.upsert_edge(b, c, 1, UpsertEdgeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + engine.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); engine.flush().unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - engine.upsert_edge(b, d, 1, UpsertEdgeOptions::default()).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + engine.upsert_edge(b, d, "RELATES_TO", UpsertEdgeOptions::default()).unwrap(); let page = traverse_depth_two_page( &engine, @@ -4545,13 +4699,13 @@ // Create edge and flush to segment let e1 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) .unwrap(); engine.flush().unwrap(); // Upsert same triple, should find existing in segment and reuse ID let e2 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions { weight: 0.9, ..Default::default() }) + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions { weight: 0.9, ..Default::default() }) .unwrap(); assert_eq!(e1, e2, "same triple should reuse edge ID across flush"); @@ -4561,7 +4715,7 @@ // Different triple still gets new ID let e3 = engine - .upsert_edge(1, 2, 20, UpsertEdgeOptions::default()) + .upsert_edge(1, 2, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); assert_ne!(e1, e3); @@ -4575,10 +4729,10 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let n1 = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let n2 = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let n1 = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let n2 = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); let eid = engine - .upsert_edge(n1, n2, 1, UpsertEdgeOptions { weight: 0.42, ..Default::default() }) + .upsert_edge(n1, n2, "RELATES_TO", UpsertEdgeOptions { weight: 0.42, ..Default::default() }) .unwrap(); engine.flush().unwrap(); @@ -4610,7 +4764,7 @@ // Create edge and flush (segment 1 has the edge) let e1 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) .unwrap(); engine.flush().unwrap(); @@ -4621,7 +4775,7 @@ // Now both segments are on disk, memtable is clean. // Upserting the same triple should get a NEW ID, not resurrect the deleted edge. let e2 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions { weight: 0.9, ..Default::default() }) + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions { weight: 0.9, ..Default::default() }) .unwrap(); assert_ne!( e1, e2, @@ -4641,23 +4795,23 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let e = engine.upsert_node(1, "e", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let e = engine.upsert_node("Person", "e", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 0.1, ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 0.1, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions { weight: 0.9, ..Default::default() }) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions { weight: 0.9, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, d, 1, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) + .upsert_edge(a, d, "RELATES_TO", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, e, 1, UpsertEdgeOptions { weight: 0.3, ..Default::default() }) + .upsert_edge(a, e, "RELATES_TO", UpsertEdgeOptions { weight: 0.3, ..Default::default() }) .unwrap(); // Top 2 by weight: c (0.9) and d (0.5) @@ -4679,20 +4833,20 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); // Edges with explicit valid_from to control recency engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 1.0, valid_from: Some(1000), valid_to: None, ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 1.0, valid_from: Some(1000), valid_to: None, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions { weight: 1.0, valid_from: Some(3000), valid_to: None, ..Default::default() }) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions { weight: 1.0, valid_from: Some(3000), valid_to: None, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, d, 1, UpsertEdgeOptions { weight: 1.0, valid_from: Some(2000), valid_to: None, ..Default::default() }) + .upsert_edge(a, d, "RELATES_TO", UpsertEdgeOptions { weight: 1.0, valid_from: Some(2000), valid_to: None, ..Default::default() }) .unwrap(); // Top 2 by recency: c (3000) and d (2000) @@ -4712,10 +4866,10 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); let now = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) @@ -4724,13 +4878,13 @@ // b: high weight but old, c: medium weight and recent, d: low weight very recent engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 1.0, valid_from: Some(now - 7_200_000), valid_to: None, ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 1.0, valid_from: Some(now - 7_200_000), valid_to: None, ..Default::default() }) .unwrap(); // 2 hours ago engine - .upsert_edge(a, c, 1, UpsertEdgeOptions { weight: 0.8, valid_from: Some(now - 600_000), valid_to: None, ..Default::default() }) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions { weight: 0.8, valid_from: Some(now - 600_000), valid_to: None, ..Default::default() }) .unwrap(); // 10 min ago engine - .upsert_edge(a, d, 1, UpsertEdgeOptions { weight: 0.3, valid_from: Some(now - 60_000), valid_to: None, ..Default::default() }) + .upsert_edge(a, d, "RELATES_TO", UpsertEdgeOptions { weight: 0.3, valid_from: Some(now - 60_000), valid_to: None, ..Default::default() }) .unwrap(); // 1 min ago // With strong decay (lambda=1.0), recent edges should beat old heavy ones @@ -4752,10 +4906,10 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = engine @@ -4772,14 +4926,14 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 0.3, ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 0.3, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions { weight: 0.7, ..Default::default() }) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions { weight: 0.7, ..Default::default() }) .unwrap(); // k=10 but only 2 neighbors. Returns all 2, sorted desc @@ -4794,29 +4948,29 @@ } #[test] - fn test_top_k_with_type_filter() { + fn test_top_k_with_label_filter() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 0.9, ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 0.9, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, c, 2, UpsertEdgeOptions { weight: 0.8, ..Default::default() }) + .upsert_edge(a, c, "WORKS_AT", UpsertEdgeOptions { weight: 0.8, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, d, 1, UpsertEdgeOptions { weight: 0.7, ..Default::default() }) + .upsert_edge(a, d, "RELATES_TO", UpsertEdgeOptions { weight: 0.7, ..Default::default() }) .unwrap(); - // Top 1 of edge type 1 only + // Top 1 for the RELATES_TO edge label only. let result = engine - .top_k_neighbors(a, 1, &TopKOptions { type_filter: Some(vec![1]), ..Default::default() }) + .top_k_neighbors(a, 1, &TopKOptions { edge_label_filter: Some(vec!["RELATES_TO".to_string()]), ..Default::default() }) .unwrap(); assert_eq!(result.len(), 1); assert_eq!(result[0].node_id, b); @@ -4830,15 +4984,15 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 0.9, ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 0.9, ..Default::default() }) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions { weight: 0.8, ..Default::default() }) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions { weight: 0.8, ..Default::default() }) .unwrap(); engine.delete_node(b).unwrap(); @@ -4858,16 +5012,16 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) .unwrap(); engine.flush().unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions { weight: 0.9, ..Default::default() }) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions { weight: 0.9, ..Default::default() }) .unwrap(); // Top 1 should be c (0.9) from memtable, beating b (0.5) from segment @@ -4886,7 +5040,7 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); let result = engine.top_k_neighbors(a, 5, &TopKOptions { scoring: ScoringMode::DecayAdjusted { lambda: -1.0 }, ..Default::default() }); assert!(result.is_err()); @@ -4905,18 +5059,18 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); let e_ab = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -4938,18 +5092,18 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -4970,21 +5124,21 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5008,17 +5162,17 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e_ca = engine - .upsert_edge(c, a, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5042,13 +5196,13 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let e_aa = engine - .upsert_edge(a, a, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e_ab = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5071,10 +5225,10 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5112,11 +5266,11 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let _c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let _c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5136,14 +5290,14 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5164,14 +5318,14 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5186,36 +5340,163 @@ } #[test] - fn test_subgraph_edge_type_filter() { - // A→B (type 1), A→C (type 2), B→D (type 1). - // Filter by type 1 → A, B, D (not C). + fn test_subgraph_edge_label_filter() { + // A→B (label 1), A→C (label 2), B→D (label 1). + // Filter by label 1 → A, B, D (not C). let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 2, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine - .extract_subgraph(a, 2, &SubgraphOptions { edge_type_filter: Some(vec![1]), ..Default::default() }) + .extract_subgraph(a, 2, &SubgraphOptions { edge_label_filter: Some(vec!["RELATES_TO".to_string()]), ..Default::default() }) .unwrap(); let node_ids: NodeIdSet = sg.nodes.iter().map(|n| n.id).collect(); assert_eq!(node_ids, NodeIdSet::from_iter([a, b, d])); assert_eq!(sg.edges.len(), 2); - // All edges should be type 1 - assert!(sg.edges.iter().all(|e| e.type_id == 1)); + // All edges should be label 1 + assert!(sg.edges.iter().all(|e| e.label == "RELATES_TO".to_string())); + + engine.close().unwrap(); + } + + #[test] + fn test_subgraph_node_label_filter_uses_single_any_all_eligibility() { + let dir = TempDir::new().unwrap(); + let engine = DatabaseEngine::open(&dir.path().join("testdb"), &DbOptions::default()).unwrap(); + + let root = engine + .upsert_node( + &["Person", "Featured"], + "root", + UpsertNodeOptions::default(), + ) + .unwrap(); + let bridge = engine + .upsert_node("Person", "bridge", UpsertNodeOptions::default()) + .unwrap(); + let target = engine + .upsert_node( + &["Person", "Featured"], + "target", + UpsertNodeOptions::default(), + ) + .unwrap(); + let blocked = engine + .upsert_node( + &["Company", "Featured"], + "blocked", + UpsertNodeOptions::default(), + ) + .unwrap(); + + engine + .upsert_edge(root, bridge, "RELATES_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(bridge, target, "RELATES_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(root, target, "RELATES_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(root, blocked, "RELATES_TO", UpsertEdgeOptions::default()) + .unwrap(); + + let person = engine + .extract_subgraph( + root, + 2, + &SubgraphOptions { + node_label_filter: Some(graph_node_label_filter( + &["Person"], + LabelMatchMode::Any, + )), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + person.nodes.iter().map(|node| node.id).collect::(), + NodeIdSet::from_iter([root, bridge, target]) + ); + assert_eq!(person.edges.len(), 3); + + let any_featured_or_company = engine + .extract_subgraph( + root, + 1, + &SubgraphOptions { + node_label_filter: Some(graph_node_label_filter( + &["Featured", "Company"], + LabelMatchMode::Any, + )), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + any_featured_or_company + .nodes + .iter() + .map(|node| node.id) + .collect::(), + NodeIdSet::from_iter([root, target, blocked]) + ); + assert_eq!(any_featured_or_company.edges.len(), 2); + + let all_person_featured = engine + .extract_subgraph( + root, + 2, + &SubgraphOptions { + node_label_filter: Some(graph_node_label_filter( + &["Person", "Featured"], + LabelMatchMode::All, + )), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + all_person_featured + .nodes + .iter() + .map(|node| node.id) + .collect::(), + NodeIdSet::from_iter([root, target]) + ); + assert_eq!(all_person_featured.edges.len(), 1); + + let start_filtered = engine + .extract_subgraph( + bridge, + 2, + &SubgraphOptions { + node_label_filter: Some(graph_node_label_filter( + &["Person", "Featured"], + LabelMatchMode::All, + )), + ..Default::default() + }, + ) + .unwrap(); + assert!(start_filtered.nodes.is_empty()); + assert!(start_filtered.edges.is_empty()); engine.close().unwrap(); } @@ -5231,16 +5512,16 @@ }; let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5261,14 +5542,14 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.delete_node(b).unwrap(); @@ -5291,14 +5572,14 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.delete_edge(e1).unwrap(); @@ -5321,14 +5602,14 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(200), ..Default::default() }) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions { weight: 1.0, valid_from: Some(100), valid_to: Some(200), ..Default::default() }) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions { weight: 1.0, valid_from: Some(300), valid_to: Some(400), ..Default::default() }) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions { weight: 1.0, valid_from: Some(300), valid_to: Some(400), ..Default::default() }) .unwrap(); let sg_150 = engine @@ -5353,14 +5634,14 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "hub", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "hub", UpsertNodeOptions::default()).unwrap(); let mut expected = NodeIdSet::from_iter([a]); for i in 0..50 { let n = engine - .upsert_node(1, &format!("spoke_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("spoke_{}", i), UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(a, n, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, n, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); expected.insert(n); } @@ -5383,22 +5664,22 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = engine.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - let e = engine.upsert_node(1, "e", UpsertNodeOptions::default()).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = engine.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + let e = engine.upsert_node("Person", "e", UpsertNodeOptions::default()).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(d, e, 1, UpsertEdgeOptions::default()) + .upsert_edge(d, e, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5421,14 +5702,14 @@ let mut props_a = BTreeMap::new(); props_a.insert("name".to_string(), PropValue::String("Alice".to_string())); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { props: props_a, weight: 0.9, ..Default::default() }).unwrap(); + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { props: props_a, weight: 0.9, ..Default::default() }).unwrap(); let mut props_b = BTreeMap::new(); props_b.insert("name".to_string(), PropValue::String("Bob".to_string())); - let b = engine.upsert_node(2, "b", UpsertNodeOptions { props: props_b, weight: 0.7, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Company", "b", UpsertNodeOptions { props: props_b, weight: 0.7, ..Default::default() }).unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let sg = engine @@ -5436,7 +5717,7 @@ .unwrap(); let node_a = sg.nodes.iter().find(|n| n.id == a).unwrap(); - assert_eq!(node_a.type_id, 1); + assert_eq!(node_a.labels.as_slice(), ["Person"]); assert_eq!(node_a.key, "a"); assert_eq!( node_a.props.get("name"), @@ -5444,7 +5725,7 @@ ); let node_b = sg.nodes.iter().find(|n| n.id == b).unwrap(); - assert_eq!(node_b.type_id, 2); + assert_eq!(node_b.labels.as_slice(), ["Company"]); assert_eq!( node_b.props.get("name"), Some(&PropValue::String("Bob".to_string())) @@ -5459,14 +5740,14 @@ fn test_neighbors_batch_basic() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let result = db @@ -5485,21 +5766,21 @@ let mut ids = Vec::new(); for i in 0..5 { ids.push( - db.upsert_node(1, &format!("n{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) + db.upsert_node("Person", &format!("n{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) .unwrap(), ); } - db.upsert_edge(ids[0], ids[1], 10, UpsertEdgeOptions::default()) + db.upsert_edge(ids[0], ids[1], "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(ids[0], ids[2], 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(ids[0], ids[2], "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); - db.upsert_edge(ids[1], ids[2], 20, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + db.upsert_edge(ids[1], ids[2], "REPORTS_TO", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); - db.upsert_edge(ids[2], ids[3], 10, UpsertEdgeOptions { weight: 1.5, ..Default::default() }) + db.upsert_edge(ids[2], ids[3], "KNOWS", UpsertEdgeOptions { weight: 1.5, ..Default::default() }) .unwrap(); - db.upsert_edge(ids[3], ids[4], 20, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) + db.upsert_edge(ids[3], ids[4], "REPORTS_TO", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) .unwrap(); - db.upsert_edge(ids[4], ids[0], 10, UpsertEdgeOptions::default()) + db.upsert_edge(ids[4], ids[0], "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let batch = db @@ -5521,22 +5802,22 @@ } #[test] - fn test_neighbors_batch_with_type_filter() { + fn test_neighbors_batch_with_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 20, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); let result = db - .neighbors_batch(&[a], &NeighborOptions { direction: Direction::Outgoing, type_filter: Some(vec![10]), ..Default::default() }) + .neighbors_batch(&[a], &NeighborOptions { direction: Direction::Outgoing, edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }) .unwrap(); assert_eq!(result.get(&a).unwrap().len(), 1); - assert_eq!(result[&a][0].edge_type_id, 10); + assert_eq!(result[&a][0].label, "KNOWS".to_string()); db.close().unwrap(); } @@ -5544,19 +5825,19 @@ fn test_neighbors_batch_cross_segment() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let c = db.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(b, d, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }) + let d = db.upsert_node("Person", "d", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }) .unwrap(); let result = db @@ -5571,14 +5852,14 @@ fn test_neighbors_batch_dedup_across_sources() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); // Re-upsert same edge triple. Should dedup (triple lookup across segments) - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); // Batch and individual MUST return the same results @@ -5603,13 +5884,13 @@ fn test_neighbors_batch_respects_tombstones() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -5628,12 +5909,12 @@ fn test_neighbors_batch_direction_both() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(c, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let result = db @@ -5648,12 +5929,12 @@ fn test_neighbors_batch_self_loop_dedup() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); // Self-loop: A→A appears in both adj_out and adj_in - db.upsert_edge(a, a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let result = db @@ -5688,12 +5969,12 @@ fn test_neighbors_batch_unsorted_input() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, a, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + db.upsert_edge(c, a, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); // Input in reverse order. Batch method sorts internally @@ -5714,7 +5995,7 @@ let mut node_ids = Vec::new(); for i in 0..50 { node_ids.push( - db.upsert_node(1, &format!("n{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) + db.upsert_node("Person", &format!("n{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) .unwrap(), ); } @@ -5722,11 +6003,11 @@ let t1 = (i + 1) % 50; let t2 = (i + 25) % 50; db.upsert_edge( - node_ids[i], node_ids[t1], 10, UpsertEdgeOptions::default(), + node_ids[i], node_ids[t1], "KNOWS", UpsertEdgeOptions::default(), ) .unwrap(); db.upsert_edge( - node_ids[i], node_ids[t2], 20, UpsertEdgeOptions { weight: 0.5, ..Default::default() }, + node_ids[i], node_ids[t2], "REPORTS_TO", UpsertEdgeOptions { weight: 0.5, ..Default::default() }, ) .unwrap(); } @@ -5737,7 +6018,7 @@ for i in 0..10 { let t = (i + 5) % 50; db.upsert_edge( - node_ids[i], node_ids[t], 30, UpsertEdgeOptions { weight: 0.3, ..Default::default() }, + node_ids[i], node_ids[t], "RATES", UpsertEdgeOptions { weight: 0.3, ..Default::default() }, ) .unwrap(); } @@ -5771,8 +6052,8 @@ fn test_self_loop_neighbors_outgoing() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, a, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let out = db @@ -5803,14 +6084,14 @@ fn test_self_loop_survives_flush_and_compact() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); let e = db - .upsert_edge(a, a, 10, UpsertEdgeOptions { weight: 2.5, ..Default::default() }) + .upsert_edge(a, a, "KNOWS", UpsertEdgeOptions { weight: 2.5, ..Default::default() }) .unwrap(); db.flush().unwrap(); // Add a second segment to enable compaction - let _b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let _b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -5838,8 +6119,8 @@ fn test_self_loop_in_top_k() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, a, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }) .unwrap(); let top = db @@ -5858,15 +6139,15 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); // a→b (w=2.0), a→c (w=3.0), b→c (w=1.5) - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 1.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 1.5, ..Default::default() }).unwrap(); let ea = db.degree_cache_entry(a); assert_eq!(ea.out_degree, 2); @@ -5900,10 +6181,10 @@ let path = dir.path().join("db"); { let db = open_imm(&path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }).unwrap(); - db.upsert_edge(b, a, 10, UpsertEdgeOptions { weight: 2.5, ..Default::default() }).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }).unwrap(); + db.upsert_edge(b, a, "KNOWS", UpsertEdgeOptions { weight: 2.5, ..Default::default() }).unwrap(); db.flush().unwrap(); db.close().unwrap(); } @@ -5931,14 +6212,14 @@ let path = dir.path().join("db"); { let db = open_imm(&path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // a→b in segment - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); // a→c in WAL (unflushed) - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); db.close().unwrap(); } @@ -5963,11 +6244,11 @@ let path = dir.path().join("db"); { let db = open_imm(&path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let e1 = db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let e1 = db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); db.flush().unwrap(); // Delete one edge. Tombstone in WAL. db.delete_edge(e1).unwrap(); @@ -5995,12 +6276,12 @@ let path = dir.path().join("db"); { let db = open_imm(&path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Self-loop on a - db.upsert_edge(a, a, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); // Normal edge a→b - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.close().unwrap(); } @@ -6036,25 +6317,25 @@ let mut nodes = Vec::new(); for i in 0..10 { - nodes.push(db.upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()).unwrap()); + nodes.push(db.upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()).unwrap()); } // Chain: 0→1→2→...→9 for i in 0..9 { - db.upsert_edge(nodes[i], nodes[i + 1], 10, UpsertEdgeOptions { weight: (i as f32) + 0.5, ..Default::default() }).unwrap(); + db.upsert_edge(nodes[i], nodes[i + 1], "KNOWS", UpsertEdgeOptions { weight: (i as f32) + 0.5, ..Default::default() }).unwrap(); } // Hub: node 0 → all others (2..10) for i in 2..10 { - db.upsert_edge(nodes[0], nodes[i], 20, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(nodes[0], nodes[i], "REPORTS_TO", UpsertEdgeOptions::default()).unwrap(); } // Self-loop on node 5 - db.upsert_edge(nodes[5], nodes[5], 10, UpsertEdgeOptions { weight: 7.0, ..Default::default() }).unwrap(); + db.upsert_edge(nodes[5], nodes[5], "KNOWS", UpsertEdgeOptions { weight: 7.0, ..Default::default() }).unwrap(); // Flush half to segments db.flush().unwrap(); // Add edge in WAL (unflushed) - db.upsert_edge(nodes[9], nodes[0], 10, UpsertEdgeOptions { weight: 0.1, ..Default::default() }).unwrap(); + db.upsert_edge(nodes[9], nodes[0], "KNOWS", UpsertEdgeOptions { weight: 0.1, ..Default::default() }).unwrap(); db.close().unwrap(); } @@ -6097,15 +6378,15 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Before any edges: cache should be empty for these nodes assert_eq!(db.degree_cache_entry(a).out_degree, 0); assert_eq!(db.degree_cache_entry(b).in_degree, 0); // Insert edge a→b - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.5, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.5, ..Default::default() }).unwrap(); let ea = db.degree_cache_entry(a); assert_eq!(ea.out_degree, 1); @@ -6127,16 +6408,16 @@ let opts = DbOptions { edge_uniqueness: true, ..Default::default() }; let db = DatabaseEngine::open(&dir.path().join("db"), &opts).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Insert edge a→b with weight 2.0 - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); assert_eq!(db.degree_cache_entry(a).out_degree, 1); assert!((db.degree_cache_entry(a).out_weight_sum - 2.0).abs() < 1e-10); - // Update same edge (same from, to, type_id) with weight 5.0 - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 5.0, ..Default::default() }).unwrap(); + // Update same edge (same from, to, label_id) with weight 5.0 + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() }).unwrap(); // Degree should NOT change, only weight let ea = db.degree_cache_entry(a); @@ -6155,10 +6436,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); - let e1 = db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); + let e1 = db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); assert_eq!(db.degree_cache_entry(a).out_degree, 1); // Delete the edge (still in memtable, not flushed) @@ -6179,12 +6460,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); - let e1 = db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); + let e1 = db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); // Flush to segments. Edges are now segment-only. db.flush().unwrap(); @@ -6209,14 +6490,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // a→b, b→c, c→a (triangle) - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(c, a, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(c, a, "KNOWS", UpsertEdgeOptions::default()).unwrap(); assert_eq!(db.degree_cache_entry(a).out_degree, 1); assert_eq!(db.degree_cache_entry(a).in_degree, 1); @@ -6249,11 +6530,11 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Insert self-loop on a - let sl = db.upsert_edge(a, a, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }).unwrap(); + let sl = db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }).unwrap(); let ea = db.degree_cache_entry(a); assert_eq!(ea.out_degree, 1); @@ -6264,7 +6545,7 @@ assert_eq!((ea.out_degree + ea.in_degree - ea.self_loop_count) as u64, 1); // Add normal edge a→b - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let ea2 = db.degree_cache_entry(a); assert_eq!(ea2.out_degree, 2); @@ -6293,10 +6574,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); - let e1 = db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let e1 = db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.delete_edge(e1).unwrap(); let ea_after_first = db.degree_cache_entry(a); @@ -6323,14 +6604,14 @@ { let db = open_imm(&path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Insert edges - let e1 = db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let e1 = db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() }).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 2); assert_eq!(db.degree(a, &DegreeOptions { direction: Direction::Both, ..Default::default() }).unwrap(), 2); @@ -6350,7 +6631,7 @@ assert_eq!(db.degree(b, &DegreeOptions { direction: Direction::Incoming, ..Default::default() }).unwrap(), 0); // Add more edges and flush again for compaction - db.upsert_edge(c, a, 10, UpsertEdgeOptions { weight: 4.0, ..Default::default() }).unwrap(); + db.upsert_edge(c, a, "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() }).unwrap(); db.flush().unwrap(); // Compact @@ -6378,28 +6659,28 @@ #[test] fn test_degree_cache_filtered_queries_bypass_cache() { - // Verify that type-filtered and temporal queries still use the walk path + // Verify that label-filtered and temporal queries still use the walk path // (they should return different results from the cache-backed unfiltered path - // when type filtering narrows the result set). + // when label filtering narrows the result set). let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); - // a→b with type 10, a→c with type 20 - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(a, c, 20, UpsertEdgeOptions::default()).unwrap(); + // a→b with label 10, a→c with label 20 + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()).unwrap(); // Unfiltered: both edges assert_eq!(db.degree(a, &DegreeOptions::default()).unwrap(), 2); - // Type-filtered: only type 10 (uses walk path, not cache) - assert_eq!(db.degree(a, &DegreeOptions { direction: Direction::Outgoing, type_filter: Some(vec![10]), ..Default::default() }).unwrap(), 1); + // Label-filtered: only label 10 (uses walk path, not cache) + assert_eq!(db.degree(a, &DegreeOptions { direction: Direction::Outgoing, edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }).unwrap(), 1); - // Type-filtered: only type 20 - assert_eq!(db.degree(a, &DegreeOptions { direction: Direction::Outgoing, type_filter: Some(vec![20]), ..Default::default() }).unwrap(), 1); + // Label-filtered: only label 20 + assert_eq!(db.degree(a, &DegreeOptions { direction: Direction::Outgoing, edge_label_filter: Some(vec!["REPORTS_TO".to_string()]), ..Default::default() }).unwrap(), 1); // Temporal: at_epoch=Some(now) should still work via walk let now = std::time::SystemTime::now() @@ -6421,19 +6702,19 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Timeless edge. Cache should be used. - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let entry_a = db.degree_cache_entry(a); assert_eq!(entry_a.temporal_edge_count, 0); assert_eq!(entry_a.out_degree, 1); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Temporal edge (valid_to = now + 10 seconds) let future_expiry = now_millis() + 10_000; - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) .unwrap(); // Node a now has temporal_edge_count > 0 @@ -6459,12 +6740,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let future_expiry = now_millis() + 10_000; let e = db - .upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) .unwrap(); assert_eq!(db.degree_cache_entry(a).temporal_edge_count, 1); assert_eq!(db.degree_cache_entry(b).temporal_edge_count, 1); @@ -6478,21 +6759,21 @@ fn test_degree_cache_temporal_to_timeless_update() { // Updating an edge from temporal to timeless should decrement // temporal_edge_count. Requires edge_uniqueness so the second - // upsert replaces the first (same from/to/type_id). + // upsert replaces the first (same from/to/label_id). let dir = TempDir::new().unwrap(); let opts = DbOptions { edge_uniqueness: true, ..Default::default() }; let db = DatabaseEngine::open(&dir.path().join("db"), &opts).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let future_expiry = now_millis() + 10_000; - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) .unwrap(); assert_eq!(db.degree_cache_entry(a).temporal_edge_count, 1); // Re-upsert same edge with valid_to = None (timeless) - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); assert_eq!(db.degree_cache_entry(a).temporal_edge_count, 0); assert_eq!(db.degree_cache_entry(b).temporal_edge_count, 0); } @@ -6503,10 +6784,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); - let e = db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let e = db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); assert_eq!(db.degree_cache_entry(a).temporal_edge_count, 0); // Invalidate (sets valid_to to past → temporal + not valid) @@ -6524,11 +6805,11 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); let future_expiry = now_millis() + 10_000; let e = db - .upsert_edge(a, a, 10, UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) + .upsert_edge(a, a, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(future_expiry), ..Default::default() }) .unwrap(); assert_eq!(db.degree_cache_entry(a).temporal_edge_count, 1); // once, not twice @@ -6544,15 +6825,15 @@ let path = dir.path().join("db"); { let db = open_imm(&path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Timeless edge - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // Temporal edge let future = now_millis() + 100_000; - db.upsert_edge(a, c, 10, UpsertEdgeOptions { weight: 2.0, valid_from: None, valid_to: Some(future), ..Default::default() }) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions { weight: 2.0, valid_from: None, valid_to: Some(future), ..Default::default() }) .unwrap(); db.flush().unwrap(); db.close().unwrap(); @@ -6560,11 +6841,11 @@ { let db = open_imm(&path); // After reopen + rebuild, temporal count should reflect the temporal edge - let entry_a = db.degree_cache_entry(db.get_node_by_key(1, "a").unwrap().unwrap().id); + let entry_a = db.degree_cache_entry(db.get_node_by_key("Person", "a").unwrap().unwrap().id); assert_eq!(entry_a.temporal_edge_count, 1); - let entry_b = db.degree_cache_entry(db.get_node_by_key(1, "b").unwrap().unwrap().id); + let entry_b = db.degree_cache_entry(db.get_node_by_key("Person", "b").unwrap().unwrap().id); assert_eq!(entry_b.temporal_edge_count, 0); - let entry_c = db.degree_cache_entry(db.get_node_by_key(1, "c").unwrap().unwrap().id); + let entry_c = db.degree_cache_entry(db.get_node_by_key("Person", "c").unwrap().unwrap().id); assert_eq!(entry_c.temporal_edge_count, 1); } } @@ -6576,11 +6857,11 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Already expired (valid_to = 1ms after epoch) - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(1), ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: None, valid_to: Some(1), ..Default::default() }).unwrap(); let entry = db.degree_cache_entry(a); assert_eq!(entry.out_degree, 0); // not counted (expired) @@ -6599,12 +6880,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Future-dated edge: valid_from = far future, valid_to = infinity let far_future = i64::MAX - 1_000_000; - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(far_future), valid_to: None, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(far_future), valid_to: None, ..Default::default() }).unwrap(); let entry_a = db.degree_cache_entry(a); let entry_b = db.degree_cache_entry(b); @@ -6635,10 +6916,10 @@ let b; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let far_future = i64::MAX - 1_000_000; - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(far_future), valid_to: None, ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(far_future), valid_to: None, ..Default::default() }).unwrap(); db.flush().unwrap(); db.close().unwrap(); } @@ -6658,15 +6939,15 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let before = db.active_degree_overlay_for_test(); let mut txn = db.begin_write_txn().unwrap(); txn.upsert_edge( TxnNodeRef::Id(a), TxnNodeRef::Id(b), - 10, + &"KNOWS".to_string(), UpsertEdgeOptions { weight: 2.5, ..Default::default() @@ -6699,8 +6980,8 @@ }; let db = DatabaseEngine::open(&dir.path().join("db"), &opts).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let before_rollback = db.active_degree_overlay_for_test(); let mut rollback_txn = db.begin_write_txn().unwrap(); @@ -6708,7 +6989,7 @@ .upsert_edge( TxnNodeRef::Id(a), TxnNodeRef::Id(b), - 10, + &"KNOWS".to_string(), UpsertEdgeOptions::default(), ) .unwrap(); @@ -6724,7 +7005,7 @@ .upsert_edge( TxnNodeRef::Id(a), TxnNodeRef::Id(b), - 10, + &"KNOWS".to_string(), UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -6734,7 +7015,7 @@ db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() @@ -6763,17 +7044,17 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let overlay_before_node_only = db.active_degree_overlay_for_test(); - db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); assert!(std::sync::Arc::ptr_eq( &overlay_before_node_only, &db.active_degree_overlay_for_test() )); let stale = db.published_read_view_for_test(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(stale.degree_entry_for_test(a).out_degree, 0); assert_eq!(db.degree_cache_entry(a).out_degree, 1); @@ -6786,14 +7067,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); let now = now_millis(); let first = EdgeRecord { id: 42, from: a, to: b, - type_id: 10, + label_id: 10, props: std::collections::BTreeMap::new(), created_at: now, updated_at: now, @@ -6806,7 +7087,7 @@ second.updated_at = now + 1; second.weight = 5.0; - db.write_op_batch(&[WalOp::UpsertEdge(first), WalOp::UpsertEdge(second)]) + write_internal_wal_op_batch(&db, &[WalOp::UpsertEdge(first), WalOp::UpsertEdge(second)]) .unwrap(); let entry_a = db.degree_cache_entry(a); @@ -6824,9 +7105,9 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -6840,11 +7121,11 @@ assert_eq!(entry_a.out_degree, 1); assert!((entry_a.out_weight_sum - 2.0).abs() < 1e-10); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() @@ -6863,12 +7144,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -6888,12 +7169,12 @@ let db = open_imm(&dir.path().join("db")); let hub = db - .upsert_node(1, "hub", UpsertNodeOptions::default()) + .upsert_node("Person", "hub", UpsertNodeOptions::default()) .unwrap(); let mut leaves = Vec::new(); for idx in 0..320 { leaves.push( - db.upsert_node(1, &format!("leaf-{idx}"), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("leaf-{idx}"), UpsertNodeOptions::default()) .unwrap(), ); } @@ -6904,7 +7185,7 @@ db.upsert_edge( hub, leaf, - 10, + "KNOWS", UpsertEdgeOptions { weight: 1.5, ..Default::default() @@ -6943,13 +7224,13 @@ }; let db = DatabaseEngine::open(&dir.path().join("db"), &opts).unwrap(); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -6960,7 +7241,7 @@ .upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() @@ -6972,7 +7253,7 @@ db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 5.0, ..Default::default() @@ -7009,12 +7290,12 @@ let c; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); db.close().unwrap(); } @@ -7042,12 +7323,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); db.set_prune_policy( @@ -7055,7 +7336,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.0), - type_id: None, + label: None, }, ) .unwrap(); @@ -7204,7 +7485,7 @@ id, from, to, - type_id: 10, + label_id: 10, props: std::collections::BTreeMap::new(), created_at: 1_000, updated_at: 1_500, @@ -7221,7 +7502,7 @@ id, from, to, - type_id: 10, + label_id: 10, props: std::collections::BTreeMap::new(), created_at: 1_000, updated_at: now, @@ -7237,13 +7518,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -7253,7 +7534,7 @@ db.upsert_edge( a, c, - 20, + "REPORTS_TO", UpsertEdgeOptions { weight: 3.0, ..Default::default() @@ -7275,7 +7556,7 @@ &db, a, DegreeOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, 1, @@ -7303,7 +7584,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.0), - type_id: None, + label: None, }, ) .unwrap(); @@ -7326,12 +7607,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -7384,12 +7665,12 @@ let a; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -7417,16 +7698,16 @@ let temporal_dir = TempDir::new().unwrap(); let temporal = open_imm(&temporal_dir.path().join("db")); let x = temporal - .upsert_node(1, "x", UpsertNodeOptions::default()) + .upsert_node("Person", "x", UpsertNodeOptions::default()) .unwrap(); let y = temporal - .upsert_node(1, "y", UpsertNodeOptions::default()) + .upsert_node("Person", "y", UpsertNodeOptions::default()) .unwrap(); temporal .upsert_edge( x, y, - 10, + "KNOWS", UpsertEdgeOptions { weight: 4.0, valid_to: Some(now_millis() + 10_000), @@ -7454,9 +7735,9 @@ let a; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertEdge(historical_temporal_edge(42, a, b, 2.0))) + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + write_internal_wal_op(&db, &WalOp::UpsertEdge(historical_temporal_edge(42, a, b, 2.0))) .unwrap(); assert_scalar_degree_family_routes( @@ -7471,7 +7752,7 @@ ); db.flush().unwrap(); - db.write_op(&WalOp::DeleteEdge { + write_internal_wal_op(&db, &WalOp::DeleteEdge { id: 42, deleted_at: now_millis(), }) @@ -7513,13 +7794,13 @@ let a; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertEdge(historical_temporal_edge(42, a, b, 2.0))) + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + write_internal_wal_op(&db, &WalOp::UpsertEdge(historical_temporal_edge(42, a, b, 2.0))) .unwrap(); db.flush().unwrap(); - db.write_op(&WalOp::UpsertEdge(current_timeless_edge(42, a, b, 5.0))) + write_internal_wal_op(&db, &WalOp::UpsertEdge(current_timeless_edge(42, a, b, 5.0))) .unwrap(); assert_scalar_degree_family_routes( @@ -7556,12 +7837,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 20, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); db.reset_degree_query_routes(); @@ -7580,7 +7861,7 @@ .degrees( &[a, b, c], &DegreeOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) @@ -7593,7 +7874,7 @@ db.upsert_edge( b, c, - 10, + "KNOWS", UpsertEdgeOptions { valid_to: Some(now_millis() + 10_000), ..Default::default() @@ -7622,13 +7903,13 @@ let d; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -7638,7 +7919,7 @@ db.upsert_edge( c, a, - 10, + "KNOWS", UpsertEdgeOptions { weight: 6.0, ..Default::default() @@ -7648,7 +7929,7 @@ db.upsert_edge( a, a, - 10, + "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() @@ -7667,11 +7948,11 @@ assert_eq!(db.segments_for_test().len(), 1); assert_degree_family_all_directions_fast_match_walk(&db, a, &initial_batch); - d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, d, - 10, + "KNOWS", UpsertEdgeOptions { weight: 8.0, ..Default::default() @@ -7705,15 +7986,15 @@ let a; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -7735,7 +8016,7 @@ db.upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() @@ -7757,7 +8038,7 @@ db.upsert_edge( a, d, - 10, + "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() @@ -7817,14 +8098,14 @@ let edge_ab; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); edge_ab = db .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -7834,7 +8115,7 @@ db.upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() @@ -7902,12 +8183,12 @@ let b; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 4.0, ..Default::default() @@ -7976,13 +8257,13 @@ let a; { let db = open_imm(&path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); db.close().unwrap(); @@ -8026,15 +8307,15 @@ let mut nodes = Vec::new(); for i in 0..5 { - nodes.push(db.upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()).unwrap()); + nodes.push(db.upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()).unwrap()); } // Star topology: 0 → 1,2,3,4 for i in 1..5 { - db.upsert_edge(nodes[0], nodes[i], 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(nodes[0], nodes[i], "KNOWS", UpsertEdgeOptions::default()).unwrap(); } // Self-loop on node 2 - db.upsert_edge(nodes[2], nodes[2], 10, UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); + db.upsert_edge(nodes[2], nodes[2], "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() }).unwrap(); let batch = db.degrees(&nodes, &DegreeOptions { direction: Direction::Both, ..Default::default() }).unwrap(); @@ -8056,11 +8337,11 @@ let db = open_imm(&dir.path().join("db")); // A - B - C (all connected via directed edges, WCC ignores direction) - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); assert_eq!(comps.len(), 3); @@ -8079,16 +8360,16 @@ let db = open_imm(&dir.path().join("db")); // Component 1: A - B - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // Component 2: C - D - E - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - let e = db.upsert_node(1, "e", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(d, e, 10, UpsertEdgeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + let e = db.upsert_node("Person", "e", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(d, e, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); assert_eq!(comps.len(), 5); @@ -8110,9 +8391,9 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // No edges. Each node is its own component. let comps = db.connected_components(&ComponentOptions::default()).unwrap(); @@ -8132,10 +8413,10 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); // Self-loop on A, no connection to B. - db.upsert_edge(a, a, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); assert_eq!(comps.len(), 2); @@ -8150,13 +8431,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Multiple edges between A and B shouldn't create multiple links. - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(a, b, 20, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, a, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "REPORTS_TO", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, a, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); assert_eq!(comps.len(), 3); @@ -8173,11 +8454,11 @@ let db = open_imm(&dir.path().join("db")); // A - B - C - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let e1 = db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let e1 = db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // Delete edge A-B: now A is isolated, B-C connected. db.delete_edge(e1).unwrap(); @@ -8200,73 +8481,157 @@ } #[test] - fn test_wcc_edge_type_filter() { + fn test_wcc_edge_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - // Edge type 10: A - B - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - // Edge type 20: B - C - db.upsert_edge(b, c, 20, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + // Edge label 10: A - B + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + // Edge label 20: B - C + db.upsert_edge(b, c, "REPORTS_TO", UpsertEdgeOptions::default()).unwrap(); - // Filter by type 10 only: only A-B connected. - let comps = db.connected_components(&ComponentOptions { edge_type_filter: Some(vec![10]), ..Default::default() }).unwrap(); + // Filter by label 10 only: only A-B connected. + let comps = db.connected_components(&ComponentOptions { edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }).unwrap(); assert_eq!(comps[&a], a); assert_eq!(comps[&b], a); - assert_eq!(comps[&c], c); // isolated when type 20 excluded + assert_eq!(comps[&c], c); // isolated when label 20 excluded - // Filter by type 20 only: only B-C connected. - let comps2 = db.connected_components(&ComponentOptions { edge_type_filter: Some(vec![20]), ..Default::default() }).unwrap(); - assert_eq!(comps2[&a], a); // isolated when type 10 excluded + // Filter by label 20 only: only B-C connected. + let comps2 = db.connected_components(&ComponentOptions { edge_label_filter: Some(vec!["REPORTS_TO".to_string()]), ..Default::default() }).unwrap(); + assert_eq!(comps2[&a], a); // isolated when label 10 excluded assert_eq!(comps2[&b], comps2[&c]); // connected db.close().unwrap(); } #[test] - fn test_wcc_node_type_filter() { + fn test_wcc_node_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); // Type 1: A, B. Type 2: C - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(2, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); - - // Filter by node type 1: only A and B visible. - let comps = db.connected_components(&ComponentOptions { node_type_filter: Some(vec![1]), ..Default::default() }).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Company", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + + // Filter by node label 1: only A and B visible. + let comps = db.connected_components(&ComponentOptions { node_label_filter: Some(graph_node_label_filter(&["Person"], LabelMatchMode::Any)), ..Default::default() }).unwrap(); assert_eq!(comps.len(), 2); assert_eq!(comps[&a], a); assert_eq!(comps[&b], a); assert!(!comps.contains_key(&c)); - // Filter by node type 2: only C visible, isolated. - let comps2 = db.connected_components(&ComponentOptions { node_type_filter: Some(vec![2]), ..Default::default() }).unwrap(); + // Filter by node label 2: only C visible, isolated. + let comps2 = db.connected_components(&ComponentOptions { node_label_filter: Some(graph_node_label_filter(&["Company"], LabelMatchMode::Any)), ..Default::default() }).unwrap(); assert_eq!(comps2.len(), 1); assert_eq!(comps2[&c], c); db.close().unwrap(); } + #[test] + fn test_wcc_and_component_of_node_label_filter_support_any_all_multi_label() { + let dir = TempDir::new().unwrap(); + let db = open_imm(&dir.path().join("db")); + + let a = db + .upsert_node( + &["Person", "Employee"], + "a", + UpsertNodeOptions::default(), + ) + .unwrap(); + let b = db + .upsert_node( + &["Person", "Employee"], + "b", + UpsertNodeOptions::default(), + ) + .unwrap(); + let c = db + .upsert_node("Person", "c", UpsertNodeOptions::default()) + .unwrap(); + let d = db + .upsert_node("Employee", "d", UpsertNodeOptions::default()) + .unwrap(); + + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.upsert_edge(b, d, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let all_filter = graph_node_label_filter(&["Person", "Employee"], LabelMatchMode::All); + let all_components = db + .connected_components(&ComponentOptions { + node_label_filter: Some(all_filter.clone()), + ..Default::default() + }) + .unwrap(); + assert_eq!(all_components.len(), 2); + assert_eq!(all_components[&a], a); + assert_eq!(all_components[&b], a); + assert!(!all_components.contains_key(&c)); + assert!(!all_components.contains_key(&d)); + assert_eq!( + db.component_of( + a, + &ComponentOptions { + node_label_filter: Some(all_filter), + ..Default::default() + }, + ) + .unwrap(), + vec![a, b] + ); + + let any_filter = graph_node_label_filter(&["Employee"], LabelMatchMode::Any); + let any_components = db + .connected_components(&ComponentOptions { + node_label_filter: Some(any_filter.clone()), + ..Default::default() + }) + .unwrap(); + assert_eq!(any_components.len(), 3); + assert_eq!(any_components[&a], a); + assert_eq!(any_components[&b], a); + assert_eq!(any_components[&d], a); + assert!(!any_components.contains_key(&c)); + assert_eq!( + db.component_of( + a, + &ComponentOptions { + node_label_filter: Some(any_filter), + ..Default::default() + }, + ) + .unwrap(), + vec![a, b, d] + ); + + db.close().unwrap(); + } + #[test] fn test_wcc_after_flush() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); // Add more data in memtable after flush. - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); // All three in one component (segment edge + memtable edge). @@ -8282,12 +8647,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -8307,11 +8672,11 @@ { let db = open_imm(&db_path); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); db.close().unwrap(); } @@ -8339,12 +8704,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // Only directed edges: A→B, C→B - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(c, b, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(c, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); // All three in one component (WCC ignores direction). @@ -8361,11 +8726,11 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(c, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, a, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(c, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, a, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // Run twice. Must produce identical results. let comps1 = db.connected_components(&ComponentOptions::default()).unwrap(); @@ -8384,11 +8749,11 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); // low weight - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); // low weight + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // Register prune policy that hides weight <= 0.5. db.set_prune_policy( @@ -8396,7 +8761,7 @@ PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ).unwrap(); @@ -8410,18 +8775,85 @@ db.close().unwrap(); } + #[test] + fn test_wcc_unconstrained_prune_policy_checks_unique_multi_label_nodes_once() { + let dir = TempDir::new().unwrap(); + let db = open_imm(&dir.path().join("db")); + + let a = db + .upsert_node( + &["Person", "Employee"], + "a", + UpsertNodeOptions::default(), + ) + .unwrap(); + let b = db + .upsert_node( + &["Person", "Manager"], + "b", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + let c = db + .upsert_node( + &["Company", "Employee"], + "c", + UpsertNodeOptions::default(), + ) + .unwrap(); + let d = db + .upsert_node("Company", "d", UpsertNodeOptions::default()) + .unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.upsert_edge(a, d, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + db.flush().unwrap(); + + db.set_prune_policy( + "low_weight", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: None, + }, + ) + .unwrap(); + + db.reset_query_execution_counters_for_test(); + let comps = db.connected_components(&ComponentOptions::default()).unwrap(); + let counters = db.query_execution_counter_snapshot_for_test(); + + assert_eq!(comps.len(), 3); + assert!(!comps.contains_key(&b)); + assert_eq!(comps[&a], a); + assert_eq!(comps[&d], a); + assert_eq!(comps[&c], c); + assert_eq!( + counters.node_visibility_meta_reads, 4, + "unconstrained WCC prune filtering should verify once per unique node, not once per label membership" + ); + + db.close().unwrap(); + } + #[test] fn test_wcc_memtable_only() { // WCC should work with memtable-only data (no flush). let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); assert_eq!(comps.len(), 4); @@ -8440,12 +8872,12 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // D is isolated. let mut members = db.component_of(a, &ComponentOptions::default()).unwrap(); @@ -8467,7 +8899,7 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); + db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); let members = db.component_of(99999, &ComponentOptions::default()).unwrap(); assert!(members.is_empty()); @@ -8480,9 +8912,9 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.delete_node(a).unwrap(); let members = db.component_of(a, &ComponentOptions::default()).unwrap(); @@ -8496,40 +8928,40 @@ } #[test] - fn test_component_of_node_type_filter() { + fn test_component_of_node_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(2, "b", UpsertNodeOptions::default()).unwrap(); // type 2 - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Company", "b", UpsertNodeOptions::default()).unwrap(); // Company label + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); - // Filter by type 1: B (type 2) is invisible, so A and C are isolated. - let members = db.component_of(a, &ComponentOptions { node_type_filter: Some(vec![1]), ..Default::default() }).unwrap(); + // Filter by Person label: B (Company) is invisible, so A and C are isolated. + let members = db.component_of(a, &ComponentOptions { node_label_filter: Some(graph_node_label_filter(&["Person"], LabelMatchMode::Any)), ..Default::default() }).unwrap(); assert_eq!(members, vec![a]); - // Start from type 2 node with type 1 filter → empty. - let members_b = db.component_of(b, &ComponentOptions { node_type_filter: Some(vec![1]), ..Default::default() }).unwrap(); + // Start from Company node with Person filter -> empty. + let members_b = db.component_of(b, &ComponentOptions { node_label_filter: Some(graph_node_label_filter(&["Person"], LabelMatchMode::Any)), ..Default::default() }).unwrap(); assert!(members_b.is_empty()); db.close().unwrap(); } #[test] - fn test_component_of_edge_type_filter() { + fn test_component_of_edge_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 20, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "REPORTS_TO", UpsertEdgeOptions::default()).unwrap(); - // Filter by edge type 10: only A-B connected. - let members = db.component_of(a, &ComponentOptions { edge_type_filter: Some(vec![10]), ..Default::default() }).unwrap(); + // Filter by KNOWS edge label: only A-B connected. + let members = db.component_of(a, &ComponentOptions { edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }).unwrap(); assert_eq!(members, vec![a, b]); db.close().unwrap(); @@ -8540,13 +8972,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let members = db.component_of(a, &ComponentOptions::default()).unwrap(); assert_eq!(members, vec![a, b, c]); @@ -8559,13 +8991,13 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -8586,11 +9018,11 @@ { let db = open_imm(&db_path); - a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.flush().unwrap(); db.close().unwrap(); } @@ -8605,18 +9037,18 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); // low weight - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.1, ..Default::default() }).unwrap(); // low weight + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); db.set_prune_policy( "low_weight", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ).unwrap(); @@ -8640,14 +9072,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - let e = db.upsert_node(1, "e", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(d, e, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + let e = db.upsert_node("Person", "e", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(d, e, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let comps = db.connected_components(&ComponentOptions::default()).unwrap(); @@ -8671,26 +9103,25 @@ #[test] fn test_wcc_agrees_with_component_of_filtered() { - // Cross-consistency under both node_type_filter and edge_type_filter. + // Cross-consistency under both node_label_filter and edge_label_filter. let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(2, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(b, c, 20, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(c, d, 10, UpsertEdgeOptions::default()).unwrap(); - db.upsert_edge(a, d, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Company", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "REPORTS_TO", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(a, d, "KNOWS", UpsertEdgeOptions::default()).unwrap(); - // Node type filter = [1], edge type filter = [10] - let ntf = Some(&[1u32][..]); - let etf = Some(&[10u32][..]); + let ntf = Some(&["Person"][..]); + let etf = Some(&["KNOWS"][..]); - let comps = db.connected_components(&ComponentOptions { edge_type_filter: etf.map(|s| s.to_vec()), node_type_filter: ntf.map(|s| s.to_vec()), ..Default::default() }).unwrap(); + let comps = db.connected_components(&ComponentOptions { edge_label_filter: etf.map(graph_filter_names), node_label_filter: ntf.map(|labels| graph_node_label_filter(labels, LabelMatchMode::Any)), ..Default::default() }).unwrap(); for &node in &[a, c, d] { - let members = db.component_of(node, &ComponentOptions { edge_type_filter: etf.map(|s| s.to_vec()), node_type_filter: ntf.map(|s| s.to_vec()), ..Default::default() }).unwrap(); + let members = db.component_of(node, &ComponentOptions { edge_label_filter: etf.map(graph_filter_names), node_label_filter: ntf.map(|labels| graph_node_label_filter(labels, LabelMatchMode::Any)), ..Default::default() }).unwrap(); let comp_id = comps[&node]; for &member in &members { assert_eq!(comps[&member], comp_id, @@ -8709,8 +9140,8 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, a, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, a, "KNOWS", UpsertEdgeOptions::default()).unwrap(); let members = db.component_of(a, &ComponentOptions::default()).unwrap(); assert_eq!(members, vec![a]); @@ -8725,9 +9156,9 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // Starting from B, should still find A (Direction::Both in BFS). let members = db.component_of(b, &ComponentOptions::default()).unwrap(); @@ -8742,20 +9173,20 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); - let d = db.upsert_node(1, "d", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); + let d = db.upsert_node("Person", "d", UpsertNodeOptions::default()).unwrap(); let now = now_millis(); // A→B: valid window [0, 1000). Expired at `now` but valid at epoch 500. - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(1000), ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(1000), ..Default::default() }).unwrap(); // B→C: always valid - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()).unwrap(); // C→D: not yet valid (valid_from far in the future) let future = now + 100_000_000; - db.upsert_edge(c, d, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(future), valid_to: None, ..Default::default() }).unwrap(); + db.upsert_edge(c, d, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(future), valid_to: None, ..Default::default() }).unwrap(); // With at_epoch=None (defaults to now): // A→B expired → A isolated @@ -8779,14 +9210,14 @@ let dir2 = TempDir::new().unwrap(); let db2 = open_imm(&dir2.path().join("db")); - let a2 = db2.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b2 = db2.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c2 = db2.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a2 = db2.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b2 = db2.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c2 = db2.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // A→B: valid [0, 1000) - db2.upsert_edge(a2, b2, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(1000), ..Default::default() }).unwrap(); + db2.upsert_edge(a2, b2, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(1000), ..Default::default() }).unwrap(); // B→C: valid [0, i64::MAX) - db2.upsert_edge(b2, c2, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: None, ..Default::default() }).unwrap(); + db2.upsert_edge(b2, c2, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: None, ..Default::default() }).unwrap(); // at_epoch=500: both edges valid → all connected let comps_t500 = db2.connected_components(&ComponentOptions { at_epoch: Some(500), ..Default::default() }).unwrap(); @@ -8808,14 +9239,14 @@ let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); - let a = db.upsert_node(1, "a", UpsertNodeOptions::default()).unwrap(); - let b = db.upsert_node(1, "b", UpsertNodeOptions::default()).unwrap(); - let c = db.upsert_node(1, "c", UpsertNodeOptions::default()).unwrap(); + let a = db.upsert_node("Person", "a", UpsertNodeOptions::default()).unwrap(); + let b = db.upsert_node("Person", "b", UpsertNodeOptions::default()).unwrap(); + let c = db.upsert_node("Person", "c", UpsertNodeOptions::default()).unwrap(); // A→B: valid [0, 1000) - db.upsert_edge(a, b, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(1000), ..Default::default() }).unwrap(); + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: Some(1000), ..Default::default() }).unwrap(); // B→C: always valid from epoch 0 - db.upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: None, ..Default::default() }).unwrap(); + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 1.0, valid_from: Some(0), valid_to: None, ..Default::default() }).unwrap(); // at_epoch=2000: A→B expired → A alone, B-C together. let members_a = db.component_of(a, &ComponentOptions { at_epoch: Some(2000), ..Default::default() }).unwrap(); diff --git a/src/engine/tests/label_catalog.rs b/src/engine/tests/label_catalog.rs new file mode 100644 index 0000000..cded743 --- /dev/null +++ b/src/engine/tests/label_catalog.rs @@ -0,0 +1,2263 @@ +#[test] +fn test_label_catalog_fresh_manifest_defaults() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let manifest = engine.manifest().unwrap(); + assert_eq!( + manifest.label_token_schema_version, + LABEL_TOKEN_SCHEMA_VERSION + ); + assert!(manifest.node_label_tokens.is_empty()); + assert!(manifest.edge_label_tokens.is_empty()); + assert_eq!(manifest.next_node_label_id, 1); + assert_eq!(manifest.next_edge_label_id, 1); + assert!(engine.list_node_labels().unwrap().is_empty()); + assert!(engine.list_edge_labels().unwrap().is_empty()); + + engine.close().unwrap(); +} + +#[test] +fn test_label_catalog_ensure_get_list_and_namespace_independence() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + assert_eq!(engine.ensure_node_label("Person").unwrap(), 1); + assert_eq!(engine.ensure_edge_label("Person").unwrap(), 1); + assert_eq!(engine.ensure_edge_label("WORKS_AT").unwrap(), 2); + assert_eq!(engine.ensure_node_label("Person").unwrap(), 1); + assert_eq!(engine.ensure_edge_label("Person").unwrap(), 1); + + assert_eq!(engine.get_node_label_id("Person").unwrap(), Some(1)); + assert_eq!(engine.get_edge_label_id("Person").unwrap(), Some(1)); + assert_eq!(engine.get_edge_label_id("WORKS_AT").unwrap(), Some(2)); + assert_eq!(engine.get_node_label(1).unwrap().as_deref(), Some("Person")); + assert_eq!(engine.get_edge_label(1).unwrap().as_deref(), Some("Person")); + assert_eq!( + engine.get_edge_label(2).unwrap().as_deref(), + Some("WORKS_AT") + ); + + assert_eq!( + engine.list_node_labels().unwrap(), + vec![NodeLabelInfo { + label: "Person".to_string(), + label_id: 1, + }] + ); + assert_eq!( + engine.list_edge_labels().unwrap(), + vec![ + EdgeLabelInfo { + label: "Person".to_string(), + label_id: 1, + }, + EdgeLabelInfo { + label: "WORKS_AT".to_string(), + label_id: 2, + }, + ] + ); + + engine.close().unwrap(); +} + +#[test] +fn test_published_label_catalog_snapshot_rebuilds_only_for_token_creation() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let initial = engine.published_label_catalog_snapshot_for_test(); + assert_eq!(engine.ensure_node_label("Person").unwrap(), 1); + let after_node_label = engine.published_label_catalog_snapshot_for_test(); + assert!(!std::sync::Arc::ptr_eq(&initial, &after_node_label)); + + assert_eq!(engine.ensure_node_label("Person").unwrap(), 1); + let after_existing_ensure = engine.published_label_catalog_snapshot_for_test(); + assert!(std::sync::Arc::ptr_eq( + &after_node_label, + &after_existing_ensure + )); + + let alice = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let after_existing_node_write = engine.published_label_catalog_snapshot_for_test(); + assert!(std::sync::Arc::ptr_eq( + &after_existing_ensure, + &after_existing_node_write + )); + + assert_eq!(engine.ensure_edge_label("KNOWS").unwrap(), 1); + let after_edge_label = engine.published_label_catalog_snapshot_for_test(); + assert!(!std::sync::Arc::ptr_eq( + &after_existing_node_write, + &after_edge_label + )); + + engine + .upsert_edge(alice, alice, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let after_existing_edge_write = engine.published_label_catalog_snapshot_for_test(); + assert!(std::sync::Arc::ptr_eq( + &after_edge_label, + &after_existing_edge_write + )); + + engine + .upsert_node("Company", "acme", UpsertNodeOptions::default()) + .unwrap(); + let after_first_use_node_write = engine.published_label_catalog_snapshot_for_test(); + assert!(!std::sync::Arc::ptr_eq( + &after_existing_edge_write, + &after_first_use_node_write + )); + assert_eq!(engine.get_node_label_id("Company").unwrap(), Some(2)); + + engine.close().unwrap(); +} + +#[test] +fn test_label_catalog_name_validation() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let long_name = "x".repeat(256); + for invalid in ["", " Person", "Person ", "bad\nname", "bad\u{0}name"] { + assert!(matches!( + engine.ensure_node_label(invalid), + Err(EngineError::InvalidOperation(_)) + )); + assert!(matches!( + engine.ensure_edge_label(invalid), + Err(EngineError::InvalidOperation(_)) + )); + assert!(matches!( + engine.get_node_label_id(invalid), + Err(EngineError::InvalidOperation(_)) + )); + assert!(matches!( + engine.get_edge_label_id(invalid), + Err(EngineError::InvalidOperation(_)) + )); + } + assert!(matches!( + engine.ensure_node_label(&long_name), + Err(EngineError::InvalidOperation(_)) + )); + assert!(matches!( + engine.ensure_edge_label(&long_name), + Err(EngineError::InvalidOperation(_)) + )); + assert!(matches!( + engine.get_node_label_id(&long_name), + Err(EngineError::InvalidOperation(_)) + )); + assert!(matches!( + engine.get_edge_label_id(&long_name), + Err(EngineError::InvalidOperation(_)) + )); + + engine.close().unwrap(); +} + +#[test] +fn test_batch_upsert_inputs_create_named_label_tokens() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let alice = engine + .batch_upsert_nodes(vec![NodeInput { + labels: vec!["Person".to_string()], + key: "alice".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }]) + .unwrap()[0]; + assert_eq!(engine.get_node_label_id("Person").unwrap(), Some(1)); + assert_eq!( + engine.get_node(alice).unwrap().unwrap().labels.as_slice(), + ["Person"] + ); + + let edge_id = engine + .batch_upsert_edges(vec![EdgeInput { + from: alice, + to: alice, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }]) + .unwrap()[0]; + assert_eq!(engine.get_edge_label_id("KNOWS").unwrap(), Some(1)); + assert_eq!( + engine.get_edge(edge_id).unwrap().unwrap().label, + "KNOWS" + ); + + engine.close().unwrap(); +} + +#[test] +fn test_core_point_named_apis_return_hydrated_views_across_sources() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let mut alice_props = BTreeMap::new(); + alice_props.insert("name".to_string(), PropValue::String("Alice".to_string())); + let alice = engine + .upsert_node( + "Person", + "alice", + UpsertNodeOptions { + props: alice_props, + weight: 0.7, + ..Default::default() + }, + ) + .unwrap(); + let bob = engine + .upsert_node("Person", "bob", UpsertNodeOptions::default()) + .unwrap(); + let knows = engine + .upsert_edge( + alice, + bob, + "KNOWS", + UpsertEdgeOptions { + weight: 2.5, + valid_from: Some(10), + valid_to: Some(100), + ..Default::default() + }, + ) + .unwrap(); + + let alice_view = engine.get_node(alice).unwrap().unwrap(); + assert_eq!(alice_view.labels.as_slice(), ["Person"]); + assert_eq!(alice_view.key, "alice"); + assert_eq!( + alice_view.props.get("name"), + Some(&PropValue::String("Alice".to_string())) + ); + assert!((alice_view.weight - 0.7).abs() < f32::EPSILON); + + let knows_view = engine.get_edge(knows).unwrap().unwrap(); + assert_eq!(knows_view.label, "KNOWS"); + assert_eq!(knows_view.from, alice); + assert_eq!(knows_view.to, bob); + assert_eq!(knows_view.valid_from, 10); + assert_eq!(knows_view.valid_to, 100); + + assert_eq!( + engine + .get_node_by_key("Person", "alice") + .unwrap() + .unwrap() + .id, + alice + ); + assert_eq!( + engine + .get_edge_by_triple(alice, bob, "KNOWS") + .unwrap() + .unwrap() + .id, + knows + ); + + let nodes = engine.get_nodes(&[alice, 999, bob]).unwrap(); + assert_eq!(nodes[0].as_ref().unwrap().labels.as_slice(), ["Person"]); + assert!(nodes[1].is_none()); + assert_eq!(nodes[2].as_ref().unwrap().key, "bob"); + + let edges = engine.get_edges(&[knows, 999]).unwrap(); + assert_eq!(edges[0].as_ref().unwrap().label, "KNOWS"); + assert!(edges[1].is_none()); + + let key_results = engine + .get_nodes_by_keys(&[ + NodeKeyQuery { + label: "Person".to_string(), + key: "alice".to_string(), + }, + NodeKeyQuery { + label: "Person".to_string(), + key: "bob".to_string(), + }, + NodeKeyQuery { + label: "MissingButValid".to_string(), + key: "alice".to_string(), + }, + ]) + .unwrap(); + assert_eq!(key_results[0].as_ref().unwrap().id, alice); + assert_eq!( + key_results[0].as_ref().unwrap().labels.as_slice(), + ["Person"] + ); + assert_eq!(key_results[1].as_ref().unwrap().id, bob); + assert_eq!( + key_results[1].as_ref().unwrap().labels.as_slice(), + ["Person"] + ); + assert!(key_results[2].is_none()); + assert_eq!(engine.get_node_label_id("MissingButValid").unwrap(), None); + + assert!(engine + .get_node_by_key("MissingButValid", "alice") + .unwrap() + .is_none()); + assert!(engine + .get_edge_by_triple(alice, bob, "MISSING") + .unwrap() + .is_none()); + assert!(matches!( + engine.get_node_by_key(" Person", "alice"), + Err(EngineError::InvalidOperation(_)) + )); + assert!(matches!( + engine.get_edge_by_triple(alice, bob, "KNOWS\n"), + Err(EngineError::InvalidOperation(_)) + )); + + let invalidated = engine.invalidate_edge(knows, 55).unwrap().unwrap(); + assert_eq!(invalidated.label, "KNOWS"); + assert_eq!(invalidated.valid_to, 55); + + engine.flush().unwrap(); + assert_eq!( + engine.get_node(alice).unwrap().unwrap().labels.as_slice(), + ["Person"] + ); + assert_eq!(engine.get_edge(knows).unwrap().unwrap().label, "KNOWS"); + + let carol = engine + .upsert_node("Person", "carol", UpsertNodeOptions::default()) + .unwrap(); + let _mentors = engine + .upsert_edge(alice, carol, "MENTORS", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + engine.compact().unwrap(); + + assert_eq!( + engine.get_node(carol).unwrap().unwrap().labels.as_slice(), + ["Person"] + ); + assert_eq!(engine.get_edge(knows).unwrap().unwrap().label, "KNOWS"); + engine.close().unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert_eq!( + reopened.get_node(alice).unwrap().unwrap().labels.as_slice(), + ["Person"] + ); + assert_eq!( + reopened.get_edge(knows).unwrap().unwrap().label, + "KNOWS" + ); + reopened.close().unwrap(); +} + +#[test] +fn test_core_point_first_use_tokens_share_wal_batch_with_records() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let alice = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let bob = engine + .upsert_node("Person", "bob", UpsertNodeOptions::default()) + .unwrap(); + let edge_id = engine + .upsert_edge(alice, bob, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let all_ops = WalReader::read_generation(&db_path, 0).unwrap(); + let ops = marker_free_wal_records(&all_ops); + assert!(matches!( + &ops[0].1, + WalOp::EnsureNodeLabel { label, label_id } if label == "Person" && *label_id == 1 + )); + assert!(matches!( + &ops[1].1, + WalOp::UpsertNode(node) if node.id == alice && node.label_ids.as_slice() == [1] + )); + assert!(matches!( + &ops[2].1, + WalOp::UpsertNode(node) if node.id == bob && node.label_ids.as_slice() == [1] + )); + assert!(matches!( + &ops[3].1, + WalOp::EnsureEdgeLabel { label, label_id } if label == "KNOWS" && *label_id == 1 + )); + assert!( + matches!(&ops[4].1, WalOp::UpsertEdge(edge) if edge.id == edge_id && edge.label_id == 1) + ); + + let disk_manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert!(disk_manifest.node_label_tokens.is_empty()); + assert!(disk_manifest.edge_label_tokens.is_empty()); + + drop(engine); + let reopened = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert_eq!(reopened.get_node_label_id("Person").unwrap(), Some(1)); + assert_eq!(reopened.get_edge_label_id("KNOWS").unwrap(), Some(1)); + assert_eq!( + reopened.get_node(alice).unwrap().unwrap().labels.as_slice(), + ["Person"] + ); + assert_eq!( + reopened.get_edge(edge_id).unwrap().unwrap().label, + "KNOWS" + ); + reopened.close().unwrap(); +} + +#[test] +fn test_batch_upserts_stage_distinct_named_tokens_once_before_records() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let node_ids = engine + .batch_upsert_nodes(vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "alice".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "bob".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Company".to_string()], + key: "acme".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + ]) + .unwrap(); + assert_eq!(node_ids.len(), 3); + + let edge_ids = engine + .batch_upsert_edges(vec![ + EdgeInput { + from: node_ids[0], + to: node_ids[1], + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: node_ids[0], + to: node_ids[2], + label: "WORKS_AT".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: node_ids[0], + to: node_ids[1], + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 2.0, + valid_from: None, + valid_to: None, + }, + ]) + .unwrap(); + assert_eq!(edge_ids[0], edge_ids[2]); + + let all_ops = WalReader::read_generation(&db_path, 0).unwrap(); + let ops = marker_free_wal_records(&all_ops); + assert!(matches!( + &ops[0].1, + WalOp::EnsureNodeLabel { label, label_id } if label == "Person" && *label_id == 1 + )); + assert!(matches!( + &ops[1].1, + WalOp::EnsureNodeLabel { label, label_id } if label == "Company" && *label_id == 2 + )); + assert!(matches!(&ops[2].1, WalOp::UpsertNode(node) if node.label_ids.as_slice() == [1])); + assert!(matches!(&ops[3].1, WalOp::UpsertNode(node) if node.label_ids.as_slice() == [1])); + assert!(matches!(&ops[4].1, WalOp::UpsertNode(node) if node.label_ids.as_slice() == [2])); + assert!(matches!( + &ops[5].1, + WalOp::EnsureEdgeLabel { label, label_id } if label == "KNOWS" && *label_id == 1 + )); + assert!(matches!( + &ops[6].1, + WalOp::EnsureEdgeLabel { label, label_id } if label == "WORKS_AT" && *label_id == 2 + )); + assert!(matches!(&ops[7].1, WalOp::UpsertEdge(edge) if edge.label_id == 1)); + assert!(matches!(&ops[8].1, WalOp::UpsertEdge(edge) if edge.label_id == 2)); + assert!(matches!(&ops[9].1, WalOp::UpsertEdge(edge) if edge.label_id == 1)); + assert_eq!( + ops.iter() + .filter( + |(_, op)| matches!(op, WalOp::EnsureNodeLabel { label, .. } if label == "Person") + ) + .count(), + 1 + ); + assert_eq!( + ops.iter() + .filter(|(_, op)| matches!(op, WalOp::EnsureEdgeLabel { label, .. } if label == "KNOWS")) + .count(), + 1 + ); + + engine.close().unwrap(); +} + +#[test] +fn test_label_resolution_plan_caches_distinct_node_labels_per_request() { + let mut manifest = default_manifest(); + manifest.node_label_tokens.insert("Person".to_string(), 7); + manifest.next_node_label_id = 8; + let catalog = RuntimeLabelCatalog::from_manifest(&manifest).unwrap(); + let mut plan = LabelResolutionPlan::from_catalog(&catalog); + + let label_ids = plan + .resolve_node_label_ids_for_request( + ["Person", "Person", "Company", "Company", "Person"].into_iter(), + ) + .unwrap(); + + assert_eq!(label_ids, vec![7, 7, 8, 8, 7]); + assert_eq!(plan.node_label_resolve_calls, 2); + assert_eq!(plan.node_labels_to_create, vec![("Company".to_string(), 8)]); +} + +#[test] +fn test_node_label_set_resolution_validates_before_token_reservation() { + let manifest = default_manifest(); + let catalog = RuntimeLabelCatalog::from_manifest(&manifest).unwrap(); + let mut plan = LabelResolutionPlan::from_catalog(&catalog); + + let err = match ValidatedNodeLabelList::new(["LeakyLabel", "LeakyLabel"]) { + Ok(_) => panic!("duplicate labels should be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("duplicate label")); + assert_eq!(plan.node_label_resolve_calls, 0); + assert!(plan.node_labels_to_create.is_empty()); + assert!(plan.new_node_label_to_id.is_empty()); + + let too_many = [ + "L1", "L2", "L3", "L4", "L5", "L6", "L7", "L8", "L9", "L10", "L11", + ]; + let err = match ValidatedNodeLabelList::new(too_many) { + Ok(_) => panic!("too many labels should be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("at most 10 labels")); + assert_eq!(plan.node_label_resolve_calls, 0); + assert!(plan.node_labels_to_create.is_empty()); + assert!(plan.new_node_label_to_id.is_empty()); + + let err = match ValidatedNodeLabelList::new([" LeakyLabel"]) { + Ok(_) => panic!("invalid label should be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("leading or trailing whitespace")); + assert_eq!(plan.node_label_resolve_calls, 0); + assert!(plan.node_labels_to_create.is_empty()); + assert!(plan.new_node_label_to_id.is_empty()); + + let labels = ValidatedNodeLabelList::new(["Person", "Company"]).unwrap(); + let label_set = plan + .resolve_validated_node_label_set_for_write(&labels) + .unwrap(); + assert_eq!(label_set.as_slice(), &[1, 2]); + assert_eq!(plan.node_label_resolve_calls, 2); + assert_eq!( + plan.node_labels_to_create, + vec![("Person".to_string(), 1), ("Company".to_string(), 2)] + ); +} + +#[test] +fn test_node_label_set_batch_validation_happens_before_token_reservation() { + let manifest = default_manifest(); + let catalog = RuntimeLabelCatalog::from_manifest(&manifest).unwrap(); + let plan = LabelResolutionPlan::from_catalog(&catalog); + let requests = [vec!["WouldHaveBeenReserved"], vec!["LeakyLabel", "LeakyLabel"]]; + + let mut validated_labels = Vec::with_capacity(requests.len()); + let err = requests + .iter() + .find_map(|labels| match ValidatedNodeLabelList::new(labels.iter().copied()) { + Ok(labels) => { + validated_labels.push(labels); + None + } + Err(err) => Some(err), + }) + .expect("duplicate labels should be rejected"); + + assert!(err.to_string().contains("duplicate label")); + assert_eq!(validated_labels.len(), 1); + assert_eq!(plan.node_label_resolve_calls, 0); + assert!(plan.node_labels_to_create.is_empty()); + assert!(plan.new_node_label_to_id.is_empty()); +} + +#[test] +fn test_node_label_set_resolution_is_distinct_and_deterministic() { + let mut manifest = default_manifest(); + manifest.node_label_tokens.insert("Person".to_string(), 7); + manifest.next_node_label_id = 8; + let catalog = RuntimeLabelCatalog::from_manifest(&manifest).unwrap(); + let mut plan = LabelResolutionPlan::from_catalog(&catalog); + let requests = [ + vec!["Company", "Person"], + vec!["Team", "Company", "Person"], + ]; + + let validated_labels = requests + .iter() + .map(|labels| ValidatedNodeLabelList::new(labels.iter().copied())) + .collect::, _>>() + .unwrap(); + let label_sets = plan + .resolve_validated_node_label_sets_for_request(&validated_labels) + .unwrap(); + + assert_eq!(label_sets[0].as_slice(), &[7, 8]); + assert_eq!(label_sets[1].as_slice(), &[7, 8, 9]); + assert_eq!(plan.node_label_resolve_calls, 3); + assert_eq!( + plan.node_labels_to_create, + vec![("Company".to_string(), 8), ("Team".to_string(), 9)] + ); +} + +#[test] +fn test_resolved_node_label_filter_read_semantics_are_numeric_and_deterministic() { + let mut manifest = default_manifest(); + manifest.node_label_tokens.insert("Person".to_string(), 7); + manifest.node_label_tokens.insert("Company".to_string(), 3); + manifest.next_node_label_id = 8; + let catalog = RuntimeLabelCatalog::from_manifest(&manifest).unwrap(); + let snapshot = ReadLabelCatalogSnapshot::from_runtime(&catalog); + + let any = snapshot + .resolve_node_label_filter_request(Some(&NodeLabelFilter { + labels: vec![ + "Missing".to_string(), + "Person".to_string(), + "Company".to_string(), + ], + mode: LabelMatchMode::Any, + })) + .unwrap(); + assert_eq!(any.mode(), Some(LabelMatchMode::Any)); + assert!(!any.is_empty_constraint()); + assert_eq!(any.label_ids().unwrap().as_slice(), &[3, 7]); + assert!(matches!( + any, + ResolvedNodeLabelFilter::LabelSet { + unknown_label_count: 1, + .. + } + )); + + let all = snapshot + .resolve_node_label_filter_request(Some(&NodeLabelFilter { + labels: vec!["Person".to_string(), "Missing".to_string()], + mode: LabelMatchMode::All, + })) + .unwrap(); + assert_eq!(all.mode(), Some(LabelMatchMode::All)); + assert!(all.is_empty_constraint()); + assert!(matches!( + all, + ResolvedNodeLabelFilter::Empty { + unknown_label_count: 1, + .. + } + )); + + assert_eq!( + snapshot.resolve_node_label_filter_request(None).unwrap(), + ResolvedNodeLabelFilter::Unconstrained + ); +} + +#[test] +fn test_label_resolution_plan_caches_distinct_edge_labels_per_request() { + let mut manifest = default_manifest(); + manifest.edge_label_tokens.insert("KNOWS".to_string(), 4); + manifest.next_edge_label_id = 5; + let catalog = RuntimeLabelCatalog::from_manifest(&manifest).unwrap(); + let mut plan = LabelResolutionPlan::from_catalog(&catalog); + + let label_ids = plan + .resolve_edge_label_ids_for_request( + ["KNOWS", "KNOWS", "WORKS_AT", "WORKS_AT", "KNOWS"].into_iter(), + ) + .unwrap(); + + assert_eq!(label_ids, vec![4, 4, 5, 5, 4]); + assert_eq!(plan.edge_label_resolve_calls, 2); + assert_eq!(plan.edge_labels_to_create, vec![("WORKS_AT".to_string(), 5)]); +} + +#[test] +fn test_graph_patch_stages_named_tokens_before_dependent_ops() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let result = engine + .graph_patch(GraphPatch { + upsert_nodes: vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "alice".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Company".to_string()], + key: "acme".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "bob".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + ], + upsert_edges: vec![ + EdgeInput { + from: 1, + to: 2, + label: "WORKS_AT".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: 3, + to: 2, + label: "WORKS_AT".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }, + ], + invalidate_edges: Vec::new(), + delete_node_ids: Vec::new(), + delete_edge_ids: Vec::new(), + }) + .unwrap(); + assert_eq!(result.node_ids, vec![1, 2, 3]); + assert_eq!(result.edge_ids.len(), 2); + + let all_ops = WalReader::read_generation(&db_path, 0).unwrap(); + let ops = marker_free_wal_records(&all_ops); + assert!(matches!( + &ops[0].1, + WalOp::EnsureNodeLabel { label, label_id } if label == "Person" && *label_id == 1 + )); + assert!(matches!( + &ops[1].1, + WalOp::EnsureNodeLabel { label, label_id } if label == "Company" && *label_id == 2 + )); + assert!(matches!( + &ops[2].1, + WalOp::EnsureEdgeLabel { label, label_id } if label == "WORKS_AT" && *label_id == 1 + )); + assert!(matches!(&ops[3].1, WalOp::UpsertNode(node) if node.label_ids.as_slice() == [1])); + assert!(matches!(&ops[4].1, WalOp::UpsertNode(node) if node.label_ids.as_slice() == [2])); + assert!(matches!(&ops[5].1, WalOp::UpsertNode(node) if node.label_ids.as_slice() == [1])); + assert!(matches!(&ops[6].1, WalOp::UpsertEdge(edge) if edge.label_id == 1)); + assert!(matches!(&ops[7].1, WalOp::UpsertEdge(edge) if edge.label_id == 1)); + assert_eq!( + ops.iter() + .filter( + |(_, op)| matches!(op, WalOp::EnsureNodeLabel { label, .. } if label == "Person") + ) + .count(), + 1 + ); + assert_eq!( + ops.iter() + .filter(|(_, op)| matches!(op, WalOp::EnsureEdgeLabel { label, .. } if label == "WORKS_AT")) + .count(), + 1 + ); + + engine.close().unwrap(); +} + +#[test] +fn test_graph_patch_failure_does_not_publish_staged_token() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let err = engine + .graph_patch(GraphPatch { + upsert_nodes: vec![NodeInput { + labels: vec!["LeakyPatchLabel".to_string()], + key: "bad-vector".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: Some(vec![1.0, 0.0]), + sparse_vector: None, + }], + ..Default::default() + }) + .unwrap_err(); + assert!(err.to_string().contains("dimension")); + assert_eq!(engine.get_node_label_id("LeakyPatchLabel").unwrap(), None); + + engine.close().unwrap(); +} + +#[test] +fn test_graph_patch_invalidation_uses_staged_edge_overlay() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let mut v1 = BTreeMap::new(); + v1.insert("version".to_string(), PropValue::Int(1)); + let edge_id = engine + .upsert_edge( + a, + b, + "KNOWS", + UpsertEdgeOptions { + props: v1, + weight: 1.0, + valid_from: Some(10), + valid_to: Some(9_999), + }, + ) + .unwrap(); + + let mut v2 = BTreeMap::new(); + v2.insert("version".to_string(), PropValue::Int(2)); + let result = engine + .graph_patch(GraphPatch { + upsert_edges: vec![EdgeInput { + from: a, + to: b, + label: "KNOWS".to_string(), + props: v2, + weight: 2.5, + valid_from: Some(20), + valid_to: Some(i64::MAX), + }], + invalidate_edges: vec![(edge_id, 1234)], + ..Default::default() + }) + .unwrap(); + assert_eq!(result.edge_ids, vec![edge_id]); + + let edge = engine.get_edge(edge_id).unwrap().unwrap(); + assert_eq!(edge.props.get("version"), Some(&PropValue::Int(2))); + assert!((edge.weight - 2.5).abs() < f32::EPSILON); + assert_eq!(edge.valid_from, 20); + assert_eq!(edge.valid_to, 1234); + + engine.close().unwrap(); +} + +#[test] +fn test_graph_patch_delete_node_cascades_staged_new_edge_once() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + + let result = engine + .graph_patch(GraphPatch { + upsert_edges: vec![EdgeInput { + from: a, + to: b, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }], + delete_node_ids: vec![a], + ..Default::default() + }) + .unwrap(); + let edge_id = result.edge_ids[0]; + + assert!(engine.get_edge(edge_id).unwrap().is_none()); + assert!(engine.get_node(a).unwrap().is_none()); + let ops = WalReader::read_generation(&db_path, 0).unwrap(); + assert_eq!( + ops.iter() + .filter(|(_, op)| matches!(op, WalOp::DeleteEdge { id, .. } if *id == edge_id)) + .count(), + 1 + ); + + engine.close().unwrap(); +} + +#[test] +fn test_graph_patch_delete_node_cascades_staged_existing_edge_update() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let edge_id = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let mut props = BTreeMap::new(); + props.insert("version".to_string(), PropValue::Int(2)); + let result = engine + .graph_patch(GraphPatch { + upsert_edges: vec![EdgeInput { + from: a, + to: b, + label: "KNOWS".to_string(), + props, + weight: 3.0, + valid_from: None, + valid_to: None, + }], + delete_node_ids: vec![a], + ..Default::default() + }) + .unwrap(); + + assert_eq!(result.edge_ids, vec![edge_id]); + assert!(engine.get_edge(edge_id).unwrap().is_none()); + assert!(engine.get_node(a).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_graph_patch_explicit_delete_and_cascade_emit_one_edge_tombstone() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let edge_id = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + engine + .graph_patch(GraphPatch { + delete_edge_ids: vec![edge_id], + delete_node_ids: vec![a], + ..Default::default() + }) + .unwrap(); + + assert!(engine.get_edge(edge_id).unwrap().is_none()); + assert!(engine.get_node(a).unwrap().is_none()); + assert!(engine.get_node(b).unwrap().is_some()); + let ops = WalReader::read_generation(&db_path, 0).unwrap(); + assert_eq!( + ops.iter() + .filter(|(_, op)| matches!(op, WalOp::DeleteEdge { id, .. } if *id == edge_id)) + .count(), + 1 + ); + + engine.close().unwrap(); +} + +#[test] +fn test_named_batch_planning_reuses_existing_ids_across_sources() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let node_segment = engine + .upsert_node("Person", "segment", UpsertNodeOptions::default()) + .unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let c = engine + .upsert_node("Person", "c", UpsertNodeOptions::default()) + .unwrap(); + let d = engine + .upsert_node("Person", "d", UpsertNodeOptions::default()) + .unwrap(); + let e = engine + .upsert_node("Person", "e", UpsertNodeOptions::default()) + .unwrap(); + let f = engine + .upsert_node("Person", "f", UpsertNodeOptions::default()) + .unwrap(); + let edge_segment = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + + let node_immutable = engine + .upsert_node("Person", "immutable", UpsertNodeOptions::default()) + .unwrap(); + let edge_immutable = engine + .upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.freeze_memtable().unwrap(); + + let node_active = engine + .upsert_node("Person", "active", UpsertNodeOptions::default()) + .unwrap(); + let edge_active = engine + .upsert_edge(e, f, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let node_ids = engine + .batch_upsert_nodes(vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "segment".to_string(), + props: BTreeMap::new(), + weight: 2.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "immutable".to_string(), + props: BTreeMap::new(), + weight: 2.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "active".to_string(), + props: BTreeMap::new(), + weight: 2.0, + dense_vector: None, + sparse_vector: None, + }, + ]) + .unwrap(); + assert_eq!(node_ids, vec![node_segment, node_immutable, node_active]); + + let edge_ids = engine + .batch_upsert_edges(vec![ + EdgeInput { + from: a, + to: b, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 2.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: c, + to: d, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 2.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: e, + to: f, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 2.0, + valid_from: None, + valid_to: None, + }, + ]) + .unwrap(); + assert_eq!(edge_ids, vec![edge_segment, edge_immutable, edge_active]); + + engine.close().unwrap(); +} + +#[test] +fn test_named_graph_patch_reuses_existing_ids_across_sources() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let node_segment = engine + .upsert_node("Person", "segment", UpsertNodeOptions::default()) + .unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let c = engine + .upsert_node("Person", "c", UpsertNodeOptions::default()) + .unwrap(); + let d = engine + .upsert_node("Person", "d", UpsertNodeOptions::default()) + .unwrap(); + let e = engine + .upsert_node("Person", "e", UpsertNodeOptions::default()) + .unwrap(); + let f = engine + .upsert_node("Person", "f", UpsertNodeOptions::default()) + .unwrap(); + let edge_segment = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + + let node_immutable = engine + .upsert_node("Person", "immutable", UpsertNodeOptions::default()) + .unwrap(); + let edge_immutable = engine + .upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.freeze_memtable().unwrap(); + + let node_active = engine + .upsert_node("Person", "active", UpsertNodeOptions::default()) + .unwrap(); + let edge_active = engine + .upsert_edge(e, f, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let result = engine + .graph_patch(GraphPatch { + upsert_nodes: vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "segment".to_string(), + props: BTreeMap::new(), + weight: 2.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "immutable".to_string(), + props: BTreeMap::new(), + weight: 2.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "active".to_string(), + props: BTreeMap::new(), + weight: 2.0, + dense_vector: None, + sparse_vector: None, + }, + ], + upsert_edges: vec![ + EdgeInput { + from: a, + to: b, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 2.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: c, + to: d, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 2.0, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: e, + to: f, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 2.0, + valid_from: None, + valid_to: None, + }, + ], + ..Default::default() + }) + .unwrap(); + + assert_eq!(result.node_ids, vec![node_segment, node_immutable, node_active]); + assert_eq!(result.edge_ids, vec![edge_segment, edge_immutable, edge_active]); + assert_eq!( + engine + .get_node_by_key("Person", "segment") + .unwrap() + .unwrap() + .id, + node_segment + ); + assert_eq!( + engine + .get_node_by_key("Person", "immutable") + .unwrap() + .unwrap() + .id, + node_immutable + ); + assert_eq!( + engine + .get_node_by_key("Person", "active") + .unwrap() + .unwrap() + .id, + node_active + ); + assert_eq!( + engine.get_edge_by_triple(a, b, "KNOWS").unwrap().unwrap().id, + edge_segment + ); + assert_eq!( + engine.get_edge_by_triple(c, d, "KNOWS").unwrap().unwrap().id, + edge_immutable + ); + assert_eq!( + engine.get_edge_by_triple(e, f, "KNOWS").unwrap().unwrap().id, + edge_active + ); + + engine.close().unwrap(); +} + +#[test] +fn test_named_batch_edge_lookup_respects_tombstone_shadowing() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let old_edge = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + engine.delete_edge(old_edge).unwrap(); + + let ids = engine + .batch_upsert_edges(vec![EdgeInput { + from: a, + to: b, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }]) + .unwrap(); + + assert_ne!(ids[0], old_edge); + assert!(engine.get_edge(old_edge).unwrap().is_none()); + assert_eq!( + engine + .get_edge_by_triple(a, b, "KNOWS") + .unwrap() + .unwrap() + .id, + ids[0] + ); + + engine.close().unwrap(); +} + +#[test] +fn test_explicit_ensure_is_wal_durable_without_foreground_manifest_write() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + assert_eq!(engine.ensure_node_label("Person").unwrap(), 1); + assert_eq!(engine.ensure_edge_label("KNOWS").unwrap(), 1); + + let disk_manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert!(disk_manifest.node_label_tokens.is_empty()); + assert!(disk_manifest.edge_label_tokens.is_empty()); + + drop(engine); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(reopened.get_node_label_id("Person").unwrap(), Some(1)); + assert_eq!(reopened.get_edge_label_id("KNOWS").unwrap(), Some(1)); + reopened.close().unwrap(); +} + +#[test] +fn test_metadata_writes_checkpoint_existing_wal_tokens() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + assert_eq!(db.ensure_node_label("Article").unwrap(), 1); + assert_eq!(db.ensure_edge_label("MENTIONS").unwrap(), 1); + assert_eq!(db.ensure_node_label("Expiring").unwrap(), 2); + let disk_manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert!(disk_manifest.node_label_tokens.is_empty()); + assert!(disk_manifest.edge_label_tokens.is_empty()); + + db.ensure_node_property_index("Article", "status", SecondaryIndexKind::Equality) + .unwrap(); + db.ensure_edge_property_index("MENTIONS", "rank", SecondaryIndexKind::Equality) + .unwrap(); + db.set_prune_policy( + "expiring", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.1), + label: Some("Expiring".to_string()), + }, + ) + .unwrap(); + + let disk_manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert_eq!(disk_manifest.node_label_tokens.get("Article"), Some(&1)); + assert_eq!(disk_manifest.node_label_tokens.get("Expiring"), Some(&2)); + assert_eq!(disk_manifest.edge_label_tokens.get("MENTIONS"), Some(&1)); + assert!(disk_manifest.secondary_indexes.iter().any(|entry| { + entry.target + == SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "status".to_string(), + } + })); + assert!(disk_manifest.secondary_indexes.iter().any(|entry| { + entry.target + == SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "rank".to_string(), + } + })); + assert_eq!( + disk_manifest + .prune_policies + .get("expiring") + .and_then(|policy| policy.label.as_deref()), + Some("Expiring") + ); + + db.close().unwrap(); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(reopened.get_node_label_id("Article").unwrap(), Some(1)); + assert_eq!(reopened.get_node_label_id("Expiring").unwrap(), Some(2)); + assert_eq!(reopened.get_edge_label_id("MENTIONS").unwrap(), Some(1)); + assert_eq!( + reopened.list_prune_policies().unwrap()[0] + .policy + .label + .as_deref(), + Some("Expiring") + ); + reopened.close().unwrap(); +} + +#[test] +fn test_flush_persists_label_tokens_before_wal_retirement() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + assert_eq!(engine.ensure_node_label("Person").unwrap(), 1); + assert_eq!(engine.ensure_edge_label("KNOWS").unwrap(), 1); + let alice = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_edge(alice, alice, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + + let disk_manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert_eq!(disk_manifest.node_label_tokens.get("Person"), Some(&1)); + assert_eq!(disk_manifest.edge_label_tokens.get("KNOWS"), Some(&1)); + assert!(!wal_generation_path(&db_path, 0).exists()); + + engine.close().unwrap(); +} + +#[test] +fn test_background_publish_does_not_checkpoint_active_group_commit_token() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::GroupCommit { + interval_ms: 60_000, + soft_trigger_bytes: 1 << 20, + hard_cap_bytes: 1 << 21, + }, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + assert_eq!(engine.ensure_node_label("DurableBeforeFlush").unwrap(), 1); + engine + .upsert_node("DurableBeforeFlush", "alice", UpsertNodeOptions::default()) + .unwrap(); + engine.freeze_memtable().unwrap(); + + assert_eq!(engine.ensure_node_label("ActiveOnly").unwrap(), 2); + engine.flush().unwrap(); + + let disk_manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert_eq!( + disk_manifest.node_label_tokens.get("DurableBeforeFlush"), + Some(&1) + ); + assert_eq!(disk_manifest.node_label_tokens.get("ActiveOnly"), None); + assert!( + !wal_generation_path(&db_path, 0).exists(), + "retired WAL generation should be removable after its token is checkpointed" + ); + + engine.close().unwrap(); + let closed_manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert_eq!( + closed_manifest.node_label_tokens.get("ActiveOnly"), + Some(&2) + ); +} + +#[test] +fn test_label_token_wal_replay_restores_catalog_and_records() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + std::fs::create_dir_all(&db_path).unwrap(); + write_manifest(&db_path, &default_manifest()).unwrap(); + + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureEdgeLabel { + label: "KNOWS".to_string(), + label_id: 1, + }, + 2, + ) + .unwrap(); + writer + .append( + &WalOp::UpsertNode(NodeRecord { + id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), + key: "alice".to_string(), + props: BTreeMap::new(), + created_at: 1, + updated_at: 1, + weight: 1.0, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 3, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Person").unwrap(), Some(1)); + assert_eq!(engine.get_edge_label_id("KNOWS").unwrap(), Some(1)); + assert_eq!(engine.get_node(1).unwrap().unwrap().key, "alice"); + engine.close().unwrap(); +} + +#[test] +fn test_label_token_wal_replay_rejects_conflicting_name_or_id() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + std::fs::create_dir_all(&db_path).unwrap(); + write_manifest(&db_path, &default_manifest()).unwrap(); + + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 2, + }, + 2, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + assert!(matches!( + DatabaseEngine::open(&db_path, &DbOptions::default()), + Err(EngineError::CorruptWal(_)) + )); + + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + std::fs::create_dir_all(&db_path).unwrap(); + write_manifest(&db_path, &default_manifest()).unwrap(); + + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Company".to_string(), + label_id: 1, + }, + 2, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + assert!(matches!( + DatabaseEngine::open(&db_path, &DbOptions::default()), + Err(EngineError::CorruptWal(_)) + )); + + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + std::fs::create_dir_all(&db_path).unwrap(); + write_manifest(&db_path, &default_manifest()).unwrap(); + + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureEdgeLabel { + label: "KNOWS".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureEdgeLabel { + label: "LIKES".to_string(), + label_id: 1, + }, + 2, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + assert!(matches!( + DatabaseEngine::open(&db_path, &DbOptions::default()), + Err(EngineError::CorruptWal(_)) + )); + + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + std::fs::create_dir_all(&db_path).unwrap(); + write_manifest(&db_path, &default_manifest()).unwrap(); + + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureEdgeLabel { + label: "KNOWS".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureEdgeLabel { + label: "KNOWS".to_string(), + label_id: 2, + }, + 2, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + assert!(matches!( + DatabaseEngine::open(&db_path, &DbOptions::default()), + Err(EngineError::CorruptWal(_)) + )); +} + +#[test] +fn test_wal_replay_rejects_dependent_record_with_missing_token() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + std::fs::create_dir_all(&db_path).unwrap(); + write_manifest(&db_path, &default_manifest()).unwrap(); + + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::UpsertNode(NodeRecord { + id: 1, + label_ids: NodeLabelSet::single(99).unwrap(), + key: "missing-token".to_string(), + props: BTreeMap::new(), + created_at: 1, + updated_at: 1, + weight: 1.0, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 1, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + assert!(matches!( + DatabaseEngine::open(&db_path, &DbOptions::default()), + Err(EngineError::CorruptWal(_)) + )); +} + +#[test] +fn test_public_label_and_edge_label_scans_are_read_only_and_hydrate_views() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let alice = db + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let bob = db + .upsert_node("Person", "bob", UpsertNodeOptions::default()) + .unwrap(); + let acme = db + .upsert_node("Company", "acme", UpsertNodeOptions::default()) + .unwrap(); + let knows = db + .upsert_edge(alice, bob, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let works_at = db + .upsert_edge(alice, acme, "WORKS_AT", UpsertEdgeOptions::default()) + .unwrap(); + + assert_eq!(db.nodes_by_labels("Person").unwrap(), vec![alice, bob]); + assert_eq!(db.count_nodes_by_labels("Person").unwrap(), 2); + let people = db.get_nodes_by_labels("Person").unwrap(); + assert_eq!(people.iter().map(|node| node.id).collect::>(), vec![alice, bob]); + assert!(people + .iter() + .all(|node| node.labels.as_slice() == ["Person"])); + + let page = db + .nodes_by_labels_paged( + "Person", + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(page.items, vec![alice]); + assert_eq!(page.next_cursor, Some(alice)); + let hydrated_page = db + .get_nodes_by_labels_paged( + "Person", + &PageRequest { + limit: Some(1), + after: page.next_cursor, + }, + ) + .unwrap(); + assert_eq!( + hydrated_page + .items + .iter() + .map(|node| (node.id, node.labels[0].as_str())) + .collect::>(), + vec![(bob, "Person")] + ); + + assert_eq!(db.edges_by_label("KNOWS").unwrap(), vec![knows]); + assert_eq!(db.count_edges_by_label("WORKS_AT").unwrap(), 1); + assert_eq!( + db.get_edges_by_label("WORKS_AT") + .unwrap() + .iter() + .map(|edge| (edge.id, edge.label.as_str())) + .collect::>(), + vec![(works_at, "WORKS_AT")] + ); + assert_eq!( + db.edges_by_label_paged("KNOWS", &PageRequest::default()) + .unwrap() + .items, + vec![knows] + ); + assert_eq!( + db.get_edges_by_label_paged("KNOWS", &PageRequest::default()) + .unwrap() + .items + .iter() + .map(|edge| (edge.id, edge.label.as_str())) + .collect::>(), + vec![(knows, "KNOWS")] + ); + + let node_catalog_len = db.list_node_labels().unwrap().len(); + let edge_catalog_len = db.list_edge_labels().unwrap().len(); + assert_eq!(db.nodes_by_labels("Missing").unwrap(), Vec::::new()); + assert_eq!( + db.nodes_by_labels_paged("Missing", &PageRequest::default()) + .unwrap() + .items, + Vec::::new() + ); + assert_eq!(db.get_nodes_by_labels("Missing").unwrap(), Vec::::new()); + assert_eq!( + db.get_nodes_by_labels_paged("Missing", &PageRequest::default()) + .unwrap() + .items, + Vec::::new() + ); + assert_eq!(db.count_nodes_by_labels("Missing").unwrap(), 0); + assert_eq!(db.edges_by_label("MISSING").unwrap(), Vec::::new()); + assert_eq!( + db.edges_by_label_paged("MISSING", &PageRequest::default()) + .unwrap() + .items, + Vec::::new() + ); + assert_eq!(db.get_edges_by_label("MISSING").unwrap(), Vec::::new()); + assert_eq!( + db.get_edges_by_label_paged("MISSING", &PageRequest::default()) + .unwrap() + .items, + Vec::::new() + ); + assert_eq!(db.count_edges_by_label("MISSING").unwrap(), 0); + assert_eq!(db.get_node_label_id("Missing").unwrap(), None); + assert_eq!(db.get_edge_label_id("MISSING").unwrap(), None); + assert_eq!(db.list_node_labels().unwrap().len(), node_catalog_len); + assert_eq!(db.list_edge_labels().unwrap().len(), edge_catalog_len); + + assert!(db.nodes_by_labels(" Person").is_err()); + assert!(db.edges_by_label("KNOWS\n").is_err()); +} + +#[test] +fn test_public_label_property_and_time_queries_preserve_empty_and_validation_semantics() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut red_low = BTreeMap::new(); + red_low.insert("color".to_string(), PropValue::String("red".to_string())); + red_low.insert("score".to_string(), PropValue::Int(10)); + let mut red_high = BTreeMap::new(); + red_high.insert("color".to_string(), PropValue::String("red".to_string())); + red_high.insert("score".to_string(), PropValue::Int(20)); + let mut other = BTreeMap::new(); + other.insert("color".to_string(), PropValue::String("red".to_string())); + other.insert("score".to_string(), PropValue::Int(15)); + + let article_a = db + .upsert_node( + "Article", + "a", + UpsertNodeOptions { + props: red_low, + ..Default::default() + }, + ) + .unwrap(); + let article_b = db + .upsert_node( + "Article", + "b", + UpsertNodeOptions { + props: red_high, + ..Default::default() + }, + ) + .unwrap(); + db.upsert_node( + "Note", + "n", + UpsertNodeOptions { + props: other, + ..Default::default() + }, + ) + .unwrap(); + + let red = PropValue::String("red".to_string()); + assert_eq!( + db.find_nodes("Article", "color", &red).unwrap(), + vec![article_a, article_b] + ); + assert_eq!( + db.find_nodes_paged( + "Article", + "color", + &red, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap() + .items, + vec![article_a] + ); + + let lower = PropertyRangeBound::Included(PropValue::Int(10)); + let upper = PropertyRangeBound::Included(PropValue::Int(20)); + assert_eq!( + db.find_nodes_range("Article", "score", Some(&lower), Some(&upper)) + .unwrap(), + vec![article_a, article_b] + ); + let range_page = db + .find_nodes_range_paged( + "Article", + "score", + Some(&lower), + Some(&upper), + &PropertyRangePageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(range_page.items, vec![article_a]); + assert!(range_page.next_cursor.is_some()); + + assert_eq!( + db.find_nodes_by_time_range("Article", i64::MIN, i64::MAX) + .unwrap(), + vec![article_a, article_b] + ); + assert_eq!( + db.find_nodes_by_time_range_paged( + "Article", + i64::MIN, + i64::MAX, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap() + .items, + vec![article_a] + ); + + let catalog_len = db.list_node_labels().unwrap().len(); + assert_eq!(db.find_nodes("Missing", "color", &red).unwrap(), Vec::::new()); + assert_eq!( + db.find_nodes_paged("Missing", "color", &red, &PageRequest::default()) + .unwrap() + .items, + Vec::::new() + ); + assert_eq!( + db.find_nodes_range("Missing", "score", Some(&lower), Some(&upper)) + .unwrap(), + Vec::::new() + ); + assert_eq!( + db.find_nodes_range_paged( + "Missing", + "score", + Some(&lower), + Some(&upper), + &PropertyRangePageRequest::default(), + ) + .unwrap() + .items, + Vec::::new() + ); + assert_eq!( + db.find_nodes_by_time_range("Missing", i64::MIN, i64::MAX) + .unwrap(), + Vec::::new() + ); + assert_eq!( + db.find_nodes_by_time_range_paged( + "Missing", + i64::MIN, + i64::MAX, + &PageRequest::default(), + ) + .unwrap() + .items, + Vec::::new() + ); + assert_eq!(db.get_node_label_id("Missing").unwrap(), None); + assert_eq!(db.list_node_labels().unwrap().len(), catalog_len); + + let mixed_upper = PropertyRangeBound::Included(PropValue::Float(1.0)); + assert!( + db.find_nodes_range("Missing", "score", Some(&lower), Some(&mixed_upper)) + .is_err() + ); + assert!(db.find_nodes(" Article", "color", &red).is_err()); +} + +#[test] +fn test_property_index_apis_use_names_and_persist_metadata() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + + { + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let node_info = db + .ensure_node_property_index("Article", "status", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(node_info.label, "Article"); + assert_eq!(node_info.prop_key, "status"); + assert_eq!(db.get_node_label_id("Article").unwrap(), Some(1)); + + let edge_info = db + .ensure_edge_property_index("MENTIONS", "rank", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(edge_info.label, "MENTIONS"); + assert_eq!(edge_info.prop_key, "rank"); + assert_eq!(db.get_edge_label_id("MENTIONS").unwrap(), Some(1)); + + assert_eq!( + db.list_node_property_indexes() + .unwrap() + .iter() + .map(|info| (info.label.as_str(), info.prop_key.as_str())) + .collect::>(), + vec![("Article", "status")] + ); + assert_eq!( + db.list_edge_property_indexes() + .unwrap() + .iter() + .map(|info| (info.label.as_str(), info.prop_key.as_str())) + .collect::>(), + vec![("MENTIONS", "rank")] + ); + db.close().unwrap(); + } + + { + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(db.get_node_label_id("Article").unwrap(), Some(1)); + assert_eq!(db.get_edge_label_id("MENTIONS").unwrap(), Some(1)); + assert_eq!( + db.list_node_property_indexes() + .unwrap() + .iter() + .map(|info| (info.label.as_str(), info.prop_key.as_str())) + .collect::>(), + vec![("Article", "status")] + ); + assert_eq!( + db.list_edge_property_indexes() + .unwrap() + .iter() + .map(|info| (info.label.as_str(), info.prop_key.as_str())) + .collect::>(), + vec![("MENTIONS", "rank")] + ); + db.close().unwrap(); + } +} + +#[test] +fn test_property_index_drop_unknown_label_is_read_only() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + assert!(!db + .drop_node_property_index("Missing", "status", SecondaryIndexKind::Equality) + .unwrap()); + assert!(!db + .drop_edge_property_index("MISSING", "status", SecondaryIndexKind::Equality) + .unwrap()); + assert_eq!(db.get_node_label_id("Missing").unwrap(), None); + assert_eq!(db.get_edge_label_id("MISSING").unwrap(), None); + assert!(db + .drop_node_property_index(" Missing", "status", SecondaryIndexKind::Equality) + .is_err()); + assert!(db + .drop_edge_property_index("MISSING\n", "status", SecondaryIndexKind::Equality) + .is_err()); +} + +#[test] +fn test_prune_policy_apis_use_names_and_persist_metadata() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + + { + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + db.set_prune_policy( + "article-low-weight", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.25), + label: Some("Article".to_string()), + }, + ) + .unwrap(); + assert_eq!(db.get_node_label_id("Article").unwrap(), Some(1)); + + let policies = db.list_prune_policies().unwrap(); + assert_eq!(policies.len(), 1); + assert_eq!(policies[0].name, "article-low-weight"); + assert_eq!(policies[0].policy.label.as_deref(), Some("Article")); + assert_eq!(policies[0].policy.max_weight, Some(0.25)); + db.close().unwrap(); + } + + { + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(db.get_node_label_id("Article").unwrap(), Some(1)); + let policies = db.list_prune_policies().unwrap(); + assert_eq!(policies.len(), 1); + assert_eq!(policies[0].name, "article-low-weight"); + assert_eq!(policies[0].policy.label.as_deref(), Some("Article")); + db.close().unwrap(); + } +} + +#[test] +fn test_prune_policy_validation_does_not_leak_tokens() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let err = db + .set_prune_policy( + "invalid-name", + PrunePolicy { + max_age_ms: None, + max_weight: None, + label: Some(" LeakyPolicy".to_string()), + }, + ) + .unwrap_err(); + match err { + EngineError::InvalidOperation(message) => { + assert!(message.contains("leading or trailing whitespace")); + } + other => panic!("expected invalid label error, got {other:?}"), + } + assert_eq!(db.get_node_label_id("LeakyPolicy").unwrap(), None); + assert!( + db.get_node_label_id(" LeakyPolicy") + .unwrap_err() + .to_string() + .contains("leading or trailing whitespace") + ); + + let err = db + .set_prune_policy( + "invalid", + PrunePolicy { + max_age_ms: None, + max_weight: None, + label: Some("LeakyPolicy".to_string()), + }, + ) + .unwrap_err(); + assert!(matches!(err, EngineError::InvalidOperation(_))); + assert_eq!(db.get_node_label_id("LeakyPolicy").unwrap(), None); + assert!(db.list_prune_policies().unwrap().is_empty()); +} + +#[test] +fn test_open_rejects_prune_policy_with_missing_label_token() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + std::fs::create_dir_all(&db_path).unwrap(); + let mut manifest = default_manifest(); + manifest.prune_policies.insert( + "broken".to_string(), + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.1), + label: Some("Missing".to_string()), + }, + ); + write_manifest(&db_path, &manifest).unwrap(); + + match DatabaseEngine::open(&db_path, &DbOptions::default()) { + Ok(_) => panic!("open should reject prune policy with missing label token"), + Err(EngineError::ManifestError(message)) => { + assert!(message.contains("prune policy references missing node label")); + } + Err(other) => panic!("expected manifest error, got {other:?}"), + } +} + +#[test] +fn test_prune_unknown_label_scope_creates_token_without_deleting() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("catalog_db"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let result = db + .prune(&PrunePolicy { + max_age_ms: None, + max_weight: Some(0.1), + label: Some("MissingButValid".to_string()), + }) + .unwrap(); + assert_eq!(result.nodes_pruned, 0); + assert_eq!(result.edges_pruned, 0); + assert_eq!(db.get_node_label_id("MissingButValid").unwrap(), Some(1)); +} diff --git a/src/engine/tests/lifecycle.rs b/src/engine/tests/lifecycle.rs index 48a13ee..455e794 100644 --- a/src/engine/tests/lifecycle.rs +++ b/src/engine/tests/lifecycle.rs @@ -1,13 +1,22 @@ // Lifecycle tests: open/close, WAL, flush, compaction, restart, group commit, backpressure. -type LegacyNode = (u64, u32, Vec<(String, PropValue)>); +fn lifecycle_filter_names(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_string()).collect() +} + +fn lifecycle_node_label_filter(names: &[&str], mode: LabelMatchMode) -> NodeLabelFilter { + NodeLabelFilter { + labels: lifecycle_filter_names(names), + mode, + } +} fn traverse_depth_two( engine: &DatabaseEngine, start: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, - node_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[&str]>, + node_label_filter: Option<&[&str]>, limit: usize, at_epoch: Option, ) -> Vec { @@ -18,8 +27,9 @@ fn traverse_depth_two( &TraverseOptions { min_depth: 2, direction, - edge_type_filter: edge_type_filter.map(|s| s.to_vec()), - node_type_filter: node_type_filter.map(|s| s.to_vec()), + edge_label_filter: edge_label_filter.map(lifecycle_filter_names), + emit_node_label_filter: node_label_filter + .map(|labels| lifecycle_node_label_filter(labels, LabelMatchMode::Any)), at_epoch, decay_lambda: None, limit: (limit > 0).then_some(limit), @@ -57,6 +67,34 @@ fn wait_for_property_index_state( } } +fn wait_for_edge_property_index_state( + engine: &DatabaseEngine, + index_id: u64, + expected_state: SecondaryIndexState, +) -> EdgePropertyIndexInfo { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + if let Some(info) = engine + .list_edge_property_indexes() + .unwrap() + .into_iter() + .find(|info| info.index_id == index_id) + { + if info.state == expected_state { + return info; + } + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for edge property index {} to reach {:?}; current indexes: {:?}", + index_id, + expected_state, + engine.list_edge_property_indexes().unwrap() + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } +} + fn wait_for_published_property_index_state( engine: &DatabaseEngine, index_id: u64, @@ -94,19 +132,271 @@ fn wait_for_published_property_index_state( fn corrupt_sidecar_header_in_place(path: &std::path::Path) { use std::io::{Seek, SeekFrom, Write}; + let payload_offset = component_payload_offset_for_test(path); let mut file = std::fs::OpenOptions::new() .write(true) .open(path) .unwrap(); assert!( - file.metadata().unwrap().len() >= 8, + file.metadata().unwrap().len() >= payload_offset + 8, "sidecar must be large enough for an in-place header corruption" ); - file.seek(SeekFrom::Start(0)).unwrap(); + file.seek(SeekFrom::Start(payload_offset)).unwrap(); file.write_all(&1_000_000u64.to_le_bytes()).unwrap(); file.sync_all().unwrap(); } +fn component_payload_offset_for_test(path: &std::path::Path) -> u64 { + let data = std::fs::read(path).unwrap(); + if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] + == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC + { + crate::segment_components::decode_identity_header(&data) + .unwrap() + .payload_offset + } else { + 0 + } +} + +fn corrupt_equality_sidecar_tail_group_order_in_place(path: &std::path::Path, value_hash: u64) { + use std::io::{Seek, SeekFrom, Write}; + + const SECONDARY_EQ_ENTRY_SIZE: usize = 20; + let data = std::fs::read(path).unwrap(); + let payload_offset = component_payload_offset_for_test(path) as usize; + let payload = &data[payload_offset..]; + assert!(payload.len() >= 8, "equality sidecar payload missing count"); + let count = u64::from_le_bytes(payload[0..8].try_into().unwrap()) as usize; + let index_bytes = 8 + count * SECONDARY_EQ_ENTRY_SIZE; + assert!( + payload.len() >= index_bytes, + "equality sidecar payload missing index" + ); + + for index in 0..count { + let entry_off = 8 + index * SECONDARY_EQ_ENTRY_SIZE; + let entry_value_hash = + u64::from_le_bytes(payload[entry_off..entry_off + 8].try_into().unwrap()); + if entry_value_hash != value_hash { + continue; + } + let group_offset = + u64::from_le_bytes(payload[entry_off + 8..entry_off + 16].try_into().unwrap()) + as usize; + let id_count = + u32::from_le_bytes(payload[entry_off + 16..entry_off + 20].try_into().unwrap()) + as usize; + assert!( + id_count >= 2, + "target equality sidecar group must have at least two IDs" + ); + assert!( + payload.len() >= group_offset + 16, + "target equality sidecar group missing first two IDs" + ); + let first = u64::from_le_bytes(payload[group_offset..group_offset + 8].try_into().unwrap()); + let second = + u64::from_le_bytes(payload[group_offset + 8..group_offset + 16].try_into().unwrap()); + assert!( + first < second, + "test setup expects initially sorted equality postings" + ); + + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start((payload_offset + group_offset) as u64)) + .unwrap(); + file.write_all(&second.to_le_bytes()).unwrap(); + file.write_all(&first.to_le_bytes()).unwrap(); + file.sync_all().unwrap(); + return; + } + + panic!("target equality sidecar group hash {value_hash} not found"); +} + +fn corrupt_range_sidecar_tail_sort_order_in_place(path: &std::path::Path) { + use std::io::{Seek, SeekFrom, Write}; + + const SECONDARY_RANGE_ENTRY_SIZE: usize = 16; + let data = std::fs::read(path).unwrap(); + let payload_offset = component_payload_offset_for_test(path) as usize; + let payload = &data[payload_offset..]; + assert!(payload.len() >= 8, "range sidecar payload missing count"); + let count = u64::from_le_bytes(payload[0..8].try_into().unwrap()) as usize; + assert!( + count >= 3, + "range sidecar test needs at least three entries to corrupt an unqueried tail" + ); + let index_bytes = 8 + count * SECONDARY_RANGE_ENTRY_SIZE; + assert!( + payload.len() >= index_bytes, + "range sidecar payload missing fixed entries" + ); + let previous_off = 8 + (count - 2) * SECONDARY_RANGE_ENTRY_SIZE; + let tail_off = 8 + (count - 1) * SECONDARY_RANGE_ENTRY_SIZE; + let previous_encoded = + u64::from_le_bytes(payload[previous_off..previous_off + 8].try_into().unwrap()); + + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start((payload_offset + tail_off) as u64)) + .unwrap(); + file.write_all(&previous_encoded.to_le_bytes()).unwrap(); + file.sync_all().unwrap(); +} + +fn component_payload_from_bytes_for_test(data: Vec) -> Vec { + if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] + == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + return data[start..end].to_vec(); + } + data +} + +fn try_read_manifest_component_payload_for_test( + seg_dir: &std::path::Path, + kind: SegmentComponentKind, +) -> Option> { + let manifest = read_component_manifest_for_test(seg_dir); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind)?; + match &record.handle { + crate::segment_components::ComponentHandleV1::ExternalFile { relative_path, .. } => { + let data = std::fs::read(seg_dir.join(relative_path)).unwrap(); + Some(component_payload_from_bytes_for_test(data)) + } + crate::segment_components::ComponentHandleV1::PackedRange { offset, len, .. } => { + let core = std::fs::read(seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME)) + .unwrap(); + let core_payload = component_payload_from_bytes_for_test(core); + let start = *offset as usize; + let end = start + *len as usize; + Some(core_payload[start..end].to_vec()) + } + } +} + +fn read_external_component_payload_for_test(path: &std::path::Path) -> Vec { + let data = std::fs::read(path) + .unwrap_or_else(|error| panic!("failed to read {}: {}", path.display(), error)); + component_payload_from_bytes_for_test(data) +} + +fn read_manifest_component_payload_for_test( + seg_dir: &std::path::Path, + kind: SegmentComponentKind, +) -> Vec { + let missing_kind = kind.clone(); + try_read_manifest_component_payload_for_test(seg_dir, kind) + .unwrap_or_else(|| panic!("missing component payload {:?}", missing_kind)) +} + +fn read_component_manifest_for_test( + seg_dir: &std::path::Path, +) -> crate::segment_components::SegmentComponentManifestV1 { + let data = + std::fs::read(seg_dir.join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME)) + .unwrap(); + crate::segment_components::decode_manifest_envelope(&data).unwrap() +} + +#[derive(Debug, Clone)] +struct PackedCoreSnapshot { + bytes: Vec, + len: u64, + modified: std::time::SystemTime, +} + +fn packed_core_snapshot_for_test(seg_dir: &std::path::Path) -> PackedCoreSnapshot { + let path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); + let metadata = std::fs::metadata(&path) + .unwrap_or_else(|error| panic!("failed to stat {}: {}", path.display(), error)); + let bytes = std::fs::read(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {}", path.display(), error)); + assert_eq!( + metadata.len(), + bytes.len() as u64, + "segment.core metadata length should match bytes read" + ); + PackedCoreSnapshot { + bytes, + len: metadata.len(), + modified: metadata.modified().expect("segment.core modified time"), + } +} + +fn assert_packed_core_unchanged_for_test( + seg_dir: &std::path::Path, + before: &PackedCoreSnapshot, + label: &str, +) { + let after = packed_core_snapshot_for_test(seg_dir); + assert_eq!(after.len, before.len, "{label} changed segment.core length"); + assert_eq!( + after.modified, before.modified, + "{label} changed segment.core mtime" + ); + assert_eq!( + after.bytes, before.bytes, + "{label} changed segment.core bytes" + ); +} + +fn assert_no_legacy_property_components(seg_dir: &std::path::Path) { + let manifest = read_component_manifest_for_test(seg_dir); + assert!(manifest.components.iter().all(|record| { + !matches!( + record.kind, + SegmentComponentKind::LegacyNodePropertyIndex + | SegmentComponentKind::NodePropertyHashMetadata + ) + })); +} + +fn read_u64_le_for_test(data: &[u8], offset: usize, label: &str) -> u64 { + let end = offset + .checked_add(8) + .unwrap_or_else(|| panic!("{label} u64 offset overflow at {offset}")); + let bytes = data + .get(offset..end) + .unwrap_or_else(|| panic!("{label} missing u64 at [{offset}, {end})")); + u64::from_le_bytes(bytes.try_into().unwrap()) +} + +fn read_u32_le_for_test(data: &[u8], offset: usize, label: &str) -> u32 { + let end = offset + .checked_add(4) + .unwrap_or_else(|| panic!("{label} u32 offset overflow at {offset}")); + let bytes = data + .get(offset..end) + .unwrap_or_else(|| panic!("{label} missing u32 at [{offset}, {end})")); + u32::from_le_bytes(bytes.try_into().unwrap()) +} + +fn read_u16_le_for_test(data: &[u8], offset: usize, label: &str) -> u16 { + let end = offset + .checked_add(2) + .unwrap_or_else(|| panic!("{label} u16 offset overflow at {offset}")); + let bytes = data + .get(offset..end) + .unwrap_or_else(|| panic!("{label} missing u16 at [{offset}, {end})")); + u16::from_le_bytes(bytes.try_into().unwrap()) +} + fn wait_for_pending_secondary_index_followup_count( engine: &DatabaseEngine, expected_count: usize, @@ -139,71 +429,6 @@ fn wait_for_path_absent(path: &std::path::Path) { } } -fn install_legacy_property_hash_sidecars( - seg_dir: &std::path::Path, - nodes: &[LegacyNode], -) { - const LEGACY_NODE_META_ENTRY_SIZE: usize = 60; - const LEGACY_PROP_INDEX_ENTRY_SIZE: usize = 32; - - let mut sorted_nodes = nodes.to_vec(); - sorted_nodes.sort_unstable_by_key(|(node_id, _, _)| *node_id); - - let mut node_meta = std::fs::read(seg_dir.join("node_meta.dat")).unwrap(); - let node_count = u64::from_le_bytes(node_meta[0..8].try_into().unwrap()) as usize; - assert_eq!(node_count, sorted_nodes.len()); - - let mut prop_hash_bytes = Vec::new(); - let mut prop_hash_offset = 0u64; - let mut prop_groups: BTreeMap<(u32, u64, u64), Vec> = BTreeMap::new(); - - for (index, (node_id, type_id, props)) in sorted_nodes.iter().enumerate() { - let entry_off = 8 + index * LEGACY_NODE_META_ENTRY_SIZE; - let prop_hash_count = props.len() as u32; - node_meta[entry_off + 38..entry_off + 46].copy_from_slice(&prop_hash_offset.to_le_bytes()); - node_meta[entry_off + 46..entry_off + 50].copy_from_slice(&prop_hash_count.to_le_bytes()); - - for (key, value) in props { - let key_hash = hash_prop_key(key); - let value_hash = hash_prop_value(value); - prop_hash_bytes.extend_from_slice(&key_hash.to_le_bytes()); - prop_hash_bytes.extend_from_slice(&value_hash.to_le_bytes()); - prop_groups - .entry((*type_id, key_hash, value_hash)) - .or_default() - .push(*node_id); - prop_hash_offset += 16; - } - } - - for ids in prop_groups.values_mut() { - ids.sort_unstable(); - ids.dedup(); - } - - std::fs::write(seg_dir.join("node_meta.dat"), node_meta).unwrap(); - std::fs::write(seg_dir.join("node_prop_hashes.dat"), prop_hash_bytes).unwrap(); - - let mut prop_index = Vec::new(); - prop_index.extend_from_slice(&(prop_groups.len() as u64).to_le_bytes()); - let data_start = 8 + prop_groups.len() as u64 * LEGACY_PROP_INDEX_ENTRY_SIZE as u64; - let mut data_offset = data_start; - for ((type_id, key_hash, value_hash), ids) in &prop_groups { - prop_index.extend_from_slice(&type_id.to_le_bytes()); - prop_index.extend_from_slice(&key_hash.to_le_bytes()); - prop_index.extend_from_slice(&value_hash.to_le_bytes()); - prop_index.extend_from_slice(&data_offset.to_le_bytes()); - prop_index.extend_from_slice(&(ids.len() as u32).to_le_bytes()); - data_offset += ids.len() as u64 * 8; - } - for ids in prop_groups.values() { - for node_id in ids { - prop_index.extend_from_slice(&node_id.to_le_bytes()); - } - } - std::fs::write(seg_dir.join("prop_index.dat"), prop_index).unwrap(); -} - // --- Low-level write_op API tests --- #[test] @@ -299,11 +524,20 @@ fn test_open_canonicalizes_vector_payloads_from_wal_replay() { // Write directly to the active WAL generation file (gen 0) let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); writer .append( &WalOp::UpsertNode(NodeRecord { id: 7, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "manual-vector".to_string(), props: BTreeMap::new(), created_at: 100, @@ -313,7 +547,7 @@ fn test_open_canonicalizes_vector_payloads_from_wal_replay() { sparse_vector: Some(vec![(4, 0.25), (2, 2.0), (4, 0.5), (7, 0.0)]), last_write_seq: 0, }), - 1, + 2, ) .unwrap(); writer.sync().unwrap(); @@ -327,7 +561,7 @@ fn test_open_canonicalizes_vector_payloads_from_wal_replay() { } #[test] -fn test_open_rejects_compacted_dense_segment_missing_hnsw_graph() { +fn test_open_tolerates_compacted_dense_segment_missing_hnsw_graph() { let dir = TempDir::new().unwrap(); let opts = DbOptions { compact_after_n_flushes: 0, @@ -342,7 +576,7 @@ fn test_open_rejects_compacted_dense_segment_missing_hnsw_graph() { engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -354,7 +588,7 @@ fn test_open_rejects_compacted_dense_segment_missing_hnsw_graph() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -372,11 +606,9 @@ fn test_open_rejects_compacted_dense_segment_missing_hnsw_graph() { let seg_dir = crate::segment_writer::segment_dir(dir.path(), seg_id); std::fs::remove_file(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - match DatabaseEngine::open(dir.path(), &opts) { - Err(EngineError::CorruptRecord(_)) => {} - Err(other) => panic!("expected CorruptRecord, got {}", other), - Ok(_) => panic!("expected reopen to fail for missing dense HNSW graph"), - } + let reopened = DatabaseEngine::open(dir.path(), &opts).unwrap(); + assert!(reopened.get_node_by_key("Person", "b").unwrap().is_some()); + reopened.close().unwrap(); } #[test] @@ -395,7 +627,7 @@ fn test_open_rejects_compacted_dense_segment_truncated_vector_blob() { engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -407,7 +639,7 @@ fn test_open_rejects_compacted_dense_segment_truncated_vector_blob() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -423,10 +655,29 @@ fn test_open_rejects_compacted_dense_segment_truncated_vector_blob() { drop(engine); let seg_dir = crate::segment_writer::segment_dir(dir.path(), seg_id); - let dense_blob_path = seg_dir.join(crate::segment_writer::NODE_DENSE_VECTOR_BLOB_FILENAME); - let mut dense_blob = std::fs::read(&dense_blob_path).unwrap(); - dense_blob.truncate(dense_blob.len() - 4); - std::fs::write(&dense_blob_path, dense_blob).unwrap(); + let manifest = read_component_manifest_for_test(&seg_dir); + let record = manifest + .components + .iter() + .find(|record| record.kind == SegmentComponentKind::NodeDenseVectorBlob) + .expect("dense vector blob component should exist"); + let crate::segment_components::ComponentHandleV1::PackedRange { offset, len, .. } = + &record.handle + else { + panic!("compacted dense vector source truth should be packed"); + }; + assert!(*len >= 4); + let core_path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); + let truncated_payload_len = *offset + *len - 4; + let truncated_file_len = crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN as u64 + + truncated_payload_len; + assert!(truncated_file_len < std::fs::metadata(&core_path).unwrap().len()); + std::fs::OpenOptions::new() + .write(true) + .open(&core_path) + .unwrap() + .set_len(truncated_file_len) + .unwrap(); match DatabaseEngine::open(dir.path(), &opts) { Err(EngineError::CorruptRecord(_)) => {} @@ -436,7 +687,7 @@ fn test_open_rejects_compacted_dense_segment_truncated_vector_blob() { } #[test] -fn test_open_rejects_standard_compacted_dense_segment_missing_hnsw_graph() { +fn test_open_tolerates_standard_compacted_dense_segment_missing_hnsw_graph() { let dir = TempDir::new().unwrap(); let opts = DbOptions { compact_after_n_flushes: 0, @@ -451,7 +702,7 @@ fn test_open_rejects_standard_compacted_dense_segment_missing_hnsw_graph() { let node_id = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.5, @@ -463,7 +714,7 @@ fn test_open_rejects_standard_compacted_dense_segment_missing_hnsw_graph() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.75, @@ -476,7 +727,7 @@ fn test_open_rejects_standard_compacted_dense_segment_missing_hnsw_graph() { assert_eq!( node_id, - engine.get_node_by_key(1, "shared").unwrap().unwrap().id + engine.get_node_by_key("Person", "shared").unwrap().unwrap().id ); assert_eq!(compaction_path_for(&engine), CompactionPath::UnifiedV3); @@ -488,11 +739,12 @@ fn test_open_rejects_standard_compacted_dense_segment_missing_hnsw_graph() { let seg_dir = crate::segment_writer::segment_dir(dir.path(), seg_id); std::fs::remove_file(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - match DatabaseEngine::open(dir.path(), &opts) { - Err(EngineError::CorruptRecord(_)) => {} - Err(other) => panic!("expected CorruptRecord, got {}", other), - Ok(_) => panic!("expected reopen to fail for standard-compacted dense segment"), - } + let reopened = DatabaseEngine::open(dir.path(), &opts).unwrap(); + assert_eq!( + node_id, + reopened.get_node_by_key("Person", "shared").unwrap().unwrap().id + ); + reopened.close().unwrap(); } #[test] @@ -519,7 +771,7 @@ fn test_open_rejects_invalid_vector_payloads_from_wal_replay() { .append( &WalOp::UpsertNode(NodeRecord { id: 8, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "bad-vector".to_string(), props: BTreeMap::new(), created_at: 100, @@ -567,7 +819,7 @@ fn test_open_rejects_malformed_vector_frame_from_wal_replay() { // V3 WAL frame: [seq:u64][walop_bytes]. The whole thing is the CRC-protected payload. let walop_bytes = crate::encoding::encode_wal_op(&WalOp::UpsertNode(NodeRecord { id: 9, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "bad-frame".to_string(), props: BTreeMap::new(), created_at: 100, @@ -613,14 +865,11 @@ fn test_write_and_read_back() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(1, "alice"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(1, "alice"))) .unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(2, "bob"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(2, "bob"))) .unwrap(); - engine - .write_op(&WalOp::UpsertEdge(make_edge(1, 1, 2))) + write_internal_wal_op(&engine, &WalOp::UpsertEdge(make_edge(1, 1, 2))) .unwrap(); assert_eq!(engine.node_count().unwrap(), 2); @@ -643,24 +892,20 @@ fn test_delete_operations() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(1, "alice"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(1, "alice"))) .unwrap(); - engine - .write_op(&WalOp::UpsertEdge(make_edge(1, 1, 1))) + write_internal_wal_op(&engine, &WalOp::UpsertEdge(make_edge(1, 1, 1))) .unwrap(); assert!(engine.get_node(1).unwrap().is_some()); assert!(engine.get_edge(1).unwrap().is_some()); - engine - .write_op(&WalOp::DeleteNode { + write_internal_wal_op(&engine, &WalOp::DeleteNode { id: 1, deleted_at: 9999, }) .unwrap(); - engine - .write_op(&WalOp::DeleteEdge { + write_internal_wal_op(&engine, &WalOp::DeleteEdge { id: 1, deleted_at: 9999, }) @@ -682,13 +927,11 @@ fn test_close_and_reopen_recovers_state() { { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); for i in 1..=10 { - engine - .write_op(&WalOp::UpsertNode(make_node(i, &format!("node:{}", i)))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(i, &format!("node:{}", i)))) .unwrap(); } for i in 1..=5 { - engine - .write_op(&WalOp::UpsertEdge(make_edge(i, i, i + 5))) + write_internal_wal_op(&engine, &WalOp::UpsertEdge(make_edge(i, i, i + 5))) .unwrap(); } engine.close().unwrap(); @@ -731,11 +974,9 @@ fn test_manifest_id_counters_survive_restart() { { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(42, "high_id"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(42, "high_id"))) .unwrap(); - engine - .write_op(&WalOp::UpsertEdge(make_edge(99, 42, 42))) + write_internal_wal_op(&engine, &WalOp::UpsertEdge(make_edge(99, 42, 42))) .unwrap(); engine.close().unwrap(); } @@ -755,14 +996,11 @@ fn test_wal_replay_with_deletes() { { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(1, "will_delete"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(1, "will_delete"))) .unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(2, "will_keep"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(2, "will_keep"))) .unwrap(); - engine - .write_op(&WalOp::DeleteNode { + write_internal_wal_op(&engine, &WalOp::DeleteNode { id: 1, deleted_at: 5000, }) @@ -774,8 +1012,8 @@ fn test_wal_replay_with_deletes() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); assert!(engine.get_node(1).unwrap().is_none()); assert!(engine.get_node(2).unwrap().is_some()); - // After close() flushes to segments, use get_nodes_by_type for total count - assert_eq!(engine.get_nodes_by_type(1).unwrap().len(), 1); + // After close() flushes to segments, use get_nodes_by_labels for total count + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 1); engine.close().unwrap(); } } @@ -790,16 +1028,16 @@ fn test_write_op_batch() { let ops: Vec = (1..=50) .map(|i| WalOp::UpsertNode(make_node(i, &format!("batch:{}", i)))) .collect(); - engine.write_op_batch(&ops).unwrap(); + write_internal_wal_op_batch(&engine, &ops).unwrap(); assert_eq!(engine.node_count().unwrap(), 50); assert_eq!(engine.get_node(25).unwrap().unwrap().key, "batch:25"); engine.close().unwrap(); - // Verify recovery (close flushes to segments, use get_nodes_by_type) + // Verify recovery (close flushes to segments, use get_nodes_by_labels) let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert_eq!(engine.get_nodes_by_type(1).unwrap().len(), 50); + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 50); engine.close().unwrap(); } @@ -817,14 +1055,14 @@ fn test_write_op_batch_survives_restart() { for i in 1..=10 { ops.push(WalOp::UpsertEdge(make_edge(i, i, i + 10))); } - engine.write_op_batch(&ops).unwrap(); + write_internal_wal_op_batch(&engine, &ops).unwrap(); engine.close().unwrap(); } { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // close() flushes to segments; use cross-source counts - assert_eq!(engine.get_nodes_by_type(1).unwrap().len(), 20); + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 20); // Verify edges individually (edge_count is memtable-only) for i in 1..=10 { assert!(engine.get_edge(i).unwrap().is_some(), "edge {} missing", i); @@ -850,7 +1088,7 @@ fn test_write_op_batch_normalizes_node_vectors() { let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); let ops = vec![WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "vector-batch".to_string(), props: BTreeMap::new(), created_at: 100, @@ -860,7 +1098,7 @@ fn test_write_op_batch_normalizes_node_vectors() { sparse_vector: Some(vec![(8, 0.0), (3, 1.0), (3, 2.0)]), last_write_seq: 0, })]; - engine.write_op_batch(&ops).unwrap(); + write_internal_wal_op_batch(&engine, &ops).unwrap(); let node = engine.get_node(1).unwrap().unwrap(); assert_eq!(node.dense_vector, Some(vec![0.1, 0.2])); @@ -882,12 +1120,11 @@ fn test_upsert_overwrites_on_replay() { { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(1, "v1"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(1, "v1"))) .unwrap(); let mut updated = make_node(1, "v2"); updated.weight = 0.99; - engine.write_op(&WalOp::UpsertNode(updated)).unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(updated)).unwrap(); engine.close().unwrap(); } @@ -897,7 +1134,7 @@ fn test_upsert_overwrites_on_replay() { assert_eq!(node.key, "v2"); assert!((node.weight - 0.99).abs() < f32::EPSILON); // close() flushes to segments; use cross-source count - assert_eq!(engine.get_nodes_by_type(1).unwrap().len(), 1); + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 1); engine.close().unwrap(); } } @@ -913,7 +1150,7 @@ fn test_flush_creates_segment() { engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -923,7 +1160,7 @@ fn test_flush_creates_segment() { .unwrap(); engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -966,7 +1203,7 @@ fn test_data_readable_after_flush() { let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -976,7 +1213,7 @@ fn test_data_readable_after_flush() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -985,7 +1222,7 @@ fn test_data_readable_after_flush() { ) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -1011,7 +1248,7 @@ fn test_neighbors_after_flush() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1021,7 +1258,7 @@ fn test_neighbors_after_flush() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1031,7 +1268,7 @@ fn test_neighbors_after_flush() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -1040,13 +1277,13 @@ fn test_neighbors_after_flush() { ) .unwrap(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine .upsert_edge( a, c, - 20, + "REPORTS_TO", UpsertEdgeOptions { weight: 0.8, ..Default::default() @@ -1062,18 +1299,18 @@ fn test_neighbors_after_flush() { assert!(ids.contains(&b)); assert!(ids.contains(&c)); - // Type filter should still work - let typed = engine + // Edge-label filter should still work + let labeled = engine .neighbors( a, &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) .unwrap(); - assert_eq!(typed.len(), 1); - assert_eq!(typed[0].node_id, b); + assert_eq!(labeled.len(), 1); + assert_eq!(labeled[0].node_id, b); engine.close().unwrap(); } @@ -1088,7 +1325,7 @@ fn test_traverse_depth_two_reproduces_basic_two_hop() { // Build chain: a -> b -> c -> d let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1098,7 +1335,7 @@ fn test_traverse_depth_two_reproduces_basic_two_hop() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1108,7 +1345,7 @@ fn test_traverse_depth_two_reproduces_basic_two_hop() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -1118,7 +1355,7 @@ fn test_traverse_depth_two_reproduces_basic_two_hop() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.5, @@ -1127,13 +1364,13 @@ fn test_traverse_depth_two_reproduces_basic_two_hop() { ) .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // 2-hop from a: should reach c (via b), but NOT d (3 hops) or a/b (origin/1-hop) @@ -1154,7 +1391,7 @@ fn test_traverse_depth_two_excludes_origin_and_hop1() { // Build graph with back-edge: a -> b -> a (cycle) let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1164,7 +1401,7 @@ fn test_traverse_depth_two_excludes_origin_and_hop1() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1174,7 +1411,7 @@ fn test_traverse_depth_two_excludes_origin_and_hop1() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -1183,13 +1420,13 @@ fn test_traverse_depth_two_excludes_origin_and_hop1() { ) .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, a, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // back to origin engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // 2-hop from a: b is 1-hop, then from b we reach a (origin, excluded) and c @@ -1210,7 +1447,7 @@ fn test_traverse_depth_two_respects_limit() { // a -> b, a -> c, b -> d, b -> e, c -> f let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1220,7 +1457,7 @@ fn test_traverse_depth_two_respects_limit() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1230,7 +1467,7 @@ fn test_traverse_depth_two_respects_limit() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -1240,7 +1477,7 @@ fn test_traverse_depth_two_respects_limit() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.5, @@ -1250,7 +1487,7 @@ fn test_traverse_depth_two_respects_limit() { .unwrap(); let e = engine .upsert_node( - 1, + "Person", "e", UpsertNodeOptions { weight: 0.5, @@ -1260,7 +1497,7 @@ fn test_traverse_depth_two_respects_limit() { .unwrap(); let f = engine .upsert_node( - 1, + "Person", "f", UpsertNodeOptions { weight: 0.5, @@ -1269,19 +1506,19 @@ fn test_traverse_depth_two_respects_limit() { ) .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, e, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, e, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, f, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, f, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // Without limit: 3 2-hop results (d, e, f) @@ -1296,16 +1533,16 @@ fn test_traverse_depth_two_respects_limit() { } #[test] -fn test_traverse_depth_two_respects_edge_type_filter() { +fn test_traverse_depth_two_respects_edge_label_filter() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // a -[type1]-> b -[type1]-> c, b -[type2]-> d + // a -[label1]-> b -[label1]-> c, b -[label2]-> d let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1315,7 +1552,7 @@ fn test_traverse_depth_two_respects_edge_type_filter() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1325,7 +1562,7 @@ fn test_traverse_depth_two_respects_edge_type_filter() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -1335,7 +1572,7 @@ fn test_traverse_depth_two_respects_edge_type_filter() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.5, @@ -1344,17 +1581,25 @@ fn test_traverse_depth_two_respects_edge_type_filter() { ) .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, d, 2, UpsertEdgeOptions::default()) + .upsert_edge(b, d, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); - // Filter type 1 only: a->b (hop1), b->c (hop2). b->d is type 2, excluded. - let hop2 = traverse_depth_two(&engine, a, Direction::Outgoing, Some(&[1]), None, 0, None); + // Filter RELATES_TO only: a->b (hop1), b->c (hop2). b->d is WORKS_AT, excluded. + let hop2 = traverse_depth_two( + &engine, + a, + Direction::Outgoing, + Some(&["RELATES_TO"]), + None, + 0, + None, + ); assert_eq!(hop2.len(), 1); assert_eq!(hop2[0].node_id, c); @@ -1371,7 +1616,7 @@ fn test_traverse_depth_two_incoming() { // Chain: a -> b -> c -> d (incoming 2-hop from d should reach b) let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1381,7 +1626,7 @@ fn test_traverse_depth_two_incoming() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1391,7 +1636,7 @@ fn test_traverse_depth_two_incoming() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -1401,7 +1646,7 @@ fn test_traverse_depth_two_incoming() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.5, @@ -1410,13 +1655,13 @@ fn test_traverse_depth_two_incoming() { ) .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(c, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // Incoming 2-hop from d: hop1 = c, hop2 = b (not a, that's 3 hops) @@ -1441,7 +1686,7 @@ fn test_traverse_depth_two_nonexistent_or_hidden_start() { // Add a node but delete it, same result let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1451,7 +1696,7 @@ fn test_traverse_depth_two_nonexistent_or_hidden_start() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1460,7 +1705,7 @@ fn test_traverse_depth_two_nonexistent_or_hidden_start() { ) .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.delete_node(a).unwrap(); @@ -1480,7 +1725,7 @@ fn test_cross_source_reads_memtable_plus_segment() { // Write batch 1, flush to segment let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -1490,7 +1735,7 @@ fn test_cross_source_reads_memtable_plus_segment() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -1499,14 +1744,14 @@ fn test_cross_source_reads_memtable_plus_segment() { ) .unwrap(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // Write batch 2, stays in memtable let c = engine .upsert_node( - 1, + "Person", "charlie", UpsertNodeOptions { weight: 0.7, @@ -1518,7 +1763,7 @@ fn test_cross_source_reads_memtable_plus_segment() { .upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.9, ..Default::default() @@ -1550,7 +1795,7 @@ fn test_upsert_dedup_across_flush_boundary() { // Insert and flush let id1 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -1560,12 +1805,12 @@ fn test_upsert_dedup_across_flush_boundary() { .unwrap(); engine.flush().unwrap(); - // Upsert same (type_id, key), should find existing in segment + // Upsert same (label, key), should find existing in segment. let mut props = BTreeMap::new(); props.insert("version".to_string(), PropValue::Int(2)); let id2 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props, @@ -1595,7 +1840,7 @@ fn test_tombstone_hides_segment_data() { let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -1605,7 +1850,7 @@ fn test_tombstone_hides_segment_data() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -1614,7 +1859,7 @@ fn test_tombstone_hides_segment_data() { ) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -1641,7 +1886,7 @@ fn test_tombstone_survives_second_flush() { let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -1671,7 +1916,7 @@ fn test_multiple_flushes_accumulate_segments() { for i in 0..3 { let id = engine .upsert_node( - 1, + "Person", &format!("batch:{}", i), UpsertNodeOptions { weight: 0.5, @@ -1703,7 +1948,7 @@ fn test_flush_updates_manifest() { engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -1730,7 +1975,7 @@ fn test_id_counters_survive_flush() { for i in 0..5 { engine .upsert_node( - 1, + "Person", &format!("n:{}", i), UpsertNodeOptions { weight: 0.5, @@ -1745,7 +1990,7 @@ fn test_id_counters_survive_flush() { // New allocations should continue from where they left off let new_id = engine .upsert_node( - 1, + "Person", "after_flush", UpsertNodeOptions { weight: 0.5, @@ -1770,7 +2015,7 @@ fn test_segment_data_survives_reopen() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -1780,7 +2025,7 @@ fn test_segment_data_survives_reopen() { .unwrap(); b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -1789,7 +2034,7 @@ fn test_segment_data_survives_reopen() { ) .unwrap(); eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); engine.close().unwrap(); @@ -1820,7 +2065,7 @@ fn test_deleted_edge_excluded_from_segment_neighbors() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -1830,7 +2075,7 @@ fn test_deleted_edge_excluded_from_segment_neighbors() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -1840,7 +2085,7 @@ fn test_deleted_edge_excluded_from_segment_neighbors() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -1849,10 +2094,10 @@ fn test_deleted_edge_excluded_from_segment_neighbors() { ) .unwrap(); let e1 = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -1877,7 +2122,7 @@ fn test_upsert_after_delete_across_flush_gets_new_id() { let id1 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -1894,7 +2139,7 @@ fn test_upsert_after_delete_across_flush_gets_new_id() { // Re-insert same key, should get a fresh ID, not reuse deleted one let id2 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.7, @@ -1928,7 +2173,7 @@ fn test_auto_flush_triggers_on_threshold() { for i in 0..20 { let id = engine .upsert_node( - 1, + "Person", &format!("node:{}", i), UpsertNodeOptions { weight: 0.5, @@ -1968,7 +2213,7 @@ fn test_auto_flush_disabled_when_zero() { for i in 0..100 { engine .upsert_node( - 1, + "Person", &format!("node:{}", i), UpsertNodeOptions { weight: 0.5, @@ -2000,7 +2245,7 @@ fn test_compact_requires_two_segments() { // 1 segment → no-op engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -2024,7 +2269,7 @@ fn test_compact_merges_two_segments() { let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -2036,7 +2281,7 @@ fn test_compact_merges_two_segments() { let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -2071,7 +2316,7 @@ fn test_compact_applies_tombstones() { // Segment 1: alice + bob + edge let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -2081,7 +2326,7 @@ fn test_compact_applies_tombstones() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -2090,7 +2335,7 @@ fn test_compact_applies_tombstones() { ) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -2139,7 +2384,7 @@ fn test_compact_node_last_write_wins() { props_v1.insert("version".to_string(), PropValue::Int(1)); let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: props_v1, @@ -2155,7 +2400,7 @@ fn test_compact_node_last_write_wins() { props_v2.insert("version".to_string(), PropValue::Int(2)); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: props_v2, @@ -2189,7 +2434,7 @@ fn test_compact_preserves_neighbors() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -2199,7 +2444,7 @@ fn test_compact_preserves_neighbors() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -2208,13 +2453,13 @@ fn test_compact_preserves_neighbors() { ) .unwrap(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -2226,7 +2471,7 @@ fn test_compact_preserves_neighbors() { .upsert_edge( a, c, - 20, + "REPORTS_TO", UpsertEdgeOptions { weight: 0.8, ..Default::default() @@ -2243,18 +2488,18 @@ fn test_compact_preserves_neighbors() { assert!(ids.contains(&b)); assert!(ids.contains(&c)); - // Type filter still works after compaction - let typed = engine + // Edge-label filter still works after compaction + let labeled = engine .neighbors( a, &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) .unwrap(); - assert_eq!(typed.len(), 1); - assert_eq!(typed[0].node_id, b); + assert_eq!(labeled.len(), 1); + assert_eq!(labeled[0].node_id, b); engine.close().unwrap(); } @@ -2268,7 +2513,7 @@ fn test_compact_cleans_up_old_segment_dirs() { engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -2279,7 +2524,7 @@ fn test_compact_cleans_up_old_segment_dirs() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -2313,7 +2558,7 @@ fn test_compact_updates_manifest() { engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -2324,7 +2569,7 @@ fn test_compact_updates_manifest() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -2357,7 +2602,7 @@ fn test_compact_data_survives_reopen() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -2368,7 +2613,7 @@ fn test_compact_data_survives_reopen() { engine.flush().unwrap(); b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -2377,7 +2622,7 @@ fn test_compact_data_survives_reopen() { ) .unwrap(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); engine.compact().unwrap(); @@ -2410,7 +2655,7 @@ fn test_compact_three_segments() { for i in 0..3 { let id = engine .upsert_node( - 1, + "Person", &format!("n:{}", i), UpsertNodeOptions { weight: 0.5, @@ -2451,7 +2696,7 @@ fn test_compact_with_unflushed_tombstone() { // Segment 1: alice + bob let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -2461,7 +2706,7 @@ fn test_compact_with_unflushed_tombstone() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.6, @@ -2474,7 +2719,7 @@ fn test_compact_with_unflushed_tombstone() { // Segment 2: charlie engine .upsert_node( - 1, + "Person", "charlie", UpsertNodeOptions { weight: 0.7, @@ -2516,7 +2761,7 @@ fn test_compact_with_unflushed_update() { props_v1.insert("v".to_string(), PropValue::Int(1)); let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: props_v1, @@ -2530,7 +2775,7 @@ fn test_compact_with_unflushed_update() { // Segment 2: bob engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.5, @@ -2545,7 +2790,7 @@ fn test_compact_with_unflushed_update() { props_v2.insert("v".to_string(), PropValue::Int(2)); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: props_v2, @@ -2577,7 +2822,7 @@ fn test_compact_removes_dangling_edges_after_node_delete() { // Segment 1: A→B→C chain let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -2587,7 +2832,7 @@ fn test_compact_removes_dangling_edges_after_node_delete() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -2597,7 +2842,7 @@ fn test_compact_removes_dangling_edges_after_node_delete() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -2606,10 +2851,10 @@ fn test_compact_removes_dangling_edges_after_node_delete() { ) .unwrap(); let e_ab = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let e_bc = engine - .upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -2668,7 +2913,7 @@ fn test_orphan_segment_does_not_reuse_id() { ) .unwrap(); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine.close().unwrap(); @@ -2679,15 +2924,19 @@ fn test_orphan_segment_does_not_reuse_id() { // the segment but before updating the manifest). let orphan_dir = db_path.join("segments").join("seg_0099"); std::fs::create_dir_all(&orphan_dir).unwrap(); - // Write a minimal nodes.dat so it looks like a real segment - std::fs::write(orphan_dir.join("nodes.dat"), [0u8; 0]).unwrap(); + // Write a minimal packed-core marker so it looks like a real segment directory. + std::fs::write( + orphan_dir.join(crate::segment_components::PACKED_CORE_FILENAME), + [0u8; 0], + ) + .unwrap(); // Reopen. next_segment_id should skip past the orphan { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Insert more data and flush. Should get segment ID > 99 engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -2746,7 +2995,7 @@ fn test_map_props_roundtrip_memtable_and_segment() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id = engine .upsert_node( - 1, + "Person", "map_node", UpsertNodeOptions { props: props.clone(), @@ -2789,7 +3038,7 @@ fn install_noop_prune_policy(engine: &mut DatabaseEngine) { PrunePolicy { max_age_ms: None, max_weight: Some(0.0), - type_id: Some(u32::MAX), + label: Some("SpecialNode1024".to_string()), }, ) .unwrap(); @@ -2814,10 +3063,9 @@ fn build_clean_compaction_fixture(engine: &mut DatabaseEngine) -> (Vec, Vec let id = next_node_id; next_node_id += 1; let created_at = 1_000 + (seg as i64 * 100) + (i as i64 * 2); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { id, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: format!("s{}_n{}", seg, i), props, created_at, @@ -2836,12 +3084,11 @@ fn build_clean_compaction_fixture(engine: &mut DatabaseEngine) -> (Vec, Vec let eid = next_edge_id; next_edge_id += 1; let created_at = 5_000 + (seg as i64 * 100) + (i as i64 * 2); - engine - .write_op(&WalOp::UpsertEdge(EdgeRecord { + write_internal_wal_op(&engine, &WalOp::UpsertEdge(EdgeRecord { id: eid, from: seg_node_ids[i], to: seg_node_ids[i + 1], - type_id: 1, + label_id: 1, props: BTreeMap::new(), created_at, updated_at: created_at + 1, @@ -2859,10 +3106,305 @@ fn build_clean_compaction_fixture(engine: &mut DatabaseEngine) -> (Vec, Vec (all_node_ids, all_edge_ids, segment_starts) } -fn assert_compacted_index_files_match( - left: &DatabaseEngine, - right: &DatabaseEngine, - left_db_dir: &std::path::Path, +fn build_vector_compaction_and_flush_fixture( + compact_engine: &DatabaseEngine, + flush_engine: &DatabaseEngine, +) -> (Vec, Vec) { + let mut compact_node_ids = Vec::new(); + let mut flush_node_ids = Vec::new(); + let mut compact_edge_ids = Vec::new(); + let mut flush_edge_ids = Vec::new(); + let mut next_node_id = 1u64; + let mut next_edge_id = 1u64; + + for seg in 0..3u64 { + let mut compact_seg_ids = Vec::new(); + let mut flush_seg_ids = Vec::new(); + for i in 0..6u64 { + let dense_vector = vec![ + 1.0 + seg as f32 * 0.1, + 0.2 + i as f32 * 0.03, + 0.4 + seg as f32 * 0.05, + 0.6 + i as f32 * 0.02, + ]; + let sparse_vector = vec![ + (seg as u32, 1.0 + i as f32 * 0.1), + (seg as u32 + 10, 0.5 + seg as f32 * 0.05), + ]; + let mut props = BTreeMap::new(); + props.insert("seg".to_string(), PropValue::UInt(seg)); + props.insert("slot".to_string(), PropValue::UInt(i)); + let node_id = next_node_id; + next_node_id += 1; + let created_at = 10_000 + (seg as i64 * 100) + (i as i64 * 2); + let compact_node = NodeRecord { + id: node_id, + label_ids: NodeLabelSet::single(1).unwrap(), + key: format!("s{}_n{}", seg, i), + props: props.clone(), + created_at, + updated_at: created_at + 1, + weight: 1.0, + dense_vector: Some(dense_vector.clone()), + sparse_vector: Some(sparse_vector.clone()), + last_write_seq: 0, + }; + let flush_node = NodeRecord { + props, + dense_vector: Some(dense_vector), + sparse_vector: Some(sparse_vector), + ..compact_node.clone() + }; + + write_internal_wal_op(&compact_engine, &WalOp::UpsertNode(compact_node)) + .unwrap(); + write_internal_wal_op(&flush_engine, &WalOp::UpsertNode(flush_node)) + .unwrap(); + + compact_seg_ids.push(node_id); + flush_seg_ids.push(node_id); + compact_node_ids.push(node_id); + flush_node_ids.push(node_id); + } + for i in 0..3usize { + let edge_id = next_edge_id; + next_edge_id += 1; + let created_at = 20_000 + (seg as i64 * 100) + (i as i64 * 2); + let compact_edge = EdgeRecord { + id: edge_id, + from: compact_seg_ids[i], + to: compact_seg_ids[i + 1], + label_id: 1, + props: BTreeMap::new(), + created_at, + updated_at: created_at + 1, + weight: 0.5 + seg as f32 * 0.1 + i as f32 * 0.05, + valid_from: seg as i64, + valid_to: i64::MAX, + last_write_seq: 0, + }; + let flush_edge = EdgeRecord { + from: flush_seg_ids[i], + to: flush_seg_ids[i + 1], + ..compact_edge.clone() + }; + + write_internal_wal_op(&compact_engine, &WalOp::UpsertEdge(compact_edge)) + .unwrap(); + write_internal_wal_op(&flush_engine, &WalOp::UpsertEdge(flush_edge)) + .unwrap(); + + compact_edge_ids.push(edge_id); + flush_edge_ids.push(edge_id); + } + compact_engine.flush().unwrap(); + } + flush_engine.flush().unwrap(); + + assert_eq!(compact_node_ids, flush_node_ids); + assert_eq!(compact_edge_ids, flush_edge_ids); + (compact_node_ids, compact_edge_ids) +} + +fn record_payload_spans_for_test( + payload: &[u8], + label: &str, +) -> BTreeMap { + const RECORD_INDEX_ENTRY_SIZE: usize = 16; + + assert!( + payload.len() >= 8, + "{label} payload is missing count header" + ); + let count = read_u64_le_for_test(payload, 0, label) as usize; + let index_end = 8 + count * RECORD_INDEX_ENTRY_SIZE; + assert!( + index_end <= payload.len(), + "{label} index exceeds payload length: index_end={index_end}, len={}", + payload.len() + ); + + let mut entries = Vec::with_capacity(count); + let mut last_id = None; + for index in 0..count { + let entry_off = 8 + index * RECORD_INDEX_ENTRY_SIZE; + let id = read_u64_le_for_test(payload, entry_off, label); + let offset = read_u64_le_for_test(payload, entry_off + 8, label); + if let Some(previous_id) = last_id { + assert!( + previous_id < id, + "{label} record ids are not strictly sorted: {previous_id} before {id}" + ); + } + assert!( + offset >= index_end as u64, + "{label} record {id} points into index/header: offset={offset}, data_start={index_end}" + ); + assert!( + offset <= payload.len() as u64, + "{label} record {id} offset exceeds payload length: offset={offset}, len={}", + payload.len() + ); + entries.push((id, offset)); + last_id = Some(id); + } + + if count == 0 { + assert_eq!( + payload.len(), + index_end, + "{label} empty payload should contain only count header" + ); + return BTreeMap::new(); + } + + let mut by_offset = entries.clone(); + by_offset.sort_unstable_by_key(|(_, offset)| *offset); + assert_eq!( + by_offset[0].1, index_end as u64, + "{label} first record should start immediately after the index" + ); + + let mut spans = BTreeMap::new(); + for (index, (id, offset)) in by_offset.iter().copied().enumerate() { + let next_offset = by_offset + .get(index + 1) + .map(|(_, next)| *next) + .unwrap_or(payload.len() as u64); + assert!( + next_offset > offset, + "{label} record {id} has empty or overlapping span: offset={offset}, next={next_offset}" + ); + let data_len = u32::try_from(next_offset - offset) + .unwrap_or_else(|_| panic!("{label} record {id} span exceeds u32")); + spans.insert(id, (offset, data_len)); + } + spans +} + +fn record_payload_ids_for_test(payload: &[u8], label: &str) -> Vec { + record_payload_spans_for_test(payload, label) + .into_keys() + .collect() +} + +fn assert_metadata_offsets_match_record_payload_for_test( + seg_dir: &std::path::Path, + record_kind: SegmentComponentKind, + metadata_kind: SegmentComponentKind, + metadata_entry_size: usize, + label: &str, +) -> Vec { + let record_payload = read_manifest_component_payload_for_test(seg_dir, record_kind); + let record_spans = record_payload_spans_for_test(&record_payload, label); + let metadata_payload = read_manifest_component_payload_for_test(seg_dir, metadata_kind.clone()); + assert!( + metadata_payload.len() >= 8, + "{label} metadata payload missing count header" + ); + let count = read_u64_le_for_test(&metadata_payload, 0, label) as usize; + assert_eq!( + count, + record_spans.len(), + "{label} metadata count should match record count" + ); + + let (fixed_entries_offset, fixed_entry_size, expected_metadata_len) = + if metadata_kind == SegmentComponentKind::NodeMetadata { + assert!( + metadata_payload.len() >= 48, + "{label} node metadata payload missing v10 header" + ); + let fixed_entry_size = read_u16_le_for_test(&metadata_payload, 8, label) as usize; + let label_offset_entry_size = + read_u16_le_for_test(&metadata_payload, 10, label) as usize; + assert_eq!(fixed_entry_size, 48, "{label} node metadata fixed row size"); + assert_eq!( + label_offset_entry_size, 8, + "{label} node metadata label offset row size" + ); + let fixed_entries_offset = read_u64_le_for_test(&metadata_payload, 16, label) as usize; + let label_offsets_offset = read_u64_le_for_test(&metadata_payload, 24, label) as usize; + let label_ids_offset = read_u64_le_for_test(&metadata_payload, 32, label) as usize; + let label_id_count = read_u64_le_for_test(&metadata_payload, 40, label) as usize; + assert_eq!(fixed_entries_offset, 48, "{label} fixed table offset"); + assert_eq!( + label_offsets_offset, + fixed_entries_offset + count * fixed_entry_size, + "{label} label offset table offset" + ); + assert_eq!( + label_ids_offset, + label_offsets_offset + (count + 1) * label_offset_entry_size, + "{label} label ID region offset" + ); + ( + fixed_entries_offset, + fixed_entry_size, + label_ids_offset + label_id_count * 4, + ) + } else { + (8, metadata_entry_size, 8 + count * metadata_entry_size) + }; + + assert_eq!( + metadata_payload.len(), + expected_metadata_len, + "{label} metadata payload length mismatch" + ); + + let mut ids = Vec::with_capacity(count); + let mut last_id = None; + for index in 0..count { + let off = fixed_entries_offset + index * fixed_entry_size; + let id = read_u64_le_for_test(&metadata_payload, off, label); + let metadata_offset = read_u64_le_for_test(&metadata_payload, off + 8, label); + let metadata_len = read_u32_le_for_test(&metadata_payload, off + 16, label); + if let Some(previous_id) = last_id { + assert!( + previous_id < id, + "{label} metadata ids are not strictly sorted: {previous_id} before {id}" + ); + } + let (record_offset, record_len) = record_spans + .get(&id) + .copied() + .unwrap_or_else(|| panic!("{label} metadata references missing record {id}")); + assert_eq!( + metadata_offset, record_offset, + "{label} metadata offset mismatch for record {id}" + ); + assert_eq!( + metadata_len, record_len, + "{label} metadata length mismatch for record {id}" + ); + ids.push(id); + last_id = Some(id); + } + ids +} + +fn assert_compacted_record_payloads_match_metadata(seg_dir: &std::path::Path) { + assert_metadata_offsets_match_record_payload_for_test( + seg_dir, + SegmentComponentKind::NodeRecords, + SegmentComponentKind::NodeMetadata, + 60, + "node records", + ); + assert_metadata_offsets_match_record_payload_for_test( + seg_dir, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::EdgeMetadata, + 80, + "edge records", + ); +} + +fn assert_compacted_index_files_match( + left: &DatabaseEngine, + right: &DatabaseEngine, + left_db_dir: &std::path::Path, right_db_dir: &std::path::Path, ) { let left_dir = segment_dir(left_db_dir, left.segments_for_test()[0].segment_id); @@ -2870,54 +3412,160 @@ fn assert_compacted_index_files_match( assert_segment_common_artifacts_match(&left_dir, &right_dir); } -fn assert_segment_common_artifacts_match(left_dir: &std::path::Path, right_dir: &std::path::Path) { - for filename in [ - "format.ver", - "key_index.dat", - "node_type_index.dat", - "edge_type_index.dat", - "edge_triple_index.dat", - "timestamp_index.dat", - "adj_out.idx", - "adj_out.dat", - "adj_in.idx", - "adj_in.dat", - "tombstones.dat", +fn assert_segment_uses_packed_core_layout(seg_dir: &std::path::Path) { + assert!( + seg_dir + .join(crate::segment_components::PACKED_CORE_FILENAME) + .exists(), + "segment.core missing in {}", + seg_dir.display() + ); + let manifest = read_component_manifest_for_test(seg_dir); + assert!(matches!( + manifest + .components + .iter() + .find(|record| record.kind == SegmentComponentKind::PackedSegmentContainer) + .expect("missing PackedSegmentContainer") + .handle, + crate::segment_components::ComponentHandleV1::ExternalFile { .. } + )); + for kind in [ + SegmentComponentKind::NodeRecords, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::NodeMetadata, + SegmentComponentKind::EdgeMetadata, + SegmentComponentKind::Tombstones, + SegmentComponentKind::KeyIndex, + SegmentComponentKind::NodeLabelIndex, + SegmentComponentKind::EdgeLabelIndex, + SegmentComponentKind::EdgeTripleIndex, + SegmentComponentKind::AdjOutIndex, + SegmentComponentKind::AdjOutPostings, + SegmentComponentKind::AdjInIndex, + SegmentComponentKind::AdjInPostings, + SegmentComponentKind::TimestampIndex, + SegmentComponentKind::EdgeWeightIndex, + SegmentComponentKind::EdgeUpdatedAtIndex, + SegmentComponentKind::EdgeValidFromIndex, + SegmentComponentKind::EdgeValidToIndex, ] { - assert_eq!( - std::fs::read(left_dir.join(filename)).unwrap(), - std::fs::read(right_dir.join(filename)).unwrap(), - "{} mismatch", - filename + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap_or_else(|| panic!("missing component {:?}", kind)); + assert!( + matches!( + record.handle, + crate::segment_components::ComponentHandleV1::PackedRange { .. } + ), + "{:?} should be packed in {}", + kind, + seg_dir.display() + ); + } + assert_only_manifested_segment_files(seg_dir, &manifest); +} + +fn assert_only_manifested_segment_files( + seg_dir: &std::path::Path, + manifest: &crate::segment_components::SegmentComponentManifestV1, +) { + let mut expected = std::collections::BTreeSet::from([ + std::path::PathBuf::from(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME), + ]); + for record in &manifest.components { + if let crate::segment_components::ComponentHandleV1::ExternalFile { + relative_path, .. + } = &record.handle + { + expected.insert(std::path::PathBuf::from(relative_path)); + } + } + for record in &manifest.unknown_optional_components { + if record.wire.handle.handle_tag == 1 { + if let Some(relative_path) = &record.wire.handle.relative_path { + expected.insert(std::path::PathBuf::from(relative_path)); + } + } + } + + let mut actual = Vec::new(); + collect_regular_segment_files(seg_dir, seg_dir, &mut actual); + for relative_path in actual { + assert!( + expected.contains(&relative_path), + "unexpected unmanifested segment file {} in {}", + relative_path.display(), + seg_dir.display() ); } +} - for filename in ["prop_index.dat", "node_prop_hashes.dat"] { +fn collect_regular_segment_files( + root: &std::path::Path, + dir: &std::path::Path, + files: &mut Vec, +) { + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let file_type = entry.file_type().unwrap(); + if file_type.is_dir() { + collect_regular_segment_files(root, &path, files); + } else if file_type.is_file() { + files.push(path.strip_prefix(root).unwrap().to_path_buf()); + } + } +} + +fn assert_segment_common_artifacts_match(left_dir: &std::path::Path, right_dir: &std::path::Path) { + for kind in [ + SegmentComponentKind::KeyIndex, + SegmentComponentKind::NodeLabelIndex, + SegmentComponentKind::EdgeLabelIndex, + SegmentComponentKind::EdgeTripleIndex, + SegmentComponentKind::EdgeWeightIndex, + SegmentComponentKind::EdgeUpdatedAtIndex, + SegmentComponentKind::EdgeValidFromIndex, + SegmentComponentKind::EdgeValidToIndex, + SegmentComponentKind::TimestampIndex, + SegmentComponentKind::AdjOutIndex, + SegmentComponentKind::AdjOutPostings, + SegmentComponentKind::AdjInIndex, + SegmentComponentKind::AdjInPostings, + SegmentComponentKind::Tombstones, + ] { assert_eq!( - left_dir.join(filename).exists(), - right_dir.join(filename).exists(), - "{} presence mismatch", - filename + read_manifest_component_payload_for_test(left_dir, kind.clone()), + read_manifest_component_payload_for_test(right_dir, kind.clone()), + "{:?} mismatch", + kind ); } // Byte-identical vector artifacts (deterministic). - for filename in [ - crate::segment_writer::NODE_VECTOR_META_FILENAME, - crate::segment_writer::NODE_DENSE_VECTOR_BLOB_FILENAME, - crate::segment_writer::NODE_SPARSE_VECTOR_BLOB_FILENAME, - crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME, - crate::sparse_postings::SPARSE_POSTINGS_FILENAME, + for kind in [ + SegmentComponentKind::NodeVectorMetadata, + SegmentComponentKind::NodeDenseVectorBlob, + SegmentComponentKind::NodeSparseVectorBlob, + SegmentComponentKind::SparsePostingIndex, + SegmentComponentKind::SparsePostings, ] { - let left_exists = left_dir.join(filename).exists(); - let right_exists = right_dir.join(filename).exists(); - assert_eq!(left_exists, right_exists, "{} presence mismatch", filename); - if left_exists { + let left_payload = try_read_manifest_component_payload_for_test(left_dir, kind.clone()); + let right_payload = try_read_manifest_component_payload_for_test(right_dir, kind.clone()); + assert_eq!( + left_payload.is_some(), + right_payload.is_some(), + "{:?} presence mismatch", + kind + ); + if let (Some(left_payload), Some(right_payload)) = (left_payload, right_payload) { assert_eq!( - std::fs::read(left_dir.join(filename)).unwrap(), - std::fs::read(right_dir.join(filename)).unwrap(), - "{} mismatch", - filename + left_payload, right_payload, + "{:?} mismatch", + kind ); } } @@ -2981,10 +3629,12 @@ fn assert_segment_common_artifacts_match(left_dir: &std::path::Path, right_dir: if left_meta_exists { for dir in [left_dir, right_dir] { - let meta = - std::fs::read(dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap(); - let graph = - std::fs::read(dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME)).unwrap(); + let meta = read_external_component_payload_for_test( + &dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), + ); + let graph = read_external_component_payload_for_test( + &dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME), + ); // Verify non-empty and structurally valid (header parses, sizes consistent). assert!(meta.len() >= 36, "HNSW meta too short in {}", dir.display()); assert!(!graph.is_empty(), "HNSW graph empty in {}", dir.display()); @@ -2999,37 +3649,60 @@ fn assert_segment_common_artifacts_match(left_dir: &std::path::Path, right_dir: } } +fn assert_record_artifacts_match(left_dir: &std::path::Path, right_dir: &std::path::Path) { + for kind in [ + SegmentComponentKind::NodeRecords, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::NodeMetadata, + SegmentComponentKind::EdgeMetadata, + ] { + assert_eq!( + read_manifest_component_payload_for_test(left_dir, kind.clone()), + read_manifest_component_payload_for_test(right_dir, kind.clone()), + "{:?} mismatch", + kind + ); + } +} + fn assert_segment_metadata_semantics_match(left: &SegmentReader, right: &SegmentReader) { assert_eq!(left.node_meta_count(), right.node_meta_count()); for index in 0..left.node_meta_count() as usize { let left_meta = left.node_meta_at(index).unwrap(); let right_meta = right.node_meta_at(index).unwrap(); - assert_eq!(left_meta.0, right_meta.0, "node {} id mismatch", index); assert_eq!( - left_meta.2, right_meta.2, + left_meta.node_id, right_meta.node_id, + "node {} id mismatch", + index + ); + assert_eq!( + left_meta.data_len, right_meta.data_len, "node {} data_len mismatch", index ); - assert_eq!(left_meta.3, right_meta.3, "node {} type mismatch", index); assert_eq!( - left_meta.4, right_meta.4, + left_meta.label_ids, right_meta.label_ids, + "node {} label set mismatch", + index + ); + assert_eq!( + left_meta.updated_at, right_meta.updated_at, "node {} updated_at mismatch", index ); assert_eq!( - left_meta.5.to_bits(), - right_meta.5.to_bits(), + left_meta.weight.to_bits(), + right_meta.weight.to_bits(), "node {} weight mismatch", index ); - assert_eq!(left_meta.6, right_meta.6, "node {} key_len mismatch", index); assert_eq!( - left_meta.8, right_meta.8, - "node {} prop_hash_count mismatch", + left_meta.key_len, right_meta.key_len, + "node {} key_len mismatch", index ); assert_eq!( - left_meta.9, right_meta.9, + left_meta.last_write_seq, right_meta.last_write_seq, "node {} last_write_seq mismatch", index ); @@ -3056,7 +3729,7 @@ fn assert_segment_metadata_semantics_match(left: &SegmentReader, right: &Segment ); assert_eq!(left_meta.3, right_meta.3, "edge {} from mismatch", index); assert_eq!(left_meta.4, right_meta.4, "edge {} to mismatch", index); - assert_eq!(left_meta.5, right_meta.5, "edge {} type mismatch", index); + assert_eq!(left_meta.5, right_meta.5, "edge {} label mismatch", index); assert_eq!( left_meta.6, right_meta.6, "edge {} updated_at mismatch", @@ -3086,15 +3759,15 @@ fn assert_segment_metadata_semantics_match(left: &SegmentReader, right: &Segment } } -fn assert_node_batches_match(left: &[Option], right: &[Option]) { +fn assert_node_batches_match(left: &[Option], right: &[Option]) { assert_eq!(left.len(), right.len()); for (idx, (left_node, right_node)) in left.iter().zip(right.iter()).enumerate() { match (left_node, right_node) { (Some(left_node), Some(right_node)) => { assert_eq!(left_node.id, right_node.id, "node {} id mismatch", idx); assert_eq!( - left_node.type_id, right_node.type_id, - "node {} type mismatch", + left_node.labels, right_node.labels, + "node {} labels mismatch", idx ); assert_eq!(left_node.key, right_node.key, "node {} key mismatch", idx); @@ -3116,7 +3789,7 @@ fn assert_node_batches_match(left: &[Option], right: &[Option], right: &[Option]) { +fn assert_edge_batches_match(left: &[Option], right: &[Option]) { assert_eq!(left.len(), right.len()); for (idx, (left_edge, right_edge)) in left.iter().zip(right.iter()).enumerate() { match (left_edge, right_edge) { @@ -3129,8 +3802,8 @@ fn assert_edge_batches_match(left: &[Option], right: &[Option> = vec![ @@ -3757,7 +4389,7 @@ fn test_fast_merge_matches_single_flush_artifacts_for_vector_segments() { dense_query: Some(query.clone()), sparse_query: None, k: 5, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -3796,6 +4428,63 @@ fn test_fast_merge_matches_single_flush_artifacts_for_vector_segments() { flush_engine.close().unwrap(); } +#[test] +fn test_v3_matches_single_flush_artifacts_for_vector_segments() { + let compact_dir = TempDir::new().unwrap(); + let flush_dir = TempDir::new().unwrap(); + let opts = DbOptions { + edge_uniqueness: true, + compact_after_n_flushes: 0, + dense_vector: Some(DenseVectorConfig { + dimension: 4, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + + let mut compact_engine = DatabaseEngine::open(compact_dir.path(), &opts).unwrap(); + let flush_engine = DatabaseEngine::open(flush_dir.path(), &opts).unwrap(); + let (node_ids, edge_ids) = + build_vector_compaction_and_flush_fixture(&compact_engine, &flush_engine); + install_noop_prune_policy(&mut compact_engine); + assert_eq!( + compaction_path_for(&compact_engine), + CompactionPath::UnifiedV3 + ); + + compact_engine.compact().unwrap().unwrap(); + + let compact_nodes = compact_engine.get_nodes(&node_ids).unwrap(); + let flush_nodes = flush_engine.get_nodes(&node_ids).unwrap(); + assert_node_batches_match(&compact_nodes, &flush_nodes); + + let compact_edges = compact_engine.get_edges(&edge_ids).unwrap(); + let flush_edges = flush_engine.get_edges(&edge_ids).unwrap(); + assert_edge_batches_match(&compact_edges, &flush_edges); + + let compact_seg_dir = segment_dir( + compact_dir.path(), + compact_engine.segments_for_test()[0].segment_id, + ); + let flush_seg_dir = segment_dir( + flush_dir.path(), + flush_engine.segments_for_test()[0].segment_id, + ); + assert_segment_uses_packed_core_layout(&compact_seg_dir); + assert_segment_uses_packed_core_layout(&flush_seg_dir); + assert_segment_common_artifacts_match(&compact_seg_dir, &flush_seg_dir); + assert_record_artifacts_match(&compact_seg_dir, &flush_seg_dir); + assert_compacted_record_payloads_match_metadata(&compact_seg_dir); + assert_segment_metadata_semantics_match( + &compact_engine.segments_for_test()[0], + &flush_engine.segments_for_test()[0], + ); + + compact_engine.close().unwrap(); + flush_engine.close().unwrap(); +} + #[test] fn test_standard_path_used_for_overlapping_segments() { let dir = TempDir::new().unwrap(); @@ -3809,7 +4498,7 @@ fn test_standard_path_used_for_overlapping_segments() { for _seg in 0..3 { for i in 0..10 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3839,6 +4528,100 @@ fn test_standard_path_used_for_overlapping_segments() { engine.close().unwrap(); } +#[test] +fn test_v3_filtered_compaction_packed_record_payloads_match_winners() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + compact_after_n_flushes: 0, + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let mut node_1_old = make_node(1, "node-1-old"); + node_1_old.weight = 1.0; + let mut node_2 = make_node(2, "node-2"); + node_2.weight = 2.0; + let mut node_3 = make_node(3, "node-3"); + node_3.weight = 3.0; + write_internal_wal_op(&engine, &WalOp::UpsertNode(node_1_old)).unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(node_2)).unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(node_3)).unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertEdge(make_edge(10, 1, 2))).unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertEdge(make_edge(11, 2, 3))).unwrap(); + let mut edge_12_old = make_edge(12, 3, 1); + edge_12_old.weight = 1.25; + write_internal_wal_op(&engine, &WalOp::UpsertEdge(edge_12_old)).unwrap(); + engine.flush().unwrap(); + + let mut node_1_new = make_node(1, "node-1-new"); + node_1_new.weight = 4.0; + write_internal_wal_op(&engine, &WalOp::UpsertNode(node_1_new)).unwrap(); + write_internal_wal_op(&engine, &WalOp::DeleteNode { + id: 3, + deleted_at: 9_000, + }) + .unwrap(); + write_internal_wal_op(&engine, &WalOp::DeleteEdge { + id: 10, + deleted_at: 9_001, + }) + .unwrap(); + let mut edge_12_new = make_edge(12, 2, 1); + edge_12_new.weight = 2.5; + write_internal_wal_op(&engine, &WalOp::UpsertEdge(edge_12_new)).unwrap(); + engine.flush().unwrap(); + + assert_eq!(compaction_path_for(&engine), CompactionPath::UnifiedV3); + let stats = engine.compact().unwrap().unwrap(); + assert_eq!(stats.nodes_kept, 2); + assert_eq!(stats.edges_kept, 1); + + let compacted_seg_dir = segment_dir(dir.path(), engine.segments_for_test()[0].segment_id); + assert_segment_uses_packed_core_layout(&compacted_seg_dir); + + let node_payload = + read_manifest_component_payload_for_test(&compacted_seg_dir, SegmentComponentKind::NodeRecords); + let edge_payload = + read_manifest_component_payload_for_test(&compacted_seg_dir, SegmentComponentKind::EdgeRecords); + assert_eq!( + record_payload_ids_for_test(&node_payload, "v3 filtered node records"), + vec![1, 2] + ); + assert_eq!( + record_payload_ids_for_test(&edge_payload, "v3 filtered edge records"), + vec![12] + ); + let node_metadata_ids = assert_metadata_offsets_match_record_payload_for_test( + &compacted_seg_dir, + SegmentComponentKind::NodeRecords, + SegmentComponentKind::NodeMetadata, + 60, + "v3 filtered node records", + ); + let edge_metadata_ids = assert_metadata_offsets_match_record_payload_for_test( + &compacted_seg_dir, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::EdgeMetadata, + 80, + "v3 filtered edge records", + ); + assert_eq!(node_metadata_ids, vec![1, 2]); + assert_eq!(edge_metadata_ids, vec![12]); + + let node_1 = engine.get_node(1).unwrap().unwrap(); + assert_eq!(node_1.key, "node-1-new"); + assert_eq!(node_1.weight.to_bits(), 4.0f32.to_bits()); + assert!(engine.get_node(3).unwrap().is_none()); + assert!(engine.get_edge(10).unwrap().is_none()); + assert!(engine.get_edge(11).unwrap().is_none()); + let edge_12 = engine.get_edge(12).unwrap().unwrap(); + assert_eq!((edge_12.from, edge_12.to), (2, 1)); + assert_eq!(edge_12.weight.to_bits(), 2.5f32.to_bits()); + + engine.close().unwrap(); +} + // --- Auto-compaction tests --- #[test] @@ -3853,7 +4636,7 @@ fn test_auto_compact_triggers_after_n_flushes() { // Flush 1 and 2: no compaction yet for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3861,7 +4644,7 @@ fn test_auto_compact_triggers_after_n_flushes() { for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3870,7 +4653,7 @@ fn test_auto_compact_triggers_after_n_flushes() { // Flush 3: should trigger auto-compact (3 segments → 1) for i in 0..10 { engine - .upsert_node(1, &format!("c{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3906,7 +4689,7 @@ fn test_auto_compact_disabled_when_zero() { for i in 0..5 { engine .upsert_node( - 1, + "Person", &format!("f{}_n{}", flush, i), UpsertNodeOptions::default(), ) @@ -3933,7 +4716,7 @@ fn test_auto_compact_counter_resets_on_manual_compact() { for seg in 0..2u64 { for i in 0..5 { engine - .upsert_node(1, &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3950,7 +4733,7 @@ fn test_auto_compact_counter_resets_on_manual_compact() { for seg in 2..6u64 { for i in 0..5 { engine - .upsert_node(1, &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3961,7 +4744,7 @@ fn test_auto_compact_counter_resets_on_manual_compact() { // 5th flush triggers auto-compact for i in 0..5 { engine - .upsert_node(1, &format!("s6_n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("s6_n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -3986,7 +4769,7 @@ fn test_auto_compact_data_integrity() { for seg in 0..6u64 { for i in 0..10 { let id = engine - .upsert_node(1, &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) .unwrap(); all_ids.push(id); } @@ -4019,7 +4802,7 @@ fn test_auto_compact_not_triggered_during_compact_flush() { // But we only have 1 segment after flush, so compact() returns None (< 2 segments). for i in 0..5 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4029,7 +4812,7 @@ fn test_auto_compact_not_triggered_during_compact_flush() { // Second flush: now 2 segments, auto-compact should fire for i in 0..5 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4070,7 +4853,7 @@ fn test_bg_compact_basic() { // Two flushes to trigger background compaction for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4078,7 +4861,7 @@ fn test_bg_compact_basic() { for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4118,13 +4901,13 @@ fn test_lifecycle_pump_applies_finished_bg_compact_without_foreground_write() { for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4159,7 +4942,7 @@ fn test_lifecycle_pump_applies_bg_flush_without_foreground_write() { }; let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); - db.upsert_node(1, "pump_flush", UpsertNodeOptions::default()) + db.upsert_node("Person", "pump_flush", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); db.enqueue_one_flush().unwrap(); @@ -4174,7 +4957,7 @@ fn test_lifecycle_pump_applies_bg_flush_without_foreground_write() { } assert_eq!(db.segment_count().unwrap(), 1); - assert!(db.get_node_by_key(1, "pump_flush").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "pump_flush").unwrap().is_some()); db.close().unwrap(); } @@ -4194,13 +4977,13 @@ fn test_bg_compact_writes_during() { // Two flushes to trigger bg compact for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4209,7 +4992,7 @@ fn test_bg_compact_writes_during() { // Immediately write more data. Should NOT block for i in 0..20 { engine - .upsert_node(1, &format!("c{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -4250,13 +5033,13 @@ fn test_flushes_while_bg_compact_is_outstanding_count_toward_next_run() { for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4266,7 +5049,7 @@ fn test_flushes_while_bg_compact_is_outstanding_count_toward_next_run() { for i in 0..10 { engine - .upsert_node(1, &format!("c{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4281,7 +5064,7 @@ fn test_flushes_while_bg_compact_is_outstanding_count_toward_next_run() { for i in 0..10 { engine - .upsert_node(1, &format!("d{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("d{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4313,13 +5096,13 @@ fn test_bg_compact_flush_during() { // Two flushes → triggers bg compact for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); // bg compact starts here @@ -4327,7 +5110,7 @@ fn test_bg_compact_flush_during() { // Write more data and flush. Adds a NEW segment while bg compact runs for i in 0..10 { engine - .upsert_node(1, &format!("c{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); // new segment added; bg compact may still be running @@ -4369,7 +5152,7 @@ fn test_bg_compact_no_double() { // First flush: only 1 segment, bg compact needs >= 2, so no bg compact for i in 0..5 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4378,7 +5161,7 @@ fn test_bg_compact_no_double() { // Second flush: 2 segments, bg compact starts for i in 0..5 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4389,7 +5172,7 @@ fn test_bg_compact_no_double() { // should NOT start a second bg compact for i in 0..5 { engine - .upsert_node(1, &format!("c{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4434,13 +5217,13 @@ fn test_bg_compact_manual_after_bg() { // Two flushes → triggers bg compact for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); // bg compact starts @@ -4448,13 +5231,13 @@ fn test_bg_compact_manual_after_bg() { // Add more segments for i in 0..10 { engine - .upsert_node(1, &format!("c{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("d{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("d{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4492,13 +5275,13 @@ fn test_bg_compact_drop_waits() { for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); // bg compact starts @@ -4536,13 +5319,13 @@ fn test_bg_compact_immediate_mode() { for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4584,13 +5367,13 @@ fn test_bg_compact_group_commit_mode() { for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4598,7 +5381,7 @@ fn test_bg_compact_group_commit_mode() { // Write more data while bg compact may be running for i in 0..10 { engine - .upsert_node(1, &format!("c{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -4640,13 +5423,13 @@ fn test_bg_compact_cancel() { // Two flushes → triggers bg compact for i in 0..10 { engine - .upsert_node(1, &format!("a{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("a{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 0..10 { engine - .upsert_node(1, &format!("b{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("b{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4688,7 +5471,7 @@ fn test_orphan_segment_cleanup_on_open() { // Write + flush to create a real segment for i in 0..5 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4742,7 +5525,7 @@ fn test_orphan_cleanup_preserves_valid_segments() { for seg in 0..3 { for i in 0..5 { engine - .upsert_node(1, &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("s{}_n{}", seg, i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4807,13 +5590,13 @@ fn test_immediate_mode_basic_operations() { // Write nodes and edges let n1 = engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(n1, n2, 1, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // Read back immediately @@ -4844,7 +5627,7 @@ fn test_immediate_mode_batch_operations() { let inputs: Vec = (0..50) .map(|i| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("node_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -4853,7 +5636,7 @@ fn test_immediate_mode_batch_operations() { }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs).unwrap(); assert_eq!(ids.len(), 50); for &id in &ids { @@ -4870,14 +5653,14 @@ fn test_immediate_mode_flush_compact_cycle() { // Insert, flush, insert more, flush, compact for i in 0..100 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 100..200 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -4914,7 +5697,7 @@ fn test_group_commit_basic_write_close_reopen() { let mut ids = Vec::new(); for i in 0..20 { let id = engine - .upsert_node(1, &format!("gc_node_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("gc_node_{}", i), UpsertNodeOptions::default()) .unwrap(); ids.push(id); } @@ -4954,16 +5737,16 @@ fn test_group_commit_with_edges() { let (dir, engine) = temp_db_group_commit(); let n1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = engine .upsert_edge( n1, n2, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 0.5, ..Default::default() @@ -5001,7 +5784,7 @@ fn test_group_commit_batch_operations() { let inputs: Vec = (0..100) .map(|i| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("batch_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -5010,7 +5793,7 @@ fn test_group_commit_batch_operations() { }) .collect(); - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs).unwrap(); assert_eq!(ids.len(), 100); engine.close().unwrap(); @@ -5042,7 +5825,7 @@ fn test_sync_forces_immediate_flush() { // Write a node let id = engine - .upsert_node(1, "sync_test", UpsertNodeOptions::default()) + .upsert_node("Person", "sync_test", UpsertNodeOptions::default()) .unwrap(); // Force sync. After this, data must be on disk @@ -5073,7 +5856,7 @@ fn test_sync_noop_in_immediate_mode() { let (_dir, engine) = temp_db_immediate(); engine - .upsert_node(1, "test", UpsertNodeOptions::default()) + .upsert_node("Person", "test", UpsertNodeOptions::default()) .unwrap(); // sync() should be a no-op in Immediate mode and not error engine.sync().unwrap(); @@ -5087,7 +5870,7 @@ fn test_group_commit_flush_cycle() { // Write → flush → write → flush under GroupCommit for i in 0..50 { engine - .upsert_node(1, &format!("pre_flush_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("pre_flush_{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -5095,7 +5878,7 @@ fn test_group_commit_flush_cycle() { for i in 0..50 { engine .upsert_node( - 1, + "Person", &format!("post_flush_{}", i), UpsertNodeOptions::default(), ) @@ -5135,7 +5918,7 @@ fn test_drop_joins_sync_thread() { for i in 0..10 { engine - .upsert_node(1, &format!("drop_test_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("drop_test_{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -5177,7 +5960,7 @@ fn test_backpressure_blocks_writer_at_hard_cap() { // will drain them. If backpressure is broken, buffered_bytes grows unbounded. for i in 0..200 { engine - .upsert_node(1, &format!("bp_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("bp_{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -5203,7 +5986,7 @@ fn test_clean_shutdown_drains_all_buffered_data() { // Write 100 nodes rapidly (most will be buffered, not yet synced) for i in 0..100 { engine - .upsert_node(1, &format!("drain_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("drain_{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -5256,7 +6039,7 @@ fn test_drop_drains_buffered_data() { for i in 0..50 { engine .upsert_node( - 1, + "Person", &format!("drop_drain_{}", i), UpsertNodeOptions::default(), ) @@ -5299,7 +6082,8 @@ fn test_sync_failure_poisons_engine() { use crate::wal_sync::WalSyncState; let dir = TempDir::new().unwrap(); - let writer = WalWriter::open(dir.path()).unwrap(); + let wal_dir = TempDir::new().unwrap(); + let writer = WalWriter::open_generation(wal_dir.path(), 0).unwrap(); let state = WalSyncState { wal_writer: writer, @@ -5329,7 +6113,7 @@ fn test_sync_failure_poisons_engine() { engine.reset_publish_counters_for_test(); // Attempt to write. Should get WalSyncFailed error - let result = engine.upsert_node(1, "should_fail", UpsertNodeOptions::default()); + let result = engine.upsert_node("Person", "should_fail", UpsertNodeOptions::default()); assert!(result.is_err()); let err_msg = format!("{}", result.unwrap_err()); assert!( @@ -5342,7 +6126,7 @@ fn test_sync_failure_poisons_engine() { let counters = engine.publish_counter_snapshot_for_test(); assert!(std::sync::Arc::ptr_eq(&before.view.sources, &after.view.sources)); assert_eq!(before.engine_seq, after.engine_seq); - assert!(engine.get_node_by_key(1, "should_fail").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "should_fail").unwrap().is_none()); assert!(counters.skipped >= 1); assert_eq!(counters.snapshot_only, 0); assert_eq!(counters.rebuild_sources, 0); @@ -5359,7 +6143,7 @@ fn test_integration_1000_writes_group_commit() { props.insert("index".to_string(), PropValue::Int(i as i64)); engine .upsert_node( - 1, + "Person", &format!("int_{}", i), UpsertNodeOptions { props, @@ -5398,13 +6182,13 @@ fn test_integration_1000_writes_group_commit() { #[test] fn test_integration_write_flush_write_flush_group_commit() { - // Exercises truncate_and_reset through multiple flush cycles + // Exercises generation WAL rotation through multiple flush cycles. let (dir, engine) = temp_db_group_commit(); // Cycle 1: write → flush for i in 0..100 { engine - .upsert_node(1, &format!("c1_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c1_{}", i), UpsertNodeOptions::default()) .unwrap(); } let seg1 = engine.flush().unwrap(); @@ -5413,7 +6197,7 @@ fn test_integration_write_flush_write_flush_group_commit() { // Cycle 2: write → flush for i in 0..100 { engine - .upsert_node(1, &format!("c2_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c2_{}", i), UpsertNodeOptions::default()) .unwrap(); } let seg2 = engine.flush().unwrap(); @@ -5422,7 +6206,7 @@ fn test_integration_write_flush_write_flush_group_commit() { // Cycle 3: write → flush for i in 0..100 { engine - .upsert_node(1, &format!("c3_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("c3_{}", i), UpsertNodeOptions::default()) .unwrap(); } let seg3 = engine.flush().unwrap(); @@ -5462,7 +6246,7 @@ fn test_group_commit_delete_and_compact_cycle() { let mut ids = Vec::new(); for i in 0..100 { let id = engine - .upsert_node(1, &format!("gc_del_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("gc_del_{}", i), UpsertNodeOptions::default()) .unwrap(); ids.push(id); } @@ -5608,7 +6392,7 @@ fn test_backpressure_flush_triggers_at_hard_cap_immediate() { for i in 0..50 { let id = engine .upsert_node( - 1, + "Person", &format!("bp_imm_{}", i), UpsertNodeOptions { weight: 0.5, @@ -5658,7 +6442,7 @@ fn test_backpressure_flush_triggers_at_hard_cap_group_commit() { for i in 0..50 { let id = engine .upsert_node( - 1, + "Person", &format!("bp_gc_{}", i), UpsertNodeOptions { weight: 0.5, @@ -5720,7 +6504,7 @@ fn test_backpressure_disabled_when_zero() { for i in 0..100 { engine .upsert_node( - 1, + "Person", &format!("no_bp_{}", i), UpsertNodeOptions { weight: 0.5, @@ -5754,7 +6538,7 @@ fn test_backpressure_fires_before_soft_threshold() { for i in 0..30 { engine .upsert_node( - 1, + "Person", &format!("early_bp_{}", i), UpsertNodeOptions { weight: 0.5, @@ -5791,7 +6575,7 @@ fn test_backpressure_with_edges_and_deletes() { for i in 0..20 { let id = engine .upsert_node( - 1, + "Person", &format!("n_{}", i), UpsertNodeOptions { weight: 0.5, @@ -5809,7 +6593,7 @@ fn test_backpressure_with_edges_and_deletes() { .upsert_edge( node_ids[i], node_ids[i + 1], - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 0.5, ..Default::default() @@ -5856,7 +6640,7 @@ fn test_backpressure_with_batch_upserts() { // First batch: fills memtable let inputs1: Vec = (0..20) .map(|i| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("batch1_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -5864,12 +6648,12 @@ fn test_backpressure_with_batch_upserts() { sparse_vector: None, }) .collect(); - let ids1 = engine.batch_upsert_nodes(&inputs1).unwrap(); + let ids1 = engine.batch_upsert_nodes(inputs1).unwrap(); // Second batch: should trigger backpressure flush before appending let inputs2: Vec = (0..20) .map(|i| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("batch2_{}", i), props: BTreeMap::new(), weight: 1.0, @@ -5877,7 +6661,7 @@ fn test_backpressure_with_batch_upserts() { sparse_vector: None, }) .collect(); - let ids2 = engine.batch_upsert_nodes(&inputs2).unwrap(); + let ids2 = engine.batch_upsert_nodes(inputs2).unwrap(); assert!( engine.segment_count().unwrap() >= 1, @@ -5916,7 +6700,7 @@ fn test_backpressure_flush_then_write_cycle_group_commit() { for i in 0..200 { let id = engine .upsert_node( - 1, + "Person", &format!("stress_{}", i), UpsertNodeOptions { weight: 0.5, @@ -5983,7 +6767,7 @@ fn test_backpressure_interacts_with_auto_compact() { for i in 0..100 { engine .upsert_node( - 1, + "Person", &format!("ac_{}", i), UpsertNodeOptions { weight: 0.5, @@ -6024,20 +6808,42 @@ fn test_backpressure_invalidate_edge() { let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); let n1 = engine - .upsert_node(1, "src", UpsertNodeOptions::default()) + .upsert_node("Person", "src", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "dst", UpsertNodeOptions::default()) + .upsert_node("Person", "dst", UpsertNodeOptions::default()) .unwrap(); // Create many edges to fill memtable let mut edge_ids = Vec::new(); + let edge_labels = [ + "RELATES_TO", + "WORKS_AT", + "LIKES", + "MENTIONS", + "OWNS", + "FOLLOWS", + "FRIENDS_WITH", + "COLLABORATES_WITH", + "RELATED_TO", + "KNOWS", + "BLOCKS", + "DEPENDS_ON", + "ASSIGNED_TO", + "REVIEWED_BY", + "PUBLISHED_BY", + "TAGGED_WITH", + "EDGE_LABEL_17", + "EDGE_LABEL_18", + "EDGE_LABEL_19", + "REPORTS_TO", + ]; for i in 0..20 { let eid = engine .upsert_edge( n1, n2, - i as u32, + edge_labels[i], UpsertEdgeOptions { weight: 0.5, ..Default::default() @@ -6069,13 +6875,13 @@ fn test_compact_all_records_tombstoned() { let db = open_imm(&dir.path().join("db")); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -6094,7 +6900,7 @@ fn test_compact_all_records_tombstoned() { // DB should still be functional after compaction of all-tombstone data let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); assert!(db.get_node(c).unwrap().is_some()); db.close().unwrap(); @@ -6108,18 +6914,18 @@ fn test_engine_seq_monotonic_across_writes() { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id1 = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let id2 = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let id3 = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - let seq1 = db.get_node(id1).unwrap().unwrap().last_write_seq; - let seq2 = db.get_node(id2).unwrap().unwrap().last_write_seq; - let seq3 = db.get_node(id3).unwrap().unwrap().last_write_seq; + let seq1 = internal_node_record(&db, id1).unwrap().unwrap().last_write_seq; + let seq2 = internal_node_record(&db, id2).unwrap().unwrap().last_write_seq; + let seq3 = internal_node_record(&db, id3).unwrap().unwrap().last_write_seq; assert!(seq1 > 0, "seq must be > 0"); assert!(seq2 > seq1, "seq2 ({}) must be > seq1 ({})", seq2, seq1); @@ -6133,17 +6939,17 @@ fn test_engine_seq_survives_flush() { let dir = tempfile::tempdir().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); // After flush, next write should continue with higher seq let id3 = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - let seq3 = db.get_node(id3).unwrap().unwrap().last_write_seq; + let seq3 = internal_node_record(&db, id3).unwrap().unwrap().last_write_seq; assert!( seq3 >= 3, "seq after flush must continue monotonically, got {}", @@ -6158,9 +6964,9 @@ fn test_engine_seq_survives_reopen() { let dir = tempfile::tempdir().unwrap(); { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); db.close().unwrap(); @@ -6168,9 +6974,9 @@ fn test_engine_seq_survives_reopen() { { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id3 = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - let seq3 = db.get_node(id3).unwrap().unwrap().last_write_seq; + let seq3 = internal_node_record(&db, id3).unwrap().unwrap().last_write_seq; // After reopen with flush, manifest persisted next_engine_seq, // so seq must continue from where it left off assert!(seq3 >= 3, "seq after reopen must be >= 3, got {}", seq3); @@ -6183,9 +6989,9 @@ fn test_engine_seq_correct_after_replay() { let dir = tempfile::tempdir().unwrap(); { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); // Close WITHOUT flush; WAL will be replayed on reopen. db.close().unwrap(); @@ -6193,16 +6999,16 @@ fn test_engine_seq_correct_after_replay() { { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); // After replay, the memtable records should have seqs assigned - let n1 = db.get_node(1).unwrap().unwrap(); - let n2 = db.get_node(2).unwrap().unwrap(); + let n1 = internal_node_record(&db, 1).unwrap().unwrap(); + let n2 = internal_node_record(&db, 2).unwrap().unwrap(); assert!(n1.last_write_seq > 0); assert!(n2.last_write_seq > n1.last_write_seq); // New writes should continue beyond replayed seqs let id3 = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - let seq3 = db.get_node(id3).unwrap().unwrap().last_write_seq; + let seq3 = internal_node_record(&db, id3).unwrap().unwrap().last_write_seq; assert!(seq3 > n2.last_write_seq); db.close().unwrap(); } @@ -6217,59 +7023,68 @@ fn test_last_write_seq_exact_equality_across_reopen() { let (seq_a, seq_b, seq_c); - // Phase 1: write 3 nodes, capture exact seqs, close_fast (no flush) + // Step 1: write 3 nodes, capture exact seqs, close_fast (no flush) { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id_a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let id_b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let id_c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - seq_a = db.get_node(id_a).unwrap().unwrap().last_write_seq; - seq_b = db.get_node(id_b).unwrap().unwrap().last_write_seq; - seq_c = db.get_node(id_c).unwrap().unwrap().last_write_seq; + seq_a = internal_node_record(&db, id_a).unwrap().unwrap().last_write_seq; + seq_b = internal_node_record(&db, id_b).unwrap().unwrap().last_write_seq; + seq_c = internal_node_record(&db, id_c).unwrap().unwrap().last_write_seq; assert!(seq_a < seq_b && seq_b < seq_c); db.close_fast().unwrap(); } - // Phase 2: reopen (WAL replay). Seqs must be exactly the same. + // Step 2: reopen (WAL replay). Seqs must be exactly the same. { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); assert_eq!( - db.get_node(1).unwrap().unwrap().last_write_seq, + internal_node_record(&db, 1).unwrap().unwrap().last_write_seq, seq_a, "node a seq changed after replay" ); assert_eq!( - db.get_node(2).unwrap().unwrap().last_write_seq, + internal_node_record(&db, 2).unwrap().unwrap().last_write_seq, seq_b, "node b seq changed after replay" ); assert_eq!( - db.get_node(3).unwrap().unwrap().last_write_seq, + internal_node_record(&db, 3).unwrap().unwrap().last_write_seq, seq_c, "node c seq changed after replay" ); // New write must continue strictly after let id_d = db - .upsert_node(1, "d", UpsertNodeOptions::default()) + .upsert_node("Person", "d", UpsertNodeOptions::default()) .unwrap(); - let seq_d = db.get_node(id_d).unwrap().unwrap().last_write_seq; + let seq_d = internal_node_record(&db, id_d).unwrap().unwrap().last_write_seq; assert!(seq_d > seq_c, "new write seq must be > replayed max"); db.close_fast().unwrap(); } - // Phase 3: reopen again, still exact (double replay). + // Step 3: reopen again, still exact (double replay). { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert_eq!(db.get_node(1).unwrap().unwrap().last_write_seq, seq_a); - assert_eq!(db.get_node(2).unwrap().unwrap().last_write_seq, seq_b); - assert_eq!(db.get_node(3).unwrap().unwrap().last_write_seq, seq_c); + assert_eq!( + internal_node_record(&db, 1).unwrap().unwrap().last_write_seq, + seq_a + ); + assert_eq!( + internal_node_record(&db, 2).unwrap().unwrap().last_write_seq, + seq_b + ); + assert_eq!( + internal_node_record(&db, 3).unwrap().unwrap().last_write_seq, + seq_c + ); db.close().unwrap(); } } @@ -6286,14 +7101,14 @@ fn test_last_write_seq_exact_across_freeze_reopen() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id_f = db - .upsert_node(1, "frozen_node", UpsertNodeOptions::default()) + .upsert_node("Person", "frozen_node", UpsertNodeOptions::default()) .unwrap(); - seq_frozen = db.get_node(id_f).unwrap().unwrap().last_write_seq; + seq_frozen = internal_node_record(&db, id_f).unwrap().unwrap().last_write_seq; db.freeze_memtable().unwrap(); let id_a = db - .upsert_node(1, "active_node", UpsertNodeOptions::default()) + .upsert_node("Person", "active_node", UpsertNodeOptions::default()) .unwrap(); - seq_active = db.get_node(id_a).unwrap().unwrap().last_write_seq; + seq_active = internal_node_record(&db, id_a).unwrap().unwrap().last_write_seq; assert!(seq_active > seq_frozen); db.close_fast().unwrap(); } @@ -6301,8 +7116,8 @@ fn test_last_write_seq_exact_across_freeze_reopen() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Frozen node is in immutable_epochs, active node in memtable - let f = db.get_node_by_key(1, "frozen_node").unwrap().unwrap(); - let a = db.get_node_by_key(1, "active_node").unwrap().unwrap(); + let f = internal_node_record(&db, 1).unwrap().unwrap(); + let a = internal_node_record(&db, 2).unwrap().unwrap(); assert_eq!(f.last_write_seq, seq_frozen, "frozen seq changed on replay"); assert_eq!(a.last_write_seq, seq_active, "active seq changed on replay"); @@ -6313,8 +7128,8 @@ fn test_last_write_seq_exact_across_freeze_reopen() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let f = db.get_node_by_key(1, "frozen_node").unwrap().unwrap(); - let a = db.get_node_by_key(1, "active_node").unwrap().unwrap(); + let f = internal_node_record(&db, 1).unwrap().unwrap(); + let a = internal_node_record(&db, 2).unwrap().unwrap(); assert_eq!( f.last_write_seq, seq_frozen, "frozen seq changed after flush+reopen" @@ -6333,23 +7148,23 @@ fn test_compaction_preserves_last_write_seq() { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id1 = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - let seq1 = db.get_node(id1).unwrap().unwrap().last_write_seq; + let seq1 = internal_node_record(&db, id1).unwrap().unwrap().last_write_seq; db.flush().unwrap(); let id2 = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - let seq2 = db.get_node(id2).unwrap().unwrap().last_write_seq; + let seq2 = internal_node_record(&db, id2).unwrap().unwrap().last_write_seq; db.flush().unwrap(); // Both nodes are now in segments. Compact. db.compact().unwrap(); // After compaction, seqs should be preserved - let n1 = db.get_node(id1).unwrap().unwrap(); - let n2 = db.get_node(id2).unwrap().unwrap(); + let n1 = internal_node_record(&db, id1).unwrap().unwrap(); + let n2 = internal_node_record(&db, id2).unwrap().unwrap(); assert_eq!(n1.last_write_seq, seq1); assert_eq!(n2.last_write_seq, seq2); @@ -6363,7 +7178,7 @@ fn test_batch_ops_get_distinct_seq() { let inputs: Vec = (0..5) .map(|i| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n{}", i), props: BTreeMap::new(), weight: 1.0, @@ -6372,11 +7187,11 @@ fn test_batch_ops_get_distinct_seq() { }) .collect(); - let ids = db.batch_upsert_nodes(&inputs).unwrap(); + let ids = db.batch_upsert_nodes(inputs).unwrap(); let seqs: Vec = ids .iter() - .map(|&id| db.get_node(id).unwrap().unwrap().last_write_seq) + .map(|&id| internal_node_record(&db, id).unwrap().unwrap().last_write_seq) .collect(); // Each op in the batch should get a distinct, increasing seq @@ -6397,26 +7212,26 @@ fn test_compaction_preserves_edge_last_write_seq() { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let nid1 = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let nid2 = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let eid = db - .upsert_edge(nid1, nid2, 1, UpsertEdgeOptions::default()) + .upsert_edge(nid1, nid2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - let edge_seq = db.get_edge(eid).unwrap().unwrap().last_write_seq; + let edge_seq = internal_edge_record(&db, eid).unwrap().unwrap().last_write_seq; assert!(edge_seq > 0); db.flush().unwrap(); // Add second segment so compaction has something to merge - db.upsert_node(1, "c", UpsertNodeOptions::default()) + db.upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); - let edge_after = db.get_edge(eid).unwrap().unwrap(); + let edge_after = internal_edge_record(&db, eid).unwrap().unwrap(); assert_eq!(edge_after.last_write_seq, edge_seq); db.close().unwrap(); @@ -6426,20 +7241,23 @@ fn test_compaction_preserves_edge_last_write_seq() { #[test] fn test_get_edge_hydrates_last_write_seq_from_segment() { - // Regression: M1. get_edge() must hydrate last_write_seq from edge_meta.dat. + // Regression: M1. get_edge() must hydrate last_write_seq from edge metadata. let dir = tempfile::tempdir().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let nid1 = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let nid2 = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let eid = db - .upsert_edge(nid1, nid2, 1, UpsertEdgeOptions::default()) + .upsert_edge(nid1, nid2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - let memtable_seq = db.get_edge(eid).unwrap().unwrap().last_write_seq; + let memtable_seq = internal_edge_record(&db, eid) + .unwrap() + .unwrap() + .last_write_seq; assert!( memtable_seq > 0, "edge in memtable must have last_write_seq > 0" @@ -6447,8 +7265,8 @@ fn test_get_edge_hydrates_last_write_seq_from_segment() { db.flush().unwrap(); - // Edge is now in a segment. get_edge must hydrate last_write_seq from edge_meta.dat. - let segment_edge = db.get_edge(eid).unwrap().unwrap(); + // Edge is now in a segment. get_edge must hydrate last_write_seq from edge metadata. + let segment_edge = internal_edge_record(&db, eid).unwrap().unwrap(); assert_eq!( segment_edge.last_write_seq, memtable_seq, "get_edge from segment must preserve last_write_seq (got {}, expected {})", @@ -6460,22 +7278,22 @@ fn test_get_edge_hydrates_last_write_seq_from_segment() { #[test] fn test_get_nodes_batch_hydrates_last_write_seq_from_segment() { - // Regression: M2. get_nodes_batch must hydrate last_write_seq from node_meta.dat. + // Regression: M2. get_nodes_batch must hydrate last_write_seq from node metadata. let dir = tempfile::tempdir().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id1 = db - .upsert_node(1, "n1", UpsertNodeOptions::default()) + .upsert_node("Person", "n1", UpsertNodeOptions::default()) .unwrap(); let id2 = db - .upsert_node(1, "n2", UpsertNodeOptions::default()) + .upsert_node("Person", "n2", UpsertNodeOptions::default()) .unwrap(); let id3 = db - .upsert_node(1, "n3", UpsertNodeOptions::default()) + .upsert_node("Person", "n3", UpsertNodeOptions::default()) .unwrap(); - let seq1 = db.get_node(id1).unwrap().unwrap().last_write_seq; - let seq2 = db.get_node(id2).unwrap().unwrap().last_write_seq; - let seq3 = db.get_node(id3).unwrap().unwrap().last_write_seq; + let seq1 = internal_node_record(&db, id1).unwrap().unwrap().last_write_seq; + let seq2 = internal_node_record(&db, id2).unwrap().unwrap().last_write_seq; + let seq3 = internal_node_record(&db, id3).unwrap().unwrap().last_write_seq; db.flush().unwrap(); @@ -6502,36 +7320,45 @@ fn test_get_nodes_batch_hydrates_last_write_seq_from_segment() { #[test] fn test_get_edges_batch_hydrates_last_write_seq_from_segment() { - // Regression: M3. get_edges_batch must hydrate last_write_seq from edge_meta.dat. + // Regression: M3. get_edges_batch must hydrate last_write_seq from edge metadata. let dir = tempfile::tempdir().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let nid1 = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let nid2 = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let nid3 = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let eid1 = db - .upsert_edge(nid1, nid2, 1, UpsertEdgeOptions::default()) + .upsert_edge(nid1, nid2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let eid2 = db - .upsert_edge(nid2, nid3, 1, UpsertEdgeOptions::default()) + .upsert_edge(nid2, nid3, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let eid3 = db - .upsert_edge(nid1, nid3, 1, UpsertEdgeOptions::default()) + .upsert_edge(nid1, nid3, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - let eseq1 = db.get_edge(eid1).unwrap().unwrap().last_write_seq; - let eseq2 = db.get_edge(eid2).unwrap().unwrap().last_write_seq; - let eseq3 = db.get_edge(eid3).unwrap().unwrap().last_write_seq; + let eseq1 = internal_edge_record(&db, eid1) + .unwrap() + .unwrap() + .last_write_seq; + let eseq2 = internal_edge_record(&db, eid2) + .unwrap() + .unwrap() + .last_write_seq; + let eseq3 = internal_edge_record(&db, eid3) + .unwrap() + .unwrap() + .last_write_seq; db.flush().unwrap(); // Batch read from segment via get_edges (uses get_edges_batch internally) - let results = db.get_edges(&[eid1, eid2, eid3]).unwrap(); + let results = internal_edge_records(&db, &[eid1, eid2, eid3]).unwrap(); assert_eq!( results[0].as_ref().unwrap().last_write_seq, eseq1, @@ -6559,7 +7386,7 @@ fn test_tombstone_last_write_seq_survives_flush_reopen() { { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id = db - .upsert_node(1, "doomed", UpsertNodeOptions::default()) + .upsert_node("Person", "doomed", UpsertNodeOptions::default()) .unwrap(); db.delete_node(id).unwrap(); // The delete op gets its own engine_seq @@ -6594,7 +7421,7 @@ fn test_tombstone_survives_flush_reopen() { { let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); deleted_id = db - .upsert_node(1, "doomed", UpsertNodeOptions::default()) + .upsert_node("Person", "doomed", UpsertNodeOptions::default()) .unwrap(); db.delete_node(deleted_id).unwrap(); db.flush().unwrap(); @@ -6618,7 +7445,7 @@ fn test_source_list_find_node_across_segments() { let dir = tempfile::tempdir().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id = db - .upsert_node(1, "seg-node", UpsertNodeOptions::default()) + .upsert_node("Person", "seg-node", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -6628,7 +7455,7 @@ fn test_source_list_find_node_across_segments() { // Upsert same key again (goes to memtable, segment has older version) db.upsert_node( - 1, + "Person", "seg-node", UpsertNodeOptions { weight: 2.0, @@ -6647,7 +7474,7 @@ fn test_source_list_find_node_tombstoned_in_memtable_segment_has_record() { let dir = tempfile::tempdir().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id = db - .upsert_node(1, "will-die", UpsertNodeOptions::default()) + .upsert_node("Person", "will-die", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -6671,46 +7498,46 @@ fn test_source_list_find_edge_by_triple_across_segment() { ) .unwrap(); let n1 = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - let eid = db.upsert_edge(n1, n2, 1, Default::default()).unwrap(); + let eid = db.upsert_edge(n1, n2, "RELATES_TO", Default::default()).unwrap(); db.flush().unwrap(); // Edge is in segment. get_edge_by_triple uses SourceList.find_edge_by_triple. - let edge = db.get_edge_by_triple(n1, n2, 1).unwrap().unwrap(); + let edge = db.get_edge_by_triple(n1, n2, "RELATES_TO").unwrap().unwrap(); assert_eq!(edge.id, eid); // Delete the edge, then check triple lookup returns None db.delete_edge(eid).unwrap(); - assert!(db.get_edge_by_triple(n1, n2, 1).unwrap().is_none()); + assert!(db.get_edge_by_triple(n1, n2, "RELATES_TO").unwrap().is_none()); db.close().unwrap(); } #[test] -fn test_source_list_find_node_by_key_across_segment() { +fn test_source_list_find_node_by_label_key_across_segment() { let dir = tempfile::tempdir().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id = db - .upsert_node(1, "keyed", UpsertNodeOptions::default()) + .upsert_node("Person", "keyed", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); // Key lookup should find the node in the segment - let node = db.get_node_by_key(1, "keyed").unwrap().unwrap(); + let node = db.get_node_by_key("Person", "keyed").unwrap().unwrap(); assert_eq!(node.id, id); // Delete and verify key lookup returns None db.delete_node(id).unwrap(); - assert!(db.get_node_by_key(1, "keyed").unwrap().is_none()); + assert!(db.get_node_by_key("Person", "keyed").unwrap().is_none()); db.close().unwrap(); } -// --- WAL generation / freeze / immutable memtable tests (Phase 21 CP3) --- +// --- WAL generation / freeze / immutable memtable tests --- #[test] fn test_freeze_creates_immutable_memtable() { @@ -6719,9 +7546,9 @@ fn test_freeze_creates_immutable_memtable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Write some data - db.upsert_node(1, "alice", UpsertNodeOptions::default()) + db.upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "bob", UpsertNodeOptions::default()) + db.upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); assert_eq!(db.immutable_memtable_count(), 0); @@ -6759,15 +7586,20 @@ fn test_write_after_freeze_goes_to_new_generation() { let db_path = dir.path().join("freeze_write"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + // This test is about data writes crossing WAL generations. Seed the label + // token through the internal catalog so the first public write does not add + // a token-definition WAL op to generation 0. + seed_internal_node_labels(&db, &[1]).unwrap(); + // Write before freeze let id_a = db - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); // Write after freeze - should go to new active memtable let id_b = db - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); // Active memtable should only have the post-freeze write @@ -6806,10 +7638,10 @@ fn test_flush_with_wal_generations() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Write some data, freeze, then write more - db.upsert_node(1, "alice", UpsertNodeOptions::default()) + db.upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_node(1, "bob", UpsertNodeOptions::default()) + db.upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); // Flush should process all (freeze active + flush both immutables) @@ -6820,7 +7652,7 @@ fn test_flush_with_wal_generations() { assert_eq!(db.immutable_memtable_count(), 0); // Both nodes should be readable from segments - let nodes = db.get_nodes_by_type(1).unwrap(); + let nodes = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(nodes.len(), 2); // Old WAL generation files should be retired @@ -6841,10 +7673,10 @@ fn test_replay_multiple_wal_generations() { // Session 1: write data, freeze (creates gen 0 frozen + gen 1 active), close without flush { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "alice", UpsertNodeOptions::default()) + db.upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_node(1, "bob", UpsertNodeOptions::default()) + db.upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); // close_fast: doesn't flush immutables, just syncs active WAL and writes manifest @@ -6863,86 +7695,22 @@ fn test_replay_multiple_wal_generations() { // Session 2: reopen, verify all data is present via WAL replay { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let nodes = db.get_nodes_by_type(1).unwrap(); + let nodes = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( nodes.len(), 2, "both nodes should be replayed from WAL generations" ); - let alice = db.get_node_by_key(1, "alice").unwrap(); + let alice = db.get_node_by_key("Person", "alice").unwrap(); assert!(alice.is_some(), "alice should be found after WAL replay"); - let bob = db.get_node_by_key(1, "bob").unwrap(); + let bob = db.get_node_by_key("Person", "bob").unwrap(); assert!(bob.is_some(), "bob should be found after WAL replay"); db.close().unwrap(); } } -#[test] -fn test_data_wal_migration() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("migration_test"); - - // Simulate old-format DB: create a data.wal file manually - std::fs::create_dir_all(&db_path).unwrap(); - - // Write a legacy data.wal with a node - { - let mut writer = WalWriter::open(&db_path).unwrap(); - let node = NodeRecord { - id: 1, - type_id: 1, - key: "legacy_node".to_string(), - props: BTreeMap::new(), - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }; - writer.append(&WalOp::UpsertNode(node), 1).unwrap(); - writer.sync().unwrap(); - } - - // Verify data.wal exists and wal_0.wal does not - assert!(db_path.join("data.wal").exists()); - assert!(!wal_generation_path(&db_path, 0).exists()); - - // Open with the engine; should trigger migration. - { - let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - // data.wal should have been renamed to wal_0.wal - assert!( - !db_path.join("data.wal").exists(), - "data.wal should be migrated away" - ); - assert!( - wal_generation_path(&db_path, 0).exists(), - "wal_0.wal should exist after migration" - ); - - // Node should be readable from replayed WAL - let node = db.get_node_by_key(1, "legacy_node").unwrap(); - assert!( - node.is_some(), - "legacy node should be found after migration replay" - ); - - db.close().unwrap(); - } - - // Reopen to verify it works with the new format - { - let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let node = db.get_node_by_key(1, "legacy_node").unwrap(); - assert!(node.is_some(), "legacy node should persist across reopens"); - db.close().unwrap(); - } -} - #[test] fn test_freeze_and_read_from_immutable() { let dir = TempDir::new().unwrap(); @@ -6957,10 +7725,10 @@ fn test_freeze_and_read_from_immutable() { // Write nodes let id_a = db - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let id_b = db - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); // Freeze: data moves to immutable epoch. @@ -6990,18 +7758,18 @@ fn test_freeze_and_read_from_immutable() { assert_eq!(bob.unwrap().key, "bob"); // Key lookups should also work - let alice_by_key = db.get_node_by_key(1, "alice").unwrap(); + let alice_by_key = db.get_node_by_key("Person", "alice").unwrap(); assert!( alice_by_key.is_some(), "alice should be findable by key while in-flight" ); - // Type query should return both - let all = db.get_nodes_by_type(1).unwrap(); + // Label query should return both + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 2, - "get_nodes_by_type should see in-flight epoch data" + "get_nodes_by_labels should see in-flight epoch data" ); // Release worker, verify data moves to segment @@ -7030,23 +7798,23 @@ fn test_multiple_freezes_before_flush() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); // Write and freeze three times - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); assert_eq!(db.immutable_epoch_count(), 1); - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); assert_eq!(db.immutable_epoch_count(), 2); - db.upsert_node(1, "c", UpsertNodeOptions::default()) + db.upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); assert_eq!(db.immutable_epoch_count(), 3); // All data should be readable across 3 immutable epochs - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 3, @@ -7066,7 +7834,7 @@ fn test_multiple_freezes_before_flush() { // All 3 epochs still visible while first is in-flight assert_eq!(db.immutable_epoch_count(), 3); assert_eq!(db.in_flight_count(), 3); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 3, "all nodes visible during in-flight flush"); // Release and drain @@ -7076,7 +7844,7 @@ fn test_multiple_freezes_before_flush() { // Data should now be in segments assert_eq!(db.segment_count().unwrap(), 3); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 3); db.close().unwrap(); @@ -7090,9 +7858,9 @@ fn test_wal_generation_survives_close_fast() { // Write data and close_fast (no flush) { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "node1", UpsertNodeOptions::default()) + db.upsert_node("Person", "node1", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "node2", UpsertNodeOptions::default()) + db.upsert_node("Person", "node2", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); } @@ -7100,7 +7868,7 @@ fn test_wal_generation_survives_close_fast() { // Reopen and verify data is present { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 2, "data should survive close_fast + reopen"); db.close().unwrap(); } @@ -7113,7 +7881,7 @@ fn test_flush_retires_wal_generations() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Write data - db.upsert_node(1, "node1", UpsertNodeOptions::default()) + db.upsert_node("Person", "node1", UpsertNodeOptions::default()) .unwrap(); // After flush, the old WAL generation should be deleted @@ -7153,7 +7921,7 @@ fn test_bg_flush_writes_continue_during_flush() { // Write first batch and freeze for i in 0..50 { - db.upsert_node(1, &format!("pre:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("pre:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.freeze_memtable().unwrap(); @@ -7166,12 +7934,12 @@ fn test_bg_flush_writes_continue_during_flush() { // Write more data to the active memtable while flush is in-flight for i in 0..50 { - db.upsert_node(1, &format!("post:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("post:{}", i), UpsertNodeOptions::default()) .unwrap(); } // All 100 nodes visible: 50 from in-flight immutable epoch + 50 from active - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 100, @@ -7184,7 +7952,7 @@ fn test_bg_flush_writes_continue_during_flush() { db.flush().unwrap(); // All 100 nodes still visible, now from segments - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 100, "all nodes should be visible after bg flush"); assert!(db.segment_count().unwrap() >= 2, "should have at least 2 segments"); assert_eq!(db.immutable_epoch_count(), 0); @@ -7210,7 +7978,7 @@ fn test_bg_flush_multiple_immutables() { for batch in 0..4 { for i in 0..10 { db.upsert_node( - 1, + "Person", &format!("batch{}:node{}", batch, i), UpsertNodeOptions::default(), ) @@ -7231,7 +7999,7 @@ fn test_bg_flush_multiple_immutables() { // All 4 epochs in-flight, all 40 nodes visible assert_eq!(db.immutable_epoch_count(), 4); assert_eq!(db.in_flight_count(), 4); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 40, "all 40 nodes visible during in-flight flush"); // Release and drain all @@ -7246,7 +8014,7 @@ fn test_bg_flush_multiple_immutables() { ); // All 40 nodes in segments - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 40, "all 40 nodes should be present in segments"); // Reopen and verify persistence @@ -7254,7 +8022,7 @@ fn test_bg_flush_multiple_immutables() { db.close().unwrap(); let db2 = DatabaseEngine::open(&path, &DbOptions::default()).unwrap(); - let all2 = db2.get_nodes_by_type(1).unwrap(); + let all2 = db2.get_nodes_by_labels("Person").unwrap(); assert_eq!(all2.len(), 40, "all 40 nodes should survive reopen"); db2.close().unwrap(); } @@ -7268,7 +8036,7 @@ fn test_bg_flush_close_drains_all() { // Write data for i in 0..20 { - db.upsert_node(1, &format!("drain:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("drain:{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -7276,7 +8044,7 @@ fn test_bg_flush_close_drains_all() { db.flush().unwrap(); // All data should be in segments now - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 20); // Close should succeed without losing data @@ -7285,7 +8053,7 @@ fn test_bg_flush_close_drains_all() { // Reopen and verify let db2 = DatabaseEngine::open(&path, &DbOptions::default()).unwrap(); - let all2 = db2.get_nodes_by_type(1).unwrap(); + let all2 = db2.get_nodes_by_labels("Person").unwrap(); assert_eq!(all2.len(), 20, "all nodes should survive close + reopen"); db2.close().unwrap(); } @@ -7301,7 +8069,7 @@ fn test_bg_flush_close_fast_preserves_recovery() { // Write data to memtable but don't flush for i in 0..15 { - db.upsert_node(1, &format!("fast:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("fast:{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -7312,7 +8080,7 @@ fn test_bg_flush_close_fast_preserves_recovery() { // Reopen: WAL replay should recover all data. { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 15, @@ -7333,14 +8101,14 @@ fn test_bg_flush_close_fast_with_frozen_memtables() { // Write and freeze for i in 0..10 { - db.upsert_node(1, &format!("frozen:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("frozen:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.freeze_memtable().unwrap(); // Write more to active memtable for i in 10..20 { - db.upsert_node(1, &format!("active:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("active:{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -7351,7 +8119,7 @@ fn test_bg_flush_close_fast_with_frozen_memtables() { // Reopen: WAL should recover all data from both generations. { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 20, @@ -7368,7 +8136,7 @@ fn test_shutdown_bg_flush_resets_stale_in_flight_epochs() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "stale-in-flight", UpsertNodeOptions::default()) + db.upsert_node("Person", "stale-in-flight", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); @@ -7389,7 +8157,7 @@ fn test_shutdown_bg_flush_resets_stale_in_flight_epochs() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let node = db.get_node_by_key(1, "stale-in-flight").unwrap(); + let node = db.get_node_by_key("Person", "stale-in-flight").unwrap(); assert!(node.is_some()); db.close().unwrap(); } @@ -7405,18 +8173,18 @@ fn test_stale_frozen_epochs_cleaned_on_reopen_then_flush_works() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("stale_epoch"); - // Phase 1: write, freeze, close_fast (leaves FrozenPendingFlush) + // Step 1: write, freeze, close_fast (leaves FrozenPendingFlush) { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "before_crash", UpsertNodeOptions::default()) + db.upsert_node("Person", "before_crash", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_node(1, "active_at_crash", UpsertNodeOptions::default()) + db.upsert_node("Person", "active_at_crash", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); } - // Phase 2: reopen. Frozen epoch is rebuilt as immutable, not cleaned up. + // Step 2: reopen. Frozen epoch is rebuilt as immutable, not cleaned up. { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); @@ -7430,22 +8198,22 @@ fn test_stale_frozen_epochs_cleaned_on_reopen_then_flush_works() { assert_eq!(db.immutable_epoch_count(), 1); // Both nodes should be recovered via WAL replay - assert!(db.get_node_by_key(1, "before_crash").unwrap().is_some()); - assert!(db.get_node_by_key(1, "active_at_crash").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "before_crash").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "active_at_crash").unwrap().is_some()); // Now do a new write cycle: freeze + flush should work cleanly - db.upsert_node(1, "after_reopen", UpsertNodeOptions::default()) + db.upsert_node("Person", "after_reopen", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_node(1, "post_freeze", UpsertNodeOptions::default()) + db.upsert_node("Person", "post_freeze", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); // Verify all 4 nodes present - assert!(db.get_node_by_key(1, "before_crash").unwrap().is_some()); - assert!(db.get_node_by_key(1, "active_at_crash").unwrap().is_some()); - assert!(db.get_node_by_key(1, "after_reopen").unwrap().is_some()); - assert!(db.get_node_by_key(1, "post_freeze").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "before_crash").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "active_at_crash").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "after_reopen").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "post_freeze").unwrap().is_some()); // All epochs drained after flush assert!( @@ -7456,10 +8224,10 @@ fn test_stale_frozen_epochs_cleaned_on_reopen_then_flush_works() { db.close().unwrap(); } - // Phase 3: final reopen to confirm persistence + // Step 3: final reopen to confirm persistence { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 4, "all 4 nodes should survive full cycle"); db.close().unwrap(); } @@ -7472,29 +8240,29 @@ fn test_repeated_crash_after_freeze_preserves_data() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("double_crash"); - // Phase 1: write, freeze, simulate crash + // Step 1: write, freeze, simulate crash { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "survivor", UpsertNodeOptions::default()) + db.upsert_node("Person", "survivor", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); db.close_fast().unwrap(); } - // Phase 2: reopen (rebuilds frozen as immutable), then crash again + // Step 2: reopen (rebuilds frozen as immutable), then crash again { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert!(db.get_node_by_key(1, "survivor").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "survivor").unwrap().is_some()); assert_eq!(db.immutable_epoch_count(), 1); // Crash without flushing; close_fast doesn't flush. db.close_fast().unwrap(); } - // Phase 3: reopen again. Data must survive the double crash. + // Step 3: reopen again. Data must survive the double crash. { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); assert!( - db.get_node_by_key(1, "survivor").unwrap().is_some(), + db.get_node_by_key("Person", "survivor").unwrap().is_some(), "data must survive two crashes without flush" ); assert_eq!(db.immutable_epoch_count(), 1); @@ -7506,10 +8274,10 @@ fn test_repeated_crash_after_freeze_preserves_data() { db.close().unwrap(); } - // Phase 4: final verification + // Step 4: final verification { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert!(db.get_node_by_key(1, "survivor").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "survivor").unwrap().is_some()); db.close().unwrap(); } } @@ -7533,7 +8301,7 @@ fn test_multi_freeze_flush_retires_each_wal_gen() { // Freeze 3 separate batches → creates WAL gens 0, 1, 2 (active = 3) for batch in 0..3 { - db.upsert_node(1, &format!("batch{}", batch), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("batch{}", batch), UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); } @@ -7579,7 +8347,7 @@ fn test_multi_freeze_flush_retires_each_wal_gen() { // All data visible from immutable epochs for batch in 0..3 { assert!( - db.get_node_by_key(1, &format!("batch{}", batch)) + db.get_node_by_key("Person", &format!("batch{}", batch)) .unwrap() .is_some(), "batch{} should be visible during in-flight", @@ -7616,7 +8384,7 @@ fn test_multi_freeze_flush_retires_each_wal_gen() { assert_eq!(db.segment_count().unwrap(), 3); for batch in 0..3 { assert!( - db.get_node_by_key(1, &format!("batch{}", batch)) + db.get_node_by_key("Person", &format!("batch{}", batch)) .unwrap() .is_some(), "batch{} should be visible in segments", @@ -7643,7 +8411,7 @@ fn test_flush_wait_loop_handles_worker_failure() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); // Create one epoch and inject a failure - db.upsert_node(1, "fail_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "fail_node", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); @@ -7658,7 +8426,7 @@ fn test_flush_wait_loop_handles_worker_failure() { assert_eq!(db.immutable_epoch_count(), 1); assert_eq!(db.in_flight_count(), 0); assert!( - db.get_node_by_key(1, "fail_node").unwrap().is_some(), + db.get_node_by_key("Person", "fail_node").unwrap().is_some(), "data should remain visible after worker failure" ); @@ -7667,7 +8435,7 @@ fn test_flush_wait_loop_handles_worker_failure() { assert_eq!(db.immutable_epoch_count(), 0); assert_eq!(db.segment_count().unwrap(), 1); assert!( - db.get_node_by_key(1, "fail_node").unwrap().is_some(), + db.get_node_by_key("Person", "fail_node").unwrap().is_some(), "data should be in segment after retry" ); @@ -7689,7 +8457,7 @@ fn test_write_after_reported_flush_failure_retries_in_background() { }; let db = DatabaseEngine::open(&db_path, &opts).unwrap(); - db.upsert_node(1, "fail_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "fail_node", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); @@ -7702,7 +8470,7 @@ fn test_write_after_reported_flush_failure_retries_in_background() { // Subsequent ordinary write should not be wedged by the already-reported // sticky error. It should restart the worker and enqueue the failed epoch. - db.upsert_node(1, "retry_trigger", UpsertNodeOptions::default()) + db.upsert_node("Person", "retry_trigger", UpsertNodeOptions::default()) .unwrap(); assert_eq!( db.in_flight_count(), @@ -7714,17 +8482,17 @@ fn test_write_after_reported_flush_failure_retries_in_background() { assert!(seg.is_some(), "retried epoch should flush successfully"); assert_eq!(db.immutable_epoch_count(), 0); assert!( - db.get_node_by_key(1, "fail_node").unwrap().is_some(), + db.get_node_by_key("Person", "fail_node").unwrap().is_some(), "failed epoch data should be published after retry" ); // Sticky error should clear after successful adoption of the failed epoch. - db.upsert_node(1, "after_clear", UpsertNodeOptions::default()) + db.upsert_node("Person", "after_clear", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); - assert!(db.get_node_by_key(1, "retry_trigger").unwrap().is_some()); - assert!(db.get_node_by_key(1, "after_clear").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "retry_trigger").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "after_clear").unwrap().is_some()); db.close().unwrap(); } @@ -7738,13 +8506,13 @@ fn test_get_edges_batch_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge e1 moves to immutable memtable @@ -7752,10 +8520,10 @@ fn test_get_edges_batch_sees_immutable() { // Create another edge in active memtable let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e2 = db - .upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // get_edges should see both @@ -7783,12 +8551,12 @@ fn test_neighbors_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: adjacency moves to immutable memtable @@ -7796,9 +8564,9 @@ fn test_neighbors_sees_immutable() { // Create another edge in active memtable let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // neighbors() should see both edges @@ -7839,7 +8607,7 @@ fn test_find_nodes_sees_immutable() { ); let _id_a = db .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: props.clone(), @@ -7853,7 +8621,7 @@ fn test_find_nodes_sees_immutable() { let _id_b = db .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props: props.clone(), @@ -7864,7 +8632,7 @@ fn test_find_nodes_sees_immutable() { // find_nodes should see both let found = db - .find_nodes(1, "status", &PropValue::String("active".to_string())) + .find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(); assert_eq!( found.len(), @@ -7876,63 +8644,63 @@ fn test_find_nodes_sees_immutable() { } #[test] -fn test_nodes_by_type_paged_sees_immutable() { +fn test_nodes_by_labels_paged_sees_immutable() { let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("type_paged_imm"); + let db_path = dir.path().join("label_paged_imm"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); // Freeze db.freeze_memtable().unwrap(); - db.upsert_node(1, "c", UpsertNodeOptions::default()) + db.upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - // nodes_by_type_paged should see all 3 - let page = db.nodes_by_type_paged(1, &PageRequest::default()).unwrap(); + // nodes_by_labels_paged should see all 3 + let page = db.nodes_by_labels_paged("Person", &PageRequest::default()).unwrap(); assert_eq!( page.items.len(), 3, - "nodes_by_type_paged should see nodes from both active and immutable memtables" + "nodes_by_labels_paged should see nodes from both active and immutable memtables" ); db.close().unwrap(); } #[test] -fn test_edges_by_type_paged_sees_immutable() { +fn test_edges_by_label_paged_sees_immutable() { let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("edge_type_paged_imm"); + let db_path = dir.path().join("edge_label_paged_imm"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze db.freeze_memtable().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - // edges_by_type_paged should see both edges - let page = db.edges_by_type_paged(10, &PageRequest::default()).unwrap(); + // edges_by_label_paged should see both edges + let page = db.edges_by_label_paged("KNOWS", &PageRequest::default()).unwrap(); assert_eq!( page.items.len(), 2, - "edges_by_type_paged should see edges from both active and immutable memtables" + "edges_by_label_paged should see edges from both active and immutable memtables" ); db.close().unwrap(); @@ -7945,20 +8713,20 @@ fn test_neighbors_batch_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let results = db @@ -7997,7 +8765,7 @@ fn test_dense_search_sees_immutable() { // Insert a node with dense vector db.upsert_node( - 1, + "Person", "vec_a", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0, 0.0]), @@ -8011,7 +8779,7 @@ fn test_dense_search_sees_immutable() { // Insert another node with dense vector in active memtable db.upsert_node( - 1, + "Person", "vec_b", UpsertNodeOptions { dense_vector: Some(vec![0.0, 1.0, 0.0]), @@ -8027,7 +8795,7 @@ fn test_dense_search_sees_immutable() { dense_query: Some(vec![1.0, 0.0, 0.0]), sparse_query: None, k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -8052,7 +8820,7 @@ fn test_sparse_search_sees_immutable() { // Insert node with sparse vector db.upsert_node( - 1, + "Person", "sp_a", UpsertNodeOptions { sparse_vector: Some(vec![(0, 1.0), (1, 0.5)]), @@ -8066,7 +8834,7 @@ fn test_sparse_search_sees_immutable() { // Insert another node with sparse vector in active memtable db.upsert_node( - 1, + "Person", "sp_b", UpsertNodeOptions { sparse_vector: Some(vec![(0, 0.5), (2, 1.0)]), @@ -8082,7 +8850,7 @@ fn test_sparse_search_sees_immutable() { dense_query: None, sparse_query: Some(vec![(0, 1.0)]), k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -8106,21 +8874,21 @@ fn test_degree_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // degree should count both edges @@ -8148,15 +8916,15 @@ fn test_top_k_neighbors_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.5, ..Default::default() @@ -8168,12 +8936,12 @@ fn test_top_k_neighbors_sees_immutable() { db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.8, ..Default::default() @@ -8206,17 +8974,17 @@ fn test_find_nodes_by_time_range_sees_immutable() { let db_path = dir.path().join("time_range_imm"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); // Freeze db.freeze_memtable().unwrap(); - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); // Both should appear in a wide time range query - let found = db.find_nodes_by_time_range(1, 0, i64::MAX).unwrap(); + let found = db.find_nodes_by_time_range("Person", 0, i64::MAX).unwrap(); assert_eq!( found.len(), 2, @@ -8234,10 +9002,10 @@ fn test_immutable_tombstones_respected() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); // Flush a and b to segments @@ -8249,8 +9017,8 @@ fn test_immutable_tombstones_respected() { // Freeze: tombstone for b moves to immutable memtable db.freeze_memtable().unwrap(); - // b should not be visible via nodes_by_type - let all = db.nodes_by_type(1).unwrap(); + // b should not be visible via nodes_by_labels + let all = db.nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 1, @@ -8280,7 +9048,7 @@ fn test_multiple_immutable_memtables_newest_wins() { // Write node A with weight 1.0 let id_a = db .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 1.0, @@ -8294,7 +9062,7 @@ fn test_multiple_immutable_memtables_newest_wins() { // Update node A with weight 2.0 db.upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 2.0, @@ -8315,8 +9083,8 @@ fn test_multiple_immutable_memtables_newest_wins() { node.weight ); - // Also test that nodes_by_type sees exactly 1 node (not duplicated) - let all = db.nodes_by_type(1).unwrap(); + // Also test that nodes_by_labels sees exactly 1 node (not duplicated) + let all = db.nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 1, @@ -8327,7 +9095,7 @@ fn test_multiple_immutable_memtables_newest_wins() { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String("red".to_string())); db.upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props, @@ -8337,7 +9105,7 @@ fn test_multiple_immutable_memtables_newest_wins() { .unwrap(); // Now active has B, immutable[0] has A (weight 2.0), immutable[1] has A (weight 1.0) - let found = db.nodes_by_type(1).unwrap(); + let found = db.nodes_by_labels("Person").unwrap(); assert_eq!( found.len(), 2, @@ -8355,7 +9123,7 @@ fn test_multiple_immutable_tombstone_shadows_older() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id = db - .upsert_node(1, "doomed", UpsertNodeOptions::default()) + .upsert_node("Person", "doomed", UpsertNodeOptions::default()) .unwrap(); // Freeze → immutable 1 (oldest, has the record) @@ -8370,7 +9138,7 @@ fn test_multiple_immutable_tombstone_shadows_older() { db.get_node(id).unwrap().is_none(), "tombstone in newer immutable should shadow record in older immutable" ); - assert_eq!(db.nodes_by_type(1).unwrap().len(), 0); + assert_eq!(db.nodes_by_labels("Person").unwrap().len(), 0); db.close().unwrap(); } @@ -8383,27 +9151,27 @@ fn test_export_adjacency_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge a→b moves to immutable db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let export = db.export_adjacency(&ExportOptions::default()).unwrap(); // Should have 3 nodes and 2 edges (a→b from immutable, a→c from active) assert_eq!(export.node_ids.len(), 3, "export should see all 3 nodes"); - let edges_from_a: Vec<_> = export.edges.iter().filter(|e| e.0 == a).collect(); + let edges_from_a: Vec<_> = export.edges.iter().filter(|e| e.from == a).collect(); assert_eq!( edges_from_a.len(), 2, @@ -8421,21 +9189,21 @@ fn test_connected_components_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge a→b moves to immutable db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // All three should be in the same component via a→b (immutable) + b→c (active) @@ -8462,21 +9230,21 @@ fn test_shortest_path_through_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge a→b moves to immutable db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // shortest_path(a, c) should find 2-hop path via b @@ -8516,7 +9284,7 @@ fn test_find_nodes_paged_sees_immutable() { props.insert("role".to_string(), PropValue::String("admin".to_string())); db.upsert_node( - 1, + "Person", "user_a", UpsertNodeOptions { props: props.clone(), @@ -8529,7 +9297,7 @@ fn test_find_nodes_paged_sees_immutable() { db.freeze_memtable().unwrap(); db.upsert_node( - 1, + "Person", "user_b", UpsertNodeOptions { props: props.clone(), @@ -8540,8 +9308,7 @@ fn test_find_nodes_paged_sees_immutable() { // find_nodes_paged should see both let page = db - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "role", &PropValue::String("admin".to_string()), &PageRequest::default(), @@ -8567,7 +9334,7 @@ fn test_upsert_node_dedup_across_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id1 = db - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); // Freeze: alice moves to immutable memtable @@ -8576,7 +9343,7 @@ fn test_upsert_node_dedup_across_immutable() { // Upsert alice again; should find her in immutable and reuse ID. let id2 = db .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.9, @@ -8613,13 +9380,13 @@ fn test_edge_uniqueness_across_immutable() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge e1 moves to immutable memtable @@ -8630,7 +9397,7 @@ fn test_edge_uniqueness_across_immutable() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.7, ..Default::default() @@ -8661,7 +9428,7 @@ fn test_batch_upsert_node_dedup_across_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id_alice = db - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); // Freeze @@ -8670,7 +9437,7 @@ fn test_batch_upsert_node_dedup_across_immutable() { // Batch upsert that includes alice (should reuse ID) + new node bob let inputs = vec![ NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "alice".to_string(), props: BTreeMap::new(), weight: 0.8, @@ -8678,7 +9445,7 @@ fn test_batch_upsert_node_dedup_across_immutable() { sparse_vector: None, }, NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "bob".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -8686,7 +9453,7 @@ fn test_batch_upsert_node_dedup_across_immutable() { sparse_vector: None, }, ]; - let ids = db.batch_upsert_nodes(&inputs).unwrap(); + let ids = db.batch_upsert_nodes(inputs).unwrap(); assert_eq!( ids[0], id_alice, @@ -8709,27 +9476,27 @@ fn test_batch_upsert_edge_uniqueness_across_immutable() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze db.freeze_memtable().unwrap(); - // Batch: re-upsert (a->b, type 10) + new (a->c, type 10) + // Batch: re-upsert (a->b, label 10) + new (a->c, label 10) let inputs = vec![ EdgeInput { from: a, to: b, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 0.5, valid_from: None, @@ -8738,14 +9505,14 @@ fn test_batch_upsert_edge_uniqueness_across_immutable() { EdgeInput { from: a, to: c, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }, ]; - let ids = db.batch_upsert_edges(&inputs).unwrap(); + let ids = db.batch_upsert_edges(inputs).unwrap(); assert_eq!( ids[0], e1, @@ -8765,16 +9532,16 @@ fn test_delete_node_cascades_immutable_edges() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge e1 (a->b) moves to immutable memtable @@ -8782,7 +9549,7 @@ fn test_delete_node_cascades_immutable_edges() { // Add another edge in active memtable let e2 = db - .upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Delete node a; should cascade-delete both e1 (immutable) and e2 (active). @@ -8819,13 +9586,13 @@ fn test_invalidate_edge_in_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge e1 moves to immutable memtable @@ -8856,13 +9623,13 @@ fn test_graph_patch_dedup_across_immutable() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); let a_id = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b_id = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a_id, b_id, 10, UpsertEdgeOptions::default()) + .upsert_edge(a_id, b_id, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze @@ -8873,7 +9640,7 @@ fn test_graph_patch_dedup_across_immutable() { let patch = GraphPatch { upsert_nodes: vec![ NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "a".to_string(), props: BTreeMap::new(), weight: 0.5, @@ -8881,7 +9648,7 @@ fn test_graph_patch_dedup_across_immutable() { sparse_vector: None, }, NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "c".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -8892,7 +9659,7 @@ fn test_graph_patch_dedup_across_immutable() { upsert_edges: vec![EdgeInput { from: a_id, to: b_id, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 0.3, valid_from: None, @@ -8903,7 +9670,7 @@ fn test_graph_patch_dedup_across_immutable() { delete_edge_ids: vec![], }; - let result = db.graph_patch(&patch).unwrap(); + let result = db.graph_patch(patch).unwrap(); assert_eq!( result.node_ids[0], a_id, @@ -8926,13 +9693,13 @@ fn test_graph_patch_delete_cascades_immutable_edges() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge e1 moves to immutable memtable @@ -8946,7 +9713,7 @@ fn test_graph_patch_delete_cascades_immutable_edges() { delete_node_ids: vec![a], delete_edge_ids: vec![], }; - db.graph_patch(&patch).unwrap(); + db.graph_patch(patch).unwrap(); assert!( db.get_node(a).unwrap().is_none(), @@ -8962,16 +9729,16 @@ fn test_graph_patch_delete_cascades_immutable_edges() { #[test] fn test_prune_finds_targets_in_immutable_memtable() { - // Prune with no type_id filter must scan immutable memtables for targets. + // Prune with no label filter must scan immutable memtables for targets. // This is the bug we fixed: collect_prune_targets skipped immutable memtables - // in the else (no type_id) branch. + // in the else (no label) branch. let dir = TempDir::new().unwrap(); let db_path = dir.path().join("prune_imm"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Insert a low-weight node db.upsert_node( - 1, + "Person", "old_low", UpsertNodeOptions { weight: 0.1, @@ -8985,7 +9752,7 @@ fn test_prune_finds_targets_in_immutable_memtable() { // Insert a high-weight node in active memtable db.upsert_node( - 1, + "Person", "new_high", UpsertNodeOptions { weight: 5.0, @@ -8994,12 +9761,12 @@ fn test_prune_finds_targets_in_immutable_memtable() { ) .unwrap(); - // Prune nodes with weight <= 0.5 (no type filter) + // Prune nodes with weight <= 0.5 (no label filter) let result = db .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -9009,7 +9776,7 @@ fn test_prune_finds_targets_in_immutable_memtable() { ); // The low-weight node should be gone, high-weight should survive - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 1, "only the high-weight node should survive"); assert_eq!(all[0].key, "new_high"); @@ -9027,7 +9794,7 @@ fn test_prune_respects_tombstones_in_immutable_memtable() { // Create and flush a low-weight node to segment let id = db .upsert_node( - 1, + "Person", "target", UpsertNodeOptions { weight: 0.1, @@ -9048,7 +9815,7 @@ fn test_prune_respects_tombstones_in_immutable_memtable() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -9069,10 +9836,10 @@ fn test_id_allocation_stable_across_freeze() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); // Freeze @@ -9080,10 +9847,10 @@ fn test_id_allocation_stable_across_freeze() { // New nodes after freeze should get IDs > existing IDs let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let d = db - .upsert_node(1, "d", UpsertNodeOptions::default()) + .upsert_node("Person", "d", UpsertNodeOptions::default()) .unwrap(); assert!( @@ -9096,11 +9863,11 @@ fn test_id_allocation_stable_across_freeze() { // Same for edges let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); let e2 = db - .upsert_edge(c, d, 10, UpsertEdgeOptions::default()) + .upsert_edge(c, d, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert!( e2 > e1, @@ -9108,7 +9875,7 @@ fn test_id_allocation_stable_across_freeze() { ); // All 4 nodes should be readable - assert_eq!(db.get_nodes_by_type(1).unwrap().len(), 4); + assert_eq!(db.get_nodes_by_labels("Person").unwrap().len(), 4); db.close().unwrap(); } @@ -9127,17 +9894,17 @@ fn test_traversal_sees_immutable_edges() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: both edges move to immutable memtable @@ -9176,13 +9943,13 @@ fn test_graph_patch_invalidate_edge_in_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze @@ -9196,7 +9963,7 @@ fn test_graph_patch_invalidate_edge_in_immutable() { delete_node_ids: vec![], delete_edge_ids: vec![], }; - db.graph_patch(&patch).unwrap(); + db.graph_patch(patch).unwrap(); // Edge should still exist but have valid_to = 500 let edge = db.get_edge(e1).unwrap().unwrap(); @@ -9219,37 +9986,37 @@ fn test_dedup_across_active_immutable_and_segments() { // Tier 1: write and flush to segment let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let seg_edge = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); // Tier 2: write and freeze to immutable let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let imm_edge = db - .upsert_edge(a, c, 20, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); // Tier 3: write to active memtable let d = db - .upsert_node(1, "d", UpsertNodeOptions::default()) + .upsert_node("Person", "d", UpsertNodeOptions::default()) .unwrap(); let act_edge = db - .upsert_edge(a, d, 30, UpsertEdgeOptions::default()) + .upsert_edge(a, d, "RATES", UpsertEdgeOptions::default()) .unwrap(); // Re-upsert node "a"; should find it in segment and reuse ID. let a2 = db .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.99, @@ -9261,7 +10028,7 @@ fn test_dedup_across_active_immutable_and_segments() { // Re-upsert edge a->c; should find it in immutable memtable. let imm_edge2 = db - .upsert_edge(a, c, 20, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); assert_eq!( imm_edge, imm_edge2, @@ -9270,12 +10037,12 @@ fn test_dedup_across_active_immutable_and_segments() { // Re-upsert edge a->b; should find it in segment. let seg_edge2 = db - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(seg_edge, seg_edge2, "must reuse edge ID from segment"); // All 4 nodes and 3 edges should exist - assert_eq!(db.get_nodes_by_type(1).unwrap().len(), 4); + assert_eq!(db.get_nodes_by_labels("Person").unwrap().len(), 4); let nbrs = db .neighbors( a, @@ -9313,19 +10080,19 @@ fn test_write_dedup_across_multiple_immutables() { // Freeze 1: alice + edge a->b in oldest immutable let id_alice = db - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let id_bob = db - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); let e1 = db - .upsert_edge(id_alice, id_bob, 10, UpsertEdgeOptions::default()) + .upsert_edge(id_alice, id_bob, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); // Freeze 2: charlie in newer immutable let id_charlie = db - .upsert_node(1, "charlie", UpsertNodeOptions::default()) + .upsert_node("Person", "charlie", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); @@ -9333,7 +10100,7 @@ fn test_write_dedup_across_multiple_immutables() { // charlie in immutable[0], and bob in immutable[1]. No new IDs allocated. let inputs = vec![ NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "alice".to_string(), props: BTreeMap::new(), weight: 0.8, @@ -9341,7 +10108,7 @@ fn test_write_dedup_across_multiple_immutables() { sparse_vector: None, }, NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "bob".to_string(), props: BTreeMap::new(), weight: 0.9, @@ -9349,7 +10116,7 @@ fn test_write_dedup_across_multiple_immutables() { sparse_vector: None, }, NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "charlie".to_string(), props: BTreeMap::new(), weight: 0.7, @@ -9357,7 +10124,7 @@ fn test_write_dedup_across_multiple_immutables() { sparse_vector: None, }, ]; - let ids = db.batch_upsert_nodes(&inputs).unwrap(); + let ids = db.batch_upsert_nodes(inputs).unwrap(); assert_eq!( ids[0], id_alice, "alice should reuse ID from older immutable" @@ -9370,12 +10137,12 @@ fn test_write_dedup_across_multiple_immutables() { // Re-upsert edge a->b; should find it in older immutable. let e2 = db - .upsert_edge(id_alice, id_bob, 10, UpsertEdgeOptions::default()) + .upsert_edge(id_alice, id_bob, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(e1, e2, "edge should reuse ID from older immutable"); // Total should still be 3 nodes - assert_eq!(db.get_nodes_by_type(1).unwrap().len(), 3); + assert_eq!(db.get_nodes_by_labels("Person").unwrap().len(), 3); db.close().unwrap(); } @@ -9390,21 +10157,21 @@ fn test_degrees_batch_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: edge a→b moves to immutable db.freeze_memtable().unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Batch degrees should count edges from both active and immutable @@ -9435,15 +10202,15 @@ fn test_sum_edge_weights_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.5, ..Default::default() @@ -9455,12 +10222,12 @@ fn test_sum_edge_weights_sees_immutable() { db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 1.5, ..Default::default() @@ -9509,21 +10276,21 @@ fn test_neighbors_paged_sees_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let page = db @@ -9565,7 +10332,7 @@ fn test_dense_search_tombstone_in_immutable_hides_result() { // Insert two nodes with dense vectors, flush to segments let id_a = db .upsert_node( - 1, + "Person", "vec_a", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0, 0.0]), @@ -9575,7 +10342,7 @@ fn test_dense_search_tombstone_in_immutable_hides_result() { .unwrap(); let id_b = db .upsert_node( - 1, + "Person", "vec_b", UpsertNodeOptions { dense_vector: Some(vec![0.9, 0.1, 0.0]), @@ -9596,7 +10363,7 @@ fn test_dense_search_tombstone_in_immutable_hides_result() { dense_query: Some(vec![1.0, 0.0, 0.0]), sparse_query: None, k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -9628,7 +10395,7 @@ fn test_sparse_search_tombstone_in_immutable_hides_result() { let id_a = db .upsert_node( - 1, + "Person", "sp_a", UpsertNodeOptions { sparse_vector: Some(vec![(0, 1.0), (1, 0.5)]), @@ -9638,7 +10405,7 @@ fn test_sparse_search_tombstone_in_immutable_hides_result() { .unwrap(); let id_b = db .upsert_node( - 1, + "Person", "sp_b", UpsertNodeOptions { sparse_vector: Some(vec![(0, 0.8), (2, 1.0)]), @@ -9658,7 +10425,7 @@ fn test_sparse_search_tombstone_in_immutable_hides_result() { dense_query: None, sparse_query: Some(vec![(0, 1.0)]), k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -9701,7 +10468,7 @@ fn test_dense_scoped_search_sees_immutable() { // Hub node (scope start) let hub = db .upsert_node( - 1, + "Person", "hub", UpsertNodeOptions { dense_vector: Some(vec![0.5, 0.5, 0.0]), @@ -9712,7 +10479,7 @@ fn test_dense_scoped_search_sees_immutable() { let id_a = db .upsert_node( - 1, + "Person", "vec_a", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0, 0.0]), @@ -9720,7 +10487,7 @@ fn test_dense_scoped_search_sees_immutable() { }, ) .unwrap(); - db.upsert_edge(hub, id_a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(hub, id_a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: node A + edge hub→A move to immutable @@ -9728,7 +10495,7 @@ fn test_dense_scoped_search_sees_immutable() { let id_b = db .upsert_node( - 1, + "Person", "vec_b", UpsertNodeOptions { dense_vector: Some(vec![0.0, 1.0, 0.0]), @@ -9736,7 +10503,7 @@ fn test_dense_scoped_search_sees_immutable() { }, ) .unwrap(); - db.upsert_edge(hub, id_b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(hub, id_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Scoped search from hub: traversal discovers A (immutable) and B (active). @@ -9746,13 +10513,13 @@ fn test_dense_scoped_search_sees_immutable() { dense_query: Some(vec![1.0, 0.0, 0.0]), sparse_query: None, k: 10, - type_filter: None, + label_filter: None, ef_search: None, scope: Some(VectorSearchScope { start_node_id: hub, max_depth: 1, direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, at_epoch: None, }), dense_weight: None, @@ -9782,15 +10549,15 @@ fn test_dijkstra_shortest_path_through_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 1.0, ..Default::default() @@ -9802,12 +10569,12 @@ fn test_dijkstra_shortest_path_through_immutable() { db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( b, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -9847,21 +10614,21 @@ fn test_all_shortest_paths_through_immutable() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze db.freeze_memtable().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // BFS variant (no weight_field) @@ -9909,9 +10676,9 @@ fn test_crash_after_freeze_before_flush() { // Write data to gen 0 let id_a = db - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(id_a, id_a, 10, UpsertEdgeOptions::default()) + db.upsert_edge(id_a, id_a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: gen 0 becomes frozen, gen 1 becomes active @@ -9921,7 +10688,7 @@ fn test_crash_after_freeze_before_flush() { // Write data to gen 1 (active) let id_b = db - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); assert_ne!(id_a, id_b); @@ -9948,13 +10715,13 @@ fn test_crash_after_freeze_before_flush() { // Reopen and verify ALL data recovered { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let alice = db.get_node_by_key(1, "alice").unwrap(); + let alice = db.get_node_by_key("Person", "alice").unwrap(); assert!(alice.is_some(), "alice from gen 0 should be recovered"); - let bob = db.get_node_by_key(1, "bob").unwrap(); + let bob = db.get_node_by_key("Person", "bob").unwrap(); assert!(bob.is_some(), "bob from gen 1 should be recovered"); // Edge from gen 0 should also be recovered - let all_nodes = db.get_nodes_by_type(1).unwrap(); + let all_nodes = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all_nodes.len(), 2, "both nodes should be present"); db.close().unwrap(); @@ -9972,9 +10739,9 @@ fn test_crash_with_flushed_segment_and_unflushed_wal() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Write and flush -- segment is created and manifest is updated - db.upsert_node(1, "flushed_1", UpsertNodeOptions::default()) + db.upsert_node("Person", "flushed_1", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "flushed_2", UpsertNodeOptions::default()) + db.upsert_node("Person", "flushed_2", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -9982,9 +10749,9 @@ fn test_crash_with_flushed_segment_and_unflushed_wal() { assert!(db.segment_count().unwrap() >= 1); // Write more data (unflushed) - db.upsert_node(1, "unflushed_1", UpsertNodeOptions::default()) + db.upsert_node("Person", "unflushed_1", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "unflushed_2", UpsertNodeOptions::default()) + db.upsert_node("Person", "unflushed_2", UpsertNodeOptions::default()) .unwrap(); // Simulate crash after writes but before another flush @@ -9996,18 +10763,18 @@ fn test_crash_with_flushed_segment_and_unflushed_wal() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Flushed data should be in segments - let f1 = db.get_node_by_key(1, "flushed_1").unwrap(); + let f1 = db.get_node_by_key("Person", "flushed_1").unwrap(); assert!(f1.is_some(), "flushed_1 from segment should be present"); - let f2 = db.get_node_by_key(1, "flushed_2").unwrap(); + let f2 = db.get_node_by_key("Person", "flushed_2").unwrap(); assert!(f2.is_some(), "flushed_2 from segment should be present"); // Unflushed data should be recovered from WAL replay - let u1 = db.get_node_by_key(1, "unflushed_1").unwrap(); + let u1 = db.get_node_by_key("Person", "unflushed_1").unwrap(); assert!(u1.is_some(), "unflushed_1 should be recovered from WAL"); - let u2 = db.get_node_by_key(1, "unflushed_2").unwrap(); + let u2 = db.get_node_by_key("Person", "unflushed_2").unwrap(); assert!(u2.is_some(), "unflushed_2 should be recovered from WAL"); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 4, "all 4 nodes should be present"); db.close().unwrap(); @@ -10024,9 +10791,9 @@ fn test_crash_after_segment_write_before_manifest_publish() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "alice", UpsertNodeOptions::default()) + db.upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "bob", UpsertNodeOptions::default()) + db.upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); // Freeze: data goes to immutable, epoch recorded as FrozenPendingFlush @@ -10040,7 +10807,11 @@ fn test_crash_after_segment_write_before_manifest_publish() { // wrote the segment but crash happened before manifest publish). let orphan_seg = segment_dir(&db_path, 9999); std::fs::create_dir_all(&orphan_seg).unwrap(); - std::fs::write(orphan_seg.join("nodes.dat"), b"dummy").unwrap(); + std::fs::write( + orphan_seg.join(crate::segment_components::PACKED_CORE_FILENAME), + b"dummy", + ) + .unwrap(); // Verify: manifest has FrozenPendingFlush, orphan segment exists let manifest = load_manifest(&db_path).unwrap().unwrap(); @@ -10060,14 +10831,14 @@ fn test_crash_after_segment_write_before_manifest_publish() { // Data recovered from WAL replay assert!( - db.get_node_by_key(1, "alice").unwrap().is_some(), + db.get_node_by_key("Person", "alice").unwrap().is_some(), "alice recovered from WAL" ); assert!( - db.get_node_by_key(1, "bob").unwrap().is_some(), + db.get_node_by_key("Person", "bob").unwrap().is_some(), "bob recovered from WAL" ); - assert_eq!(db.get_nodes_by_type(1).unwrap().len(), 2); + assert_eq!(db.get_nodes_by_labels("Person").unwrap().len(), 2); // Orphan segment cleaned up assert!( @@ -10091,17 +10862,17 @@ fn test_crash_with_multiple_frozen_generations() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Gen 0: write and freeze - db.upsert_node(1, "gen0_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "gen0_node", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); // Gen 1: write and freeze - db.upsert_node(1, "gen1_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "gen1_node", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); // Gen 2 (active): write - db.upsert_node(1, "gen2_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "gen2_node", UpsertNodeOptions::default()) .unwrap(); assert_eq!(db.immutable_memtable_count(), 2); @@ -10129,19 +10900,19 @@ fn test_crash_with_multiple_frozen_generations() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); assert!( - db.get_node_by_key(1, "gen0_node").unwrap().is_some(), + db.get_node_by_key("Person", "gen0_node").unwrap().is_some(), "gen0 data recovered" ); assert!( - db.get_node_by_key(1, "gen1_node").unwrap().is_some(), + db.get_node_by_key("Person", "gen1_node").unwrap().is_some(), "gen1 data recovered" ); assert!( - db.get_node_by_key(1, "gen2_node").unwrap().is_some(), + db.get_node_by_key("Person", "gen2_node").unwrap().is_some(), "gen2 data recovered" ); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 3, "all 3 generations of data should be present"); db.close().unwrap(); @@ -10165,7 +10936,7 @@ fn test_crash_after_publish_before_wal_retire() { // Write data and flush -- creates segment and retires WAL node_id = db - .upsert_node(1, "published_node", UpsertNodeOptions::default()) + .upsert_node("Person", "published_node", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -10192,7 +10963,7 @@ fn test_crash_after_publish_before_wal_retire() { let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); let node = NodeRecord { id: node_id, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "published_node".to_string(), props: BTreeMap::new(), created_at: 1000, @@ -10222,7 +10993,7 @@ fn test_crash_after_publish_before_wal_retire() { assert_eq!(node.unwrap().key, "published_node"); // No duplicate data -- still just 1 node - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 1, @@ -10259,7 +11030,7 @@ fn test_reopen_fails_if_published_pending_retire_segment_is_missing() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); node_id = db - .upsert_node(1, "published_node", UpsertNodeOptions::default()) + .upsert_node("Person", "published_node", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); seg_id = db.manifest().unwrap().segments[0].id; @@ -10279,7 +11050,7 @@ fn test_reopen_fails_if_published_pending_retire_segment_is_missing() { let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); let node = NodeRecord { id: node_id, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "published_node".to_string(), props: BTreeMap::new(), created_at: 1000, @@ -10339,7 +11110,7 @@ fn test_crash_after_wal_delete_before_epoch_removal() { let seg_id; { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "survivor", UpsertNodeOptions::default()) + db.upsert_node("Person", "survivor", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); seg_id = db.manifest().unwrap().segments[0].id; @@ -10370,7 +11141,7 @@ fn test_crash_after_wal_delete_before_epoch_removal() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Data should be intact from segment - let node = db.get_node_by_key(1, "survivor").unwrap(); + let node = db.get_node_by_key("Person", "survivor").unwrap(); assert!(node.is_some(), "node should be readable from segment"); // Stale epoch should be cleaned from manifest @@ -10393,7 +11164,7 @@ fn test_orphan_segment_ignored_on_reopen_cp8() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "real_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "real_node", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); assert_eq!(db.segment_count().unwrap(), 1); @@ -10418,7 +11189,7 @@ fn test_orphan_segment_ignored_on_reopen_cp8() { // Real segment data should be intact assert_eq!(db.segment_count().unwrap(), 1); - let node = db.get_node_by_key(1, "real_node").unwrap(); + let node = db.get_node_by_key("Person", "real_node").unwrap(); assert!(node.is_some(), "real node should still be readable"); db.close().unwrap(); @@ -10434,7 +11205,7 @@ fn test_orphan_wal_generation_ignored() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "real_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "real_node", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); } @@ -10444,7 +11215,7 @@ fn test_orphan_wal_generation_ignored() { let mut writer = WalWriter::open_generation(&db_path, 99).unwrap(); let orphan_node = NodeRecord { id: 999, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "orphan_ghost".to_string(), props: BTreeMap::new(), created_at: 5000, @@ -10468,7 +11239,7 @@ fn test_orphan_wal_generation_ignored() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Real data should be present - let real = db.get_node_by_key(1, "real_node").unwrap(); + let real = db.get_node_by_key("Person", "real_node").unwrap(); assert!(real.is_some(), "real_node should be recovered"); // Orphan data should NOT be present @@ -10478,7 +11249,7 @@ fn test_orphan_wal_generation_ignored() { "orphan ghost node from unreferenced WAL gen 99 should NOT be replayed" ); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 1, "only real_node should exist"); // Orphan WAL file should be cleaned up @@ -10504,7 +11275,7 @@ fn test_reopen_replays_frozen_epochs_oldest_first() { // Gen 0: create node with initial value db.upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 1.0, @@ -10516,7 +11287,7 @@ fn test_reopen_replays_frozen_epochs_oldest_first() { // Gen 1: update the same node (different weight to distinguish) db.upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 2.0, @@ -10528,7 +11299,7 @@ fn test_reopen_replays_frozen_epochs_oldest_first() { // Gen 2: update again db.upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 3.0, @@ -10543,7 +11314,7 @@ fn test_reopen_replays_frozen_epochs_oldest_first() { // Reopen -- newest value should win { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let node = db.get_node_by_key(1, "shared").unwrap(); + let node = db.get_node_by_key("Person", "shared").unwrap(); assert!(node.is_some(), "shared node should be recovered"); let node = node.unwrap(); assert!( @@ -10553,7 +11324,7 @@ fn test_reopen_replays_frozen_epochs_oldest_first() { ); // Should be exactly 1 node (not 3 copies) - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 1, "upsert dedup should produce exactly 1 node"); db.close().unwrap(); @@ -10574,18 +11345,18 @@ fn test_published_pending_retire_not_replayed() { // Write 5 nodes and an edge, flush to segment let ids: Vec = (0..5) .map(|i| { - db.upsert_node(1, &format!("node_{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("node_{}", i), UpsertNodeOptions::default()) .unwrap() }) .collect(); let _edge_id = db - .upsert_edge(ids[0], ids[1], 10, UpsertEdgeOptions::default()) + .upsert_edge(ids[0], ids[1], "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); assert!(db.segment_count().unwrap() >= 1); // Write more data after flush (this goes to the new active WAL gen) - db.upsert_node(1, "post_flush_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "post_flush_node", UpsertNodeOptions::default()) .unwrap(); db.close().unwrap(); @@ -10609,7 +11380,7 @@ fn test_published_pending_retire_not_replayed() { for i in 0..5 { let node = NodeRecord { id: i + 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: format!("node_{}", i), props: BTreeMap::new(), created_at: 1000, @@ -10629,7 +11400,7 @@ fn test_published_pending_retire_not_replayed() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Should have exactly 6 nodes: 5 from segment + 1 post-flush - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 6, @@ -10664,24 +11435,24 @@ fn test_reopen_after_flush_then_more_writes() { // Write and flush for i in 0..10 { - db.upsert_node(1, &format!("flushed_{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("flushed_{}", i), UpsertNodeOptions::default()) .unwrap(); } db.flush().unwrap(); // Write more (not flushed) for i in 0..10 { - db.upsert_node(1, &format!("unflushed_{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("unflushed_{}", i), UpsertNodeOptions::default()) .unwrap(); } // Add edge spanning flushed and unflushed nodes - let flushed_node = db.get_node_by_key(1, "flushed_0").unwrap().unwrap(); - let unflushed_node = db.get_node_by_key(1, "unflushed_0").unwrap().unwrap(); + let flushed_node = db.get_node_by_key("Person", "flushed_0").unwrap().unwrap(); + let unflushed_node = db.get_node_by_key("Person", "unflushed_0").unwrap().unwrap(); db.upsert_edge( flushed_node.id, unflushed_node.id, - 10, + "KNOWS", UpsertEdgeOptions::default(), ) .unwrap(); @@ -10697,7 +11468,7 @@ fn test_reopen_after_flush_then_more_writes() { for i in 0..10 { let key = format!("flushed_{}", i); assert!( - db.get_node_by_key(1, &key).unwrap().is_some(), + db.get_node_by_key("Person", &key).unwrap().is_some(), "{} should be present from segment", key ); @@ -10707,13 +11478,13 @@ fn test_reopen_after_flush_then_more_writes() { for i in 0..10 { let key = format!("unflushed_{}", i); assert!( - db.get_node_by_key(1, &key).unwrap().is_some(), + db.get_node_by_key("Person", &key).unwrap().is_some(), "{} should be present from WAL replay", key ); } - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 20, "all 20 nodes should be present"); db.close().unwrap(); @@ -10731,21 +11502,21 @@ fn test_multiple_flush_reopen_cycles() { // Cycle 1: write + flush for i in 0..5 { - db.upsert_node(1, &format!("cycle1_{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("cycle1_{}", i), UpsertNodeOptions::default()) .unwrap(); } db.flush().unwrap(); // Cycle 2: write + flush for i in 0..5 { - db.upsert_node(1, &format!("cycle2_{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("cycle2_{}", i), UpsertNodeOptions::default()) .unwrap(); } db.flush().unwrap(); // Cycle 3: write (no flush -- stays in WAL) for i in 0..5 { - db.upsert_node(1, &format!("cycle3_{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("cycle3_{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -10760,14 +11531,14 @@ fn test_multiple_flush_reopen_cycles() { for i in 0..5 { let key = format!("cycle{}_{}", cycle, i); assert!( - db.get_node_by_key(1, &key).unwrap().is_some(), + db.get_node_by_key("Person", &key).unwrap().is_some(), "{} should be present", key ); } } - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 15, @@ -10786,38 +11557,38 @@ fn test_close_fast_then_close_normally() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("fast_then_normal"); - // Phase 1: Write, freeze, close_fast (simulate crash) + // Step 1: Write, freeze, close_fast (simulate crash) { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "surviving_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "surviving_node", UpsertNodeOptions::default()) .unwrap(); let a = db - .upsert_node(1, "node_a", UpsertNodeOptions::default()) + .upsert_node("Person", "node_a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "node_b", UpsertNodeOptions::default()) + .upsert_node("Person", "node_b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); // Write more to active - db.upsert_node(1, "active_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "active_node", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); } - // Phase 2: Reopen, verify data, flush, close normally + // Step 2: Reopen, verify data, flush, close normally { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // All data should be present via WAL replay - assert!(db.get_node_by_key(1, "surviving_node").unwrap().is_some()); - assert!(db.get_node_by_key(1, "node_a").unwrap().is_some()); - assert!(db.get_node_by_key(1, "node_b").unwrap().is_some()); - assert!(db.get_node_by_key(1, "active_node").unwrap().is_some()); - let all = db.get_nodes_by_type(1).unwrap(); + assert!(db.get_node_by_key("Person", "surviving_node").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "node_a").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "node_b").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "active_node").unwrap().is_some()); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 4, "all 4 nodes should be present after recovery"); // Now flush and close normally @@ -10828,15 +11599,15 @@ fn test_close_fast_then_close_normally() { db.close().unwrap(); } - // Phase 3: Reopen again, verify everything is clean + // Step 3: Reopen again, verify everything is clean { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert!(db.get_node_by_key(1, "surviving_node").unwrap().is_some()); - assert!(db.get_node_by_key(1, "node_a").unwrap().is_some()); - assert!(db.get_node_by_key(1, "node_b").unwrap().is_some()); - assert!(db.get_node_by_key(1, "active_node").unwrap().is_some()); - let all = db.get_nodes_by_type(1).unwrap(); + assert!(db.get_node_by_key("Person", "surviving_node").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "node_a").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "node_b").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "active_node").unwrap().is_some()); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 4); // Manifest should be clean -- no pending flush epochs @@ -10865,13 +11636,13 @@ fn test_crash_recovery_preserves_edges() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); node_a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); node_b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); edge_ab = db - .upsert_edge(node_a, node_b, 10, UpsertEdgeOptions::default()) + .upsert_edge(node_a, node_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Freeze: nodes a,b and edge_ab move to immutable @@ -10879,10 +11650,10 @@ fn test_crash_recovery_preserves_edges() { // Write more in active gen node_c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); edge_bc = db - .upsert_edge(node_b, node_c, 20, UpsertEdgeOptions::default()) + .upsert_edge(node_b, node_c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); // Simulate crash @@ -10935,13 +11706,13 @@ fn test_crash_recovery_preserves_deletes() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let node_a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let node_b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let _edge = db - .upsert_edge(node_a, node_b, 10, UpsertEdgeOptions::default()) + .upsert_edge(node_a, node_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); @@ -10959,18 +11730,18 @@ fn test_crash_recovery_preserves_deletes() { // node_a should be deleted assert!( - db.get_node_by_key(1, "a").unwrap().is_none(), + db.get_node_by_key("Person", "a").unwrap().is_none(), "deleted node_a should not be visible after recovery" ); // node_b should still exist assert!( - db.get_node_by_key(1, "b").unwrap().is_some(), + db.get_node_by_key("Person", "b").unwrap().is_some(), "non-deleted node_b should survive recovery" ); // Only 1 node should be visible - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 1, "only node_b should be visible"); db.close().unwrap(); @@ -10988,16 +11759,16 @@ fn test_reopen_engine_seq_continuity() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Write several items - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_node(1, "c", UpsertNodeOptions::default()) + db.upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_node(1, "d", UpsertNodeOptions::default()) + db.upsert_node("Person", "d", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); @@ -11008,17 +11779,17 @@ fn test_reopen_engine_seq_continuity() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Write a new node -- its seq should be > pre-crash values - db.upsert_node(1, "post_crash", UpsertNodeOptions::default()) + db.upsert_node("Person", "post_crash", UpsertNodeOptions::default()) .unwrap(); // The node should exist - assert!(db.get_node_by_key(1, "post_crash").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "post_crash").unwrap().is_some()); // All pre-crash data should be present - assert!(db.get_node_by_key(1, "a").unwrap().is_some()); - assert!(db.get_node_by_key(1, "b").unwrap().is_some()); - assert!(db.get_node_by_key(1, "c").unwrap().is_some()); - assert!(db.get_node_by_key(1, "d").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "a").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "b").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "c").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "d").unwrap().is_some()); db.close().unwrap(); } @@ -11033,7 +11804,7 @@ fn test_repeated_crash_reopen_cycles() { // Crash cycle 1 { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "cycle1_node", UpsertNodeOptions::default()) + db.upsert_node("Person", "cycle1_node", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); } @@ -11042,11 +11813,11 @@ fn test_repeated_crash_reopen_cycles() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Verify cycle 1 data - assert!(db.get_node_by_key(1, "cycle1_node").unwrap().is_some()); - db.upsert_node(1, "cycle2_node", UpsertNodeOptions::default()) + assert!(db.get_node_by_key("Person", "cycle1_node").unwrap().is_some()); + db.upsert_node("Person", "cycle2_node", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_node(1, "cycle2_active", UpsertNodeOptions::default()) + db.upsert_node("Person", "cycle2_active", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); } @@ -11055,13 +11826,13 @@ fn test_repeated_crash_reopen_cycles() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Verify cycle 1 + 2 data - assert!(db.get_node_by_key(1, "cycle1_node").unwrap().is_some()); - assert!(db.get_node_by_key(1, "cycle2_node").unwrap().is_some()); - assert!(db.get_node_by_key(1, "cycle2_active").unwrap().is_some()); - db.upsert_node(1, "cycle3_node", UpsertNodeOptions::default()) + assert!(db.get_node_by_key("Person", "cycle1_node").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "cycle2_node").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "cycle2_active").unwrap().is_some()); + db.upsert_node("Person", "cycle3_node", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_node(1, "cycle3_unflushed", UpsertNodeOptions::default()) + db.upsert_node("Person", "cycle3_unflushed", UpsertNodeOptions::default()) .unwrap(); db.close_fast().unwrap(); } @@ -11069,13 +11840,13 @@ fn test_repeated_crash_reopen_cycles() { // Final verification { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert!(db.get_node_by_key(1, "cycle1_node").unwrap().is_some()); - assert!(db.get_node_by_key(1, "cycle2_node").unwrap().is_some()); - assert!(db.get_node_by_key(1, "cycle2_active").unwrap().is_some()); - assert!(db.get_node_by_key(1, "cycle3_node").unwrap().is_some()); - assert!(db.get_node_by_key(1, "cycle3_unflushed").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "cycle1_node").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "cycle2_node").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "cycle2_active").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "cycle3_node").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "cycle3_unflushed").unwrap().is_some()); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 5, @@ -11096,7 +11867,7 @@ fn test_backpressure_triggers_on_total_bytes() { // just the active memtable. let dir = TempDir::new().unwrap(); let db_path = dir.path().join("bp_total"); - // Each node is ~190 bytes (120 base + key + type/time index overhead). + // Each node is ~190 bytes (120 base + key + label/time index overhead). // Threshold at 350 bytes: 1 node won't trigger, but 2 nodes will. let opts = DbOptions { memtable_flush_threshold: 350, // triggers when total > 350 bytes @@ -11109,7 +11880,7 @@ fn test_backpressure_triggers_on_total_bytes() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); // Write 1 node (under threshold) and freeze. - db.upsert_node(1, "frozen", UpsertNodeOptions::default()) + db.upsert_node("Person", "frozen", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); assert_eq!(db.immutable_memtable_count(), 1); @@ -11117,7 +11888,7 @@ fn test_backpressure_triggers_on_total_bytes() { // Now write 1 more node. Active ~190 + immutable ~190 = ~380, exceeding // the 350-byte threshold. auto-flush should fire (async). - db.upsert_node(1, "active", UpsertNodeOptions::default()) + db.upsert_node("Person", "active", UpsertNodeOptions::default()) .unwrap(); // Auto-flush is now async, so drain pending flushes before asserting. @@ -11154,7 +11925,7 @@ fn test_max_immutable_memtables_blocks() { // Write and freeze twice to reach max_immutable_memtables=2 for i in 0..5 { - db.upsert_node(1, &format!("g1:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("g1:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.freeze_memtable().unwrap(); @@ -11162,7 +11933,7 @@ fn test_max_immutable_memtables_blocks() { assert_eq!(db.segment_count().unwrap(), 0); for i in 0..5 { - db.upsert_node(1, &format!("g2:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("g2:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.freeze_memtable().unwrap(); @@ -11172,7 +11943,7 @@ fn test_max_immutable_memtables_blocks() { // Now write to the active memtable. The next write triggers // backpressure because immutable count == max_immutable_memtables. for i in 0..5 { - db.upsert_node(1, &format!("g3:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("g3:{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -11183,7 +11954,7 @@ fn test_max_immutable_memtables_blocks() { ); // All data should be readable - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 15, "all 15 nodes should be visible"); db.close().unwrap(); @@ -11208,7 +11979,7 @@ fn test_max_immutable_memtables_disabled_when_zero() { for batch in 0..5 { for i in 0..3 { db.upsert_node( - 1, + "Person", &format!("b{}:{}", batch, i), UpsertNodeOptions::default(), ) @@ -11250,7 +12021,7 @@ fn test_close_drains_all_immutables() { for batch in 0..3 { for i in 0..5 { db.upsert_node( - 1, + "Person", &format!("b{}:{}", batch, i), UpsertNodeOptions::default(), ) @@ -11261,7 +12032,7 @@ fn test_close_drains_all_immutables() { // Write more to active memtable for i in 0..5 { - db.upsert_node(1, &format!("active:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("active:{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -11275,7 +12046,7 @@ fn test_close_drains_all_immutables() { // Reopen and verify all data is in segments { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 20, "all 20 nodes should survive close + reopen"); // Data should be in segments, not memtable (WAL was retired) @@ -11316,7 +12087,7 @@ fn test_close_fast_preserves_wal_for_recovery() { for batch in 0..3 { for i in 0..5 { db.upsert_node( - 1, + "Person", &format!("b{}:{}", batch, i), UpsertNodeOptions::default(), ) @@ -11327,7 +12098,7 @@ fn test_close_fast_preserves_wal_for_recovery() { // Write to active memtable for i in 0..5 { - db.upsert_node(1, &format!("active:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("active:{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -11341,7 +12112,7 @@ fn test_close_fast_preserves_wal_for_recovery() { // Reopen: WAL replay recovers everything. { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!( all.len(), 20, @@ -11378,14 +12149,14 @@ fn test_compaction_respects_flush_published_segments() { // Create first segment for i in 0..10 { - db.upsert_node(1, &format!("seg1:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("seg1:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.flush().unwrap(); // Create second segment for i in 0..10 { - db.upsert_node(1, &format!("seg2:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("seg2:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.flush().unwrap(); @@ -11400,7 +12171,7 @@ fn test_compaction_respects_flush_published_segments() { // Create a third segment (published after compaction) for i in 0..10 { - db.upsert_node(1, &format!("seg3:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("seg3:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.flush().unwrap(); @@ -11409,13 +12180,13 @@ fn test_compaction_respects_flush_published_segments() { assert_eq!(db.segment_count().unwrap(), 2); // All 30 nodes should be readable - let all = db.get_nodes_by_type(1).unwrap(); + let all = db.get_nodes_by_labels("Person").unwrap(); assert_eq!(all.len(), 30); // Compact again to verify new segments coexist properly db.compact().unwrap(); assert_eq!(db.segment_count().unwrap(), 1); - assert_eq!(db.get_nodes_by_type(1).unwrap().len(), 30); + assert_eq!(db.get_nodes_by_labels("Person").unwrap().len(), 30); db.close().unwrap(); } @@ -11442,14 +12213,14 @@ fn test_close_with_active_and_immutable_data() { // Write to active, then freeze let id = db - .upsert_node(1, "frozen_a", UpsertNodeOptions::default()) + .upsert_node("Person", "frozen_a", UpsertNodeOptions::default()) .unwrap(); node_ids.push(id); db.freeze_memtable().unwrap(); // Write more to new active let id = db - .upsert_node(1, "active_b", UpsertNodeOptions::default()) + .upsert_node("Person", "active_b", UpsertNodeOptions::default()) .unwrap(); node_ids.push(id); @@ -11462,7 +12233,7 @@ fn test_close_with_active_and_immutable_data() { for &id in &node_ids { assert!(db.get_node(id).unwrap().is_some()); } - assert_eq!(db.get_nodes_by_type(1).unwrap().len(), 2); + assert_eq!(db.get_nodes_by_labels("Person").unwrap().len(), 2); assert!(db.segment_count().unwrap() >= 1, "close() should have flushed"); db.close().unwrap(); } @@ -11504,14 +12275,14 @@ fn test_backpressure_bytes_and_count_combined() { // Freeze once to reach max_immutable_memtables=1 for i in 0..3 { - db.upsert_node(1, &format!("x:{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("x:{}", i), UpsertNodeOptions::default()) .unwrap(); } db.freeze_memtable().unwrap(); assert_eq!(db.immutable_memtable_count(), 1); // Next write should trigger count-based backpressure - db.upsert_node(1, "trigger", UpsertNodeOptions::default()) + db.upsert_node("Person", "trigger", UpsertNodeOptions::default()) .unwrap(); // Flush should have happened @@ -11539,7 +12310,7 @@ fn test_data_visible_while_in_flight() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); let id = db - .upsert_node(1, "visible", UpsertNodeOptions::default()) + .upsert_node("Person", "visible", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); assert_eq!(db.immutable_epoch_count(), 1); @@ -11555,7 +12326,7 @@ fn test_data_visible_while_in_flight() { // All read paths must see the frozen data assert!(db.get_node(id).unwrap().is_some()); - assert!(db.get_node_by_key(1, "visible").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "visible").unwrap().is_some()); // Release worker, wait for completion release_tx.send(()).unwrap(); @@ -11586,7 +12357,7 @@ fn test_multiple_epochs_all_visible_during_flush() { // Gen 1: oldest frozen let id1 = db .upsert_node( - 1, + "Person", "gen1_key", UpsertNodeOptions { weight: 1.0, @@ -11599,7 +12370,7 @@ fn test_multiple_epochs_all_visible_during_flush() { // Gen 2: newest frozen let id2 = db .upsert_node( - 1, + "Person", "gen2_key", UpsertNodeOptions { weight: 2.0, @@ -11612,7 +12383,7 @@ fn test_multiple_epochs_all_visible_during_flush() { // Gen 3: active memtable let id3 = db .upsert_node( - 1, + "Person", "gen3_key", UpsertNodeOptions { weight: 3.0, @@ -11637,9 +12408,9 @@ fn test_multiple_epochs_all_visible_during_flush() { // Precedence: if same key existed across generations, active wins // (we used different keys, so just check all exist) - assert!(db.get_node_by_key(1, "gen1_key").unwrap().is_some()); - assert!(db.get_node_by_key(1, "gen2_key").unwrap().is_some()); - assert!(db.get_node_by_key(1, "gen3_key").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "gen1_key").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "gen2_key").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "gen3_key").unwrap().is_some()); // Release, drain release_tx.send(()).unwrap(); @@ -11672,7 +12443,7 @@ fn test_auto_flush_is_async_not_blocking() { let mut ids = Vec::new(); for i in 0..5 { let id = db - .upsert_node(1, &format!("af_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("af_{}", i), UpsertNodeOptions::default()) .unwrap(); ids.push(id); } @@ -11711,7 +12482,7 @@ fn test_apply_removes_epoch_after_publish() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); let id = db - .upsert_node(1, "apply_test", UpsertNodeOptions::default()) + .upsert_node("Person", "apply_test", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); assert_eq!(db.immutable_epoch_count(), 1); @@ -11741,7 +12512,7 @@ fn test_worker_failure_keeps_epoch_visible() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); let id = db - .upsert_node(1, "fail_test", UpsertNodeOptions::default()) + .upsert_node("Person", "fail_test", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); @@ -11759,7 +12530,7 @@ fn test_worker_failure_keeps_epoch_visible() { // Data still readable assert!(db.get_node(id).unwrap().is_some()); - assert!(db.get_node_by_key(1, "fail_test").unwrap().is_some()); + assert!(db.get_node_by_key("Person", "fail_test").unwrap().is_some()); db.close().unwrap(); } @@ -11779,10 +12550,10 @@ fn test_backpressure_counts_all_epochs_including_in_flight() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); // Freeze twice to reach max - db.upsert_node(1, "bp1", UpsertNodeOptions::default()) + db.upsert_node("Person", "bp1", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); - db.upsert_node(1, "bp2", UpsertNodeOptions::default()) + db.upsert_node("Person", "bp2", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); assert_eq!(db.immutable_epoch_count(), 2); @@ -11819,7 +12590,7 @@ fn test_flush_sync_barrier_drains_all_epochs() { // Create 3 frozen memtables for i in 0..3 { - db.upsert_node(1, &format!("sync_{}", i), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("sync_{}", i), UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); } @@ -11833,7 +12604,7 @@ fn test_flush_sync_barrier_drains_all_epochs() { // All data readable from segments for i in 0..3 { assert!(db - .get_node_by_key(1, &format!("sync_{}", i)) + .get_node_by_key("Person", &format!("sync_{}", i)) .unwrap() .is_some()); } @@ -11886,7 +12657,7 @@ fn test_async_flush_latency_profile() { for i in 0..WRITE_COUNT { let start = Instant::now(); - db.upsert_node(1, &format!("n{}", i), write_opts(i)) + db.upsert_node("Person", &format!("n{}", i), write_opts(i)) .unwrap(); if (i + 1) % SYNC_FLUSH_INTERVAL == 0 { db.flush().unwrap(); @@ -11912,7 +12683,7 @@ fn test_async_flush_latency_profile() { for i in 0..WRITE_COUNT { let start = Instant::now(); - db.upsert_node(1, &format!("n{}", i), write_opts(i)) + db.upsert_node("Person", &format!("n{}", i), write_opts(i)) .unwrap(); latencies.push(start.elapsed().as_micros()); } @@ -11982,7 +12753,7 @@ fn test_property_index_manifest_reopens_and_reseeds_active_memtable() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); index_id = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap() .index_id; db.close().unwrap(); @@ -12003,7 +12774,7 @@ fn test_property_index_manifest_reopens_and_reseeds_active_memtable() { props.insert("color".to_string(), PropValue::String("red".to_string())); let node_id = db .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props, @@ -12043,7 +12814,7 @@ fn test_ensure_property_index_while_flush_in_flight_preserves_manifest_and_seedi ); let node_id = db .upsert_node( - 1, + "Person", "frozen", UpsertNodeOptions { props, @@ -12058,7 +12829,7 @@ fn test_ensure_property_index_while_flush_in_flight_preserves_manifest_and_seedi ready_rx.recv().unwrap(); let info = db - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); assert_eq!(info.state, SecondaryIndexState::Building); let status_hash = hash_prop_value(&PropValue::String("active".to_string())); @@ -12080,7 +12851,7 @@ fn test_ensure_property_index_while_flush_in_flight_preserves_manifest_and_seedi assert!(crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id).exists()); db.reset_property_query_routes(); assert_eq!( - db.find_nodes(1, "status", &PropValue::String("active".to_string())) + db.find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(), vec![node_id] ); @@ -12113,7 +12884,7 @@ fn test_ready_property_index_downgrades_when_flush_publish_missed_declaration_sn ); let node_id = db .upsert_node( - 1, + "Person", "frozen", UpsertNodeOptions { props, @@ -12128,7 +12899,7 @@ fn test_ready_property_index_downgrades_when_flush_publish_missed_declaration_sn publish_ready_rx.recv().unwrap(); let info = db - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); @@ -12150,7 +12921,7 @@ fn test_ready_property_index_downgrades_when_flush_publish_missed_declaration_sn db.reset_property_query_routes(); assert_eq!( - db.find_nodes(1, "status", &PropValue::String("active".to_string())) + db.find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(), vec![node_id] ); @@ -12160,11 +12931,13 @@ fn test_ready_property_index_downgrades_when_flush_publish_missed_declaration_sn repair_release_tx.send(()).unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); db.reset_property_query_routes(); assert_eq!( - db.find_nodes(1, "status", &PropValue::String("active".to_string())) + db.find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(), vec![node_id] ); @@ -12187,7 +12960,7 @@ fn test_published_property_query_route_stays_snapshot_stable_across_build_comple ); let node_id = db .upsert_node( - 1, + "Person", "snapshot-stable", UpsertNodeOptions { props, @@ -12198,7 +12971,7 @@ fn test_published_property_query_route_stays_snapshot_stable_across_build_comple let (build_ready_rx, build_release_tx) = db.set_secondary_index_build_pause(); let info = db - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); build_ready_rx.recv().unwrap(); @@ -12230,7 +13003,7 @@ fn test_published_property_query_route_stays_snapshot_stable_across_build_comple db.reset_property_query_routes(); assert_eq!( - db.find_nodes(1, "status", &PropValue::String("active".to_string())) + db.find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(), vec![node_id] ); @@ -12258,7 +13031,7 @@ fn test_ready_property_index_downgrades_when_bg_compaction_missed_declaration_sn first_props.insert("status".to_string(), active.clone()); let node_a = db .upsert_node( - 1, + "Person", "seg_a", UpsertNodeOptions { props: first_props, @@ -12272,7 +13045,7 @@ fn test_ready_property_index_downgrades_when_bg_compaction_missed_declaration_sn second_props.insert("status".to_string(), active.clone()); let node_b = db .upsert_node( - 1, + "Person", "seg_b", UpsertNodeOptions { props: second_props, @@ -12287,7 +13060,7 @@ fn test_ready_property_index_downgrades_when_bg_compaction_missed_declaration_sn let expected_ids = vec![node_a, node_b]; let info = db - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); @@ -12310,7 +13083,7 @@ fn test_ready_property_index_downgrades_when_bg_compaction_missed_declaration_sn db.reset_property_query_routes(); let mut results = db - .find_nodes(1, "status", &PropValue::String("active".to_string())) + .find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(); results.sort_unstable(); assert_eq!(results, expected_ids); @@ -12320,11 +13093,13 @@ fn test_ready_property_index_downgrades_when_bg_compaction_missed_declaration_sn repair_release_tx.send(()).unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); db.reset_property_query_routes(); assert_eq!( - db.find_nodes(1, "status", &PropValue::String("active".to_string())) + db.find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(), expected_ids ); @@ -12346,7 +13121,7 @@ fn test_failed_property_indexes_survive_reopen_and_queries_fallback() { color_props.insert("color".to_string(), PropValue::String("red".to_string())); let color_id = db .upsert_node( - 1, + "Person", "color", UpsertNodeOptions { props: color_props, @@ -12358,7 +13133,7 @@ fn test_failed_property_indexes_survive_reopen_and_queries_fallback() { score_props.insert("score".to_string(), PropValue::Int(10)); let score_id = db .upsert_node( - 1, + "Person", "score", UpsertNodeOptions { props: score_props, @@ -12368,11 +13143,10 @@ fn test_failed_property_indexes_survive_reopen_and_queries_fallback() { .unwrap(); let eq = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); let range = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -12395,13 +13169,12 @@ fn test_failed_property_indexes_survive_reopen_and_queries_fallback() { db.rebuild_secondary_index_catalog().unwrap(); assert_eq!( - db.find_nodes(1, "color", &PropValue::String("red".to_string())) + db.find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap(), vec![color_id] ); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -12432,14 +13205,13 @@ fn test_failed_property_indexes_survive_reopen_and_queries_fallback() { db.reset_property_query_routes(); assert_eq!( - db.find_nodes(1, "color", &PropValue::String("red".to_string())) + db.find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap() .len(), 1 ); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -12468,7 +13240,7 @@ fn test_zero_declaration_flush_and_compaction_skip_equality_artifacts() { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String("red".to_string())); db.upsert_node( - 1, + "Person", key, UpsertNodeOptions { props, @@ -12479,8 +13251,7 @@ fn test_zero_declaration_flush_and_compaction_skip_equality_artifacts() { } db.flush().unwrap(); let first_seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); - assert!(!first_seg_dir.join("prop_index.dat").exists()); - assert!(!first_seg_dir.join("node_prop_hashes.dat").exists()); + assert_no_legacy_property_components(&first_seg_dir); assert!(!first_seg_dir .join(crate::segment_writer::SECONDARY_INDEX_DIRNAME) .exists()); @@ -12489,7 +13260,7 @@ fn test_zero_declaration_flush_and_compaction_skip_equality_artifacts() { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String("blue".to_string())); db.upsert_node( - 1, + "Person", key, UpsertNodeOptions { props, @@ -12502,8 +13273,7 @@ fn test_zero_declaration_flush_and_compaction_skip_equality_artifacts() { let stats = db.compact().unwrap().unwrap(); assert_eq!(stats.segments_merged, 2); let compacted_seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); - assert!(!compacted_seg_dir.join("prop_index.dat").exists()); - assert!(!compacted_seg_dir.join("node_prop_hashes.dat").exists()); + assert_no_legacy_property_components(&compacted_seg_dir); assert!(!compacted_seg_dir .join(crate::segment_writer::SECONDARY_INDEX_DIRNAME) .exists()); @@ -12522,7 +13292,7 @@ fn test_equality_index_backfills_existing_segments_and_compaction_preserves_side props.insert("color".to_string(), red.clone()); let first_id = db .upsert_node( - 1, + "Person", "first", UpsertNodeOptions { props: props.clone(), @@ -12533,7 +13303,7 @@ fn test_equality_index_backfills_existing_segments_and_compaction_preserves_side db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); @@ -12544,7 +13314,7 @@ fn test_equality_index_backfills_existing_segments_and_compaction_preserves_side let second_id = db .upsert_node( - 1, + "Person", "second", UpsertNodeOptions { props, @@ -12566,11 +13336,10 @@ fn test_equality_index_backfills_existing_segments_and_compaction_preserves_side let compacted_sidecar = crate::segment_writer::node_prop_eq_sidecar_path(&compacted_seg_dir, info.index_id); assert!(compacted_sidecar.exists()); - assert!(!compacted_seg_dir.join("prop_index.dat").exists()); - assert!(!compacted_seg_dir.join("node_prop_hashes.dat").exists()); + assert_no_legacy_property_components(&compacted_seg_dir); db.reset_property_query_routes(); - let mut ids = db.find_nodes(1, "color", &red).unwrap(); + let mut ids = db.find_nodes("Person", "color", &red).unwrap(); ids.sort_unstable(); assert_eq!(ids, vec![first_id, second_id]); let routes = db.property_query_route_snapshot(); @@ -12593,7 +13362,7 @@ fn test_missing_equality_sidecar_reopens_and_repairs_to_ready() { let mut props = BTreeMap::new(); props.insert("color".to_string(), red.clone()); db.upsert_node( - 1, + "Person", "repair-me", UpsertNodeOptions { props, @@ -12604,7 +13373,7 @@ fn test_missing_equality_sidecar_reopens_and_repairs_to_ready() { db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); index_id = info.index_id; @@ -12617,71 +13386,143 @@ fn test_missing_equality_sidecar_reopens_and_repairs_to_ready() { std::fs::remove_file(&sidecar_path).unwrap(); assert!(!sidecar_path.exists()); - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - wait_for_property_index_state(&reopened, index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); - assert_eq!( - reopened - .find_nodes(1, "color", &PropValue::String("red".to_string())) - .unwrap() - .len(), - 1 - ); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + wait_for_property_index_state(&reopened, index_id, SecondaryIndexState::Ready); + let repaired_sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, index_id); + assert!(repaired_sidecar_path.exists()); + assert_eq!( + reopened + .find_nodes("Person", "color", &PropValue::String("red".to_string())) + .unwrap() + .len(), + 1 + ); + reopened.close().unwrap(); +} + +#[test] +fn test_corrupt_equality_sidecar_reopens_failed_and_queries_fallback() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let index_id; + let seg_id; + let red = PropValue::String("red".to_string()); + + { + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let mut props = BTreeMap::new(); + props.insert("color".to_string(), red.clone()); + let node_id = db + .upsert_node( + "Person", + "broken", + UpsertNodeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); + db.flush().unwrap(); + + let info = db + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + seg_id = db.segments_for_test()[0].segment_id; + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); + db.close().unwrap(); + } + + let seg_dir = segment_dir(&db_path, seg_id); + let sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, index_id); + std::fs::write(&sidecar_path, [1u8, 2, 3]).unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = reopened + .list_node_property_indexes().unwrap() + .into_iter() + .find(|info| info.index_id == index_id) + .unwrap(); + assert_eq!(info.state, SecondaryIndexState::Failed); + assert!(info.last_error.is_some()); + + reopened.reset_property_query_routes(); + assert_eq!(reopened.find_nodes("Person", "color", &red).unwrap().len(), 1); + let routes = reopened.property_query_route_snapshot(); + assert_eq!(routes.equality_scan_fallback, 1); + assert_eq!(routes.equality_index_lookup, 0); + reopened.close().unwrap(); } #[test] -fn test_corrupt_equality_sidecar_reopens_failed_and_queries_fallback() { +fn test_ready_equality_sidecar_tail_corruption_does_not_full_scan_on_open() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let red = PropValue::String("red".to_string()); + let blue = PropValue::String("blue".to_string()); let index_id; let seg_id; - let red = PropValue::String("red".to_string()); + let red_id; { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut props = BTreeMap::new(); - props.insert("color".to_string(), red.clone()); - let node_id = db + let mut red_props = BTreeMap::new(); + red_props.insert("color".to_string(), red.clone()); + red_id = db .upsert_node( - 1, - "broken", + "Person", + "red", UpsertNodeOptions { - props, + props: red_props, + ..Default::default() + }, + ) + .unwrap(); + for key in ["blue-a", "blue-b"] { + let mut blue_props = BTreeMap::new(); + blue_props.insert("color".to_string(), blue.clone()); + db.upsert_node( + "Person", + key, + UpsertNodeOptions { + props: blue_props, ..Default::default() }, ) .unwrap(); + } db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); index_id = info.index_id; seg_id = db.segments_for_test()[0].segment_id; - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); db.close().unwrap(); } let seg_dir = segment_dir(&db_path, seg_id); let sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, index_id); - std::fs::write(&sidecar_path, [1u8, 2, 3]).unwrap(); + corrupt_equality_sidecar_tail_group_order_in_place(&sidecar_path, hash_prop_value(&blue)); let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let info = reopened - .list_node_property_indexes().unwrap() + .list_node_property_indexes() + .unwrap() .into_iter() .find(|info| info.index_id == index_id) .unwrap(); - assert_eq!(info.state, SecondaryIndexState::Failed); - assert!(info.last_error.is_some()); + assert_eq!(info.state, SecondaryIndexState::Ready); + assert!(info.last_error.is_none()); reopened.reset_property_query_routes(); - assert_eq!(reopened.find_nodes(1, "color", &red).unwrap().len(), 1); + assert_eq!(reopened.find_nodes("Person", "color", &red).unwrap(), vec![red_id]); let routes = reopened.property_query_route_snapshot(); - assert_eq!(routes.equality_scan_fallback, 1); - assert_eq!(routes.equality_index_lookup, 0); + assert_eq!(routes.equality_scan_fallback, 0); + assert_eq!(routes.equality_index_lookup, 1); reopened.close().unwrap(); } @@ -12698,7 +13539,7 @@ fn test_missing_equality_sidecar_while_open_queries_fallback_and_repairs() { props.insert("color".to_string(), red.clone()); let node_id = db .upsert_node( - 1, + "Person", "repair-live", UpsertNodeOptions { props, @@ -12709,20 +13550,23 @@ fn test_missing_equality_sidecar_while_open_queries_fallback_and_repairs() { db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - let seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); + let segment_id = db.segments_for_test()[0].segment_id; + let seg_dir = segment_dir(&db_path, segment_id); let sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); std::fs::remove_file(&sidecar_path).unwrap(); assert!(!sidecar_path.exists()); + db.reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); let mut unrelated_props = BTreeMap::new(); unrelated_props.insert("color".to_string(), blue.clone()); let unrelated_id = db .upsert_node( - 1, + "Person", "live-counter-node", UpsertNodeOptions { props: unrelated_props, @@ -12733,7 +13577,7 @@ fn test_missing_equality_sidecar_while_open_queries_fallback_and_repairs() { db.upsert_edge( node_id, unrelated_id, - 7, + "FRIENDS_WITH", UpsertEdgeOptions { ..Default::default() }, @@ -12747,7 +13591,7 @@ fn test_missing_equality_sidecar_while_open_queries_fallback_and_repairs() { let (repair_ready_rx, repair_release_tx) = db.set_secondary_index_build_pause(); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); repair_ready_rx.recv().unwrap(); let routes = db.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 1); @@ -12773,7 +13617,7 @@ fn test_missing_equality_sidecar_while_open_queries_fallback_and_repairs() { later_props.insert("color".to_string(), blue); let later_id = db .upsert_node( - 1, + "Person", "repair-counter-node", UpsertNodeOptions { props: later_props, @@ -12784,7 +13628,7 @@ fn test_missing_equality_sidecar_while_open_queries_fallback_and_repairs() { db.upsert_edge( node_id, later_id, - 8, + "COLLABORATES_WITH", UpsertEdgeOptions { ..Default::default() }, @@ -12798,7 +13642,9 @@ fn test_missing_equality_sidecar_while_open_queries_fallback_and_repairs() { repair_release_tx.send(()).unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); let manifest_after_repair = crate::manifest::load_manifest_readonly(&db_path) .unwrap() @@ -12824,7 +13670,7 @@ fn test_corrupt_equality_sidecar_while_open_queries_fallback_and_marks_failed() props.insert("color".to_string(), red.clone()); let node_id = db .upsert_node( - 1, + "Person", "fail-live", UpsertNodeOptions { props, @@ -12835,7 +13681,7 @@ fn test_corrupt_equality_sidecar_while_open_queries_fallback_and_marks_failed() db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); @@ -12844,7 +13690,7 @@ fn test_corrupt_equality_sidecar_while_open_queries_fallback_and_marks_failed() corrupt_sidecar_header_in_place(&sidecar_path); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let routes = db.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 1); assert_eq!(routes.equality_index_lookup, 0); @@ -12867,7 +13713,7 @@ fn test_compaction_with_corrupt_ready_sidecar_succeeds_and_marks_failed() { props.insert("color".to_string(), red.clone()); let first_id = db .upsert_node( - 1, + "Person", "first", UpsertNodeOptions { props: props.clone(), @@ -12878,13 +13724,13 @@ fn test_compaction_with_corrupt_ready_sidecar_succeeds_and_marks_failed() { db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); let second_id = db .upsert_node( - 1, + "Person", "second", UpsertNodeOptions { props, @@ -12911,7 +13757,7 @@ fn test_compaction_with_corrupt_ready_sidecar_succeeds_and_marks_failed() { assert!(compacted_sidecar.exists()); db.reset_property_query_routes(); - let mut ids = db.find_nodes(1, "color", &red).unwrap(); + let mut ids = db.find_nodes("Person", "color", &red).unwrap(); ids.sort_unstable(); assert_eq!(ids, vec![first_id, second_id]); let routes = db.property_query_route_snapshot(); @@ -12932,7 +13778,7 @@ fn test_compaction_with_missing_ready_sidecar_rebuilds_equality_index_via_target props.insert("color".to_string(), red.clone()); let first_id = db .upsert_node( - 1, + "Person", "first-missing-sidecar", UpsertNodeOptions { props: props.clone(), @@ -12943,13 +13789,13 @@ fn test_compaction_with_missing_ready_sidecar_rebuilds_equality_index_via_target db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); let second_id = db .upsert_node( - 1, + "Person", "second-missing-sidecar", UpsertNodeOptions { props, @@ -12982,7 +13828,7 @@ fn test_compaction_with_missing_ready_sidecar_rebuilds_equality_index_via_target assert!(compacted_sidecar.exists()); db.reset_property_query_routes(); - let mut ids = db.find_nodes(1, "color", &red).unwrap(); + let mut ids = db.find_nodes("Person", "color", &red).unwrap(); ids.sort_unstable(); assert_eq!(ids, vec![first_id, second_id]); let routes = db.property_query_route_snapshot(); @@ -13001,7 +13847,7 @@ fn test_compaction_with_missing_ready_sidecar_rebuilds_equality_index_via_target assert!(reopened_info.last_error.is_none()); reopened.reset_property_query_routes(); - let mut reopened_ids = reopened.find_nodes(1, "color", &red).unwrap(); + let mut reopened_ids = reopened.find_nodes("Person", "color", &red).unwrap(); reopened_ids.sort_unstable(); assert_eq!(reopened_ids, vec![first_id, second_id]); let reopened_routes = reopened.property_query_route_snapshot(); @@ -13022,7 +13868,7 @@ fn test_drop_equality_index_routes_to_fallback_and_cleans_sidecar() { props.insert("color".to_string(), red.clone()); let node_id = db .upsert_node( - 1, + "Person", "drop-equality", UpsertNodeOptions { props, @@ -13033,12 +13879,12 @@ fn test_drop_equality_index_routes_to_fallback_and_cleans_sidecar() { db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let indexed_routes = db.property_query_route_snapshot(); assert_eq!(indexed_routes.equality_scan_fallback, 0); assert_eq!(indexed_routes.equality_index_lookup, 1); @@ -13048,7 +13894,7 @@ fn test_drop_equality_index_routes_to_fallback_and_cleans_sidecar() { assert!(sidecar_path.exists()); assert!(db - .drop_node_property_index(1, "color", SecondaryIndexKind::Equality) + .drop_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap()); assert!( db.list_node_property_indexes().unwrap() @@ -13062,97 +13908,34 @@ fn test_drop_equality_index_routes_to_fallback_and_cleans_sidecar() { ); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let fallback_routes = db.property_query_route_snapshot(); assert_eq!(fallback_routes.equality_scan_fallback, 1); assert_eq!(fallback_routes.equality_index_lookup, 0); wait_for_path_absent(&sidecar_path); + let manifest = read_component_manifest_for_test(&seg_dir); + assert!(manifest.components.iter().all(|record| { + !matches!( + &record.kind, + crate::segment_components::SegmentComponentKind::NodePropertyEqualityIndex { + index_id + } if *index_id == info.index_id + ) + })); db.close().unwrap(); let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); assert!(reopened.list_node_property_indexes().unwrap().is_empty()); reopened.reset_property_query_routes(); - assert_eq!(reopened.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(reopened.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let reopened_routes = reopened.property_query_route_snapshot(); assert_eq!(reopened_routes.equality_scan_fallback, 1); assert_eq!(reopened_routes.equality_index_lookup, 0); reopened.close().unwrap(); } -#[test] -fn test_legacy_property_hash_backfill_and_compaction_parity() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let red = PropValue::String("red".to_string()); - let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let mut props = BTreeMap::new(); - props.insert("color".to_string(), red.clone()); - let first_id = db - .upsert_node( - 1, - "legacy-first", - UpsertNodeOptions { - props: props.clone(), - ..Default::default() - }, - ) - .unwrap(); - db.flush().unwrap(); - - let first_seg_id = db.segments_for_test()[0].segment_id; - db.close().unwrap(); - drop(db); - - let first_seg_dir = segment_dir(&db_path, first_seg_id); - install_legacy_property_hash_sidecars( - &first_seg_dir, - &[(first_id, 1, vec![("color".to_string(), red.clone())])], - ); - assert!(first_seg_dir.join("prop_index.dat").exists()); - assert!(first_seg_dir.join("node_prop_hashes.dat").exists()); - - let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) - .unwrap(); - wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - let first_sidecar = - crate::segment_writer::node_prop_eq_sidecar_path(&first_seg_dir, info.index_id); - assert!(first_sidecar.exists()); - - let second_id = db - .upsert_node( - 1, - "legacy-second", - UpsertNodeOptions { - props, - ..Default::default() - }, - ) - .unwrap(); - db.flush().unwrap(); - - let stats = db.compact().unwrap().unwrap(); - assert_eq!(stats.segments_merged, 2); - let compacted_seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); - let compacted_sidecar = - crate::segment_writer::node_prop_eq_sidecar_path(&compacted_seg_dir, info.index_id); - assert!(compacted_sidecar.exists()); - - db.reset_property_query_routes(); - let mut ids = db.find_nodes(1, "color", &red).unwrap(); - ids.sort_unstable(); - assert_eq!(ids, vec![first_id, second_id]); - let routes = db.property_query_route_snapshot(); - assert_eq!(routes.equality_scan_fallback, 0); - assert_eq!(routes.equality_index_lookup, 1); - - db.close().unwrap(); -} - #[test] fn test_equality_backfill_survives_compaction_during_build() { let dir = TempDir::new().unwrap(); @@ -13164,7 +13947,7 @@ fn test_equality_backfill_survives_compaction_during_build() { let mut props = BTreeMap::new(); props.insert("color".to_string(), red.clone()); db.upsert_node( - 1, + "Person", key, UpsertNodeOptions { props, @@ -13177,7 +13960,7 @@ fn test_equality_backfill_survives_compaction_during_build() { let (ready_rx, release_tx) = db.set_secondary_index_build_pause(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); ready_rx .recv_timeout(std::time::Duration::from_secs(5)) @@ -13191,7 +13974,7 @@ fn test_equality_backfill_survives_compaction_during_build() { wait_for_published_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap().len(), 2); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap().len(), 2); let routes = db.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 0); assert_eq!(routes.equality_index_lookup, 1); @@ -13214,7 +13997,7 @@ fn test_equality_index_close_while_build_paused_reopens_and_resumes() { props.insert("color".to_string(), red.clone()); let node_id = db .upsert_node( - 1, + "Person", "close-paused-eq", UpsertNodeOptions { props, @@ -13227,7 +14010,7 @@ fn test_equality_index_close_while_build_paused_reopens_and_resumes() { let seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); let (ready_rx, release_tx) = db.set_secondary_index_build_pause(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); ready_rx .recv_timeout(std::time::Duration::from_secs(5)) @@ -13262,10 +14045,12 @@ fn test_equality_index_close_while_build_paused_reopens_and_resumes() { let reopened = DatabaseEngine::open(&db_path, &opts).unwrap(); wait_for_property_index_state(&reopened, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); reopened.reset_property_query_routes(); - assert_eq!(reopened.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(reopened.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let routes = reopened.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 0); assert_eq!(routes.equality_index_lookup, 1); @@ -13287,7 +14072,7 @@ fn test_drop_equality_index_while_build_paused_stale_sidecar_does_not_resurrect( props.insert("color".to_string(), red.clone()); let node_id = db .upsert_node( - 1, + "Person", "drop-paused-eq", UpsertNodeOptions { props, @@ -13300,14 +14085,14 @@ fn test_drop_equality_index_while_build_paused_stale_sidecar_does_not_resurrect( let seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); let (ready_rx, release_tx) = db.set_secondary_index_build_pause(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); ready_rx .recv_timeout(std::time::Duration::from_secs(5)) .unwrap(); assert!(db - .drop_node_property_index(1, "color", SecondaryIndexKind::Equality) + .drop_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap()); assert!(db.list_node_property_indexes().unwrap().is_empty()); let manifest_after_drop = crate::manifest::load_manifest_readonly(&db_path) @@ -13330,7 +14115,7 @@ fn test_drop_equality_index_while_build_paused_stale_sidecar_does_not_resurrect( let reopened = DatabaseEngine::open(&db_path, &opts).unwrap(); assert!(reopened.list_node_property_indexes().unwrap().is_empty()); reopened.reset_property_query_routes(); - assert_eq!(reopened.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(reopened.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let routes = reopened.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 1); assert_eq!(routes.equality_index_lookup, 0); @@ -13352,7 +14137,7 @@ fn test_property_range_index_manifest_reopens_and_reseeds_active_memtable() { props.insert("score".to_string(), PropValue::Int(10)); let node_id = db .upsert_node( - 1, + "Person", "frozen-range", UpsertNodeOptions { props, @@ -13367,8 +14152,7 @@ fn test_property_range_index_manifest_reopens_and_reseeds_active_memtable() { ready_rx.recv().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13391,8 +14175,7 @@ fn test_property_range_index_manifest_reopens_and_reseeds_active_memtable() { assert!(crate::segment_writer::node_prop_range_sidecar_path(&seg_dir, info.index_id).exists()); db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13426,7 +14209,7 @@ fn test_ready_property_range_index_downgrades_when_flush_publish_missed_declarat props.insert("score".to_string(), PropValue::Int(10)); let node_id = db .upsert_node( - 1, + "Person", "frozen-range", UpsertNodeOptions { props, @@ -13441,8 +14224,7 @@ fn test_ready_property_range_index_downgrades_when_flush_publish_missed_declarat publish_ready_rx.recv().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13470,8 +14252,7 @@ fn test_ready_property_range_index_downgrades_when_flush_publish_missed_declarat db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13485,12 +14266,13 @@ fn test_ready_property_range_index_downgrades_when_flush_publish_missed_declarat repair_release_tx.send(()).unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_range_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13517,7 +14299,7 @@ fn test_missing_range_sidecar_reopens_and_repairs_to_ready() { let mut props = BTreeMap::new(); props.insert("score".to_string(), PropValue::Int(10)); db.upsert_node( - 1, + "Person", "repair-me-range", UpsertNodeOptions { props, @@ -13528,8 +14310,7 @@ fn test_missing_range_sidecar_reopens_and_repairs_to_ready() { db.flush().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13549,11 +14330,12 @@ fn test_missing_range_sidecar_reopens_and_repairs_to_ready() { let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); wait_for_property_index_state(&reopened, index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_range_sidecar_path(&seg_dir, index_id); + assert!(repaired_sidecar_path.exists()); assert_eq!( reopened - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13578,7 +14360,7 @@ fn test_corrupt_range_sidecar_reopens_failed_and_queries_fallback() { props.insert("score".to_string(), PropValue::Int(10)); let node_id = db .upsert_node( - 1, + "Person", "broken-range", UpsertNodeOptions { props, @@ -13589,8 +14371,7 @@ fn test_corrupt_range_sidecar_reopens_failed_and_queries_fallback() { db.flush().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13601,8 +14382,7 @@ fn test_corrupt_range_sidecar_reopens_failed_and_queries_fallback() { index_id = info.index_id; seg_id = db.segments_for_test()[0].segment_id; assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13629,8 +14409,7 @@ fn test_corrupt_range_sidecar_reopens_failed_and_queries_fallback() { reopened.reset_property_query_routes(); assert_eq!( reopened - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13646,6 +14425,83 @@ fn test_corrupt_range_sidecar_reopens_failed_and_queries_fallback() { reopened.close().unwrap(); } +#[test] +fn test_ready_range_sidecar_tail_corruption_does_not_full_scan_on_open() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let index_id; + let seg_id; + let first_id; + + { + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let mut first_id_for_setup = None; + for (key, score) in [("score-10", 10), ("score-20", 20), ("score-30", 30)] { + let mut props = BTreeMap::new(); + props.insert("score".to_string(), PropValue::Int(score)); + let id = db + .upsert_node( + "Person", + key, + UpsertNodeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); + if score == 10 { + first_id_for_setup = Some(id); + } + } + first_id = first_id_for_setup.unwrap(); + db.flush().unwrap(); + + let info = db + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + seg_id = db.segments_for_test()[0].segment_id; + db.close().unwrap(); + } + + let seg_dir = segment_dir(&db_path, seg_id); + let sidecar_path = crate::segment_writer::node_prop_range_sidecar_path(&seg_dir, index_id); + corrupt_range_sidecar_tail_sort_order_in_place(&sidecar_path); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = reopened + .list_node_property_indexes() + .unwrap() + .into_iter() + .find(|info| info.index_id == index_id) + .unwrap(); + assert_eq!(info.state, SecondaryIndexState::Ready); + assert!(info.last_error.is_none()); + + reopened.reset_property_query_routes(); + assert_eq!( + reopened + .find_nodes_range("Person", + "score", + Some(&PropertyRangeBound::Included(PropValue::Int(10))), + Some(&PropertyRangeBound::Included(PropValue::Int(10))), + ) + .unwrap(), + vec![first_id] + ); + let routes = reopened.property_query_route_snapshot(); + assert_eq!(routes.range_scan_fallback, 0); + assert_eq!(routes.range_index_lookup, 1); + + reopened.close().unwrap(); +} + #[test] fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { let dir = TempDir::new().unwrap(); @@ -13656,7 +14512,7 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { props.insert("score".to_string(), PropValue::Int(10)); let node_id = db .upsert_node( - 1, + "Person", "repair-live-range", UpsertNodeOptions { props, @@ -13667,8 +14523,7 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { db.flush().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13677,14 +14532,17 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - let seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); + let segment_id = db.segments_for_test()[0].segment_id; + let seg_dir = segment_dir(&db_path, segment_id); let sidecar_path = crate::segment_writer::node_prop_range_sidecar_path(&seg_dir, info.index_id); std::fs::remove_file(&sidecar_path).unwrap(); assert!(!sidecar_path.exists()); + db.reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); let unrelated_id = db .upsert_node( - 1, + "Person", "live-counter-range", UpsertNodeOptions { ..Default::default() @@ -13694,7 +14552,7 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { db.upsert_edge( node_id, unrelated_id, - 7, + "FRIENDS_WITH", UpsertEdgeOptions { ..Default::default() }, @@ -13709,8 +14567,7 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { let (repair_ready_rx, repair_release_tx) = db.set_secondary_index_build_pause(); db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13732,7 +14589,7 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { let later_id = db .upsert_node( - 1, + "Person", "repair-counter-range", UpsertNodeOptions { ..Default::default() @@ -13742,7 +14599,7 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { db.upsert_edge( node_id, later_id, - 8, + "COLLABORATES_WITH", UpsertEdgeOptions { ..Default::default() }, @@ -13756,7 +14613,9 @@ fn test_missing_range_sidecar_while_open_queries_fallback_and_repairs() { repair_release_tx.send(()).unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_range_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); let manifest_after_repair = crate::manifest::load_manifest_readonly(&db_path) .unwrap() @@ -13778,7 +14637,7 @@ fn test_corrupt_range_sidecar_while_open_queries_fallback_and_marks_failed() { props.insert("score".to_string(), PropValue::Int(10)); let node_id = db .upsert_node( - 1, + "Person", "corrupt-live-range", UpsertNodeOptions { props, @@ -13789,8 +14648,7 @@ fn test_corrupt_range_sidecar_while_open_queries_fallback_and_marks_failed() { db.flush().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13805,7 +14663,7 @@ fn test_corrupt_range_sidecar_while_open_queries_fallback_and_marks_failed() { let unrelated_id = db .upsert_node( - 1, + "Person", "failed-counter-range", UpsertNodeOptions { ..Default::default() @@ -13815,7 +14673,7 @@ fn test_corrupt_range_sidecar_while_open_queries_fallback_and_marks_failed() { db.upsert_edge( node_id, unrelated_id, - 9, + "RELATED_TO", UpsertEdgeOptions { ..Default::default() }, @@ -13829,8 +14687,7 @@ fn test_corrupt_range_sidecar_while_open_queries_fallback_and_marks_failed() { db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13866,7 +14723,7 @@ fn test_compaction_with_corrupt_ready_range_sidecar_succeeds_and_marks_failed() props.insert("score".to_string(), PropValue::Int(10)); let first_id = db .upsert_node( - 1, + "Person", "first-range", UpsertNodeOptions { props: props.clone(), @@ -13877,8 +14734,7 @@ fn test_compaction_with_corrupt_ready_range_sidecar_succeeds_and_marks_failed() db.flush().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13889,7 +14745,7 @@ fn test_compaction_with_corrupt_ready_range_sidecar_succeeds_and_marks_failed() let second_id = db .upsert_node( - 1, + "Person", "second-range", UpsertNodeOptions { props, @@ -13921,8 +14777,7 @@ fn test_compaction_with_corrupt_ready_range_sidecar_succeeds_and_marks_failed() db.reset_property_query_routes(); let mut ids = db - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -13947,7 +14802,7 @@ fn test_compaction_with_missing_ready_range_sidecar_rebuilds_index_via_targeted_ props.insert("score".to_string(), PropValue::Int(10)); let first_id = db .upsert_node( - 1, + "Person", "first-missing-range-sidecar", UpsertNodeOptions { props: props.clone(), @@ -13958,8 +14813,7 @@ fn test_compaction_with_missing_ready_range_sidecar_rebuilds_index_via_targeted_ db.flush().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -13970,7 +14824,7 @@ fn test_compaction_with_missing_ready_range_sidecar_rebuilds_index_via_targeted_ let second_id = db .upsert_node( - 1, + "Person", "second-missing-range-sidecar", UpsertNodeOptions { props, @@ -14004,8 +14858,7 @@ fn test_compaction_with_missing_ready_range_sidecar_rebuilds_index_via_targeted_ db.reset_property_query_routes(); let mut ids = db - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14030,8 +14883,7 @@ fn test_compaction_with_missing_ready_range_sidecar_rebuilds_index_via_targeted_ reopened.reset_property_query_routes(); let mut reopened_ids = reopened - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14056,7 +14908,7 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { props.insert("score".to_string(), PropValue::Int(10)); let first_id = db .upsert_node( - 1, + "Person", "drop-range-first", UpsertNodeOptions { props: props.clone(), @@ -14067,8 +14919,7 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { db.flush().unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -14079,8 +14930,7 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14096,10 +14946,10 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { let first_sidecar_path = crate::segment_writer::node_prop_range_sidecar_path(&first_seg_dir, info.index_id); assert!(first_sidecar_path.exists()); + let packed_core_before_drop = packed_core_snapshot_for_test(&first_seg_dir); assert!(db - .drop_node_property_index( - 1, + .drop_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -14119,8 +14969,7 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14133,10 +14982,24 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { assert_eq!(fallback_routes.range_index_lookup, 0); wait_for_path_absent(&first_sidecar_path); + assert_packed_core_unchanged_for_test( + &first_seg_dir, + &packed_core_before_drop, + "declared range index drop cleanup", + ); + let first_manifest = read_component_manifest_for_test(&first_seg_dir); + assert!(first_manifest.components.iter().all(|record| { + !matches!( + &record.kind, + crate::segment_components::SegmentComponentKind::NodePropertyRangeIndex { + index_id + } if *index_id == info.index_id + ) + })); let second_id = db .upsert_node( - 1, + "Person", "drop-range-second", UpsertNodeOptions { props, @@ -14160,8 +15023,7 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { db.reset_property_query_routes(); let mut ids = db - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14179,8 +15041,7 @@ fn test_drop_range_index_routes_to_fallback_cleans_sidecar_and_stays_dropped() { assert!(reopened.list_node_property_indexes().unwrap().is_empty()); reopened.reset_property_query_routes(); let mut reopened_ids = reopened - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14204,7 +15065,7 @@ fn test_range_backfill_survives_compaction_during_build() { let mut props = BTreeMap::new(); props.insert("score".to_string(), PropValue::Int(10)); db.upsert_node( - 1, + "Person", key, UpsertNodeOptions { props, @@ -14217,8 +15078,7 @@ fn test_range_backfill_survives_compaction_during_build() { let (ready_rx, release_tx) = db.set_secondary_index_build_pause(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -14238,8 +15098,7 @@ fn test_range_backfill_survives_compaction_during_build() { db.reset_property_query_routes(); assert_eq!( - db.find_nodes_range( - 1, + db.find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14269,7 +15128,7 @@ fn test_range_index_close_fast_while_build_paused_reopens_and_resumes() { props.insert("score".to_string(), PropValue::Int(10)); let node_id = db .upsert_node( - 1, + "Person", "close-fast-paused-range", UpsertNodeOptions { props, @@ -14282,8 +15141,7 @@ fn test_range_index_close_fast_while_build_paused_reopens_and_resumes() { let seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); let (ready_rx, release_tx) = db.set_secondary_index_build_pause(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -14324,13 +15182,14 @@ fn test_range_index_close_fast_while_build_paused_reopens_and_resumes() { let reopened = DatabaseEngine::open(&db_path, &opts).unwrap(); wait_for_property_index_state(&reopened, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_range_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); reopened.reset_property_query_routes(); assert_eq!( reopened - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(10))), Some(&PropertyRangeBound::Included(PropValue::Int(10))), @@ -14355,7 +15214,7 @@ fn test_open_rejects_conflicting_range_declarations_for_same_property() { SecondaryIndexManifestEntry { index_id: 1, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Range { @@ -14367,7 +15226,7 @@ fn test_open_rejects_conflicting_range_declarations_for_same_property() { SecondaryIndexManifestEntry { index_id: 2, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Range { @@ -14398,7 +15257,7 @@ fn test_open_rejects_duplicate_secondary_index_ids_in_manifest() { SecondaryIndexManifestEntry { index_id: 1, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -14408,7 +15267,7 @@ fn test_open_rejects_duplicate_secondary_index_ids_in_manifest() { SecondaryIndexManifestEntry { index_id: 1, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -14439,7 +15298,7 @@ fn test_open_rejects_duplicate_equality_declarations_for_same_property() { SecondaryIndexManifestEntry { index_id: 1, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -14449,7 +15308,7 @@ fn test_open_rejects_duplicate_equality_declarations_for_same_property() { SecondaryIndexManifestEntry { index_id: 2, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -14476,7 +15335,7 @@ fn test_shared_handle_clone_observes_state_and_close_is_family_wide() { let clone = db.clone(); let id = db - .upsert_node(1, "shared", UpsertNodeOptions::default()) + .upsert_node("Person", "shared", UpsertNodeOptions::default()) .unwrap(); assert_eq!(clone.get_node(id).unwrap().unwrap().key, "shared"); @@ -14484,7 +15343,7 @@ fn test_shared_handle_clone_observes_state_and_close_is_family_wide() { assert!(matches!(db.get_node(id), Err(EngineError::DatabaseClosed))); assert!(matches!( - db.upsert_node(1, "after-close", UpsertNodeOptions::default()), + db.upsert_node("Person", "after-close", UpsertNodeOptions::default()), Err(EngineError::DatabaseClosed) )); assert!(matches!(db.list_prune_policies(), Err(EngineError::DatabaseClosed))); @@ -14526,18 +15385,20 @@ fn test_shared_handle_reads_hold_old_published_snapshot_until_republish() { let (write_done_tx, write_done_rx) = std::sync::mpsc::sync_channel(1); std::thread::spawn(move || { - let result = writer.upsert_node(1, "during-publish", UpsertNodeOptions::default()); + let result = writer.upsert_node("Person", "during-publish", UpsertNodeOptions::default()); let _ = write_done_tx.send(result); }); ready_rx.recv().unwrap(); assert!(reader.get_node(1).unwrap().is_none()); - assert!(reader.get_node_by_key(1, "during-publish").unwrap().is_none()); + assert!(reader.get_node_by_key("Person", "during-publish").unwrap().is_none()); assert!(reader.find_existing_node(1, "during-publish").unwrap().is_none()); let batch_nodes = reader.get_nodes(&[1]).unwrap(); assert!(batch_nodes[0].is_none()); - let batch_keys = reader.get_nodes_by_keys(&[(1, "during-publish")]).unwrap(); + let batch_keys = reader + .get_nodes_by_keys(&[NodeKeyQuery { label: "Person".to_string(), key: "during-publish".to_string() }]) + .unwrap(); assert!(batch_keys[0].is_none()); release_tx.send(()).unwrap(); @@ -14549,7 +15410,7 @@ fn test_shared_handle_reads_hold_old_published_snapshot_until_republish() { assert_eq!(id, 1); assert_eq!(reader.get_node(id).unwrap().unwrap().key, "during-publish"); assert_eq!( - reader.get_node_by_key(1, "during-publish").unwrap().unwrap().id, + reader.get_node_by_key("Person", "during-publish").unwrap().unwrap().id, id ); assert_eq!( @@ -14562,21 +15423,23 @@ fn test_shared_handle_reads_hold_old_published_snapshot_until_republish() { let batch_nodes = reader.get_nodes(&[id]).unwrap(); assert_eq!(batch_nodes[0].as_ref().unwrap().key, "during-publish"); - let batch_keys = reader.get_nodes_by_keys(&[(1, "during-publish")]).unwrap(); + let batch_keys = reader + .get_nodes_by_keys(&[NodeKeyQuery { label: "Person".to_string(), key: "during-publish".to_string() }]) + .unwrap(); assert_eq!(batch_keys[0].as_ref().unwrap().id, id); db.close().unwrap(); } #[test] -fn test_shared_handle_snapshot_queries_keep_visible_type_enumeration_until_republish() { +fn test_shared_handle_snapshot_queries_keep_visible_label_enumeration_until_republish() { let dir = TempDir::new().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let reader = db.clone(); let writer = db.clone(); let id = db - .upsert_node(1, "snapshot-type", UpsertNodeOptions::default()) + .upsert_node("Person", "snapshot-label", UpsertNodeOptions::default()) .unwrap(); let initial_export = reader.export_adjacency(&ExportOptions::default()).unwrap(); @@ -14600,7 +15463,7 @@ fn test_shared_handle_snapshot_queries_keep_visible_type_enumeration_until_repub assert_eq!( pinned_export.node_ids, vec![id], - "old published snapshot must keep the pre-delete node type visible" + "old published snapshot must keep the pre-delete node label visible" ); let pinned_components = reader .connected_components(&ComponentOptions::default()) @@ -14608,7 +15471,7 @@ fn test_shared_handle_snapshot_queries_keep_visible_type_enumeration_until_repub assert_eq!( pinned_components.get(&id), Some(&id), - "connected_components must enumerate node types from snapshot-visible membership" + "connected_components must enumerate node labels from snapshot-visible membership" ); release_tx.send(()).unwrap(); @@ -14627,12 +15490,70 @@ fn test_shared_handle_snapshot_queries_keep_visible_type_enumeration_until_repub db.close().unwrap(); } +#[test] +fn test_transaction_pinned_snapshot_label_history() { + let dir = TempDir::new().unwrap(); + let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); + + let id = db + .upsert_node("Person", "snapshot-label-history", UpsertNodeOptions::default()) + .unwrap(); + let pinned = db.published_read_view_for_test(); + let pinned_person_id = pinned + .label_catalog + .resolve_node_label_for_read("Person") + .unwrap() + .unwrap(); + assert_eq!( + pinned.get_node(id).unwrap().unwrap().label_ids.as_slice(), + &[pinned_person_id] + ); + + let mut txn = db.begin_write_txn().unwrap(); + assert!(txn.add_node_label(TxnNodeRef::Id(id), "Employee").unwrap()); + assert!(txn.remove_node_label(TxnNodeRef::Id(id), "Person").unwrap()); + txn.commit().unwrap(); + + assert_eq!( + pinned.get_node(id).unwrap().unwrap().label_ids.as_slice(), + &[pinned_person_id], + "pinned reader must keep the old node labels after committed label changes" + ); + assert_eq!( + pinned + .get_node_by_label_key(pinned_person_id, "snapshot-label-history") + .unwrap() + .map(|node| node.id), + Some(id) + ); + assert!(pinned + .label_catalog + .resolve_node_label_for_read("Employee") + .unwrap() + .is_none()); + + let latest = db.get_node(id).unwrap().unwrap(); + assert_eq!(latest.labels, vec!["Employee".to_string()]); + assert!(db + .get_node_by_key("Person", "snapshot-label-history") + .unwrap() + .is_none()); + assert_eq!( + db.get_node_by_key("Employee", "snapshot-label-history") + .unwrap() + .map(|node| node.id), + Some(id) + ); + + db.close().unwrap(); +} + #[test] fn test_shared_handle_close_waits_for_admitted_read_and_rejects_later_reads() { let dir = TempDir::new().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let id = db - .upsert_node(1, "close-read", UpsertNodeOptions::default()) + .upsert_node("Person", "close-read", UpsertNodeOptions::default()) .unwrap(); let reader = db.clone(); @@ -14691,7 +15612,7 @@ fn test_shared_handle_close_waits_for_admitted_write_to_finish() { let (close_done_tx, close_done_rx) = std::sync::mpsc::sync_channel(1); std::thread::spawn(move || { - let result = writer.upsert_node(1, "close-barrier", UpsertNodeOptions::default()); + let result = writer.upsert_node("Person", "close-barrier", UpsertNodeOptions::default()); let _ = write_done_tx.send(result); }); @@ -14739,18 +15660,20 @@ fn test_shared_handle_single_object_allows_point_read_during_paused_write() { let (write_done_tx, write_done_rx) = std::sync::mpsc::sync_channel(1); std::thread::spawn(move || { - let result = writer.upsert_node(1, "single-object", UpsertNodeOptions::default()); + let result = writer.upsert_node("Person", "single-object", UpsertNodeOptions::default()); let _ = write_done_tx.send(result); }); ready_rx.recv().unwrap(); assert!(reader.get_node(1).unwrap().is_none()); - assert!(reader.get_node_by_key(1, "single-object").unwrap().is_none()); + assert!(reader.get_node_by_key("Person", "single-object").unwrap().is_none()); assert!(reader.find_existing_node(1, "single-object").unwrap().is_none()); let batch_nodes = reader.get_nodes(&[1]).unwrap(); assert!(batch_nodes[0].is_none()); - let batch_keys = reader.get_nodes_by_keys(&[(1, "single-object")]).unwrap(); + let batch_keys = reader + .get_nodes_by_keys(&[NodeKeyQuery { label: "Person".to_string(), key: "single-object".to_string() }]) + .unwrap(); assert!(batch_keys[0].is_none()); release_tx.send(()).unwrap(); @@ -14762,7 +15685,7 @@ fn test_shared_handle_single_object_allows_point_read_during_paused_write() { assert_eq!(id, 1); assert_eq!(reader.get_node(id).unwrap().unwrap().key, "single-object"); assert_eq!( - reader.get_node_by_key(1, "single-object").unwrap().unwrap().id, + reader.get_node_by_key("Person", "single-object").unwrap().unwrap().id, id ); assert_eq!( @@ -14786,14 +15709,14 @@ fn test_coordinator_sequences_same_key_upserts_to_one_id() { let barrier_a = std::sync::Arc::clone(&barrier); let handle_a = std::thread::spawn(move || { barrier_a.wait(); - writer_a.upsert_node(1, "same-key", UpsertNodeOptions::default()) + writer_a.upsert_node("Person", "same-key", UpsertNodeOptions::default()) }); let writer_b = std::sync::Arc::clone(&db); let barrier_b = std::sync::Arc::clone(&barrier); let handle_b = std::thread::spawn(move || { barrier_b.wait(); - writer_b.upsert_node(1, "same-key", UpsertNodeOptions::default()) + writer_b.upsert_node("Person", "same-key", UpsertNodeOptions::default()) }); barrier.wait(); @@ -14802,7 +15725,7 @@ fn test_coordinator_sequences_same_key_upserts_to_one_id() { let id_b = handle_b.join().unwrap().unwrap(); assert_eq!(id_a, 1); assert_eq!(id_b, 1); - assert_eq!(db.get_node_by_key(1, "same-key").unwrap().unwrap().id, 1); + assert_eq!(db.get_node_by_key("Person", "same-key").unwrap().unwrap().id, 1); db.close().unwrap(); } @@ -14816,10 +15739,10 @@ fn test_coordinator_sequences_unique_edge_upserts_to_one_id() { }; let db = std::sync::Arc::new(DatabaseEngine::open(dir.path(), &opts).unwrap()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let barrier = std::sync::Arc::new(std::sync::Barrier::new(3)); @@ -14827,14 +15750,14 @@ fn test_coordinator_sequences_unique_edge_upserts_to_one_id() { let barrier_a = std::sync::Arc::clone(&barrier); let handle_a = std::thread::spawn(move || { barrier_a.wait(); - writer_a.upsert_edge(a, b, 9, UpsertEdgeOptions::default()) + writer_a.upsert_edge(a, b, "RELATED_TO", UpsertEdgeOptions::default()) }); let writer_b = std::sync::Arc::clone(&db); let barrier_b = std::sync::Arc::clone(&barrier); let handle_b = std::thread::spawn(move || { barrier_b.wait(); - writer_b.upsert_edge(a, b, 9, UpsertEdgeOptions::default()) + writer_b.upsert_edge(a, b, "RELATED_TO", UpsertEdgeOptions::default()) }); barrier.wait(); @@ -14843,7 +15766,7 @@ fn test_coordinator_sequences_unique_edge_upserts_to_one_id() { let id_b = handle_b.join().unwrap().unwrap(); assert_eq!(id_a, 1); assert_eq!(id_b, 1); - assert_eq!(db.get_edge_by_triple(a, b, 9).unwrap().unwrap().id, 1); + assert_eq!(db.get_edge_by_triple(a, b, "RELATED_TO").unwrap().unwrap().id, 1); db.close().unwrap(); } @@ -14861,12 +15784,12 @@ fn test_core_write_queue_capacity_blocks_pre_admission_until_slot_frees() { let (done_b_tx, done_b_rx) = std::sync::mpsc::sync_channel(1); std::thread::spawn(move || { - let _ = done_a_tx.send(writer_a.upsert_node(1, "queued-a", UpsertNodeOptions::default())); + let _ = done_a_tx.send(writer_a.upsert_node("Person", "queued-a", UpsertNodeOptions::default())); }); ready_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); std::thread::spawn(move || { - let _ = done_b_tx.send(writer_b.upsert_node(1, "queued-b", UpsertNodeOptions::default())); + let _ = done_b_tx.send(writer_b.upsert_node("Person", "queued-b", UpsertNodeOptions::default())); }); assert!( @@ -14911,12 +15834,12 @@ fn test_core_write_queue_close_rejects_pre_admission_submitter() { let (close_done_tx, close_done_rx) = std::sync::mpsc::sync_channel(1); std::thread::spawn(move || { - let _ = done_a_tx.send(writer_a.upsert_node(1, "close-head", UpsertNodeOptions::default())); + let _ = done_a_tx.send(writer_a.upsert_node("Person", "close-head", UpsertNodeOptions::default())); }); ready_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); std::thread::spawn(move || { - let _ = done_b_tx.send(writer_b.upsert_node(1, "close-blocked", UpsertNodeOptions::default())); + let _ = done_b_tx.send(writer_b.upsert_node("Person", "close-blocked", UpsertNodeOptions::default())); }); assert!( done_b_rx @@ -14957,7 +15880,7 @@ fn test_core_write_queue_close_rejects_pre_admission_submitter() { let reopened = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); assert_eq!(reopened.get_node(1).unwrap().unwrap().key, "close-head"); - assert!(reopened.get_node_by_key(1, "close-blocked").unwrap().is_none()); + assert!(reopened.get_node_by_key("Person", "close-blocked").unwrap().is_none()); reopened.close().unwrap(); } @@ -14975,7 +15898,7 @@ fn test_coordinator_head_of_line_backpressure_retry_preserves_order() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); assert_eq!( - db.upsert_node(1, "seed", UpsertNodeOptions::default()).unwrap(), + db.upsert_node("Person", "seed", UpsertNodeOptions::default()).unwrap(), 1 ); db.freeze_memtable().unwrap(); @@ -14987,12 +15910,12 @@ fn test_coordinator_head_of_line_backpressure_retry_preserves_order() { let (done_b_tx, done_b_rx) = std::sync::mpsc::sync_channel(1); std::thread::spawn(move || { - let _ = done_a_tx.send(writer_a.upsert_node(1, "head-a", UpsertNodeOptions::default())); + let _ = done_a_tx.send(writer_a.upsert_node("Person", "head-a", UpsertNodeOptions::default())); }); ready_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); std::thread::spawn(move || { - let _ = done_b_tx.send(writer_b.upsert_node(1, "queued-b", UpsertNodeOptions::default())); + let _ = done_b_tx.send(writer_b.upsert_node("Person", "queued-b", UpsertNodeOptions::default())); }); assert!( @@ -15018,8 +15941,8 @@ fn test_coordinator_head_of_line_backpressure_retry_preserves_order() { .unwrap(), 3 ); - assert_eq!(db.get_node_by_key(1, "head-a").unwrap().unwrap().id, 2); - assert_eq!(db.get_node_by_key(1, "queued-b").unwrap().unwrap().id, 3); + assert_eq!(db.get_node_by_key("Person", "head-a").unwrap().unwrap().id, 2); + assert_eq!(db.get_node_by_key("Person", "queued-b").unwrap().unwrap().id, 3); db.close().unwrap(); } @@ -15029,7 +15952,7 @@ fn test_coordinator_flush_barrier_stays_head_of_line() { let dir = TempDir::new().unwrap(); let db = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); - db.upsert_node(1, "flush-head", UpsertNodeOptions::default()) + db.upsert_node("Person", "flush-head", UpsertNodeOptions::default()) .unwrap(); db.freeze_memtable().unwrap(); @@ -15045,7 +15968,7 @@ fn test_coordinator_flush_barrier_stays_head_of_line() { ready_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); std::thread::spawn(move || { - let _ = write_done_tx.send(writer.upsert_node(1, "queued-after-flush", UpsertNodeOptions::default())); + let _ = write_done_tx.send(writer.upsert_node("Person", "queued-after-flush", UpsertNodeOptions::default())); }); assert!( @@ -15083,11 +16006,11 @@ fn test_compact_with_progress_waits_for_admitted_work_and_blocks_later_mutations let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); let seg_a = db - .upsert_node(1, "seg-a", UpsertNodeOptions::default()) + .upsert_node("Person", "seg-a", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); let seg_b = db - .upsert_node(1, "seg-b", UpsertNodeOptions::default()) + .upsert_node("Person", "seg-b", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -15104,7 +16027,7 @@ fn test_compact_with_progress_waits_for_admitted_work_and_blocks_later_mutations std::thread::spawn(move || { let _ = write_a_done_tx.send(writer_a.upsert_node( - 1, + "Person", "ahead-of-compact", UpsertNodeOptions::default(), )); @@ -15128,7 +16051,7 @@ fn test_compact_with_progress_waits_for_admitted_work_and_blocks_later_mutations std::thread::spawn(move || { let _ = write_b_done_tx.send(writer_b.upsert_node( - 1, + "Person", "behind-compact", UpsertNodeOptions::default(), )); @@ -15192,7 +16115,7 @@ fn test_publish_counters_use_snapshot_only_for_normal_write() { let before = db.published_state(); db.reset_publish_counters_for_test(); let node_id = db - .upsert_node(1, "snapshot-only", UpsertNodeOptions::default()) + .upsert_node("Person", "snapshot-only", UpsertNodeOptions::default()) .unwrap(); let after = db.published_state(); @@ -15217,7 +16140,7 @@ fn test_publish_counters_rebuild_sources_for_flush() { }; let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); - db.upsert_node(1, "flush-me", UpsertNodeOptions::default()) + db.upsert_node("Person", "flush-me", UpsertNodeOptions::default()) .unwrap(); db.reset_publish_counters_for_test(); @@ -15238,7 +16161,7 @@ fn test_publish_counters_rebuild_sources_for_prune_policy_change_and_skip_noop() let policy = PrunePolicy { max_age_ms: Some(60_000), max_weight: Some(0.5), - type_id: Some(1), + label: Some("Person".to_string()), }; db.reset_publish_counters_for_test(); @@ -15273,7 +16196,7 @@ fn test_publish_counters_rebuild_sources_for_property_index_change_and_skip_exis PropValue::String("active".to_string()), ); db.upsert_node( - 1, + "Person", "indexed", UpsertNodeOptions { props, @@ -15285,7 +16208,7 @@ fn test_publish_counters_rebuild_sources_for_property_index_change_and_skip_exis let (ready_rx, release_tx) = db.set_secondary_index_build_pause(); db.reset_publish_counters_for_test(); let info = db - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); let counters = db.publish_counter_snapshot_for_test(); @@ -15303,7 +16226,7 @@ fn test_publish_counters_rebuild_sources_for_property_index_change_and_skip_exis db.reset_publish_counters_for_test(); let existing = db - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); assert_eq!(existing.index_id, info.index_id); @@ -15344,7 +16267,7 @@ fn test_property_index_followups_coalesce_while_first_followup_is_in_flight() { props.insert("color".to_string(), red.clone()); let node_id = db .upsert_node( - 1, + "Person", "coalesce", UpsertNodeOptions { props, @@ -15355,25 +16278,28 @@ fn test_property_index_followups_coalesce_while_first_followup_is_in_flight() { db.flush().unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - let seg_dir = segment_dir(&db_path, db.segments_for_test()[0].segment_id); + let segment_id = db.segments_for_test()[0].segment_id; + let seg_dir = segment_dir(&db_path, segment_id); let sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); std::fs::remove_file(&sidecar_path).unwrap(); assert!(!sidecar_path.exists()); + db.reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); let (repair_ready_rx, repair_release_tx) = db.set_secondary_index_build_pause(); let (followup_ready_rx, followup_release_tx) = db.set_runtime_publish_pause(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); followup_ready_rx .recv_timeout(std::time::Duration::from_secs(5)) .unwrap(); assert_eq!(db.pending_secondary_index_followup_count_for_test(), 1); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); assert_eq!(db.pending_secondary_index_followup_count_for_test(), 1); followup_release_tx.send(()).unwrap(); @@ -15391,7 +16317,9 @@ fn test_property_index_followups_coalesce_while_first_followup_is_in_flight() { repair_release_tx.send(()).unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); - assert!(sidecar_path.exists()); + let repaired_sidecar_path = + crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); + assert!(repaired_sidecar_path.exists()); db.close().unwrap(); } @@ -15409,27 +16337,99 @@ fn test_targeted_equality_stats_refresh_replaces_reader_after_ready_transition() ) .unwrap(); + let mut node_ids = Vec::new(); for (key, color) in [("red-a", "red"), ("blue-a", "blue"), ("red-b", "red")] { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String(color.to_string())); - db.upsert_node(1, key, UpsertNodeOptions { props, ..Default::default() }) + let node_id = db + .upsert_node( + "Person", + key, + UpsertNodeOptions { + props, + ..Default::default() + }, + ) .unwrap(); + node_ids.push(node_id); } db.flush().unwrap(); let old_reader = db.segments_for_test()[0].clone(); + let segment_id = old_reader.segment_id; + let seg_dir = segment_dir(&db_path, segment_id); + let old_component_manifest = read_component_manifest_for_test(&seg_dir); + let packed_core_before = packed_core_snapshot_for_test(&seg_dir); + let base_stats_path = seg_dir.join(crate::planner_stats::PLANNER_STATS_FILENAME); + let base_stats_payload_before = read_external_component_payload_for_test(&base_stats_path); let old_read_view = db.published_read_view_for_test(); let old_stats_view = Arc::clone(&old_read_view.planner_stats); assert!(old_reader.planner_stats().unwrap().equality_index_stats.is_empty()); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); wait_for_published_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); let new_reader = db.segments_for_test()[0].clone(); assert!(!Arc::ptr_eq(&old_reader, &new_reader)); + assert!(new_reader.component_manifest_generation() > old_reader.component_manifest_generation()); + let new_component_manifest = read_component_manifest_for_test(&seg_dir); + assert!(new_component_manifest.generation > old_component_manifest.generation); + assert_packed_core_unchanged_for_test( + &seg_dir, + &packed_core_before, + "equality/planner-stats optional refresh", + ); + assert_eq!( + new_component_manifest.segment_data_id, + old_component_manifest.segment_data_id + ); + assert_eq!( + read_external_component_payload_for_test(&base_stats_path), + base_stats_payload_before, + "optional refresh must not overwrite the base planner_stats.dat path" + ); + let stats_record = new_component_manifest + .components + .iter() + .find(|record| record.kind == crate::segment_components::SegmentComponentKind::PlannerStats) + .expect("planner stats record should exist"); + let crate::segment_components::ComponentHandleV1::ExternalFile { + relative_path: stats_relative_path, + .. + } = &stats_record.handle + else { + panic!("planner stats refresh should use an external file handle"); + }; + assert!(stats_relative_path.starts_with("planner_stats.g")); + assert_ne!(stats_relative_path, crate::planner_stats::PLANNER_STATS_FILENAME); + assert!(seg_dir.join(stats_relative_path).exists()); + let eq_record = new_component_manifest + .components + .iter() + .find(|record| { + record.kind + == crate::segment_components::SegmentComponentKind::NodePropertyEqualityIndex { + index_id: info.index_id, + } + }) + .expect("equality sidecar record should exist"); + let crate::segment_components::ComponentHandleV1::ExternalFile { + relative_path: eq_relative_path, + .. + } = &eq_record.handle + else { + panic!("equality sidecar refresh should use an external file handle"); + }; + assert!(eq_relative_path.starts_with(&format!( + "secondary_indexes/node_prop_eq_{}.g", + info.index_id + ))); + assert!(!seg_dir + .join(format!("secondary_indexes/node_prop_eq_{}.dat", info.index_id)) + .exists()); let stats = new_reader.planner_stats().expect("targeted stats should be available"); assert_eq!( stats.build_kind, @@ -15449,6 +16449,11 @@ fn test_targeted_equality_stats_refresh_replaces_reader_after_ready_transition() .equality_index_stats .iter() .all(|stats| stats.index_id != info.index_id)); + let old_reader_node = old_reader + .get_node(node_ids[0]) + .unwrap() + .expect("old reader should keep packed core mapped after refresh"); + assert_eq!(old_reader_node.key, "red-a"); assert!(!old_stats_view.equality_index_rollups.contains_key(&info.index_id)); assert!(!old_read_view .planner_stats @@ -15462,6 +16467,304 @@ fn test_targeted_equality_stats_refresh_replaces_reader_after_ready_transition() db.close().unwrap(); } +#[test] +fn test_ready_index_adopts_refreshed_sidecar_when_stats_refresh_fails() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let red = PropValue::String("red".to_string()); + let db = DatabaseEngine::open( + &db_path, + &DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }, + ) + .unwrap(); + + let mut props = BTreeMap::new(); + props.insert("color".to_string(), red.clone()); + let node_id = db + .upsert_node("Person", "stats-fails-adopt", UpsertNodeOptions { props, ..Default::default() }) + .unwrap(); + db.flush().unwrap(); + + let info = db + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + let ready_entry = + wait_for_published_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); + + let segment_id = db.segments_for_test()[0].segment_id; + let seg_dir = segment_dir(&db_path, segment_id); + let sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); + std::fs::remove_file(&sidecar_path).unwrap(); + db.reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + let missing_reader = db.segments_for_test()[0].clone(); + + let groups = build_secondary_eq_groups_for_segment(&missing_reader, 1, "color").unwrap(); + install_secondary_eq_sidecar(&seg_dir, &ready_entry, &groups).unwrap(); + let manifest_after_sidecar = read_component_manifest_for_test(&seg_dir); + assert!(manifest_after_sidecar.generation > missing_reader.component_manifest_generation()); + + let blocked_stats_path = seg_dir.join(format!( + "planner_stats.g{:016}.dat", + manifest_after_sidecar.generation + 1 + )); + std::fs::create_dir(&blocked_stats_path).unwrap(); + + let ready = SecondaryIndexReadyApplied::from_ready_entry(&ready_entry, vec![segment_id]).unwrap(); + let manifest_write_lock = { + let core_guard = db.runtime.core.lock().unwrap(); + Arc::clone(&core_guard.as_ref().unwrap().manifest_write_lock) + }; + let refreshed = refresh_ready_secondary_index_planner_stats( + &db_path, + &manifest_write_lock, + &ready, + &AtomicBool::new(false), + ); + assert_eq!(refreshed.len(), 1); + assert_eq!( + read_component_manifest_for_test(&seg_dir).generation, + manifest_after_sidecar.generation, + "the forced stats failure must not be required for sidecar adoption" + ); + + db.runtime + .republish_secondary_index_state_and_refreshed_stats_if_open(&ready, refreshed); + let adopted_reader = db.segments_for_test()[0].clone(); + assert!(adopted_reader.component_manifest_generation() > missing_reader.component_manifest_generation()); + + db.reset_property_query_routes(); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); + let routes = db.property_query_route_snapshot(); + assert_eq!(routes.equality_scan_fallback, 0); + assert_eq!(routes.equality_index_lookup, 1); + + db.close().unwrap(); +} + +#[test] +fn test_optional_refresh_preserves_root_manifest_and_republishes_once() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = DatabaseEngine::open( + &db_path, + &DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }, + ) + .unwrap(); + + for (key, color) in [("refresh-a", "red"), ("refresh-b", "blue")] { + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String(color.to_string())); + db.upsert_node("Person", key, UpsertNodeOptions { props, ..Default::default() }) + .unwrap(); + } + db.flush().unwrap(); + + let info = db + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + let ready_entry = + wait_for_published_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); + let before_reader = db.segments_for_test()[0].clone(); + let seg_dir = segment_dir(&db_path, before_reader.segment_id); + let before_component_manifest = read_component_manifest_for_test(&seg_dir); + let packed_core_before = packed_core_snapshot_for_test(&seg_dir); + let root_manifest_before = std::fs::read(db_path.join("manifest.current")).unwrap(); + let ready = SecondaryIndexReadyApplied::from_ready_entry( + &ready_entry, + vec![before_reader.segment_id], + ) + .unwrap(); + let manifest_write_lock = { + let core_guard = db.runtime.core.lock().unwrap(); + Arc::clone(&core_guard.as_ref().unwrap().manifest_write_lock) + }; + + let refreshed = refresh_ready_secondary_index_planner_stats( + &db_path, + &manifest_write_lock, + &ready, + &AtomicBool::new(false), + ); + assert_eq!(refreshed.len(), 1); + let after_component_manifest = read_component_manifest_for_test(&seg_dir); + assert!(after_component_manifest.generation > before_component_manifest.generation); + assert_packed_core_unchanged_for_test( + &seg_dir, + &packed_core_before, + "targeted planner-stats optional refresh", + ); + assert_eq!( + after_component_manifest.segment_data_id, + before_component_manifest.segment_data_id + ); + assert_eq!( + std::fs::read(db_path.join("manifest.current")).unwrap(), + root_manifest_before, + "optional refresh must not rewrite the root DB manifest" + ); + + db.reset_publish_counters_for_test(); + db.runtime + .republish_secondary_index_state_and_refreshed_stats_if_open(&ready, refreshed); + let counters = db.publish_counter_snapshot_for_test(); + assert_eq!(counters.rebuild_sources, 1); + assert_eq!(counters.source_rebuilds, 1); + let after_reader = db.segments_for_test()[0].clone(); + assert!(after_reader.component_manifest_generation() > before_reader.component_manifest_generation()); + + db.close().unwrap(); +} + +#[test] +fn test_optional_refresh_adoption_rejects_older_reader_generation() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = DatabaseEngine::open( + &db_path, + &DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }, + ) + .unwrap(); + + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + db.upsert_node("Person", "stale-adopt", UpsertNodeOptions { props, ..Default::default() }) + .unwrap(); + db.flush().unwrap(); + + let info = db + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + let ready_entry = + wait_for_published_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); + let stale_reader = db.segments_for_test()[0].clone(); + let ready = SecondaryIndexReadyApplied::from_ready_entry( + &ready_entry, + vec![stale_reader.segment_id], + ) + .unwrap(); + let manifest_write_lock = { + let core_guard = db.runtime.core.lock().unwrap(); + Arc::clone(&core_guard.as_ref().unwrap().manifest_write_lock) + }; + let refreshed = refresh_ready_secondary_index_planner_stats( + &db_path, + &manifest_write_lock, + &ready, + &AtomicBool::new(false), + ); + assert_eq!(refreshed.len(), 1); + db.runtime + .republish_secondary_index_state_and_refreshed_stats_if_open(&ready, refreshed); + let newer_reader = db.segments_for_test()[0].clone(); + assert!(newer_reader.component_manifest_generation() > stale_reader.component_manifest_generation()); + + db.runtime + .republish_secondary_index_state_and_refreshed_stats_if_open( + &ready, + vec![(stale_reader.segment_id, stale_reader)], + ); + assert!(Arc::ptr_eq(&newer_reader, &db.segments_for_test()[0])); + + db.close().unwrap(); +} + +#[test] +fn test_optional_refresh_orphan_files_are_cleaned_on_reopen() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let (segment_id, index_id, referenced_eq_path, referenced_stats_path); + let temp_path; + let orphan_stats_path; + let orphan_eq_path; + let packed_core_before; + + { + let db = DatabaseEngine::open( + &db_path, + &DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }, + ) + .unwrap(); + + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + db.upsert_node("Person", "cleanup", UpsertNodeOptions { props, ..Default::default() }) + .unwrap(); + db.flush().unwrap(); + + let info = db + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_published_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = db.segments_for_test()[0].segment_id; + let seg_dir = segment_dir(&db_path, segment_id); + packed_core_before = packed_core_snapshot_for_test(&seg_dir); + referenced_eq_path = + crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); + let manifest = read_component_manifest_for_test(&seg_dir); + let stats_record = manifest + .components + .iter() + .find(|record| { + record.kind == crate::segment_components::SegmentComponentKind::PlannerStats + }) + .unwrap(); + let crate::segment_components::ComponentHandleV1::ExternalFile { + relative_path, + .. + } = &stats_record.handle + else { + panic!("planner stats should use external file"); + }; + referenced_stats_path = seg_dir.join(relative_path); + temp_path = seg_dir.join(".planner_stats.refresh_tmp.test.g0000000000009999.dat"); + orphan_stats_path = seg_dir.join("planner_stats.g0000000000009999.dat"); + orphan_eq_path = seg_dir + .join(format!("secondary_indexes/node_prop_eq_{index_id}.g0000000000009999.dat")); + std::fs::write(&temp_path, b"temp").unwrap(); + std::fs::write(&orphan_stats_path, b"orphan stats").unwrap(); + std::fs::write(&orphan_eq_path, b"orphan eq").unwrap(); + + db.close().unwrap(); + } + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert!(!temp_path.exists()); + assert!(!orphan_stats_path.exists()); + assert!(!orphan_eq_path.exists()); + assert!(referenced_eq_path.exists()); + assert!(referenced_stats_path.exists()); + let seg_dir = segment_dir(&db_path, segment_id); + assert_packed_core_unchanged_for_test( + &seg_dir, + &packed_core_before, + "optional orphan cleanup", + ); + assert_eq!(reopened.segments_for_test()[0].segment_id, segment_id); + let ready = reopened + .list_node_property_indexes() + .unwrap() + .into_iter() + .find(|entry| entry.index_id == index_id) + .unwrap(); + assert_eq!(ready.state, SecondaryIndexState::Ready); + + reopened.close().unwrap(); +} + #[test] fn test_public_index_list_waits_for_ready_publish_after_targeted_stats_refresh() { let dir = TempDir::new().unwrap(); @@ -15479,7 +16782,7 @@ fn test_public_index_list_waits_for_ready_publish_after_targeted_stats_refresh() let mut props = BTreeMap::new(); props.insert("color".to_string(), red.clone()); let node_id = db - .upsert_node(1, "publish-boundary", UpsertNodeOptions { props, ..Default::default() }) + .upsert_node("Person", "publish-boundary", UpsertNodeOptions { props, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -15488,7 +16791,7 @@ fn test_public_index_list_waits_for_ready_publish_after_targeted_stats_refresh() let (build_ready_rx, build_release_tx) = db.set_secondary_index_build_pause(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); build_ready_rx .recv_timeout(std::time::Duration::from_secs(5)) @@ -15509,7 +16812,7 @@ fn test_public_index_list_waits_for_ready_publish_after_targeted_stats_refresh() assert_ne!(listed_while_publish_paused[0].state, SecondaryIndexState::Ready); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let routes_before_publish = db.property_query_route_snapshot(); assert_eq!(routes_before_publish.equality_scan_fallback, 1); assert_eq!(routes_before_publish.equality_index_lookup, 0); @@ -15520,7 +16823,7 @@ fn test_public_index_list_waits_for_ready_publish_after_targeted_stats_refresh() assert!(old_read_view.secondary_index_entries.is_empty()); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let routes_after_publish = db.property_query_route_snapshot(); assert_eq!(routes_after_publish.equality_scan_fallback, 0); assert_eq!(routes_after_publish.equality_index_lookup, 1); @@ -15530,7 +16833,7 @@ fn test_public_index_list_waits_for_ready_publish_after_targeted_stats_refresh() #[test] #[cfg(unix)] -fn test_targeted_stats_refresh_failure_still_publishes_ready_index() { +fn test_targeted_stats_refresh_ignores_stale_legacy_tmp_and_publishes_ready_index() { use std::os::unix::fs::PermissionsExt; let dir = TempDir::new().unwrap(); @@ -15548,7 +16851,7 @@ fn test_targeted_stats_refresh_failure_still_publishes_ready_index() { let mut props = BTreeMap::new(); props.insert("color".to_string(), red.clone()); let node_id = db - .upsert_node(1, "stats-failure-ready", UpsertNodeOptions { props, ..Default::default() }) + .upsert_node("Person", "stats-failure-ready", UpsertNodeOptions { props, ..Default::default() }) .unwrap(); db.flush().unwrap(); @@ -15561,15 +16864,15 @@ fn test_targeted_stats_refresh_failure_still_publishes_ready_index() { std::fs::set_permissions(&tmp_path, perms).unwrap(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); let ready = wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); assert_eq!(ready.state, SecondaryIndexState::Ready); assert!(ready.last_error.is_none()); - assert!(!tmp_path.exists()); + assert!(tmp_path.exists()); db.reset_property_query_routes(); - assert_eq!(db.find_nodes(1, "color", &red).unwrap(), vec![node_id]); + assert_eq!(db.find_nodes("Person", "color", &red).unwrap(), vec![node_id]); let routes = db.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 0); assert_eq!(routes.equality_index_lookup, 1); @@ -15579,7 +16882,7 @@ fn test_targeted_stats_refresh_failure_still_publishes_ready_index() { assert!(stats .equality_index_stats .iter() - .all(|stats| stats.index_id != info.index_id)); + .any(|stats| stats.index_id == info.index_id && stats.sidecar_present_at_build)); db.close().unwrap(); } @@ -15600,18 +16903,19 @@ fn test_targeted_range_stats_refresh_writes_minimal_stats_when_missing() { for (key, score) in [("score-a", 10), ("score-b", 20), ("score-c", 30)] { let mut props = BTreeMap::new(); props.insert("score".to_string(), PropValue::Int(score)); - db.upsert_node(1, key, UpsertNodeOptions { props, ..Default::default() }) + db.upsert_node("Person", key, UpsertNodeOptions { props, ..Default::default() }) .unwrap(); } db.flush().unwrap(); let segment_id = db.segments_for_test()[0].segment_id; - let stats_path = segment_dir(&db_path, segment_id).join(crate::planner_stats::PLANNER_STATS_FILENAME); + let seg_dir = segment_dir(&db_path, segment_id); + let stats_path = seg_dir.join(crate::planner_stats::PLANNER_STATS_FILENAME); std::fs::remove_file(&stats_path).unwrap(); + let packed_core_before = packed_core_snapshot_for_test(&seg_dir); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -15620,6 +16924,29 @@ fn test_targeted_range_stats_refresh_writes_minimal_stats_when_missing() { .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); wait_for_published_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); + assert_packed_core_unchanged_for_test( + &seg_dir, + &packed_core_before, + "range/planner-stats optional refresh", + ); + let manifest = read_component_manifest_for_test(&seg_dir); + let range_record = manifest + .components + .iter() + .find(|record| { + record.kind + == crate::segment_components::SegmentComponentKind::NodePropertyRangeIndex { + index_id: info.index_id, + } + }) + .expect("range sidecar record should exist"); + assert!( + matches!( + &range_record.handle, + crate::segment_components::ComponentHandleV1::ExternalFile { .. } + ), + "range optional refresh should publish an external sidecar" + ); let reader = db.segments_for_test()[0].clone(); let stats = reader.planner_stats().expect("minimal targeted stats should load"); @@ -15631,7 +16958,7 @@ fn test_targeted_range_stats_refresh_writes_minimal_stats_when_missing() { assert_eq!(stats.general_property_sampled_node_count, 0); assert!(stats.property_stats.is_empty()); assert_eq!(stats.node_count, 3); - assert_eq!(stats.type_stats.len(), 1); + assert_eq!(stats.node_label_stats.len(), 1); let range = stats .range_index_stats .iter() @@ -15659,7 +16986,7 @@ fn test_targeted_range_stats_refresh_writes_minimal_stats_when_corrupt() { for (key, score) in [("corrupt-a", 10), ("corrupt-b", 20), ("corrupt-c", 30)] { let mut props = BTreeMap::new(); props.insert("score".to_string(), PropValue::Int(score)); - db.upsert_node(1, key, UpsertNodeOptions { props, ..Default::default() }) + db.upsert_node("Person", key, UpsertNodeOptions { props, ..Default::default() }) .unwrap(); } db.flush().unwrap(); @@ -15670,8 +16997,7 @@ fn test_targeted_range_stats_refresh_writes_minimal_stats_when_corrupt() { std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); let info = db - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -15719,13 +17045,13 @@ fn test_targeted_stats_refresh_drops_stale_declared_index_blocks() { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String(color.to_string())); props.insert("tier".to_string(), PropValue::String(tier.to_string())); - db.upsert_node(1, key, UpsertNodeOptions { props, ..Default::default() }) + db.upsert_node("Person", key, UpsertNodeOptions { props, ..Default::default() }) .unwrap(); } db.flush().unwrap(); let color = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, color.index_id, SecondaryIndexState::Ready); wait_for_published_property_index_state(&db, color.index_id, SecondaryIndexState::Ready); @@ -15737,10 +17063,10 @@ fn test_targeted_stats_refresh_drops_stale_declared_index_blocks() { .any(|stats| stats.index_id == color.index_id)); assert!(db - .drop_node_property_index(1, "color", SecondaryIndexKind::Equality) + .drop_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap()); let tier = db - .ensure_node_property_index(1, "tier", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "tier", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, tier.index_id, SecondaryIndexState::Ready); wait_for_published_property_index_state(&db, tier.index_id, SecondaryIndexState::Ready); @@ -15779,7 +17105,7 @@ fn test_targeted_stats_refresh_skips_obsolete_segments_before_write_and_swap() { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String("red".to_string())); db.upsert_node( - 1, + "Person", &format!("race-{segment}"), UpsertNodeOptions { props, ..Default::default() }, ) @@ -15789,7 +17115,7 @@ fn test_targeted_stats_refresh_skips_obsolete_segments_before_write_and_swap() { let old_segments = db.segments_for_test(); let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); let ready_entry = @@ -15832,7 +17158,7 @@ fn test_secondary_index_non_ready_finalize_outcomes_do_not_request_stats_refresh let entry = SecondaryIndexManifestEntry { index_id: 991, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -15848,9 +17174,12 @@ fn test_secondary_index_non_ready_finalize_outcomes_do_not_request_stats_refresh let snapshot = SecondaryEqBuildSnapshot { dense_config: None, - type_id: 1, + target: SecondaryIndexTargetDiscriminant::Node, + target_label_id: 1, prop_key: "color".to_string(), segment_ids: Vec::new(), + segment_infos: Vec::new(), + secondary_indexes: vec![entry.clone()], }; let outcome = { let core_guard = db.runtime.core.lock().unwrap(); @@ -15863,6 +17192,7 @@ fn test_secondary_index_non_ready_finalize_outcomes_do_not_request_stats_refresh &core.next_node_id_seen, &core.next_edge_id_seen, &core.engine_seq_seen, + &core.label_catalog, entry.index_id, &snapshot, &status, @@ -15892,7 +17222,7 @@ fn test_secondary_index_non_ready_finalize_outcomes_do_not_request_stats_refresh let entry = SecondaryIndexManifestEntry { index_id: 992, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Range { @@ -15910,10 +17240,13 @@ fn test_secondary_index_non_ready_finalize_outcomes_do_not_request_stats_refresh let snapshot = SecondaryRangeBuildSnapshot { dense_config: None, - type_id: 1, + target: SecondaryIndexTargetDiscriminant::Node, + target_label_id: 1, prop_key: "score".to_string(), domain: SecondaryIndexRangeDomain::Int, segment_ids: Vec::new(), + segment_infos: Vec::new(), + secondary_indexes: vec![entry.clone()], }; let outcome = { let core_guard = db.runtime.core.lock().unwrap(); @@ -15926,6 +17259,7 @@ fn test_secondary_index_non_ready_finalize_outcomes_do_not_request_stats_refresh &core.next_node_id_seen, &core.next_edge_id_seen, &core.engine_seq_seen, + &core.label_catalog, entry.index_id, &snapshot, &status, @@ -15965,7 +17299,7 @@ fn test_targeted_stats_refresh_swaps_only_affected_reader_arcs() { let mut props = BTreeMap::new(); props.insert("color".to_string(), PropValue::String("red".to_string())); db.upsert_node( - 1, + "Person", &format!("affected-{segment}"), UpsertNodeOptions { props, ..Default::default() }, ) @@ -15973,7 +17307,7 @@ fn test_targeted_stats_refresh_swaps_only_affected_reader_arcs() { db.flush().unwrap(); } let info = db - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&db, info.index_id, SecondaryIndexState::Ready); let ready_entry = @@ -15986,11 +17320,20 @@ fn test_targeted_stats_refresh_swaps_only_affected_reader_arcs() { .unwrap(); let refreshed_segment_id = before[0].segment_id; + let manifest = crate::manifest::load_manifest_readonly(&db_path) + .unwrap() + .unwrap(); + let refreshed_info = manifest + .segments + .iter() + .find(|segment| segment.id == refreshed_segment_id) + .unwrap(); let refreshed_reader = Arc::new( - SegmentReader::open( + SegmentReader::open_with_info( &segment_dir(&db_path, refreshed_segment_id), - refreshed_segment_id, - None, + refreshed_info, + manifest.dense_vector.as_ref(), + &manifest.secondary_indexes, ) .unwrap(), ); @@ -16013,7 +17356,7 @@ fn test_open_does_not_backfill_missing_planner_stats() { let db_path = dir.path().join("testdb"); { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.upsert_node(1, "missing-stats", UpsertNodeOptions::default()) + db.upsert_node("Person", "missing-stats", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); let segment_id = db.segments_for_test()[0].segment_id; diff --git a/src/engine/tests/query_planner.rs b/src/engine/tests/query_planner.rs index 8f4d3f8..54a2f83 100644 --- a/src/engine/tests/query_planner.rs +++ b/src/engine/tests/query_planner.rs @@ -1,4 +1,4 @@ -// Phase 23 CP1 planner tests: validation, scan-backed node queries, and explain. +// Planner tests: validation, scan-backed node queries, and explain. // --- validation and oracle helpers --- @@ -9,16 +9,102 @@ fn query_test_props(entries: &[(&str, PropValue)]) -> BTreeMap std::path::PathBuf { + let manifest_bytes = std::fs::read( + seg_dir.join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME), + ) + .unwrap(); + let manifest = crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap(); + match &record.handle { + crate::segment_components::ComponentHandleV1::ExternalFile { relative_path, .. } => { + seg_dir.join(relative_path) + } + crate::segment_components::ComponentHandleV1::PackedRange { .. } => { + panic!("test component unexpectedly used a packed handle") + } + } +} + +fn ready_node_property_equality_entry( + index_id: u64, + label_id: u32, + prop_key: &str, +) -> SecondaryIndexManifestEntry { + SecondaryIndexManifestEntry { + index_id, + target: SecondaryIndexTarget::NodeProperty { + label_id, + prop_key: prop_key.to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + } +} + +fn publish_planner_stats_for_test( + seg_dir: &std::path::Path, + stats: crate::planner_stats::SegmentPlannerStatsV1, + ready_indexes: &[SecondaryIndexManifestEntry], +) { + let payload = crate::planner_stats::planner_stats_sidecar_payload(stats) + .unwrap() + .expect("planner stats payload should fit test cap"); + crate::segment_writer::publish_planner_stats_component_payload( + seg_dir, + ready_indexes, + &payload, + ) + .unwrap(); +} + +fn corrupt_planner_stats_for_segment(db_path: &std::path::Path, segment_id: u64) { + let seg_dir = crate::segment_writer::segment_dir(db_path, segment_id); + let stats_path = segment_component_path( + &seg_dir, + crate::segment_components::SegmentComponentKind::PlannerStats, + ); + std::fs::write(stats_path, b"corrupt planner stats").unwrap(); +} + fn insert_query_node( engine: &DatabaseEngine, - type_id: u32, + label: &str, + key: &str, + entries: &[(&str, PropValue)], + weight: f32, +) -> u64 { + engine + .upsert_node( + label, + key, + UpsertNodeOptions { + props: query_test_props(entries), + weight, + ..Default::default() + }, + ) + .unwrap() +} + +fn insert_query_node_with_labels( + engine: &DatabaseEngine, + labels: &[&str], key: &str, entries: &[(&str, PropValue)], weight: f32, ) -> u64 { engine .upsert_node( - type_id, + labels, key, UpsertNodeOptions { props: query_test_props(entries), @@ -29,19 +115,62 @@ fn insert_query_node( .unwrap() } +fn node_label_filter(labels: &[&str], mode: LabelMatchMode) -> NodeLabelFilter { + NodeLabelFilter { + labels: labels.iter().map(|label| (*label).to_string()).collect(), + mode, + } +} + +fn query_label_filter(labels: &[&str], mode: LabelMatchMode) -> NodeQuery { + NodeQuery { + label_filter: Some(node_label_filter(labels, mode)), + ..Default::default() + } +} + fn query_ids( - type_id: Option, + label: Option<&str>, filter_exprs: Vec, allow_full_scan: bool, ) -> NodeQuery { NodeQuery { - type_id, + label_filter: label.map(|label| node_label_filter(&[label], LabelMatchMode::All)), filter: filter_from_conjunction(filter_exprs), allow_full_scan, ..Default::default() } } +fn pattern_node_with_labels( + alias: &str, + labels: &[&str], + filter_exprs: Vec, +) -> NodePattern { + NodePattern { + alias: alias.to_string(), + label_filter: Some(node_label_filter(labels, LabelMatchMode::All)), + ids: Vec::new(), + keys: Vec::new(), + filter: filter_from_conjunction(filter_exprs), + } +} + +fn pattern_node_with_label_filter( + alias: &str, + labels: &[&str], + mode: LabelMatchMode, + filter_exprs: Vec, +) -> NodePattern { + NodePattern { + alias: alias.to_string(), + label_filter: Some(node_label_filter(labels, mode)), + ids: Vec::new(), + keys: Vec::new(), + filter: filter_from_conjunction(filter_exprs), + } +} + fn filter_from_conjunction(filter_exprs: Vec) -> Option { match filter_exprs.len() { 0 => None, @@ -64,12 +193,12 @@ macro_rules! filter_and { fn pattern_node( alias: &str, - type_id: Option, + label: Option<&str>, filter_exprs: Vec, ) -> NodePattern { NodePattern { alias: alias.to_string(), - type_id, + label_filter: label.map(|label| node_label_filter(&[label], LabelMatchMode::All)), ids: Vec::new(), keys: Vec::new(), filter: filter_from_conjunction(filter_exprs), @@ -79,7 +208,7 @@ fn pattern_node( fn pattern_node_with_ids(alias: &str, ids: Vec) -> NodePattern { NodePattern { alias: alias.to_string(), - type_id: None, + label_filter: None, ids, keys: Vec::new(), filter: None, @@ -91,15 +220,17 @@ fn pattern_edge( from_alias: &str, to_alias: &str, direction: Direction, - type_filter: Option>, + label_filter: Option>, ) -> EdgePattern { EdgePattern { alias: alias.map(str::to_string), from_alias: from_alias.to_string(), to_alias: to_alias.to_string(), direction, - type_filter, - property_predicates: Vec::new(), + label_filter: label_filter + .map(|edge_labels| edge_labels.into_iter().map(str::to_string).collect()) + .unwrap_or_default(), + filter: None, } } @@ -113,2554 +244,7001 @@ fn pattern_query(nodes: Vec, edges: Vec) -> GraphPatte } } -fn expected_match(nodes: &[(&str, u64)], edges: &[(&str, u64)]) -> QueryMatch { - QueryMatch { - nodes: nodes - .iter() - .map(|(alias, id)| ((*alias).to_string(), *id)) - .collect(), - edges: edges - .iter() - .map(|(alias, id)| ((*alias).to_string(), *id)) - .collect(), +fn seed_query_test_catalog(engine: &DatabaseEngine) { + for label in ["Person", "Company", "Article", "Topic", "City"] { + engine.ensure_node_label(label).unwrap(); + } + for label in [ + "RELATES_TO", + "LIKES", + "FRIENDS_WITH", + "COLLABORATES_WITH", + "RELATED_TO", + "KNOWS", + "BLOCKS", + ] { + engine.ensure_edge_label(label).unwrap(); } } -fn oracle_node_matches(query: &NodeQuery, node: &NodeRecord) -> bool { - if query.type_id.is_some_and(|type_id| node.type_id != type_id) { - return false; - } - if !query.ids.is_empty() && !query.ids.contains(&node.id) { - return false; - } - if !query.keys.is_empty() && !query.keys.contains(&node.key) { - return false; - } - query - .filter - .as_ref() - .is_none_or(|filter| oracle_filter_matches(filter, node)) +fn query_test_engine() -> (TempDir, DatabaseEngine) { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + seed_query_test_catalog(&engine); + (dir, engine) } -fn oracle_filter_matches(filter: &NodeFilterExpr, node: &NodeRecord) -> bool { - match filter { - NodeFilterExpr::PropertyEquals { key, value } => { - node.props.get(key).is_some_and(|candidate| candidate == value) - } - NodeFilterExpr::PropertyIn { key, values } => node - .props - .get(key) - .is_some_and(|candidate| values.iter().any(|value| candidate == value)), - NodeFilterExpr::PropertyRange { key, lower, upper } => { - let Some(value) = node.props.get(key) else { - return false; - }; - let lower_matches = lower.as_ref().is_none_or(|bound| { - let Some(ordering) = compare_range_values(value, bound.value()) else { - return false; - }; - match bound { - PropertyRangeBound::Included(_) => ordering != std::cmp::Ordering::Less, - PropertyRangeBound::Excluded(_) => ordering == std::cmp::Ordering::Greater, - } - }); - let upper_matches = upper.as_ref().is_none_or(|bound| { - let Some(ordering) = compare_range_values(value, bound.value()) else { - return false; - }; - match bound { - PropertyRangeBound::Included(_) => ordering != std::cmp::Ordering::Greater, - PropertyRangeBound::Excluded(_) => ordering == std::cmp::Ordering::Less, - } - }); - lower_matches && upper_matches - } - NodeFilterExpr::UpdatedAtRange { lower_ms, upper_ms } => { - lower_ms.is_none_or(|lower| node.updated_at >= lower) - && upper_ms.is_none_or(|upper| node.updated_at <= upper) - } - NodeFilterExpr::PropertyExists { key } => node.props.contains_key(key), - NodeFilterExpr::PropertyMissing { key } => !node.props.contains_key(key), - NodeFilterExpr::And(children) => { - children.iter().all(|child| oracle_filter_matches(child, node)) - } - NodeFilterExpr::Or(children) => { - children.iter().any(|child| oracle_filter_matches(child, node)) +fn wait_until_after_millis(epoch_ms: i64) { + for _ in 0..100 { + if now_millis() > epoch_ms { + return; } - NodeFilterExpr::Not(child) => !oracle_filter_matches(child, node), + std::thread::sleep(std::time::Duration::from_millis(1)); } + assert!( + now_millis() > epoch_ms, + "millisecond clock did not advance past {epoch_ms}" + ); } -fn oracle_query_ids(engine: &DatabaseEngine, candidate_ids: &[u64], query: &NodeQuery) -> Vec { - let mut ids = candidate_ids.to_vec(); - ids.sort_unstable(); - ids.dedup(); - engine - .get_nodes(&ids) - .unwrap() - .into_iter() - .flatten() - .filter(|node| oracle_node_matches(query, node)) - .map(|node| node.id) - .collect() -} - -fn set_query_node_updated_at(engine: &DatabaseEngine, node_id: u64, updated_at: i64) { - let node = engine.get_node(node_id).unwrap().unwrap(); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { - created_at: updated_at, - updated_at, - ..node - })) - .unwrap(); -} +#[test] +fn edge_query_normalizes_anchors_and_enforces_full_scan_opt_in() { + let (_dir, engine) = query_test_engine(); + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); -fn explain_input_node(plan: &QueryPlan) -> &QueryPlanNode { - match &plan.root { - QueryPlanNode::VerifyNodeFilter { input } => input.as_ref(), - other => panic!("expected VerifyNodeFilter root, got {other:?}"), - } -} + let err = published + .view + .normalize_edge_query(&EdgeQuery::default()) + .unwrap_err(); + assert!( + err.to_string().contains("edge query requires label"), + "unexpected error: {err}" + ); -fn explain_input_nodes(plan: &QueryPlan) -> Vec { - match explain_input_node(plan) { - QueryPlanNode::Intersect { inputs } => inputs.clone(), - node => vec![node.clone()], - } -} + let filter_only = EdgeQuery { + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(0.5), + upper: None, + }), + ..Default::default() + }; + let err = published + .view + .normalize_edge_query(&filter_only) + .unwrap_err(); + assert!( + err.to_string().contains("allow_full_scan"), + "unexpected error: {err}" + ); -fn assert_plan_input_nodes(plan: &QueryPlan, expected: Vec) { - assert_eq!(explain_input_nodes(plan), expected); + let always_false_filter_only = EdgeQuery { + filter: Some(EdgeFilterExpr::PropertyIn { + key: "status".to_string(), + values: Vec::new(), + }), + ..Default::default() + }; + let normalized = published + .view + .normalize_edge_query(&always_false_filter_only) + .unwrap(); + assert!(matches!(normalized.filter, NormalizedEdgeFilter::AlwaysFalse)); + + let anchored = EdgeQuery { + label: Some("FRIENDS_WITH".to_string()), + ids: vec![9, 3, 9, 1], + from_ids: vec![4, 4, 2], + to_ids: vec![8, 6, 8], + endpoint_ids: vec![5, 1, 5], + ..Default::default() + }; + let normalized = published.view.normalize_edge_query(&anchored).unwrap(); + let expected_label_id = engine.get_edge_label_id("FRIENDS_WITH").unwrap().unwrap(); + assert_eq!(normalized.label_id, Some(expected_label_id)); + assert_eq!(normalized.ids, vec![1, 3, 9]); + assert_eq!(normalized.from_ids, vec![2, 4]); + assert_eq!(normalized.to_ids, vec![6, 8]); + assert_eq!(normalized.endpoint_ids, vec![1, 5]); } -fn assert_plan_includes_input_nodes(plan: &QueryPlan, expected: &[QueryPlanNode]) { - let mut actual = explain_input_nodes(plan); - for expected_node in expected { - let position = actual - .iter() - .position(|node| node == expected_node) - .unwrap_or_else(|| panic!("expected plan to include {expected_node:?}; got {actual:?}")); - actual.remove(position); - } -} +#[test] +fn edge_query_filter_validation_and_canonical_in_dedupe() { + let (_dir, engine) = query_test_engine(); + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); -fn pattern_anchor_plan_node(plan: &QueryPlan) -> (&str, &QueryPlanNode) { - let pattern = match &plan.root { - QueryPlanNode::PatternExpand { .. } => &plan.root, - QueryPlanNode::VerifyEdgePredicates { input } => input.as_ref(), - other => panic!("expected pattern expand root, got {other:?}"), + let duplicate_single = EdgeQuery { + label: Some("RELATES_TO".to_string()), + filter: Some(EdgeFilterExpr::PropertyIn { + key: "score".to_string(), + values: vec![PropValue::Int(10), PropValue::Int(10)], + }), + ..Default::default() }; - match pattern { - QueryPlanNode::PatternExpand { - anchor_alias, - input, - } => match input.as_ref() { - QueryPlanNode::VerifyNodeFilter { input } => (anchor_alias.as_str(), input.as_ref()), - other => panic!("expected VerifyNodeFilter pattern input, got {other:?}"), - }, - other => panic!("expected pattern expand node, got {other:?}"), - } -} + let normalized = published + .view + .normalize_edge_query(&duplicate_single) + .unwrap(); + assert!(matches!( + normalized.filter, + NormalizedEdgeFilter::PropertyEquals { + key, + value: PropValue::Int(10), + } if key == "score" + )); -fn pattern_anchor_input_nodes(plan: &QueryPlan) -> Vec { - match pattern_anchor_plan_node(plan).1 { - QueryPlanNode::Intersect { inputs } => inputs.clone(), - node => vec![node.clone()], + let signed_zero = EdgeQuery { + label: Some("RELATES_TO".to_string()), + filter: Some(EdgeFilterExpr::PropertyIn { + key: "z".to_string(), + values: vec![PropValue::Float(-0.0), PropValue::Float(0.0)], + }), + ..Default::default() + }; + let normalized = published + .view + .normalize_edge_query(&signed_zero) + .unwrap(); + match normalized.filter { + NormalizedEdgeFilter::PropertyIn { values, value_keys, .. } => { + assert_eq!(values.len(), 2); + assert_eq!(value_keys.len(), 2); + } + other => panic!("expected signed-zero IN to preserve two canonical values, got {other:?}"), } -} -fn planned_pattern_anchor_and_edge_aliases( - engine: &DatabaseEngine, - query: &GraphPatternQuery, -) -> (String, Vec) { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let normalized = published.view.normalize_pattern_query(query).unwrap(); - let planned = published + let invalid_weight = EdgeQuery { + label: Some("RELATES_TO".to_string()), + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(f32::NAN), + upper: None, + }), + ..Default::default() + }; + let err = published .view - .plan_normalized_pattern_query(&normalized) - .unwrap(); - let aliases = planned - .expansion_order - .iter() - .map(|&edge_index| { - normalized.edges[edge_index] - .alias - .clone() - .unwrap_or_else(|| format!("edge-{edge_index}")) - }) - .collect(); - (normalized.nodes[planned.anchor_index].alias.clone(), aliases) -} + .normalize_edge_query(&invalid_weight) + .unwrap_err(); + assert!(err.to_string().contains("must not be NaN")); -fn planned_pattern_anchor_sort_and_edge_aliases( - engine: &DatabaseEngine, - query: &GraphPatternQuery, -) -> (String, String, Vec) { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let normalized = published.view.normalize_pattern_query(query).unwrap(); - let planned = published + let empty_updated_at = EdgeQuery { + label: Some("RELATES_TO".to_string()), + filter: Some(EdgeFilterExpr::UpdatedAtRange { + lower_ms: None, + upper_ms: None, + }), + ..Default::default() + }; + let err = published .view - .plan_normalized_pattern_query(&normalized) + .normalize_edge_query(&empty_updated_at) + .unwrap_err(); + assert!(err.to_string().contains("at least one bound")); + + let inverted_valid_from = EdgeQuery { + label: Some("RELATES_TO".to_string()), + filter: Some(EdgeFilterExpr::ValidFromRange { + lower_ms: Some(20), + upper_ms: Some(10), + }), + ..Default::default() + }; + let normalized = published + .view + .normalize_edge_query(&inverted_valid_from) .unwrap(); - let aliases = planned - .expansion_order - .iter() - .map(|&edge_index| { - normalized.edges[edge_index] - .alias - .clone() - .unwrap_or_else(|| format!("edge-{edge_index}")) - }) - .collect(); - ( - normalized.nodes[planned.anchor_index].alias.clone(), - planned.sort_anchor_alias, - aliases, - ) -} + assert!(matches!(normalized.filter, NormalizedEdgeFilter::AlwaysFalse)); -fn plan_contains_fallback_full_node_scan(node: &QueryPlanNode) -> bool { - match node { - QueryPlanNode::FallbackFullNodeScan => true, - QueryPlanNode::Intersect { inputs } | QueryPlanNode::Union { inputs } => { - inputs.iter().any(plan_contains_fallback_full_node_scan) - } - QueryPlanNode::VerifyNodeFilter { input } - | QueryPlanNode::VerifyEdgePredicates { input } - | QueryPlanNode::PatternExpand { input, .. } => { - plan_contains_fallback_full_node_scan(input) - } - _ => false, - } + let mixed_property_range = EdgeQuery { + label: Some("RELATES_TO".to_string()), + filter: Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(1))), + upper: Some(PropertyRangeBound::Included(PropValue::Float(2.0))), + }), + ..Default::default() + }; + let err = published + .view + .normalize_edge_query(&mixed_property_range) + .unwrap_err(); + assert!(err.to_string().contains("same PropValue variant")); } #[test] -fn test_planner_stats_view_rebuilds_only_with_read_sources() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let initial = engine.planner_stats_view_for_test(); - assert_eq!(initial.generation, 1); - assert_eq!(initial.segment_count, 0); - - insert_query_node( - &engine, - 1, - "active", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let after_write = engine.planner_stats_view_for_test(); - assert!(std::sync::Arc::ptr_eq(&initial, &after_write)); - assert_eq!(after_write.generation, initial.generation); +fn edge_query_metadata_and_hydrated_verifier_semantics() { + let (_dir, engine) = query_test_engine(); + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let edge_label_id = engine.get_edge_label_id("LIKES").unwrap().unwrap(); + let edge = EdgeRecord { + id: 42, + from: 7, + to: 9, + label_id: edge_label_id, + props: query_test_props(&[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(5)), + ]), + created_at: 1, + updated_at: 100, + weight: 0.75, + valid_from: 10, + valid_to: 20, + last_write_seq: 0, + }; - engine.flush().unwrap(); - let after_flush = engine.planner_stats_view_for_test(); - assert!(!std::sync::Arc::ptr_eq(&after_write, &after_flush)); - assert!(after_flush.generation > after_write.generation); - assert_eq!(after_flush.segment_count, 1); - assert_eq!(after_flush.available_segment_stats, 1); - assert_eq!(after_flush.full_rollup.node_count, 1); + let query = EdgeQuery { + label: Some("LIKES".to_string()), + ids: vec![42], + from_ids: vec![7], + endpoint_ids: vec![9], + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::WeightRange { + lower: Some(0.5), + upper: Some(1.0), + }, + EdgeFilterExpr::UpdatedAtRange { + lower_ms: Some(90), + upper_ms: Some(110), + }, + EdgeFilterExpr::ValidAt { epoch_ms: 10 }, + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + ])), + ..Default::default() + }; + let normalized = published.view.normalize_edge_query(&query).unwrap(); + let meta = EdgeMetadataForQuery::from(&edge); + assert!(edge_filter_requires_hydration(&normalized.filter)); + assert!(edge_query_metadata_matches(&normalized, &meta)); + assert!(edge_query_matches(&normalized, &edge)); + + let expired = EdgeQuery { + label: Some("LIKES".to_string()), + ids: vec![42], + filter: Some(EdgeFilterExpr::ValidAt { epoch_ms: 20 }), + ..Default::default() + }; + let normalized = published + .view + .normalize_edge_query(&expired) + .unwrap(); + assert!(!edge_query_metadata_matches(&normalized, &meta)); + assert!(!edge_query_matches(&normalized, &edge)); - engine.close().unwrap(); + let nan_weight = EdgeRecord { + weight: f32::NAN, + ..edge.clone() + }; + let range_query = EdgeQuery { + label: Some("LIKES".to_string()), + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(0.0), + upper: Some(1.0), + }), + ..Default::default() + }; + let normalized = published + .view + .normalize_edge_query(&range_query) + .unwrap(); + let meta = EdgeMetadataForQuery::from(&nan_weight); + assert!(!edge_query_metadata_matches(&normalized, &meta)); + assert!(!edge_query_matches(&normalized, &nan_weight)); } #[test] -fn test_planner_stats_stale_risk_uses_newer_sample_shadowing() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - for index in 0..16 { - insert_query_node( - &engine, - 1, - &format!("shadow-{index:02}"), - &[("version", PropValue::Int(1))], - 1.0, - ); - } - engine.flush().unwrap(); - for index in 0..8 { - insert_query_node( - &engine, - 1, - &format!("shadow-{index:02}"), - &[("version", PropValue::Int(2))], - 1.0, - ); - } - engine.flush().unwrap(); - - let stats_view = engine.planner_stats_view_for_test(); - assert_eq!( - stats_view.max_segment_stale_risk(), - crate::planner_stats::StalePostingRisk::High - ); +fn edge_query_executes_type_endpoint_metadata_and_explain_sources() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "c", &[], 1.0); - engine.close().unwrap(); -} + let keep = engine + .upsert_edge( + a, + b, + "KNOWS", + UpsertEdgeOptions { + weight: -0.0, + valid_from: Some(10), + valid_to: Some(20), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "KNOWS", + UpsertEdgeOptions { + weight: 2.0, + valid_from: Some(10), + valid_to: Some(20), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + b, + c, + "REPORTS_TO", + UpsertEdgeOptions { + weight: 0.0, + valid_from: Some(10), + valid_to: Some(20), + ..Default::default() + }, + ) + .unwrap(); -#[test] -fn test_planner_stats_stale_risk_uses_newer_tombstones() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = EdgeQuery { + label: Some("KNOWS".to_string()), + from_ids: vec![a], + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(0.0), + upper: Some(0.0), + }), + ..Default::default() + }; - let mut ids = Vec::new(); - for index in 0..16 { - ids.push(insert_query_node( - &engine, - 1, - &format!("delete-{index:02}"), - &[], - 1.0, - )); - } - engine.flush().unwrap(); - engine.delete_node(ids[0]).unwrap(); - engine.flush().unwrap(); + let ids = engine.query_edge_ids(&query).unwrap(); + assert_eq!(ids.edge_ids, vec![keep]); + assert_eq!(ids.next_cursor, None); - let stats_view = engine.planner_stats_view_for_test(); + let edges = engine.query_edges(&query).unwrap(); assert_eq!( - stats_view.max_segment_stale_risk(), - crate::planner_stats::StalePostingRisk::Medium + edges.edges.iter().map(|edge| edge.id).collect::>(), + vec![keep] ); - engine.close().unwrap(); + let plan = engine.explain_edge_query(&query).unwrap(); + assert_eq!(plan.kind, QueryPlanKind::EdgeQuery); + assert!(matches!( + &plan.root, + QueryPlanNode::VerifyEdgeFilter { .. } + )); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeLabelIndex)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgeEndpointAdjacency + )); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeMetadataScan)); } #[test] -fn test_write_adjacent_helper_reads_do_not_rebuild_planner_stats_view() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_query_triple_index_returns_parallel_edges() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); - let delete_a = insert_query_node(&engine, 1, "delete-a", &[], 1.0); - let delete_b = insert_query_node(&engine, 1, "delete-b", &[], 1.0); - let patch_c = insert_query_node(&engine, 2, "patch-c", &[], 1.0); - let patch_d = insert_query_node(&engine, 2, "patch-d", &[], 1.0); - let prune_e = insert_query_node(&engine, 90, "prune-e", &[], 0.1); - let prune_f = insert_query_node(&engine, 91, "prune-f", &[], 1.0); - engine - .upsert_edge(delete_a, delete_b, 10, UpsertEdgeOptions::default()) - .unwrap(); - let patch_edge = engine - .upsert_edge(patch_c, patch_d, 20, UpsertEdgeOptions::default()) + let first = engine + .upsert_edge( + a, + b, + "FRIENDS_WITH", + UpsertEdgeOptions { + weight: 1.0, + ..Default::default() + }, + ) .unwrap(); - engine - .upsert_edge(prune_e, prune_f, 30, UpsertEdgeOptions::default()) + let second = engine + .upsert_edge( + a, + b, + "FRIENDS_WITH", + UpsertEdgeOptions { + weight: 2.0, + ..Default::default() + }, + ) .unwrap(); - engine.flush().unwrap(); + assert_ne!(first, second); - let stats_before = engine.planner_stats_view_for_test(); - let generation_before = stats_before.generation; - let source_builds_before = engine.published_read_source_build_count_for_test(); - engine.reset_publish_counters_for_test(); + let query = EdgeQuery { + label: Some("FRIENDS_WITH".to_string()), + from_ids: vec![a], + to_ids: vec![b], + ..Default::default() + }; + let ids = engine.query_edge_ids(&query).unwrap(); + assert_eq!(ids.edge_ids, vec![first, second]); - engine.delete_node(delete_a).unwrap(); - engine - .graph_patch(&GraphPatch { - invalidate_edges: vec![(patch_edge, 1)], - delete_node_ids: vec![patch_c], - ..Default::default() - }) + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeTripleIndex)); +} + +#[test] +fn edge_query_reads_segment_and_active_memtable_sources() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "c", &[], 1.0); + + let flushed = engine + .upsert_edge(a, b, "BLOCKS", UpsertEdgeOptions::default()) .unwrap(); - let prune = engine - .prune(&PrunePolicy { - max_age_ms: None, - max_weight: Some(0.5), - type_id: Some(90), - }) + engine.flush().unwrap(); + let active = engine + .upsert_edge(a, c, "BLOCKS", UpsertEdgeOptions::default()) .unwrap(); - assert_eq!(prune.nodes_pruned, 1); - assert_eq!(prune.edges_pruned, 1); - - let stats_after = engine.planner_stats_view_for_test(); - let counters = engine.publish_counter_snapshot_for_test(); - assert_eq!(counters.rebuild_sources, 0); - assert_eq!(counters.source_rebuilds, 0); - assert_eq!( - engine.published_read_source_build_count_for_test(), - source_builds_before - ); - assert!(std::sync::Arc::ptr_eq(&stats_before, &stats_after)); - assert_eq!(stats_after.generation, generation_before); - engine.close().unwrap(); + let query = EdgeQuery { + label: Some("BLOCKS".to_string()), + ..Default::default() + }; + let ids = engine.query_edge_ids(&query).unwrap(); + assert_eq!(ids.edge_ids, vec![flushed, active]); } #[test] -fn test_planner_stats_corruption_degrades_without_index_repair_followup() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - insert_query_node( - &engine, - 1, - "red", - &[("status", PropValue::String("red".to_string()))], - 1.0, - ); - insert_query_node( - &engine, - 1, - "blue", - &[("status", PropValue::String("blue".to_string()))], - 1.0, - ); - engine.flush().unwrap(); - engine.close().unwrap(); +fn edge_query_metadata_sidecar_unavailable_falls_back_at_engine_level() { + #[derive(Clone, Copy)] + enum SidecarRewrite { + Missing, + Corrupt, } - let stats_path = crate::segment_writer::segment_dir(&db_path, 1) - .join(crate::planner_stats::PLANNER_STATS_FILENAME); - std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); - - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let stats_view = reopened.planner_stats_view_for_test(); - assert_eq!(stats_view.segment_count, 1); - assert_eq!(stats_view.available_segment_stats, 0); - assert_eq!(stats_view.unavailable_segment_stats, 1); - - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("red".to_string()), - }], - false, - ); - assert_eq!(reopened.query_node_ids(&query).unwrap().items.len(), 1); - let plan = reopened.explain_node_query(&query).unwrap(); - assert!(!plan.warnings.contains(&QueryPlanWarning::MissingReadyIndex)); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyEqualityIndex]); + fn edge_metadata_kind(logical_name: &str) -> crate::segment_components::SegmentComponentKind { + match logical_name { + crate::edge_metadata::EDGE_WEIGHT_INDEX_LOGICAL_NAME => { + crate::segment_components::SegmentComponentKind::EdgeWeightIndex + } + crate::edge_metadata::EDGE_UPDATED_AT_INDEX_LOGICAL_NAME => { + crate::segment_components::SegmentComponentKind::EdgeUpdatedAtIndex + } + crate::edge_metadata::EDGE_VALID_FROM_INDEX_LOGICAL_NAME => { + crate::segment_components::SegmentComponentKind::EdgeValidFromIndex + } + crate::edge_metadata::EDGE_VALID_TO_INDEX_LOGICAL_NAME => { + crate::segment_components::SegmentComponentKind::EdgeValidToIndex + } + other => panic!("unexpected edge metadata logical name {other}"), + } + } - reopened.close().unwrap(); -} + fn rewrite_sidecar(seg_dir: &std::path::Path, logical_name: &str, mode: SidecarRewrite) { + let kind = edge_metadata_kind(logical_name); + match mode { + SidecarRewrite::Missing => { + let manifest_path = seg_dir + .join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME); + let mut manifest = crate::segment_components::decode_manifest_envelope( + &std::fs::read(&manifest_path).unwrap(), + ) + .unwrap(); + manifest.components.retain(|record| record.kind != kind); + let data = crate::segment_components::encode_manifest_envelope(&manifest).unwrap(); + std::fs::write(manifest_path, data).unwrap(); + } + SidecarRewrite::Corrupt => { + let manifest_path = seg_dir + .join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME); + let manifest = crate::segment_components::decode_manifest_envelope( + &std::fs::read(&manifest_path).unwrap(), + ) + .unwrap(); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap(); + match &record.handle { + crate::segment_components::ComponentHandleV1::ExternalFile { + relative_path, + .. + } => std::fs::write(seg_dir.join(relative_path), b"corrupt metadata sidecar") + .unwrap(), + crate::segment_components::ComponentHandleV1::PackedRange { offset, .. } => { + let core_path = + seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); + let mut core = std::fs::read(&core_path).unwrap(); + let header = + crate::segment_components::decode_identity_header(&core).unwrap(); + let start = header.payload_offset as usize + *offset as usize; + core[start..start + 8].copy_from_slice(&u64::MAX.to_le_bytes()); + std::fs::write(core_path, core).unwrap(); + } + } + } + } + } -#[test] -fn test_planner_stats_zero_is_advisory_not_empty_result() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let index_id; - { + fn run_case(logical_name: &str, filter: EdgeFilterExpr, rewrite: SidecarRewrite) { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "c", &[], 1.0); + let first = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_41", + UpsertEdgeOptions { + weight: 1.0, + valid_from: Some(10), + valid_to: Some(100), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "EDGE_LABEL_41", + UpsertEdgeOptions { + weight: 2.0, + valid_from: Some(20), + valid_to: Some(200), + ..Default::default() + }, + ) .unwrap(); - index_id = info.index_id; - wait_for_property_index_state(&engine, index_id, SecondaryIndexState::Ready); - insert_query_node( - &engine, - 1, - "red", - &[("status", PropValue::String("red".to_string()))], - 1.0, - ); engine.flush().unwrap(); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_41".to_string()), + filter: Some(filter), + ..Default::default() + }; + let baseline = engine.query_edge_ids(&query).unwrap().edge_ids; + assert!( + baseline.contains(&first), + "baseline should include the selective edge for {logical_name}" + ); engine.close().unwrap(); + + let seg_dir = crate::segment_writer::segment_dir(&db_path, 1); + rewrite_sidecar(&seg_dir, logical_name, rewrite); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(reopened.query_edge_ids(&query).unwrap().edge_ids, baseline); + let plan = reopened.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeMetadataScan)); + reopened.close().unwrap(); } - let seg_dir = crate::segment_writer::segment_dir(&db_path, 1); - let mut stats = match crate::planner_stats::read_planner_stats_sidecar(&seg_dir, 1, 1, 0) { - crate::planner_stats::PlannerStatsAvailability::Available(stats) => *stats, - other => panic!("expected available planner stats, got {other:?}"), - }; - let equality = stats - .equality_index_stats - .iter_mut() - .find(|stats| stats.index_id == index_id) - .expect("expected equality stats for test index"); - equality.total_postings = 0; - equality.value_group_count = 0; - equality.max_group_postings = 0; - equality.top_value_hashes.clear(); - assert_eq!( - crate::planner_stats::write_planner_stats_sidecar_atomic(&seg_dir, stats).unwrap(), - crate::planner_stats::PlannerStatsWriteOutcome::Written + run_case( + crate::edge_metadata::EDGE_WEIGHT_INDEX_LOGICAL_NAME, + EdgeFilterExpr::WeightRange { + lower: Some(1.0), + upper: Some(1.0), + }, + SidecarRewrite::Missing, ); - - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let red = PropValue::String("red".to_string()); - let red_hash = hash_prop_value(&red); - assert_eq!( - reopened - .planner_stats_view_for_test() - .equality_segment_estimate(index_id, 1, &[red_hash]) - .unwrap(), - crate::planner_stats::PlannerStatsValueEstimate { - count: 0, - exact: true, - } + run_case( + crate::edge_metadata::EDGE_UPDATED_AT_INDEX_LOGICAL_NAME, + EdgeFilterExpr::UpdatedAtRange { + lower_ms: Some(i64::MIN), + upper_ms: Some(i64::MAX), + }, + SidecarRewrite::Corrupt, ); - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: red, - }], - false, + run_case( + crate::edge_metadata::EDGE_VALID_FROM_INDEX_LOGICAL_NAME, + EdgeFilterExpr::ValidFromRange { + lower_ms: Some(10), + upper_ms: Some(10), + }, + SidecarRewrite::Missing, + ); + run_case( + crate::edge_metadata::EDGE_VALID_TO_INDEX_LOGICAL_NAME, + EdgeFilterExpr::ValidToRange { + lower_ms: Some(100), + upper_ms: Some(100), + }, + SidecarRewrite::Corrupt, ); - assert_eq!(reopened.query_node_ids(&query).unwrap().items.len(), 1); - let plan = reopened.explain_node_query(&query).unwrap(); - assert_eq!(plan.estimated_candidates, Some(0)); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyEqualityIndex]); - - reopened.close().unwrap(); } #[test] -fn test_planner_stats_low_equality_estimate_uses_capped_materialization() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let index_id; - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - index_id = info.index_id; - wait_for_property_index_state(&engine, index_id, SecondaryIndexState::Ready); - for idx in 0..=QUERY_RANGE_CANDIDATE_CAP { - insert_query_node( - &engine, - 1, - &format!("active-{idx}"), - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - } - engine.flush().unwrap(); - engine.close().unwrap(); - } +fn edge_query_property_filter_uses_legal_universe_and_hydrates() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "c", &[], 1.0); - let node_count = (QUERY_RANGE_CANDIDATE_CAP + 1) as u64; - let seg_dir = crate::segment_writer::segment_dir(&db_path, 1); - let mut stats = match crate::planner_stats::read_planner_stats_sidecar(&seg_dir, 1, node_count, 0) - { - crate::planner_stats::PlannerStatsAvailability::Available(stats) => *stats, - other => panic!("expected available planner stats, got {other:?}"), - }; - let equality = stats - .equality_index_stats - .iter_mut() - .find(|stats| stats.index_id == index_id) - .expect("expected equality stats for test index"); - equality.total_postings = 0; - equality.value_group_count = 0; - equality.max_group_postings = 0; - equality.top_value_hashes.clear(); - assert_eq!( - crate::planner_stats::write_planner_stats_sidecar_atomic(&seg_dir, stats).unwrap(), - crate::planner_stats::PlannerStatsWriteOutcome::Written - ); + let keep = engine + .upsert_edge( + a, + b, + "DEPENDS_ON", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "DEPENDS_ON", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { + let query = EdgeQuery { + label: Some("DEPENDS_ON".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { key: "status".to_string(), value: PropValue::String("active".to_string()), - }], - false, - ); - { - let (_guard, published) = reopened.runtime.published_snapshot().unwrap(); - let normalized = published.view.normalize_node_query(&query).unwrap(); - let planned = published.view.plan_normalized_node_query(&normalized).unwrap(); - let NodePhysicalPlan::Source(source) = planned.driver else { - panic!("expected equality source driver"); - }; - assert_eq!(source.kind, NodeQueryCandidateSourceKind::PropertyEqualityIndex); - assert_eq!(source.estimate.known_upper_bound(), Some(0)); - assert!(!source.estimate.can_use_uncapped_equality_materialization()); - } - let result = reopened.query_node_ids(&query).unwrap(); - assert_eq!(result.items.len(), QUERY_RANGE_CANDIDATE_CAP + 1); - assert_eq!(result.next_cursor, None); - - reopened.close().unwrap(); -} - -#[test] -fn test_planner_stats_back_type_and_full_scan_explain_estimates() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - for idx in 0..3 { - insert_query_node(&engine, 1, &format!("type1-{idx}"), &[], 1.0); - } - for idx in 0..2 { - insert_query_node(&engine, 2, &format!("type2-{idx}"), &[], 1.0); - } - engine.flush().unwrap(); - { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let type_estimate = published.view.node_type_estimate(1).unwrap(); - assert_eq!(type_estimate.kind, PlannerEstimateKind::StatsExact); - assert_eq!(type_estimate.known_upper_bound(), Some(3)); - let full_estimate = published.view.full_scan_estimate(); - assert_eq!(full_estimate.kind, PlannerEstimateKind::StatsExact); - assert_eq!(full_estimate.known_upper_bound(), Some(5)); - } + }), + ..Default::default() + }; + let ids = engine.query_edge_ids(&query).unwrap(); + assert_eq!(ids.edge_ids, vec![keep]); - let type_query = query_ids(Some(1), Vec::new(), false); - assert_eq!(engine.query_node_ids(&type_query).unwrap().items.len(), 3); - let type_plan = engine.explain_node_query(&type_query).unwrap(); - assert_eq!(type_plan.estimated_candidates, Some(3)); - assert_plan_input_nodes(&type_plan, vec![QueryPlanNode::NodeTypeIndex]); + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan.warnings.contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); - let full_scan_query = NodeQuery { - allow_full_scan: true, + let metadata_and_property_query = EdgeQuery { + label: Some("DEPENDS_ON".to_string()), + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::WeightRange { + lower: Some(0.5), + upper: Some(1.5), + }, + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + ])), ..Default::default() }; - assert_eq!(engine.query_node_ids(&full_scan_query).unwrap().items.len(), 5); - let full_scan_plan = engine.explain_node_query(&full_scan_query).unwrap(); - assert_eq!(full_scan_plan.estimated_candidates, Some(5)); - assert_plan_input_nodes(&full_scan_plan, vec![QueryPlanNode::FallbackFullNodeScan]); assert_eq!( - full_scan_plan.warnings, - vec![QueryPlanWarning::FullScanExplicitlyAllowed] + engine + .query_edge_ids(&metadata_and_property_query) + .unwrap() + .edge_ids, + vec![keep] ); - - engine.close().unwrap(); -} - -#[test] -fn test_active_memtable_only_estimates_are_exact_cheap() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let mixed_plan = engine + .explain_edge_query(&metadata_and_property_query) .unwrap(); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - let score = engine - .ensure_node_property_index( - 1, + assert!(mixed_plan + .warnings + .contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert!(mixed_plan + .warnings + .contains(&QueryPlanWarning::VerifyOnlyFilter)); +} + +#[test] +fn edge_query_uses_ready_edge_property_equality_index() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "eq-edge-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "eq-edge-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "eq-edge-c", &[], 1.0); + + let keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_82", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "EDGE_LABEL_82", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let info = engine + .ensure_edge_property_index("EDGE_LABEL_82", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_82".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + ..Default::default() + }; + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(!plan + .warnings + .contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert!(!plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + + engine.reset_query_execution_counters_for_test(); + let ids = engine.query_edge_ids(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(ids.edge_ids, vec![keep]); + assert_eq!(counters.edge_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_calls, 0); +} + +#[test] +fn edge_query_uses_ready_edge_property_range_index() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "range-edge-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "range-edge-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "range-edge-c", &[], 1.0); + let d = insert_query_node(&engine, "Person", "range-edge-d", &[], 1.0); + + engine + .upsert_edge( + a, + b, + "EDGE_LABEL_83", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(2))]), + ..Default::default() + }, + ) + .unwrap(); + let keep = engine + .upsert_edge( + a, + c, + "EDGE_LABEL_83", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(5))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + d, + "EDGE_LABEL_83", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(9))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let info = engine + .ensure_edge_property_index("EDGE_LABEL_83", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, }, ) .unwrap(); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - insert_query_node( - &engine, - 1, - "active", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(10)), - ], - 1.0, - ); - insert_query_node( - &engine, - 1, - "inactive", - &[ - ("status", PropValue::String("inactive".to_string())), - ("score", PropValue::Int(20)), - ], - 1.0, - ); + let query = EdgeQuery { + label: Some("EDGE_LABEL_83".to_string()), + filter: Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(4))), + upper: Some(PropertyRangeBound::Excluded(PropValue::Int(9))), + }), + ..Default::default() + }; + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyRangeIndex + )); + assert!(!plan + .warnings + .contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert!(!plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_eq!(engine.query_edge_ids(&query).unwrap().edge_ids, vec![keep]); +} - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let type_estimate = published.view.node_type_estimate(1).unwrap(); - assert_eq!(type_estimate.kind, PlannerEstimateKind::ExactCheap); - assert_eq!(type_estimate.known_upper_bound(), Some(2)); - let full_estimate = published.view.full_scan_estimate(); - assert_eq!(full_estimate.kind, PlannerEstimateKind::ExactCheap); - assert_eq!(full_estimate.known_upper_bound(), Some(2)); - let (equality_estimate, followup) = published - .view - .equality_candidate_estimate( - status.index_id, - "status", - &PropValue::String("active".to_string()), +#[test] +fn edge_property_range_index_query_paginates_by_edge_id_cursor() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "range-page-edge-a", &[], 1.0); + let targets = (0..5) + .map(|idx| insert_query_node(&engine, "Person", &format!("range-page-edge-{idx}"), &[], 1.0)) + .collect::>(); + + engine + .upsert_edge( + a, + targets[0], + "SPECIAL_EDGE_831", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(10))]), + ..Default::default() + }, ) .unwrap(); - assert!(followup.is_none()); - let equality_estimate = equality_estimate.unwrap(); - assert_eq!(equality_estimate.kind, PlannerEstimateKind::ExactCheap); - assert_eq!(equality_estimate.known_upper_bound(), Some(1)); + let first = engine + .upsert_edge( + a, + targets[1], + "SPECIAL_EDGE_831", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(80))]), + ..Default::default() + }, + ) + .unwrap(); + let second = engine + .upsert_edge( + a, + targets[2], + "SPECIAL_EDGE_831", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(90))]), + ..Default::default() + }, + ) + .unwrap(); + let third = engine + .upsert_edge( + a, + targets[3], + "SPECIAL_EDGE_831", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(100))]), + ..Default::default() + }, + ) + .unwrap(); + let deleted = engine + .upsert_edge( + a, + targets[4], + "SPECIAL_EDGE_831", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(110))]), + ..Default::default() + }, + ) + .unwrap(); + engine.delete_edge(deleted).unwrap(); + engine.flush().unwrap(); - let normalized = NormalizedNodeQuery { - type_id: Some(1), - ids: Vec::new(), - keys: Vec::new(), - filter: NormalizedNodeFilter::AlwaysTrue, - allow_full_scan: false, - page: PageRequest::default(), - }; - let cap_context = published.view.query_cap_context(&normalized).unwrap(); - let mut budget = BooleanPlanningBudget::new(); - let range_probe = published - .view - .range_candidate_probe( - &normalized, - cap_context, - 1, + let info = engine + .ensure_edge_property_index("SPECIAL_EDGE_831", "score", - Some(&PropertyRangeBound::Included(PropValue::Int(10))), - Some(&PropertyRangeBound::Included(PropValue::Int(10))), - &mut budget, + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, ) .unwrap(); - let range_estimate = range_probe.source.unwrap().estimate; - assert_eq!(range_estimate.kind, PlannerEstimateKind::ExactCheap); - assert_eq!(range_estimate.known_upper_bound(), Some(1)); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let mut budget = BooleanPlanningBudget::new(); - let timestamp_probe = published - .view - .timestamp_candidate_probe(&normalized, cap_context, 1, i64::MIN, i64::MAX, &mut budget) - .unwrap(); - let timestamp_estimate = timestamp_probe.source.unwrap().estimate; - assert_eq!(timestamp_estimate.kind, PlannerEstimateKind::ExactCheap); - assert_eq!(timestamp_estimate.known_upper_bound(), Some(2)); + let mut query = EdgeQuery { + label: Some("SPECIAL_EDGE_831".to_string()), + filter: Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(80))), + upper: None, + }), + page: PageRequest { + limit: Some(2), + after: None, + }, + ..Default::default() + }; + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyRangeIndex + )); - drop(published); - drop(_guard); - engine.close().unwrap(); + let first_page = engine.query_edge_ids(&query).unwrap(); + assert_eq!(first_page.edge_ids, vec![first, second]); + assert_eq!(first_page.next_cursor, Some(second)); + + query.page.after = first_page.next_cursor; + let second_page = engine.query_edge_ids(&query).unwrap(); + assert_eq!(second_page.edge_ids, vec![third]); + assert_eq!(second_page.next_cursor, None); } #[test] -fn test_planner_stats_equality_heavy_hitter_and_residual_explain_estimates() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_property_index_does_not_make_filter_only_edge_query_legal() { + let (_dir, engine) = query_test_engine(); let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_edge_property_index("EDGE_LABEL_84", "status", SecondaryIndexKind::Equality) .unwrap(); - wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - - let values: Vec = (0..40).map(|idx| format!("status-{idx:02}")).collect(); - for value in &values { - insert_query_node( - &engine, - 1, - value, - &[("status", PropValue::String(value.clone()))], - 1.0, - ); - } - engine.flush().unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let stats_view = engine.planner_stats_view_for_test(); - let rollup = stats_view.equality_index_rollups.get(&info.index_id).unwrap(); - assert_eq!(rollup.total_postings, 40); - assert_eq!( - rollup.top_value_hashes.len(), - crate::planner_stats::PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY + let query = EdgeQuery { + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + ..Default::default() + }; + let err = engine.query_edge_ids(&query).unwrap_err(); + assert!( + err.to_string().contains("edge query requires label"), + "unexpected error: {err}" ); - let top_value = values - .iter() - .find(|value| { - rollup - .top_value_hashes - .contains_key(&hash_prop_value(&PropValue::String((*value).clone()))) - }) - .unwrap() - .clone(); - let residual_value = values - .iter() - .find(|value| { - !rollup - .top_value_hashes - .contains_key(&hash_prop_value(&PropValue::String((*value).clone()))) - }) - .unwrap() - .clone(); - drop(stats_view); +} - let top_query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String(top_value), - }], - false, - ); - assert_eq!(engine.query_node_ids(&top_query).unwrap().items.len(), 1); - let top_plan = engine.explain_node_query(&top_query).unwrap(); - assert_eq!(top_plan.estimated_candidates, Some(1)); - assert_plan_input_nodes(&top_plan, vec![QueryPlanNode::PropertyEqualityIndex]); +#[test] +fn edge_query_missing_edge_property_index_remains_verifier_only() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "missing-edge-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "missing-edge-b", &[], 1.0); + let keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_85", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); - let residual_query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { + let query = EdgeQuery { + label: Some("EDGE_LABEL_85".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { key: "status".to_string(), - value: PropValue::String(residual_value), - }], - false, - ); - assert_eq!(engine.query_node_ids(&residual_query).unwrap().items.len(), 1); - let residual_plan = engine.explain_node_query(&residual_query).unwrap(); - assert_eq!(residual_plan.estimated_candidates, Some(1)); - assert_plan_input_nodes(&residual_plan, vec![QueryPlanNode::PropertyEqualityIndex]); - - engine.close().unwrap(); + value: PropValue::String("active".to_string()), + }), + ..Default::default() + }; + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(plan.warnings.contains(&QueryPlanWarning::MissingReadyIndex)); + assert!(plan + .warnings + .contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_eq!(engine.query_edge_ids(&query).unwrap().edge_ids, vec![keep]); } #[test] -fn test_planner_stats_rare_residual_equality_beats_broad_type_source() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) +fn edge_property_in_uses_index_union_and_preserves_signed_zero() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "edge-in-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-in-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "edge-in-c", &[], 1.0); + let d = insert_query_node(&engine, "Person", "edge-in-d", &[], 1.0); + + let positive_zero = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_86", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(0.0))]), + ..Default::default() + }, + ) .unwrap(); - wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - - let value_count = - QUERY_RANGE_CANDIDATE_CAP + crate::planner_stats::PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY + 1; - let values: Vec = (0..value_count) - .map(|idx| format!("rare-status-{idx:04}")) - .collect(); - for value in &values { - insert_query_node( - &engine, - 1, - value, - &[("status", PropValue::String(value.clone()))], - 1.0, - ); + let negative_zero = engine + .upsert_edge( + a, + c, + "EDGE_LABEL_86", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(-0.0))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + d, + "EDGE_LABEL_86", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(1.0))]), + ..Default::default() + }, + ) + .unwrap(); + for idx in 0..10 { + engine + .upsert_edge( + b, + d, + "EDGE_LABEL_86", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(idx as f64 + 2.0))]), + ..Default::default() + }, + ) + .unwrap(); } engine.flush().unwrap(); - let stats_view = engine.planner_stats_view_for_test(); - let rollup = stats_view.equality_index_rollups.get(&info.index_id).unwrap(); - let residual_value = values - .iter() - .find(|value| { - !rollup - .top_value_hashes - .contains_key(&hash_prop_value(&PropValue::String((*value).clone()))) - }) - .unwrap() - .clone(); - assert!(rollup.total_postings > QUERY_RANGE_CANDIDATE_CAP as u64); - drop(stats_view); - - let residual_query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String(residual_value), - }], - false, - ); - assert_eq!(engine.query_node_ids(&residual_query).unwrap().items.len(), 1); - let residual_plan = engine.explain_node_query(&residual_query).unwrap(); - assert_eq!(residual_plan.warnings, Vec::::new()); - assert_eq!(residual_plan.estimated_candidates, Some(1)); - assert_plan_input_nodes( - &residual_plan, - vec![QueryPlanNode::PropertyEqualityIndex], - ); - - engine.close().unwrap(); -} - -#[test] -fn test_planner_stats_broad_heavy_hitter_equality_uses_cheaper_type_scan() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_edge_property_index("EDGE_LABEL_86", "z", SecondaryIndexKind::Equality) .unwrap(); - wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - - let inputs: Vec<_> = (0..=QUERY_RANGE_CANDIDATE_CAP) - .map(|index| NodeInput { - type_id: 1, - key: format!("broad-heavy-{index}"), - props: query_test_props(&[("status", PropValue::String("broad".to_string()))]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }) - .collect(); - let all_ids = engine.batch_upsert_nodes(&inputs).unwrap(); - engine.flush().unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("broad".to_string()), - }], - false, - ); - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &all_ids, &query) - ); - let plan = engine.explain_node_query(&query).unwrap(); + let query = EdgeQuery { + label: Some("EDGE_LABEL_86".to_string()), + filter: Some(EdgeFilterExpr::PropertyIn { + key: "z".to_string(), + values: vec![ + PropValue::Float(-0.0), + PropValue::Float(0.0), + PropValue::Float(-0.0), + ], + }), + ..Default::default() + }; + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(!plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); assert_eq!( - plan.warnings, - vec![ - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::CandidateCapExceeded, - QueryPlanWarning::VerifyOnlyFilter, - ] + engine.query_edge_ids(&query).unwrap().edge_ids, + vec![positive_zero, negative_zero] ); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackTypeScan]); - - engine.close().unwrap(); } #[test] -fn test_planner_stats_range_and_timestamp_explain_use_no_planning_probe() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_property_range_requires_exact_declared_domain() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "edge-domain-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-domain-b", &[], 1.0); + let edge_id = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_87", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Float(5.0))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); - let score = engine - .ensure_node_property_index( - 1, + let info = engine + .ensure_edge_property_index("EDGE_LABEL_87", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, }, ) .unwrap(); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let inputs: Vec<_> = (0..32) - .map(|index| NodeInput { - type_id: 1, - key: format!("stats-probe-{index}"), - props: query_test_props(&[("score", PropValue::Int(index))]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }) - .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + let query = EdgeQuery { + label: Some("EDGE_LABEL_87".to_string()), + filter: Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Float(4.0))), + upper: Some(PropertyRangeBound::Included(PropValue::Float(6.0))), + }), + ..Default::default() + }; + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyRangeIndex + )); + assert!(plan.warnings.contains(&QueryPlanWarning::MissingReadyIndex)); + assert!(plan + .warnings + .contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert_eq!(engine.query_edge_ids(&query).unwrap().edge_ids, vec![edge_id]); +} + +#[test] +fn edge_property_or_with_verifier_branch_falls_back_whole_or() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "edge-or-index-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-or-index-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "edge-or-index-c", &[], 1.0); + let indexed = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_88", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let exists_only = engine + .upsert_edge( + a, + c, + "EDGE_LABEL_88", + UpsertEdgeOptions { + props: query_test_props(&[("tag", PropValue::String("present".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); engine.flush().unwrap(); - engine.reset_query_planning_probe_counters_for_test(); - let range_query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(12))), - }], - false, - ); - let range_plan = engine.explain_node_query(&range_query).unwrap(); - assert_plan_input_nodes(&range_plan, vec![QueryPlanNode::PropertyRangeIndex]); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_88", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_88".to_string()), + filter: Some(EdgeFilterExpr::Or(vec![ + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + EdgeFilterExpr::PropertyExists { + key: "tag".to_string(), + }, + ])), + ..Default::default() + }; + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(plan + .warnings + .contains(&QueryPlanWarning::BooleanBranchFallback)); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); assert_eq!( - engine.query_planning_probe_snapshot_for_test().range, - 0, - "stats-covered range explain must not materialize planning candidates" + engine.query_edge_ids(&query).unwrap().edge_ids, + vec![indexed, exists_only] ); - - let timestamp_query = query_ids( - Some(1), - vec![NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(i64::MIN), - upper_ms: Some(i64::MAX), - }], - false, - ); - let timestamp_plan = engine.explain_node_query(×tamp_query).unwrap(); - assert_plan_input_nodes(×tamp_plan, vec![QueryPlanNode::TimestampIndex]); - assert_eq!( - engine.query_planning_probe_snapshot_for_test().timestamp, - 0, - "stats-covered timestamp explain must not materialize planning candidates" - ); - - engine.close().unwrap(); -} +} #[test] -fn test_planner_stats_range_and_timestamp_mixed_coverage_probe_uncovered_segments() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let all_ids; - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, - }, - ) - .unwrap(); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - - let seg1 = [ - ("covered-a", 10, 1_000), - ("covered-b", 20, 1_100), - ("covered-c", 100, 9_000), - ]; - let mut ids = Vec::new(); - for (key, score, updated_at) in seg1 { - let node_id = insert_query_node( - &engine, - 1, - key, - &[("score", PropValue::Int(score))], - 1.0, - ); - set_query_node_updated_at(&engine, node_id, updated_at); - ids.push(node_id); - } - engine.flush().unwrap(); - - let seg2 = [ - ("uncovered-a", 15, 1_200), - ("uncovered-b", 25, 1_300), - ("uncovered-c", 200, 10_000), - ]; - for (key, score, updated_at) in seg2 { - let node_id = insert_query_node( - &engine, - 1, - key, - &[("score", PropValue::Int(score))], - 1.0, - ); - set_query_node_updated_at(&engine, node_id, updated_at); - ids.push(node_id); - } - engine.flush().unwrap(); - all_ids = ids; - engine.close().unwrap(); - } +fn edge_property_indexed_not_filters_use_bounded_positive_universe_only() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "edge-not-source", &[], 1.0); + let active_keep_node = insert_query_node(&engine, "Person", "edge-not-active-keep", &[], 1.0); + let active_drop_node = insert_query_node(&engine, "Person", "edge-not-active-drop", &[], 1.0); + let inactive_flagged_node = + insert_query_node(&engine, "Person", "edge-not-inactive-flagged", &[], 1.0); + let inactive_plain_node = + insert_query_node(&engine, "Person", "edge-not-inactive-plain", &[], 1.0); + + let active_keep = engine + .upsert_edge( + source, + active_keep_node, + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let active_drop = engine + .upsert_edge( + source, + active_drop_node, + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[ + ("status", PropValue::String("active".to_string())), + ("flag", PropValue::String("drop".to_string())), + ]), + ..Default::default() + }, + ) + .unwrap(); + let inactive_flagged = engine + .upsert_edge( + source, + inactive_flagged_node, + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[ + ("status", PropValue::String("inactive".to_string())), + ("flag", PropValue::String("keep".to_string())), + ]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + source, + inactive_plain_node, + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); - let stats_path = crate::segment_writer::segment_dir(&db_path, 2) - .join(crate::planner_stats::PLANNER_STATS_FILENAME); - std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_89", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let stats_view = reopened.planner_stats_view_for_test(); - assert_eq!(stats_view.available_segment_stats, 1); - assert_eq!(stats_view.unavailable_segment_stats, 1); - assert_eq!(stats_view.timestamp_coverage.covered_segment_ids, vec![1]); - let range_index_id = *stats_view.range_index_rollups.keys().next().unwrap(); + let and_not_query = EdgeQuery { + label: Some("EDGE_LABEL_89".to_string()), + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + EdgeFilterExpr::Not(Box::new(EdgeFilterExpr::PropertyEquals { + key: "flag".to_string(), + value: PropValue::String("drop".to_string()), + })), + ])), + ..Default::default() + }; assert_eq!( - stats_view - .range_index_rollups - .get(&range_index_id) - .unwrap() - .coverage - .covered_segment_ids, - vec![1] + engine.query_edge_ids(&and_not_query).unwrap().edge_ids, + vec![active_keep] ); - drop(stats_view); + let and_not_plan = engine.explain_edge_query(&and_not_query).unwrap(); + assert!(plan_contains_node( + &and_not_plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(and_not_plan + .warnings + .contains(&QueryPlanWarning::VerifyOnlyFilter)); - let range_query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(25))), - }], - false, - ); - reopened.reset_query_planning_probe_counters_for_test(); - let range_plan = reopened.explain_node_query(&range_query).unwrap(); - assert_eq!(range_plan.warnings, Vec::::new()); - assert_eq!(range_plan.estimated_candidates, Some(5)); - assert_plan_input_nodes(&range_plan, vec![QueryPlanNode::PropertyRangeIndex]); - assert_eq!(reopened.query_planning_probe_snapshot_for_test().range, 1); + let or_not_query = EdgeQuery { + label: Some("EDGE_LABEL_89".to_string()), + filter: Some(EdgeFilterExpr::Or(vec![ + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + EdgeFilterExpr::Not(Box::new(EdgeFilterExpr::PropertyMissing { + key: "flag".to_string(), + })), + ])), + ..Default::default() + }; assert_eq!( - reopened.query_node_ids(&range_query).unwrap().items, - oracle_query_ids(&reopened, &all_ids, &range_query) + engine.query_edge_ids(&or_not_query).unwrap().edge_ids, + vec![active_keep, active_drop, inactive_flagged] ); + let or_not_plan = engine.explain_edge_query(&or_not_query).unwrap(); + assert!(!plan_contains_node( + &or_not_plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(plan_contains_node( + &or_not_plan.root, + &QueryPlanNode::EdgeLabelIndex + )); + assert!(or_not_plan + .warnings + .contains(&QueryPlanWarning::BooleanBranchFallback)); - let timestamp_query = query_ids( - Some(1), - vec![NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(1_000), - upper_ms: Some(1_300), - }], - false, - ); - reopened.reset_query_planning_probe_counters_for_test(); - let timestamp_plan = reopened.explain_node_query(×tamp_query).unwrap(); - assert_eq!(timestamp_plan.warnings, Vec::::new()); - assert_eq!(timestamp_plan.estimated_candidates, Some(5)); - assert_plan_input_nodes(×tamp_plan, vec![QueryPlanNode::TimestampIndex]); - assert_eq!( - reopened.query_planning_probe_snapshot_for_test().timestamp, - 1 - ); + let endpoint_not_query = EdgeQuery { + from_ids: vec![source], + filter: Some(EdgeFilterExpr::Not(Box::new(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("inactive".to_string()), + }))), + ..Default::default() + }; assert_eq!( - reopened.query_node_ids(×tamp_query).unwrap().items, - oracle_query_ids(&reopened, &all_ids, ×tamp_query) + engine.query_edge_ids(&endpoint_not_query).unwrap().edge_ids, + vec![active_keep, active_drop] ); - - reopened.close().unwrap(); + let endpoint_not_plan = engine.explain_edge_query(&endpoint_not_query).unwrap(); + assert!(!plan_contains_node( + &endpoint_not_plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(plan_contains_node( + &endpoint_not_plan.root, + &QueryPlanNode::EdgeEndpointAdjacency + )); } #[test] -fn test_planner_stats_adaptive_cap_allows_high_confidence_range_above_default() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_property_index_visibility_merges_active_frozen_and_segments() { + let (_dir, engine) = query_test_engine(); + let nodes = (0..8) + .map(|idx| insert_query_node(&engine, "Person", &format!("edge-vis-{idx}"), &[], 1.0)) + .collect::>(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_89", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, + let shadowed = engine + .upsert_edge( + nodes[0], + nodes[1], + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let deleted = engine + .upsert_edge( + nodes[0], + nodes[2], + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let segment_keep = engine + .upsert_edge( + nodes[0], + nodes[3], + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() }, ) .unwrap(); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - - let selected_count = - crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP + 256; - let total_count = selected_count + 1024; - let inputs: Vec<_> = (0..total_count) - .map(|index| NodeInput { - type_id: 1, - key: format!("adaptive-range-{index}"), - props: query_test_props(&[("score", PropValue::Int(index as i64))]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }) - .collect(); - let all_ids = engine.batch_upsert_nodes(&inputs).unwrap(); engine.flush().unwrap(); - let query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), - upper: Some(PropertyRangeBound::Included(PropValue::Int( - selected_count as i64 - 1, - ))), + set_query_edge_props( + &engine, + shadowed, + query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + ); + engine.delete_edge(deleted).unwrap(); + let frozen_keep = engine + .upsert_edge( + nodes[0], + nodes[4], + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.freeze_memtable().unwrap(); + let active_keep = engine + .upsert_edge( + nodes[0], + nodes[5], + "EDGE_LABEL_89", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_89".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), }), - page: PageRequest { - limit: Some(16), - after: None, - }, ..Default::default() }; - - let expected: Vec<_> = oracle_query_ids(&engine, &all_ids, &query) - .into_iter() - .take(16) - .collect(); - assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyRangeIndex]); - assert!( - plan.estimated_candidates - > Some(crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP as u64) + assert_eq!( + engine.query_edge_ids(&query).unwrap().edge_ids, + vec![segment_keep, frozen_keep, active_keep] ); - - engine.close().unwrap(); } #[test] -fn test_direct_read_apis_are_unchanged_with_planner_stats_sidecars() { +fn edge_property_equality_verifier_filters_stale_or_colliding_postings() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, - }, - ) - .unwrap(); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - wait_for_published_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + let index_id; + let segment_id; + let red_one; + let red_two; + let blue; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let nodes = (0..4) + .map(|idx| insert_query_node(&engine, "Person", &format!("edge-collision-{idx}"), &[], 1.0)) + .collect::>(); + red_one = engine + .upsert_edge( + nodes[0], + nodes[1], + "EDGE_LABEL_90", + UpsertEdgeOptions { + props: query_test_props(&[("color", PropValue::String("red".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + red_two = engine + .upsert_edge( + nodes[0], + nodes[2], + "EDGE_LABEL_90", + UpsertEdgeOptions { + props: query_test_props(&[("color", PropValue::String("red".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + blue = engine + .upsert_edge( + nodes[0], + nodes[3], + "EDGE_LABEL_90", + UpsertEdgeOptions { + props: query_test_props(&[("color", PropValue::String("blue".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); - let inputs = vec![ - NodeInput { - type_id: 1, - key: "direct-a".to_string(), - props: query_test_props(&[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(10)), - ]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }, - NodeInput { - type_id: 1, - key: "direct-b".to_string(), - props: query_test_props(&[ - ("status", PropValue::String("inactive".to_string())), - ("score", PropValue::Int(20)), - ]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }, - NodeInput { - type_id: 2, - key: "direct-c".to_string(), - props: query_test_props(&[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(10)), - ]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }, - ]; - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); - engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_90", "color", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = engine.segments_for_test()[0].segment_id; + engine.close().unwrap(); + } - assert_eq!( - engine - .find_nodes(1, "status", &PropValue::String("active".to_string())) - .unwrap(), - vec![ids[0]] - ); - assert_eq!( - engine - .find_nodes_range( - 1, - "score", - Some(&PropertyRangeBound::Included(PropValue::Int(10))), - Some(&PropertyRangeBound::Included(PropValue::Int(20))), - ) - .unwrap(), - vec![ids[0], ids[1]] + let sidecar_path = crate::segment_writer::edge_prop_eq_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, segment_id), + index_id, ); - assert_eq!( - engine - .find_nodes_by_time_range(1, i64::MIN, i64::MAX) - .unwrap(), - vec![ids[0], ids[1]] + replace_equality_sidecar_group_id_in_place( + &sidecar_path, + hash_prop_value(&PropValue::String("red".to_string())), + red_two, + blue, ); - assert_eq!(engine.nodes_by_type(1).unwrap(), vec![ids[0], ids[1]]); - engine.close().unwrap(); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = EdgeQuery { + label: Some("EDGE_LABEL_90".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "color".to_string(), + value: PropValue::String("red".to_string()), + }), + ..Default::default() + }; + let plan = reopened.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert_eq!(reopened.query_edge_ids(&query).unwrap().edge_ids, vec![red_one]); } #[test] -fn test_planner_stats_mixed_segment_fallback_estimates_once() { +fn edge_property_query_falls_back_and_marks_corrupt_equality_sidecar_failed() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let index_id; + let segment_id; + let keep; { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - for idx in 0..2 { - insert_query_node(&engine, 1, &format!("covered-{idx}"), &[], 1.0); - } - engine.flush().unwrap(); - for idx in 0..3 { - insert_query_node(&engine, 1, &format!("fallback-{idx}"), &[], 1.0); - } + let a = insert_query_node(&engine, "Person", "edge-corrupt-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-corrupt-b", &[], 1.0); + keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_91", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); engine.flush().unwrap(); + + let info = engine + .ensure_edge_property_index("EDGE_LABEL_91", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = engine.segments_for_test()[0].segment_id; engine.close().unwrap(); } - let stats_path = crate::segment_writer::segment_dir(&db_path, 2) - .join(crate::planner_stats::PLANNER_STATS_FILENAME); - std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); + let sidecar_path = crate::segment_writer::edge_prop_eq_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, segment_id), + index_id, + ); + corrupt_sidecar_header_in_place(&sidecar_path); let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let stats_view = reopened.planner_stats_view_for_test(); - assert_eq!(stats_view.segment_count, 2); - assert_eq!(stats_view.available_segment_stats, 1); - assert_eq!(stats_view.unavailable_segment_stats, 1); - assert_eq!(stats_view.type_node_count(1), 2); - assert_eq!(stats_view.type_coverage.covered_segment_ids, vec![1]); - drop(stats_view); - { - let (_guard, published) = reopened.runtime.published_snapshot().unwrap(); - let estimate = published.view.node_type_estimate(1).unwrap(); - assert_eq!(estimate.kind, PlannerEstimateKind::UpperBound); - assert_eq!(estimate.known_upper_bound(), Some(5)); - } - - let query = query_ids(Some(1), Vec::new(), false); - assert_eq!(reopened.query_node_ids(&query).unwrap().items.len(), 5); - let plan = reopened.explain_node_query(&query).unwrap(); - assert_eq!(plan.estimated_candidates, Some(5)); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::NodeTypeIndex]); - - reopened.close().unwrap(); + let query = EdgeQuery { + label: Some("EDGE_LABEL_91".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + ..Default::default() + }; + assert_eq!(reopened.query_edge_ids(&query).unwrap().edge_ids, vec![keep]); + wait_for_edge_property_index_state(&reopened, index_id, SecondaryIndexState::Failed); } #[test] -fn test_planner_estimate_sort_prefers_cheaper_count_before_source_rank() { +fn edge_property_query_enqueues_planning_followup_for_corrupt_equality_sidecar() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let index_id; + let segment_id; + let keep; { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let mut candidates = vec![ - NodePhysicalPlan::source(PlannedNodeCandidateSource::property_equality_index( - 1, - 1, - "status", - &PropValue::String("active".to_string()), - PlannerEstimate::stats_estimated( - 100, - EstimateConfidence::High, - StalePostingRisk::Low, - ), - )), - NodePhysicalPlan::source(PlannedNodeCandidateSource::fallback_type_scan( - 1, - PlannerEstimate::upper_bound(10), - )), - ]; - published - .view - .sort_physical_plans_by_selectivity(&mut candidates); - assert_eq!(candidates[0].plan_node(), QueryPlanNode::FallbackTypeScan); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let a = insert_query_node(&engine, "Person", "edge-plan-eq-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-plan-eq-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "edge-plan-eq-c", &[], 1.0); + keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_94", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "EDGE_LABEL_94", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let info = engine + .ensure_edge_property_index("EDGE_LABEL_94", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = engine.segments_for_test()[0].segment_id; + engine.close().unwrap(); } - engine.close().unwrap(); + let sidecar_path = crate::segment_writer::edge_prop_eq_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, segment_id), + index_id, + ); + corrupt_planner_stats_for_segment(&db_path, segment_id); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + wait_for_edge_property_index_state(&reopened, index_id, SecondaryIndexState::Ready); + corrupt_sidecar_header_in_place(&sidecar_path); + let query = EdgeQuery { + label: Some("EDGE_LABEL_94".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + ..Default::default() + }; + let plan = reopened.explain_edge_query(&query).unwrap(); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgeLabelIndex + )); + + let (followup_ready_rx, followup_release_tx) = reopened.set_runtime_publish_pause(); + assert_eq!(reopened.query_edge_ids(&query).unwrap().edge_ids, vec![keep]); + followup_ready_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert_eq!(reopened.pending_secondary_index_followup_count_for_test(), 1); + followup_release_tx.send(()).unwrap(); + wait_for_pending_secondary_index_followup_count(&reopened, 0); } #[test] -fn test_query_validation_and_explain_reject_type_less_scan_without_opt_in() { +fn edge_property_query_enqueues_planning_followup_for_corrupt_range_sidecar() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let index_id; + let segment_id; + let keep; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let a = insert_query_node(&engine, "Person", "edge-corrupt-range-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-corrupt-range-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "edge-corrupt-range-c", &[], 1.0); + keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_95", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(7))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "EDGE_LABEL_95", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(20))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); - let query = query_ids( - None, - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - false, + let info = engine + .ensure_edge_property_index("EDGE_LABEL_95", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = engine.segments_for_test()[0].segment_id; + engine.close().unwrap(); + } + + let sidecar_path = crate::segment_writer::edge_prop_range_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, segment_id), + index_id, ); + corrupt_planner_stats_for_segment(&db_path, segment_id); - assert!(matches!( - engine.query_node_ids(&query).unwrap_err(), - EngineError::InvalidOperation(_) + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + wait_for_edge_property_index_state(&reopened, index_id, SecondaryIndexState::Ready); + corrupt_sidecar_header_in_place(&sidecar_path); + let query = EdgeQuery { + label: Some("EDGE_LABEL_95".to_string()), + filter: Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(10))), + }), + ..Default::default() + }; + let plan = reopened.explain_edge_query(&query).unwrap(); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyRangeIndex )); - assert!(matches!( - engine.explain_node_query(&query).unwrap_err(), - EngineError::InvalidOperation(_) + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgeLabelIndex )); - let key_query = NodeQuery { - keys: vec!["alice".to_string()], - ..Default::default() - }; - assert!(matches!( - engine.query_node_ids(&key_query).unwrap_err(), - EngineError::InvalidOperation(_) - )); + let (followup_ready_rx, followup_release_tx) = reopened.set_runtime_publish_pause(); + assert_eq!(reopened.query_edge_ids(&query).unwrap().edge_ids, vec![keep]); + followup_ready_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert_eq!(reopened.pending_secondary_index_followup_count_for_test(), 1); + followup_release_tx.send(()).unwrap(); + wait_for_pending_secondary_index_followup_count(&reopened, 0); +} - let empty_range_query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: None, - upper: None, - }], - false, - ); - assert!(matches!( - engine.query_node_ids(&empty_range_query).unwrap_err(), - EngineError::InvalidOperation(_) - )); +#[test] +fn edge_property_query_falls_back_and_marks_corrupt_range_sidecar_failed() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let index_id; + let segment_id; + let keep; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let a = insert_query_node(&engine, "Person", "edge-corrupt-range-failed-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-corrupt-range-failed-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "edge-corrupt-range-failed-c", &[], 1.0); + keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_96", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(7))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "EDGE_LABEL_96", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(20))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); - let empty_time_query = query_ids( - Some(1), - vec![NodeFilterExpr::UpdatedAtRange { - lower_ms: None, - upper_ms: None, - }], - false, - ); - assert!(matches!( - engine.explain_node_query(&empty_time_query).unwrap_err(), - EngineError::InvalidOperation(_) - )); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_96", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = engine.segments_for_test()[0].segment_id; + engine.close().unwrap(); + } - let inverted_time_query = query_ids( - Some(1), - vec![NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(200), - upper_ms: Some(100), - }], - false, + let sidecar_path = crate::segment_writer::edge_prop_range_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, segment_id), + index_id, ); - assert!(engine - .query_node_ids(&inverted_time_query) - .unwrap() - .items - .is_empty()); - assert!(matches!( - explain_input_node(&engine.explain_node_query(&inverted_time_query).unwrap()), - QueryPlanNode::EmptyResult - )); + corrupt_sidecar_header_in_place(&sidecar_path); - engine.close().unwrap(); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = EdgeQuery { + label: Some("EDGE_LABEL_96".to_string()), + filter: Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(10))), + }), + ..Default::default() + }; + assert_eq!(reopened.query_edge_ids(&query).unwrap().edge_ids, vec![keep]); + wait_for_edge_property_index_state(&reopened, index_id, SecondaryIndexState::Failed); + + let failed_plan = reopened.explain_edge_query(&query).unwrap(); + assert!(!plan_contains_node( + &failed_plan.root, + &QueryPlanNode::EdgePropertyRangeIndex + )); } #[test] -fn test_query_normalization_expands_open_updated_at_bounds() { +fn edge_property_query_uses_sidecar_counts_when_planner_stats_are_missing() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - + let keep; { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - - let lower_open_query = query_ids( - Some(1), - vec![NodeFilterExpr::UpdatedAtRange { - lower_ms: None, - upper_ms: Some(123), - }], - false, - ); - let normalized = published - .view - .normalize_node_query(&lower_open_query) + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let a = insert_query_node(&engine, "Person", "edge-no-stats-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "edge-no-stats-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "edge-no-stats-c", &[], 1.0); + keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_92", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - match normalized.filter { - NormalizedNodeFilter::UpdatedAtRange { lower_ms, upper_ms } => { - assert_eq!(lower_ms, i64::MIN); - assert_eq!(upper_ms, 123); - } - _ => panic!("expected normalized updated-at range"), - } + engine + .upsert_edge( + a, + c, + "EDGE_LABEL_92", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); - let upper_open_query = query_ids( - Some(1), - vec![NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(456), - upper_ms: None, - }], - false, - ); - let normalized = published - .view - .normalize_node_query(&upper_open_query) + let info = engine + .ensure_edge_property_index("EDGE_LABEL_92", "status", SecondaryIndexKind::Equality) .unwrap(); - match normalized.filter { - NormalizedNodeFilter::UpdatedAtRange { lower_ms, upper_ms } => { - assert_eq!(lower_ms, 456); - assert_eq!(upper_ms, i64::MAX); - } - _ => panic!("expected normalized updated-at range"), - } + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + engine.close().unwrap(); } + let stats_path = crate::segment_writer::segment_dir(&db_path, 1) + .join(crate::planner_stats::PLANNER_STATS_FILENAME); + std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); - engine.close().unwrap(); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = EdgeQuery { + label: Some("EDGE_LABEL_92".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + ..Default::default() + }; + let plan = reopened.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert_eq!(reopened.query_edge_ids(&query).unwrap().edge_ids, vec![keep]); } #[test] -fn test_query_filter_validation_and_empty_result_without_scan_opt_in() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_property_in_union_materializes_when_union_cap_allows_it() { + let (_dir, engine) = query_test_engine(); + let matching_count = crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP + 904; + let nonmatching_count = 2_000usize; + let total_edges = matching_count + nonmatching_count; + let nodes = (0..=total_edges) + .map(|idx| NodeInput { + labels: vec!["Person".to_string()], + key: format!("edge-union-cap-node-{idx}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let edge_inputs = node_ids[1..] + .iter() + .enumerate() + .map(|(idx, to)| EdgeInput { + from: hub, + to: *to, + label: "EDGE_LABEL_93".to_string(), + props: query_test_props(&[( + "bucket", + PropValue::String( + if idx < matching_count { + if idx % 2 == 0 { "a" } else { "b" } + } else { + "c" + } + .to_string(), + ), + )]), + weight: 1.0, + valid_from: None, + valid_to: None, + }) + .collect::>(); + let edge_ids = engine.batch_upsert_edges(edge_inputs).unwrap(); + engine.flush().unwrap(); - for filter in [ - NodeFilterExpr::And(Vec::new()), - NodeFilterExpr::Or(Vec::new()), - NodeFilterExpr::PropertyEquals { - key: String::new(), - value: PropValue::String("x".to_string()), - }, - NodeFilterExpr::PropertyIn { - key: "status".to_string(), - values: Vec::new(), - }, - ] { - let query = NodeQuery { - type_id: Some(1), - filter: Some(filter), - ..Default::default() - }; - assert!(matches!( - engine.explain_node_query(&query).unwrap_err(), - EngineError::InvalidOperation(_) - )); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_93", "bucket", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_93".to_string()), + filter: Some(EdgeFilterExpr::PropertyIn { + key: "bucket".to_string(), + values: vec![ + PropValue::String("a".to_string()), + PropValue::String("b".to_string()), + ], + }), + ..Default::default() + }; + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let normalized = published.view.normalize_edge_query(&query).unwrap(); + let planned = published.view.plan_normalized_edge_query(&normalized).unwrap(); + match published + .view + .materialize_edge_physical_plan(&normalized, planned.cap_context, &planned.driver) + .unwrap() + { + CandidateMaterializationResult::Ready { ids, .. } => { + assert_eq!(ids, edge_ids[..matching_count]); + } + CandidateMaterializationResult::TooBroad { .. } => { + panic!("edge property IN union should materialize under the union cap") + } } +} - let always_false = NodeQuery { - filter: filter_and![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), +#[test] +fn edge_query_endpoint_visibility_does_not_hydrate_nodes() { + let (_dir, engine) = query_test_engine(); + let seg_a = insert_query_node(&engine, "Person", "endpoint-segment-a", &[], 1.0); + let seg_b = insert_query_node(&engine, "Person", "endpoint-segment-b", &[], 1.0); + let segment_edge = engine + .upsert_edge( + seg_a, + seg_b, + "EDGE_LABEL_42", + UpsertEdgeOptions { + valid_from: Some(0), + valid_to: Some(100), + ..Default::default() }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("inactive".to_string()), + ) + .unwrap(); + engine.flush().unwrap(); + + let frozen_a = insert_query_node(&engine, "Person", "endpoint-frozen-a", &[], 1.0); + let frozen_b = insert_query_node(&engine, "Person", "endpoint-frozen-b", &[], 1.0); + let frozen_edge = engine + .upsert_edge( + frozen_a, + frozen_b, + "EDGE_LABEL_42", + UpsertEdgeOptions { + valid_from: Some(0), + valid_to: Some(100), + ..Default::default() }, - ], - ..Default::default() - }; - assert!(engine.query_node_ids(&always_false).unwrap().items.is_empty()); - let plan = engine.explain_node_query(&always_false).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert!(matches!(explain_input_node(&plan), QueryPlanNode::EmptyResult)); + ) + .unwrap(); + engine.freeze_memtable().unwrap(); - let always_true_without_anchor = NodeQuery { - filter: Some(NodeFilterExpr::Not(Box::new( - always_false.filter.clone().unwrap(), - ))), + let active_a = insert_query_node(&engine, "Person", "endpoint-active-a", &[], 1.0); + let active_b = insert_query_node(&engine, "Person", "endpoint-active-b", &[], 1.0); + let active_edge = engine + .upsert_edge( + active_a, + active_b, + "EDGE_LABEL_42", + UpsertEdgeOptions { + valid_from: Some(0), + valid_to: Some(100), + ..Default::default() + }, + ) + .unwrap(); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_42".to_string()), + filter: Some(EdgeFilterExpr::ValidAt { epoch_ms: 50 }), ..Default::default() }; - assert!(matches!( - engine.query_node_ids(&always_true_without_anchor).unwrap_err(), - EngineError::InvalidOperation(_) - )); - engine.close().unwrap(); + engine.reset_query_execution_counters_for_test(); + let ids = engine.query_edge_ids(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + + assert_eq!(ids.edge_ids, vec![segment_edge, frozen_edge, active_edge]); + assert_eq!(counters.node_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_reads, 0); } #[test] -fn test_query_filter_exists_missing_not_and_or_verifier_semantics() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let tagged_null = insert_query_node( - &engine, - 1, - "tagged-null", - &[ - ("status", PropValue::String("active".to_string())), - ("tag", PropValue::Null), - ], - 1.0, - ); - let missing_tag = insert_query_node( - &engine, - 1, - "missing-tag", - &[("status", PropValue::String("inactive".to_string()))], - 1.0, - ); - let tagged_trial = insert_query_node( - &engine, - 1, - "tagged-trial", - &[ - ("status", PropValue::String("trial".to_string())), - ("tag", PropValue::String("present".to_string())), - ], - 1.0, - ); +fn edge_query_prune_endpoint_visibility_uses_metadata_only() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "prune-source", &[], 1.0); + let hidden = insert_query_node(&engine, "Person", "prune-hidden", &[], 0.1); + let visible = insert_query_node(&engine, "Person", "prune-visible", &[], 1.0); + engine + .upsert_edge(source, hidden, "EDGE_LABEL_43", UpsertEdgeOptions::default()) + .unwrap(); + let keep = engine + .upsert_edge(source, visible, "EDGE_LABEL_43", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + engine + .set_prune_policy( + "hide-light-endpoints", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: None, + }, + ) + .unwrap(); - let query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::And(vec![ - NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("trial".to_string()), + engine.reset_query_execution_counters_for_test(); + let ids = engine + .query_edge_ids(&EdgeQuery { + label: Some("EDGE_LABEL_43".to_string()), + from_ids: vec![source], + ..Default::default() + }) + .unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + + assert_eq!(ids.edge_ids, vec![keep]); + assert_eq!(counters.node_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_reads, 0); +} + +#[test] +fn edge_query_property_hydrates_only_metadata_survivors() { + let (_dir, engine) = query_test_engine(); + let nodes = (0..41) + .map(|idx| NodeInput { + labels: vec!["Person".to_string()], + key: format!("property-prefilter-node-{idx}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let mut expected = None; + for (idx, to) in node_ids[1..].iter().enumerate() { + let selective = idx == 3 || idx == 17 || idx == 29; + let props = if idx == 17 { + query_test_props(&[("status", PropValue::String("active".to_string()))]) + } else { + query_test_props(&[("status", PropValue::String("inactive".to_string()))]) + }; + let edge_id = engine + .upsert_edge( + hub, + *to, + "EDGE_LABEL_44", + UpsertEdgeOptions { + props, + weight: if selective { 9.0 } else { 1.0 }, + ..Default::default() }, - ]), - NodeFilterExpr::PropertyExists { - key: "tag".to_string(), + ) + .unwrap(); + if idx == 17 { + expected = Some(edge_id); + } + } + engine.flush().unwrap(); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_44".to_string()), + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::WeightRange { + lower: Some(9.0), + upper: Some(9.0), + }, + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), }, - NodeFilterExpr::Not(Box::new(NodeFilterExpr::PropertyMissing { - key: "tag".to_string(), - })), ])), ..Default::default() }; - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - vec![tagged_null, tagged_trial] - ); - let plan = engine.explain_node_query(&query).unwrap(); - assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackTypeScan]); - let missing_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyMissing { - key: "tag".to_string(), - }), - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&missing_query).unwrap().items, vec![missing_tag]); + engine.reset_query_execution_counters_for_test(); + let ids = engine.query_edge_ids(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); - engine.close().unwrap(); + assert_eq!(ids.edge_ids, vec![expected.unwrap()]); + assert_eq!(counters.edge_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_calls, 0); } #[test] -fn test_query_filter_in_dedupes_by_canonical_value_and_uses_union() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_query_or_filter_uses_projected_properties_without_hydration() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "edge-or-source", &[], 1.0); + let metadata_match = insert_query_node(&engine, "Person", "edge-or-metadata", &[], 1.0); + let property_match = insert_query_node(&engine, "Person", "edge-or-property", &[], 1.0); + let drop = insert_query_node(&engine, "Person", "edge-or-drop", &[], 1.0); + + let property_edge = engine + .upsert_edge( + source, + property_match, + "EDGE_LABEL_45", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + weight: 2.0, + ..Default::default() + }, + ) + .unwrap(); + let metadata_edge = engine + .upsert_edge( + source, + metadata_match, + "EDGE_LABEL_45", + UpsertEdgeOptions { + weight: 0.5, + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + source, + drop, + "EDGE_LABEL_45", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + weight: 2.0, + ..Default::default() + }, + ) + .unwrap(); - let mut map_value = BTreeMap::new(); - map_value.insert("x".to_string(), PropValue::Int(1)); - let map_value = PropValue::Map(map_value); - let array_value = PropValue::Array(vec![PropValue::Int(1), PropValue::UInt(2)]); + let query = EdgeQuery { + label: Some("EDGE_LABEL_45".to_string()), + filter: Some(EdgeFilterExpr::Or(vec![ + EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }, + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + ])), + ..Default::default() + }; - let null_id = insert_query_node(&engine, 1, "null", &[("kind", PropValue::Null)], 1.0); - let int_id = insert_query_node(&engine, 1, "int", &[("kind", PropValue::Int(1))], 1.0); - let uint_id = insert_query_node(&engine, 1, "uint", &[("kind", PropValue::UInt(1))], 1.0); - let array_id = insert_query_node(&engine, 1, "array", &[("kind", array_value.clone())], 1.0); - let map_id = insert_query_node(&engine, 1, "map", &[("kind", map_value.clone())], 1.0); - let neg_zero_id = insert_query_node( - &engine, - 1, - "neg-zero-kind", - &[("kind", PropValue::Float(-0.0))], - 1.0, - ); - let pos_zero_id = insert_query_node( - &engine, - 1, - "pos-zero-kind", - &[("kind", PropValue::Float(0.0))], - 1.0, - ); - let _missing = insert_query_node(&engine, 1, "missing", &[], 1.0); + engine.reset_query_execution_counters_for_test(); + let result = engine.query_edge_ids(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); - let index = engine - .ensure_node_property_index(1, "kind", SecondaryIndexKind::Equality) + assert_eq!(result.edge_ids, vec![property_edge, metadata_edge]); + assert_eq!(counters.edge_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_calls, 0); +} + +#[test] +fn edge_query_edges_hydrates_only_final_property_filtered_page() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "edge-output-cache-source", &[], 1.0); + let keep = insert_query_node(&engine, "Person", "edge-output-cache-keep", &[], 1.0); + let drop = insert_query_node(&engine, "Person", "edge-output-cache-drop", &[], 1.0); + let keep_edge = engine + .upsert_edge( + source, + keep, + "EDGE_LABEL_46", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + source, + drop, + "EDGE_LABEL_46", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); - let query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyIn { - key: "kind".to_string(), - values: vec![ - PropValue::Null, - PropValue::Null, - PropValue::UInt(1), - array_value.clone(), - map_value.clone(), - map_value.clone(), - ], + let query = EdgeQuery { + label: Some("EDGE_LABEL_46".to_string()), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), }), ..Default::default() }; + + engine.reset_query_execution_counters_for_test(); + let result = engine.query_edges(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!( - engine.query_node_ids(&query).unwrap().items, - vec![null_id, uint_id, array_id, map_id] - ); - let plan = engine.explain_node_query(&query).unwrap(); - assert!(!plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes( - &plan, - vec![QueryPlanNode::Union { - inputs: vec![ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], - }], + result.edges.iter().map(|edge| edge.id).collect::>(), + vec![keep_edge] ); + assert_eq!(counters.edge_record_hydration_reads, 1); + assert_eq!(counters.edge_record_hydration_calls, 1); +} - let int_only = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyIn { - key: "kind".to_string(), - values: vec![PropValue::Int(1)], - }), - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&int_only).unwrap().items, vec![int_id]); - let int_only_plan = engine.explain_node_query(&int_only).unwrap(); - assert!(!int_only_plan - .warnings - .contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes(&int_only_plan, vec![QueryPlanNode::PropertyEqualityIndex]); +#[test] +fn edge_query_excludes_edges_cascaded_by_deleted_endpoint() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let deleted = insert_query_node(&engine, "Person", "deleted", &[], 1.0); + let visible = insert_query_node(&engine, "Person", "visible", &[], 1.0); - let signed_zero_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyIn { - key: "kind".to_string(), - values: vec![PropValue::Float(-0.0), PropValue::Float(0.0)], - }), + engine + .upsert_edge(a, deleted, "PUBLISHED_BY", UpsertEdgeOptions::default()) + .unwrap(); + let keep = engine + .upsert_edge(a, visible, "PUBLISHED_BY", UpsertEdgeOptions::default()) + .unwrap(); + engine.delete_node(deleted).unwrap(); + + let ids = engine + .query_edge_ids(&EdgeQuery { + label: Some("PUBLISHED_BY".to_string()), + from_ids: vec![a], + ..Default::default() + }) + .unwrap(); + assert_eq!(ids.edge_ids, vec![keep]); +} + +#[test] +fn edge_query_excludes_edges_with_prune_hidden_endpoint() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 0.9); + let hidden = insert_query_node(&engine, "Person", "hidden", &[], 0.2); + let visible = insert_query_node(&engine, "Person", "visible", &[], 0.8); + + engine + .upsert_edge(a, hidden, "TAGGED_WITH", UpsertEdgeOptions::default()) + .unwrap(); + let keep = engine + .upsert_edge(a, visible, "TAGGED_WITH", UpsertEdgeOptions::default()) + .unwrap(); + engine + .set_prune_policy( + "low-weight", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: None, + }, + ) + .unwrap(); + + let query = EdgeQuery { + label: Some("TAGGED_WITH".to_string()), + from_ids: vec![a], ..Default::default() }; + let ids = engine.query_edge_ids(&query).unwrap(); + assert_eq!(ids.edge_ids, vec![keep]); + + let edges = engine.query_edges(&query).unwrap(); assert_eq!( - engine.query_node_ids(&signed_zero_query).unwrap().items, - vec![neg_zero_id, pos_zero_id] - ); - assert_plan_input_nodes( - &engine.explain_node_query(&signed_zero_query).unwrap(), - vec![QueryPlanNode::Union { - inputs: vec![ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], - }], + edges.edges.iter().map(|edge| edge.id).collect::>(), + vec![keep] ); - - engine.close().unwrap(); } #[test] -fn test_query_filter_large_verify_only_in_matches_verifier_semantics() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_query_valid_at_uses_half_open_validity_window() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "c", &[], 1.0); - let string_match = insert_query_node( - &engine, - 1, - "token-string-match", - &[("token", PropValue::String("value-63".to_string()))], - 1.0, - ); - let signed_zero_match = insert_query_node( - &engine, - 1, - "token-signed-zero-match", - &[("token", PropValue::Float(0.0))], - 1.0, - ); - let nested_zero_match = insert_query_node( - &engine, - 1, - "token-nested-zero-match", - &[( - "token", - PropValue::Array(vec![PropValue::Float(0.0)]), - )], - 1.0, - ); - insert_query_node( - &engine, - 1, - "token-nan-not-match", - &[("token", PropValue::Float(f64::NAN))], - 1.0, - ); - insert_query_node( - &engine, - 1, - "token-miss", - &[("token", PropValue::String("missing".to_string()))], - 1.0, - ); + engine + .upsert_edge( + a, + b, + "ASSIGNED_TO", + UpsertEdgeOptions { + valid_from: Some(0), + valid_to: Some(100), + ..Default::default() + }, + ) + .unwrap(); + let live = engine + .upsert_edge( + a, + c, + "ASSIGNED_TO", + UpsertEdgeOptions { + valid_from: Some(0), + valid_to: Some(101), + ..Default::default() + }, + ) + .unwrap(); - let mut values: Vec = (0..64) - .map(|index| PropValue::String(format!("value-{index}"))) - .collect(); - values.push(PropValue::Float(-0.0)); - values.push(PropValue::Array(vec![PropValue::Float(-0.0)])); - values.push(PropValue::Float(f64::NAN)); - let query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyIn { - key: "token".to_string(), - values, - }), + let query = EdgeQuery { + label: Some("ASSIGNED_TO".to_string()), + filter: Some(EdgeFilterExpr::ValidAt { epoch_ms: 100 }), ..Default::default() }; + let ids = engine.query_edge_ids(&query).unwrap(); + assert_eq!(ids.edge_ids, vec![live]); - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - vec![string_match, signed_zero_match, nested_zero_match] - ); - let plan = engine.explain_node_query(&query).unwrap(); - assert!(plan.warnings.contains(&QueryPlanWarning::MissingReadyIndex)); - assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackTypeScan]); - - engine.close().unwrap(); + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeMetadataScan)); } #[test] -fn test_query_filter_equality_contradictions_match_verifier_semantics() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let neg_zero = insert_query_node( - &engine, - 1, - "neg-zero", - &[("temperature", PropValue::Float(-0.0))], - 1.0, - ); - let pos_zero = insert_query_node( - &engine, - 1, - "pos-zero", - &[("temperature", PropValue::Float(0.0))], - 1.0, - ); +fn edge_query_paginates_edge_ids_by_cursor() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "c", &[], 1.0); + + let first = engine + .upsert_edge(a, b, "REVIEWED_BY", UpsertEdgeOptions::default()) + .unwrap(); + let second = engine + .upsert_edge(a, c, "REVIEWED_BY", UpsertEdgeOptions::default()) + .unwrap(); - let query = NodeQuery { - ids: vec![neg_zero, pos_zero], - filter: Some(NodeFilterExpr::And(vec![ - NodeFilterExpr::PropertyEquals { - key: "temperature".to_string(), - value: PropValue::Float(-0.0), + let first_page = engine + .query_edge_ids(&EdgeQuery { + label: Some("REVIEWED_BY".to_string()), + page: PageRequest { + limit: Some(1), + after: None, }, - NodeFilterExpr::PropertyEquals { - key: "temperature".to_string(), - value: PropValue::Float(0.0), + ..Default::default() + }) + .unwrap(); + assert_eq!(first_page.edge_ids, vec![first]); + assert_eq!(first_page.next_cursor, Some(first)); + + let second_page = engine + .query_edge_ids(&EdgeQuery { + label: Some("REVIEWED_BY".to_string()), + page: PageRequest { + limit: Some(1), + after: first_page.next_cursor, }, - ])), + ..Default::default() + }) + .unwrap(); + assert_eq!(second_page.edge_ids, vec![second]); + assert_eq!(second_page.next_cursor, None); +} + +#[test] +fn edge_query_broad_label_source_uses_streaming_fallback_page() { + let (_dir, engine) = query_test_engine(); + let edge_count = crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP + 1; + let nodes = (0..=edge_count) + .map(|idx| NodeInput { + labels: vec!["Person".to_string()], + key: format!("broad-edge-node-{idx}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let edge_inputs = node_ids[1..] + .iter() + .map(|to| EdgeInput { + from: hub, + to: *to, + label: "RATES".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }) + .collect::>(); + let edge_ids = engine.batch_upsert_edges(edge_inputs).unwrap(); + engine.flush().unwrap(); + + let query = EdgeQuery { + label: Some("RATES".to_string()), + page: PageRequest { + limit: Some(2), + after: None, + }, ..Default::default() }; - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - vec![neg_zero, pos_zero] - ); - assert_plan_input_nodes( - &engine.explain_node_query(&query).unwrap(), - vec![QueryPlanNode::ExplicitIds], - ); + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let normalized = published.view.normalize_edge_query(&query).unwrap(); + let planned = published.view.plan_normalized_edge_query(&normalized).unwrap(); + assert!(planned + .warnings + .contains(&QueryPlanWarning::CandidateCapExceeded)); + match published + .view + .materialize_edge_physical_plan(&normalized, planned.cap_context, &planned.driver) + .unwrap() + { + CandidateMaterializationResult::TooBroad { .. } => {} + CandidateMaterializationResult::Ready { ids, .. } => { + panic!("expected broad edge source to avoid materialization, got {}", ids.len()) + } + } + } - engine.close().unwrap(); + let first_page = engine.query_edge_ids(&query).unwrap(); + assert_eq!(first_page.edge_ids, edge_ids[..2]); + assert_eq!(first_page.next_cursor, Some(edge_ids[1])); + + let second_page = engine + .query_edge_ids(&EdgeQuery { + page: PageRequest { + limit: Some(2), + after: first_page.next_cursor, + }, + ..query + }) + .unwrap(); + assert_eq!(second_page.edge_ids, edge_ids[2..4]); } #[test] -fn test_query_indexed_float_signed_zero_equality_matches_verifier_semantics() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let neg_zero = insert_query_node( - &engine, - 1, - "indexed-neg-zero", - &[("temperature", PropValue::Float(-0.0))], - 1.0, - ); - let pos_zero = insert_query_node( - &engine, - 1, - "indexed-pos-zero", - &[("temperature", PropValue::Float(0.0))], - 1.0, - ); - insert_query_node( - &engine, - 1, - "indexed-one", - &[("temperature", PropValue::Float(1.0))], - 1.0, - ); +fn edge_query_selective_metadata_source_is_capped_before_too_broad() { + let (_dir, engine) = query_test_engine(); + let edge_count = crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP + 1; + let nodes = (0..=edge_count) + .map(|idx| NodeInput { + labels: vec!["Person".to_string()], + key: format!("metadata-range-node-{idx}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let edge_inputs = node_ids[1..] + .iter() + .enumerate() + .map(|(idx, to)| EdgeInput { + from: hub, + to: *to, + label: "EDGE_LABEL_33".to_string(), + props: BTreeMap::new(), + weight: if idx == 7 { 9.0 } else { 1.0 }, + valid_from: None, + valid_to: None, + }) + .collect::>(); + let edge_ids = engine.batch_upsert_edges(edge_inputs).unwrap(); engine.flush().unwrap(); - let index = engine - .ensure_node_property_index(1, "temperature", SecondaryIndexKind::Equality) - .unwrap(); - wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); + let query = EdgeQuery { + label: Some("EDGE_LABEL_33".to_string()), + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(9.0), + upper: Some(9.0), + }), + page: PageRequest { + limit: Some(1), + after: None, + }, + ..Default::default() + }; - let neg_zero_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyEquals { - key: "temperature".to_string(), - value: PropValue::Float(-0.0), + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let normalized = published.view.normalize_edge_query(&query).unwrap(); + let planned = published.view.plan_normalized_edge_query(&normalized).unwrap(); + match published + .view + .materialize_edge_physical_plan(&normalized, planned.cap_context, &planned.driver) + .unwrap() + { + CandidateMaterializationResult::Ready { ids, .. } => { + assert_eq!(ids, vec![edge_ids[7]]); + } + CandidateMaterializationResult::TooBroad { .. } => { + panic!("selective metadata sidecar source should be capped before TooBroad") + } + } +} + +#[test] +fn edge_query_broad_endpoint_anchor_does_not_fall_back_to_full_scan() { + let (_dir, engine) = query_test_engine(); + let edge_count = crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP + 1; + let nodes = (0..=edge_count) + .map(|idx| NodeInput { + labels: vec!["Person".to_string()], + key: format!("endpoint-fallback-node-{idx}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let edge_inputs = node_ids[1..] + .iter() + .map(|to| EdgeInput { + from: hub, + to: *to, + label: "EDGE_LABEL_34".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }) + .collect::>(); + let edge_ids = engine.batch_upsert_edges(edge_inputs).unwrap(); + engine.flush().unwrap(); + + let query = EdgeQuery { + from_ids: vec![hub], + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(0.0), + upper: Some(2.0), }), + page: PageRequest { + limit: Some(2), + after: None, + }, ..Default::default() }; - assert_eq!( - engine.query_node_ids(&neg_zero_query).unwrap().items, - vec![neg_zero, pos_zero] - ); - assert_plan_input_nodes( - &engine.explain_node_query(&neg_zero_query).unwrap(), - vec![QueryPlanNode::PropertyEqualityIndex], + + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(plan + .warnings + .contains(&QueryPlanWarning::CandidateCapExceeded)); + assert!(plan + .warnings + .contains(&QueryPlanWarning::RangeCandidateCapExceeded)); + + engine.reset_query_execution_counters_for_test(); + let page = engine.query_edge_ids(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(page.edge_ids, edge_ids[..2]); + assert_eq!(page.next_cursor, Some(edge_ids[1])); + assert_eq!(counters.edge_full_scan_pages, 0); + assert_eq!(counters.node_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_reads, 0); + assert!( + counters.endpoint_adjacency_candidates <= 12, + "endpoint scan should stop after the first verification chunk, got {} candidates", + counters.endpoint_adjacency_candidates ); +} - let pos_zero_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyEquals { - key: "temperature".to_string(), - value: PropValue::Float(0.0), - }), +#[test] +fn edge_query_active_memtable_endpoint_scan_is_bounded() { + let (_dir, engine) = query_test_engine(); + let edge_count = 512usize; + let nodes = (0..=edge_count) + .map(|idx| NodeInput { + labels: vec!["Person".to_string()], + key: format!("active-endpoint-node-{idx}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let edge_inputs = node_ids[1..] + .iter() + .map(|to| EdgeInput { + from: hub, + to: *to, + label: "EDGE_LABEL_35".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }) + .collect::>(); + let edge_ids = engine.batch_upsert_edges(edge_inputs).unwrap(); + + let query = EdgeQuery { + from_ids: vec![hub], + page: PageRequest { + limit: Some(2), + after: None, + }, ..Default::default() }; + + crate::memtable::reset_endpoint_cursor_entries_visited_for_test(); + let plan = engine.explain_edge_query(&query).unwrap(); assert_eq!( - engine.query_node_ids(&pos_zero_query).unwrap().items, - vec![neg_zero, pos_zero] + crate::memtable::endpoint_cursor_entries_visited_for_test(), + 0, + "edge endpoint planning must use cheap memtable count bounds, not cursor through the hub" ); - assert_plan_input_nodes( - &engine.explain_node_query(&pos_zero_query).unwrap(), - vec![QueryPlanNode::PropertyEqualityIndex], + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgeEndpointAdjacency + )); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::FallbackFullEdgeScan + )); + + engine.reset_query_execution_counters_for_test(); + let first_page = engine.query_edge_ids(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(first_page.edge_ids, edge_ids[..2]); + assert_eq!(first_page.next_cursor, Some(edge_ids[1])); + assert_eq!(counters.edge_full_scan_pages, 0); + assert!( + counters.endpoint_adjacency_candidates <= 12, + "active endpoint scan should stop after a bounded chunk, got {} candidates", + counters.endpoint_adjacency_candidates ); - engine.close().unwrap(); + let second_page = engine + .query_edge_ids(&EdgeQuery { + page: PageRequest { + limit: Some(2), + after: first_page.next_cursor, + }, + ..query + }) + .unwrap(); + assert_eq!(second_page.edge_ids, edge_ids[2..4]); } #[test] -fn test_query_filter_or_and_in_extract_complete_index_candidates() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let active = insert_query_node( - &engine, - 1, - "active", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let trial = insert_query_node( - &engine, - 1, - "trial", - &[("status", PropValue::String("trial".to_string()))], - 1.0, - ); - let _inactive = insert_query_node( - &engine, - 1, - "inactive", - &[("status", PropValue::String("inactive".to_string()))], - 1.0, - ); - let index = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) +fn edge_query_active_memtable_label_scan_pages_without_materializing_driver() { + let (_dir, engine) = query_test_engine(); + let edge_count = 512usize; + let nodes = (0..=edge_count) + .map(|idx| NodeInput { + labels: vec!["Person".to_string()], + key: format!("active-edge-label-node-{idx}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let edge_ids = engine + .batch_upsert_edges( + node_ids[1..] + .iter() + .map(|to| EdgeInput { + from: hub, + to: *to, + label: "EDGE_LABEL_36".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }) + .collect::>(), + ) .unwrap(); - wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); - - let or_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("trial".to_string()), - }, - ])), - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&or_query).unwrap().items, vec![active, trial]); - let or_plan = engine.explain_node_query(&or_query).unwrap(); - assert!(!or_plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes( - &or_plan, - vec![QueryPlanNode::Union { - inputs: vec![ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], - }], - ); - let singleton_or_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::Or(vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }])), + let query = EdgeQuery { + label: Some("EDGE_LABEL_36".to_string()), + page: PageRequest { + limit: Some(2), + after: None, + }, ..Default::default() }; - assert_eq!( - engine.query_node_ids(&singleton_or_query).unwrap().items, - vec![active] - ); - let singleton_or_plan = engine.explain_node_query(&singleton_or_query).unwrap(); - assert!(!singleton_or_plan - .warnings - .contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes( - &singleton_or_plan, - vec![QueryPlanNode::PropertyEqualityIndex], - ); - let double_not_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::Not(Box::new(NodeFilterExpr::Not(Box::new( - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), + let first_page = engine.query_edge_ids(&query).unwrap(); + assert_eq!(first_page.edge_ids, edge_ids[..2]); + assert_eq!(first_page.next_cursor, Some(edge_ids[1])); + let second_page = engine + .query_edge_ids(&EdgeQuery { + page: PageRequest { + limit: Some(2), + after: first_page.next_cursor, }, - ))))), - ..Default::default() - }; - assert_eq!( - engine.query_node_ids(&double_not_query).unwrap().items, - vec![active] - ); - let double_not_plan = engine.explain_node_query(&double_not_query).unwrap(); - assert!(!double_not_plan - .warnings - .contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes(&double_not_plan, vec![QueryPlanNode::PropertyEqualityIndex]); + ..query + }) + .unwrap(); + assert_eq!(second_page.edge_ids, edge_ids[2..4]); +} - let in_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyIn { - key: "status".to_string(), - values: vec![ - PropValue::String("active".to_string()), - PropValue::String("trial".to_string()), - ], - }), +#[test] +fn edge_query_endpoint_list_uses_batched_source_semantics() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "Person", "endpoint-a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "endpoint-b", &[], 1.0); + let c = insert_query_node(&engine, "Person", "endpoint-c", &[], 1.0); + let d = insert_query_node(&engine, "Person", "endpoint-d", &[], 1.0); + let e = insert_query_node(&engine, "Person", "endpoint-e", &[], 1.0); + + let first = engine.upsert_edge(a, d, "EDGE_LABEL_31", UpsertEdgeOptions::default()).unwrap(); + let second = engine.upsert_edge(b, d, "EDGE_LABEL_31", UpsertEdgeOptions::default()).unwrap(); + let third = engine.upsert_edge(c, e, "EDGE_LABEL_31", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(a, e, "EDGE_LABEL_32", UpsertEdgeOptions::default()).unwrap(); + engine.flush().unwrap(); + + let query = EdgeQuery { + label: Some("EDGE_LABEL_31".to_string()), + endpoint_ids: vec![b, a, b, c], ..Default::default() }; - assert_eq!(engine.query_node_ids(&in_query).unwrap().items, vec![active, trial]); - let in_plan = engine.explain_node_query(&in_query).unwrap(); - assert!(!in_plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert_plan_input_nodes( - &in_plan, - vec![QueryPlanNode::Union { - inputs: vec![ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], - }], - ); + let ids = engine.query_edge_ids(&query).unwrap(); + assert_eq!(ids.edge_ids, vec![first, second, third]); - engine.close().unwrap(); + let plan = engine.explain_edge_query(&query).unwrap(); + assert!(matches!( + &plan.root, + QueryPlanNode::VerifyEdgeFilter { .. } + )); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgeEndpointAdjacency + )); } #[test] -fn test_query_filter_or_in_union_final_verification_and_pagination() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_query_rejects_filter_only_without_full_scan_opt_in() { + let (_dir, engine) = query_test_engine(); + let err = engine + .query_edge_ids(&EdgeQuery { + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(0.0), + upper: Some(1.0), + }), + ..Default::default() + }) + .unwrap_err(); + assert!(err.to_string().contains("allow_full_scan")); +} - let stale_active = insert_query_node( - &engine, - 1, - "stale-active", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let deleted_trial = insert_query_node( - &engine, - 1, - "deleted-trial", - &[("status", PropValue::String("trial".to_string()))], - 1.0, - ); - let active = insert_query_node( - &engine, - 1, - "active", - &[("status", PropValue::String("active".to_string()))], - 1.0, +#[test] +fn edge_query_pattern_filter_normalization() { + let (_dir, engine) = query_test_engine(); + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + + let canonical = pattern_query( + vec![ + pattern_node_with_ids("a", vec![1]), + pattern_node_with_ids("b", vec![2]), + ], + vec![EdgePattern { + alias: Some("e".to_string()), + from_alias: "a".to_string(), + to_alias: "b".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["RELATED_TO".to_string(), "RELATED_TO".to_string(), "LIKES".to_string()], + filter: Some(EdgeFilterExpr::WeightRange { + lower: Some(0.25), + upper: Some(0.75), + }), + }], ); - let trial = insert_query_node( - &engine, - 1, - "trial", - &[("status", PropValue::String("trial".to_string()))], - 1.0, + let normalized = published + .view + .normalize_pattern_query(&canonical) + .unwrap(); + let mut expected_label_filter_ids = vec![ + engine.get_edge_label_id("LIKES").unwrap().unwrap(), + engine.get_edge_label_id("RELATED_TO").unwrap().unwrap(), + ]; + expected_label_filter_ids.sort_unstable(); + assert_eq!(normalized.edges[0].label_filter_ids, Some(expected_label_filter_ids)); + assert!(matches!( + normalized.edges[0].filter, + NormalizedEdgeFilter::WeightRange { + lower: Some(0.25), + upper: Some(0.75), + } + )); + + let property_filter = pattern_query( + vec![ + pattern_node_with_ids("a", vec![1]), + pattern_node_with_ids("b", vec![2]), + ], + vec![EdgePattern { + alias: Some("e".to_string()), + from_alias: "a".to_string(), + to_alias: "b".to_string(), + direction: Direction::Outgoing, + label_filter: Vec::new(), + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "kind".to_string(), + value: PropValue::String("friend".to_string()), + }), + }], ); - for index in 0..3 { - insert_query_node( - &engine, - 1, - &format!("inactive-{index}"), - &[("status", PropValue::String("inactive".to_string()))], - 1.0, - ); - } - engine.flush().unwrap(); - let index = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let normalized = published + .view + .normalize_pattern_query(&property_filter) .unwrap(); - wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); + assert!(matches!( + normalized.edges[0].filter, + NormalizedEdgeFilter::PropertyEquals { ref key, .. } if key == "kind" + )); +} - let updated = engine - .upsert_node( - 1, - "stale-active", - UpsertNodeOptions { - props: query_test_props(&[( - "status", - PropValue::String("inactive".to_string()), - )]), - ..Default::default() +#[test] +fn edge_query_pattern_filter_is_applied_during_execution() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "source", &[], 1.0); + let keep = insert_query_node(&engine, "Company", "keep", &[], 1.0); + let drop_property = insert_query_node(&engine, "Company", "drop-property", &[], 1.0); + let drop_weight = insert_query_node(&engine, "Company", "drop-weight", &[], 1.0); + + let keep_edge = engine + .upsert_edge( + source, + keep, + "FRIENDS_WITH", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + weight: 0.5, + valid_from: Some(10), + valid_to: Some(20), + }, + ) + .unwrap(); + engine + .upsert_edge( + source, + drop_property, + "FRIENDS_WITH", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), + weight: 0.5, + valid_from: Some(10), + valid_to: Some(20), + }, + ) + .unwrap(); + engine + .upsert_edge( + source, + drop_weight, + "FRIENDS_WITH", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + weight: 2.0, + valid_from: Some(10), + valid_to: Some(20), }, ) .unwrap(); - assert_eq!(updated, stale_active); - engine.delete_node(deleted_trial).unwrap(); - - let or_filter = NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("trial".to_string()), - }, - ]); - let mut or_query = NodeQuery { - type_id: Some(1), - filter: Some(or_filter.clone()), - page: PageRequest { - limit: Some(1), - after: None, - }, - ..Default::default() - }; - - let first = engine.query_node_ids(&or_query).unwrap(); - assert_eq!(first.items, vec![active]); - assert_eq!(first.next_cursor, Some(active)); - or_query.page.after = first.next_cursor; - let second = engine.query_node_ids(&or_query).unwrap(); - assert_eq!(second.items, vec![trial]); - assert_eq!(second.next_cursor, None); - or_query.page = PageRequest::default(); - assert_eq!(engine.query_node_ids(&or_query).unwrap().items, vec![active, trial]); - assert_plan_input_nodes( - &engine.explain_node_query(&or_query).unwrap(), - vec![QueryPlanNode::Union { - inputs: vec![ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], + let mut query = pattern_query( + vec![ + pattern_node_with_ids("source", vec![source]), + pattern_node("target", Some("Company"), Vec::new()), + ], + vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["FRIENDS_WITH".to_string()], + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }, + EdgeFilterExpr::ValidAt { epoch_ms: 10 }, + ])), }], ); + query.at_epoch = Some(10); - let in_query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::PropertyIn { - key: "status".to_string(), - values: vec![ - PropValue::String("trial".to_string()), - PropValue::String("active".to_string()), - PropValue::String("active".to_string()), - ], - }), - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&in_query).unwrap().items, vec![active, trial]); - - engine.close().unwrap(); + let result = engine.query_pattern(&query).unwrap(); + assert_eq!( + result.matches, + vec![expected_match( + &[("source", source), ("target", keep)], + &[("edge", keep_edge)] + )] + ); } #[test] -fn test_query_filter_and_of_or_intersects_range() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn edge_query_pattern_metadata_only_filter_does_not_hydrate_edges() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "metadata-pattern-source", &[], 1.0); + let keep = insert_query_node(&engine, "Company", "metadata-pattern-keep", &[], 1.0); + let drop = insert_query_node(&engine, "Company", "metadata-pattern-drop", &[], 1.0); + let keep_edge = engine + .upsert_edge( + source, + keep, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: 0.5, + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + source, + drop, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: 2.0, + ..Default::default() + }, + ) + .unwrap(); - let active_high = insert_query_node( - &engine, - 1, - "active-high", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(20)), - ], - 1.0, - ); - let trial_high = insert_query_node( - &engine, - 1, - "trial-high", - &[ - ("status", PropValue::String("trial".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, - ); - let _active_low = insert_query_node( - &engine, - 1, - "active-low", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(1)), + let query = pattern_query( + vec![ + pattern_node_with_ids("source", vec![source]), + pattern_node("target", Some("Company"), Vec::new()), ], - 1.0, + vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["COLLABORATES_WITH".to_string()], + filter: Some(EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }), + }], ); - let _inactive_high = insert_query_node( - &engine, - 1, - "inactive-high", - &[ - ("status", PropValue::String("inactive".to_string())), - ("score", PropValue::Int(40)), - ], - 1.0, + + engine.reset_query_execution_counters_for_test(); + let result = engine.query_pattern(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + + assert_eq!( + result.matches, + vec![expected_match( + &[("source", source), ("target", keep)], + &[("edge", keep_edge)] + )] ); - engine.flush().unwrap(); - let status_index = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - let score_index = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, + assert_eq!(counters.edge_record_hydration_reads, 0); +} + +#[test] +fn pattern_edge_posting_metadata_filters_prune_before_pending_frontier() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "posting-pattern-source", &[], 1.0); + let keep = insert_query_node(&engine, "Company", "posting-pattern-keep", &[], 1.0); + let drop_by_weight = insert_query_node(&engine, "Company", "posting-pattern-weight", &[], 1.0); + let drop_by_valid_to = insert_query_node(&engine, "Company", "posting-pattern-valid-to", &[], 1.0); + + let keep_edge = engine + .upsert_edge( + source, + keep, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: 0.5, + valid_from: Some(0), + valid_to: Some(20), + ..Default::default() }, ) .unwrap(); - wait_for_property_index_state(&engine, status_index.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(&engine, score_index.index_id, SecondaryIndexState::Ready); - - let query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::And(vec![ - NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("trial".to_string()), - }, - ]), - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), - upper: None, + engine + .upsert_edge( + source, + drop_by_weight, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: 2.0, + valid_from: Some(0), + valid_to: Some(20), + ..Default::default() }, - ])), - ..Default::default() - }; - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - vec![active_high, trial_high] - ); - assert_plan_includes_input_nodes( - &engine.explain_node_query(&query).unwrap(), - &[ - QueryPlanNode::Union { - inputs: vec![ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], + ) + .unwrap(); + engine + .upsert_edge( + source, + drop_by_valid_to, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: 0.5, + valid_from: Some(0), + valid_to: Some(10), + ..Default::default() }, - QueryPlanNode::PropertyRangeIndex, + ) + .unwrap(); + + let mut query = pattern_query( + vec![ + pattern_node_with_ids("source", vec![source]), + pattern_node("target", Some("Company"), Vec::new()), ], + vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["COLLABORATES_WITH".to_string()], + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }, + EdgeFilterExpr::ValidAt { epoch_ms: 10 }, + ])), + }], ); + query.at_epoch = Some(5); - engine.close().unwrap(); + engine.reset_query_execution_counters_for_test(); + let result = engine.query_pattern(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + + assert_eq!( + result.matches, + vec![expected_match( + &[("source", source), ("target", keep)], + &[("edge", keep_edge)] + )] + ); + assert_eq!(counters.edge_record_hydration_reads, 0); + assert_eq!(counters.pattern_edge_pending_entries, 1); } #[test] -fn test_query_filter_fallback_budget_and_empty_plan_edges() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn pattern_edge_updated_at_filter_uses_metadata_without_hydration() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "updated-pattern-source", &[], 1.0); + let old_target = insert_query_node(&engine, "Company", "updated-pattern-old", &[], 1.0); + let keep = insert_query_node(&engine, "Company", "updated-pattern-keep", &[], 1.0); + let old_edge = engine + .upsert_edge(source, old_target, "COLLABORATES_WITH", UpsertEdgeOptions::default()) + .unwrap(); + let old_updated_at = engine.get_edge(old_edge).unwrap().unwrap().updated_at; + wait_until_after_millis(old_updated_at); + let keep_edge = engine + .upsert_edge(source, keep, "COLLABORATES_WITH", UpsertEdgeOptions::default()) + .unwrap(); + let keep_updated_at = engine.get_edge(keep_edge).unwrap().unwrap().updated_at; + assert!(keep_updated_at > old_updated_at); - let active = insert_query_node( - &engine, - 1, - "active", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(1)), + let query = pattern_query( + vec![ + pattern_node_with_ids("source", vec![source]), + pattern_node("target", Some("Company"), Vec::new()), ], - 1.0, + vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["COLLABORATES_WITH".to_string()], + filter: Some(EdgeFilterExpr::UpdatedAtRange { + lower_ms: Some(keep_updated_at), + upper_ms: Some(keep_updated_at), + }), + }], ); - let scored = insert_query_node( - &engine, - 1, - "scored", - &[ - ("status", PropValue::String("inactive".to_string())), - ("score", PropValue::Int(50)), - ], - 1.0, + + engine.reset_query_execution_counters_for_test(); + let result = engine.query_pattern(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + + assert_eq!( + result.matches, + vec![expected_match( + &[("source", source), ("target", keep)], + &[("edge", keep_edge)] + )] ); - for index in 0..8 { - insert_query_node( - &engine, - 1, - &format!("filler-{index}"), - &[ - ("status", PropValue::String(format!("v{index}"))), - ("score", PropValue::Int(index)), - ], + assert_eq!(counters.edge_record_hydration_reads, 0); + assert_eq!(counters.pattern_edge_pending_entries, 2); +} + +#[test] +fn pattern_edge_property_filter_projects_only_metadata_survivors() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "property-pattern-source", &[], 1.0); + let keep = insert_query_node(&engine, "Company", "property-pattern-keep", &[], 1.0); + let metadata_drop = insert_query_node(&engine, "Company", "property-pattern-metadata-drop", &[], 1.0); + + let keep_edge = engine + .upsert_edge( + source, + keep, + "COLLABORATES_WITH", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + weight: 0.5, + ..Default::default() + }, + ) + .unwrap(); + for index in 0..10 { + let target = insert_query_node(&engine, "Company", + &format!("property-pattern-metadata-survivor-{index}"), + &[], 1.0, ); + engine + .upsert_edge( + source, + target, + "COLLABORATES_WITH", + UpsertEdgeOptions { + props: query_test_props(&[( + "status", + PropValue::String("inactive".to_string()), + )]), + weight: 0.5, + ..Default::default() + }, + ) + .unwrap(); } - engine.flush().unwrap(); - let status_index = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + engine + .upsert_edge( + source, + metadata_drop, + "COLLABORATES_WITH", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + weight: 2.0, + ..Default::default() + }, + ) .unwrap(); - wait_for_property_index_state(&engine, status_index.index_id, SecondaryIndexState::Ready); - let impossible = NodeQuery { - filter: Some(NodeFilterExpr::And(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("trial".to_string()), - }, - ])), - ..Default::default() - }; - assert!(engine.query_node_ids(&impossible).unwrap().items.is_empty()); - assert_plan_input_nodes( - &engine.explain_node_query(&impossible).unwrap(), - vec![QueryPlanNode::EmptyResult], + let query = pattern_query( + vec![ + pattern_node_with_ids("source", vec![source]), + pattern_node("target", Some("Company"), Vec::new()), + ], + vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["COLLABORATES_WITH".to_string()], + filter: Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }, + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + ])), + }], ); - let always_true_requires_anchor = NodeQuery { - filter: Some(NodeFilterExpr::Not(Box::new(NodeFilterExpr::And(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("trial".to_string()), - }, - ])))), - ..Default::default() - }; - assert!(matches!( - engine.query_node_ids(&always_true_requires_anchor), - Err(EngineError::InvalidOperation(_)) - )); - - let missing_index_or = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(40))), - upper: None, - }, - ])), - ..Default::default() - }; - assert_eq!( - engine.query_node_ids(&missing_index_or).unwrap().items, - vec![active, scored] - ); - let missing_plan = engine.explain_node_query(&missing_index_or).unwrap(); - assert_eq!( - missing_plan.warnings, - vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, - QueryPlanWarning::BooleanBranchFallback, - ] - ); - assert_plan_input_nodes(&missing_plan, vec![QueryPlanNode::FallbackTypeScan]); + engine.reset_query_execution_counters_for_test(); + let result = engine.query_pattern(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); - let budget_or = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::Or( - (0..=MAX_BOOLEAN_UNION_INPUTS) - .map(|index| NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String(format!("v{index}")), - }) - .collect(), - )), - ..Default::default() - }; - let budget_plan = engine.explain_node_query(&budget_or).unwrap(); assert_eq!( - budget_plan.warnings, - vec![ - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, - QueryPlanWarning::BooleanBranchFallback, - QueryPlanWarning::PlanningProbeBudgetExceeded, - ] + result.matches, + vec![expected_match( + &[("source", source), ("target", keep)], + &[("edge", keep_edge)] + )] ); - - engine.close().unwrap(); + assert_eq!(counters.pattern_edge_pending_entries, 11); + assert_eq!(counters.edge_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_calls, 0); } #[test] -fn test_query_or_unknown_branch_falls_back_without_partial_union() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let indexed = insert_query_node( - &engine, - 1, - "indexed", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let missing_index = insert_query_node( - &engine, - 1, - "missing-index", - &[("score", PropValue::Int(10))], - 1.0, - ); - let other = insert_query_node(&engine, 1, "other", &[], 1.0); - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) +fn pattern_edge_property_filter_reuses_match_cache_across_pending_flushes() { + let (_dir, engine) = query_test_engine(); + let root = insert_query_node(&engine, "Person", "property-cache-root", &[], 1.0); + let mid = insert_query_node(&engine, "Company", "property-cache-mid", &[], 1.0); + let leaf = insert_query_node(&engine, "Article", "property-cache-leaf", &[], 1.0); + for _ in 0..300 { + engine + .upsert_edge(root, mid, "COLLABORATES_WITH", UpsertEdgeOptions::default()) + .unwrap(); + } + let second_edge = engine + .upsert_edge( + mid, + leaf, + "RELATED_TO", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - let query = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(15))), - }, - ])), - ..Default::default() + let query = GraphPatternQuery { + limit: 400, + ..pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node("mid", Some("Company"), Vec::new()), + pattern_node("leaf", Some("Article"), Vec::new()), + ], + vec![ + EdgePattern { + alias: Some("first".to_string()), + from_alias: "root".to_string(), + to_alias: "mid".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["COLLABORATES_WITH".to_string()], + filter: None, + }, + EdgePattern { + alias: Some("second".to_string()), + from_alias: "mid".to_string(), + to_alias: "leaf".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["RELATED_TO".to_string()], + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + }, + ], + ) }; - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[indexed, missing_index, other], &query) - ); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!( - plan.warnings, - vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, - QueryPlanWarning::BooleanBranchFallback, - ] - ); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackTypeScan]); + engine.reset_query_execution_counters_for_test(); + let result = engine.query_pattern(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); - engine.close().unwrap(); + assert!(!result.truncated); + assert_eq!(result.matches.len(), 300); + assert!(result + .matches + .iter() + .all(|matched| matched.edges.get("second") == Some(&second_edge))); + assert_eq!(counters.edge_record_hydration_reads, 0); + assert_eq!(counters.edge_record_hydration_calls, 0); } #[test] -fn test_query_filter_verify_only_uses_expected_legal_universe() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn pattern_edge_metadata_filter_preserves_high_fanout_order_and_truncation() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "Person", "fanout-filter-source", &[], 1.0); + let mut expected = Vec::new(); + for index in 0..10 { + let target = insert_query_node(&engine, "Company", + &format!("fanout-filter-target-{index}"), + &[], + 1.0, + ); + let edge_id = engine + .upsert_edge( + source, + target, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: if index % 2 == 0 { 0.5 } else { 2.0 }, + ..Default::default() + }, + ) + .unwrap(); + if index % 2 == 0 { + expected.push((target, edge_id)); + } + } - let type1_inputs: Vec = (0..QUERY_RANGE_CANDIDATE_CAP + 8) - .map(|index| NodeInput { - type_id: 1, - key: format!("type1-archived-{index}"), - props: query_test_props(&[("archived", PropValue::Bool(true))]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, + let query = GraphPatternQuery { + limit: 3, + ..pattern_query( + vec![ + pattern_node_with_ids("source", vec![source]), + pattern_node("target", Some("Company"), Vec::new()), + ], + vec![EdgePattern { + alias: Some("edge".to_string()), + from_alias: "source".to_string(), + to_alias: "target".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["COLLABORATES_WITH".to_string()], + filter: Some(EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }), + }], + ) + }; + + let result = engine.query_pattern(&query).unwrap(); + assert!(result.truncated); + let expected_matches: Vec<_> = expected + .iter() + .take(3) + .map(|(target, edge)| { + expected_match(&[("source", source), ("target", *target)], &[("edge", *edge)]) }) .collect(); - let type1_ids = engine.batch_upsert_nodes(&type1_inputs).unwrap(); - let type1_archived = type1_ids[0]; - let type1_missing = insert_query_node(&engine, 1, "type1-missing", &[], 1.0); - let small_missing = insert_query_node(&engine, 2, "small-missing", &[], 1.0); - let small_archived = insert_query_node( - &engine, - 2, - "small-archived", - &[("archived", PropValue::Bool(true))], - 1.0, - ); - let active_tag = insert_query_node( - &engine, - 3, - "active-tag", - &[ - ("status", PropValue::String("active".to_string())), - ("tag", PropValue::String("present".to_string())), - ], - 1.0, - ); - let active_missing = insert_query_node( - &engine, - 3, - "active-missing", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let inactive_tag = insert_query_node( - &engine, - 3, - "inactive-tag", - &[ - ("status", PropValue::String("inactive".to_string())), - ("tag", PropValue::String("present".to_string())), - ], - 1.0, - ); + assert_eq!(result.matches, expected_matches); +} - let status_index = engine - .ensure_node_property_index(3, "status", SecondaryIndexKind::Equality) +#[test] +fn pattern_edge_filters_work_on_branching_and_self_loop_patterns() { + let (_dir, engine) = query_test_engine(); + let root = insert_query_node(&engine, "Person", "filtered-branch-root", &[], 1.0); + let left = insert_query_node(&engine, "Company", "filtered-branch-left", &[], 1.0); + let right = insert_query_node(&engine, "Article", "filtered-branch-right", &[], 1.0); + let left_edge = engine + .upsert_edge( + root, + left, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: 0.5, + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + root, + right, + "COLLABORATES_WITH", + UpsertEdgeOptions { + weight: 2.0, + ..Default::default() + }, + ) + .unwrap(); + let right_edge = engine + .upsert_edge( + root, + right, + "RELATED_TO", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("active".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let self_loop = engine + .upsert_edge( + root, + root, + "KNOWS", + UpsertEdgeOptions { + valid_from: Some(0), + valid_to: Some(20), + ..Default::default() + }, + ) .unwrap(); - wait_for_property_index_state(&engine, status_index.index_id, SecondaryIndexState::Ready); - let mut huge_ids = type1_ids.clone(); - huge_ids.push(type1_missing); - huge_ids.push(small_missing); - huge_ids.push(small_archived); - let type_small_query = NodeQuery { - type_id: Some(2), - ids: huge_ids, - filter: Some(NodeFilterExpr::PropertyMissing { - key: "archived".to_string(), - }), - ..Default::default() - }; - assert_eq!( - engine.query_node_ids(&type_small_query).unwrap().items, - vec![small_missing] + let branch_query = pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node_with_ids("left", vec![left]), + pattern_node_with_ids("right", vec![right]), + ], + vec![ + EdgePattern { + alias: Some("left_edge".to_string()), + from_alias: "root".to_string(), + to_alias: "left".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["COLLABORATES_WITH".to_string()], + filter: Some(EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }), + }, + EdgePattern { + alias: Some("right_edge".to_string()), + from_alias: "root".to_string(), + to_alias: "right".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["RELATED_TO".to_string()], + filter: Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + }, + ], ); - let type_small_plan = engine.explain_node_query(&type_small_query).unwrap(); assert_eq!( - type_small_plan.warnings, - vec![ - QueryPlanWarning::UsingFallbackScan, + engine.query_pattern(&branch_query).unwrap().matches, + vec![expected_match( + &[("left", left), ("right", right), ("root", root)], + &[("left_edge", left_edge), ("right_edge", right_edge)] + )] + ); + + let self_loop_query = GraphPatternQuery { + at_epoch: Some(5), + ..pattern_query( + vec![pattern_node_with_ids("root", vec![root])], + vec![EdgePattern { + alias: Some("loop".to_string()), + from_alias: "root".to_string(), + to_alias: "root".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["KNOWS".to_string()], + filter: Some(EdgeFilterExpr::ValidAt { epoch_ms: 10 }), + }], + ) + }; + assert_eq!( + engine.query_pattern(&self_loop_query).unwrap().matches, + vec![expected_match(&[("root", root)], &[("loop", self_loop)])] + ); +} + +fn expected_match(nodes: &[(&str, u64)], edges: &[(&str, u64)]) -> QueryMatch { + QueryMatch { + nodes: nodes + .iter() + .map(|(alias, id)| ((*alias).to_string(), *id)) + .collect(), + edges: edges + .iter() + .map(|(alias, id)| ((*alias).to_string(), *id)) + .collect(), + } +} + +fn oracle_node_matches(query: &NodeQuery, node: &NodeView) -> bool { + if let Some(filter) = query.label_filter.as_ref() { + match filter.mode { + LabelMatchMode::Any => { + if !filter + .labels + .iter() + .any(|label| node.labels.iter().any(|node_label| node_label == label)) + { + return false; + } + } + LabelMatchMode::All => { + if !filter + .labels + .iter() + .all(|label| node.labels.iter().any(|node_label| node_label == label)) + { + return false; + } + } + } + } + if !query.ids.is_empty() && !query.ids.contains(&node.id) { + return false; + } + if !query.keys.is_empty() && !query.keys.contains(&node.key) { + return false; + } + query + .filter + .as_ref() + .is_none_or(|filter| oracle_filter_matches(filter, node)) +} + +fn oracle_filter_matches(filter: &NodeFilterExpr, node: &NodeView) -> bool { + match filter { + NodeFilterExpr::PropertyEquals { key, value } => { + node.props.get(key).is_some_and(|candidate| candidate == value) + } + NodeFilterExpr::PropertyIn { key, values } => node + .props + .get(key) + .is_some_and(|candidate| values.iter().any(|value| candidate == value)), + NodeFilterExpr::PropertyRange { key, lower, upper } => { + let Some(value) = node.props.get(key) else { + return false; + }; + let lower_matches = lower.as_ref().is_none_or(|bound| { + let Some(ordering) = compare_range_values(value, bound.value()) else { + return false; + }; + match bound { + PropertyRangeBound::Included(_) => ordering != std::cmp::Ordering::Less, + PropertyRangeBound::Excluded(_) => ordering == std::cmp::Ordering::Greater, + } + }); + let upper_matches = upper.as_ref().is_none_or(|bound| { + let Some(ordering) = compare_range_values(value, bound.value()) else { + return false; + }; + match bound { + PropertyRangeBound::Included(_) => ordering != std::cmp::Ordering::Greater, + PropertyRangeBound::Excluded(_) => ordering == std::cmp::Ordering::Less, + } + }); + lower_matches && upper_matches + } + NodeFilterExpr::UpdatedAtRange { lower_ms, upper_ms } => { + lower_ms.is_none_or(|lower| node.updated_at >= lower) + && upper_ms.is_none_or(|upper| node.updated_at <= upper) + } + NodeFilterExpr::PropertyExists { key } => node.props.contains_key(key), + NodeFilterExpr::PropertyMissing { key } => !node.props.contains_key(key), + NodeFilterExpr::And(children) => { + children.iter().all(|child| oracle_filter_matches(child, node)) + } + NodeFilterExpr::Or(children) => { + children.iter().any(|child| oracle_filter_matches(child, node)) + } + NodeFilterExpr::Not(child) => !oracle_filter_matches(child, node), + } +} + +fn oracle_query_ids( + engine: &DatabaseEngine, + candidate_ids: &[u64], + query: &NodeQuery, +) -> Vec { + let mut ids = candidate_ids.to_vec(); + ids.sort_unstable(); + ids.dedup(); + engine + .get_nodes(&ids) + .unwrap() + .into_iter() + .flatten() + .filter(|node| oracle_node_matches(query, node)) + .map(|node| node.id) + .collect() +} + +fn set_query_node_updated_at(engine: &DatabaseEngine, node_id: u64, updated_at: i64) { + let node = internal_node_record(engine, node_id).unwrap().unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { + created_at: updated_at, + updated_at, + ..node + })) + .unwrap(); +} + +fn set_query_edge_props(engine: &DatabaseEngine, edge_id: u64, props: BTreeMap) { + let edge = internal_edge_record(engine, edge_id).unwrap().unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertEdge(EdgeRecord { props, ..edge })) + .unwrap(); +} + +fn replace_equality_sidecar_group_id_in_place( + path: &std::path::Path, + value_hash: u64, + from_id: u64, + to_id: u64, +) { + use std::io::{Seek, SeekFrom, Write}; + + const SECONDARY_EQ_ENTRY_SIZE: usize = 20; + let data = std::fs::read(path).unwrap(); + let payload_offset = component_payload_offset_for_test(path) as usize; + let payload = &data[payload_offset..]; + assert!(payload.len() >= 8, "equality sidecar payload missing count"); + let count = u64::from_le_bytes(payload[0..8].try_into().unwrap()) as usize; + + for index in 0..count { + let entry_off = 8 + index * SECONDARY_EQ_ENTRY_SIZE; + let entry_value_hash = + u64::from_le_bytes(payload[entry_off..entry_off + 8].try_into().unwrap()); + if entry_value_hash != value_hash { + continue; + } + let group_offset = + u64::from_le_bytes(payload[entry_off + 8..entry_off + 16].try_into().unwrap()) + as usize; + let id_count = + u32::from_le_bytes(payload[entry_off + 16..entry_off + 20].try_into().unwrap()) + as usize; + for id_index in 0..id_count { + let id_offset = group_offset + id_index * 8; + let existing = u64::from_le_bytes(payload[id_offset..id_offset + 8].try_into().unwrap()); + if existing == from_id { + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start((payload_offset + id_offset) as u64)) + .unwrap(); + file.write_all(&to_id.to_le_bytes()).unwrap(); + file.sync_all().unwrap(); + return; + } + } + panic!("target equality sidecar group did not contain id {from_id}"); + } + + panic!("target equality sidecar group hash {value_hash} not found"); +} + +fn plan_contains_node(node: &QueryPlanNode, expected: &QueryPlanNode) -> bool { + if node == expected { + return true; + } + match node { + QueryPlanNode::Intersect { inputs } | QueryPlanNode::Union { inputs } => { + inputs.iter().any(|input| plan_contains_node(input, expected)) + } + QueryPlanNode::VerifyNodeFilter { input } + | QueryPlanNode::VerifyEdgeFilter { input } + | QueryPlanNode::VerifyEdgePredicates { input } + | QueryPlanNode::PatternExpand { input, .. } + | QueryPlanNode::PatternEdgeAnchor { input, .. } => plan_contains_node(input, expected), + _ => false, + } +} + +fn explain_input_node(plan: &QueryPlan) -> &QueryPlanNode { + match &plan.root { + QueryPlanNode::VerifyNodeFilter { input } => input.as_ref(), + other => panic!("expected VerifyNodeFilter root, got {other:?}"), + } +} + +fn explain_input_nodes(plan: &QueryPlan) -> Vec { + match explain_input_node(plan) { + QueryPlanNode::Intersect { inputs } => inputs.clone(), + node => vec![node.clone()], + } +} + +fn assert_plan_input_nodes(plan: &QueryPlan, expected: Vec) { + assert_eq!(explain_input_nodes(plan), expected); +} + +fn assert_plan_includes_input_nodes(plan: &QueryPlan, expected: &[QueryPlanNode]) { + let mut actual = explain_input_nodes(plan); + for expected_node in expected { + let position = actual + .iter() + .position(|node| node == expected_node) + .unwrap_or_else(|| panic!("expected plan to include {expected_node:?}; got {actual:?}")); + actual.remove(position); + } +} + +fn pattern_anchor_plan_node(plan: &QueryPlan) -> (&str, &QueryPlanNode) { + let pattern = match &plan.root { + QueryPlanNode::PatternExpand { .. } => &plan.root, + QueryPlanNode::VerifyEdgePredicates { input } => input.as_ref(), + other => panic!("expected pattern expand root, got {other:?}"), + }; + match pattern { + QueryPlanNode::PatternExpand { + anchor_alias, + input, + } => match input.as_ref() { + QueryPlanNode::VerifyNodeFilter { input } => (anchor_alias.as_str(), input.as_ref()), + other => panic!("expected VerifyNodeFilter pattern input, got {other:?}"), + }, + other => panic!("expected pattern expand node, got {other:?}"), + } +} + +fn pattern_anchor_input_nodes(plan: &QueryPlan) -> Vec { + match pattern_anchor_plan_node(plan).1 { + QueryPlanNode::Intersect { inputs } => inputs.clone(), + node => vec![node.clone()], + } +} + +fn planned_pattern_anchor_and_edge_aliases( + engine: &DatabaseEngine, + query: &GraphPatternQuery, +) -> (String, Vec) { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let normalized = published + .view + .normalize_pattern_query(&query) + .unwrap(); + let planned = published + .view + .plan_normalized_pattern_query(&normalized) + .unwrap(); + let aliases = planned + .anchor + .expansion_order() + .iter() + .map(|&edge_index| { + normalized.edges[edge_index] + .alias + .clone() + .unwrap_or_else(|| format!("edge-{edge_index}")) + }) + .collect(); + let anchor_alias = match &planned.anchor { + PatternAnchorPlan::Node { node_index, .. } => normalized.nodes[*node_index].alias.clone(), + PatternAnchorPlan::Edge { edge_alias, .. } => { + edge_alias.clone().unwrap_or_else(|| "".to_string()) + } + }; + (anchor_alias, aliases) +} + +fn planned_pattern_anchor_sort_and_edge_aliases( + engine: &DatabaseEngine, + query: &GraphPatternQuery, +) -> (String, String, Vec) { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let normalized = published + .view + .normalize_pattern_query(&query) + .unwrap(); + let planned = published + .view + .plan_normalized_pattern_query(&normalized) + .unwrap(); + let aliases = planned + .anchor + .expansion_order() + .iter() + .map(|&edge_index| { + normalized.edges[edge_index] + .alias + .clone() + .unwrap_or_else(|| format!("edge-{edge_index}")) + }) + .collect(); + let anchor_alias = match &planned.anchor { + PatternAnchorPlan::Node { node_index, .. } => normalized.nodes[*node_index].alias.clone(), + PatternAnchorPlan::Edge { edge_alias, .. } => { + edge_alias.clone().unwrap_or_else(|| "".to_string()) + } + }; + ( + anchor_alias, + planned.anchor.sort_anchor_alias().to_string(), + aliases, + ) +} + +fn plan_contains_fallback_full_node_scan(node: &QueryPlanNode) -> bool { + match node { + QueryPlanNode::FallbackFullNodeScan => true, + QueryPlanNode::Intersect { inputs } | QueryPlanNode::Union { inputs } => { + inputs.iter().any(plan_contains_fallback_full_node_scan) + } + QueryPlanNode::VerifyNodeFilter { input } + | QueryPlanNode::VerifyEdgePredicates { input } + | QueryPlanNode::PatternExpand { input, .. } + | QueryPlanNode::PatternEdgeAnchor { input, .. } => { + plan_contains_fallback_full_node_scan(input) + } + _ => false, + } +} + +fn plan_contains_pattern_edge_anchor(node: &QueryPlanNode) -> bool { + match node { + QueryPlanNode::PatternEdgeAnchor { .. } => true, + QueryPlanNode::Intersect { inputs } | QueryPlanNode::Union { inputs } => { + inputs.iter().any(plan_contains_pattern_edge_anchor) + } + QueryPlanNode::VerifyNodeFilter { input } + | QueryPlanNode::VerifyEdgeFilter { input } + | QueryPlanNode::VerifyEdgePredicates { input } + | QueryPlanNode::PatternExpand { input, .. } => { + plan_contains_pattern_edge_anchor(input) + } + _ => false, + } +} + +#[test] +fn test_planner_stats_view_rebuilds_only_with_read_sources() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let initial = engine.planner_stats_view_for_test(); + assert!(initial.generation >= 1); + assert_eq!(initial.segment_count, 0); + + insert_query_node(&engine, "Person", + "active", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let after_write = engine.planner_stats_view_for_test(); + assert!(std::sync::Arc::ptr_eq(&initial, &after_write)); + assert_eq!(after_write.generation, initial.generation); + + engine.flush().unwrap(); + let after_flush = engine.planner_stats_view_for_test(); + assert!(!std::sync::Arc::ptr_eq(&after_write, &after_flush)); + assert!(after_flush.generation > after_write.generation); + assert_eq!(after_flush.segment_count, 1); + assert_eq!(after_flush.available_segment_stats, 1); + assert_eq!(after_flush.full_rollup.node_count, 1); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_stale_risk_uses_newer_sample_shadowing() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + for index in 0..16 { + insert_query_node(&engine, "Person", + &format!("shadow-{index:02}"), + &[("version", PropValue::Int(1))], + 1.0, + ); + } + engine.flush().unwrap(); + for index in 0..8 { + insert_query_node(&engine, "Person", + &format!("shadow-{index:02}"), + &[("version", PropValue::Int(2))], + 1.0, + ); + } + engine.flush().unwrap(); + + let stats_view = engine.planner_stats_view_for_test(); + assert_eq!( + stats_view.max_segment_stale_risk(), + crate::planner_stats::StalePostingRisk::High + ); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_stale_risk_uses_newer_tombstones() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut ids = Vec::new(); + for index in 0..16 { + ids.push(insert_query_node(&engine, "Person", + &format!("delete-{index:02}"), + &[], + 1.0, + )); + } + engine.flush().unwrap(); + engine.delete_node(ids[0]).unwrap(); + engine.flush().unwrap(); + + let stats_view = engine.planner_stats_view_for_test(); + assert_eq!( + stats_view.max_segment_stale_risk(), + crate::planner_stats::StalePostingRisk::Medium + ); + + engine.close().unwrap(); +} + +#[test] +fn test_write_adjacent_helper_reads_do_not_rebuild_planner_stats_view() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let delete_a = insert_query_node(&engine, "Person", "delete-a", &[], 1.0); + let delete_b = insert_query_node(&engine, "Person", "delete-b", &[], 1.0); + let patch_c = insert_query_node(&engine, "Company", "patch-c", &[], 1.0); + let patch_d = insert_query_node(&engine, "Company", "patch-d", &[], 1.0); + let prune_e = insert_query_node(&engine, "Metric", "prune-e", &[], 0.1); + let prune_f = insert_query_node(&engine, "NodeLabel91", "prune-f", &[], 1.0); + engine + .upsert_edge(delete_a, delete_b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let patch_edge = engine + .upsert_edge(patch_c, patch_d, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(prune_e, prune_f, "RATES", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + + let stats_before = engine.planner_stats_view_for_test(); + let generation_before = stats_before.generation; + let source_builds_before = engine.published_read_source_build_count_for_test(); + engine.reset_publish_counters_for_test(); + + engine.delete_node(delete_a).unwrap(); + engine + .graph_patch(GraphPatch { + invalidate_edges: vec![(patch_edge, 1)], + delete_node_ids: vec![patch_c], + ..Default::default() + }) + .unwrap(); + let prune = engine + .prune(&PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("Metric".to_string()), + }) + .unwrap(); + assert_eq!(prune.nodes_pruned, 1); + assert_eq!(prune.edges_pruned, 1); + + let stats_after = engine.planner_stats_view_for_test(); + let counters = engine.publish_counter_snapshot_for_test(); + assert_eq!(counters.rebuild_sources, 0); + assert_eq!(counters.source_rebuilds, 0); + assert_eq!( + engine.published_read_source_build_count_for_test(), + source_builds_before + ); + assert!(std::sync::Arc::ptr_eq(&stats_before, &stats_after)); + assert_eq!(stats_after.generation, generation_before); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_corruption_degrades_without_index_repair_followup() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + insert_query_node(&engine, "Person", + "red", + &[("status", PropValue::String("red".to_string()))], + 1.0, + ); + insert_query_node(&engine, "Person", + "blue", + &[("status", PropValue::String("blue".to_string()))], + 1.0, + ); + engine.flush().unwrap(); + engine.close().unwrap(); + } + + let stats_path = crate::segment_writer::segment_dir(&db_path, 1) + .join(crate::planner_stats::PLANNER_STATS_FILENAME); + std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let stats_view = reopened.planner_stats_view_for_test(); + assert_eq!(stats_view.segment_count, 1); + assert_eq!(stats_view.available_segment_stats, 0); + assert_eq!(stats_view.unavailable_segment_stats, 1); + + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("red".to_string()), + }], + false, + ); + assert_eq!(reopened.query_node_ids(&query).unwrap().items.len(), 1); + let plan = reopened.explain_node_query(&query).unwrap(); + assert!(!plan.warnings.contains(&QueryPlanWarning::MissingReadyIndex)); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyEqualityIndex]); + + reopened.close().unwrap(); +} + +#[test] +fn test_planner_stats_zero_is_advisory_not_empty_result() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let index_id; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + index_id = info.index_id; + wait_for_property_index_state(&engine, index_id, SecondaryIndexState::Ready); + insert_query_node(&engine, "Person", + "red", + &[("status", PropValue::String("red".to_string()))], + 1.0, + ); + engine.flush().unwrap(); + engine.close().unwrap(); + } + + let seg_dir = crate::segment_writer::segment_dir(&db_path, 1); + let mut stats = match crate::planner_stats::read_planner_stats_sidecar(&seg_dir, 1, 1, 0) { + crate::planner_stats::PlannerStatsAvailability::Available(stats) => *stats, + other => panic!("expected available planner stats, got {other:?}"), + }; + let equality = stats + .equality_index_stats + .iter_mut() + .find(|stats| stats.index_id == index_id) + .expect("expected equality stats for test index"); + equality.total_postings = 0; + equality.value_group_count = 0; + equality.max_group_postings = 0; + equality.top_value_hashes.clear(); + let ready_indexes = [ready_node_property_equality_entry(index_id, 1, "status")]; + publish_planner_stats_for_test(&seg_dir, stats, &ready_indexes); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let red = PropValue::String("red".to_string()); + let red_hash = hash_prop_value(&red); + assert_eq!( + reopened + .planner_stats_view_for_test() + .equality_segment_estimate(index_id, 1, &[red_hash]) + .unwrap(), + crate::planner_stats::PlannerStatsValueEstimate { + count: 0, + exact: true, + } + ); + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: red, + }], + false, + ); + assert_eq!(reopened.query_node_ids(&query).unwrap().items.len(), 1); + let plan = reopened.explain_node_query(&query).unwrap(); + assert_eq!(plan.estimated_candidates, Some(0)); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyEqualityIndex]); + + reopened.close().unwrap(); +} + +#[test] +fn test_planner_stats_low_equality_estimate_uses_capped_materialization() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let index_id; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + index_id = info.index_id; + wait_for_property_index_state(&engine, index_id, SecondaryIndexState::Ready); + for idx in 0..=QUERY_RANGE_CANDIDATE_CAP { + insert_query_node(&engine, "Person", + &format!("active-{idx}"), + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + } + engine.flush().unwrap(); + engine.close().unwrap(); + } + + let node_count = (QUERY_RANGE_CANDIDATE_CAP + 1) as u64; + let seg_dir = crate::segment_writer::segment_dir(&db_path, 1); + let mut stats = match crate::planner_stats::read_planner_stats_sidecar(&seg_dir, 1, node_count, 0) + { + crate::planner_stats::PlannerStatsAvailability::Available(stats) => *stats, + other => panic!("expected available planner stats, got {other:?}"), + }; + let equality = stats + .equality_index_stats + .iter_mut() + .find(|stats| stats.index_id == index_id) + .expect("expected equality stats for test index"); + equality.total_postings = 0; + equality.value_group_count = 0; + equality.max_group_postings = 0; + equality.top_value_hashes.clear(); + let ready_indexes = [ready_node_property_equality_entry(index_id, 1, "status")]; + publish_planner_stats_for_test(&seg_dir, stats, &ready_indexes); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + false, + ); + { + let (_guard, published) = reopened.runtime.published_snapshot().unwrap(); + let normalized = published.view.normalize_node_query(&query).unwrap(); + let planned = published.view.plan_normalized_node_query(&normalized).unwrap(); + let NodePhysicalPlan::Source(source) = planned.driver else { + panic!("expected equality source driver"); + }; + assert_eq!(source.kind, NodeQueryCandidateSourceKind::PropertyEqualityIndex); + assert_eq!(source.estimate.known_upper_bound(), Some(0)); + assert!(!source.estimate.can_use_uncapped_equality_materialization()); + } + let result = reopened.query_node_ids(&query).unwrap(); + assert_eq!(result.items.len(), QUERY_RANGE_CANDIDATE_CAP + 1); + assert_eq!(result.next_cursor, None); + + reopened.close().unwrap(); +} + +#[test] +fn test_planner_stats_back_label_and_full_scan_explain_estimates() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + for idx in 0..3 { + insert_query_node(&engine, "Person", &format!("label1-{idx}"), &[], 1.0); + } + for idx in 0..2 { + insert_query_node(&engine, "Company", &format!("label2-{idx}"), &[], 1.0); + } + engine.flush().unwrap(); + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let label_estimate = published.view.node_label_estimate(1).unwrap(); + assert_eq!(label_estimate.kind, PlannerEstimateKind::StatsExact); + assert_eq!(label_estimate.known_upper_bound(), Some(3)); + let full_estimate = published.view.full_scan_estimate(); + assert_eq!(full_estimate.kind, PlannerEstimateKind::StatsExact); + assert_eq!(full_estimate.known_upper_bound(), Some(5)); + } + + let label_query = query_ids(Some("Person"), Vec::new(), false); + assert_eq!(engine.query_node_ids(&label_query).unwrap().items.len(), 3); + let label_plan = engine.explain_node_query(&label_query).unwrap(); + assert_eq!(label_plan.estimated_candidates, Some(3)); + assert_plan_input_nodes(&label_plan, vec![QueryPlanNode::NodeLabelIndex]); + + let full_scan_query = NodeQuery { + allow_full_scan: true, + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&full_scan_query).unwrap().items.len(), 5); + let full_scan_plan = engine.explain_node_query(&full_scan_query).unwrap(); + assert_eq!(full_scan_plan.estimated_candidates, Some(5)); + assert_plan_input_nodes(&full_scan_plan, vec![QueryPlanNode::FallbackFullNodeScan]); + assert_eq!( + full_scan_plan.warnings, + vec![QueryPlanWarning::FullScanExplicitlyAllowed] + ); + + engine.close().unwrap(); +} + +#[test] +fn test_multi_label_any_and_all_membership_estimates_are_conservative() { + let (_dir, engine) = query_test_engine(); + let person_id = engine.get_node_label_id("Person").unwrap().unwrap(); + let company_id = engine.get_node_label_id("Company").unwrap().unwrap(); + + engine + .upsert_node( + &["Person", "Company"], + "overlap", + UpsertNodeOptions { + props: BTreeMap::new(), + ..Default::default() + }, + ) + .unwrap(); + insert_query_node(&engine, "Person", "person-a", &[], 1.0); + insert_query_node(&engine, "Person", "person-b", &[], 1.0); + insert_query_node(&engine, "Company", "company", &[], 1.0); + engine.flush().unwrap(); + + let labels = NodeLabelSet::from_canonical_ids(&[person_id, company_id]).unwrap(); + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let any = published + .view + .node_label_filter_estimate(&labels, LabelMatchMode::Any) + .unwrap(); + assert_eq!(any.estimate.kind, PlannerEstimateKind::UpperBound); + assert_eq!(any.estimate.known_upper_bound(), Some(5)); + assert_eq!(any.driver_label_id, None); + assert_eq!(published.view.full_scan_estimate().known_upper_bound(), Some(4)); + + let all = published + .view + .node_label_filter_estimate(&labels, LabelMatchMode::All) + .unwrap(); + assert_eq!(all.estimate.kind, PlannerEstimateKind::UpperBound); + assert_eq!(all.estimate.known_upper_bound(), Some(2)); + assert_eq!(all.driver_label_id, Some(company_id)); + + drop(published); + drop(_guard); + engine.close().unwrap(); +} + +#[test] +fn test_multi_label_filter_estimates_fall_back_when_stats_are_corrupt() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + seed_query_test_catalog(&engine); + engine + .upsert_node( + &["Person", "Company"], + "covered-overlap", + UpsertNodeOptions { + props: BTreeMap::new(), + ..Default::default() + }, + ) + .unwrap(); + insert_query_node(&engine, "Person", "covered-person", &[], 1.0); + engine.flush().unwrap(); + engine + .upsert_node( + &["Person", "Company"], + "fallback-overlap", + UpsertNodeOptions { + props: BTreeMap::new(), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + engine.close().unwrap(); + } + corrupt_planner_stats_for_segment(&db_path, 2); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let person_id = reopened.get_node_label_id("Person").unwrap().unwrap(); + let company_id = reopened.get_node_label_id("Company").unwrap().unwrap(); + let labels = NodeLabelSet::from_canonical_ids(&[person_id, company_id]).unwrap(); + let stats_view = reopened.planner_stats_view_for_test(); + assert_eq!(stats_view.available_segment_stats, 1); + assert_eq!(stats_view.unavailable_segment_stats, 1); + drop(stats_view); + + let (_guard, published) = reopened.runtime.published_snapshot().unwrap(); + let any = published + .view + .node_label_filter_estimate(&labels, LabelMatchMode::Any) + .unwrap(); + assert_eq!(any.estimate.kind, PlannerEstimateKind::UpperBound); + assert_eq!(any.estimate.known_upper_bound(), Some(5)); + assert_eq!(any.driver_label_id, None); + + let all = published + .view + .node_label_filter_estimate(&labels, LabelMatchMode::All) + .unwrap(); + assert_eq!(all.estimate.kind, PlannerEstimateKind::UpperBound); + assert_eq!(all.estimate.known_upper_bound(), Some(2)); + assert_eq!(all.driver_label_id, Some(company_id)); + + drop(published); + drop(_guard); + reopened.close().unwrap(); +} + +#[test] +fn test_node_query_multi_label_any_all_unknown_and_explain_notes() { + let (_dir, engine) = query_test_engine(); + let person = insert_query_node(&engine, "Person", "person", &[], 1.0); + let employee = insert_query_node(&engine, "Employee", "employee", &[], 1.0); + let both = + insert_query_node_with_labels(&engine, &["Person", "Employee"], "both", &[], 1.0); + let _company = insert_query_node(&engine, "Company", "company", &[], 1.0); + + let mut any_expected = vec![person, employee, both]; + any_expected.sort_unstable(); + let any_query = query_label_filter(&["Person", "Employee"], LabelMatchMode::Any); + assert_eq!(engine.query_node_ids(&any_query).unwrap().items, any_expected); + let any_plan = engine.explain_node_query(&any_query).unwrap(); + assert_plan_input_nodes(&any_plan, vec![QueryPlanNode::NodeLabelAnyIndex]); + assert_eq!( + any_plan.public_inputs.node_labels, + vec![ + QueryPlanPublicName { + alias: None, + name: "Person".to_string(), + known: true, + mode: Some(LabelMatchMode::Any), + }, + QueryPlanPublicName { + alias: None, + name: "Employee".to_string(), + known: true, + mode: Some(LabelMatchMode::Any), + }, + ] + ); + assert!(any_plan + .notes + .contains(&QueryPlanNote::NodeLabelAnyDedupeBeforePagination)); + assert!(any_plan + .notes + .contains(&QueryPlanNote::NodeLabelAnyFinalVerification)); + assert!(any_plan + .notes + .contains(&QueryPlanNote::StaleNodeLabelMembershipVerification)); + + let all_query = query_label_filter(&["Person", "Employee"], LabelMatchMode::All); + assert_eq!( + engine.query_node_ids(&all_query).unwrap().items, + vec![both] + ); + let all_plan = engine.explain_node_query(&all_query).unwrap(); + assert_eq!( + all_plan.public_inputs.node_labels, + vec![ + QueryPlanPublicName { + alias: None, + name: "Person".to_string(), + known: true, + mode: Some(LabelMatchMode::All), + }, + QueryPlanPublicName { + alias: None, + name: "Employee".to_string(), + known: true, + mode: Some(LabelMatchMode::All), + }, + ] + ); + assert!(all_plan + .notes + .contains(&QueryPlanNote::NodeLabelAllSupersetVerification)); + assert!(all_plan + .notes + .contains(&QueryPlanNote::StaleNodeLabelMembershipVerification)); + + let mixed_unknown_any = query_label_filter(&["Person", "Missing"], LabelMatchMode::Any); + let mut mixed_expected = vec![person, both]; + mixed_expected.sort_unstable(); + assert_eq!( + engine.query_node_ids(&mixed_unknown_any).unwrap().items, + mixed_expected + ); + let mixed_plan = engine.explain_node_query(&mixed_unknown_any).unwrap(); + assert!(mixed_plan + .warnings + .contains(&QueryPlanWarning::UnknownNodeLabel)); + assert_eq!( + mixed_plan.public_inputs.node_labels, + vec![ + QueryPlanPublicName { + alias: None, + name: "Person".to_string(), + known: true, + mode: Some(LabelMatchMode::Any), + }, + QueryPlanPublicName { + alias: None, + name: "Missing".to_string(), + known: false, + mode: Some(LabelMatchMode::Any), + }, + ] + ); + + let all_unknown_any = query_label_filter(&["Missing"], LabelMatchMode::Any); + assert!(engine + .query_node_ids(&all_unknown_any) + .unwrap() + .items + .is_empty()); + assert!(engine + .explain_node_query(&all_unknown_any) + .unwrap() + .warnings + .contains(&QueryPlanWarning::UnknownNodeLabel)); + + let mixed_unknown_all = query_label_filter(&["Person", "Missing"], LabelMatchMode::All); + assert!(engine + .query_node_ids(&mixed_unknown_all) + .unwrap() + .items + .is_empty()); + assert!(engine + .explain_node_query(&mixed_unknown_all) + .unwrap() + .warnings + .contains(&QueryPlanWarning::UnknownNodeLabel)); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_any_dedupes_before_pagination_and_hydrates_final_page() { + let (_dir, engine) = query_test_engine(); + let both_a = + insert_query_node_with_labels(&engine, &["Person", "Employee"], "both-a", &[], 1.0); + let person = insert_query_node(&engine, "Person", "person", &[], 1.0); + let employee = insert_query_node(&engine, "Employee", "employee", &[], 1.0); + let both_b = + insert_query_node_with_labels(&engine, &["Person", "Employee"], "both-b", &[], 1.0); + let expected = vec![both_a, person, employee, both_b]; + + let mut query = query_label_filter(&["Person", "Employee"], LabelMatchMode::Any); + query.page = PageRequest { + limit: Some(3), + after: None, + }; + let first = engine.query_node_ids(&query).unwrap(); + assert_eq!(first.items, expected[..3]); + assert_eq!(first.next_cursor, Some(employee)); + + query.page.after = first.next_cursor; + let second = engine.query_node_ids(&query).unwrap(); + assert_eq!(second.items, expected[3..]); + assert_eq!(second.next_cursor, None); + + query.page = PageRequest { + limit: Some(2), + after: None, + }; + engine.reset_query_execution_counters_for_test(); + let nodes = engine.query_nodes(&query).unwrap(); + assert_eq!( + nodes.items.iter().map(|node| node.id).collect::>(), + expected[..2] + ); + assert_eq!(nodes.next_cursor, Some(person)); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.node_record_hydration_reads, 2); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_single_label_cursor_ignores_trailing_stale_postings() { + let (_dir, engine) = query_test_engine(); + let keep_a = + insert_query_node_with_labels(&engine, &["Employee", "Current"], "keep-a", &[], 1.0); + let keep_b = + insert_query_node_with_labels(&engine, &["Employee", "Current"], "keep-b", &[], 1.0); + let stale_count = 20usize; + let stale_ids = (0..stale_count) + .map(|idx| { + insert_query_node_with_labels( + &engine, + &["Employee", "Former"], + &format!("stale-{idx}"), + &[], + 1.0, + ) + }) + .collect::>(); + engine.flush().unwrap(); + + for (idx, expected_id) in stale_ids.iter().copied().enumerate() { + let updated = insert_query_node_with_labels( + &engine, + &["Former"], + &format!("stale-{idx}"), + &[], + 1.0, + ); + assert_eq!(updated, expected_id); + } + + let mut query = query_label_filter(&["Employee"], LabelMatchMode::All); + query.page = PageRequest { + limit: Some(2), + after: None, + }; + let first = engine.query_node_ids(&query).unwrap(); + assert_eq!(first.items, vec![keep_a, keep_b]); + assert_eq!(first.next_cursor, None); + + query.page.after = Some(keep_b); + assert!(engine.query_node_ids(&query).unwrap().items.is_empty()); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_single_label_label_only_small_page_stays_page_shaped() { + let (_dir, engine) = query_test_engine(); + let total = 40usize; + let expected = (0..total) + .map(|idx| insert_query_node(&engine, "Person", &format!("person-{idx}"), &[], 1.0)) + .collect::>(); + engine.flush().unwrap(); + + let mut query = query_label_filter(&["Person"], LabelMatchMode::All); + query.page = PageRequest { + limit: Some(2), + after: None, + }; + + engine.reset_query_execution_counters_for_test(); + let page = engine.query_nodes(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + + assert_eq!( + page.items.iter().map(|node| node.id).collect::>(), + expected[..2] + ); + assert_eq!(page.next_cursor, Some(expected[1])); + assert_eq!(counters.node_record_hydration_reads, 2); + assert!( + counters.node_visibility_meta_reads < total, + "label-only page read should not verify the full posting list" + ); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_any_overlap_streams_and_hydrates_final_page() { + let (_dir, engine) = query_test_engine(); + let total = 40usize; + let expected = (0..total) + .map(|idx| { + insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + &format!("overlap-{idx}"), + &[], + 1.0, + ) + }) + .collect::>(); + engine.flush().unwrap(); + + let mut query = query_label_filter(&["Person", "Employee"], LabelMatchMode::Any); + query.page = PageRequest { + limit: Some(3), + after: None, + }; + + engine.reset_query_execution_counters_for_test(); + let page = engine.query_nodes(&query).unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + + assert_eq!( + page.items.iter().map(|node| node.id).collect::>(), + expected[..3] + ); + assert_eq!(page.next_cursor, Some(expected[2])); + assert_eq!(counters.node_record_hydration_reads, 3); + assert!( + counters.node_visibility_meta_reads < total, + "overlapping Any scan should dedupe raw candidates before page verification" + ); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_multi_label_property_keys_and_stale_membership() { + let (_dir, engine) = query_test_engine(); + let active_both = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "active-both", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let _inactive_both = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "inactive-both", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let _active_person = insert_query_node( + &engine, + "Person", + "active-person", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let _active_employee = insert_query_node( + &engine, + "Employee", + "active-employee", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + + let property_all = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::All)), + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&property_all).unwrap().items, + vec![active_both] + ); + + let single_key = NodeQuery { + label_filter: Some(node_label_filter(&["Person"], LabelMatchMode::All)), + keys: vec!["active-both".to_string()], + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&single_key).unwrap().items, + vec![active_both] + ); + + let ambiguous_key = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::Any)), + keys: vec!["active-both".to_string()], + ..Default::default() + }; + assert!(matches!( + engine.query_node_ids(&ambiguous_key).unwrap_err(), + EngineError::InvalidOperation(message) + if message.contains("keys require exactly one resolved label") + )); + + let stale = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "stale", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + engine.flush().unwrap(); + assert!(engine.remove_node_label(stale, "Employee").unwrap()); + + let employee_query = query_label_filter(&["Employee"], LabelMatchMode::All); + let employee_ids = engine.query_node_ids(&employee_query).unwrap().items; + assert!(!employee_ids.contains(&stale)); + let stale_all = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::All)), + ids: vec![stale], + ..Default::default() + }; + assert!(engine.query_node_ids(&stale_all).unwrap().items.is_empty()); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_multi_label_all_uses_requested_label_property_index() { + let (_dir, engine) = query_test_engine(); + let both = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "both", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let _inactive_both = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "inactive-both", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let person_only = insert_query_node( + &engine, + "Person", + "person-only", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let employee_only = insert_query_node( + &engine, + "Employee", + "employee-only", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + engine.flush().unwrap(); + + let status = engine + .ensure_node_property_index("Employee", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + + let all_query = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::All)), + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&all_query).unwrap().items, vec![both]); + let all_plan = engine.explain_node_query(&all_query).unwrap(); + assert_eq!(all_plan.warnings, Vec::::new()); + assert_plan_input_nodes(&all_plan, vec![QueryPlanNode::PropertyEqualityIndex]); + assert!(all_plan + .notes + .contains(&QueryPlanNote::NodeLabelAllSupersetVerification)); + + let any_query = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::Any)), + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ..Default::default() + }; + let mut expected_any = vec![both, person_only, employee_only]; + expected_any.sort_unstable(); + assert_eq!( + engine.query_node_ids(&any_query).unwrap().items, + expected_any + ); + let any_plan = engine.explain_node_query(&any_query).unwrap(); + assert!( + !plan_contains_node(&any_plan.root, &QueryPlanNode::PropertyEqualityIndex), + "multi-label Any must not use one label's property index as a complete source" + ); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_multi_label_all_uses_requested_label_range_and_timestamp_indexes() { + let (_dir, engine) = query_test_engine(); + let both = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "both", + &[("score", PropValue::Int(10))], + 1.0, + ); + let both_out_of_range = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "both-out-of-range", + &[("score", PropValue::Int(80))], + 1.0, + ); + let person_only = insert_query_node( + &engine, + "Person", + "person-only", + &[("score", PropValue::Int(10))], + 1.0, + ); + let employee_only = insert_query_node( + &engine, + "Employee", + "employee-only", + &[("score", PropValue::Int(10))], + 1.0, + ); + set_query_node_updated_at(&engine, both, 1_000); + set_query_node_updated_at(&engine, both_out_of_range, 2_000); + set_query_node_updated_at(&engine, person_only, 1_000); + set_query_node_updated_at(&engine, employee_only, 1_000); + engine.flush().unwrap(); + + let score = engine + .ensure_node_property_index( + "Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + + let range_query = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::All)), + filter: filter_and![NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(15))), + }], + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&range_query).unwrap().items, + vec![both] + ); + let range_plan = engine.explain_node_query(&range_query).unwrap(); + assert_eq!(range_plan.warnings, Vec::::new()); + assert_plan_input_nodes(&range_plan, vec![QueryPlanNode::PropertyRangeIndex]); + + let timestamp_query = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::All)), + filter: filter_and![NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(900), + upper_ms: Some(1_100), + }], + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(×tamp_query).unwrap().items, + vec![both] + ); + let timestamp_plan = engine.explain_node_query(×tamp_query).unwrap(); + assert_eq!(timestamp_plan.warnings, Vec::::new()); + assert_plan_input_nodes(×tamp_plan, vec![QueryPlanNode::TimestampIndex]); + + engine.close().unwrap(); +} + +#[test] +fn test_node_query_multi_label_all_large_explicit_ids_can_use_property_index() { + let (_dir, engine) = query_test_engine(); + let mut all_ids = Vec::new(); + let mut expected = Vec::new(); + for index in 0..50 { + let selected = index < 3; + let node_id = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + &format!("both-{index}"), + &[("status", PropValue::String(if selected { "target" } else { "other" }.to_string()))], + 1.0, + ); + if selected { + expected.push(node_id); + } + all_ids.push(node_id); + } + engine.flush().unwrap(); + + let status = engine + .ensure_node_property_index("Employee", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + + let query = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::All)), + ids: all_ids.clone(), + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("target".to_string()), + }], + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyEqualityIndex]); + + let tiny_query = NodeQuery { + ids: all_ids[..2].to_vec(), + ..query + }; + let tiny_plan = engine.explain_node_query(&tiny_query).unwrap(); + assert_eq!(tiny_plan.warnings, Vec::::new()); + assert_plan_input_nodes(&tiny_plan, vec![QueryPlanNode::ExplicitIds]); + + engine.close().unwrap(); +} + +#[test] +fn test_graph_pattern_multi_label_filter_all_and_any() { + let (_dir, engine) = query_test_engine(); + let anchor = insert_query_node(&engine, "Company", "anchor", &[], 1.0); + let both = + insert_query_node_with_labels(&engine, &["Person", "Employee"], "both", &[], 1.0); + let person = insert_query_node(&engine, "Person", "person", &[], 1.0); + let employee = insert_query_node(&engine, "Employee", "employee", &[], 1.0); + let both_edge = engine + .upsert_edge(anchor, both, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let person_edge = engine + .upsert_edge(anchor, person, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let employee_edge = engine + .upsert_edge(anchor, employee, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let edges = vec![pattern_edge( + Some("edge"), + "anchor", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )]; + let shorthand_all = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + pattern_node_with_labels("target", &["Person", "Employee"], Vec::new()), + ], + edges.clone(), + ); + assert_eq!( + engine.query_pattern(&shorthand_all).unwrap().matches, + vec![expected_match( + &[("anchor", anchor), ("target", both)], + &[("edge", both_edge)] + )] + ); + let shorthand_plan = engine.explain_pattern_query(&shorthand_all).unwrap(); + assert!(shorthand_plan + .notes + .contains(&QueryPlanNote::NodeLabelAllSupersetVerification)); + assert!(shorthand_plan.public_inputs.node_labels.contains(&QueryPlanPublicName { + alias: Some("target".to_string()), + name: "Person".to_string(), + known: true, + mode: Some(LabelMatchMode::All), + })); + + let explicit_any = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + pattern_node_with_label_filter( + "target", + &["Person", "Employee"], + LabelMatchMode::Any, + Vec::new(), + ), + ], + edges, + ); + let mut any_matches = engine.query_pattern(&explicit_any).unwrap().matches; + any_matches.sort_by_key(|match_| match_.nodes["target"]); + assert_eq!( + any_matches, + vec![ + expected_match( + &[("anchor", anchor), ("target", both)], + &[("edge", both_edge)] + ), + expected_match( + &[("anchor", anchor), ("target", person)], + &[("edge", person_edge)] + ), + expected_match( + &[("anchor", anchor), ("target", employee)], + &[("edge", employee_edge)] + ), + ] + ); + let any_plan = engine.explain_pattern_query(&explicit_any).unwrap(); + assert!(any_plan + .notes + .contains(&QueryPlanNote::NodeLabelAnyFinalVerification)); + assert!(!any_plan + .notes + .contains(&QueryPlanNote::NodeLabelAnyDedupeBeforePagination)); + assert!(any_plan.public_inputs.node_labels.contains(&QueryPlanPublicName { + alias: Some("target".to_string()), + name: "Employee".to_string(), + known: true, + mode: Some(LabelMatchMode::Any), + })); + + engine.close().unwrap(); +} + +#[test] +fn test_graph_pattern_label_only_targets_verify_metadata_without_hydration() { + let (_dir, engine) = query_test_engine(); + let anchor = insert_query_node(&engine, "Company", "anchor", &[], 1.0); + let both = + insert_query_node_with_labels(&engine, &["Person", "Employee"], "both", &[], 1.0); + let person = insert_query_node(&engine, "Person", "person", &[], 1.0); + let employee = insert_query_node(&engine, "Employee", "employee", &[], 1.0); + let both_edge = engine + .upsert_edge(anchor, both, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let person_edge = engine + .upsert_edge(anchor, person, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let employee_edge = engine + .upsert_edge(anchor, employee, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let edges = vec![pattern_edge( + Some("edge"), + "anchor", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )]; + + let all_query = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + pattern_node_with_labels("target", &["Person", "Employee"], Vec::new()), + ], + edges.clone(), + ); + engine.reset_query_execution_counters_for_test(); + assert_eq!( + engine.query_pattern(&all_query).unwrap().matches, + vec![expected_match( + &[("anchor", anchor), ("target", both)], + &[("edge", both_edge)] + )] + ); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.node_record_hydration_reads, 0); + assert!(counters.node_visibility_meta_reads > 0); + + let any_query = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + pattern_node_with_label_filter( + "target", + &["Person", "Employee"], + LabelMatchMode::Any, + Vec::new(), + ), + ], + edges, + ); + engine.reset_query_execution_counters_for_test(); + let mut any_matches = engine.query_pattern(&any_query).unwrap().matches; + any_matches.sort_by_key(|match_| match_.nodes["target"]); + assert_eq!( + any_matches, + vec![ + expected_match( + &[("anchor", anchor), ("target", both)], + &[("edge", both_edge)] + ), + expected_match( + &[("anchor", anchor), ("target", person)], + &[("edge", person_edge)] + ), + expected_match( + &[("anchor", anchor), ("target", employee)], + &[("edge", employee_edge)] + ), + ] + ); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.node_record_hydration_reads, 0); + assert!(counters.node_visibility_meta_reads > 0); + + engine.close().unwrap(); +} + +#[test] +fn test_graph_pattern_property_target_still_hydrates_for_predicate() { + let (_dir, engine) = query_test_engine(); + let anchor = insert_query_node(&engine, "Company", "anchor", &[], 1.0); + let active = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "active", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let inactive = insert_query_node_with_labels( + &engine, + &["Person", "Employee"], + "inactive", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let active_edge = engine + .upsert_edge(anchor, active, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(anchor, inactive, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let query = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + pattern_node_with_labels( + "target", + &["Person", "Employee"], + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ), + ], + vec![pattern_edge( + Some("edge"), + "anchor", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], + ); + + engine.reset_query_execution_counters_for_test(); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[("anchor", anchor), ("target", active)], + &[("edge", active_edge)] + )] + ); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert!(counters.node_record_hydration_reads > 0); + + engine.close().unwrap(); +} + +#[test] +fn test_explain_stale_membership_note_follows_label_posting_source() { + let (_dir, engine) = query_test_engine(); + let both = + insert_query_node_with_labels(&engine, &["Person", "Employee"], "both", &[], 1.0); + insert_query_node(&engine, "Person", "person", &[], 1.0); + insert_query_node(&engine, "Employee", "employee", &[], 1.0); + + let explicit_all = NodeQuery { + label_filter: Some(node_label_filter(&["Person", "Employee"], LabelMatchMode::All)), + ids: vec![both], + ..Default::default() + }; + let explicit_plan = engine.explain_node_query(&explicit_all).unwrap(); + assert!(explicit_plan + .notes + .contains(&QueryPlanNote::NodeLabelAllSupersetVerification)); + assert!(!explicit_plan + .notes + .contains(&QueryPlanNote::StaleNodeLabelMembershipVerification)); + + let key_lookup = NodeQuery { + label_filter: Some(node_label_filter(&["Person"], LabelMatchMode::All)), + keys: vec!["both".to_string()], + ..Default::default() + }; + let key_plan = engine.explain_node_query(&key_lookup).unwrap(); + assert!(!key_plan + .notes + .contains(&QueryPlanNote::StaleNodeLabelMembershipVerification)); + + let any_label_scan = query_label_filter(&["Person", "Employee"], LabelMatchMode::Any); + let any_plan = engine.explain_node_query(&any_label_scan).unwrap(); + assert_plan_input_nodes(&any_plan, vec![QueryPlanNode::NodeLabelAnyIndex]); + assert!(any_plan + .notes + .contains(&QueryPlanNote::NodeLabelAnyDedupeBeforePagination)); + assert!(any_plan + .notes + .contains(&QueryPlanNote::NodeLabelAnyFinalVerification)); + assert!(any_plan + .notes + .contains(&QueryPlanNote::StaleNodeLabelMembershipVerification)); + + let all_label_scan = query_label_filter(&["Person", "Employee"], LabelMatchMode::All); + let all_plan = engine.explain_node_query(&all_label_scan).unwrap(); + assert!(all_plan + .notes + .contains(&QueryPlanNote::NodeLabelAllSupersetVerification)); + assert!(all_plan + .notes + .contains(&QueryPlanNote::StaleNodeLabelMembershipVerification)); + + engine.close().unwrap(); +} + +#[test] +fn test_active_memtable_only_estimates_are_exact_cheap() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + + insert_query_node(&engine, "Person", + "active", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(10)), + ], + 1.0, + ); + insert_query_node(&engine, "Person", + "inactive", + &[ + ("status", PropValue::String("inactive".to_string())), + ("score", PropValue::Int(20)), + ], + 1.0, + ); + + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let label_estimate = published.view.node_label_estimate(1).unwrap(); + assert_eq!(label_estimate.kind, PlannerEstimateKind::ExactCheap); + assert_eq!(label_estimate.known_upper_bound(), Some(2)); + let full_estimate = published.view.full_scan_estimate(); + assert_eq!(full_estimate.kind, PlannerEstimateKind::ExactCheap); + assert_eq!(full_estimate.known_upper_bound(), Some(2)); + let (equality_estimate, followup) = published + .view + .equality_candidate_estimate( + status.index_id, + "status", + &PropValue::String("active".to_string()), + ) + .unwrap(); + assert!(followup.is_none()); + let equality_estimate = equality_estimate.unwrap(); + assert_eq!(equality_estimate.kind, PlannerEstimateKind::ExactCheap); + assert_eq!(equality_estimate.known_upper_bound(), Some(1)); + + let normalized = NormalizedNodeQuery { + single_label_id: Some(1), + label_filter: ResolvedNodeLabelFilter::known( + LabelMatchMode::All, + NodeLabelSet::single(1).unwrap(), + 0, + ), + ids: Vec::new(), + keys: Vec::new(), + filter: NormalizedNodeFilter::AlwaysTrue, + allow_full_scan: false, + page: PageRequest::default(), + warnings: Vec::new(), + }; + let cap_context = published.view.query_cap_context(&normalized).unwrap(); + let mut budget = BooleanPlanningBudget::new(); + let range_probe = published + .view + .range_candidate_probe( + &normalized, + cap_context, + 1, + "score", + Some(&PropertyRangeBound::Included(PropValue::Int(10))), + Some(&PropertyRangeBound::Included(PropValue::Int(10))), + &mut budget, + ) + .unwrap(); + let range_estimate = range_probe.source.unwrap().estimate; + assert_eq!(range_estimate.kind, PlannerEstimateKind::ExactCheap); + assert_eq!(range_estimate.known_upper_bound(), Some(1)); + + let mut budget = BooleanPlanningBudget::new(); + let timestamp_probe = published + .view + .timestamp_candidate_probe(&normalized, cap_context, 1, i64::MIN, i64::MAX, &mut budget) + .unwrap(); + let timestamp_estimate = timestamp_probe.source.unwrap().estimate; + assert_eq!(timestamp_estimate.kind, PlannerEstimateKind::ExactCheap); + assert_eq!(timestamp_estimate.known_upper_bound(), Some(2)); + + drop(published); + drop(_guard); + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_equality_heavy_hitter_and_residual_explain_estimates() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let values: Vec = (0..40).map(|idx| format!("status-{idx:02}")).collect(); + for value in &values { + insert_query_node(&engine, "Person", + value, + &[("status", PropValue::String(value.clone()))], + 1.0, + ); + } + engine.flush().unwrap(); + + let stats_view = engine.planner_stats_view_for_test(); + let rollup = stats_view.equality_index_rollups.get(&info.index_id).unwrap(); + assert_eq!(rollup.total_postings, 40); + assert_eq!( + rollup.top_value_hashes.len(), + crate::planner_stats::PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY + ); + let top_value = values + .iter() + .find(|value| { + rollup + .top_value_hashes + .contains_key(&hash_prop_value(&PropValue::String((*value).clone()))) + }) + .unwrap() + .clone(); + let residual_value = values + .iter() + .find(|value| { + !rollup + .top_value_hashes + .contains_key(&hash_prop_value(&PropValue::String((*value).clone()))) + }) + .unwrap() + .clone(); + drop(stats_view); + + let top_query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String(top_value), + }], + false, + ); + assert_eq!(engine.query_node_ids(&top_query).unwrap().items.len(), 1); + let top_plan = engine.explain_node_query(&top_query).unwrap(); + assert_eq!(top_plan.estimated_candidates, Some(1)); + assert_plan_input_nodes(&top_plan, vec![QueryPlanNode::PropertyEqualityIndex]); + + let residual_query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String(residual_value), + }], + false, + ); + assert_eq!(engine.query_node_ids(&residual_query).unwrap().items.len(), 1); + let residual_plan = engine.explain_node_query(&residual_query).unwrap(); + assert_eq!(residual_plan.estimated_candidates, Some(1)); + assert_plan_input_nodes(&residual_plan, vec![QueryPlanNode::PropertyEqualityIndex]); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_rare_residual_equality_beats_broad_label_source() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let value_count = + QUERY_RANGE_CANDIDATE_CAP + crate::planner_stats::PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY + 1; + let values: Vec = (0..value_count) + .map(|idx| format!("rare-status-{idx:04}")) + .collect(); + for value in &values { + insert_query_node(&engine, "Person", + value, + &[("status", PropValue::String(value.clone()))], + 1.0, + ); + } + engine.flush().unwrap(); + + let stats_view = engine.planner_stats_view_for_test(); + let rollup = stats_view.equality_index_rollups.get(&info.index_id).unwrap(); + let residual_value = values + .iter() + .find(|value| { + !rollup + .top_value_hashes + .contains_key(&hash_prop_value(&PropValue::String((*value).clone()))) + }) + .unwrap() + .clone(); + assert!(rollup.total_postings > QUERY_RANGE_CANDIDATE_CAP as u64); + drop(stats_view); + + let residual_query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String(residual_value), + }], + false, + ); + assert_eq!(engine.query_node_ids(&residual_query).unwrap().items.len(), 1); + let residual_plan = engine.explain_node_query(&residual_query).unwrap(); + assert_eq!(residual_plan.warnings, Vec::::new()); + assert_eq!(residual_plan.estimated_candidates, Some(1)); + assert_plan_input_nodes( + &residual_plan, + vec![QueryPlanNode::PropertyEqualityIndex], + ); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_broad_heavy_hitter_equality_uses_cheaper_label_scan() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let inputs: Vec<_> = (0..=QUERY_RANGE_CANDIDATE_CAP) + .map(|index| NodeInput { + labels: vec!["Person".to_string()], + key: format!("broad-heavy-{index}"), + props: query_test_props(&[("status", PropValue::String("broad".to_string()))]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + let all_ids = engine.batch_upsert_nodes(inputs).unwrap(); + engine.flush().unwrap(); + + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("broad".to_string()), + }], + false, + ); + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &all_ids, &query) + ); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!( + plan.warnings, + vec![ + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::CandidateCapExceeded, + QueryPlanWarning::VerifyOnlyFilter, + ] + ); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackNodeLabelScan]); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_range_and_timestamp_explain_use_no_planning_probe() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + + let inputs: Vec<_> = (0..32) + .map(|index| NodeInput { + labels: vec!["Person".to_string()], + key: format!("stats-probe-{index}"), + props: query_test_props(&[("score", PropValue::Int(index))]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + engine.batch_upsert_nodes(inputs).unwrap(); + engine.flush().unwrap(); + + engine.reset_query_planning_probe_counters_for_test(); + let range_query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(12))), + }], + false, + ); + let range_plan = engine.explain_node_query(&range_query).unwrap(); + assert_plan_input_nodes(&range_plan, vec![QueryPlanNode::PropertyRangeIndex]); + assert_eq!( + engine.query_planning_probe_snapshot_for_test().range, + 0, + "stats-covered range explain must not materialize planning candidates" + ); + + let timestamp_query = query_ids(Some("Person"), + vec![NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(i64::MIN), + upper_ms: Some(i64::MAX), + }], + false, + ); + let timestamp_plan = engine.explain_node_query(×tamp_query).unwrap(); + assert_plan_input_nodes(×tamp_plan, vec![QueryPlanNode::TimestampIndex]); + assert_eq!( + engine.query_planning_probe_snapshot_for_test().timestamp, + 0, + "stats-covered timestamp explain must not materialize planning candidates" + ); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_range_and_timestamp_mixed_coverage_probe_uncovered_segments() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let all_ids; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + + let seg1 = [ + ("covered-a", 10, 1_000), + ("covered-b", 20, 1_100), + ("covered-c", 100, 9_000), + ]; + let mut ids = Vec::new(); + for (key, score, updated_at) in seg1 { + let node_id = insert_query_node(&engine, "Person", + key, + &[("score", PropValue::Int(score))], + 1.0, + ); + set_query_node_updated_at(&engine, node_id, updated_at); + ids.push(node_id); + } + engine.flush().unwrap(); + + let seg2 = [ + ("uncovered-a", 15, 1_200), + ("uncovered-b", 25, 1_300), + ("uncovered-c", 200, 10_000), + ]; + for (key, score, updated_at) in seg2 { + let node_id = insert_query_node(&engine, "Person", + key, + &[("score", PropValue::Int(score))], + 1.0, + ); + set_query_node_updated_at(&engine, node_id, updated_at); + ids.push(node_id); + } + engine.flush().unwrap(); + all_ids = ids; + engine.close().unwrap(); + } + + let stats_path = crate::segment_writer::segment_dir(&db_path, 2) + .join(crate::planner_stats::PLANNER_STATS_FILENAME); + std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let stats_view = reopened.planner_stats_view_for_test(); + assert_eq!(stats_view.available_segment_stats, 1); + assert_eq!(stats_view.unavailable_segment_stats, 1); + assert_eq!(stats_view.timestamp_coverage.covered_segment_ids, vec![1]); + let range_index_id = *stats_view.range_index_rollups.keys().next().unwrap(); + assert_eq!( + stats_view + .range_index_rollups + .get(&range_index_id) + .unwrap() + .coverage + .covered_segment_ids, + vec![1] + ); + drop(stats_view); + + let range_query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(25))), + }], + false, + ); + reopened.reset_query_planning_probe_counters_for_test(); + let range_plan = reopened.explain_node_query(&range_query).unwrap(); + assert_eq!(range_plan.warnings, Vec::::new()); + assert_eq!(range_plan.estimated_candidates, Some(5)); + assert_plan_input_nodes(&range_plan, vec![QueryPlanNode::PropertyRangeIndex]); + assert_eq!(reopened.query_planning_probe_snapshot_for_test().range, 1); + assert_eq!( + reopened.query_node_ids(&range_query).unwrap().items, + oracle_query_ids(&reopened, &all_ids, &range_query) + ); + + let timestamp_query = query_ids(Some("Person"), + vec![NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(1_000), + upper_ms: Some(1_300), + }], + false, + ); + reopened.reset_query_planning_probe_counters_for_test(); + let timestamp_plan = reopened.explain_node_query(×tamp_query).unwrap(); + assert_eq!(timestamp_plan.warnings, Vec::::new()); + assert_eq!(timestamp_plan.estimated_candidates, Some(5)); + assert_plan_input_nodes(×tamp_plan, vec![QueryPlanNode::TimestampIndex]); + assert_eq!( + reopened.query_planning_probe_snapshot_for_test().timestamp, + 1 + ); + assert_eq!( + reopened.query_node_ids(×tamp_query).unwrap().items, + oracle_query_ids(&reopened, &all_ids, ×tamp_query) + ); + + reopened.close().unwrap(); +} + +#[test] +fn test_planner_stats_adaptive_cap_allows_high_confidence_range_above_default() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + + let selected_count = + crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP + 256; + let total_count = selected_count + 1024; + let inputs: Vec<_> = (0..total_count) + .map(|index| NodeInput { + labels: vec!["Person".to_string()], + key: format!("adaptive-range-{index}"), + props: query_test_props(&[("score", PropValue::Int(index as i64))]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + let all_ids = engine.batch_upsert_nodes(inputs).unwrap(); + engine.flush().unwrap(); + + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), + upper: Some(PropertyRangeBound::Included(PropValue::Int( + selected_count as i64 - 1, + ))), + }), + page: PageRequest { + limit: Some(16), + after: None, + }, + ..Default::default() + }; + + let expected: Vec<_> = oracle_query_ids(&engine, &all_ids, &query) + .into_iter() + .take(16) + .collect(); + assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyRangeIndex]); + assert!( + plan.estimated_candidates + > Some(crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP as u64) + ); + + engine.close().unwrap(); +} + +#[test] +fn test_direct_read_apis_are_unchanged_with_planner_stats_sidecars() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + wait_for_published_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + wait_for_published_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + + let inputs = vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "direct-a".to_string(), + props: query_test_props(&[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(10)), + ]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "direct-b".to_string(), + props: query_test_props(&[ + ("status", PropValue::String("inactive".to_string())), + ("score", PropValue::Int(20)), + ]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Company".to_string()], + key: "direct-c".to_string(), + props: query_test_props(&[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(10)), + ]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + ]; + let ids = engine.batch_upsert_nodes(inputs).unwrap(); + engine.flush().unwrap(); + + assert_eq!( + engine + .find_nodes("Person", "status", &PropValue::String("active".to_string())) + .unwrap(), + vec![ids[0]] + ); + assert_eq!( + engine + .find_nodes_range( + "Person", + "score", + Some(&PropertyRangeBound::Included(PropValue::Int(10))), + Some(&PropertyRangeBound::Included(PropValue::Int(20))), + ) + .unwrap(), + vec![ids[0], ids[1]] + ); + assert_eq!( + engine + .find_nodes_by_time_range("Person", i64::MIN, i64::MAX) + .unwrap(), + vec![ids[0], ids[1]] + ); + assert_eq!(engine.nodes_by_labels("Person").unwrap(), vec![ids[0], ids[1]]); + + engine.close().unwrap(); +} + +#[test] +fn test_planner_stats_mixed_segment_fallback_estimates_once() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + for idx in 0..2 { + insert_query_node(&engine, "Person", &format!("covered-{idx}"), &[], 1.0); + } + engine.flush().unwrap(); + for idx in 0..3 { + insert_query_node(&engine, "Person", &format!("fallback-{idx}"), &[], 1.0); + } + engine.flush().unwrap(); + engine.close().unwrap(); + } + + let stats_path = crate::segment_writer::segment_dir(&db_path, 2) + .join(crate::planner_stats::PLANNER_STATS_FILENAME); + std::fs::write(&stats_path, b"corrupt planner stats").unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let stats_view = reopened.planner_stats_view_for_test(); + assert_eq!(stats_view.segment_count, 2); + assert_eq!(stats_view.available_segment_stats, 1); + assert_eq!(stats_view.unavailable_segment_stats, 1); + assert_eq!(stats_view.node_label_count(1), 2); + assert_eq!(stats_view.node_label_coverage.covered_segment_ids, vec![1]); + drop(stats_view); + { + let (_guard, published) = reopened.runtime.published_snapshot().unwrap(); + let estimate = published.view.node_label_estimate(1).unwrap(); + assert_eq!(estimate.kind, PlannerEstimateKind::UpperBound); + assert_eq!(estimate.known_upper_bound(), Some(5)); + } + + let query = query_ids(Some("Person"), Vec::new(), false); + assert_eq!(reopened.query_node_ids(&query).unwrap().items.len(), 5); + let plan = reopened.explain_node_query(&query).unwrap(); + assert_eq!(plan.estimated_candidates, Some(5)); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::NodeLabelIndex]); + + reopened.close().unwrap(); +} + +#[test] +fn test_planner_estimate_sort_prefers_cheaper_count_before_source_rank() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let mut candidates = vec![ + NodePhysicalPlan::source(PlannedNodeCandidateSource::property_equality_index( + 1, + 1, + "status", + &PropValue::String("active".to_string()), + PlannerEstimate::stats_estimated( + 100, + EstimateConfidence::High, + StalePostingRisk::Low, + ), + )), + NodePhysicalPlan::source(PlannedNodeCandidateSource::fallback_node_label_scan( + 1, + PlannerEstimate::upper_bound(10), + )), + ]; + published + .view + .sort_physical_plans_by_selectivity(&mut candidates); + assert_eq!(candidates[0].plan_node(), QueryPlanNode::FallbackNodeLabelScan); + } + + engine.close().unwrap(); +} + +#[test] +fn test_query_validation_and_explain_reject_label_less_scan_without_opt_in() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let query = query_ids( + None, + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + false, + ); + + assert!(matches!( + engine.query_node_ids(&query).unwrap_err(), + EngineError::InvalidOperation(_) + )); + assert!(matches!( + engine.explain_node_query(&query).unwrap_err(), + EngineError::InvalidOperation(_) + )); + + let key_query = NodeQuery { + keys: vec!["alice".to_string()], + ..Default::default() + }; + assert!(matches!( + engine.query_node_ids(&key_query).unwrap_err(), + EngineError::InvalidOperation(_) + )); + + let empty_range_query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: None, + upper: None, + }], + false, + ); + assert!(matches!( + engine.query_node_ids(&empty_range_query).unwrap_err(), + EngineError::InvalidOperation(_) + )); + + let empty_time_query = query_ids(Some("Person"), + vec![NodeFilterExpr::UpdatedAtRange { + lower_ms: None, + upper_ms: None, + }], + false, + ); + assert!(matches!( + engine.explain_node_query(&empty_time_query).unwrap_err(), + EngineError::InvalidOperation(_) + )); + + let inverted_time_query = query_ids(Some("Person"), + vec![NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(200), + upper_ms: Some(100), + }], + false, + ); + assert!(engine + .query_node_ids(&inverted_time_query) + .unwrap() + .items + .is_empty()); + assert!(matches!( + explain_input_node(&engine.explain_node_query(&inverted_time_query).unwrap()), + QueryPlanNode::EmptyResult + )); + + engine.close().unwrap(); +} + +#[test] +fn test_query_normalization_expands_open_updated_at_bounds() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + engine.ensure_node_label("Person").unwrap(); + + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + + let lower_open_query = query_ids(Some("Person"), + vec![NodeFilterExpr::UpdatedAtRange { + lower_ms: None, + upper_ms: Some(123), + }], + false, + ); + let normalized = published + .view + .normalize_node_query(&lower_open_query) + .unwrap(); + match normalized.filter { + NormalizedNodeFilter::UpdatedAtRange { lower_ms, upper_ms } => { + assert_eq!(lower_ms, i64::MIN); + assert_eq!(upper_ms, 123); + } + _ => panic!("expected normalized updated-at range"), + } + + let upper_open_query = query_ids(Some("Person"), + vec![NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(456), + upper_ms: None, + }], + false, + ); + let normalized = published + .view + .normalize_node_query(&upper_open_query) + .unwrap(); + match normalized.filter { + NormalizedNodeFilter::UpdatedAtRange { lower_ms, upper_ms } => { + assert_eq!(lower_ms, 456); + assert_eq!(upper_ms, i64::MAX); + } + _ => panic!("expected normalized updated-at range"), + } + } + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_validation_and_empty_result_without_scan_opt_in() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + for filter in [ + NodeFilterExpr::And(Vec::new()), + NodeFilterExpr::Or(Vec::new()), + NodeFilterExpr::PropertyEquals { + key: String::new(), + value: PropValue::String("x".to_string()), + }, + NodeFilterExpr::PropertyIn { + key: "status".to_string(), + values: Vec::new(), + }, + ] { + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(filter), + ..Default::default() + }; + assert!(matches!( + engine.explain_node_query(&query).unwrap_err(), + EngineError::InvalidOperation(_) + )); + } + + let always_false = NodeQuery { + filter: filter_and![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("inactive".to_string()), + }, + ], + ..Default::default() + }; + assert!(engine.query_node_ids(&always_false).unwrap().items.is_empty()); + let plan = engine.explain_node_query(&always_false).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert!(matches!(explain_input_node(&plan), QueryPlanNode::EmptyResult)); + + let always_true_without_anchor = NodeQuery { + filter: Some(NodeFilterExpr::Not(Box::new( + always_false.filter.clone().unwrap(), + ))), + ..Default::default() + }; + assert!(matches!( + engine.query_node_ids(&always_true_without_anchor).unwrap_err(), + EngineError::InvalidOperation(_) + )); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_exists_missing_not_and_or_verifier_semantics() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let tagged_null = insert_query_node(&engine, "Person", + "tagged-null", + &[ + ("status", PropValue::String("active".to_string())), + ("tag", PropValue::Null), + ], + 1.0, + ); + let missing_tag = insert_query_node(&engine, "Person", + "missing-tag", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let tagged_trial = insert_query_node(&engine, "Person", + "tagged-trial", + &[ + ("status", PropValue::String("trial".to_string())), + ("tag", PropValue::String("present".to_string())), + ], + 1.0, + ); + + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::And(vec![ + NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("trial".to_string()), + }, + ]), + NodeFilterExpr::PropertyExists { + key: "tag".to_string(), + }, + NodeFilterExpr::Not(Box::new(NodeFilterExpr::PropertyMissing { + key: "tag".to_string(), + })), + ])), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + vec![tagged_null, tagged_trial] + ); + let plan = engine.explain_node_query(&query).unwrap(); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackNodeLabelScan]); + + let missing_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyMissing { + key: "tag".to_string(), + }), + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&missing_query).unwrap().items, vec![missing_tag]); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_in_dedupes_by_canonical_value_and_uses_union() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut map_value = BTreeMap::new(); + map_value.insert("x".to_string(), PropValue::Int(1)); + let map_value = PropValue::Map(map_value); + let array_value = PropValue::Array(vec![PropValue::Int(1), PropValue::UInt(2)]); + + let null_id = insert_query_node(&engine, "Person", "null", &[("kind", PropValue::Null)], 1.0); + let int_id = insert_query_node(&engine, "Person", "int", &[("kind", PropValue::Int(1))], 1.0); + let uint_id = insert_query_node(&engine, "Person", "uint", &[("kind", PropValue::UInt(1))], 1.0); + let array_id = insert_query_node(&engine, "Person", "array", &[("kind", array_value.clone())], 1.0); + let map_id = insert_query_node(&engine, "Person", "map", &[("kind", map_value.clone())], 1.0); + let neg_zero_id = insert_query_node(&engine, "Person", + "neg-zero-kind", + &[("kind", PropValue::Float(-0.0))], + 1.0, + ); + let pos_zero_id = insert_query_node(&engine, "Person", + "pos-zero-kind", + &[("kind", PropValue::Float(0.0))], + 1.0, + ); + let _missing = insert_query_node(&engine, "Person", "missing", &[], 1.0); + + let index = engine + .ensure_node_property_index("Person", "kind", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); + + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyIn { + key: "kind".to_string(), + values: vec![ + PropValue::Null, + PropValue::Null, + PropValue::UInt(1), + array_value.clone(), + map_value.clone(), + map_value.clone(), + ], + }), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + vec![null_id, uint_id, array_id, map_id] + ); + let plan = engine.explain_node_query(&query).unwrap(); + assert!(!plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes( + &plan, + vec![QueryPlanNode::Union { + inputs: vec![ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + }], + ); + + let int_only = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyIn { + key: "kind".to_string(), + values: vec![PropValue::Int(1)], + }), + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&int_only).unwrap().items, vec![int_id]); + let int_only_plan = engine.explain_node_query(&int_only).unwrap(); + assert!(!int_only_plan + .warnings + .contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes(&int_only_plan, vec![QueryPlanNode::PropertyEqualityIndex]); + + let signed_zero_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyIn { + key: "kind".to_string(), + values: vec![PropValue::Float(-0.0), PropValue::Float(0.0)], + }), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&signed_zero_query).unwrap().items, + vec![neg_zero_id, pos_zero_id] + ); + assert_plan_input_nodes( + &engine.explain_node_query(&signed_zero_query).unwrap(), + vec![QueryPlanNode::Union { + inputs: vec![ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + }], + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_large_verify_only_in_matches_verifier_semantics() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let string_match = insert_query_node(&engine, "Person", + "token-string-match", + &[("token", PropValue::String("value-63".to_string()))], + 1.0, + ); + let signed_zero_match = insert_query_node(&engine, "Person", + "token-signed-zero-match", + &[("token", PropValue::Float(0.0))], + 1.0, + ); + let nested_zero_match = insert_query_node(&engine, "Person", + "token-nested-zero-match", + &[( + "token", + PropValue::Array(vec![PropValue::Float(0.0)]), + )], + 1.0, + ); + insert_query_node(&engine, "Person", + "token-nan-not-match", + &[("token", PropValue::Float(f64::NAN))], + 1.0, + ); + insert_query_node(&engine, "Person", + "token-miss", + &[("token", PropValue::String("missing".to_string()))], + 1.0, + ); + + let mut values: Vec = (0..64) + .map(|index| PropValue::String(format!("value-{index}"))) + .collect(); + values.push(PropValue::Float(-0.0)); + values.push(PropValue::Array(vec![PropValue::Float(-0.0)])); + values.push(PropValue::Float(f64::NAN)); + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyIn { + key: "token".to_string(), + values, + }), + ..Default::default() + }; + + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + vec![string_match, signed_zero_match, nested_zero_match] + ); + let plan = engine.explain_node_query(&query).unwrap(); + assert!(plan.warnings.contains(&QueryPlanWarning::MissingReadyIndex)); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackNodeLabelScan]); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_equality_contradictions_match_verifier_semantics() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let neg_zero = insert_query_node(&engine, "Person", + "neg-zero", + &[("temperature", PropValue::Float(-0.0))], + 1.0, + ); + let pos_zero = insert_query_node(&engine, "Person", + "pos-zero", + &[("temperature", PropValue::Float(0.0))], + 1.0, + ); + + let query = NodeQuery { + ids: vec![neg_zero, pos_zero], + filter: Some(NodeFilterExpr::And(vec![ + NodeFilterExpr::PropertyEquals { + key: "temperature".to_string(), + value: PropValue::Float(-0.0), + }, + NodeFilterExpr::PropertyEquals { + key: "temperature".to_string(), + value: PropValue::Float(0.0), + }, + ])), + ..Default::default() + }; + + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + vec![neg_zero, pos_zero] + ); + assert_plan_input_nodes( + &engine.explain_node_query(&query).unwrap(), + vec![QueryPlanNode::ExplicitIds], + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_indexed_float_signed_zero_equality_matches_verifier_semantics() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let neg_zero = insert_query_node(&engine, "Person", + "indexed-neg-zero", + &[("temperature", PropValue::Float(-0.0))], + 1.0, + ); + let pos_zero = insert_query_node(&engine, "Person", + "indexed-pos-zero", + &[("temperature", PropValue::Float(0.0))], + 1.0, + ); + insert_query_node(&engine, "Person", + "indexed-one", + &[("temperature", PropValue::Float(1.0))], + 1.0, + ); + engine.flush().unwrap(); + + let index = engine + .ensure_node_property_index("Person", "temperature", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); + + let neg_zero_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyEquals { + key: "temperature".to_string(), + value: PropValue::Float(-0.0), + }), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&neg_zero_query).unwrap().items, + vec![neg_zero, pos_zero] + ); + assert_plan_input_nodes( + &engine.explain_node_query(&neg_zero_query).unwrap(), + vec![QueryPlanNode::PropertyEqualityIndex], + ); + + let pos_zero_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyEquals { + key: "temperature".to_string(), + value: PropValue::Float(0.0), + }), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&pos_zero_query).unwrap().items, + vec![neg_zero, pos_zero] + ); + assert_plan_input_nodes( + &engine.explain_node_query(&pos_zero_query).unwrap(), + vec![QueryPlanNode::PropertyEqualityIndex], + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_or_and_in_extract_complete_index_candidates() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let active = insert_query_node(&engine, "Person", + "active", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let trial = insert_query_node(&engine, "Person", + "trial", + &[("status", PropValue::String("trial".to_string()))], + 1.0, + ); + let _inactive = insert_query_node(&engine, "Person", + "inactive", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let index = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); + + let or_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("trial".to_string()), + }, + ])), + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&or_query).unwrap().items, vec![active, trial]); + let or_plan = engine.explain_node_query(&or_query).unwrap(); + assert!(!or_plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes( + &or_plan, + vec![QueryPlanNode::Union { + inputs: vec![ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + }], + ); + + let singleton_or_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::Or(vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }])), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&singleton_or_query).unwrap().items, + vec![active] + ); + let singleton_or_plan = engine.explain_node_query(&singleton_or_query).unwrap(); + assert!(!singleton_or_plan + .warnings + .contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes( + &singleton_or_plan, + vec![QueryPlanNode::PropertyEqualityIndex], + ); + + let double_not_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::Not(Box::new(NodeFilterExpr::Not(Box::new( + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + ))))), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&double_not_query).unwrap().items, + vec![active] + ); + let double_not_plan = engine.explain_node_query(&double_not_query).unwrap(); + assert!(!double_not_plan + .warnings + .contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes(&double_not_plan, vec![QueryPlanNode::PropertyEqualityIndex]); + + let in_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyIn { + key: "status".to_string(), + values: vec![ + PropValue::String("active".to_string()), + PropValue::String("trial".to_string()), + ], + }), + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&in_query).unwrap().items, vec![active, trial]); + let in_plan = engine.explain_node_query(&in_query).unwrap(); + assert!(!in_plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_plan_input_nodes( + &in_plan, + vec![QueryPlanNode::Union { + inputs: vec![ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + }], + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_or_in_union_final_verification_and_pagination() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let stale_active = insert_query_node(&engine, "Person", + "stale-active", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let deleted_trial = insert_query_node(&engine, "Person", + "deleted-trial", + &[("status", PropValue::String("trial".to_string()))], + 1.0, + ); + let active = insert_query_node(&engine, "Person", + "active", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let trial = insert_query_node(&engine, "Person", + "trial", + &[("status", PropValue::String("trial".to_string()))], + 1.0, + ); + for index in 0..3 { + insert_query_node(&engine, "Person", + &format!("inactive-{index}"), + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + } + engine.flush().unwrap(); + let index = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); + + let updated = engine + .upsert_node( + "Person", + "stale-active", + UpsertNodeOptions { + props: query_test_props(&[( + "status", + PropValue::String("inactive".to_string()), + )]), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(updated, stale_active); + engine.delete_node(deleted_trial).unwrap(); + + let or_filter = NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("trial".to_string()), + }, + ]); + let mut or_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(or_filter.clone()), + page: PageRequest { + limit: Some(1), + after: None, + }, + ..Default::default() + }; + + let first = engine.query_node_ids(&or_query).unwrap(); + assert_eq!(first.items, vec![active]); + assert_eq!(first.next_cursor, Some(active)); + or_query.page.after = first.next_cursor; + let second = engine.query_node_ids(&or_query).unwrap(); + assert_eq!(second.items, vec![trial]); + assert_eq!(second.next_cursor, None); + + or_query.page = PageRequest::default(); + assert_eq!(engine.query_node_ids(&or_query).unwrap().items, vec![active, trial]); + assert_plan_input_nodes( + &engine.explain_node_query(&or_query).unwrap(), + vec![QueryPlanNode::Union { + inputs: vec![ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + }], + ); + + let in_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::PropertyIn { + key: "status".to_string(), + values: vec![ + PropValue::String("trial".to_string()), + PropValue::String("active".to_string()), + PropValue::String("active".to_string()), + ], + }), + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&in_query).unwrap().items, vec![active, trial]); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_and_of_or_intersects_range() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let active_high = insert_query_node(&engine, "Person", + "active-high", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(20)), + ], + 1.0, + ); + let trial_high = insert_query_node(&engine, "Person", + "trial-high", + &[ + ("status", PropValue::String("trial".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ); + let _active_low = insert_query_node(&engine, "Person", + "active-low", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(1)), + ], + 1.0, + ); + let _inactive_high = insert_query_node(&engine, "Person", + "inactive-high", + &[ + ("status", PropValue::String("inactive".to_string())), + ("score", PropValue::Int(40)), + ], + 1.0, + ); + engine.flush().unwrap(); + let status_index = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let score_index = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, status_index.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(&engine, score_index.index_id, SecondaryIndexState::Ready); + + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::And(vec![ + NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("trial".to_string()), + }, + ]), + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), + upper: None, + }, + ])), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + vec![active_high, trial_high] + ); + assert_plan_includes_input_nodes( + &engine.explain_node_query(&query).unwrap(), + &[ + QueryPlanNode::Union { + inputs: vec![ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + }, + QueryPlanNode::PropertyRangeIndex, + ], + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_fallback_budget_and_empty_plan_edges() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let active = insert_query_node(&engine, "Person", + "active", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(1)), + ], + 1.0, + ); + let scored = insert_query_node(&engine, "Person", + "scored", + &[ + ("status", PropValue::String("inactive".to_string())), + ("score", PropValue::Int(50)), + ], + 1.0, + ); + for index in 0..8 { + insert_query_node(&engine, "Person", + &format!("filler-{index}"), + &[ + ("status", PropValue::String(format!("v{index}"))), + ("score", PropValue::Int(index)), + ], + 1.0, + ); + } + engine.flush().unwrap(); + let status_index = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, status_index.index_id, SecondaryIndexState::Ready); + + let impossible = NodeQuery { + filter: Some(NodeFilterExpr::And(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("trial".to_string()), + }, + ])), + ..Default::default() + }; + assert!(engine.query_node_ids(&impossible).unwrap().items.is_empty()); + assert_plan_input_nodes( + &engine.explain_node_query(&impossible).unwrap(), + vec![QueryPlanNode::EmptyResult], + ); + + let always_true_requires_anchor = NodeQuery { + filter: Some(NodeFilterExpr::Not(Box::new(NodeFilterExpr::And(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("trial".to_string()), + }, + ])))), + ..Default::default() + }; + assert!(matches!( + engine.query_node_ids(&always_true_requires_anchor), + Err(EngineError::InvalidOperation(_)) + )); + + let missing_index_or = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(40))), + upper: None, + }, + ])), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&missing_index_or).unwrap().items, + vec![active, scored] + ); + let missing_plan = engine.explain_node_query(&missing_index_or).unwrap(); + assert_eq!( + missing_plan.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + QueryPlanWarning::BooleanBranchFallback, + ] + ); + assert_plan_input_nodes(&missing_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); + + let budget_or = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::Or( + (0..=MAX_BOOLEAN_UNION_INPUTS) + .map(|index| NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String(format!("v{index}")), + }) + .collect(), + )), + ..Default::default() + }; + let budget_plan = engine.explain_node_query(&budget_or).unwrap(); + assert_eq!( + budget_plan.warnings, + vec![ + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + QueryPlanWarning::BooleanBranchFallback, + QueryPlanWarning::PlanningProbeBudgetExceeded, + ] + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_or_unknown_branch_falls_back_without_partial_union() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let indexed = insert_query_node(&engine, "Person", + "indexed", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let missing_index = insert_query_node(&engine, "Person", + "missing-index", + &[("score", PropValue::Int(10))], + 1.0, + ); + let other = insert_query_node(&engine, "Person", "other", &[], 1.0); + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(15))), + }, + ])), + ..Default::default() + }; + + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[indexed, missing_index, other], &query) + ); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!( + plan.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + QueryPlanWarning::BooleanBranchFallback, + ] + ); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackNodeLabelScan]); + + engine.close().unwrap(); +} + +#[test] +fn test_query_filter_verify_only_uses_expected_legal_universe() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let label1_inputs: Vec = (0..QUERY_RANGE_CANDIDATE_CAP + 8) + .map(|index| NodeInput { + labels: vec!["Person".to_string()], + key: format!("label1-archived-{index}"), + props: query_test_props(&[("archived", PropValue::Bool(true))]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + let label1_ids = engine.batch_upsert_nodes(label1_inputs).unwrap(); + let label1_archived = label1_ids[0]; + let label1_missing = insert_query_node(&engine, "Person", "label1-missing", &[], 1.0); + let small_missing = insert_query_node(&engine, "Company", "small-missing", &[], 1.0); + let small_archived = insert_query_node(&engine, "Company", + "small-archived", + &[("archived", PropValue::Bool(true))], + 1.0, + ); + let active_tag = insert_query_node(&engine, "Article", + "active-tag", + &[ + ("status", PropValue::String("active".to_string())), + ("tag", PropValue::String("present".to_string())), + ], + 1.0, + ); + let active_missing = insert_query_node(&engine, "Article", + "active-missing", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let inactive_tag = insert_query_node(&engine, "Article", + "inactive-tag", + &[ + ("status", PropValue::String("inactive".to_string())), + ("tag", PropValue::String("present".to_string())), + ], + 1.0, + ); + + let status_index = engine + .ensure_node_property_index("Article", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, status_index.index_id, SecondaryIndexState::Ready); + + let mut huge_ids = label1_ids.clone(); + huge_ids.push(label1_missing); + huge_ids.push(small_missing); + huge_ids.push(small_archived); + let label_small_query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Company".to_string()], mode: LabelMatchMode::All }), + ids: huge_ids, + filter: Some(NodeFilterExpr::PropertyMissing { + key: "archived".to_string(), + }), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&label_small_query).unwrap().items, + vec![small_missing] + ); + let label_small_plan = engine.explain_node_query(&label_small_query).unwrap(); + assert_eq!( + label_small_plan.warnings, + vec![ + QueryPlanWarning::UsingFallbackScan, QueryPlanWarning::VerifyOnlyFilter, ] ); - assert_plan_input_nodes(&type_small_plan, vec![QueryPlanNode::FallbackTypeScan]); + assert_plan_input_nodes(&label_small_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); let ids_small_query = NodeQuery { - type_id: Some(1), - ids: vec![type1_missing, type1_archived], + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: vec![label1_missing, label1_archived], filter: Some(NodeFilterExpr::PropertyMissing { key: "archived".to_string(), }), @@ -2668,7 +7246,7 @@ fn test_query_filter_verify_only_uses_expected_legal_universe() { }; assert_eq!( engine.query_node_ids(&ids_small_query).unwrap().items, - vec![type1_missing] + vec![label1_missing] ); let ids_small_plan = engine.explain_node_query(&ids_small_query).unwrap(); assert_eq!( @@ -2678,7 +7256,7 @@ fn test_query_filter_verify_only_uses_expected_legal_universe() { assert_plan_input_nodes(&ids_small_plan, vec![QueryPlanNode::ExplicitIds]); let equality_plus_not_missing = NodeQuery { - type_id: Some(3), + label_filter: Some(NodeLabelFilter { labels: vec!["Article".to_string()], mode: LabelMatchMode::All }), filter: Some(NodeFilterExpr::And(vec![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -2710,7 +7288,7 @@ fn test_query_filter_verify_only_uses_expected_legal_universe() { ); let or_missing = NodeQuery { - type_id: Some(3), + label_filter: Some(NodeLabelFilter { labels: vec!["Article".to_string()], mode: LabelMatchMode::All }), filter: Some(NodeFilterExpr::Or(vec![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -2735,11 +7313,11 @@ fn test_query_filter_verify_only_uses_expected_legal_universe() { QueryPlanWarning::BooleanBranchFallback, ] ); - assert_plan_input_nodes(&or_missing_plan, vec![QueryPlanNode::FallbackTypeScan]); + assert_plan_input_nodes(&or_missing_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); assert!(!engine .query_node_ids(&NodeQuery { - type_id: Some(3), + label_filter: Some(NodeLabelFilter { labels: vec!["Article".to_string()], mode: LabelMatchMode::All }), ids: vec![inactive_tag], filter: Some(NodeFilterExpr::PropertyMissing { key: "tag".to_string(), @@ -2761,7 +7339,7 @@ fn test_query_filter_range_and_timestamp_probe_budget_overflow_is_cumulative() { let inputs: Vec = (0..QUERY_RANGE_CANDIDATE_CAP + 8) .map(|index| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("budget-{index}"), props: query_test_props(&[("score", PropValue::Int(index as i64))]), weight: 1.0, @@ -2769,11 +7347,10 @@ fn test_query_filter_range_and_timestamp_probe_budget_overflow_is_cumulative() { sparse_vector: None, }) .collect(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs).unwrap(); engine.flush().unwrap(); let score_index = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -2782,15 +7359,18 @@ fn test_query_filter_range_and_timestamp_probe_budget_overflow_is_cumulative() { .unwrap(); wait_for_property_index_state(&engine, score_index.index_id, SecondaryIndexState::Ready); let segment_id = engine.segments_for_test()[0].segment_id; - let stats_path = - segment_dir(&db_path, segment_id).join(crate::planner_stats::PLANNER_STATS_FILENAME); + let seg_dir = segment_dir(&db_path, segment_id); + let stats_path = segment_component_path( + &seg_dir, + crate::segment_components::SegmentComponentKind::PlannerStats, + ); engine.close().unwrap(); std::fs::remove_file(&stats_path).unwrap(); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); assert!(engine.segments_for_test()[0].planner_stats().is_none()); let range_query = NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), filter: Some(NodeFilterExpr::And( (0..5) .map(|lower| NodeFilterExpr::PropertyRange { @@ -2814,7 +7394,7 @@ fn test_query_filter_range_and_timestamp_probe_budget_overflow_is_cumulative() { ); let timestamp_query = NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), filter: Some(NodeFilterExpr::And( (0..5) .map(|lower| NodeFilterExpr::UpdatedAtRange { @@ -2845,56 +7425,46 @@ fn test_query_pattern_boolean_anchor_uses_union_and_filters_stale_candidates() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let stale_active = insert_query_node( - &engine, - 1, + let stale_active = insert_query_node(&engine, "Person", "stale-active", &[("status", PropValue::String("active".to_string()))], 1.0, ); - let deleted_trial = insert_query_node( - &engine, - 1, + let deleted_trial = insert_query_node(&engine, "Person", "deleted-trial", &[("status", PropValue::String("trial".to_string()))], 1.0, ); - let active = insert_query_node( - &engine, - 1, + let active = insert_query_node(&engine, "Person", "active", &[("status", PropValue::String("active".to_string()))], 1.0, ); - let trial = insert_query_node( - &engine, - 1, + let trial = insert_query_node(&engine, "Person", "trial", &[("status", PropValue::String("trial".to_string()))], 1.0, ); - let inactive = insert_query_node( - &engine, - 1, + let inactive = insert_query_node(&engine, "Person", "inactive", &[("status", PropValue::String("inactive".to_string()))], 1.0, ); - let target = insert_query_node(&engine, 2, "target", &[], 1.0); + let target = insert_query_node(&engine, "Company", "target", &[], 1.0); for source in [stale_active, deleted_trial, active, trial, inactive] { engine - .upsert_edge(source, target, 10, UpsertEdgeOptions::default()) + .upsert_edge(source, target, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } engine.flush().unwrap(); let index = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); let updated = engine .upsert_node( - 1, + "Person", "stale-active", UpsertNodeOptions { props: query_test_props(&[( @@ -2912,7 +7482,7 @@ fn test_query_pattern_boolean_anchor_uses_union_and_filters_stale_candidates() { vec![ NodePattern { alias: "person".to_string(), - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), ids: Vec::new(), keys: Vec::new(), filter: Some(NodeFilterExpr::Or(vec![ @@ -2933,7 +7503,7 @@ fn test_query_pattern_boolean_anchor_uses_union_and_filters_stale_candidates() { "person", "target", Direction::Outgoing, - Some(vec![10]), + None, )], ); @@ -2961,14 +7531,12 @@ fn test_query_pattern_boolean_anchor_uses_union_and_filters_stale_candidates() { } #[test] -fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_type_scan() { +fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_label_scan() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let active = insert_query_node( - &engine, - 1, + let active = insert_query_node(&engine, "Person", "active", &[ ("status", PropValue::String("active".to_string())), @@ -2976,16 +7544,12 @@ fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_type_scan() { ], 1.0, ); - let missing = insert_query_node( - &engine, - 1, + let missing = insert_query_node(&engine, "Person", "missing", &[("status", PropValue::String("inactive".to_string()))], 1.0, ); - let skipped = insert_query_node( - &engine, - 1, + let skipped = insert_query_node(&engine, "Person", "skipped", &[ ("status", PropValue::String("inactive".to_string())), @@ -2993,15 +7557,15 @@ fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_type_scan() { ], 1.0, ); - let target = insert_query_node(&engine, 2, "target", &[], 1.0); + let target = insert_query_node(&engine, "Company", "target", &[], 1.0); for source in [active, missing, skipped] { engine - .upsert_edge(source, target, 10, UpsertEdgeOptions::default()) + .upsert_edge(source, target, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } engine.flush().unwrap(); let index = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, index.index_id, SecondaryIndexState::Ready); @@ -3009,7 +7573,7 @@ fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_type_scan() { vec![ NodePattern { alias: "person".to_string(), - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), ids: Vec::new(), keys: Vec::new(), filter: Some(NodeFilterExpr::Or(vec![ @@ -3029,7 +7593,7 @@ fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_type_scan() { "person", "target", Direction::Outgoing, - Some(vec![10]), + Some(vec!["KNOWS"]), )], ); @@ -3041,7 +7605,7 @@ fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_type_scan() { ] ); let plan = engine.explain_pattern_query(&query).unwrap(); - assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::FallbackTypeScan]); + assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::FallbackNodeLabelScan]); assert!(plan.warnings.contains(&QueryPlanWarning::UsingFallbackScan)); assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); assert!(plan.warnings.contains(&QueryPlanWarning::BooleanBranchFallback)); @@ -3051,41 +7615,311 @@ fn test_query_pattern_boolean_anchor_with_verify_only_branch_uses_type_scan() { } #[test] -fn test_query_pattern_exists_anchor_uses_type_scan_and_presence_semantics() { +fn test_query_pattern_exists_anchor_uses_label_scan_and_presence_semantics() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let present = insert_query_node( - &engine, - 1, + let present = insert_query_node(&engine, "Person", "present", &[("tag", PropValue::String("present".to_string()))], 1.0, ); - let null_tag = insert_query_node(&engine, 1, "null-tag", &[("tag", PropValue::Null)], 1.0); - let missing = insert_query_node(&engine, 1, "missing", &[], 1.0); - let target = insert_query_node(&engine, 2, "target", &[], 1.0); + let null_tag = insert_query_node(&engine, "Person", "null-tag", &[("tag", PropValue::Null)], 1.0); + let missing = insert_query_node(&engine, "Person", "missing", &[], 1.0); + let target = insert_query_node(&engine, "Company", "target", &[], 1.0); let present_edge = engine - .upsert_edge(present, target, 10, UpsertEdgeOptions::default()) + .upsert_edge(present, target, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let null_edge = engine - .upsert_edge(null_tag, target, 10, UpsertEdgeOptions::default()) + .upsert_edge(null_tag, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(missing, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let query = pattern_query( + vec![ + NodePattern { + alias: "source".to_string(), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: Vec::new(), + keys: Vec::new(), + filter: Some(NodeFilterExpr::PropertyExists { + key: "tag".to_string(), + }), + }, + pattern_node("target", None, Vec::new()), + ], + vec![pattern_edge( + Some("edge"), + "source", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], + ); + + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![ + expected_match( + &[("source", present), ("target", target)], + &[("edge", present_edge)] + ), + expected_match( + &[("source", null_tag), ("target", target)], + &[("edge", null_edge)] + ), + ] + ); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert_eq!( + pattern_anchor_input_nodes(&plan), + vec![QueryPlanNode::FallbackNodeLabelScan] + ); + assert!(plan.warnings.contains(&QueryPlanWarning::UsingFallbackScan)); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert!(!plan_contains_fallback_full_node_scan(&plan.root)); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_ids_with_verify_only_filter_is_valid_anchor() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let missing = insert_query_node(&engine, "Person", "missing", &[], 1.0); + let present = insert_query_node(&engine, "Person", + "present", + &[("tag", PropValue::String("present".to_string()))], + 1.0, + ); + let target = insert_query_node(&engine, "Company", "target", &[], 1.0); + let edge = engine + .upsert_edge(missing, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(present, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let query = pattern_query( + vec![ + NodePattern { + alias: "source".to_string(), + label_filter: None, + ids: vec![missing, present], + keys: Vec::new(), + filter: Some(NodeFilterExpr::PropertyMissing { + key: "tag".to_string(), + }), + }, + pattern_node("target", None, Vec::new()), + ], + vec![pattern_edge( + Some("edge"), + "source", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], + ); + + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("source", missing), ("target", target)], &[("edge", edge)])] + ); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::ExplicitIds]); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_rejects_labelless_verify_only_initial_anchor() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let query = pattern_query( + vec![ + NodePattern { + alias: "source".to_string(), + label_filter: None, + ids: Vec::new(), + keys: Vec::new(), + filter: Some(NodeFilterExpr::PropertyMissing { + key: "tag".to_string(), + }), + }, + pattern_node("target", None, Vec::new()), + ], + vec![pattern_edge( + Some("edge"), + "source", + "target", + Direction::Outgoing, + None, + )], + ); + + assert!(matches!( + engine.query_pattern(&query).unwrap_err(), + EngineError::InvalidOperation(message) + if message.contains("anchorable node pattern") + )); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_labelless_missing_target_verifies_after_expansion() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let anchor = insert_query_node(&engine, "Person", "anchor", &[], 1.0); + let missing = insert_query_node(&engine, "Company", "missing", &[], 1.0); + let present = insert_query_node(&engine, "Company", + "present", + &[("tag", PropValue::String("present".to_string()))], + 1.0, + ); + let edge = engine + .upsert_edge(anchor, missing, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(anchor, present, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let query = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + NodePattern { + alias: "target".to_string(), + label_filter: None, + ids: Vec::new(), + keys: Vec::new(), + filter: Some(NodeFilterExpr::PropertyMissing { + key: "tag".to_string(), + }), + }, + ], + vec![pattern_edge( + Some("edge"), + "anchor", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], + ); + + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("anchor", anchor), ("target", missing)], &[("edge", edge)])] + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_labelless_not_target_verifies_after_expansion() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let anchor = insert_query_node(&engine, "Person", "anchor", &[], 1.0); + let active = insert_query_node(&engine, "Company", + "active", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let missing_status = insert_query_node(&engine, "Company", "missing-status", &[], 1.0); + let inactive = insert_query_node(&engine, "Company", + "inactive", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let active_edge = engine + .upsert_edge(anchor, active, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let missing_edge = engine + .upsert_edge(anchor, missing_status, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(missing, target, 10, UpsertEdgeOptions::default()) + .upsert_edge(anchor, inactive, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); + let query = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + NodePattern { + alias: "target".to_string(), + label_filter: None, + ids: Vec::new(), + keys: Vec::new(), + filter: Some(NodeFilterExpr::Not(Box::new( + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("inactive".to_string()), + }, + ))), + }, + ], + vec![pattern_edge( + Some("edge"), + "anchor", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], + ); + + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![ + expected_match( + &[("anchor", anchor), ("target", active)], + &[("edge", active_edge)] + ), + expected_match( + &[("anchor", anchor), ("target", missing_status)], + &[("edge", missing_edge)] + ), + ] + ); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::ExplicitIds]); + assert!(!plan_contains_fallback_full_node_scan(&plan.root)); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_always_false_node_returns_empty_result() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = pattern_query( vec![ NodePattern { alias: "source".to_string(), - type_id: Some(1), + label_filter: None, ids: Vec::new(), keys: Vec::new(), - filter: Some(NodeFilterExpr::PropertyExists { - key: "tag".to_string(), - }), + filter: Some(NodeFilterExpr::And(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("inactive".to_string()), + }, + ])), }, pattern_node("target", None, Vec::new()), ], @@ -3094,3622 +7928,4934 @@ fn test_query_pattern_exists_anchor_uses_type_scan_and_presence_semantics() { "source", "target", Direction::Outgoing, - Some(vec![10]), + Some(vec!["KNOWS"]), )], ); - assert_eq!( - engine.query_pattern(&query).unwrap().matches, - vec![ - expected_match( - &[("source", present), ("target", target)], - &[("edge", present_edge)] - ), - expected_match( - &[("source", null_tag), ("target", target)], - &[("edge", null_edge)] - ), - ] - ); + let result = engine.query_pattern(&query).unwrap(); + assert!(result.matches.is_empty()); + assert!(!result.truncated); let plan = engine.explain_pattern_query(&query).unwrap(); - assert_eq!( - pattern_anchor_input_nodes(&plan), - vec![QueryPlanNode::FallbackTypeScan] - ); - assert!(plan.warnings.contains(&QueryPlanWarning::UsingFallbackScan)); - assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); - assert!(!plan_contains_fallback_full_node_scan(&plan.root)); + assert_eq!(pattern_anchor_plan_node(&plan).0, "source"); + assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::EmptyResult]); engine.close().unwrap(); } #[test] -fn test_query_pattern_ids_with_verify_only_filter_is_valid_anchor() { +fn test_query_pattern_target_in_matches_equivalent_or_filter() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let missing = insert_query_node(&engine, 1, "missing", &[], 1.0); - let present = insert_query_node( - &engine, - 1, - "present", - &[("tag", PropValue::String("present".to_string()))], + let anchor = insert_query_node(&engine, "Person", "anchor", &[], 1.0); + let active = insert_query_node(&engine, "Company", + "active", + &[("status", PropValue::String("active".to_string()))], 1.0, ); - let target = insert_query_node(&engine, 2, "target", &[], 1.0); - let edge = engine - .upsert_edge(missing, target, 10, UpsertEdgeOptions::default()) + let trial = insert_query_node(&engine, "Company", + "trial", + &[("status", PropValue::String("trial".to_string()))], + 1.0, + ); + let inactive = insert_query_node(&engine, "Company", + "inactive", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let active_edge = engine + .upsert_edge(anchor, active, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let trial_edge = engine + .upsert_edge(anchor, trial, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(present, target, 10, UpsertEdgeOptions::default()) + .upsert_edge(anchor, inactive, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - let query = pattern_query( - vec![ - NodePattern { - alias: "source".to_string(), - type_id: None, - ids: vec![missing, present], + let base_nodes = vec![pattern_node_with_ids("anchor", vec![anchor])]; + let edges = vec![pattern_edge( + Some("edge"), + "anchor", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )]; + let in_query = pattern_query( + { + let mut nodes = base_nodes.clone(); + nodes.push(NodePattern { + alias: "target".to_string(), + label_filter: None, + ids: Vec::new(), keys: Vec::new(), - filter: Some(NodeFilterExpr::PropertyMissing { - key: "tag".to_string(), + filter: Some(NodeFilterExpr::PropertyIn { + key: "status".to_string(), + values: vec![ + PropValue::String("active".to_string()), + PropValue::String("trial".to_string()), + ], }), - }, - pattern_node("target", None, Vec::new()), - ], - vec![pattern_edge( - Some("edge"), - "source", - "target", - Direction::Outgoing, - Some(vec![10]), - )], + }); + nodes + }, + edges.clone(), + ); + let or_query = pattern_query( + { + let mut nodes = base_nodes; + nodes.push(NodePattern { + alias: "target".to_string(), + label_filter: None, + ids: Vec::new(), + keys: Vec::new(), + filter: Some(NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("trial".to_string()), + }, + ])), + }); + nodes + }, + edges, ); + let expected = vec![ + expected_match(&[("anchor", anchor), ("target", active)], &[("edge", active_edge)]), + expected_match(&[("anchor", anchor), ("target", trial)], &[("edge", trial_edge)]), + ]; + assert_eq!(engine.query_pattern(&in_query).unwrap().matches, expected); assert_eq!( - engine.query_pattern(&query).unwrap().matches, - vec![expected_match(&[("source", missing), ("target", target)], &[("edge", edge)])] + engine.query_pattern(&or_query).unwrap().matches, + engine.query_pattern(&in_query).unwrap().matches ); - let plan = engine.explain_pattern_query(&query).unwrap(); - assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::ExplicitIds]); engine.close().unwrap(); } -#[test] -fn test_query_pattern_rejects_typeless_verify_only_initial_anchor() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +// --- scan-backed node queries --- + +#[test] +fn test_query_label_only_uses_label_index_path() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Person", "b", &[], 1.0); + let _other_label = insert_query_node(&engine, "Company", "x", &[], 1.0); + + let query = query_ids(Some("Person"), Vec::new(), false); + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[a, b, _other_label], &query) + ); + + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert!(matches!( + plan.root, + QueryPlanNode::VerifyNodeFilter { ref input } + if **input == QueryPlanNode::NodeLabelIndex + )); + + engine.close().unwrap(); +} + +#[test] +fn test_query_label_only_pagination_excludes_deleted_and_survives_reopen() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let kept = insert_query_node(&engine, "Person", "kept", &[], 1.0); + let deleted = insert_query_node(&engine, "Person", "deleted", &[], 1.0); + let overwritten = insert_query_node(&engine, "Person", "overwritten", &[], 1.0); + let _other_label = insert_query_node(&engine, "Company", "other", &[], 1.0); + engine.flush().unwrap(); + + engine.delete_node(deleted).unwrap(); + let overwritten_again = insert_query_node(&engine, "Person", + "overwritten", + &[("status", PropValue::String("new".to_string()))], + 1.0, + ); + assert_eq!(overwritten_again, overwritten); + let memtable = insert_query_node(&engine, "Person", "memtable", &[], 1.0); + + let mut expected = vec![kept, overwritten, memtable]; + expected.sort_unstable(); + + let mut query = query_ids(Some("Person"), Vec::new(), false); + query.page = PageRequest { + limit: Some(2), + after: None, + }; + + let page1 = engine.query_node_ids(&query).unwrap(); + assert_eq!(page1.items, expected[..2]); + assert_eq!(page1.next_cursor, Some(expected[1])); + + query.page.after = page1.next_cursor; + let page2 = engine.query_node_ids(&query).unwrap(); + assert_eq!(page2.items, expected[2..]); + assert_eq!(page2.next_cursor, None); + + engine.flush().unwrap(); + engine.close().unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = query_ids(Some("Person"), Vec::new(), false); + assert_eq!(reopened.query_node_ids(&query).unwrap().items, expected); + reopened.close().unwrap(); +} + +#[test] +fn test_query_label_only_pagination_across_multiple_segments() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut expected = Vec::new(); + for index in 0..6 { + expected.push(insert_query_node(&engine, "Person", + &format!("seg-a-{index}"), + &[], + 1.0, + )); + } + engine.flush().unwrap(); + + for index in 0..6 { + expected.push(insert_query_node(&engine, "Person", + &format!("seg-b-{index}"), + &[], + 1.0, + )); + } + engine.flush().unwrap(); + + let deleted = expected[3]; + engine.delete_node(deleted).unwrap(); + let memtable = insert_query_node(&engine, "Person", "memtable", &[], 1.0); + expected.retain(|id| *id != deleted); + expected.push(memtable); + expected.sort_unstable(); + + let mut query = query_ids(Some("Person"), Vec::new(), false); + query.page = PageRequest { + limit: Some(5), + after: None, + }; + + let page1 = engine.query_node_ids(&query).unwrap(); + assert_eq!(page1.items, expected[..5]); + assert_eq!(page1.next_cursor, Some(expected[4])); + + query.page.after = page1.next_cursor; + let page2 = engine.query_node_ids(&query).unwrap(); + assert_eq!(page2.items, expected[5..10]); + assert_eq!(page2.next_cursor, Some(expected[9])); + + query.page.after = page2.next_cursor; + let page3 = engine.query_node_ids(&query).unwrap(); + assert_eq!(page3.items, expected[10..]); + assert_eq!(page3.next_cursor, None); + + engine.close().unwrap(); +} + +#[test] +fn test_query_label_universe_beats_large_explicit_ids() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let filler: Vec = (0..QUERY_RANGE_CANDIDATE_CAP + 32) + .map(|index| NodeInput { + labels: vec!["Company".to_string()], + key: format!("filler-{index}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + let mut all_ids = engine.batch_upsert_nodes(filler).unwrap(); + engine.flush().unwrap(); + + let mut expected = Vec::new(); + for index in 0..8 { + let id = insert_query_node(&engine, "Person", &format!("small-{index}"), &[], 1.0); + all_ids.push(id); + expected.push(id); + } + all_ids.sort_unstable(); + expected.sort_unstable(); - let query = pattern_query( - vec![ - NodePattern { - alias: "source".to_string(), - type_id: None, - ids: Vec::new(), - keys: Vec::new(), - filter: Some(NodeFilterExpr::PropertyMissing { - key: "tag".to_string(), - }), - }, - pattern_node("target", None, Vec::new()), - ], - vec![pattern_edge( - Some("edge"), - "source", - "target", - Direction::Outgoing, - Some(vec![10]), - )], - ); + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: all_ids, + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); - assert!(matches!( - engine.query_pattern(&query).unwrap_err(), - EngineError::InvalidOperation(message) - if message.contains("anchorable node pattern") - )); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::NodeLabelIndex]); engine.close().unwrap(); } #[test] -fn test_query_pattern_typeless_missing_target_verifies_after_expansion() { +fn test_query_small_explicit_ids_beat_large_label_universe() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let anchor = insert_query_node(&engine, 1, "anchor", &[], 1.0); - let missing = insert_query_node(&engine, 2, "missing", &[], 1.0); - let present = insert_query_node( - &engine, - 2, - "present", - &[("tag", PropValue::String("present".to_string()))], - 1.0, - ); - let edge = engine - .upsert_edge(anchor, missing, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine - .upsert_edge(anchor, present, 10, UpsertEdgeOptions::default()) - .unwrap(); + let mut node_ids = Vec::new(); + for index in 0..300 { + node_ids.push(insert_query_node(&engine, "Person", + &format!("node-{index}"), + &[], + 1.0, + )); + } - let query = pattern_query( - vec![ - pattern_node_with_ids("anchor", vec![anchor]), - NodePattern { - alias: "target".to_string(), - type_id: None, - ids: Vec::new(), - keys: Vec::new(), - filter: Some(NodeFilterExpr::PropertyMissing { - key: "tag".to_string(), - }), - }, - ], - vec![pattern_edge( - Some("edge"), - "anchor", - "target", - Direction::Outgoing, - Some(vec![10]), - )], - ); + let expected = vec![node_ids[12], node_ids[223]]; + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: expected.clone(), + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); - assert_eq!( - engine.query_pattern(&query).unwrap().matches, - vec![expected_match(&[("anchor", anchor), ("target", missing)], &[("edge", edge)])] - ); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::ExplicitIds]); engine.close().unwrap(); } #[test] -fn test_query_pattern_typeless_not_target_verifies_after_expansion() { +fn test_query_explain_omits_label_scan_when_property_index_drives_execution() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let anchor = insert_query_node(&engine, 1, "anchor", &[], 1.0); - let active = insert_query_node( - &engine, - 2, + let active = insert_query_node(&engine, "Person", "active", &[("status", PropValue::String("active".to_string()))], 1.0, ); - let missing_status = insert_query_node(&engine, 2, "missing-status", &[], 1.0); - let inactive = insert_query_node( - &engine, - 2, + let inactive = insert_query_node(&engine, "Person", "inactive", &[("status", PropValue::String("inactive".to_string()))], 1.0, ); - let active_edge = engine - .upsert_edge(anchor, active, 10, UpsertEdgeOptions::default()) - .unwrap(); - let missing_edge = engine - .upsert_edge(anchor, missing_status, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine - .upsert_edge(anchor, inactive, 10, UpsertEdgeOptions::default()) + let other_label = insert_query_node(&engine, "Company", + "other", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - let query = pattern_query( - vec![ - pattern_node_with_ids("anchor", vec![anchor]), - NodePattern { - alias: "target".to_string(), - type_id: None, - ids: Vec::new(), - keys: Vec::new(), - filter: Some(NodeFilterExpr::Not(Box::new( - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("inactive".to_string()), - }, - ))), - }, - ], - vec![pattern_edge( - Some("edge"), - "anchor", - "target", - Direction::Outgoing, - Some(vec![10]), - )], + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + false, ); assert_eq!( - engine.query_pattern(&query).unwrap().matches, - vec![ - expected_match( - &[("anchor", anchor), ("target", active)], - &[("edge", active_edge)] - ), - expected_match( - &[("anchor", anchor), ("target", missing_status)], - &[("edge", missing_edge)] - ), - ] + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[active, inactive, other_label], &query) ); - let plan = engine.explain_pattern_query(&query).unwrap(); - assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::ExplicitIds]); - assert!(!plan_contains_fallback_full_node_scan(&plan.root)); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyEqualityIndex]); engine.close().unwrap(); } #[test] -fn test_query_pattern_always_false_node_returns_empty_result() { +fn test_query_label_universe_beats_large_key_upper_bound() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let query = pattern_query( - vec![ - NodePattern { - alias: "source".to_string(), - type_id: None, - ids: Vec::new(), - keys: Vec::new(), - filter: Some(NodeFilterExpr::And(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("inactive".to_string()), - }, - ])), - }, - pattern_node("target", None, Vec::new()), - ], - vec![pattern_edge( - Some("edge"), - "source", - "target", - Direction::Outgoing, - Some(vec![10]), - )], - ); + for index in 0..300 { + insert_query_node(&engine, "Company", &format!("filler-{index}"), &[], 1.0); + } + engine.flush().unwrap(); - let result = engine.query_pattern(&query).unwrap(); - assert!(result.matches.is_empty()); - assert!(!result.truncated); - let plan = engine.explain_pattern_query(&query).unwrap(); - assert_eq!(pattern_anchor_plan_node(&plan).0, "source"); - assert_eq!(pattern_anchor_input_nodes(&plan), vec![QueryPlanNode::EmptyResult]); + let mut expected = Vec::new(); + let mut keys = Vec::new(); + for index in 0..QUERY_RANGE_CANDIDATE_CAP + 32 { + keys.push(format!("missing-{index}")); + } + for index in 0..8 { + let key = format!("small-{index}"); + expected.push(insert_query_node(&engine, "Person", &key, &[], 1.0)); + keys.push(key); + } + expected.sort_unstable(); + + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + keys, + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); + + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::NodeLabelIndex]); engine.close().unwrap(); } #[test] -fn test_query_pattern_target_in_matches_equivalent_or_filter() { +fn test_query_label_universe_verifies_large_ids_and_predicate() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let anchor = insert_query_node(&engine, 1, "anchor", &[], 1.0); - let active = insert_query_node( - &engine, - 2, - "active", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let trial = insert_query_node( - &engine, - 2, - "trial", - &[("status", PropValue::String("trial".to_string()))], - 1.0, - ); - let inactive = insert_query_node( - &engine, - 2, - "inactive", - &[("status", PropValue::String("inactive".to_string()))], - 1.0, - ); - let active_edge = engine - .upsert_edge(anchor, active, 10, UpsertEdgeOptions::default()) - .unwrap(); - let trial_edge = engine - .upsert_edge(anchor, trial, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine - .upsert_edge(anchor, inactive, 10, UpsertEdgeOptions::default()) - .unwrap(); - - let base_nodes = vec![pattern_node_with_ids("anchor", vec![anchor])]; - let edges = vec![pattern_edge( - Some("edge"), - "anchor", - "target", - Direction::Outgoing, - Some(vec![10]), - )]; - let in_query = pattern_query( - { - let mut nodes = base_nodes.clone(); - nodes.push(NodePattern { - alias: "target".to_string(), - type_id: None, - ids: Vec::new(), - keys: Vec::new(), - filter: Some(NodeFilterExpr::PropertyIn { - key: "status".to_string(), - values: vec![ - PropValue::String("active".to_string()), - PropValue::String("trial".to_string()), - ], - }), - }); - nodes - }, - edges.clone(), + let filler: Vec = (0..QUERY_RANGE_CANDIDATE_CAP + 32) + .map(|index| NodeInput { + labels: vec!["Company".to_string()], + key: format!("filler-{index}"), + props: query_test_props(&[("status", PropValue::String("keep".to_string()))]), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + let mut all_ids = engine.batch_upsert_nodes(filler).unwrap(); + engine.flush().unwrap(); + + let keep = insert_query_node(&engine, "Person", + "keep", + &[("status", PropValue::String("keep".to_string()))], + 1.0, ); - let or_query = pattern_query( - { - let mut nodes = base_nodes; - nodes.push(NodePattern { - alias: "target".to_string(), - type_id: None, - ids: Vec::new(), - keys: Vec::new(), - filter: Some(NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("trial".to_string()), - }, - ])), - }); - nodes - }, - edges, + let drop = insert_query_node(&engine, "Person", + "drop", + &[("status", PropValue::String("drop".to_string()))], + 1.0, ); + all_ids.push(keep); + all_ids.push(drop); + all_ids.sort_unstable(); - let expected = vec![ - expected_match(&[("anchor", anchor), ("target", active)], &[("edge", active_edge)]), - expected_match(&[("anchor", anchor), ("target", trial)], &[("edge", trial_edge)]), - ]; - assert_eq!(engine.query_pattern(&in_query).unwrap().matches, expected); + let query = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: all_ids, + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("keep".to_string()), + }], + ..Default::default() + }; + assert_eq!(engine.query_node_ids(&query).unwrap().items, vec![keep]); + + let plan = engine.explain_node_query(&query).unwrap(); assert_eq!( - engine.query_pattern(&or_query).unwrap().matches, - engine.query_pattern(&in_query).unwrap().matches + plan.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + ] ); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackNodeLabelScan]); engine.close().unwrap(); } -// --- scan-backed node queries --- - #[test] -fn test_query_type_only_uses_type_index_path() { +fn test_query_label_scan_predicates_pagination_and_hydration_parity() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = insert_query_node(&engine, 1, "a", &[], 1.0); - let b = insert_query_node(&engine, 1, "b", &[], 1.0); - let _other_type = insert_query_node(&engine, 2, "x", &[], 1.0); + let a = insert_query_node(&engine, "Person", + "a", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(10)), + ], + 1.0, + ); + let b = insert_query_node(&engine, "Person", + "b", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(20)), + ], + 1.0, + ); + let c = insert_query_node(&engine, "Person", + "c", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ); + let _other_label = insert_query_node(&engine, "Company", + "x", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ); + + let mut query = query_ids(Some("Person"), + vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), + upper: Some(PropertyRangeBound::Excluded(PropValue::Int(30))), + }, + NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(0), + upper_ms: None, + }, + ], + false, + ); + query.page = PageRequest { + limit: Some(1), + after: None, + }; + + let first = engine.query_node_ids(&query).unwrap(); + assert_eq!(first.items, vec![a]); + assert_eq!(first.next_cursor, Some(a)); + + query.page.after = first.next_cursor; + let second = engine.query_node_ids(&query).unwrap(); + assert!(!second.items.contains(&c)); + assert_eq!(second.items.len(), 1); + assert_eq!(second.next_cursor, None); - let query = query_ids(Some(1), Vec::new(), false); + query.page = PageRequest::default(); + let ids = engine.query_node_ids(&query).unwrap(); assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b, _other_type], &query) + ids.items, + oracle_query_ids(&engine, &[a, b, c, _other_label], &query) + ); + let nodes = engine.query_nodes(&query).unwrap(); + assert_eq!( + ids.items, + nodes.items.iter().map(|node| node.id).collect::>() ); let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); + assert_eq!(plan.kind, QueryPlanKind::NodeQuery); + assert_eq!( + plan.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::VerifyOnlyFilter, + ] + ); assert!(matches!( - plan.root, - QueryPlanNode::VerifyNodeFilter { ref input } - if **input == QueryPlanNode::NodeTypeIndex + explain_input_node(&plan), + QueryPlanNode::TimestampIndex )); engine.close().unwrap(); } +// --- anchor semantics --- + #[test] -fn test_query_type_only_pagination_excludes_deleted_and_survives_reopen() { +fn test_query_multi_anchor_and_semantics_and_conflicts() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let kept = insert_query_node(&engine, 1, "kept", &[], 1.0); - let deleted = insert_query_node(&engine, 1, "deleted", &[], 1.0); - let overwritten = insert_query_node(&engine, 1, "overwritten", &[], 1.0); - let _other_type = insert_query_node(&engine, 2, "other", &[], 1.0); - engine.flush().unwrap(); - - engine.delete_node(deleted).unwrap(); - let overwritten_again = insert_query_node( - &engine, - 1, - "overwritten", - &[("status", PropValue::String("new".to_string()))], + let alice = insert_query_node(&engine, "Person", + "alice", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let bob = insert_query_node(&engine, "Person", + "bob", + &[("status", PropValue::String("active".to_string()))], 1.0, ); - assert_eq!(overwritten_again, overwritten); - let memtable = insert_query_node(&engine, 1, "memtable", &[], 1.0); - - let mut expected = vec![kept, overwritten, memtable]; - expected.sort_unstable(); - let mut query = query_ids(Some(1), Vec::new(), false); - query.page = PageRequest { - limit: Some(2), - after: None, + let matched = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: vec![alice, bob], + keys: vec!["alice".to_string()], + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ..Default::default() }; + assert_eq!( + engine.query_node_ids(&matched).unwrap().items, + oracle_query_ids(&engine, &[alice, bob], &matched) + ); - let page1 = engine.query_node_ids(&query).unwrap(); - assert_eq!(page1.items, expected[..2]); - assert_eq!(page1.next_cursor, Some(expected[1])); - - query.page.after = page1.next_cursor; - let page2 = engine.query_node_ids(&query).unwrap(); - assert_eq!(page2.items, expected[2..]); - assert_eq!(page2.next_cursor, None); + let conflict = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: vec![bob], + keys: vec!["alice".to_string()], + ..Default::default() + }; + assert!(engine.query_node_ids(&conflict).unwrap().items.is_empty()); - engine.flush().unwrap(); engine.close().unwrap(); - - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let query = query_ids(Some(1), Vec::new(), false); - assert_eq!(reopened.query_node_ids(&query).unwrap().items, expected); - reopened.close().unwrap(); } #[test] -fn test_query_type_only_pagination_across_multiple_segments() { +fn test_query_key_lookup_anchor_normalization_and_source_choice() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut expected = Vec::new(); - for index in 0..6 { - expected.push(insert_query_node( - &engine, - 1, - &format!("seg-a-{index}"), - &[], - 1.0, - )); - } - engine.flush().unwrap(); + let alice = insert_query_node(&engine, "Person", + "alice", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let bob = insert_query_node(&engine, "Person", + "bob", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + let carol = insert_query_node(&engine, "Person", + "carol", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + let other_label = insert_query_node(&engine, "Company", + "alice", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + + let key_only = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + keys: vec!["bob".to_string(), "alice".to_string(), "alice".to_string()], + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&key_only).unwrap().items, + oracle_query_ids(&engine, &[alice, bob, carol, other_label], &key_only) + ); + let key_only_plan = engine.explain_node_query(&key_only).unwrap(); + assert_eq!(key_only_plan.warnings, Vec::::new()); + assert_plan_input_nodes(&key_only_plan, vec![QueryPlanNode::KeyLookup]); + + let key_preferred = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + ids: vec![alice, bob, carol], + keys: vec!["bob".to_string()], + filter: filter_and![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&key_preferred).unwrap().items, + oracle_query_ids(&engine, &[alice, bob, carol, other_label], &key_preferred) + ); + let key_preferred_plan = engine.explain_node_query(&key_preferred).unwrap(); + assert_eq!(key_preferred_plan.warnings, Vec::::new()); + assert_plan_input_nodes(&key_preferred_plan, vec![QueryPlanNode::KeyLookup]); - for index in 0..6 { - expected.push(insert_query_node( - &engine, - 1, - &format!("seg-b-{index}"), - &[], - 1.0, - )); - } - engine.flush().unwrap(); + engine.close().unwrap(); +} - let deleted = expected[3]; - engine.delete_node(deleted).unwrap(); - let memtable = insert_query_node(&engine, 1, "memtable", &[], 1.0); - expected.retain(|id| *id != deleted); - expected.push(memtable); - expected.sort_unstable(); +// --- indexed candidate-source planning --- - let mut query = query_ids(Some(1), Vec::new(), false); - query.page = PageRequest { - limit: Some(5), - after: None, - }; +#[test] +fn test_query_intersects_ready_equality_indexes_against_oracle() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let page1 = engine.query_node_ids(&query).unwrap(); - assert_eq!(page1.items, expected[..5]); - assert_eq!(page1.next_cursor, Some(expected[4])); + let a = insert_query_node(&engine, "Person", + "a", + &[ + ("status", PropValue::String("active".to_string())), + ("tier", PropValue::String("gold".to_string())), + ], + 1.0, + ); + let b = insert_query_node(&engine, "Person", + "b", + &[ + ("status", PropValue::String("active".to_string())), + ("tier", PropValue::String("silver".to_string())), + ], + 1.0, + ); + let c = insert_query_node(&engine, "Person", + "c", + &[ + ("status", PropValue::String("inactive".to_string())), + ("tier", PropValue::String("gold".to_string())), + ], + 1.0, + ); + let d = insert_query_node(&engine, "Person", + "d", + &[ + ("status", PropValue::String("active".to_string())), + ("tier", PropValue::String("gold".to_string())), + ], + 1.0, + ); - query.page.after = page1.next_cursor; - let page2 = engine.query_node_ids(&query).unwrap(); - assert_eq!(page2.items, expected[5..10]); - assert_eq!(page2.next_cursor, Some(expected[9])); + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let tier = engine + .ensure_node_property_index("Person", "tier", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(&engine, tier.index_id, SecondaryIndexState::Ready); - query.page.after = page2.next_cursor; - let page3 = engine.query_node_ids(&query).unwrap(); - assert_eq!(page3.items, expected[10..]); - assert_eq!(page3.next_cursor, None); + let query = query_ids(Some("Person"), + vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "tier".to_string(), + value: PropValue::String("gold".to_string()), + }, + ], + false, + ); + + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[a, b, c, d], &query) + ); + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes( + &plan, + vec![ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + ); engine.close().unwrap(); } #[test] -fn test_query_type_universe_beats_large_explicit_ids() { +fn test_query_intersects_equality_and_range_indexes_against_oracle() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let filler: Vec = (0..QUERY_RANGE_CANDIDATE_CAP + 32) - .map(|index| NodeInput { - type_id: 2, - key: format!("filler-{index}"), - props: BTreeMap::new(), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }) - .collect(); - let mut all_ids = engine.batch_upsert_nodes(&filler).unwrap(); - engine.flush().unwrap(); + let a = insert_query_node(&engine, "Person", + "a", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(10)), + ], + 1.0, + ); + let b = insert_query_node(&engine, "Person", + "b", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(20)), + ], + 1.0, + ); + let c = insert_query_node(&engine, "Person", + "c", + &[ + ("status", PropValue::String("inactive".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ); + let d = insert_query_node(&engine, "Person", + "d", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(40)), + ], + 1.0, + ); - let mut expected = Vec::new(); - for index in 0..8 { - let id = insert_query_node(&engine, 1, &format!("small-{index}"), &[], 1.0); - all_ids.push(id); - expected.push(id); - } - all_ids.sort_unstable(); - expected.sort_unstable(); + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - let query = NodeQuery { - type_id: Some(1), - ids: all_ids, - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); + let query = query_ids(Some("Person"), + vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(20))), + }, + ], + false, + ); + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[a, b, c, d], &query) + ); let plan = engine.explain_node_query(&query).unwrap(); assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::NodeTypeIndex]); + assert_plan_includes_input_nodes( + &plan, + &[ + QueryPlanNode::PropertyRangeIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + ); engine.close().unwrap(); } #[test] -fn test_query_small_explicit_ids_beat_large_type_universe() { +fn test_query_intersects_equality_equality_and_range_indexes() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut type_ids = Vec::new(); - for index in 0..300 { - type_ids.push(insert_query_node( - &engine, - 1, - &format!("node-{index}"), - &[], - 1.0, - )); - } + let a = insert_query_node(&engine, "Person", + "a", + &[ + ("status", PropValue::String("active".to_string())), + ("tier", PropValue::String("gold".to_string())), + ("score", PropValue::Int(10)), + ], + 1.0, + ); + let b = insert_query_node(&engine, "Person", + "b", + &[ + ("status", PropValue::String("active".to_string())), + ("tier", PropValue::String("gold".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ); + let c = insert_query_node(&engine, "Person", + "c", + &[ + ("status", PropValue::String("active".to_string())), + ("tier", PropValue::String("silver".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ); + let d = insert_query_node(&engine, "Person", + "d", + &[ + ("status", PropValue::String("inactive".to_string())), + ("tier", PropValue::String("gold".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ); - let expected = vec![type_ids[12], type_ids[223]]; - let query = NodeQuery { - type_id: Some(1), - ids: expected.clone(), - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let tier = engine + .ensure_node_property_index("Person", "tier", SecondaryIndexKind::Equality) + .unwrap(); + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(&engine, tier.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + + let query = query_ids(Some("Person"), + vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyEquals { + key: "tier".to_string(), + value: PropValue::String("gold".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(20))), + }, + ], + false, + ); + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[a, b, c, d], &query) + ); let plan = engine.explain_node_query(&query).unwrap(); assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::ExplicitIds]); + assert_plan_input_nodes( + &plan, + vec![ + QueryPlanNode::PropertyRangeIndex, + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + ); engine.close().unwrap(); } #[test] -fn test_query_explain_omits_type_scan_when_property_index_drives_execution() { +fn test_query_intersects_timestamp_and_property_sources() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let active = insert_query_node( - &engine, - 1, - "active", + let a = insert_query_node(&engine, "Person", + "a", &[("status", PropValue::String("active".to_string()))], 1.0, ); - let inactive = insert_query_node( - &engine, - 1, - "inactive", - &[("status", PropValue::String("inactive".to_string()))], + let b = insert_query_node(&engine, "Person", + "b", + &[("status", PropValue::String("active".to_string()))], 1.0, ); - let other_type = insert_query_node( - &engine, - 2, - "other", + let c = insert_query_node(&engine, "Person", + "c", &[("status", PropValue::String("active".to_string()))], 1.0, ); + let d = insert_query_node(&engine, "Person", + "d", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + set_query_node_updated_at(&engine, a, 1_000); + set_query_node_updated_at(&engine, b, 2_000); + set_query_node_updated_at(&engine, c, 3_000); + set_query_node_updated_at(&engine, d, 2_500); let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], + let query = query_ids(Some("Person"), + vec![ + NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(1_500), + upper_ms: Some(2_500), + }, + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + ], false, ); assert_eq!( engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[active, inactive, other_type], &query) + oracle_query_ids(&engine, &[a, b, c, d], &query) ); let plan = engine.explain_node_query(&query).unwrap(); assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::PropertyEqualityIndex]); + assert_plan_includes_input_nodes( + &plan, + &[ + QueryPlanNode::TimestampIndex, + QueryPlanNode::PropertyEqualityIndex, + ], + ); + + engine.close().unwrap(); +} + +#[test] +fn test_query_ready_index_sources_match_oracle_across_storage_states() { + fn lifecycle_query() -> NodeQuery { + query_ids(Some("Person"), + vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(15))), + }, + NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(i64::MIN), + upper_ms: Some(i64::MAX), + }, + ], + false, + ) + } + + fn insert_lifecycle_segment_nodes(engine: &DatabaseEngine) -> Vec { + vec![ + insert_query_node(engine, "Person", + "a", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(10)), + ], + 1.0, + ), + insert_query_node(engine, "Person", + "b", + &[ + ("status", PropValue::String("inactive".to_string())), + ("score", PropValue::Int(10)), + ], + 1.0, + ), + insert_query_node(engine, "Person", + "c", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ), + insert_query_node(engine, "Person", + "d", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(30)), + ], + 1.0, + ), + ] + } + + fn insert_lifecycle_active_nodes(engine: &DatabaseEngine) -> Vec { + vec![ + insert_query_node(engine, "Person", + "e", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(12)), + ], + 1.0, + ), + insert_query_node(engine, "Person", + "f", + &[ + ("status", PropValue::String("active".to_string())), + ("score", PropValue::Int(50)), + ], + 1.0, + ), + ] + } + + fn ensure_lifecycle_indexes(engine: &DatabaseEngine) { + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(engine, status.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(engine, score.index_id, SecondaryIndexState::Ready); + } + + fn assert_tiny_lifecycle_query_uses_fallback( + engine: &DatabaseEngine, + all_ids: &[u64], + query: &NodeQuery, + ) { + assert_eq!( + engine.query_node_ids(query).unwrap().items, + oracle_query_ids(engine, all_ids, query) + ); + let plan = engine.explain_node_query(query).unwrap(); + assert_eq!(plan.warnings, vec![QueryPlanWarning::UsingFallbackScan]); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackNodeLabelScan]); + } + + let dir = TempDir::new().unwrap(); + let query = lifecycle_query(); + + { + let db_path = dir.path().join("memtable-only"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let mut all_ids = insert_lifecycle_segment_nodes(&engine); + all_ids.extend(insert_lifecycle_active_nodes(&engine)); + ensure_lifecycle_indexes(&engine); + assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); + engine.close().unwrap(); + } + + { + let db_path = dir.path().join("mixed"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let mut all_ids = insert_lifecycle_segment_nodes(&engine); + engine.flush().unwrap(); + all_ids.extend(insert_lifecycle_active_nodes(&engine)); + ensure_lifecycle_indexes(&engine); + assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); + engine.close().unwrap(); + } + + { + let db_path = dir.path().join("compacted-reopened"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let mut all_ids = insert_lifecycle_segment_nodes(&engine); + engine.flush().unwrap(); + all_ids.extend(insert_lifecycle_active_nodes(&engine)); + engine.flush().unwrap(); + ensure_lifecycle_indexes(&engine); + assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); + engine.compact().unwrap().unwrap(); + assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); + engine.close().unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_tiny_lifecycle_query_uses_fallback(&reopened, &all_ids, &query); + reopened.close().unwrap(); + } +} + +#[test] +fn test_query_selective_ready_indexes_match_oracle_across_storage_states() { + fn selective_query() -> NodeQuery { + query_ids(Some("Person"), + vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("target".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(15))), + }, + NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(1_000), + upper_ms: Some(1_010), + }, + ], + false, + ) + } + + fn ensure_selective_indexes(engine: &DatabaseEngine) { + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_property_index_state(engine, status.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(engine, score.index_id, SecondaryIndexState::Ready); + wait_for_published_property_index_state(engine, status.index_id, SecondaryIndexState::Ready); + wait_for_published_property_index_state(engine, score.index_id, SecondaryIndexState::Ready); + } + + fn insert_selective_nodes(engine: &DatabaseEngine, start: usize, count: usize) -> Vec { + let mut ids = Vec::with_capacity(count); + for index in start..start + count { + let selected = index % 64 == 0; + let node_id = insert_query_node(engine, "Person", + &format!("selective-{index}"), + &[ + ( + "status", + PropValue::String(if selected { "target" } else { "other" }.to_string()), + ), + ( + "score", + PropValue::Int(if selected { 10 } else { 1_000 + index as i64 }), + ), + ], + 1.0, + ); + set_query_node_updated_at( + engine, + node_id, + if selected { 1_005 } else { 10_000 + index as i64 }, + ); + ids.push(node_id); + } + ids + } - engine.close().unwrap(); -} + fn assert_selective_indexes_match_oracle( + engine: &DatabaseEngine, + all_ids: &[u64], + query: &NodeQuery, + ) { + assert_eq!( + engine.query_node_ids(query).unwrap().items, + oracle_query_ids(engine, all_ids, query) + ); + let plan = engine.explain_node_query(query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_includes_input_nodes( + &plan, + &[ + QueryPlanNode::PropertyEqualityIndex, + QueryPlanNode::PropertyRangeIndex, + QueryPlanNode::TimestampIndex, + ], + ); + } -#[test] -fn test_query_type_universe_beats_large_key_upper_bound() { let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = selective_query(); - for index in 0..300 { - insert_query_node(&engine, 2, &format!("filler-{index}"), &[], 1.0); + { + let db_path = dir.path().join("memtable-only-selective"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + ensure_selective_indexes(&engine); + let all_ids = insert_selective_nodes(&engine, 0, 512); + assert_selective_indexes_match_oracle(&engine, &all_ids, &query); + engine.close().unwrap(); } - engine.flush().unwrap(); - let mut expected = Vec::new(); - let mut keys = Vec::new(); - for index in 0..QUERY_RANGE_CANDIDATE_CAP + 32 { - keys.push(format!("missing-{index}")); - } - for index in 0..8 { - let key = format!("small-{index}"); - expected.push(insert_query_node(&engine, 1, &key, &[], 1.0)); - keys.push(key); + { + let db_path = dir.path().join("mixed-selective"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + ensure_selective_indexes(&engine); + let mut all_ids = insert_selective_nodes(&engine, 0, 256); + engine.flush().unwrap(); + all_ids.extend(insert_selective_nodes(&engine, 256, 256)); + assert_selective_indexes_match_oracle(&engine, &all_ids, &query); + engine.close().unwrap(); } - expected.sort_unstable(); - let query = NodeQuery { - type_id: Some(1), - keys, - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&query).unwrap().items, expected); - - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::NodeTypeIndex]); + { + let db_path = dir.path().join("compacted-reopened-selective"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + ensure_selective_indexes(&engine); + let mut all_ids = insert_selective_nodes(&engine, 0, 256); + engine.flush().unwrap(); + all_ids.extend(insert_selective_nodes(&engine, 256, 256)); + engine.flush().unwrap(); + assert_selective_indexes_match_oracle(&engine, &all_ids, &query); + engine.compact().unwrap().unwrap(); + assert_selective_indexes_match_oracle(&engine, &all_ids, &query); + engine.close().unwrap(); - engine.close().unwrap(); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_selective_indexes_match_oracle(&reopened, &all_ids, &query); + reopened.close().unwrap(); + } } #[test] -fn test_query_type_universe_verifies_large_ids_and_predicate() { +fn test_query_bounded_range_uses_index_and_broad_sources_fallback() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let filler: Vec = (0..QUERY_RANGE_CANDIDATE_CAP + 32) - .map(|index| NodeInput { - type_id: 2, - key: format!("filler-{index}"), - props: query_test_props(&[("status", PropValue::String("keep".to_string()))]), + let mut inputs = Vec::with_capacity(QUERY_RANGE_CANDIDATE_CAP + 1); + for i in 0..=QUERY_RANGE_CANDIDATE_CAP { + inputs.push(NodeInput { + labels: vec!["Person".to_string()], + key: format!("n{i}"), + props: query_test_props(&[ + ("score", PropValue::Int(i as i64)), + ( + "status", + PropValue::String(if i == 0 { "needle" } else { "other" }.to_string()), + ), + ]), weight: 1.0, dense_vector: None, sparse_vector: None, - }) - .collect(); - let mut all_ids = engine.batch_upsert_nodes(&filler).unwrap(); + }); + } + let all_ids = engine.batch_upsert_nodes(inputs).unwrap(); engine.flush().unwrap(); - - let keep = insert_query_node( - &engine, - 1, - "keep", - &[("status", PropValue::String("keep".to_string()))], - 1.0, - ); - let drop = insert_query_node( - &engine, - 1, - "drop", - &[("status", PropValue::String("drop".to_string()))], - 1.0, - ); - all_ids.push(keep); - all_ids.push(drop); - all_ids.sort_unstable(); - - let query = NodeQuery { - type_id: Some(1), - ids: all_ids, - filter: filter_and![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("keep".to_string()), - }], - ..Default::default() - }; - assert_eq!(engine.query_node_ids(&query).unwrap().items, vec![keep]); - - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!( - plan.warnings, - vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, - ] - ); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackTypeScan]); - - engine.close().unwrap(); -} - -#[test] -fn test_query_type_scan_predicates_pagination_and_hydration_parity() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let a = insert_query_node( - &engine, - 1, - "a", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(10)), - ], - 1.0, - ); - let b = insert_query_node( - &engine, - 1, - "b", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(20)), - ], - 1.0, - ); - let c = insert_query_node( - &engine, - 1, - "c", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, - ); - let _other_type = insert_query_node( - &engine, - 2, - "x", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, - ); - - let mut query = query_ids( - Some(1), - vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), - upper: Some(PropertyRangeBound::Excluded(PropValue::Int(30))), - }, - NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(0), - upper_ms: None, + let score = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, }, - ], - false, - ); - query.page = PageRequest { - limit: Some(1), - after: None, - }; + ) + .unwrap(); + let status = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - let first = engine.query_node_ids(&query).unwrap(); - assert_eq!(first.items, vec![a]); - assert_eq!(first.next_cursor, Some(a)); + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let range_lower = PropertyRangeBound::Included(PropValue::Int(0)); + let range_upper = + PropertyRangeBound::Included(PropValue::Int(QUERY_RANGE_CANDIDATE_CAP as i64)); + let (range_candidates, followup) = published + .view + .ready_range_candidate_ids( + score.index_id, + SecondaryIndexRangeDomain::Int, + Some(&range_lower), + Some(&range_upper), + QUERY_RANGE_CANDIDATE_CAP + 1, + ) + .unwrap(); + assert!(followup.is_none()); + assert_eq!( + range_candidates.unwrap().len(), + QUERY_RANGE_CANDIDATE_CAP + 1 + ); - query.page.after = first.next_cursor; - let second = engine.query_node_ids(&query).unwrap(); - assert!(!second.items.contains(&c)); - assert_eq!(second.items.len(), 1); - assert_eq!(second.next_cursor, None); + let timestamp_candidates = published + .view + .timestamp_candidate_ids(1, i64::MIN, i64::MAX, QUERY_RANGE_CANDIDATE_CAP + 1) + .unwrap(); + assert_eq!(timestamp_candidates.len(), QUERY_RANGE_CANDIDATE_CAP + 1); + } - query.page = PageRequest::default(); - let ids = engine.query_node_ids(&query).unwrap(); - assert_eq!( - ids.items, - oracle_query_ids(&engine, &[a, b, c, _other_type], &query) + let bounded = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(12))), + }], + false, ); - let nodes = engine.query_nodes(&query).unwrap(); assert_eq!( - ids.items, - nodes.items.iter().map(|node| node.id).collect::>() + engine.query_node_ids(&bounded).unwrap().items, + oracle_query_ids(&engine, &all_ids, &bounded) ); + let bounded_plan = engine.explain_node_query(&bounded).unwrap(); + assert_eq!(bounded_plan.warnings, Vec::::new()); + assert_plan_input_nodes(&bounded_plan, vec![QueryPlanNode::PropertyRangeIndex]); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.kind, QueryPlanKind::NodeQuery); + let broad_range = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), + upper: Some(PropertyRangeBound::Included(PropValue::Int( + QUERY_RANGE_CANDIDATE_CAP as i64, + ))), + }], + false, + ); assert_eq!( - plan.warnings, + engine.query_node_ids(&broad_range).unwrap().items, + oracle_query_ids(&engine, &all_ids, &broad_range) + ); + let broad_range_plan = engine.explain_node_query(&broad_range).unwrap(); + assert_eq!( + broad_range_plan.warnings, vec![ - QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::RangeCandidateCapExceeded, QueryPlanWarning::VerifyOnlyFilter, ] ); - assert!(matches!( - explain_input_node(&plan), - QueryPlanNode::TimestampIndex - )); - - engine.close().unwrap(); -} + assert_plan_input_nodes(&broad_range_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); -// --- anchor semantics --- - -#[test] -fn test_query_multi_anchor_and_semantics_and_conflicts() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let alice = insert_query_node( - &engine, - 1, - "alice", - &[("status", PropValue::String("active".to_string()))], - 1.0, + let broad_timestamp = query_ids(Some("Person"), + vec![NodeFilterExpr::UpdatedAtRange { + lower_ms: Some(i64::MIN), + upper_ms: Some(i64::MAX), + }], + false, ); - let bob = insert_query_node( - &engine, - 1, - "bob", - &[("status", PropValue::String("active".to_string()))], - 1.0, + assert_eq!( + engine.query_node_ids(&broad_timestamp).unwrap().items, + oracle_query_ids(&engine, &all_ids, &broad_timestamp) ); - - let matched = NodeQuery { - type_id: Some(1), - ids: vec![alice, bob], - keys: vec!["alice".to_string()], - filter: filter_and![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - ..Default::default() - }; + let broad_timestamp_plan = engine.explain_node_query(&broad_timestamp).unwrap(); assert_eq!( - engine.query_node_ids(&matched).unwrap().items, - oracle_query_ids(&engine, &[alice, bob], &matched) + broad_timestamp_plan.warnings, + vec![ + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::TimestampCandidateCapExceeded, + QueryPlanWarning::VerifyOnlyFilter, + ] + ); + assert_plan_input_nodes( + &broad_timestamp_plan, + vec![QueryPlanNode::FallbackNodeLabelScan], ); - let conflict = NodeQuery { - type_id: Some(1), - ids: vec![bob], - keys: vec!["alice".to_string()], + let broad_or = NodeQuery { + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), + filter: Some(NodeFilterExpr::Or(vec![ + NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("needle".to_string()), + }, + NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), + upper: Some(PropertyRangeBound::Included(PropValue::Int( + QUERY_RANGE_CANDIDATE_CAP as i64, + ))), + }, + ])), ..Default::default() }; - assert!(engine.query_node_ids(&conflict).unwrap().items.is_empty()); + assert_eq!( + engine.query_node_ids(&broad_or).unwrap().items, + oracle_query_ids(&engine, &all_ids, &broad_or) + ); + let broad_or_plan = engine.explain_node_query(&broad_or).unwrap(); + assert_eq!( + broad_or_plan.warnings, + vec![ + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::RangeCandidateCapExceeded, + QueryPlanWarning::VerifyOnlyFilter, + QueryPlanWarning::BooleanBranchFallback, + ] + ); + assert_plan_input_nodes(&broad_or_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); engine.close().unwrap(); } #[test] -fn test_query_key_lookup_anchor_normalization_and_source_choice() { +fn test_query_missing_building_and_failed_indexes_fallback() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let alice = insert_query_node( - &engine, - 1, - "alice", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let bob = insert_query_node( - &engine, - 1, - "bob", + let a = insert_query_node(&engine, "Person", + "a", &[("status", PropValue::String("active".to_string()))], 1.0, ); - let carol = insert_query_node( - &engine, - 1, - "carol", + let b = insert_query_node(&engine, "Person", + "b", &[("status", PropValue::String("inactive".to_string()))], 1.0, ); - let other_type = insert_query_node( - &engine, - 2, - "alice", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - - let key_only = NodeQuery { - type_id: Some(1), - keys: vec!["bob".to_string(), "alice".to_string(), "alice".to_string()], - filter: filter_and![NodeFilterExpr::PropertyEquals { + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { key: "status".to_string(), value: PropValue::String("active".to_string()), }], - ..Default::default() - }; - assert_eq!( - engine.query_node_ids(&key_only).unwrap().items, - oracle_query_ids(&engine, &[alice, bob, carol, other_type], &key_only) + false, ); - let key_only_plan = engine.explain_node_query(&key_only).unwrap(); - assert_eq!(key_only_plan.warnings, Vec::::new()); - assert_plan_input_nodes(&key_only_plan, vec![QueryPlanNode::KeyLookup]); - let key_preferred = NodeQuery { - type_id: Some(1), - ids: vec![alice, bob, carol], - keys: vec!["bob".to_string()], - filter: filter_and![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - ..Default::default() - }; assert_eq!( - engine.query_node_ids(&key_preferred).unwrap().items, - oracle_query_ids(&engine, &[alice, bob, carol, other_type], &key_preferred) - ); - let key_preferred_plan = engine.explain_node_query(&key_preferred).unwrap(); - assert_eq!(key_preferred_plan.warnings, Vec::::new()); - assert_plan_input_nodes(&key_preferred_plan, vec![QueryPlanNode::KeyLookup]); - - engine.close().unwrap(); -} - -// --- indexed candidate-source planning --- - -#[test] -fn test_query_intersects_ready_equality_indexes_against_oracle() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let a = insert_query_node( - &engine, - 1, - "a", - &[ - ("status", PropValue::String("active".to_string())), - ("tier", PropValue::String("gold".to_string())), - ], - 1.0, - ); - let b = insert_query_node( - &engine, - 1, - "b", - &[ - ("status", PropValue::String("active".to_string())), - ("tier", PropValue::String("silver".to_string())), - ], - 1.0, - ); - let c = insert_query_node( - &engine, - 1, - "c", - &[ - ("status", PropValue::String("inactive".to_string())), - ("tier", PropValue::String("gold".to_string())), - ], - 1.0, + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[a, b], &query) ); - let d = insert_query_node( - &engine, - 1, - "d", - &[ - ("status", PropValue::String("active".to_string())), - ("tier", PropValue::String("gold".to_string())), - ], - 1.0, + let missing_plan = engine.explain_node_query(&query).unwrap(); + assert_eq!( + missing_plan.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + ] ); + assert_plan_input_nodes(&missing_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let (build_ready_rx, build_release_tx) = engine.set_secondary_index_build_pause(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); - let tier = engine - .ensure_node_property_index(1, "tier", SecondaryIndexKind::Equality) + build_ready_rx + .recv_timeout(std::time::Duration::from_secs(5)) .unwrap(); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(&engine, tier.index_id, SecondaryIndexState::Ready); + let building_plan = engine.explain_node_query(&query).unwrap(); + assert_eq!( + building_plan.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + ] + ); + assert_plan_input_nodes(&building_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); + build_release_tx.send(()).unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let query = query_ids( - Some(1), + engine.shutdown_secondary_index_worker(); + engine + .with_runtime_manifest_write(|manifest| { + let entry = manifest + .secondary_indexes + .iter_mut() + .find(|entry| entry.index_id == info.index_id) + .unwrap(); + entry.state = SecondaryIndexState::Failed; + entry.last_error = Some("forced failure".to_string()); + Ok(()) + }) + .unwrap(); + engine.rebuild_secondary_index_catalog().unwrap(); + let failed_plan = engine.explain_node_query(&query).unwrap(); + assert_eq!( + failed_plan.warnings, vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "tier".to_string(), - value: PropValue::String("gold".to_string()), - }, - ], - false, + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + ] ); - + assert_plan_input_nodes(&failed_plan, vec![QueryPlanNode::FallbackNodeLabelScan]); assert_eq!( engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b, c, d], &query) - ); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes( - &plan, - vec![ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], + oracle_query_ids(&engine, &[a, b], &query) ); engine.close().unwrap(); } #[test] -fn test_query_intersects_equality_and_range_indexes_against_oracle() { +fn test_query_ready_sidecar_removed_after_open_remains_usable_until_reopen() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = insert_query_node( - &engine, - 1, - "a", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(10)), - ], + let active = insert_query_node(&engine, "Person", + "active", + &[("status", PropValue::String("active".to_string()))], 1.0, ); - let b = insert_query_node( - &engine, - 1, - "b", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(20)), - ], + let inactive = insert_query_node(&engine, "Person", + "inactive", + &[("status", PropValue::String("inactive".to_string()))], 1.0, ); - let c = insert_query_node( - &engine, - 1, - "c", - &[ - ("status", PropValue::String("inactive".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, + engine.flush().unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + false, ); - let d = insert_query_node( - &engine, - 1, - "d", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(40)), - ], - 1.0, + let planned; + let normalized; + let policy_cutoffs; + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + normalized = published.view.normalize_node_query(&query).unwrap(); + planned = published.view.plan_normalized_node_query(&normalized).unwrap(); + policy_cutoffs = published.view.query_policy_cutoffs(); + } + assert_plan_input_nodes( + &engine.explain_node_query(&query).unwrap(), + vec![QueryPlanNode::PropertyEqualityIndex], ); + assert!(engine.declared_index_runtime_coverage_len_for_test() > 0); - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, - }, - ) - .unwrap(); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + let seg_dir = segment_dir(&db_path, engine.segments_for_test()[0].segment_id); + let sidecar_path = segment_component_path( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodePropertyEqualityIndex { + index_id: info.index_id, + }, + ); + std::fs::remove_file(&sidecar_path).unwrap(); - let query = query_ids( - Some(1), - vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(20))), - }, - ], - false, + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let (page, followups) = published + .view + .query_node_page_planned(&normalized, &planned, false, policy_cutoffs.as_ref()) + .unwrap(); + assert_eq!(page.ids, vec![active]); + assert!(followups.is_empty()); + } + + assert_plan_input_nodes( + &engine.explain_node_query(&query).unwrap(), + vec![QueryPlanNode::PropertyEqualityIndex], ); + let segment_id = engine.segments_for_test()[0].segment_id; + engine + .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + let explain = engine.explain_node_query(&query).unwrap(); assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b, c, d], &query) + explain.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::UsingFallbackScan, + QueryPlanWarning::VerifyOnlyFilter, + ] ); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_includes_input_nodes( - &plan, - &[ - QueryPlanNode::PropertyRangeIndex, - QueryPlanNode::PropertyEqualityIndex, - ], + assert_eq!( + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[active, inactive], &query) ); engine.close().unwrap(); } #[test] -fn test_query_intersects_equality_equality_and_range_indexes() { +fn test_query_explicit_anchor_does_not_scan_ready_property_index() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = insert_query_node( - &engine, - 1, - "a", - &[ - ("status", PropValue::String("active".to_string())), - ("tier", PropValue::String("gold".to_string())), - ("score", PropValue::Int(10)), - ], - 1.0, - ); - let b = insert_query_node( - &engine, - 1, - "b", - &[ - ("status", PropValue::String("active".to_string())), - ("tier", PropValue::String("gold".to_string())), - ("score", PropValue::Int(30)), - ], + let active = insert_query_node(&engine, "Person", + "active", + &[("status", PropValue::String("active".to_string()))], 1.0, ); - let c = insert_query_node( - &engine, - 1, - "c", - &[ - ("status", PropValue::String("active".to_string())), - ("tier", PropValue::String("silver".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, + engine.flush().unwrap(); + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let seg_dir = segment_dir(&db_path, engine.segments_for_test()[0].segment_id); + let sidecar_path = segment_component_path( + &seg_dir, + crate::segment_components::SegmentComponentKind::NodePropertyEqualityIndex { + index_id: info.index_id, + }, ); - let d = insert_query_node( - &engine, - 1, - "d", - &[ - ("status", PropValue::String("inactive".to_string())), - ("tier", PropValue::String("gold".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, + std::fs::remove_file(&sidecar_path).unwrap(); + + let mut query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + false, ); + query.ids = vec![active]; - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - let tier = engine - .ensure_node_property_index(1, "tier", SecondaryIndexKind::Equality) - .unwrap(); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, - }, - ) - .unwrap(); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(&engine, tier.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); + let (_followup_ready_rx, followup_release_tx) = engine.set_runtime_publish_pause(); + assert_eq!(engine.query_node_ids(&query).unwrap().items, vec![active]); + assert_eq!(engine.pending_secondary_index_followup_count_for_test(), 0); + followup_release_tx.send(()).unwrap(); - let query = query_ids( - Some(1), - vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyEquals { - key: "tier".to_string(), - value: PropValue::String("gold".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(20))), - }, - ], - false, + let plan = engine.explain_node_query(&query).unwrap(); + assert_eq!(plan.warnings, Vec::::new()); + assert_plan_input_nodes(&plan, vec![QueryPlanNode::ExplicitIds]); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pagination_does_not_skip_after_rejected_candidates() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let reject_first = insert_query_node(&engine, "Person", + "reject-first", + &[("score", PropValue::Int(1))], + 1.0, + ); + let accept_first = insert_query_node(&engine, "Person", + "accept-first", + &[("score", PropValue::Int(10))], + 1.0, ); - - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b, c, d], &query) + let reject_second = insert_query_node(&engine, "Person", + "reject-second", + &[("score", PropValue::Int(2))], + 1.0, ); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes( - &plan, - vec![ - QueryPlanNode::PropertyRangeIndex, - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyEqualityIndex, - ], + let accept_second = insert_query_node(&engine, "Person", + "accept-second", + &[("score", PropValue::Int(20))], + 1.0, ); + let mut query = NodeQuery { + ids: vec![reject_first, accept_first, reject_second, accept_second], + filter: filter_and![NodeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), + upper: None, + }], + page: PageRequest { + limit: Some(1), + after: None, + }, + ..Default::default() + }; + + let first = engine.query_node_ids(&query).unwrap(); + assert_eq!(first.items, vec![accept_first]); + assert_eq!(first.next_cursor, Some(accept_first)); + + query.page.after = first.next_cursor; + let second = engine.query_node_ids(&query).unwrap(); + assert_eq!(second.items, vec![accept_second]); + assert_eq!(second.next_cursor, None); + + query.page.after = Some(accept_second); + let third = engine.query_node_ids(&query).unwrap(); + assert!(third.items.is_empty()); + assert!(third.next_cursor.is_none()); + engine.close().unwrap(); } +// --- full-scan opt-in --- + #[test] -fn test_query_intersects_timestamp_and_property_sources() { +fn test_query_explicit_full_scan_opt_in_and_explain_warning() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = insert_query_node( - &engine, - 1, + let a = insert_query_node(&engine, "Person", "a", - &[("status", PropValue::String("active".to_string()))], + &[("tenant", PropValue::String("t1".to_string()))], 1.0, ); - let b = insert_query_node( - &engine, - 1, + let b = insert_query_node(&engine, "Company", "b", - &[("status", PropValue::String("active".to_string()))], + &[("tenant", PropValue::String("t1".to_string()))], 1.0, ); - let c = insert_query_node( - &engine, - 1, + let _c = insert_query_node(&engine, "Article", "c", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let d = insert_query_node( - &engine, - 1, - "d", - &[("status", PropValue::String("inactive".to_string()))], + &[("tenant", PropValue::String("t2".to_string()))], 1.0, ); - set_query_node_updated_at(&engine, a, 1_000); - set_query_node_updated_at(&engine, b, 2_000); - set_query_node_updated_at(&engine, c, 3_000); - set_query_node_updated_at(&engine, d, 2_500); - - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); let query = query_ids( - Some(1), - vec![ - NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(1_500), - upper_ms: Some(2_500), - }, - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - ], - false, + None, + vec![NodeFilterExpr::PropertyEquals { + key: "tenant".to_string(), + value: PropValue::String("t1".to_string()), + }], + true, ); - assert_eq!( engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b, c, d], &query) + oracle_query_ids(&engine, &[a, b, _c], &query) ); + let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_includes_input_nodes( - &plan, - &[ - QueryPlanNode::TimestampIndex, - QueryPlanNode::PropertyEqualityIndex, - ], + assert_eq!( + plan.warnings, + vec![ + QueryPlanWarning::MissingReadyIndex, + QueryPlanWarning::FullScanExplicitlyAllowed, + QueryPlanWarning::VerifyOnlyFilter, + ] ); + assert!(matches!( + plan.root, + QueryPlanNode::VerifyNodeFilter { .. } + )); engine.close().unwrap(); } -#[test] -fn test_query_ready_index_sources_match_oracle_across_storage_states() { - fn lifecycle_query() -> NodeQuery { - query_ids( - Some(1), - vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(15))), - }, - NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(i64::MIN), - upper_ms: Some(i64::MAX), - }, - ], - false, - ) - } - - fn insert_lifecycle_segment_nodes(engine: &DatabaseEngine) -> Vec { - vec![ - insert_query_node( - engine, - 1, - "a", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(10)), - ], - 1.0, - ), - insert_query_node( - engine, - 1, - "b", - &[ - ("status", PropValue::String("inactive".to_string())), - ("score", PropValue::Int(10)), - ], - 1.0, - ), - insert_query_node( - engine, - 1, - "c", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, - ), - insert_query_node( - engine, - 1, - "d", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(30)), - ], - 1.0, - ), - ] - } +// --- visibility matrix --- - fn insert_lifecycle_active_nodes(engine: &DatabaseEngine) -> Vec { - vec![ - insert_query_node( - engine, - 1, - "e", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(12)), - ], - 1.0, - ), - insert_query_node( - engine, - 1, - "f", - &[ - ("status", PropValue::String("active".to_string())), - ("score", PropValue::Int(50)), - ], - 1.0, - ), - ] - } +#[test] +fn test_query_scan_parity_after_flush_reopen_overwrite_delete_and_prune() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); - fn ensure_lifecycle_indexes(engine: &DatabaseEngine) { - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, - }, - ) - .unwrap(); - wait_for_property_index_state(engine, status.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(engine, score.index_id, SecondaryIndexState::Ready); - } + let keep; + let deleted; + let low; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + keep = insert_query_node(&engine, "Person", + "keep", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + deleted = insert_query_node(&engine, "Person", + "delete", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + low = insert_query_node(&engine, "Person", + "low", + &[("status", PropValue::String("active".to_string()))], + 0.1, + ); + engine.flush().unwrap(); - fn assert_tiny_lifecycle_query_uses_fallback( - engine: &DatabaseEngine, - all_ids: &[u64], - query: &NodeQuery, - ) { + insert_query_node(&engine, "Person", + "keep", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, + ); + insert_query_node(&engine, "Person", + "keep", + &[("status", PropValue::String("active".to_string()))], + 1.0, + ); + engine.delete_node(deleted).unwrap(); + engine + .set_prune_policy( + "low-weight", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("Person".to_string()), + }, + ) + .unwrap(); + + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + false, + ); assert_eq!( - engine.query_node_ids(query).unwrap().items, - oracle_query_ids(engine, all_ids, query) + engine.query_node_ids(&query).unwrap().items, + oracle_query_ids(&engine, &[keep, deleted, low], &query) ); - let plan = engine.explain_node_query(query).unwrap(); - assert_eq!(plan.warnings, vec![QueryPlanWarning::UsingFallbackScan]); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::FallbackTypeScan]); + assert!(engine.get_node(low).unwrap().is_none()); + engine.close().unwrap(); } + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let query = query_ids(Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + false, + ); + assert_eq!( + reopened.query_node_ids(&query).unwrap().items, + oracle_query_ids(&reopened, &[keep, deleted, low], &query) + ); + reopened.close().unwrap(); +} + +// --- graph pattern execution --- + +#[test] +fn test_query_pattern_validation_rejects_invalid_shapes() { let dir = TempDir::new().unwrap(); - let query = lifecycle_query(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - { - let db_path = dir.path().join("memtable-only"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut all_ids = insert_lifecycle_segment_nodes(&engine); - all_ids.extend(insert_lifecycle_active_nodes(&engine)); - ensure_lifecycle_indexes(&engine); - assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); - engine.close().unwrap(); - } + let empty = pattern_query(Vec::new(), Vec::new()); + assert!(matches!( + engine.query_pattern(&empty).unwrap_err(), + EngineError::InvalidOperation(_) + )); - { - let db_path = dir.path().join("mixed"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut all_ids = insert_lifecycle_segment_nodes(&engine); - engine.flush().unwrap(); - all_ids.extend(insert_lifecycle_active_nodes(&engine)); - ensure_lifecycle_indexes(&engine); - assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); - engine.close().unwrap(); - } + let node_only = pattern_query(vec![pattern_node("a", Some("Person"), Vec::new())], Vec::new()); + assert!(matches!( + engine.explain_pattern_query(&node_only).unwrap_err(), + EngineError::InvalidOperation(_) + )); - { - let db_path = dir.path().join("compacted-reopened"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut all_ids = insert_lifecycle_segment_nodes(&engine); - engine.flush().unwrap(); - all_ids.extend(insert_lifecycle_active_nodes(&engine)); - engine.flush().unwrap(); - ensure_lifecycle_indexes(&engine); - assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); - engine.compact().unwrap().unwrap(); - assert_tiny_lifecycle_query_uses_fallback(&engine, &all_ids, &query); - engine.close().unwrap(); + let duplicate_node = pattern_query( + vec![ + pattern_node("a", Some("Person"), Vec::new()), + pattern_node("a", Some("Company"), Vec::new()), + ], + vec![pattern_edge(Some("e"), "a", "a", Direction::Outgoing, None)], + ); + assert!(matches!( + engine.query_pattern(&duplicate_node).unwrap_err(), + EngineError::InvalidOperation(_) + )); - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert_tiny_lifecycle_query_uses_fallback(&reopened, &all_ids, &query); - reopened.close().unwrap(); - } -} + let empty_node_alias = pattern_query( + vec![pattern_node("", Some("Person"), Vec::new())], + vec![pattern_edge(Some("e"), "", "", Direction::Outgoing, None)], + ); + assert!(matches!( + engine.query_pattern(&empty_node_alias).unwrap_err(), + EngineError::InvalidOperation(_) + )); -#[test] -fn test_query_selective_ready_indexes_match_oracle_across_storage_states() { - fn selective_query() -> NodeQuery { - query_ids( - Some(1), - vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("target".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(5))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(15))), - }, - NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(1_000), - upper_ms: Some(1_010), - }, - ], - false, - ) - } + let bad_reference = pattern_query( + vec![pattern_node("a", Some("Person"), Vec::new())], + vec![pattern_edge(Some("e"), "a", "missing", Direction::Outgoing, None)], + ); + assert!(matches!( + engine.query_pattern(&bad_reference).unwrap_err(), + EngineError::InvalidOperation(_) + )); - fn ensure_selective_indexes(engine: &DatabaseEngine) { - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, - }, - ) - .unwrap(); - wait_for_property_index_state(engine, status.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(engine, score.index_id, SecondaryIndexState::Ready); - wait_for_published_property_index_state(engine, status.index_id, SecondaryIndexState::Ready); - wait_for_published_property_index_state(engine, score.index_id, SecondaryIndexState::Ready); - } + let empty_edge_alias = pattern_query( + vec![pattern_node("a", Some("Person"), Vec::new())], + vec![pattern_edge(Some(""), "a", "a", Direction::Outgoing, None)], + ); + assert!(matches!( + engine.query_pattern(&empty_edge_alias).unwrap_err(), + EngineError::InvalidOperation(_) + )); - fn insert_selective_nodes(engine: &DatabaseEngine, start: usize, count: usize) -> Vec { - let mut ids = Vec::with_capacity(count); - for index in start..start + count { - let selected = index % 64 == 0; - let node_id = insert_query_node( - engine, - 1, - &format!("selective-{index}"), - &[ - ( - "status", - PropValue::String(if selected { "target" } else { "other" }.to_string()), - ), - ( - "score", - PropValue::Int(if selected { 10 } else { 1_000 + index as i64 }), - ), - ], - 1.0, - ); - set_query_node_updated_at( - engine, - node_id, - if selected { 1_005 } else { 10_000 + index as i64 }, - ); - ids.push(node_id); - } - ids - } + let duplicate_edge_alias = pattern_query( + vec![ + pattern_node("a", Some("Person"), Vec::new()), + pattern_node("b", Some("Company"), Vec::new()), + ], + vec![ + pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None), + pattern_edge(Some("e"), "b", "a", Direction::Outgoing, None), + ], + ); + assert!(matches!( + engine.query_pattern(&duplicate_edge_alias).unwrap_err(), + EngineError::InvalidOperation(_) + )); - fn assert_selective_indexes_match_oracle( - engine: &DatabaseEngine, - all_ids: &[u64], - query: &NodeQuery, - ) { - assert_eq!( - engine.query_node_ids(query).unwrap().items, - oracle_query_ids(engine, all_ids, query) - ); - let plan = engine.explain_node_query(query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_includes_input_nodes( - &plan, - &[ - QueryPlanNode::PropertyEqualityIndex, - QueryPlanNode::PropertyRangeIndex, - QueryPlanNode::TimestampIndex, + let zero_limit = GraphPatternQuery { + limit: 0, + ..pattern_query( + vec![ + pattern_node("a", Some("Person"), Vec::new()), + pattern_node("b", Some("Company"), Vec::new()), ], - ); - } - - let dir = TempDir::new().unwrap(); - let query = selective_query(); + vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], + ) + }; + assert!(matches!( + engine.query_pattern(&zero_limit).unwrap_err(), + EngineError::InvalidOperation(_) + )); - { - let db_path = dir.path().join("memtable-only-selective"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - ensure_selective_indexes(&engine); - let all_ids = insert_selective_nodes(&engine, 0, 512); - assert_selective_indexes_match_oracle(&engine, &all_ids, &query); - engine.close().unwrap(); - } + let mut key_without_label = pattern_node("a", None, Vec::new()); + key_without_label.keys.push("a".to_string()); + let key_query = pattern_query( + vec![key_without_label, pattern_node("b", Some("Company"), Vec::new())], + vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], + ); + assert!(matches!( + engine.query_pattern(&key_query).unwrap_err(), + EngineError::InvalidOperation(_) + )); - { - let db_path = dir.path().join("mixed-selective"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - ensure_selective_indexes(&engine); - let mut all_ids = insert_selective_nodes(&engine, 0, 256); - engine.flush().unwrap(); - all_ids.extend(insert_selective_nodes(&engine, 256, 256)); - assert_selective_indexes_match_oracle(&engine, &all_ids, &query); - engine.close().unwrap(); - } + let disconnected_extra_node = pattern_query( + vec![ + pattern_node("a", Some("Person"), Vec::new()), + pattern_node("b", Some("Company"), Vec::new()), + pattern_node("c", Some("Article"), Vec::new()), + ], + vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], + ); + assert!(matches!( + engine.query_pattern(&disconnected_extra_node).unwrap_err(), + EngineError::InvalidOperation(_) + )); - { - let db_path = dir.path().join("compacted-reopened-selective"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - ensure_selective_indexes(&engine); - let mut all_ids = insert_selective_nodes(&engine, 0, 256); - engine.flush().unwrap(); - all_ids.extend(insert_selective_nodes(&engine, 256, 256)); - engine.flush().unwrap(); - assert_selective_indexes_match_oracle(&engine, &all_ids, &query); - engine.compact().unwrap().unwrap(); - assert_selective_indexes_match_oracle(&engine, &all_ids, &query); - engine.close().unwrap(); + let disconnected_components = pattern_query( + vec![ + pattern_node("a", Some("Person"), Vec::new()), + pattern_node("b", Some("Company"), Vec::new()), + pattern_node("c", Some("Article"), Vec::new()), + pattern_node("d", Some("Topic"), Vec::new()), + ], + vec![ + pattern_edge(Some("ab"), "a", "b", Direction::Outgoing, None), + pattern_edge(Some("cd"), "c", "d", Direction::Outgoing, None), + ], + ); + assert!(matches!( + engine.query_pattern(&disconnected_components).unwrap_err(), + EngineError::InvalidOperation(_) + )); - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert_selective_indexes_match_oracle(&reopened, &all_ids, &query); - reopened.close().unwrap(); - } + let unanchored = pattern_query( + vec![pattern_node("a", None, Vec::new()), pattern_node("b", None, Vec::new())], + vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], + ); + assert!(matches!( + engine.explain_pattern_query(&unanchored).unwrap_err(), + EngineError::InvalidOperation(_) + )); + + engine.close().unwrap(); } #[test] -fn test_query_bounded_range_uses_index_and_broad_sources_fallback() { +fn test_query_pattern_anchor_selection_uses_label_cardinality() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut inputs = Vec::with_capacity(QUERY_RANGE_CANDIDATE_CAP + 1); - for i in 0..=QUERY_RANGE_CANDIDATE_CAP { - inputs.push(NodeInput { - type_id: 1, - key: format!("n{i}"), - props: query_test_props(&[ - ("score", PropValue::Int(i as i64)), - ( - "status", - PropValue::String(if i == 0 { "needle" } else { "other" }.to_string()), - ), - ]), - weight: 1.0, - dense_vector: None, - sparse_vector: None, - }); + for index in 0..8 { + insert_query_node(&engine, "Person", &format!("wide-{index}"), &[], 1.0); } - let all_ids = engine.batch_upsert_nodes(&inputs).unwrap(); - engine.flush().unwrap(); - let score = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, - }, - ) - .unwrap(); - let status = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - wait_for_property_index_state(&engine, score.index_id, SecondaryIndexState::Ready); - wait_for_property_index_state(&engine, status.index_id, SecondaryIndexState::Ready); + insert_query_node(&engine, "Company", "narrow", &[], 1.0); + + let query = pattern_query( + vec![ + pattern_node("aaa_wide", Some("Person"), Vec::new()), + pattern_node("zzz_narrow", Some("Company"), Vec::new()), + ], + vec![pattern_edge( + Some("edge"), + "aaa_wide", + "zzz_narrow", + Direction::Both, + None, + )], + ); { let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let range_lower = PropertyRangeBound::Included(PropValue::Int(0)); - let range_upper = - PropertyRangeBound::Included(PropValue::Int(QUERY_RANGE_CANDIDATE_CAP as i64)); - let (range_candidates, followup) = published - .view - .ready_range_candidate_ids( - score.index_id, - SecondaryIndexRangeDomain::Int, - Some(&range_lower), - Some(&range_upper), - QUERY_RANGE_CANDIDATE_CAP + 1, - ) - .unwrap(); - assert!(followup.is_none()); - assert_eq!( - range_candidates.unwrap().len(), - QUERY_RANGE_CANDIDATE_CAP + 1 - ); - - let timestamp_candidates = published + let normalized = published.view.normalize_pattern_query(&query).unwrap(); + let planned = published .view - .timestamp_candidate_ids(1, i64::MIN, i64::MAX, QUERY_RANGE_CANDIDATE_CAP + 1) + .plan_normalized_pattern_query(&normalized) .unwrap(); - assert_eq!(timestamp_candidates.len(), QUERY_RANGE_CANDIDATE_CAP + 1); + match &planned.anchor { + PatternAnchorPlan::Node { node_index, .. } => { + assert_eq!(normalized.nodes[*node_index].alias, "zzz_narrow"); + } + PatternAnchorPlan::Edge { .. } => panic!("expected node anchor"), + } } - let bounded = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), - upper: Some(PropertyRangeBound::Included(PropValue::Int(12))), - }], - false, - ); - assert_eq!( - engine.query_node_ids(&bounded).unwrap().items, - oracle_query_ids(&engine, &all_ids, &bounded) - ); - let bounded_plan = engine.explain_node_query(&bounded).unwrap(); - assert_eq!(bounded_plan.warnings, Vec::::new()); - assert_plan_input_nodes(&bounded_plan, vec![QueryPlanNode::PropertyRangeIndex]); + engine.close().unwrap(); +} - let broad_range = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), - upper: Some(PropertyRangeBound::Included(PropValue::Int( - QUERY_RANGE_CANDIDATE_CAP as i64, - ))), - }], - false, +#[test] +fn test_query_pattern_linear_uses_reverse_direction_from_selective_anchor() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let alice = insert_query_node(&engine, "Person", + "alice", + &[("status", PropValue::String("active".to_string()))], + 1.0, ); - assert_eq!( - engine.query_node_ids(&broad_range).unwrap().items, - oracle_query_ids(&engine, &all_ids, &broad_range) + let _bob = insert_query_node(&engine, "Person", + "bob", + &[("status", PropValue::String("inactive".to_string()))], + 1.0, ); - let broad_range_plan = engine.explain_node_query(&broad_range).unwrap(); - assert_eq!( - broad_range_plan.warnings, - vec![ - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::RangeCandidateCapExceeded, - QueryPlanWarning::VerifyOnlyFilter, - ] + let acme = insert_query_node(&engine, "Company", + "acme", + &[("tier", PropValue::String("enterprise".to_string()))], + 1.0, ); - assert_plan_input_nodes(&broad_range_plan, vec![QueryPlanNode::FallbackTypeScan]); + let edge = engine + .upsert_edge(alice, acme, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); - let broad_timestamp = query_ids( - Some(1), - vec![NodeFilterExpr::UpdatedAtRange { - lower_ms: Some(i64::MIN), - upper_ms: Some(i64::MAX), - }], - false, + let query = pattern_query( + vec![ + pattern_node("person", Some("Person"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }], + ), + pattern_node_with_ids("company", vec![acme]), + ], + vec![pattern_edge( + Some("works_at"), + "person", + "company", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], ); + + let result = engine.query_pattern(&query).unwrap(); assert_eq!( - engine.query_node_ids(&broad_timestamp).unwrap().items, - oracle_query_ids(&engine, &all_ids, &broad_timestamp) + result.matches, + vec![expected_match( + &[("company", acme), ("person", alice)], + &[("works_at", edge)] + )] ); - let broad_timestamp_plan = engine.explain_node_query(&broad_timestamp).unwrap(); - assert_eq!( - broad_timestamp_plan.warnings, - vec![ - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::TimestampCandidateCapExceeded, - QueryPlanWarning::VerifyOnlyFilter, - ] + assert!(!result.truncated); + + let plan = engine.explain_pattern_query(&query).unwrap(); + assert_eq!(plan.kind, QueryPlanKind::PatternQuery); + assert!(matches!(plan.root, QueryPlanNode::PatternExpand { .. })); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_labelless_predicate_target_verifies_after_expansion() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let anchor = insert_query_node(&engine, "Person", "anchor", &[], 1.0); + let good = insert_query_node(&engine, "Company", + "good", + &[("status", PropValue::String("match".to_string()))], + 1.0, ); - assert_plan_input_nodes( - &broad_timestamp_plan, - vec![QueryPlanNode::FallbackTypeScan], + let bad = insert_query_node(&engine, "Article", + "bad", + &[("status", PropValue::String("skip".to_string()))], + 1.0, ); + let good_edge = engine + .upsert_edge(anchor, good, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(anchor, bad, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); - let broad_or = NodeQuery { - type_id: Some(1), - filter: Some(NodeFilterExpr::Or(vec![ - NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("needle".to_string()), - }, - NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(0))), - upper: Some(PropertyRangeBound::Included(PropValue::Int( - QUERY_RANGE_CANDIDATE_CAP as i64, - ))), - }, - ])), - ..Default::default() - }; - assert_eq!( - engine.query_node_ids(&broad_or).unwrap().items, - oracle_query_ids(&engine, &all_ids, &broad_or) + let query = pattern_query( + vec![ + pattern_node_with_ids("anchor", vec![anchor]), + pattern_node( + "target", + None, + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("match".to_string()), + }], + ), + ], + vec![pattern_edge( + Some("edge"), + "anchor", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], ); - let broad_or_plan = engine.explain_node_query(&broad_or).unwrap(); + + { + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let normalized = published.view.normalize_pattern_query(&query).unwrap(); + let planned = published + .view + .plan_normalized_pattern_query(&normalized) + .unwrap(); + match &planned.anchor { + PatternAnchorPlan::Node { node_index, .. } => { + assert_eq!(normalized.nodes[*node_index].alias, "anchor"); + } + PatternAnchorPlan::Edge { .. } => panic!("expected node anchor"), + } + } + assert_eq!( - broad_or_plan.warnings, - vec![ - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::RangeCandidateCapExceeded, - QueryPlanWarning::VerifyOnlyFilter, - QueryPlanWarning::BooleanBranchFallback, - ] + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[("anchor", anchor), ("target", good)], + &[("edge", good_edge)] + )] ); - assert_plan_input_nodes(&broad_or_plan, vec![QueryPlanNode::FallbackTypeScan]); engine.close().unwrap(); } #[test] -fn test_query_missing_building_and_failed_indexes_fallback() { +fn test_query_pattern_order_limit_truncated_and_direction_both() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = insert_query_node( - &engine, - 1, - "a", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let b = insert_query_node( - &engine, - 1, - "b", - &[("status", PropValue::String("inactive".to_string()))], - 1.0, - ); - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - false, + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + let low = insert_query_node(&engine, "Company", "low", &[], 1.0); + let high = insert_query_node(&engine, "Company", "high", &[], 1.0); + let high_edge = engine + .upsert_edge(root, high, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let low_edge = engine + .upsert_edge(low, root, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let base = pattern_query( + vec![pattern_node_with_ids("root", vec![root]), pattern_node("target", Some("Company"), Vec::new())], + vec![pattern_edge(Some("edge"), "root", "target", Direction::Both, Some(vec!["KNOWS"]))], ); + let limited = GraphPatternQuery { limit: 1, ..base.clone() }; + let limited_result = engine.query_pattern(&limited).unwrap(); assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b], &query) + limited_result.matches, + vec![expected_match( + &[("root", root), ("target", low)], + &[("edge", low_edge)] + )] ); - let missing_plan = engine.explain_node_query(&query).unwrap(); + assert!(limited_result.truncated); + + let full_result = engine.query_pattern(&base).unwrap(); assert_eq!( - missing_plan.warnings, + full_result.matches, vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, + expected_match(&[("root", root), ("target", low)], &[("edge", low_edge)]), + expected_match(&[("root", root), ("target", high)], &[("edge", high_edge)]), ] ); - assert_plan_input_nodes(&missing_plan, vec![QueryPlanNode::FallbackTypeScan]); + assert!(!full_result.truncated); - let (build_ready_rx, build_release_tx) = engine.set_secondary_index_build_pause(); - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - build_ready_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - let building_plan = engine.explain_node_query(&query).unwrap(); - assert_eq!( - building_plan.warnings, + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_high_fanout_limit_keeps_deterministic_top_matches() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + let mut targets = Vec::new(); + for index in 0..32 { + targets.push(insert_query_node(&engine, "Company", + &format!("target-{index:02}"), + &[], + 1.0, + )); + } + + let mut edges_by_target = BTreeMap::new(); + for &target in targets.iter().rev() { + let edge = engine + .upsert_edge(root, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + edges_by_target.insert(target, edge); + } + + let query = GraphPatternQuery { + limit: 5, + ..pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node("target", Some("Company"), Vec::new()), + ], + vec![pattern_edge( + Some("edge"), + "root", + "target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + )], + ) + }; + + let result = engine.query_pattern(&query).unwrap(); + let expected: Vec = targets + .iter() + .take(5) + .map(|&target| { + expected_match( + &[("root", root), ("target", target)], + &[("edge", edges_by_target[&target])], + ) + }) + .collect(); + assert_eq!(result.matches, expected); + assert!(result.truncated); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_fanout_cost_can_choose_larger_lower_expansion_anchor() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let hub = insert_query_node(&engine, "Person", "hub", &[], 1.0); + let mut mids = Vec::new(); + for index in 0..300 { + let mid = insert_query_node(&engine, "Article", &format!("mid-{index:03}"), &[], 1.0); + engine + .upsert_edge(hub, mid, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + mids.push(mid); + } + let mut anchors = Vec::new(); + for (index, &mid) in mids.iter().enumerate().take(20) { + let anchor = insert_query_node(&engine, "Company", &format!("anchor-{index:02}"), &[], 1.0); + engine + .upsert_edge(mid, anchor, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + anchors.push(anchor); + } + engine.flush().unwrap(); + + let query = pattern_query( vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, - ] + pattern_node("small_hub", Some("Person"), Vec::new()), + pattern_node("larger_anchor", Some("Company"), Vec::new()), + pattern_node("middle", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("hub_to_middle"), + "small_hub", + "middle", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("middle_to_anchor"), + "middle", + "larger_anchor", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), + ], ); - assert_plan_input_nodes(&building_plan, vec![QueryPlanNode::FallbackTypeScan]); - build_release_tx.send(()).unwrap(); - wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - engine.shutdown_secondary_index_worker(); + let (anchor_alias, _) = planned_pattern_anchor_and_edge_aliases(&engine, &query); + assert_eq!(anchor_alias, "larger_anchor"); + let result = engine.query_pattern(&query).unwrap(); + assert_eq!(result.matches.len(), anchors.len()); + assert!(!result.truncated); + + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_fanout_physical_anchor_does_not_change_result_order() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let hub_low = insert_query_node(&engine, "Person", "hub-low", &[], 1.0); + let hub_high = insert_query_node(&engine, "Person", "hub-high", &[], 1.0); + let mut mids = Vec::new(); + for index in 0..300 { + mids.push(insert_query_node(&engine, "Article", + &format!("mid-{index:03}"), + &[], + 1.0, + )); + } + let mut anchors = Vec::new(); + for index in 0..20 { + anchors.push(insert_query_node(&engine, "Company", + &format!("anchor-{index:02}"), + &[], + 1.0, + )); + } + for &mid in &mids[..150] { + engine + .upsert_edge(hub_low, mid, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + for &mid in &mids[150..] { + engine + .upsert_edge(hub_high, mid, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } engine - .with_runtime_manifest_write(|manifest| { - let entry = manifest - .secondary_indexes - .iter_mut() - .find(|entry| entry.index_id == info.index_id) - .unwrap(); - entry.state = SecondaryIndexState::Failed; - entry.last_error = Some("forced failure".to_string()); - Ok(()) - }) + .upsert_edge(mids[0], anchors[19], "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - engine.rebuild_secondary_index_catalog().unwrap(); - let failed_plan = engine.explain_node_query(&query).unwrap(); - assert_eq!( - failed_plan.warnings, + engine + .upsert_edge(mids[150], anchors[0], "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + + let mut query = pattern_query( vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, - ] - ); - assert_plan_input_nodes(&failed_plan, vec![QueryPlanNode::FallbackTypeScan]); - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b], &query) + pattern_node("a_small_hub", Some("Person"), Vec::new()), + pattern_node("z_larger_anchor", Some("Company"), Vec::new()), + pattern_node("middle", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("hub_to_middle"), + "a_small_hub", + "middle", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("middle_to_anchor"), + "middle", + "z_larger_anchor", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), + ], ); + query.limit = 1; + + let (physical_anchor, sort_anchor, _) = + planned_pattern_anchor_sort_and_edge_aliases(&engine, &query); + assert_eq!(physical_anchor, "middle_to_anchor"); + assert_eq!(sort_anchor, "a_small_hub"); + + let result = engine.query_pattern(&query).unwrap(); + assert_eq!(result.matches.len(), 1); + assert!(result.truncated); + assert_eq!(result.matches[0].nodes["a_small_hub"], hub_low); + assert_eq!(result.matches[0].nodes["z_larger_anchor"], anchors[19]); engine.close().unwrap(); } #[test] -fn test_query_ready_sidecar_failure_returns_followup_only_for_selected_source() { +fn test_query_pattern_fanout_delays_high_hub_expansion() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let active = insert_query_node( - &engine, - 1, - "active", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let inactive = insert_query_node( - &engine, - 1, - "inactive", - &[("status", PropValue::String("inactive".to_string()))], - 1.0, - ); - engine.flush().unwrap(); - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + let low = insert_query_node(&engine, "Article", "low", &[], 1.0); + engine + .upsert_edge(root, low, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - false, - ); - let planned; - let normalized; - let policy_cutoffs; - { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - normalized = published.view.normalize_node_query(&query).unwrap(); - planned = published.view.plan_normalized_node_query(&normalized).unwrap(); - policy_cutoffs = published.view.query_policy_cutoffs(); - } - assert_plan_input_nodes( - &engine.explain_node_query(&query).unwrap(), - vec![QueryPlanNode::PropertyEqualityIndex], - ); - assert!(engine.declared_index_runtime_coverage_len_for_test() > 0); - - let seg_dir = segment_dir(&db_path, engine.segments_for_test()[0].segment_id); - let sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); - std::fs::remove_file(&sidecar_path).unwrap(); - - { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let (page, followups) = published - .view - .query_node_page_planned(&normalized, &planned, false, policy_cutoffs.as_ref()) + for index in 0..128 { + let target = insert_query_node(&engine, "Company", &format!("hub-target-{index:03}"), &[], 1.0); + engine + .upsert_edge(root, target, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - assert_eq!(page.ids, vec![active]); - assert_eq!(followups.len(), 1); } + engine.flush().unwrap(); - assert_plan_input_nodes( - &engine.explain_node_query(&query).unwrap(), - vec![QueryPlanNode::PropertyEqualityIndex], - ); - - let segment_id = engine.segments_for_test()[0].segment_id; - engine - .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) - .unwrap(); - let explain = engine.explain_node_query(&query).unwrap(); - assert_eq!( - explain.warnings, + let query = pattern_query( vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::UsingFallbackScan, - QueryPlanWarning::VerifyOnlyFilter, - ] - ); - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[active, inactive], &query) + pattern_node_with_ids("root", vec![root]), + pattern_node("hub_target", Some("Company"), Vec::new()), + pattern_node("low_target", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("aaa_hub"), + "root", + "hub_target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("zzz_low"), + "root", + "low_target", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), + ], ); + let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); + assert_eq!(edge_aliases.first().map(String::as_str), Some("zzz_low")); + engine.close().unwrap(); } #[test] -fn test_query_explicit_anchor_does_not_scan_ready_property_index() { +fn test_query_pattern_mutable_edges_preserve_deterministic_expansion_order() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let active = insert_query_node( - &engine, - 1, - "active", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - engine.flush().unwrap(); - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + let low = insert_query_node(&engine, "Article", "low", &[], 1.0); + engine + .upsert_edge(root, low, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + for index in 0..96 { + let target = insert_query_node(&engine, "Company", &format!("target-{index:02}"), &[], 1.0); + engine + .upsert_edge(root, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + engine.flush().unwrap(); - let seg_dir = segment_dir(&db_path, engine.segments_for_test()[0].segment_id); - let sidecar_path = crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id); - std::fs::remove_file(&sidecar_path).unwrap(); + let mutable_target = insert_query_node(&engine, "MissingLabel", "mutable", &[], 1.0); + engine + .upsert_edge(root, mutable_target, "MISSING_EDGE_LABEL", UpsertEdgeOptions::default()) + .unwrap(); - let mut query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - false, + let query = pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node("hub_target", Some("Company"), Vec::new()), + pattern_node("low_target", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("aaa_hub"), + "root", + "hub_target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("zzz_low"), + "root", + "low_target", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), + ], ); - query.ids = vec![active]; - - let (_followup_ready_rx, followup_release_tx) = engine.set_runtime_publish_pause(); - assert_eq!(engine.query_node_ids(&query).unwrap().items, vec![active]); - assert_eq!(engine.pending_secondary_index_followup_count_for_test(), 0); - followup_release_tx.send(()).unwrap(); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!(plan.warnings, Vec::::new()); - assert_plan_input_nodes(&plan, vec![QueryPlanNode::ExplicitIds]); + let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); + assert_eq!(edge_aliases.first().map(String::as_str), Some("aaa_hub")); engine.close().unwrap(); } #[test] -fn test_query_pagination_does_not_skip_after_rejected_candidates() { +fn test_query_pattern_target_filter_selectivity_reduces_fanout_cost() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + engine + .ensure_node_property_index("Company", "status", SecondaryIndexKind::Equality) + .unwrap(); - let reject_first = insert_query_node( - &engine, - 1, - "reject-first", - &[("score", PropValue::Int(1))], - 1.0, - ); - let accept_first = insert_query_node( - &engine, - 1, - "accept-first", - &[("score", PropValue::Int(10))], - 1.0, - ); - let reject_second = insert_query_node( - &engine, - 1, - "reject-second", - &[("score", PropValue::Int(2))], - 1.0, - ); - let accept_second = insert_query_node( - &engine, - 1, - "accept-second", - &[("score", PropValue::Int(20))], - 1.0, - ); - - let mut query = NodeQuery { - ids: vec![reject_first, accept_first, reject_second, accept_second], - filter: filter_and![NodeFilterExpr::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(10))), - upper: None, - }], - page: PageRequest { - limit: Some(1), - after: None, - }, - ..Default::default() - }; - - let first = engine.query_node_ids(&query).unwrap(); - assert_eq!(first.items, vec![accept_first]); - assert_eq!(first.next_cursor, Some(accept_first)); + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + for index in 0..128 { + let status = if index == 127 { "selected" } else { "other" }; + let target = insert_query_node(&engine, "Company", + &format!("candidate-{index:03}"), + &[("status", PropValue::String(status.to_string()))], + 1.0, + ); + engine + .upsert_edge(root, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + for index in 0..16 { + let target = insert_query_node(&engine, "Article", &format!("low-{index:02}"), &[], 1.0); + engine + .upsert_edge(root, target, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + } + engine.flush().unwrap(); - query.page.after = first.next_cursor; - let second = engine.query_node_ids(&query).unwrap(); - assert_eq!(second.items, vec![accept_second]); - assert_eq!(second.next_cursor, None); + let query = pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node("selected_target", Some("Company"), + vec![NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("selected".to_string()), + }], + ), + pattern_node("low_target", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("zzz_selective"), + "root", + "selected_target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("aaa_low"), + "root", + "low_target", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), + ], + ); - query.page.after = Some(accept_second); - let third = engine.query_node_ids(&query).unwrap(); - assert!(third.items.is_empty()); - assert!(third.next_cursor.is_none()); + let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); + assert_eq!( + edge_aliases.first().map(String::as_str), + Some("zzz_selective") + ); engine.close().unwrap(); } -// --- full-scan opt-in --- - #[test] -fn test_query_explicit_full_scan_opt_in_and_explain_warning() { +fn test_query_pattern_absent_edge_label_uses_complete_zero_fanout() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = insert_query_node( - &engine, - 1, - "a", - &[("tenant", PropValue::String("t1".to_string()))], - 1.0, - ); - let b = insert_query_node( - &engine, - 2, - "b", - &[("tenant", PropValue::String("t1".to_string()))], - 1.0, - ); - let _c = insert_query_node( - &engine, - 3, - "c", - &[("tenant", PropValue::String("t2".to_string()))], - 1.0, - ); + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + let missing_target = insert_query_node(&engine, "Topic", "missing-target", &[], 1.0); + for index in 0..64 { + let target = insert_query_node(&engine, "Company", &format!("target-{index:02}"), &[], 1.0); + engine + .upsert_edge(root, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + engine.ensure_edge_label("SPECIAL_EDGE_999").unwrap(); + engine.flush().unwrap(); - let query = query_ids( - None, - vec![NodeFilterExpr::PropertyEquals { - key: "tenant".to_string(), - value: PropValue::String("t1".to_string()), - }], - true, + let query = pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node("hub_target", Some("Company"), Vec::new()), + pattern_node_with_ids("missing_target", vec![missing_target]), + ], + vec![ + pattern_edge( + Some("aaa_hub"), + "root", + "hub_target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("zzz_missing"), + "root", + "missing_target", + Direction::Outgoing, + Some(vec!["SPECIAL_EDGE_999"]), + ), + ], ); + + let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[a, b, _c], &query) + edge_aliases.first().map(String::as_str), + Some("zzz_missing") ); + let result = engine.query_pattern(&query).unwrap(); + assert!(result.matches.is_empty()); - let plan = engine.explain_node_query(&query).unwrap(); - assert_eq!( - plan.warnings, + engine.close().unwrap(); +} + +#[test] +fn test_query_pattern_both_bound_constraint_stays_before_unbound_expansion() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Company", "b", &[], 1.0); + let c = insert_query_node(&engine, "Article", "c", &[], 1.0); + engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(a, c, "RATES", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(b, a, "BLOCKS", UpsertEdgeOptions::default()) + .unwrap(); + for index in 0..96 { + let dummy = insert_query_node(&engine, "Topic", &format!("dummy-{index:03}"), &[], 1.0); + engine + .upsert_edge(b, dummy, "BLOCKS", UpsertEdgeOptions::default()) + .unwrap(); + } + engine.flush().unwrap(); + + let query = pattern_query( vec![ - QueryPlanWarning::MissingReadyIndex, - QueryPlanWarning::FullScanExplicitlyAllowed, - QueryPlanWarning::VerifyOnlyFilter, - ] + pattern_node_with_ids("a", vec![a]), + pattern_node("b", Some("Company"), Vec::new()), + pattern_node("c", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("aaa_bind_b"), + "a", + "b", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("zzz_constraint"), + "b", + "a", + Direction::Outgoing, + Some(vec!["BLOCKS"]), + ), + pattern_edge( + Some("zzz_unbound"), + "a", + "c", + Direction::Outgoing, + Some(vec!["RATES"]), + ), + ], ); - assert!(matches!( - plan.root, - QueryPlanNode::VerifyNodeFilter { .. } - )); + + let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); + assert_eq!(edge_aliases[0], "aaa_bind_b"); + assert_eq!(edge_aliases[1], "zzz_constraint"); engine.close().unwrap(); } -// --- visibility matrix --- - #[test] -fn test_query_scan_parity_after_flush_reopen_overwrite_delete_and_prune() { +fn test_query_pattern_missing_fanout_stats_preserves_deterministic_order() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + let low = insert_query_node(&engine, "Article", "low", &[], 1.0); + engine + .upsert_edge(root, low, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + for index in 0..64 { + let target = insert_query_node(&engine, "Company", &format!("target-{index:02}"), &[], 1.0); + engine + .upsert_edge(root, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + engine.flush().unwrap(); + engine.close().unwrap(); + } + + let stats_path = crate::segment_writer::segment_dir(&db_path, 1) + .join(crate::planner_stats::PLANNER_STATS_FILENAME); + std::fs::remove_file(stats_path).unwrap(); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let root = reopened.get_node_by_key("Person", "root").unwrap().unwrap().id; + let query = pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node("hub_target", Some("Company"), Vec::new()), + pattern_node("low_target", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("aaa_hub"), + "root", + "hub_target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("zzz_low"), + "root", + "low_target", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), + ], + ); + + let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&reopened, &query); + assert_eq!(edge_aliases.first().map(String::as_str), Some("aaa_hub")); + + reopened.close().unwrap(); +} + +fn fanout_parity_query(root: u64) -> GraphPatternQuery { + pattern_query( + vec![ + pattern_node_with_ids("root", vec![root]), + pattern_node("hub_target", Some("Company"), Vec::new()), + pattern_node("low_target", Some("Article"), Vec::new()), + ], + vec![ + pattern_edge( + Some("hub_edge"), + "root", + "hub_target", + Direction::Outgoing, + Some(vec!["KNOWS"]), + ), + pattern_edge( + Some("low_edge"), + "root", + "low_target", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), + ], + ) +} + +fn insert_fanout_parity_tail(engine: &DatabaseEngine, root: u64, start: usize, count: usize) { + for index in start..start + count { + let target = insert_query_node(engine, "Company", &format!("target-{index:02}"), &[], 1.0); + engine + .upsert_edge(root, target, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } +} + +fn assert_fanout_parity_result(engine: &DatabaseEngine, root: u64, expected_matches: &[QueryMatch]) { + let query = fanout_parity_query(root); + let result = engine.query_pattern(&query).unwrap(); + assert_eq!(result.matches, expected_matches); + assert!(!result.truncated); +} + +#[test] +fn test_query_pattern_fanout_stats_preserve_results_across_storage_states() { + { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("memtable"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + insert_fanout_parity_tail(&engine, root, 0, 8); + let low = insert_query_node(&engine, "Article", "low", &[], 1.0); + let low_edge = engine + .upsert_edge(root, low, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + let expected: Vec<_> = (0..8) + .map(|index| { + let target = engine + .get_node_by_key("Company", &format!("target-{index:02}")) + .unwrap() + .unwrap() + .id; + let hub_edge = engine + .get_edge_by_triple(root, target, "KNOWS") + .unwrap() + .unwrap() + .id; + expected_match( + &[("hub_target", target), ("low_target", low), ("root", root)], + &[("hub_edge", hub_edge), ("low_edge", low_edge)], + ) + }) + .collect(); + assert_fanout_parity_result(&engine, root, &expected); + engine.close().unwrap(); + } + + { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("flushed"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + insert_fanout_parity_tail(&engine, root, 0, 8); + let low = insert_query_node(&engine, "Article", "low", &[], 1.0); + let low_edge = engine + .upsert_edge(root, low, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + let expected: Vec<_> = (0..8) + .map(|index| { + let target = engine + .get_node_by_key("Company", &format!("target-{index:02}")) + .unwrap() + .unwrap() + .id; + let hub_edge = engine + .get_edge_by_triple(root, target, "KNOWS") + .unwrap() + .unwrap() + .id; + expected_match( + &[("hub_target", target), ("low_target", low), ("root", root)], + &[("hub_edge", hub_edge), ("low_edge", low_edge)], + ) + }) + .collect(); + assert_fanout_parity_result(&engine, root, &expected); + engine.close().unwrap(); + } + + { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("reopened"); + let root = { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + insert_fanout_parity_tail(&engine, root, 0, 8); + let low = insert_query_node(&engine, "Article", "low", &[], 1.0); + engine + .upsert_edge(root, low, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine.flush().unwrap(); + engine.close().unwrap(); + root + }; + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let low = reopened.get_node_by_key("Article", "low").unwrap().unwrap().id; + let low_edge = reopened + .get_edge_by_triple(root, low, "REPORTS_TO") + .unwrap() + .unwrap() + .id; + let expected: Vec<_> = (0..8) + .map(|index| { + let target = reopened + .get_node_by_key("Company", &format!("target-{index:02}")) + .unwrap() + .unwrap() + .id; + let hub_edge = reopened + .get_edge_by_triple(root, target, "KNOWS") + .unwrap() + .unwrap() + .id; + expected_match( + &[("hub_target", target), ("low_target", low), ("root", root)], + &[("hub_edge", hub_edge), ("low_edge", low_edge)], + ) + }) + .collect(); + assert_fanout_parity_result(&reopened, root, &expected); + reopened.close().unwrap(); + } - let keep; - let deleted; - let low; { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("compacted"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - keep = insert_query_node( - &engine, - 1, - "keep", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - deleted = insert_query_node( - &engine, - 1, - "delete", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - low = insert_query_node( - &engine, - 1, - "low", - &[("status", PropValue::String("active".to_string()))], - 0.1, - ); + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + insert_fanout_parity_tail(&engine, root, 0, 4); engine.flush().unwrap(); - - insert_query_node( - &engine, - 1, - "keep", - &[("status", PropValue::String("inactive".to_string()))], - 1.0, - ); - insert_query_node( - &engine, - 1, - "keep", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - engine.delete_node(deleted).unwrap(); - engine - .set_prune_policy( - "low-weight", - PrunePolicy { - max_age_ms: None, - max_weight: Some(0.5), - type_id: Some(1), - }, - ) + insert_fanout_parity_tail(&engine, root, 4, 4); + let low = insert_query_node(&engine, "Article", "low", &[], 1.0); + let low_edge = engine + .upsert_edge(root, low, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - false, - ); - assert_eq!( - engine.query_node_ids(&query).unwrap().items, - oracle_query_ids(&engine, &[keep, deleted, low], &query) - ); - assert!(engine.get_node(low).unwrap().is_none()); + engine.flush().unwrap(); + engine.compact().unwrap().unwrap(); + let expected: Vec<_> = (0..8) + .map(|index| { + let target = engine + .get_node_by_key("Company", &format!("target-{index:02}")) + .unwrap() + .unwrap() + .id; + let hub_edge = engine + .get_edge_by_triple(root, target, "KNOWS") + .unwrap() + .unwrap() + .id; + expected_match( + &[("hub_target", target), ("low_target", low), ("root", root)], + &[("hub_edge", hub_edge), ("low_edge", low_edge)], + ) + }) + .collect(); + assert_fanout_parity_result(&engine, root, &expected); engine.close().unwrap(); } - - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let query = query_ids( - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - false, - ); - assert_eq!( - reopened.query_node_ids(&query).unwrap().items, - oracle_query_ids(&reopened, &[keep, deleted, low], &query) - ); - reopened.close().unwrap(); } -// --- graph pattern execution --- - #[test] -fn test_query_pattern_validation_rejects_invalid_shapes() { +fn test_query_pattern_distinct_edge_aliases_may_share_edge_id() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let empty = pattern_query(Vec::new(), Vec::new()); - assert!(matches!( - engine.query_pattern(&empty).unwrap_err(), - EngineError::InvalidOperation(_) - )); - - let node_only = pattern_query(vec![pattern_node("a", Some(1), Vec::new())], Vec::new()); - assert!(matches!( - engine.explain_pattern_query(&node_only).unwrap_err(), - EngineError::InvalidOperation(_) - )); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Company", "b", &[], 1.0); + let edge = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); - let duplicate_node = pattern_query( + let query = pattern_query( + vec![pattern_node_with_ids("a", vec![a]), pattern_node("b", Some("Company"), Vec::new())], vec![ - pattern_node("a", Some(1), Vec::new()), - pattern_node("a", Some(2), Vec::new()), + pattern_edge(Some("first"), "a", "b", Direction::Outgoing, Some(vec!["KNOWS"])), + pattern_edge(Some("second"), "a", "b", Direction::Outgoing, Some(vec!["KNOWS"])), ], - vec![pattern_edge(Some("e"), "a", "a", Direction::Outgoing, None)], ); - assert!(matches!( - engine.query_pattern(&duplicate_node).unwrap_err(), - EngineError::InvalidOperation(_) - )); - let empty_node_alias = pattern_query( - vec![pattern_node("", Some(1), Vec::new())], - vec![pattern_edge(Some("e"), "", "", Direction::Outgoing, None)], + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[("a", a), ("b", b)], + &[("first", edge), ("second", edge)] + )] ); - assert!(matches!( - engine.query_pattern(&empty_node_alias).unwrap_err(), - EngineError::InvalidOperation(_) - )); - let bad_reference = pattern_query( - vec![pattern_node("a", Some(1), Vec::new())], - vec![pattern_edge(Some("e"), "a", "missing", Direction::Outgoing, None)], - ); - assert!(matches!( - engine.query_pattern(&bad_reference).unwrap_err(), - EngineError::InvalidOperation(_) - )); + engine.close().unwrap(); +} - let empty_edge_alias = pattern_query( - vec![pattern_node("a", Some(1), Vec::new())], - vec![pattern_edge(Some(""), "a", "a", Direction::Outgoing, None)], - ); - assert!(matches!( - engine.query_pattern(&empty_edge_alias).unwrap_err(), - EngineError::InvalidOperation(_) - )); +#[test] +fn test_query_pattern_branching_distinct_aliases_and_no_match() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let duplicate_edge_alias = pattern_query( + let root = insert_query_node(&engine, "Person", "root", &[], 1.0); + let left = insert_query_node(&engine, "Company", "left", &[], 1.0); + let right = insert_query_node(&engine, "Article", "right", &[], 1.0); + let shared = insert_query_node(&engine, "Topic", "shared", &[], 1.0); + let left_edge = engine + .upsert_edge(root, left, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let right_edge = engine + .upsert_edge(root, right, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(root, shared, "RATES", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(root, shared, "EDGE_LABEL_40", UpsertEdgeOptions::default()) + .unwrap(); + + let query = pattern_query( vec![ - pattern_node("a", Some(1), Vec::new()), - pattern_node("b", Some(2), Vec::new()), + pattern_node_with_ids("root", vec![root]), + pattern_node("left", Some("Company"), Vec::new()), + pattern_node("right", Some("Article"), Vec::new()), ], vec![ - pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None), - pattern_edge(Some("e"), "b", "a", Direction::Outgoing, None), + pattern_edge(Some("left_edge"), "root", "left", Direction::Outgoing, Some(vec!["KNOWS"])), + pattern_edge( + Some("right_edge"), + "root", + "right", + Direction::Outgoing, + Some(vec!["REPORTS_TO"]), + ), ], ); - assert!(matches!( - engine.query_pattern(&duplicate_edge_alias).unwrap_err(), - EngineError::InvalidOperation(_) - )); - - let zero_limit = GraphPatternQuery { - limit: 0, - ..pattern_query( - vec![ - pattern_node("a", Some(1), Vec::new()), - pattern_node("b", Some(2), Vec::new()), - ], - vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], - ) - }; - assert!(matches!( - engine.query_pattern(&zero_limit).unwrap_err(), - EngineError::InvalidOperation(_) - )); - - let mut key_without_type = pattern_node("a", None, Vec::new()); - key_without_type.keys.push("a".to_string()); - let key_query = pattern_query( - vec![key_without_type, pattern_node("b", Some(2), Vec::new())], - vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], - ); - assert!(matches!( - engine.query_pattern(&key_query).unwrap_err(), - EngineError::InvalidOperation(_) - )); - - let disconnected_extra_node = pattern_query( - vec![ - pattern_node("a", Some(1), Vec::new()), - pattern_node("b", Some(2), Vec::new()), - pattern_node("c", Some(3), Vec::new()), - ], - vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[("left", left), ("right", right), ("root", root)], + &[("left_edge", left_edge), ("right_edge", right_edge)] + )] ); - assert!(matches!( - engine.query_pattern(&disconnected_extra_node).unwrap_err(), - EngineError::InvalidOperation(_) - )); - let disconnected_components = pattern_query( + let distinct_alias_query = pattern_query( vec![ - pattern_node("a", Some(1), Vec::new()), - pattern_node("b", Some(2), Vec::new()), - pattern_node("c", Some(3), Vec::new()), - pattern_node("d", Some(4), Vec::new()), + pattern_node_with_ids("root", vec![root]), + pattern_node("x", Some("Topic"), Vec::new()), + pattern_node("y", Some("Topic"), Vec::new()), ], vec![ - pattern_edge(Some("ab"), "a", "b", Direction::Outgoing, None), - pattern_edge(Some("cd"), "c", "d", Direction::Outgoing, None), + pattern_edge(Some("x_edge"), "root", "x", Direction::Outgoing, Some(vec!["RATES"])), + pattern_edge(Some("y_edge"), "root", "y", Direction::Outgoing, Some(vec!["REFERENCES"])), ], ); - assert!(matches!( - engine.query_pattern(&disconnected_components).unwrap_err(), - EngineError::InvalidOperation(_) - )); + assert!(engine + .query_pattern(&distinct_alias_query) + .unwrap() + .matches + .is_empty()); - let unanchored = pattern_query( - vec![pattern_node("a", None, Vec::new()), pattern_node("b", None, Vec::new())], - vec![pattern_edge(Some("e"), "a", "b", Direction::Outgoing, None)], + let no_match_query = pattern_query( + vec![pattern_node_with_ids("root", vec![root]), pattern_node("missing", Some("MissingLabel"), Vec::new())], + vec![pattern_edge(None, "root", "missing", Direction::Outgoing, None)], ); - assert!(matches!( - engine.explain_pattern_query(&unanchored).unwrap_err(), - EngineError::InvalidOperation(_) - )); + assert!(engine + .query_pattern(&no_match_query) + .unwrap() + .matches + .is_empty()); engine.close().unwrap(); } #[test] -fn test_query_pattern_anchor_selection_uses_type_cardinality() { +fn test_query_pattern_cycle_closing_edge_and_self_loop() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - for index in 0..8 { - insert_query_node(&engine, 1, &format!("wide-{index}"), &[], 1.0); - } - insert_query_node(&engine, 2, "narrow", &[], 1.0); + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Company", "b", &[], 1.0); + let c = insert_query_node(&engine, "Article", "c", &[], 1.0); + let ab = engine.upsert_edge(a, b, "KNOWS", Default::default()).unwrap(); + let bc = engine.upsert_edge(b, c, "KNOWS", Default::default()).unwrap(); + let ca = engine.upsert_edge(c, a, "KNOWS", Default::default()).unwrap(); + let aa = engine.upsert_edge(a, a, "MISSING_EDGE_LABEL", Default::default()).unwrap(); - let query = pattern_query( + let cycle = pattern_query( vec![ - pattern_node("aaa_wide", Some(1), Vec::new()), - pattern_node("zzz_narrow", Some(2), Vec::new()), + pattern_node_with_ids("a", vec![a]), + pattern_node("b", Some("Company"), Vec::new()), + pattern_node("c", Some("Article"), Vec::new()), ], - vec![pattern_edge( - Some("edge"), - "aaa_wide", - "zzz_narrow", - Direction::Both, - None, - )], + vec![ + pattern_edge(Some("ab"), "a", "b", Direction::Outgoing, Some(vec!["KNOWS"])), + pattern_edge(Some("bc"), "b", "c", Direction::Outgoing, Some(vec!["KNOWS"])), + pattern_edge(Some("ca"), "c", "a", Direction::Outgoing, Some(vec!["KNOWS"])), + ], + ); + assert_eq!( + engine.query_pattern(&cycle).unwrap().matches, + vec![expected_match( + &[("a", a), ("b", b), ("c", c)], + &[("ab", ab), ("bc", bc), ("ca", ca)] + )] ); - { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let normalized = published.view.normalize_pattern_query(&query).unwrap(); - let planned = published - .view - .plan_normalized_pattern_query(&normalized) - .unwrap(); - assert_eq!(normalized.nodes[planned.anchor_index].alias, "zzz_narrow"); - } + let self_loop = pattern_query( + vec![pattern_node_with_ids("a", vec![a])], + vec![pattern_edge(Some("loop"), "a", "a", Direction::Outgoing, Some(vec!["MISSING_EDGE_LABEL"]))], + ); + assert_eq!( + engine.query_pattern(&self_loop).unwrap().matches, + vec![expected_match(&[("a", a)], &[("loop", aa)])] + ); engine.close().unwrap(); } #[test] -fn test_query_pattern_linear_uses_reverse_direction_from_selective_anchor() { +fn test_query_pattern_edge_property_post_filters_and_explain_warning() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let alice = insert_query_node( - &engine, - 1, - "alice", - &[("status", PropValue::String("active".to_string()))], - 1.0, - ); - let _bob = insert_query_node( - &engine, - 1, - "bob", - &[("status", PropValue::String("inactive".to_string()))], - 1.0, + let a = insert_query_node(&engine, "Person", "a", &[], 1.0); + let b = insert_query_node(&engine, "Company", "b", &[], 1.0); + let c = insert_query_node(&engine, "Company", "c", &[], 1.0); + let good = engine + .upsert_edge( + a, + b, + "KNOWS", + UpsertEdgeOptions { + props: query_test_props(&[ + ("rel", PropValue::String("friend".to_string())), + ("score", PropValue::Int(5)), + ]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "KNOWS", + UpsertEdgeOptions { + props: query_test_props(&[ + ("rel", PropValue::String("friend".to_string())), + ("score", PropValue::Int(1)), + ]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let mut edge = pattern_edge(Some("e"), "a", "target", Direction::Outgoing, Some(vec!["KNOWS"])); + edge.filter = Some(EdgeFilterExpr::And(vec![ + EdgeFilterExpr::PropertyEquals { + key: "rel".to_string(), + value: PropValue::String("friend".to_string()), + }, + EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(3))), + upper: None, + }, + ])); + let query = pattern_query( + vec![pattern_node_with_ids("a", vec![a]), pattern_node("target", Some("Company"), Vec::new())], + vec![edge], ); - let acme = insert_query_node( - &engine, - 2, - "acme", - &[("tier", PropValue::String("enterprise".to_string()))], - 1.0, + + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("a", a), ("target", b)], &[("e", good)])] ); - let edge = engine - .upsert_edge(alice, acme, 10, UpsertEdgeOptions::default()) + + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan + .warnings + .contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert!(matches!( + plan.root, + QueryPlanNode::VerifyEdgePredicates { .. } + )); + + engine.close().unwrap(); +} + +#[test] +fn pattern_edge_property_equality_uses_edge_anchor_when_selective() { + let (_dir, engine) = query_test_engine(); + let mut sources = Vec::new(); + let mut targets = Vec::new(); + for index in 0..256 { + sources.push(insert_query_node(&engine, "NodeLabel101", + &format!("edge-anchor-eq-source-{index}"), + &[], + 1.0, + )); + targets.push(insert_query_node(&engine, "NodeLabel102", + &format!("edge-anchor-eq-target-{index}"), + &[], + 1.0, + )); + } + + let hot_edge = engine + .upsert_edge( + sources[7], + targets[9], + "EDGE_LABEL_201", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + for index in 0..256 { + engine + .upsert_edge( + sources[index], + targets[index], + "EDGE_LABEL_201", + UpsertEdgeOptions { + props: query_test_props(&[( + "status", + PropValue::String("cold".to_string()), + )]), + ..Default::default() + }, + ) + .unwrap(); + } + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_201", "status", SecondaryIndexKind::Equality) .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + let mut edge = pattern_edge(Some("rel"), "source", "target", Direction::Outgoing, Some(vec!["EDGE_LABEL_201"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( vec![ - pattern_node( - "person", - Some(1), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("active".to_string()), - }], - ), - pattern_node_with_ids("company", vec![acme]), + pattern_node("source", Some("NodeLabel101"), Vec::new()), + pattern_node("target", Some("NodeLabel102"), Vec::new()), ], - vec![pattern_edge( - Some("works_at"), - "person", - "company", - Direction::Outgoing, - Some(vec![10]), - )], + vec![edge], ); - let result = engine.query_pattern(&query).unwrap(); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(!plan.warnings.contains(&QueryPlanWarning::EdgePropertyPostFilter)); + assert!(!plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + + engine.reset_query_execution_counters_for_test(); assert_eq!( - result.matches, + engine.query_pattern(&query).unwrap().matches, vec![expected_match( - &[("company", acme), ("person", alice)], - &[("works_at", edge)] + &[("source", sources[7]), ("target", targets[9])], + &[("rel", hot_edge)] )] ); - assert!(!result.truncated); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.endpoint_adjacency_candidates, 0); + assert_eq!(counters.edge_record_hydration_reads, 0); +} + +#[test] +fn pattern_edge_property_range_uses_edge_anchor_when_selective() { + let (_dir, engine) = query_test_engine(); + let left = insert_query_node(&engine, "NodeLabel103", "edge-anchor-range-left", &[], 1.0); + let mut keep_target = 0; + let mut keep_edge = 0; + for index in 0..128 { + let target = insert_query_node(&engine, "NodeLabel104", + &format!("edge-anchor-range-target-{index}"), + &[], + 1.0, + ); + let score = if index == 77 { 1_000 } else { index as i64 }; + let edge_id = engine + .upsert_edge( + left, + target, + "EDGE_LABEL_202", + UpsertEdgeOptions { + props: query_test_props(&[("score", PropValue::Int(score))]), + ..Default::default() + }, + ) + .unwrap(); + if index == 77 { + keep_target = target; + keep_edge = edge_id; + } + } + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_202", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let mut edge = pattern_edge(Some("rel"), "left", "right", Direction::Outgoing, Some(vec!["EDGE_LABEL_202"])); + edge.filter = Some(EdgeFilterExpr::PropertyRange { + key: "score".to_string(), + lower: Some(PropertyRangeBound::Included(PropValue::Int(1_000))), + upper: Some(PropertyRangeBound::Included(PropValue::Int(1_000))), + }); + let query = pattern_query( + vec![ + pattern_node_with_ids("left", vec![left]), + pattern_node("right", Some("NodeLabel104"), Vec::new()), + ], + vec![edge], + ); let plan = engine.explain_pattern_query(&query).unwrap(); - assert_eq!(plan.kind, QueryPlanKind::PatternQuery); - assert!(matches!(plan.root, QueryPlanNode::PatternExpand { .. })); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyRangeIndex + )); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[("left", left), ("right", keep_target)], + &[("rel", keep_edge)] + )] + ); +} + +#[test] +fn pattern_edge_anchor_direction_both_and_self_loop_bind_correctly() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "NodeLabel105", "edge-anchor-dir-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel105", "edge-anchor-dir-b", &[], 1.0); + let ab = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_203", + UpsertEdgeOptions { + props: query_test_props(&[("kind", PropValue::String("keep".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let aa = engine + .upsert_edge( + a, + a, + "EDGE_LABEL_203", + UpsertEdgeOptions { + props: query_test_props(&[("kind", PropValue::String("loop".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_203", "kind", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let mut outgoing = pattern_edge(Some("e"), "left", "right", Direction::Outgoing, Some(vec!["EDGE_LABEL_203"])); + outgoing.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "kind".to_string(), + value: PropValue::String("keep".to_string()), + }); + let outgoing_query = pattern_query( + vec![ + pattern_node("left", Some("NodeLabel105"), Vec::new()), + pattern_node("right", Some("NodeLabel105"), Vec::new()), + ], + vec![outgoing], + ); + assert_eq!( + engine.query_pattern(&outgoing_query).unwrap().matches, + vec![expected_match(&[("left", a), ("right", b)], &[("e", ab)])] + ); + + let mut incoming = pattern_edge(Some("e"), "left", "right", Direction::Incoming, Some(vec!["EDGE_LABEL_203"])); + incoming.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "kind".to_string(), + value: PropValue::String("keep".to_string()), + }); + let incoming_query = pattern_query( + vec![ + pattern_node("left", Some("NodeLabel105"), Vec::new()), + pattern_node("right", Some("NodeLabel105"), Vec::new()), + ], + vec![incoming], + ); + assert_eq!( + engine.query_pattern(&incoming_query).unwrap().matches, + vec![expected_match(&[("left", b), ("right", a)], &[("e", ab)])] + ); + + let mut both = pattern_edge(Some("e"), "left", "right", Direction::Both, Some(vec!["EDGE_LABEL_203"])); + both.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "kind".to_string(), + value: PropValue::String("keep".to_string()), + }); + let both_query = pattern_query( + vec![ + pattern_node("left", Some("NodeLabel105"), Vec::new()), + pattern_node("right", Some("NodeLabel105"), Vec::new()), + ], + vec![both], + ); + assert_eq!( + engine.query_pattern(&both_query).unwrap().matches, + vec![ + expected_match(&[("left", a), ("right", b)], &[("e", ab)]), + expected_match(&[("left", b), ("right", a)], &[("e", ab)]), + ] + ); - engine.close().unwrap(); + let mut loop_edge = pattern_edge(Some("loop"), "same", "same", Direction::Both, Some(vec!["EDGE_LABEL_203"])); + loop_edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "kind".to_string(), + value: PropValue::String("loop".to_string()), + }); + let loop_query = pattern_query( + vec![pattern_node("same", Some("NodeLabel105"), Vec::new())], + vec![loop_edge], + ); + assert_eq!( + engine.query_pattern(&loop_query).unwrap().matches, + vec![expected_match(&[("same", a)], &[("loop", aa)])] + ); } #[test] -fn test_query_pattern_typeless_predicate_target_verifies_after_expansion() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let anchor = insert_query_node(&engine, 1, "anchor", &[], 1.0); - let good = insert_query_node( - &engine, - 2, - "good", - &[("status", PropValue::String("match".to_string()))], +fn pattern_edge_anchor_verifies_endpoint_node_filters_after_binding() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "NodeLabel106", "edge-anchor-node-filter-source", &[], 1.0); + let keep = insert_query_node(&engine, "NodeLabel107", + "edge-anchor-node-filter-keep", + &[("state", PropValue::String("ok".to_string()))], 1.0, ); - let bad = insert_query_node( - &engine, - 3, - "bad", - &[("status", PropValue::String("skip".to_string()))], + let drop = insert_query_node(&engine, "NodeLabel107", + "edge-anchor-node-filter-drop", + &[("state", PropValue::String("drop".to_string()))], 1.0, ); - let good_edge = engine - .upsert_edge(anchor, good, 10, UpsertEdgeOptions::default()) + let keep_edge = engine + .upsert_edge( + source, + keep, + "EDGE_LABEL_204", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) .unwrap(); engine - .upsert_edge(anchor, bad, 10, UpsertEdgeOptions::default()) + .upsert_edge( + source, + drop, + "EDGE_LABEL_204", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_204", "status", SecondaryIndexKind::Equality) .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + let mut edge = pattern_edge(Some("rel"), "source", "target", Direction::Outgoing, Some(vec!["EDGE_LABEL_204"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( vec![ - pattern_node_with_ids("anchor", vec![anchor]), - pattern_node( - "target", - None, + pattern_node("source", Some("NodeLabel106"), Vec::new()), + pattern_node("target", Some("NodeLabel107"), vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("match".to_string()), + key: "state".to_string(), + value: PropValue::String("ok".to_string()), }], ), ], - vec![pattern_edge( - Some("edge"), - "anchor", - "target", - Direction::Outgoing, - Some(vec![10]), - )], + vec![edge], ); - { - let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let normalized = published.view.normalize_pattern_query(&query).unwrap(); - let planned = published - .view - .plan_normalized_pattern_query(&normalized) - .unwrap(); - assert_eq!(normalized.nodes[planned.anchor_index].alias, "anchor"); - } - + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + engine.reset_query_execution_counters_for_test(); assert_eq!( engine.query_pattern(&query).unwrap().matches, vec![expected_match( - &[("anchor", anchor), ("target", good)], - &[("edge", good_edge)] + &[("source", source), ("target", keep)], + &[("rel", keep_edge)] )] ); - - engine.close().unwrap(); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert!(counters.node_record_hydration_reads > 0); + assert_eq!(counters.edge_record_hydration_reads, 0); } #[test] -fn test_query_pattern_order_limit_truncated_and_direction_both() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - let low = insert_query_node(&engine, 2, "low", &[], 1.0); - let high = insert_query_node(&engine, 2, "high", &[], 1.0); - let high_edge = engine - .upsert_edge(root, high, 10, UpsertEdgeOptions::default()) +fn pattern_metadata_edge_anchor_uses_metadata_without_property_hydration() { + let (_dir, engine) = query_test_engine(); + let left = insert_query_node(&engine, "NodeLabel108", "edge-anchor-meta-left", &[], 1.0); + let keep = insert_query_node(&engine, "NodeLabel109", "edge-anchor-meta-keep", &[], 1.0); + let drop = insert_query_node(&engine, "NodeLabel109", "edge-anchor-meta-drop", &[], 1.0); + let keep_edge = engine + .upsert_edge( + left, + keep, + "EDGE_LABEL_205", + UpsertEdgeOptions { + weight: 0.25, + ..Default::default() + }, + ) .unwrap(); - let low_edge = engine - .upsert_edge(low, root, 10, UpsertEdgeOptions::default()) + engine + .upsert_edge( + left, + drop, + "EDGE_LABEL_205", + UpsertEdgeOptions { + weight: 9.0, + ..Default::default() + }, + ) .unwrap(); + engine.flush().unwrap(); - let base = pattern_query( - vec![pattern_node_with_ids("root", vec![root]), pattern_node("target", Some(2), Vec::new())], - vec![pattern_edge(Some("edge"), "root", "target", Direction::Both, Some(vec![10]))], + let mut edge = pattern_edge(Some("rel"), "left", "right", Direction::Outgoing, Some(vec!["EDGE_LABEL_205"])); + edge.filter = Some(EdgeFilterExpr::WeightRange { + lower: None, + upper: Some(1.0), + }); + let query = pattern_query( + vec![ + pattern_node("left", Some("NodeLabel108"), Vec::new()), + pattern_node("right", Some("NodeLabel109"), Vec::new()), + ], + vec![edge], ); - let limited = GraphPatternQuery { limit: 1, ..base.clone() }; - let limited_result = engine.query_pattern(&limited).unwrap(); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!( + plan_contains_node(&plan.root, &QueryPlanNode::EdgeWeightIndex) + || plan_contains_node(&plan.root, &QueryPlanNode::EdgeMetadataScan) + ); + engine.reset_query_execution_counters_for_test(); assert_eq!( - limited_result.matches, + engine.query_pattern(&query).unwrap().matches, vec![expected_match( - &[("root", root), ("target", low)], - &[("edge", low_edge)] + &[("left", left), ("right", keep)], + &[("rel", keep_edge)] )] ); - assert!(limited_result.truncated); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.edge_record_hydration_reads, 0); +} - let full_result = engine.query_pattern(&base).unwrap(); +#[test] +fn pattern_label_only_edge_anchor_binds_unanchored_aliases() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "SearchNode120", "label-only-anchor-a", &[], 1.0); + let b = insert_query_node(&engine, "SearchNode120", "label-only-anchor-b", &[], 1.0); + let c = insert_query_node(&engine, "SearchNode120", "label-only-anchor-c", &[], 1.0); + let d = insert_query_node(&engine, "SearchNode120", "label-only-anchor-d", &[], 1.0); + let ab = engine.upsert_edge(a, b, "EDGE_LABEL_215", UpsertEdgeOptions::default()).unwrap(); + let cd = engine.upsert_edge(c, d, "EDGE_LABEL_215", UpsertEdgeOptions::default()).unwrap(); + engine.upsert_edge(a, d, "EDGE_LABEL_216", UpsertEdgeOptions::default()).unwrap(); + + let query = pattern_query( + vec![ + pattern_node("from", None, Vec::new()), + pattern_node("to", None, Vec::new()), + ], + vec![pattern_edge( + Some("e"), + "from", + "to", + Direction::Outgoing, + Some(vec!["EDGE_LABEL_215"]), + )], + ); + + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeLabelIndex)); assert_eq!( - full_result.matches, + engine.query_pattern(&query).unwrap().matches, vec![ - expected_match(&[("root", root), ("target", low)], &[("edge", low_edge)]), - expected_match(&[("root", root), ("target", high)], &[("edge", high_edge)]), + expected_match(&[("from", a), ("to", b)], &[("e", ab)]), + expected_match(&[("from", c), ("to", d)], &[("e", cd)]), ] ); - assert!(!full_result.truncated); - - engine.close().unwrap(); } #[test] -fn test_query_pattern_high_fanout_limit_keeps_deterministic_top_matches() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - let mut targets = Vec::new(); - for index in 0..32 { - targets.push(insert_query_node( - &engine, - 2, - &format!("target-{index:02}"), - &[], - 1.0, - )); - } - - let mut edges_by_target = BTreeMap::new(); - for &target in targets.iter().rev() { - let edge = engine - .upsert_edge(root, target, 10, UpsertEdgeOptions::default()) - .unwrap(); - edges_by_target.insert(target, edge); - } - - let query = GraphPatternQuery { - limit: 5, - ..pattern_query( - vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("target", Some(2), Vec::new()), - ], - vec![pattern_edge( - Some("edge"), - "root", - "target", - Direction::Outgoing, - Some(vec![10]), - )], - ) - }; - - let result = engine.query_pattern(&query).unwrap(); - let expected: Vec = targets +fn pattern_broad_label_only_edge_anchor_is_rejected_when_over_cap() { + let (_dir, engine) = query_test_engine(); + let edge_count = crate::planner_stats::PLANNER_STATS_DEFAULT_SELECTED_SOURCE_CAP + 1; + let nodes = (0..=edge_count) + .map(|index| NodeInput { + labels: vec!["SearchNode120".to_string()], + key: format!("broad-edge-label-pattern-node-{index}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect::>(); + let node_ids = engine.batch_upsert_nodes(nodes).unwrap(); + let hub = node_ids[0]; + let edges = node_ids[1..] .iter() - .take(5) - .map(|&target| { - expected_match( - &[("root", root), ("target", target)], - &[("edge", edges_by_target[&target])], - ) + .map(|target| EdgeInput { + from: hub, + to: *target, + label: "EDGE_LABEL_216".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, }) - .collect(); - assert_eq!(result.matches, expected); - assert!(result.truncated); - - engine.close().unwrap(); -} - -#[test] -fn test_query_pattern_fanout_cost_can_choose_larger_lower_expansion_anchor() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let hub = insert_query_node(&engine, 1, "hub", &[], 1.0); - let mut mids = Vec::new(); - for index in 0..300 { - let mid = insert_query_node(&engine, 3, &format!("mid-{index:03}"), &[], 1.0); - engine - .upsert_edge(hub, mid, 10, UpsertEdgeOptions::default()) - .unwrap(); - mids.push(mid); - } - let mut anchors = Vec::new(); - for (index, &mid) in mids.iter().enumerate().take(20) { - let anchor = insert_query_node(&engine, 2, &format!("anchor-{index:02}"), &[], 1.0); - engine - .upsert_edge(mid, anchor, 20, UpsertEdgeOptions::default()) - .unwrap(); - anchors.push(anchor); - } - engine.flush().unwrap(); + .collect::>(); + engine.batch_upsert_edges(edges).unwrap(); let query = pattern_query( vec![ - pattern_node("small_hub", Some(1), Vec::new()), - pattern_node("larger_anchor", Some(2), Vec::new()), - pattern_node("middle", Some(3), Vec::new()), - ], - vec![ - pattern_edge( - Some("hub_to_middle"), - "small_hub", - "middle", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("middle_to_anchor"), - "middle", - "larger_anchor", - Direction::Outgoing, - Some(vec![20]), - ), + pattern_node("from", None, Vec::new()), + pattern_node("to", None, Vec::new()), ], + vec![pattern_edge( + Some("e"), + "from", + "to", + Direction::Outgoing, + Some(vec!["EDGE_LABEL_216"]), + )], ); - let (anchor_alias, _) = planned_pattern_anchor_and_edge_aliases(&engine, &query); - assert_eq!(anchor_alias, "larger_anchor"); - let result = engine.query_pattern(&query).unwrap(); - assert_eq!(result.matches.len(), anchors.len()); - assert!(!result.truncated); - - engine.close().unwrap(); + assert!(matches!( + engine.explain_pattern_query(&query), + Err(EngineError::InvalidOperation(message)) + if message.contains("anchorable node pattern or edge pattern") + )); } #[test] -fn test_query_pattern_fanout_physical_anchor_does_not_change_result_order() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let hub_low = insert_query_node(&engine, 1, "hub-low", &[], 1.0); - let hub_high = insert_query_node(&engine, 1, "hub-high", &[], 1.0); - let mut mids = Vec::new(); - for index in 0..300 { - mids.push(insert_query_node( - &engine, - 3, - &format!("mid-{index:03}"), - &[], - 1.0, - )); - } - let mut anchors = Vec::new(); - for index in 0..20 { - anchors.push(insert_query_node( - &engine, - 2, - &format!("anchor-{index:02}"), - &[], - 1.0, - )); - } - for &mid in &mids[..150] { - engine - .upsert_edge(hub_low, mid, 10, UpsertEdgeOptions::default()) - .unwrap(); - } - for &mid in &mids[150..] { - engine - .upsert_edge(hub_high, mid, 10, UpsertEdgeOptions::default()) - .unwrap(); - } - engine - .upsert_edge(mids[0], anchors[19], 20, UpsertEdgeOptions::default()) +fn pattern_edge_anchor_expands_branching_pattern_from_both_endpoints() { + let (_dir, engine) = query_test_engine(); + let left = insert_query_node(&engine, "NodeLabel121", "branch-edge-anchor-left", &[], 1.0); + let right = insert_query_node(&engine, "NodeLabel122", "branch-edge-anchor-right", &[], 1.0); + let left_leaf = insert_query_node(&engine, "NodeLabel123", "branch-edge-anchor-left-leaf", &[], 1.0); + let right_leaf = insert_query_node(&engine, "NodeLabel124", "branch-edge-anchor-right-leaf", &[], 1.0); + let anchor_edge = engine + .upsert_edge( + left, + right, + "EDGE_LABEL_217", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let left_edge = engine + .upsert_edge(left, left_leaf, "EDGE_LABEL_218", UpsertEdgeOptions::default()) .unwrap(); - engine - .upsert_edge(mids[150], anchors[0], 20, UpsertEdgeOptions::default()) + let right_edge = engine + .upsert_edge(right, right_leaf, "EDGE_LABEL_219", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_217", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let mut query = pattern_query( + let mut anchor = pattern_edge(Some("anchor"), "left", "right", Direction::Outgoing, Some(vec!["EDGE_LABEL_217"])); + anchor.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); + let query = pattern_query( vec![ - pattern_node("a_small_hub", Some(1), Vec::new()), - pattern_node("z_larger_anchor", Some(2), Vec::new()), - pattern_node("middle", Some(3), Vec::new()), + pattern_node("left", None, Vec::new()), + pattern_node("right", None, Vec::new()), + pattern_node("left_leaf", None, Vec::new()), + pattern_node("right_leaf", None, Vec::new()), ], vec![ + anchor, pattern_edge( - Some("hub_to_middle"), - "a_small_hub", - "middle", + Some("left_edge"), + "left", + "left_leaf", Direction::Outgoing, - Some(vec![10]), + Some(vec!["EDGE_LABEL_218"]), ), pattern_edge( - Some("middle_to_anchor"), - "middle", - "z_larger_anchor", + Some("right_edge"), + "right", + "right_leaf", Direction::Outgoing, - Some(vec![20]), + Some(vec!["EDGE_LABEL_219"]), ), ], ); - query.limit = 1; - let (physical_anchor, sort_anchor, _) = - planned_pattern_anchor_sort_and_edge_aliases(&engine, &query); - assert_eq!(physical_anchor, "z_larger_anchor"); - assert_eq!(sort_anchor, "a_small_hub"); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[ + ("left", left), + ("right", right), + ("left_leaf", left_leaf), + ("right_leaf", right_leaf), + ], + &[ + ("anchor", anchor_edge), + ("left_edge", left_edge), + ("right_edge", right_edge), + ], + )] + ); +} - let result = engine.query_pattern(&query).unwrap(); - assert_eq!(result.matches.len(), 1); - assert!(result.truncated); - assert_eq!(result.matches[0].nodes["a_small_hub"], hub_low); - assert_eq!(result.matches[0].nodes["z_larger_anchor"], anchors[19]); +#[test] +fn pattern_edge_anchor_remaining_property_filter_uses_projection() { + let (_dir, engine) = query_test_engine(); + let left = insert_query_node(&engine, "NodeLabel131", "branch-projection-left", &[], 1.0); + let mid = insert_query_node(&engine, "NodeLabel132", "branch-projection-mid", &[], 1.0); + let keep = insert_query_node(&engine, "NodeLabel133", "branch-projection-keep", &[], 1.0); + let drop = insert_query_node(&engine, "NodeLabel133", "branch-projection-drop", &[], 1.0); + let anchor_edge = engine + .upsert_edge( + left, + mid, + "EDGE_LABEL_226", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let keep_edge = engine + .upsert_edge( + mid, + keep, + "EDGE_LABEL_227", + UpsertEdgeOptions { + props: query_test_props(&[("role", PropValue::String("keep".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + mid, + drop, + "EDGE_LABEL_227", + UpsertEdgeOptions { + props: query_test_props(&[("role", PropValue::String("drop".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_226", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let mut anchor = pattern_edge(Some("anchor"), "left", "mid", Direction::Outgoing, Some(vec!["EDGE_LABEL_226"])); + anchor.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); + let mut remaining = pattern_edge(Some("remaining"), "mid", "leaf", Direction::Outgoing, Some(vec!["EDGE_LABEL_227"])); + remaining.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "role".to_string(), + value: PropValue::String("keep".to_string()), + }); + let query = pattern_query( + vec![ + pattern_node("left", Some("NodeLabel131"), Vec::new()), + pattern_node("mid", Some("NodeLabel132"), Vec::new()), + pattern_node("leaf", Some("NodeLabel133"), Vec::new()), + ], + vec![anchor, remaining], + ); - engine.close().unwrap(); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + engine.reset_query_execution_counters_for_test(); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[("left", left), ("mid", mid), ("leaf", keep)], + &[("anchor", anchor_edge), ("remaining", keep_edge)] + )] + ); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.edge_record_hydration_reads, 0); } #[test] -fn test_query_pattern_fanout_delays_high_hub_expansion() { +fn pattern_edge_exists_and_missing_filters_use_label_anchor_only() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "NodeLabel125", "edge-exists-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel125", "edge-exists-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel125", "edge-exists-c", &[], 1.0); + let present = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_220", + UpsertEdgeOptions { + props: query_test_props(&[("flag", PropValue::String("yes".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let missing = engine.upsert_edge(a, c, "EDGE_LABEL_220", UpsertEdgeOptions::default()).unwrap(); + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_220", "flag", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + for (filter, expected_edge, expected_target) in [ + ( + EdgeFilterExpr::PropertyExists { + key: "flag".to_string(), + }, + present, + b, + ), + ( + EdgeFilterExpr::PropertyMissing { + key: "flag".to_string(), + }, + missing, + c, + ), + ] { + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_220"])); + edge.filter = Some(filter); + let query = pattern_query( + vec![ + pattern_node("a", None, Vec::new()), + pattern_node("b", None, Vec::new()), + ], + vec![edge], + ); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeLabelIndex)); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(plan.warnings.contains(&QueryPlanWarning::VerifyOnlyFilter)); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("a", a), ("b", expected_target)], &[("e", expected_edge)])] + ); + } +} + +#[test] +fn pattern_edge_property_unavailable_sidecar_uses_label_fallback() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - let low = insert_query_node(&engine, 3, "low", &[], 1.0); - engine - .upsert_edge(root, low, 20, UpsertEdgeOptions::default()) - .unwrap(); - for index in 0..128 { - let target = insert_query_node(&engine, 2, &format!("hub-target-{index:03}"), &[], 1.0); + let index_id; + let segment_id; + let keep; + let a; + let b; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + a = insert_query_node(&engine, "NodeLabel126", "pattern-corrupt-edge-a", &[], 1.0); + b = insert_query_node(&engine, "NodeLabel126", "pattern-corrupt-edge-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel126", "pattern-corrupt-edge-c", &[], 1.0); + keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_221", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); engine - .upsert_edge(root, target, 10, UpsertEdgeOptions::default()) + .upsert_edge( + a, + c, + "EDGE_LABEL_221", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("cold".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_221", "status", SecondaryIndexKind::Equality) .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = engine.segments_for_test()[0].segment_id; + engine.close().unwrap(); } - engine.flush().unwrap(); + let sidecar_path = crate::segment_writer::edge_prop_eq_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, segment_id), + index_id, + ); + corrupt_planner_stats_for_segment(&db_path, segment_id); + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + wait_for_edge_property_index_state(&reopened, index_id, SecondaryIndexState::Ready); + corrupt_sidecar_header_in_place(&sidecar_path); + + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_221"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("hub_target", Some(2), Vec::new()), - pattern_node("low_target", Some(3), Vec::new()), - ], - vec![ - pattern_edge( - Some("aaa_hub"), - "root", - "hub_target", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("zzz_low"), - "root", - "low_target", - Direction::Outgoing, - Some(vec![20]), - ), + pattern_node("a", None, Vec::new()), + pattern_node("b", None, Vec::new()), ], + vec![edge], + ); + let plan = reopened.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeLabelIndex)); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(plan.warnings.contains(&QueryPlanWarning::MissingReadyIndex)); + assert_eq!( + reopened.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("a", a), ("b", b)], &[("e", keep)])] ); - - let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); - assert_eq!(edge_aliases.first().map(String::as_str), Some("zzz_low")); - - engine.close().unwrap(); } #[test] -fn test_query_pattern_mutable_edges_preserve_deterministic_expansion_order() { +fn pattern_edge_anchor_runtime_sidecar_failure_uses_local_label_fallback() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - let low = insert_query_node(&engine, 3, "low", &[], 1.0); + let a = insert_query_node(&engine, "NodeLabel130", "pattern-runtime-fallback-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel130", "pattern-runtime-fallback-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel130", "pattern-runtime-fallback-c", &[], 1.0); + let keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_225", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); engine - .upsert_edge(root, low, 20, UpsertEdgeOptions::default()) + .upsert_edge( + a, + c, + "EDGE_LABEL_225", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("cold".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - for index in 0..96 { - let target = insert_query_node(&engine, 2, &format!("target-{index:02}"), &[], 1.0); - engine - .upsert_edge(root, target, 10, UpsertEdgeOptions::default()) - .unwrap(); - } engine.flush().unwrap(); - - let mutable_target = insert_query_node(&engine, 99, "mutable", &[], 1.0); - engine - .upsert_edge(root, mutable_target, 99, UpsertEdgeOptions::default()) + let info = engine + .ensure_edge_property_index("EDGE_LABEL_225", "status", SecondaryIndexKind::Equality) .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + let sidecar_path = crate::segment_writer::edge_prop_eq_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, engine.segments_for_test()[0].segment_id), + info.index_id, + ); + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_225"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("hub_target", Some(2), Vec::new()), - pattern_node("low_target", Some(3), Vec::new()), - ], - vec![ - pattern_edge( - Some("aaa_hub"), - "root", - "hub_target", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("zzz_low"), - "root", - "low_target", - Direction::Outgoing, - Some(vec![20]), - ), + pattern_node("a", Some("NodeLabel130"), Vec::new()), + pattern_node("b", Some("NodeLabel130"), Vec::new()), ], + vec![edge], ); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); - let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); - assert_eq!(edge_aliases.first().map(String::as_str), Some("aaa_hub")); + let (_guard, published) = engine.runtime.published_snapshot().unwrap(); + let normalized = published.view.normalize_pattern_query(&query).unwrap(); + let planned = published + .view + .plan_normalized_pattern_query(&normalized) + .unwrap(); + corrupt_sidecar_header_in_place(&sidecar_path); - engine.close().unwrap(); + engine.reset_query_execution_counters_for_test(); + let outcome = published + .view + .query_pattern_planned(&normalized, planned) + .unwrap(); + assert!(!outcome.followups.is_empty()); + assert_eq!( + outcome.value.matches, + vec![expected_match(&[("a", a), ("b", b)], &[("e", keep)])] + ); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.endpoint_adjacency_candidates, 0); + assert_eq!(counters.public_edge_query_calls, 0); } #[test] -fn test_query_pattern_target_filter_selectivity_reduces_fanout_cost() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); +fn pattern_edge_property_failed_index_state_uses_label_fallback() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "NodeLabel127", "pattern-failed-edge-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel127", "pattern-failed-edge-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel127", "pattern-failed-edge-c", &[], 1.0); + let keep = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_222", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); engine - .ensure_node_property_index(2, "status", SecondaryIndexKind::Equality) + .upsert_edge( + a, + c, + "EDGE_LABEL_222", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("cold".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - for index in 0..128 { - let status = if index == 127 { "selected" } else { "other" }; - let target = insert_query_node( - &engine, - 2, - &format!("candidate-{index:03}"), - &[("status", PropValue::String(status.to_string()))], - 1.0, - ); - engine - .upsert_edge(root, target, 10, UpsertEdgeOptions::default()) - .unwrap(); - } - for index in 0..16 { - let target = insert_query_node(&engine, 3, &format!("low-{index:02}"), &[], 1.0); - engine - .upsert_edge(root, target, 20, UpsertEdgeOptions::default()) - .unwrap(); - } engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_222", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + engine.shutdown_secondary_index_worker(); + engine + .with_runtime_manifest_write(|manifest| { + let entry = manifest + .secondary_indexes + .iter_mut() + .find(|entry| entry.index_id == info.index_id) + .unwrap(); + entry.state = SecondaryIndexState::Failed; + entry.last_error = Some("forced pattern fallback".to_string()); + Ok(()) + }) + .unwrap(); + engine.rebuild_secondary_index_catalog().unwrap(); + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_222"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node( - "selected_target", - Some(2), - vec![NodeFilterExpr::PropertyEquals { - key: "status".to_string(), - value: PropValue::String("selected".to_string()), - }], - ), - pattern_node("low_target", Some(3), Vec::new()), - ], - vec![ - pattern_edge( - Some("zzz_selective"), - "root", - "selected_target", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("aaa_low"), - "root", - "low_target", - Direction::Outgoing, - Some(vec![20]), - ), + pattern_node("a", None, Vec::new()), + pattern_node("b", None, Vec::new()), ], + vec![edge], ); - - let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::EdgeLabelIndex)); + assert!(!plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); assert_eq!( - edge_aliases.first().map(String::as_str), - Some("zzz_selective") + engine.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("a", a), ("b", b)], &[("e", keep)])] ); - - engine.close().unwrap(); } #[test] -fn test_query_pattern_absent_edge_type_uses_complete_zero_fanout() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - let missing_target = insert_query_node(&engine, 4, "missing-target", &[], 1.0); - for index in 0..64 { - let target = insert_query_node(&engine, 2, &format!("target-{index:02}"), &[], 1.0); - engine - .upsert_edge(root, target, 10, UpsertEdgeOptions::default()) - .unwrap(); - } +fn pattern_edge_anchor_mixed_sources_dedupes_and_uses_newest_edge_props() { + let (_dir, engine) = query_test_engine(); + let source = insert_query_node(&engine, "NodeLabel128", "pattern-mixed-source", &[], 1.0); + let target_a = insert_query_node(&engine, "NodeLabel128", "pattern-mixed-target-a", &[], 1.0); + let target_b = insert_query_node(&engine, "NodeLabel128", "pattern-mixed-target-b", &[], 1.0); + let target_c = insert_query_node(&engine, "NodeLabel128", "pattern-mixed-target-c", &[], 1.0); + let edge_a = engine + .upsert_edge( + source, + target_a, + "EDGE_LABEL_223", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let edge_b = engine + .upsert_edge( + source, + target_b, + "EDGE_LABEL_223", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_223", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + set_query_edge_props( + &engine, + edge_a, + query_test_props(&[("status", PropValue::String("cold".to_string()))]), + ); + set_query_edge_props( + &engine, + edge_b, + query_test_props(&[("status", PropValue::String("cold".to_string()))]), + ); + engine.freeze_memtable().unwrap(); + set_query_edge_props( + &engine, + edge_a, + query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ); + let edge_c = engine + .upsert_edge( + source, + target_c, + "EDGE_LABEL_223", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let mut edge = pattern_edge(Some("e"), "source", "target", Direction::Outgoing, Some(vec!["EDGE_LABEL_223"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("hub_target", Some(2), Vec::new()), - pattern_node_with_ids("missing_target", vec![missing_target]), - ], - vec![ - pattern_edge( - Some("aaa_hub"), - "root", - "hub_target", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("zzz_missing"), - "root", - "missing_target", - Direction::Outgoing, - Some(vec![999]), - ), + pattern_node("source", None, Vec::new()), + pattern_node("target", None, Vec::new()), ], + vec![edge], ); - - let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); assert_eq!( - edge_aliases.first().map(String::as_str), - Some("zzz_missing") + engine.query_pattern(&query).unwrap().matches, + vec![ + expected_match(&[("source", source), ("target", target_a)], &[("e", edge_a)]), + expected_match(&[("source", source), ("target", target_c)], &[("e", edge_c)]), + ] ); - let result = engine.query_pattern(&query).unwrap(); - assert!(result.matches.is_empty()); - - engine.close().unwrap(); } #[test] -fn test_query_pattern_both_bound_constraint_stays_before_unbound_expansion() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let a = insert_query_node(&engine, 1, "a", &[], 1.0); - let b = insert_query_node(&engine, 2, "b", &[], 1.0); - let c = insert_query_node(&engine, 3, "c", &[], 1.0); - engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine - .upsert_edge(a, c, 30, UpsertEdgeOptions::default()) - .unwrap(); - engine - .upsert_edge(b, a, 11, UpsertEdgeOptions::default()) +fn pattern_edge_anchor_execution_does_not_call_public_edge_queries() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "NodeLabel129", "pattern-public-edge-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel129", "pattern-public-edge-b", &[], 1.0); + let edge_id = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_224", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - for index in 0..96 { - let dummy = insert_query_node(&engine, 4, &format!("dummy-{index:03}"), &[], 1.0); - engine - .upsert_edge(b, dummy, 11, UpsertEdgeOptions::default()) - .unwrap(); - } engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_224", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_224"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( vec![ - pattern_node_with_ids("a", vec![a]), - pattern_node("b", Some(2), Vec::new()), - pattern_node("c", Some(3), Vec::new()), - ], - vec![ - pattern_edge( - Some("aaa_bind_b"), - "a", - "b", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("zzz_constraint"), - "b", - "a", - Direction::Outgoing, - Some(vec![11]), - ), - pattern_edge( - Some("zzz_unbound"), - "a", - "c", - Direction::Outgoing, - Some(vec![30]), - ), + pattern_node("a", None, Vec::new()), + pattern_node("b", None, Vec::new()), ], + vec![edge], ); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); - let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&engine, &query); - assert_eq!(edge_aliases[0], "aaa_bind_b"); - assert_eq!(edge_aliases[1], "zzz_constraint"); - - engine.close().unwrap(); + engine.reset_query_execution_counters_for_test(); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("a", a), ("b", b)], &[("e", edge_id)])] + ); + let counters = engine.query_execution_counter_snapshot_for_test(); + assert_eq!(counters.public_edge_query_calls, 0); } #[test] -fn test_query_pattern_missing_fanout_stats_preserves_deterministic_order() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - let low = insert_query_node(&engine, 3, "low", &[], 1.0); +fn pattern_node_anchor_still_wins_when_node_filter_is_more_selective() { + let (_dir, engine) = query_test_engine(); + let mut sources = Vec::new(); + for index in 0..64 { + sources.push(insert_query_node(&engine, "SearchNode110", + &format!("node-anchor-still-wins-source-{index}"), + &[( + "tenant", + PropValue::String(if index == 3 { "one" } else { "many" }.to_string()), + )], + 1.0, + )); + } + let target = insert_query_node(&engine, "NodeLabel111", "node-anchor-still-wins-target", &[], 1.0); + let keep_edge = engine + .upsert_edge( + sources[3], + target, + "EDGE_LABEL_206", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("broad".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + for (index, source) in sources.iter().enumerate() { + if index == 3 { + continue; + } engine - .upsert_edge(root, low, 20, UpsertEdgeOptions::default()) + .upsert_edge( + *source, + target, + "EDGE_LABEL_206", + UpsertEdgeOptions { + props: query_test_props(&[( + "status", + PropValue::String("broad".to_string()), + )]), + ..Default::default() + }, + ) .unwrap(); - for index in 0..64 { - let target = insert_query_node(&engine, 2, &format!("target-{index:02}"), &[], 1.0); - engine - .upsert_edge(root, target, 10, UpsertEdgeOptions::default()) - .unwrap(); - } - engine.flush().unwrap(); - engine.close().unwrap(); } + engine.flush().unwrap(); + let node_index = engine + .ensure_node_property_index("SearchNode110", "tenant", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, node_index.index_id, SecondaryIndexState::Ready); + let edge_index = engine + .ensure_edge_property_index("EDGE_LABEL_206", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, edge_index.index_id, SecondaryIndexState::Ready); - let stats_path = crate::segment_writer::segment_dir(&db_path, 1) - .join(crate::planner_stats::PLANNER_STATS_FILENAME); - std::fs::remove_file(stats_path).unwrap(); - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let root = reopened.get_node_by_key(1, "root").unwrap().unwrap().id; + let mut edge = pattern_edge(Some("rel"), "source", "target", Direction::Outgoing, Some(vec!["EDGE_LABEL_206"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("broad".to_string()), + }); let query = pattern_query( vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("hub_target", Some(2), Vec::new()), - pattern_node("low_target", Some(3), Vec::new()), - ], - vec![ - pattern_edge( - Some("aaa_hub"), - "root", - "hub_target", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("zzz_low"), - "root", - "low_target", - Direction::Outgoing, - Some(vec![20]), + pattern_node("source", Some("SearchNode110"), + vec![NodeFilterExpr::PropertyEquals { + key: "tenant".to_string(), + value: PropValue::String("one".to_string()), + }], ), + pattern_node("target", Some("NodeLabel111"), Vec::new()), ], + vec![edge], ); - let (_, edge_aliases) = planned_pattern_anchor_and_edge_aliases(&reopened, &query); - assert_eq!(edge_aliases.first().map(String::as_str), Some("aaa_hub")); - - reopened.close().unwrap(); -} - -fn fanout_parity_query(root: u64) -> GraphPatternQuery { - pattern_query( - vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("hub_target", Some(2), Vec::new()), - pattern_node("low_target", Some(3), Vec::new()), - ], - vec![ - pattern_edge( - Some("hub_edge"), - "root", - "hub_target", - Direction::Outgoing, - Some(vec![10]), - ), - pattern_edge( - Some("low_edge"), - "root", - "low_target", - Direction::Outgoing, - Some(vec![20]), - ), - ], - ) + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(!plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node(&plan.root, &QueryPlanNode::PropertyEqualityIndex)); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![expected_match( + &[("source", sources[3]), ("target", target)], + &[("rel", keep_edge)] + )] + ); } -fn insert_fanout_parity_tail(engine: &DatabaseEngine, root: u64, start: usize, count: usize) { - for index in start..start + count { - let target = insert_query_node(engine, 2, &format!("target-{index:02}"), &[], 1.0); +#[test] +fn pattern_edge_property_in_anchor_preserves_signed_zero() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "NodeLabel112", "pattern-in-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel112", "pattern-in-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel112", "pattern-in-c", &[], 1.0); + let d = insert_query_node(&engine, "NodeLabel112", "pattern-in-d", &[], 1.0); + let positive_zero = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_207", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(0.0))]), + ..Default::default() + }, + ) + .unwrap(); + let negative_zero = engine + .upsert_edge( + a, + c, + "EDGE_LABEL_207", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(-0.0))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + d, + "EDGE_LABEL_207", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(1.0))]), + ..Default::default() + }, + ) + .unwrap(); + for index in 0..12 { engine - .upsert_edge(root, target, 10, UpsertEdgeOptions::default()) + .upsert_edge( + b, + d, + "EDGE_LABEL_207", + UpsertEdgeOptions { + props: query_test_props(&[("z", PropValue::Float(index as f64 + 2.0))]), + ..Default::default() + }, + ) .unwrap(); } -} + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_207", "z", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); -fn assert_fanout_parity_result(engine: &DatabaseEngine, root: u64, expected_matches: &[QueryMatch]) { - let query = fanout_parity_query(root); - let result = engine.query_pattern(&query).unwrap(); - assert_eq!(result.matches, expected_matches); - assert!(!result.truncated); + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_207"])); + edge.filter = Some(EdgeFilterExpr::PropertyIn { + key: "z".to_string(), + values: vec![PropValue::Float(-0.0), PropValue::Float(0.0)], + }); + let query = pattern_query( + vec![ + pattern_node("a", Some("NodeLabel112"), Vec::new()), + pattern_node("b", Some("NodeLabel112"), Vec::new()), + ], + vec![edge], + ); + + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert_eq!( + engine.query_pattern(&query).unwrap().matches, + vec![ + expected_match(&[("a", a), ("b", b)], &[("e", positive_zero)]), + expected_match(&[("a", a), ("b", c)], &[("e", negative_zero)]), + ] + ); } #[test] -fn test_query_pattern_fanout_stats_preserve_results_across_storage_states() { +fn pattern_edge_property_equality_anchor_verifies_hash_collisions() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let index_id; + let segment_id; + let red_one; + let red_two; + let blue; + let a; + let b; { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("memtable"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - insert_fanout_parity_tail(&engine, root, 0, 8); - let low = insert_query_node(&engine, 3, "low", &[], 1.0); - let low_edge = engine - .upsert_edge(root, low, 20, UpsertEdgeOptions::default()) + a = insert_query_node(&engine, "NodeLabel113", "pattern-collision-a", &[], 1.0); + b = insert_query_node(&engine, "NodeLabel113", "pattern-collision-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel113", "pattern-collision-c", &[], 1.0); + let d = insert_query_node(&engine, "NodeLabel113", "pattern-collision-d", &[], 1.0); + red_one = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_208", + UpsertEdgeOptions { + props: query_test_props(&[("color", PropValue::String("red".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - let expected: Vec<_> = (0..8) - .map(|index| { - let target = engine - .get_node_by_key(2, &format!("target-{index:02}")) - .unwrap() - .unwrap() - .id; - let hub_edge = engine - .get_edge_by_triple(root, target, 10) - .unwrap() - .unwrap() - .id; - expected_match( - &[("hub_target", target), ("low_target", low), ("root", root)], - &[("hub_edge", hub_edge), ("low_edge", low_edge)], - ) - }) - .collect(); - assert_fanout_parity_result(&engine, root, &expected); - engine.close().unwrap(); - } - - { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("flushed"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - insert_fanout_parity_tail(&engine, root, 0, 8); - let low = insert_query_node(&engine, 3, "low", &[], 1.0); - let low_edge = engine - .upsert_edge(root, low, 20, UpsertEdgeOptions::default()) + red_two = engine + .upsert_edge( + a, + c, + "EDGE_LABEL_208", + UpsertEdgeOptions { + props: query_test_props(&[("color", PropValue::String("red".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + blue = engine + .upsert_edge( + a, + d, + "EDGE_LABEL_208", + UpsertEdgeOptions { + props: query_test_props(&[("color", PropValue::String("blue".to_string()))]), + ..Default::default() + }, + ) .unwrap(); engine.flush().unwrap(); - let expected: Vec<_> = (0..8) - .map(|index| { - let target = engine - .get_node_by_key(2, &format!("target-{index:02}")) - .unwrap() - .unwrap() - .id; - let hub_edge = engine - .get_edge_by_triple(root, target, 10) - .unwrap() - .unwrap() - .id; - expected_match( - &[("hub_target", target), ("low_target", low), ("root", root)], - &[("hub_edge", hub_edge), ("low_edge", low_edge)], - ) - }) - .collect(); - assert_fanout_parity_result(&engine, root, &expected); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_208", "color", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + index_id = info.index_id; + segment_id = engine.segments_for_test()[0].segment_id; engine.close().unwrap(); } - { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("reopened"); - let root = { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - insert_fanout_parity_tail(&engine, root, 0, 8); - let low = insert_query_node(&engine, 3, "low", &[], 1.0); - engine - .upsert_edge(root, low, 20, UpsertEdgeOptions::default()) - .unwrap(); - engine.flush().unwrap(); - engine.close().unwrap(); - root - }; - let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let low = reopened.get_node_by_key(3, "low").unwrap().unwrap().id; - let low_edge = reopened - .get_edge_by_triple(root, low, 20) - .unwrap() - .unwrap() - .id; - let expected: Vec<_> = (0..8) - .map(|index| { - let target = reopened - .get_node_by_key(2, &format!("target-{index:02}")) - .unwrap() - .unwrap() - .id; - let hub_edge = reopened - .get_edge_by_triple(root, target, 10) - .unwrap() - .unwrap() - .id; - expected_match( - &[("hub_target", target), ("low_target", low), ("root", root)], - &[("hub_edge", hub_edge), ("low_edge", low_edge)], - ) - }) - .collect(); - assert_fanout_parity_result(&reopened, root, &expected); - reopened.close().unwrap(); - } + let sidecar_path = crate::segment_writer::edge_prop_eq_sidecar_path( + &crate::segment_writer::segment_dir(&db_path, segment_id), + index_id, + ); + replace_equality_sidecar_group_id_in_place( + &sidecar_path, + hash_prop_value(&PropValue::String("red".to_string())), + red_two, + blue, + ); - { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("compacted"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - insert_fanout_parity_tail(&engine, root, 0, 4); - engine.flush().unwrap(); - insert_fanout_parity_tail(&engine, root, 4, 4); - let low = insert_query_node(&engine, 3, "low", &[], 1.0); - let low_edge = engine - .upsert_edge(root, low, 20, UpsertEdgeOptions::default()) - .unwrap(); - engine.flush().unwrap(); - engine.compact().unwrap().unwrap(); - let expected: Vec<_> = (0..8) - .map(|index| { - let target = engine - .get_node_by_key(2, &format!("target-{index:02}")) - .unwrap() - .unwrap() - .id; - let hub_edge = engine - .get_edge_by_triple(root, target, 10) - .unwrap() - .unwrap() - .id; - expected_match( - &[("hub_target", target), ("low_target", low), ("root", root)], - &[("hub_edge", hub_edge), ("low_edge", low_edge)], - ) - }) - .collect(); - assert_fanout_parity_result(&engine, root, &expected); - engine.close().unwrap(); - } + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_208"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "color".to_string(), + value: PropValue::String("red".to_string()), + }); + let query = pattern_query( + vec![ + pattern_node("a", Some("NodeLabel113"), Vec::new()), + pattern_node("b", Some("NodeLabel113"), Vec::new()), + ], + vec![edge], + ); + let plan = reopened.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); + assert!(plan_contains_node( + &plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert_eq!( + reopened.query_pattern(&query).unwrap().matches, + vec![expected_match(&[("a", a), ("b", b)], &[("e", red_one)])] + ); } #[test] -fn test_query_pattern_distinct_edge_aliases_may_share_edge_id() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let a = insert_query_node(&engine, 1, "a", &[], 1.0); - let b = insert_query_node(&engine, 2, "b", &[], 1.0); - let edge = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) +fn pattern_edge_range_anchor_keeps_numeric_domains_exact() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "NodeLabel114", "pattern-domain-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel114", "pattern-domain-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel114", "pattern-domain-c", &[], 1.0); + let d = insert_query_node(&engine, "NodeLabel114", "pattern-domain-d", &[], 1.0); + + let int_edge = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_209", + UpsertEdgeOptions { + props: query_test_props(&[("metric", PropValue::Int(5))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + c, + "EDGE_LABEL_209", + UpsertEdgeOptions { + props: query_test_props(&[("metric", PropValue::UInt(5))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + a, + d, + "EDGE_LABEL_209", + UpsertEdgeOptions { + props: query_test_props(&[("metric", PropValue::Float(5.0))]), + ..Default::default() + }, + ) + .unwrap(); + let uint_edge = engine + .upsert_edge( + b, + c, + "EDGE_LABEL_210", + UpsertEdgeOptions { + props: query_test_props(&[("metric", PropValue::UInt(7))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + b, + d, + "EDGE_LABEL_210", + UpsertEdgeOptions { + props: query_test_props(&[("metric", PropValue::Int(7))]), + ..Default::default() + }, + ) + .unwrap(); + let float_edge = engine + .upsert_edge( + c, + d, + "EDGE_LABEL_211", + UpsertEdgeOptions { + props: query_test_props(&[("metric", PropValue::Float(9.5))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + c, + a, + "EDGE_LABEL_211", + UpsertEdgeOptions { + props: query_test_props(&[("metric", PropValue::Int(9))]), + ..Default::default() + }, + ) .unwrap(); + engine.flush().unwrap(); - let query = pattern_query( - vec![pattern_node_with_ids("a", vec![a]), pattern_node("b", Some(2), Vec::new())], - vec![ - pattern_edge(Some("first"), "a", "b", Direction::Outgoing, Some(vec![10])), - pattern_edge(Some("second"), "a", "b", Direction::Outgoing, Some(vec![10])), - ], - ); - - assert_eq!( - engine.query_pattern(&query).unwrap().matches, - vec![expected_match( - &[("a", a), ("b", b)], - &[("first", edge), ("second", edge)] - )] - ); + for (label, domain) in [ + ("EDGE_LABEL_209", SecondaryIndexRangeDomain::Int), + ("EDGE_LABEL_210", SecondaryIndexRangeDomain::UInt), + ("EDGE_LABEL_211", SecondaryIndexRangeDomain::Float), + ] { + let info = engine + .ensure_edge_property_index( + label, + "metric", + SecondaryIndexKind::Range { domain }, + ) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + } - engine.close().unwrap(); + let cases = [ + ("EDGE_LABEL_209", PropValue::Int(5), int_edge), + ("EDGE_LABEL_210", PropValue::UInt(7), uint_edge), + ("EDGE_LABEL_211", PropValue::Float(9.5), float_edge), + ]; + for (label, value, expected_edge) in cases { + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec![label])); + edge.filter = Some(EdgeFilterExpr::PropertyRange { + key: "metric".to_string(), + lower: Some(PropertyRangeBound::Included(value.clone())), + upper: Some(PropertyRangeBound::Included(value)), + }); + let query = pattern_query( + vec![ + pattern_node("a", Some("NodeLabel114"), Vec::new()), + pattern_node("b", Some("NodeLabel114"), Vec::new()), + ], + vec![edge], + ); + let result = engine.query_pattern(&query).unwrap(); + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].edges["e"], expected_edge); + } } #[test] -fn test_query_pattern_branching_distinct_aliases_and_no_match() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let root = insert_query_node(&engine, 1, "root", &[], 1.0); - let left = insert_query_node(&engine, 2, "left", &[], 1.0); - let right = insert_query_node(&engine, 3, "right", &[], 1.0); - let shared = insert_query_node(&engine, 4, "shared", &[], 1.0); - let left_edge = engine - .upsert_edge(root, left, 10, UpsertEdgeOptions::default()) +fn pattern_edge_or_and_not_use_bounded_fallback_without_partial_index_results() { + let (_dir, engine) = query_test_engine(); + let a = insert_query_node(&engine, "NodeLabel115", "pattern-or-a", &[], 1.0); + let b = insert_query_node(&engine, "NodeLabel115", "pattern-or-b", &[], 1.0); + let c = insert_query_node(&engine, "NodeLabel115", "pattern-or-c", &[], 1.0); + let d = insert_query_node(&engine, "NodeLabel115", "pattern-or-d", &[], 1.0); + let hot = engine + .upsert_edge( + a, + b, + "EDGE_LABEL_212", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - let right_edge = engine - .upsert_edge(root, right, 20, UpsertEdgeOptions::default()) + let archived = engine + .upsert_edge( + a, + c, + "EDGE_LABEL_212", + UpsertEdgeOptions { + props: query_test_props(&[("flag", PropValue::String("archived".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - engine - .upsert_edge(root, shared, 30, UpsertEdgeOptions::default()) + let cold = engine + .upsert_edge( + a, + d, + "EDGE_LABEL_212", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("cold".to_string()))]), + ..Default::default() + }, + ) .unwrap(); - engine - .upsert_edge(root, shared, 40, UpsertEdgeOptions::default()) + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_212", "status", SecondaryIndexKind::Equality) .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); - let query = pattern_query( - vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("left", Some(2), Vec::new()), - pattern_node("right", Some(3), Vec::new()), - ], + let mut or_edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_212"])); + or_edge.filter = Some(EdgeFilterExpr::Or(vec![ + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }, + EdgeFilterExpr::PropertyEquals { + key: "flag".to_string(), + value: PropValue::String("archived".to_string()), + }, + ])); + let or_query = pattern_query( vec![ - pattern_edge(Some("left_edge"), "root", "left", Direction::Outgoing, Some(vec![10])), - pattern_edge( - Some("right_edge"), - "root", - "right", - Direction::Outgoing, - Some(vec![20]), - ), + pattern_node("a", Some("NodeLabel115"), Vec::new()), + pattern_node("b", Some("NodeLabel115"), Vec::new()), ], + vec![or_edge], ); + let or_plan = engine.explain_pattern_query(&or_query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&or_plan.root)); + assert!(!plan_contains_node( + &or_plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert!(or_plan + .warnings + .contains(&QueryPlanWarning::BooleanBranchFallback)); assert_eq!( - engine.query_pattern(&query).unwrap().matches, - vec![expected_match( - &[("left", left), ("right", right), ("root", root)], - &[("left_edge", left_edge), ("right_edge", right_edge)] - )] + engine.query_pattern(&or_query).unwrap().matches, + vec![ + expected_match(&[("a", a), ("b", b)], &[("e", hot)]), + expected_match(&[("a", a), ("b", c)], &[("e", archived)]), + ] ); - let distinct_alias_query = pattern_query( - vec![ - pattern_node_with_ids("root", vec![root]), - pattern_node("x", Some(4), Vec::new()), - pattern_node("y", Some(4), Vec::new()), - ], + let mut not_edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_212"])); + not_edge.filter = Some(EdgeFilterExpr::Not(Box::new( + EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }, + ))); + let not_query = pattern_query( vec![ - pattern_edge(Some("x_edge"), "root", "x", Direction::Outgoing, Some(vec![30])), - pattern_edge(Some("y_edge"), "root", "y", Direction::Outgoing, Some(vec![40])), + pattern_node("a", Some("NodeLabel115"), Vec::new()), + pattern_node("b", Some("NodeLabel115"), Vec::new()), ], + vec![not_edge], ); - assert!(engine - .query_pattern(&distinct_alias_query) - .unwrap() - .matches - .is_empty()); - - let no_match_query = pattern_query( - vec![pattern_node_with_ids("root", vec![root]), pattern_node("missing", Some(99), Vec::new())], - vec![pattern_edge(None, "root", "missing", Direction::Outgoing, None)], + let not_plan = engine.explain_pattern_query(¬_query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(¬_plan.root)); + assert!(!plan_contains_node( + ¬_plan.root, + &QueryPlanNode::EdgePropertyEqualityIndex + )); + assert_eq!( + engine.query_pattern(¬_query).unwrap().matches, + vec![ + expected_match(&[("a", a), ("b", c)], &[("e", archived)]), + expected_match(&[("a", a), ("b", d)], &[("e", cold)]), + ] ); - assert!(engine - .query_pattern(&no_match_query) - .unwrap() - .matches - .is_empty()); - - engine.close().unwrap(); } #[test] -fn test_query_pattern_cycle_closing_edge_and_self_loop() { +fn pattern_edge_anchor_excludes_deleted_and_prune_hidden_endpoints() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let source = insert_query_node(&engine, "NodeLabel116", "pattern-visible-source", &[], 1.0); + let keep = insert_query_node(&engine, "SearchNode117", "pattern-visible-keep", &[], 1.0); + let hidden = insert_query_node(&engine, "SearchNode117", "pattern-visible-hidden", &[], 0.1); + let deleted_target = insert_query_node(&engine, "SearchNode117", "pattern-visible-deleted", &[], 1.0); + let keep_edge = engine + .upsert_edge( + source, + keep, + "EDGE_LABEL_213", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge( + source, + hidden, + "EDGE_LABEL_213", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + let deleted_edge = engine + .upsert_edge( + source, + deleted_target, + "EDGE_LABEL_213", + UpsertEdgeOptions { + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_213", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + engine.delete_edge(deleted_edge).unwrap(); + engine + .set_prune_policy( + "low-weight-targets", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("SearchNode117".to_string()), + }, + ) + .unwrap(); - let a = insert_query_node(&engine, 1, "a", &[], 1.0); - let b = insert_query_node(&engine, 2, "b", &[], 1.0); - let c = insert_query_node(&engine, 3, "c", &[], 1.0); - let ab = engine.upsert_edge(a, b, 10, Default::default()).unwrap(); - let bc = engine.upsert_edge(b, c, 10, Default::default()).unwrap(); - let ca = engine.upsert_edge(c, a, 10, Default::default()).unwrap(); - let aa = engine.upsert_edge(a, a, 99, Default::default()).unwrap(); - - let cycle = pattern_query( - vec![ - pattern_node_with_ids("a", vec![a]), - pattern_node("b", Some(2), Vec::new()), - pattern_node("c", Some(3), Vec::new()), - ], + let mut edge = pattern_edge(Some("e"), "source", "target", Direction::Outgoing, Some(vec!["EDGE_LABEL_213"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); + let query = pattern_query( vec![ - pattern_edge(Some("ab"), "a", "b", Direction::Outgoing, Some(vec![10])), - pattern_edge(Some("bc"), "b", "c", Direction::Outgoing, Some(vec![10])), - pattern_edge(Some("ca"), "c", "a", Direction::Outgoing, Some(vec![10])), + pattern_node("source", Some("NodeLabel116"), Vec::new()), + pattern_node("target", Some("SearchNode117"), Vec::new()), ], + vec![edge], ); + let plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&plan.root)); assert_eq!( - engine.query_pattern(&cycle).unwrap().matches, + engine.query_pattern(&query).unwrap().matches, vec![expected_match( - &[("a", a), ("b", b), ("c", c)], - &[("ab", ab), ("bc", bc), ("ca", ca)] + &[("source", source), ("target", keep)], + &[("e", keep_edge)] )] ); - - let self_loop = pattern_query( - vec![pattern_node_with_ids("a", vec![a])], - vec![pattern_edge(Some("loop"), "a", "a", Direction::Outgoing, Some(vec![99]))], - ); - assert_eq!( - engine.query_pattern(&self_loop).unwrap().matches, - vec![expected_match(&[("a", a)], &[("loop", aa)])] - ); - - engine.close().unwrap(); } #[test] -fn test_query_pattern_edge_property_post_filters_and_explain_warning() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let a = insert_query_node(&engine, 1, "a", &[], 1.0); - let b = insert_query_node(&engine, 2, "b", &[], 1.0); - let c = insert_query_node(&engine, 2, "c", &[], 1.0); - let good = engine +fn pattern_edge_anchor_preserves_logical_order_and_truncation() { + let (_dir, engine) = query_test_engine(); + let a1 = insert_query_node(&engine, "NodeLabel118", "order-a1", &[], 1.0); + let a2 = insert_query_node(&engine, "NodeLabel118", "order-a2", &[], 1.0); + let b1 = insert_query_node(&engine, "NodeLabel119", "order-b1", &[], 1.0); + let b2 = insert_query_node(&engine, "NodeLabel119", "order-b2", &[], 1.0); + let e2 = engine .upsert_edge( - a, - b, - 10, + a2, + b2, + "EDGE_LABEL_214", UpsertEdgeOptions { - props: query_test_props(&[ - ("rel", PropValue::String("friend".to_string())), - ("score", PropValue::Int(5)), - ]), + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), ..Default::default() }, ) .unwrap(); - engine + let e1 = engine .upsert_edge( - a, - c, - 10, + a1, + b1, + "EDGE_LABEL_214", UpsertEdgeOptions { - props: query_test_props(&[ - ("rel", PropValue::String("friend".to_string())), - ("score", PropValue::Int(1)), - ]), + props: query_test_props(&[("status", PropValue::String("hot".to_string()))]), ..Default::default() }, ) .unwrap(); - engine.flush().unwrap(); - - let mut edge = pattern_edge(Some("e"), "a", "target", Direction::Outgoing, Some(vec![10])); - edge.property_predicates = vec![ - EdgePostFilterPredicate::PropertyEquals { - key: "rel".to_string(), - value: PropValue::String("friend".to_string()), - }, - EdgePostFilterPredicate::PropertyRange { - key: "score".to_string(), - lower: Some(PropertyRangeBound::Included(PropValue::Int(3))), - upper: None, - }, - ]; + let mut edge = pattern_edge(Some("e"), "a", "b", Direction::Outgoing, Some(vec!["EDGE_LABEL_214"])); + edge.filter = Some(EdgeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("hot".to_string()), + }); let query = pattern_query( - vec![pattern_node_with_ids("a", vec![a]), pattern_node("target", Some(2), Vec::new())], - vec![edge], + vec![ + pattern_node("a", Some("NodeLabel118"), Vec::new()), + pattern_node("b", Some("NodeLabel119"), Vec::new()), + ], + vec![edge.clone()], ); + let node_anchor_matches = engine.query_pattern(&query).unwrap().matches; + engine.flush().unwrap(); + let info = engine + .ensure_edge_property_index("EDGE_LABEL_214", "status", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + let edge_anchor_plan = engine.explain_pattern_query(&query).unwrap(); + assert!(plan_contains_pattern_edge_anchor(&edge_anchor_plan.root)); + assert_eq!(engine.query_pattern(&query).unwrap().matches, node_anchor_matches); + + let limited = GraphPatternQuery { + limit: 1, + ..query + }; + let limited_result = engine.query_pattern(&limited).unwrap(); assert_eq!( - engine.query_pattern(&query).unwrap().matches, - vec![expected_match(&[("a", a), ("target", b)], &[("e", good)])] + limited_result.matches, + vec![expected_match(&[("a", a1), ("b", b1)], &[("e", e1)])] ); - - let plan = engine.explain_pattern_query(&query).unwrap(); - assert!(plan - .warnings - .contains(&QueryPlanWarning::EdgePropertyPostFilter)); - assert!(matches!( - plan.root, - QueryPlanNode::VerifyEdgePredicates { .. } - )); - - engine.close().unwrap(); + assert!(limited_result.truncated); + assert_ne!(e1, e2); } #[test] @@ -6724,17 +12870,17 @@ fn test_query_pattern_visibility_temporal_delete_prune_and_reopen() { let deleted; { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - anchor = insert_query_node(&engine, 1, "anchor", &[], 1.0); - keep = insert_query_node(&engine, 2, "keep", &[], 1.0); - hidden = insert_query_node(&engine, 2, "hidden", &[], 0.1); - expired = insert_query_node(&engine, 2, "expired", &[], 1.0); - deleted = insert_query_node(&engine, 2, "deleted", &[], 1.0); + anchor = insert_query_node(&engine, "Person", "anchor", &[], 1.0); + keep = insert_query_node(&engine, "Company", "keep", &[], 1.0); + hidden = insert_query_node(&engine, "Company", "hidden", &[], 0.1); + expired = insert_query_node(&engine, "Company", "expired", &[], 1.0); + deleted = insert_query_node(&engine, "Company", "deleted", &[], 1.0); engine .upsert_edge( anchor, keep, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(100), valid_to: Some(300), @@ -6746,7 +12892,7 @@ fn test_query_pattern_visibility_temporal_delete_prune_and_reopen() { .upsert_edge( anchor, hidden, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(100), valid_to: Some(300), @@ -6758,7 +12904,7 @@ fn test_query_pattern_visibility_temporal_delete_prune_and_reopen() { .upsert_edge( anchor, expired, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(0), valid_to: Some(50), @@ -6770,7 +12916,7 @@ fn test_query_pattern_visibility_temporal_delete_prune_and_reopen() { .upsert_edge( anchor, deleted, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(100), valid_to: Some(300), @@ -6785,7 +12931,7 @@ fn test_query_pattern_visibility_temporal_delete_prune_and_reopen() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(2), + label: Some("Company".to_string()), }, ) .unwrap(); @@ -6794,8 +12940,8 @@ fn test_query_pattern_visibility_temporal_delete_prune_and_reopen() { let query = GraphPatternQuery { at_epoch: Some(150), ..pattern_query( - vec![pattern_node_with_ids("anchor", vec![anchor]), pattern_node("target", Some(2), Vec::new())], - vec![pattern_edge(None, "anchor", "target", Direction::Outgoing, Some(vec![10]))], + vec![pattern_node_with_ids("anchor", vec![anchor]), pattern_node("target", Some("Company"), Vec::new())], + vec![pattern_edge(None, "anchor", "target", Direction::Outgoing, Some(vec!["KNOWS"]))], ) }; assert_eq!( @@ -6820,8 +12966,8 @@ fn test_query_pattern_visibility_temporal_delete_prune_and_reopen() { let query = GraphPatternQuery { at_epoch: Some(150), ..pattern_query( - vec![pattern_node_with_ids("anchor", vec![anchor]), pattern_node("target", Some(2), Vec::new())], - vec![pattern_edge(None, "anchor", "target", Direction::Outgoing, Some(vec![10]))], + vec![pattern_node_with_ids("anchor", vec![anchor]), pattern_node("target", Some("Company"), Vec::new())], + vec![pattern_edge(None, "anchor", "target", Direction::Outgoing, Some(vec!["KNOWS"]))], ) }; assert_eq!( @@ -6843,7 +12989,7 @@ fn test_query_broad_index_warning_priority_and_selective_source_choice() { let mut inputs = Vec::with_capacity(QUERY_RANGE_CANDIDATE_CAP + 1); for index in 0..=QUERY_RANGE_CANDIDATE_CAP { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[ ("status", PropValue::String("inactive".to_string())), @@ -6861,18 +13007,17 @@ fn test_query_broad_index_warning_priority_and_selective_source_choice() { sparse_vector: None, }); } - let all_ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let all_ids = engine.batch_upsert_nodes(inputs).unwrap(); engine.flush().unwrap(); for key in ["status", "tenant", "cohort"] { let info = engine - .ensure_node_property_index(1, key, SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", key, SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); } - let cap_query = query_ids( - Some(1), + let cap_query = query_ids(Some("Person"), vec![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -6895,8 +13040,7 @@ fn test_query_broad_index_warning_priority_and_selective_source_choice() { .contains(&QueryPlanWarning::CandidateCapExceeded)); assert_plan_input_nodes(&cap_plan, vec![QueryPlanNode::PropertyEqualityIndex]); - let broad_skip_query = query_ids( - Some(1), + let broad_skip_query = query_ids(Some("Person"), vec![ NodeFilterExpr::PropertyEquals { key: "cohort".to_string(), @@ -6934,7 +13078,7 @@ fn test_query_large_explicit_ids_become_membership_check_for_cheaper_index() { let mut inputs = Vec::with_capacity(5_000); for index in 0..5_000 { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[( "tenant", @@ -6945,15 +13089,15 @@ fn test_query_large_explicit_ids_become_membership_check_for_cheaper_index() { sparse_vector: None, }); } - let all_ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let all_ids = engine.batch_upsert_nodes(inputs).unwrap(); engine.flush().unwrap(); let info = engine - .ensure_node_property_index(1, "tenant", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "tenant", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); let query = NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), ids: all_ids.clone(), filter: filter_and![NodeFilterExpr::PropertyEquals { key: "tenant".to_string(), @@ -6994,7 +13138,7 @@ fn test_query_large_keys_become_membership_check_for_cheaper_index() { let mut inputs = Vec::with_capacity(50); for index in 0..50 { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[( "tenant", @@ -7005,10 +13149,10 @@ fn test_query_large_keys_become_membership_check_for_cheaper_index() { sparse_vector: None, }); } - let all_ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let all_ids = engine.batch_upsert_nodes(inputs).unwrap(); engine.flush().unwrap(); let info = engine - .ensure_node_property_index(1, "tenant", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "tenant", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); @@ -7017,7 +13161,7 @@ fn test_query_large_keys_become_membership_check_for_cheaper_index() { .collect(); keys.push("n-0".to_string()); let query = NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), keys, filter: filter_and![NodeFilterExpr::PropertyEquals { key: "tenant".to_string(), @@ -7043,23 +13187,17 @@ fn test_query_full_scan_pagination_proves_extra_match_and_skips_tombstone() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let first = insert_query_node( - &engine, - 1, + let first = insert_query_node(&engine, "Person", "first", &[("status", PropValue::String("keep".to_string()))], 1.0, ); - let deleted = insert_query_node( - &engine, - 1, + let deleted = insert_query_node(&engine, "Person", "deleted", &[("status", PropValue::String("keep".to_string()))], 1.0, ); - let last = insert_query_node( - &engine, - 1, + let last = insert_query_node(&engine, "Person", "last", &[("status", PropValue::String("keep".to_string()))], 1.0, @@ -7101,7 +13239,7 @@ fn test_query_unknown_selected_index_source_falls_back_when_execution_cap_exceed let mut inputs = Vec::with_capacity(QUERY_RANGE_CANDIDATE_CAP + 1); for index in 0..=QUERY_RANGE_CANDIDATE_CAP { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[("status", PropValue::String("inactive".to_string()))]), weight: 1.0, @@ -7109,22 +13247,20 @@ fn test_query_unknown_selected_index_source_falls_back_when_execution_cap_exceed sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); - let active = insert_query_node( - &engine, - 1, + let ids = engine.batch_upsert_nodes(inputs).unwrap(); + let active = insert_query_node(&engine, "Person", "active", &[("status", PropValue::String("active".to_string()))], 1.0, ); engine.flush().unwrap(); let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); let public_query = NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), filter: filter_and![NodeFilterExpr::PropertyEquals { key: "status".to_string(), value: PropValue::String("inactive".to_string()), @@ -7138,7 +13274,10 @@ fn test_query_unknown_selected_index_source_falls_back_when_execution_cap_exceed { let (_guard, published) = engine.runtime.published_snapshot().unwrap(); - let normalized = published.view.normalize_node_query(&public_query).unwrap(); + let normalized = published + .view + .normalize_node_query(&public_query) + .unwrap(); let cap_context = published.view.query_cap_context(&normalized).unwrap(); let planned = PlannedNodeQuery { driver: NodePhysicalPlan::source(PlannedNodeCandidateSource::property_equality_index( @@ -7161,7 +13300,7 @@ fn test_query_unknown_selected_index_source_falls_back_when_execution_cap_exceed assert_eq!(page.next_cursor, page.ids.last().copied()); let union_query = NodeQuery { - type_id: Some(1), + label_filter: Some(NodeLabelFilter { labels: vec!["Person".to_string()], mode: LabelMatchMode::All }), filter: Some(NodeFilterExpr::Or(vec![ NodeFilterExpr::PropertyEquals { key: "status".to_string(), @@ -7178,7 +13317,10 @@ fn test_query_unknown_selected_index_source_falls_back_when_execution_cap_exceed }, ..Default::default() }; - let normalized = published.view.normalize_node_query(&union_query).unwrap(); + let normalized = published + .view + .normalize_node_query(&union_query) + .unwrap(); let cap_context = published.view.query_cap_context(&normalized).unwrap(); let planned = PlannedNodeQuery { driver: NodePhysicalPlan::union(vec![ @@ -7222,7 +13364,7 @@ fn test_query_limited_equality_read_skips_shadowed_ids_before_cap() { let mut inputs = Vec::with_capacity(3); for index in 0..3 { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[("status", PropValue::String("old".to_string()))]), weight: 1.0, @@ -7230,19 +13372,19 @@ fn test_query_limited_equality_read_skips_shadowed_ids_before_cap() { sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs).unwrap(); let surviving_old_id = *ids.last().unwrap(); engine.flush().unwrap(); let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); for (index, id) in ids.iter().enumerate().take(2) { let updated_id = engine .upsert_node( - 1, + "Person", &format!("n-{index}"), UpsertNodeOptions { props: query_test_props(&[( @@ -7284,7 +13426,7 @@ fn test_query_limited_equality_read_skips_newer_segment_shadowed_ids_before_cap( let mut inputs = Vec::with_capacity(3); for index in 0..3 { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[("status", PropValue::String("old".to_string()))]), weight: 1.0, @@ -7292,19 +13434,19 @@ fn test_query_limited_equality_read_skips_newer_segment_shadowed_ids_before_cap( sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs).unwrap(); let surviving_old_id = *ids.last().unwrap(); engine.flush().unwrap(); let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); for (index, id) in ids.iter().enumerate().take(2) { let updated_id = engine .upsert_node( - 1, + "Person", &format!("n-{index}"), UpsertNodeOptions { props: query_test_props(&[( @@ -7344,14 +13486,14 @@ fn test_query_limited_equality_read_enforces_raw_posting_cap_for_stale_segments( let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); let mut inputs = Vec::new(); for index in 0..24 { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[("status", PropValue::String("old".to_string()))]), weight: 1.0, @@ -7359,14 +13501,14 @@ fn test_query_limited_equality_read_enforces_raw_posting_cap_for_stale_segments( sparse_vector: None, }); } - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let ids = engine.batch_upsert_nodes(inputs).unwrap(); let surviving_old_id = *ids.last().unwrap(); engine.flush().unwrap(); for (index, id) in ids.iter().copied().enumerate().take(23) { let updated_id = engine .upsert_node( - 1, + "Person", &format!("n-{index}"), UpsertNodeOptions { props: query_test_props(&[( @@ -7417,14 +13559,14 @@ fn test_query_stats_backed_equality_materialization_uses_raw_ids_only() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); let mut inputs = Vec::new(); for index in 0..24 { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("n-{index}"), props: query_test_props(&[("status", PropValue::String("old".to_string()))]), weight: 1.0, @@ -7432,14 +13574,14 @@ fn test_query_stats_backed_equality_materialization_uses_raw_ids_only() { sparse_vector: None, }); } - let all_ids = engine.batch_upsert_nodes(&inputs).unwrap(); + let all_ids = engine.batch_upsert_nodes(inputs).unwrap(); let surviving_old_id = *all_ids.last().unwrap(); engine.flush().unwrap(); for (index, id) in all_ids.iter().copied().enumerate().take(23) { let updated_id = engine .upsert_node( - 1, + "Person", &format!("n-{index}"), UpsertNodeOptions { props: query_test_props(&[( @@ -7453,8 +13595,7 @@ fn test_query_stats_backed_equality_materialization_uses_raw_ids_only() { assert_eq!(updated_id, id); } - let query = query_ids( - Some(1), + let query = query_ids(Some("Person"), vec![NodeFilterExpr::PropertyEquals { key: "status".to_string(), value: PropValue::String("old".to_string()), @@ -7530,7 +13671,8 @@ fn test_query_unknown_estimates_use_stable_rank_then_key() { } #[test] -fn test_query_empty_edge_filter_rejected_and_segment_unnamed_parallel_edges_dedup() { +fn test_query_empty_label_filter_matches_unconstrained_and_segment_unnamed_parallel_edges_dedup() +{ let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open( @@ -7542,52 +13684,57 @@ fn test_query_empty_edge_filter_rejected_and_segment_unnamed_parallel_edges_dedu ) .unwrap(); - let source = insert_query_node(&engine, 1, "source", &[], 1.0); - let target = insert_query_node(&engine, 2, "target", &[], 1.0); - - let invalid = pattern_query( - vec![ - pattern_node_with_ids("source", vec![source]), - pattern_node("target", Some(2), Vec::new()), - ], - vec![pattern_edge( - None, - "source", - "target", - Direction::Outgoing, - Some(Vec::new()), - )], - ); - assert!(matches!( - engine.query_pattern(&invalid).unwrap_err(), - EngineError::InvalidOperation(_) - )); + let source = insert_query_node(&engine, "Person", "source", &[], 1.0); + let target = insert_query_node(&engine, "Company", "target", &[], 1.0); let edges: Vec = (0..16) .map(|_| EdgeInput { from: source, to: target, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edges).unwrap(); + engine.batch_upsert_edges(edges).unwrap(); engine.flush().unwrap(); + let unconstrained = pattern_query( + vec![ + pattern_node_with_ids("source", vec![source]), + pattern_node("target", Some("Company"), Vec::new()), + ], + vec![pattern_edge( + None, + "source", + "target", + Direction::Outgoing, + Some(Vec::new()), + )], + ); + let result = engine.query_pattern(&unconstrained).unwrap(); + assert_eq!( + result.matches, + vec![expected_match( + &[("source", source), ("target", target)], + &[] + )] + ); + assert!(!result.truncated); + let query = pattern_query( vec![ pattern_node_with_ids("source", vec![source]), - pattern_node("target", Some(2), Vec::new()), + pattern_node("target", Some("Company"), Vec::new()), ], vec![pattern_edge( None, "source", "target", Direction::Outgoing, - Some(vec![10]), + Some(vec!["KNOWS"]), )], ); let result = engine.query_pattern(&query).unwrap(); @@ -7616,13 +13763,13 @@ fn test_query_pattern_segment_named_parallel_edges_preserve_edge_matches() { ) .unwrap(); - let source = insert_query_node(&engine, 1, "source", &[], 1.0); - let target = insert_query_node(&engine, 2, "target", &[], 1.0); + let source = insert_query_node(&engine, "Person", "source", &[], 1.0); + let target = insert_query_node(&engine, "Company", "target", &[], 1.0); let edges = vec![ EdgeInput { from: source, to: target, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -7631,27 +13778,27 @@ fn test_query_pattern_segment_named_parallel_edges_preserve_edge_matches() { EdgeInput { from: source, to: target, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }, ]; - let edge_ids = engine.batch_upsert_edges(&edges).unwrap(); + let edge_ids = engine.batch_upsert_edges(edges).unwrap(); engine.flush().unwrap(); let query = pattern_query( vec![ pattern_node_with_ids("source", vec![source]), - pattern_node("target", Some(2), Vec::new()), + pattern_node("target", Some("Company"), Vec::new()), ], vec![pattern_edge( Some("edge"), "source", "target", Direction::Outgoing, - Some(vec![10]), + Some(vec!["KNOWS"]), )], ); @@ -7678,12 +13825,12 @@ fn test_query_pattern_frontier_budget_rejects_pathological_expansion() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let anchor = insert_query_node(&engine, 1, "anchor", &[], 1.0); - let sink = insert_query_node(&engine, 3, "sink", &[], 1.0); + let anchor = insert_query_node(&engine, "Person", "anchor", &[], 1.0); + let sink = insert_query_node(&engine, "Article", "sink", &[], 1.0); let mid_inputs: Vec = (0..=PATTERN_FRONTIER_BUDGET) .map(|index| NodeInput { - type_id: 2, + labels: vec!["Company".to_string()], key: format!("mid-{index}"), props: BTreeMap::new(), weight: 1.0, @@ -7691,30 +13838,31 @@ fn test_query_pattern_frontier_budget_rejects_pathological_expansion() { sparse_vector: None, }) .collect(); - let mids = engine.batch_upsert_nodes(&mid_inputs).unwrap(); + let mids = engine.batch_upsert_nodes(mid_inputs).unwrap(); let edge_inputs: Vec = mids .iter() .map(|&mid| EdgeInput { from: anchor, to: mid, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&edge_inputs).unwrap(); + engine.batch_upsert_edges(edge_inputs).unwrap(); + engine.ensure_edge_label("BLOCKS").unwrap(); let query = pattern_query( vec![ pattern_node_with_ids("anchor", vec![anchor]), - pattern_node("mid", Some(2), Vec::new()), + pattern_node("mid", Some("Company"), Vec::new()), pattern_node_with_ids("sink", vec![sink]), ], vec![ - pattern_edge(None, "anchor", "mid", Direction::Outgoing, Some(vec![10])), - pattern_edge(None, "mid", "sink", Direction::Outgoing, Some(vec![11])), + pattern_edge(None, "anchor", "mid", Direction::Outgoing, Some(vec!["KNOWS"])), + pattern_edge(None, "mid", "sink", Direction::Outgoing, Some(vec!["BLOCKS"])), ], ); diff --git a/src/engine/tests/read.rs b/src/engine/tests/read.rs index cc036b9..b2bb833 100644 --- a/src/engine/tests/read.rs +++ b/src/engine/tests/read.rs @@ -1,11 +1,31 @@ -// Read tests: type index, find, neighbors, pagination, temporal, decay, traversal, top-k, PPR, export. +// Read tests: label index, find, neighbors, pagination, temporal, decay, traversal, top-k, PPR, export. + +fn read_filter_names(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_string()).collect() +} + +fn read_node_label_filter(names: &[&str], mode: LabelMatchMode) -> NodeLabelFilter { + NodeLabelFilter { + labels: read_filter_names(names), + mode, + } +} + +fn read_node_key_queries(keys: &[(&str, &str)]) -> Vec { + keys.iter() + .map(|&(label, key)| NodeKeyQuery { + label: label.to_string(), + key: key.to_string(), + }) + .collect() +} fn traverse_depth_two_read( engine: &DatabaseEngine, start: u64, direction: Direction, - edge_type_filter: Option<&[u32]>, - node_type_filter: Option<&[u32]>, + edge_label_filter: Option<&[&str]>, + node_label_filter: Option<&[&str]>, at_epoch: Option, ) -> Vec { engine @@ -15,8 +35,9 @@ fn traverse_depth_two_read( &TraverseOptions { min_depth: 2, direction, - edge_type_filter: edge_type_filter.map(|s| s.to_vec()), - node_type_filter: node_type_filter.map(|s| s.to_vec()), + edge_label_filter: edge_label_filter.map(read_filter_names), + emit_node_label_filter: node_label_filter + .map(|labels| read_node_label_filter(labels, LabelMatchMode::Any)), at_epoch, ..Default::default() }, @@ -25,17 +46,17 @@ fn traverse_depth_two_read( .items } -// --- Type index tests --- +// --- Label index tests --- #[test] -fn test_nodes_by_type_memtable_only() { +fn test_nodes_by_labels_memtable_only() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -45,7 +66,7 @@ fn test_nodes_by_type_memtable_only() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.5, @@ -55,7 +76,7 @@ fn test_nodes_by_type_memtable_only() { .unwrap(); let c = engine .upsert_node( - 2, + "Company", "charlie", UpsertNodeOptions { weight: 0.5, @@ -64,24 +85,58 @@ fn test_nodes_by_type_memtable_only() { ) .unwrap(); - let mut type1 = engine.nodes_by_type(1).unwrap(); - type1.sort(); - assert_eq!(type1, vec![a, b]); - assert_eq!(engine.nodes_by_type(2).unwrap(), vec![c]); - assert!(engine.nodes_by_type(99).unwrap().is_empty()); + let mut person_ids = engine.nodes_by_labels("Person").unwrap(); + person_ids.sort(); + assert_eq!(person_ids, vec![a, b]); + assert_eq!(engine.nodes_by_labels("Company").unwrap(), vec![c]); + assert!(engine.nodes_by_labels("MissingLabel").unwrap().is_empty()); + + engine.close().unwrap(); +} + +#[test] +fn test_nodes_by_labels_multi_label_all() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let both = engine + .upsert_node( + &["Person", "Employee"], + "alice", + UpsertNodeOptions::default(), + ) + .unwrap(); + let person_only = engine + .upsert_node("Person", "bob", UpsertNodeOptions::default()) + .unwrap(); + let _employee_only = engine + .upsert_node("Employee", "cara", UpsertNodeOptions::default()) + .unwrap(); + + assert_eq!( + engine + .nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(), + vec![both] + ); + assert_eq!( + engine.nodes_by_labels("Person").unwrap(), + vec![both, person_only] + ); engine.close().unwrap(); } #[test] -fn test_edges_by_type_memtable_only() { +fn test_edges_by_label_memtable_only() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -91,7 +146,7 @@ fn test_edges_by_type_memtable_only() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -100,30 +155,30 @@ fn test_edges_by_type_memtable_only() { ) .unwrap(); let e1 = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let e2 = engine - .upsert_edge(a, b, 20, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - assert_eq!(engine.edges_by_type(10).unwrap(), vec![e1]); - assert_eq!(engine.edges_by_type(20).unwrap(), vec![e2]); - assert!(engine.edges_by_type(99).unwrap().is_empty()); + assert_eq!(engine.edges_by_label("KNOWS").unwrap(), vec![e1]); + assert_eq!(engine.edges_by_label("REPORTS_TO").unwrap(), vec![e2]); + assert!(engine.edges_by_label("MISSING_EDGE_LABEL").unwrap().is_empty()); engine.close().unwrap(); } #[test] -fn test_nodes_by_type_cross_source() { +fn test_nodes_by_labels_cross_source() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // Segment: type 1 nodes + // Segment: Person-labeled nodes let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -133,7 +188,7 @@ fn test_nodes_by_type_cross_source() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.5, @@ -143,10 +198,10 @@ fn test_nodes_by_type_cross_source() { .unwrap(); engine.flush().unwrap(); - // Memtable: more type 1 + type 2 + // Memtable: more Person + Company labels. let c = engine .upsert_node( - 1, + "Person", "charlie", UpsertNodeOptions { weight: 0.5, @@ -156,7 +211,7 @@ fn test_nodes_by_type_cross_source() { .unwrap(); let d = engine .upsert_node( - 2, + "Company", "delta", UpsertNodeOptions { weight: 0.5, @@ -165,16 +220,16 @@ fn test_nodes_by_type_cross_source() { ) .unwrap(); - let mut type1 = engine.nodes_by_type(1).unwrap(); - type1.sort(); - assert_eq!(type1, vec![a, b, c]); - assert_eq!(engine.nodes_by_type(2).unwrap(), vec![d]); + let mut person_ids = engine.nodes_by_labels("Person").unwrap(); + person_ids.sort(); + assert_eq!(person_ids, vec![a, b, c]); + assert_eq!(engine.nodes_by_labels("Company").unwrap(), vec![d]); engine.close().unwrap(); } #[test] -fn test_nodes_by_type_excludes_deleted() { +fn test_nodes_by_labels_excludes_deleted() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); @@ -182,7 +237,7 @@ fn test_nodes_by_type_excludes_deleted() { let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -192,7 +247,7 @@ fn test_nodes_by_type_excludes_deleted() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.5, @@ -205,14 +260,14 @@ fn test_nodes_by_type_excludes_deleted() { // Delete alice (cross-source tombstone: segment data, memtable tombstone) engine.delete_node(a).unwrap(); - let type1 = engine.nodes_by_type(1).unwrap(); - assert_eq!(type1, vec![b]); + let person_ids = engine.nodes_by_labels("Person").unwrap(); + assert_eq!(person_ids, vec![b]); engine.close().unwrap(); } #[test] -fn test_type_index_survives_flush_and_reopen() { +fn test_label_index_survives_flush_and_reopen() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); @@ -222,7 +277,7 @@ fn test_type_index_survives_flush_and_reopen() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -232,7 +287,7 @@ fn test_type_index_survives_flush_and_reopen() { .unwrap(); b = engine .upsert_node( - 2, + "Company", "bob", UpsertNodeOptions { weight: 0.5, @@ -241,25 +296,25 @@ fn test_type_index_survives_flush_and_reopen() { ) .unwrap(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); engine.close().unwrap(); } - // Reopen. Type index should be available from segment + // Reopen. Label index should be available from segment let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert_eq!(engine.nodes_by_type(1).unwrap(), vec![a]); - assert_eq!(engine.nodes_by_type(2).unwrap(), vec![b]); - assert_eq!(engine.edges_by_type(10).unwrap().len(), 1); + assert_eq!(engine.nodes_by_labels("Person").unwrap(), vec![a]); + assert_eq!(engine.nodes_by_labels("Company").unwrap(), vec![b]); + assert_eq!(engine.edges_by_label("KNOWS").unwrap().len(), 1); engine.close().unwrap(); } -// --- get_nodes_by_type / get_edges_by_type / count tests --- +// --- get_nodes_by_labels / get_edges_by_label / count tests --- #[test] -fn test_get_nodes_by_type_memtable_only() { +fn test_get_nodes_by_labels_memtable_only() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); @@ -268,7 +323,7 @@ fn test_get_nodes_by_type_memtable_only() { props.insert("name".to_string(), PropValue::String("Alice".to_string())); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: props.clone(), @@ -280,7 +335,7 @@ fn test_get_nodes_by_type_memtable_only() { props.insert("name".to_string(), PropValue::String("Bob".to_string())); engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { props, @@ -291,7 +346,7 @@ fn test_get_nodes_by_type_memtable_only() { .unwrap(); engine .upsert_node( - 2, + "Company", "charlie", UpsertNodeOptions { weight: 0.7, @@ -300,25 +355,60 @@ fn test_get_nodes_by_type_memtable_only() { ) .unwrap(); - let type1 = engine.get_nodes_by_type(1).unwrap(); - assert_eq!(type1.len(), 2); - assert!(type1.iter().all(|n| n.type_id == 1)); - assert!(type1.iter().any(|n| n.key == "alice")); - assert!(type1.iter().any(|n| n.key == "bob")); + let people = engine.get_nodes_by_labels("Person").unwrap(); + assert_eq!(people.len(), 2); + assert!(people.iter().all(|n| n.labels.as_slice() == ["Person"])); + assert!(people.iter().any(|n| n.key == "alice")); + assert!(people.iter().any(|n| n.key == "bob")); - let type2 = engine.get_nodes_by_type(2).unwrap(); - assert_eq!(type2.len(), 1); - assert_eq!(type2[0].key, "charlie"); + let companies = engine.get_nodes_by_labels("Company").unwrap(); + assert_eq!(companies.len(), 1); + assert_eq!(companies[0].key, "charlie"); - // Non-existent type - let empty = engine.get_nodes_by_type(99).unwrap(); + // Non-existent label + let empty = engine.get_nodes_by_labels("MissingLabel").unwrap(); assert!(empty.is_empty()); engine.close().unwrap(); } #[test] -fn test_get_nodes_by_type_cross_source() { +fn test_get_nodes_by_labels_multi_label_all() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let both = engine + .upsert_node( + &["Person", "Employee"], + "alice", + UpsertNodeOptions::default(), + ) + .unwrap(); + let _person_only = engine + .upsert_node("Person", "bob", UpsertNodeOptions::default()) + .unwrap(); + let _employee_only = engine + .upsert_node("Employee", "cara", UpsertNodeOptions::default()) + .unwrap(); + + let nodes = engine + .get_nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(); + assert_eq!( + nodes.iter().map(|node| node.id).collect::>(), + vec![both] + ); + assert_eq!( + nodes[0].labels, + vec!["Person".to_string(), "Employee".to_string()] + ); + + engine.close().unwrap(); +} + +#[test] +fn test_get_nodes_by_labels_cross_source() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let opts = DbOptions { @@ -327,21 +417,21 @@ fn test_get_nodes_by_type_cross_source() { }; let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - // Type 1 nodes in segment + // Label 1 nodes in segment engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); - // Type 1 node in memtable + // Label 1 node in memtable engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - let records = engine.get_nodes_by_type(1).unwrap(); + let records = engine.get_nodes_by_labels("Person").unwrap(); assert_eq!(records.len(), 3); let keys: Vec<&str> = records.iter().map(|n| n.key.as_str()).collect(); assert!(keys.contains(&"a")); @@ -350,7 +440,7 @@ fn test_get_nodes_by_type_cross_source() { // Verify records carry full data (props, weight, timestamps) for r in &records { - assert_eq!(r.type_id, 1); + assert_eq!(r.labels.as_slice(), ["Person"]); assert!(r.weight > 0.0); assert!(r.created_at > 0); } @@ -359,14 +449,14 @@ fn test_get_nodes_by_type_cross_source() { } #[test] -fn test_get_nodes_by_type_excludes_deleted() { +fn test_get_nodes_by_labels_excludes_deleted() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 0.5, @@ -376,7 +466,7 @@ fn test_get_nodes_by_type_excludes_deleted() { .unwrap(); engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.5, @@ -388,7 +478,7 @@ fn test_get_nodes_by_type_excludes_deleted() { engine.delete_node(a).unwrap(); - let records = engine.get_nodes_by_type(1).unwrap(); + let records = engine.get_nodes_by_labels("Person").unwrap(); assert_eq!(records.len(), 1); assert_eq!(records[0].key, "bob"); @@ -396,14 +486,14 @@ fn test_get_nodes_by_type_excludes_deleted() { } #[test] -fn test_get_nodes_by_type_excludes_pruned() { +fn test_get_nodes_by_labels_excludes_pruned() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.1, @@ -413,7 +503,7 @@ fn test_get_nodes_by_type_excludes_pruned() { .unwrap(); engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -429,12 +519,12 @@ fn test_get_nodes_by_type_excludes_pruned() { PrunePolicy { max_weight: Some(0.5), max_age_ms: None, - type_id: None, + label: None, }, ) .unwrap(); - let records = engine.get_nodes_by_type(1).unwrap(); + let records = engine.get_nodes_by_labels("Person").unwrap(); assert_eq!(records.len(), 1); assert_eq!(records[0].key, "high"); @@ -442,7 +532,7 @@ fn test_get_nodes_by_type_excludes_pruned() { } #[test] -fn test_get_nodes_by_type_post_compaction() { +fn test_get_nodes_by_labels_post_compaction() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let opts = DbOptions { @@ -452,27 +542,27 @@ fn test_get_nodes_by_type_post_compaction() { let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine.compact().unwrap(); - let records = engine.get_nodes_by_type(1).unwrap(); + let records = engine.get_nodes_by_labels("Person").unwrap(); assert_eq!(records.len(), 3); engine.close().unwrap(); } #[test] -fn test_get_edges_by_type_memtable_and_segment() { +fn test_get_edges_by_label_memtable_and_segment() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let opts = DbOptions { @@ -483,39 +573,39 @@ fn test_get_edges_by_type_memtable_and_segment() { let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - // Type 10 edge in segment + // Label 10 edge in segment engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); - // Type 10 edge in memtable + // Label 10 edge in memtable engine .upsert_edge( b, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.8, ..Default::default() }, ) .unwrap(); - // Type 20 edge in memtable + // Label 20 edge in memtable engine .upsert_edge( a, c, - 20, + "REPORTS_TO", UpsertEdgeOptions { weight: 0.5, ..Default::default() @@ -523,30 +613,30 @@ fn test_get_edges_by_type_memtable_and_segment() { ) .unwrap(); - let type10 = engine.get_edges_by_type(10).unwrap(); - assert_eq!(type10.len(), 2); - assert!(type10.iter().all(|e| e.type_id == 10)); + let label10 = engine.get_edges_by_label("KNOWS").unwrap(); + assert_eq!(label10.len(), 2); + assert!(label10.iter().all(|e| e.label == "KNOWS")); - let type20 = engine.get_edges_by_type(20).unwrap(); - assert_eq!(type20.len(), 1); - assert_eq!(type20[0].type_id, 20); + let label20 = engine.get_edges_by_label("REPORTS_TO").unwrap(); + assert_eq!(label20.len(), 1); + assert_eq!(label20[0].label, "REPORTS_TO"); // Verify records carry full data - for e in &type10 { + for e in &label10 { assert!(e.weight > 0.0); assert!(e.from > 0); assert!(e.to > 0); } // Empty type - let empty = engine.get_edges_by_type(99).unwrap(); + let empty = engine.get_edges_by_label("MISSING_EDGE_LABEL").unwrap(); assert!(empty.is_empty()); engine.close().unwrap(); } #[test] -fn test_get_edges_by_type_excludes_deleted() { +fn test_get_edges_by_label_excludes_deleted() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let opts = DbOptions { @@ -557,35 +647,35 @@ fn test_get_edges_by_type_excludes_deleted() { let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); engine.delete_edge(e1).unwrap(); - let type10 = engine.get_edges_by_type(10).unwrap(); - assert_eq!(type10.len(), 1); - assert_eq!(type10[0].from, b); - assert_eq!(type10[0].to, c); + let label10 = engine.get_edges_by_label("KNOWS").unwrap(); + assert_eq!(label10.len(), 1); + assert_eq!(label10[0].from, b); + assert_eq!(label10[0].to, c); engine.close().unwrap(); } #[test] -fn test_count_nodes_by_type() { +fn test_count_nodes_by_labels_single_label() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let opts = DbOptions { @@ -594,32 +684,259 @@ fn test_count_nodes_by_type() { }; let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - // 3 type-1 nodes across memtable + segment + // 3 Person-labeled nodes across memtable + segment engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) + .unwrap(); + + // 1 Company-labeled node + engine + .upsert_node("Company", "x", UpsertNodeOptions::default()) + .unwrap(); + + assert_eq!(engine.count_nodes_by_labels("Person").unwrap(), 3); + assert_eq!(engine.count_nodes_by_labels("Company").unwrap(), 1); + assert_eq!(engine.count_nodes_by_labels("MissingLabel").unwrap(), 0); + + engine.close().unwrap(); +} + +#[test] +fn test_count_nodes_by_labels_multi_label_all() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + engine + .upsert_node( + &["Person", "Employee"], + "alice", + UpsertNodeOptions::default(), + ) + .unwrap(); + engine + .upsert_node( + &["Person", "Employee"], + "bob", + UpsertNodeOptions::default(), + ) + .unwrap(); + engine + .upsert_node("Person", "cara", UpsertNodeOptions::default()) + .unwrap(); + + assert_eq!( + engine + .count_nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(), + 2 + ); + assert_eq!(engine.count_nodes_by_labels("Person").unwrap(), 3); + + engine.close().unwrap(); +} + +#[test] +fn test_count_nodes_by_labels_suppresses_stale_memberships() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let opts = DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }; + + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let id = engine + .upsert_node( + &["Person", "Employee"], + "alice", + UpsertNodeOptions::default(), + ) + .unwrap(); + engine.flush().unwrap(); + assert_eq!( + engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(), + id + ); + + assert_eq!( + engine + .count_nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(), + 0 + ); + engine.flush().unwrap(); + engine.close().unwrap(); + } + + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert_eq!( + engine + .count_nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(), + 0 + ); + engine.compact().unwrap(); + assert_eq!( + engine + .count_nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(), + 0 + ); + engine.close().unwrap(); + } +} + +#[test] +fn test_nodes_by_labels_all_superset_verifies_after_replacement() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let opts = DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }; + + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let id = engine + .upsert_node( + &["Person", "Employee"], + "alice", + UpsertNodeOptions::default(), + ) + .unwrap(); + engine.flush().unwrap(); + assert_eq!( + engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(), + id + ); + + assert!(engine + .nodes_by_labels(&["Person", "Employee"]) + .unwrap() + .is_empty()); + assert!(engine + .get_nodes_by_labels(&["Person", "Employee"]) + .unwrap() + .is_empty()); + engine.flush().unwrap(); + engine.close().unwrap(); + } + + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert!(engine + .nodes_by_labels(&["Person", "Employee"]) + .unwrap() + .is_empty()); + assert!(engine + .get_nodes_by_labels(&["Person", "Employee"]) + .unwrap() + .is_empty()); + engine.compact().unwrap(); + assert!(engine + .nodes_by_labels(&["Person", "Employee"]) + .unwrap() + .is_empty()); + engine.close().unwrap(); + } +} + +#[test] +fn test_count_nodes_by_labels_unknown_and_invalid_inputs() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + + assert_eq!(engine.count_nodes_by_labels("Missing").unwrap(), 0); + assert_eq!( + engine + .count_nodes_by_labels(vec!["Person".to_string(), "Missing".to_string()]) + .unwrap(), + 0 + ); + assert!(engine.count_nodes_by_labels(Vec::::new()).is_err()); + assert!(engine + .count_nodes_by_labels(vec!["Person".to_string(), "Person".to_string()]) + .is_err()); + + engine.close().unwrap(); +} + +#[test] +fn test_count_nodes_by_labels_respects_policies() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + engine + .upsert_node( + &["Person", "Employee"], + "low", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_node( + &["Person", "Employee"], + "high", + UpsertNodeOptions { + weight: 0.9, + ..Default::default() + }, + ) .unwrap(); - // 1 type-2 node + assert_eq!( + engine + .count_nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(), + 2 + ); + engine - .upsert_node(2, "x", UpsertNodeOptions::default()) + .set_prune_policy( + "low-weight", + PrunePolicy { + max_weight: Some(0.5), + max_age_ms: None, + label: None, + }, + ) .unwrap(); - assert_eq!(engine.count_nodes_by_type(1).unwrap(), 3); - assert_eq!(engine.count_nodes_by_type(2).unwrap(), 1); - assert_eq!(engine.count_nodes_by_type(99).unwrap(), 0); + assert_eq!( + engine + .count_nodes_by_labels(vec!["Person".to_string(), "Employee".to_string()]) + .unwrap(), + 1 + ); engine.close().unwrap(); } #[test] -fn test_count_edges_by_type() { +fn test_count_edges_by_label() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let opts = DbOptions { @@ -630,42 +947,42 @@ fn test_count_edges_by_type() { let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .upsert_edge(a, c, 20, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - assert_eq!(engine.count_edges_by_type(10).unwrap(), 2); - assert_eq!(engine.count_edges_by_type(20).unwrap(), 1); - assert_eq!(engine.count_edges_by_type(99).unwrap(), 0); + assert_eq!(engine.count_edges_by_label("KNOWS").unwrap(), 2); + assert_eq!(engine.count_edges_by_label("REPORTS_TO").unwrap(), 1); + assert_eq!(engine.count_edges_by_label("MISSING_EDGE_LABEL").unwrap(), 0); engine.close().unwrap(); } #[test] -fn test_count_nodes_by_type_respects_policies() { +fn test_count_nodes_by_labels_single_label_respects_policies() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.1, @@ -675,7 +992,7 @@ fn test_count_nodes_by_type_respects_policies() { .unwrap(); engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -684,7 +1001,7 @@ fn test_count_nodes_by_type_respects_policies() { ) .unwrap(); - assert_eq!(engine.count_nodes_by_type(1).unwrap(), 2); + assert_eq!(engine.count_nodes_by_labels("Person").unwrap(), 2); engine .set_prune_policy( @@ -692,30 +1009,30 @@ fn test_count_nodes_by_type_respects_policies() { PrunePolicy { max_weight: Some(0.5), max_age_ms: None, - type_id: None, + label: None, }, ) .unwrap(); // Now the low-weight node is excluded - assert_eq!(engine.count_nodes_by_type(1).unwrap(), 1); + assert_eq!(engine.count_nodes_by_labels("Person").unwrap(), 1); engine.close().unwrap(); } -// --- Paginated type-index query tests --- +// --- Paginated node-label query tests --- #[test] -fn test_nodes_by_type_paged_basic() { +fn test_nodes_by_labels_paged_basic() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // Create 10 nodes of type 1 + // Create 10 Person-labeled nodes. let mut ids: Vec = Vec::new(); for i in 0..10 { let id = engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); ids.push(id); } @@ -723,8 +1040,7 @@ fn test_nodes_by_type_paged_basic() { // Page through 3 at a time let page1 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: None, @@ -736,8 +1052,7 @@ fn test_nodes_by_type_paged_basic() { assert!(page1.next_cursor.is_some()); let page2 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: page1.next_cursor, @@ -749,8 +1064,7 @@ fn test_nodes_by_type_paged_basic() { assert!(page2.next_cursor.is_some()); let page3 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: page2.next_cursor, @@ -762,8 +1076,7 @@ fn test_nodes_by_type_paged_basic() { assert!(page3.next_cursor.is_some()); let page4 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: page3.next_cursor, @@ -776,24 +1089,67 @@ fn test_nodes_by_type_paged_basic() { } #[test] -fn test_nodes_by_type_paged_roundtrip() { - // Page through all results 1-at-a-time, collect, should equal unpaginated +fn test_nodes_by_labels_paged_multi_label_all() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - for i in 0..20 { - engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) - .unwrap(); - } - + let alice = engine + .upsert_node(&["Person", "Admin"], "alice", UpsertNodeOptions::default()) + .unwrap(); + let bob = engine + .upsert_node(&["Person", "Admin"], "bob", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_node("Person", "carol", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_node("Admin", "dave", UpsertNodeOptions::default()) + .unwrap(); + + let page1 = engine + .nodes_by_labels_paged( + &["Person", "Admin"], + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(page1.items, vec![alice]); + assert_eq!(page1.next_cursor, Some(alice)); + + let page2 = engine + .nodes_by_labels_paged( + &["Person", "Admin"], + &PageRequest { + limit: Some(1), + after: page1.next_cursor, + }, + ) + .unwrap(); + assert_eq!(page2.items, vec![bob]); + assert_eq!(page2.next_cursor, None); +} + +#[test] +fn test_nodes_by_labels_paged_roundtrip() { + // Page through all results 1-at-a-time, collect, should equal unpaginated + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + for i in 0..20 { + engine + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) + .unwrap(); + } + let mut all_paged: Vec = Vec::new(); let mut cursor: Option = None; loop { let page = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(4), after: cursor, @@ -807,39 +1163,38 @@ fn test_nodes_by_type_paged_roundtrip() { } } - let mut all_unpaged = engine.nodes_by_type(1).unwrap(); + let mut all_unpaged = engine.nodes_by_labels("Person").unwrap(); all_unpaged.sort(); assert_eq!(all_paged, all_unpaged); } #[test] -fn test_nodes_by_type_paged_default_returns_all() { +fn test_nodes_by_labels_paged_default_returns_all() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); for i in 0..5 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } let result = engine - .nodes_by_type_paged(1, &PageRequest::default()) + .nodes_by_labels_paged("Person", &PageRequest::default()) .unwrap(); assert_eq!(result.items.len(), 5); assert!(result.next_cursor.is_none()); } #[test] -fn test_nodes_by_type_paged_empty_type() { +fn test_nodes_by_labels_paged_empty_label() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let result = engine - .nodes_by_type_paged( - 99, + .nodes_by_labels_paged("MissingLabel", &PageRequest { limit: Some(10), after: None, @@ -851,20 +1206,19 @@ fn test_nodes_by_type_paged_empty_type() { } #[test] -fn test_nodes_by_type_paged_cursor_past_end() { +fn test_nodes_by_labels_paged_cursor_past_end() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); for i in 0..3 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } let result = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(10), after: Some(u64::MAX), @@ -876,7 +1230,7 @@ fn test_nodes_by_type_paged_cursor_past_end() { } #[test] -fn test_nodes_by_type_paged_cross_source() { +fn test_nodes_by_labels_paged_cross_source() { // IDs from memtable + segments should merge and paginate correctly let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); @@ -885,7 +1239,7 @@ fn test_nodes_by_type_paged_cross_source() { // Create 5 nodes, flush to segment for i in 0..5 { engine - .upsert_node(1, &format!("seg{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("seg{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -893,7 +1247,7 @@ fn test_nodes_by_type_paged_cross_source() { // Create 5 more in memtable for i in 0..5 { engine - .upsert_node(1, &format!("mem{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("mem{}", i), UpsertNodeOptions::default()) .unwrap(); } @@ -902,8 +1256,7 @@ fn test_nodes_by_type_paged_cross_source() { let mut cursor: Option = None; loop { let page = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: cursor, @@ -925,25 +1278,24 @@ fn test_nodes_by_type_paged_cross_source() { } #[test] -fn test_nodes_by_type_paged_respects_tombstones() { +fn test_nodes_by_labels_paged_respects_tombstones() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let id2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let id3 = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.delete_node(id2).unwrap(); let result = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(10), after: None, @@ -957,17 +1309,17 @@ fn test_nodes_by_type_paged_respects_tombstones() { } #[test] -fn test_nodes_by_type_paged_respects_prune_policies() { +fn test_nodes_by_labels_paged_respects_prune_policies() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); engine - .upsert_node(1, "keep", UpsertNodeOptions::default()) + .upsert_node("Person", "keep", UpsertNodeOptions::default()) .unwrap(); engine .upsert_node( - 1, + "Person", "prune_me", UpsertNodeOptions { weight: 0.1, @@ -982,14 +1334,13 @@ fn test_nodes_by_type_paged_respects_prune_policies() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let result = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(10), after: None, @@ -1000,29 +1351,29 @@ fn test_nodes_by_type_paged_respects_prune_policies() { } #[test] -fn test_edges_by_type_paged_basic() { +fn test_edges_by_label_paged_basic() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let n1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let n3 = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let mut edge_ids: Vec = Vec::new(); for _ in 0..6 { let eid = engine - .upsert_edge(n1, n2, 5, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "OWNS", UpsertEdgeOptions::default()) .unwrap(); edge_ids.push(eid); let eid = engine - .upsert_edge(n2, n3, 5, UpsertEdgeOptions::default()) + .upsert_edge(n2, n3, "OWNS", UpsertEdgeOptions::default()) .unwrap(); edge_ids.push(eid); } @@ -1030,8 +1381,7 @@ fn test_edges_by_type_paged_basic() { // Page 2 at a time let page1 = engine - .edges_by_type_paged( - 5, + .edges_by_label_paged("OWNS", &PageRequest { limit: Some(2), after: None, @@ -1042,8 +1392,7 @@ fn test_edges_by_type_paged_basic() { assert!(page1.next_cursor.is_some()); let page2 = engine - .edges_by_type_paged( - 5, + .edges_by_label_paged("OWNS", &PageRequest { limit: Some(2), after: page1.next_cursor, @@ -1054,21 +1403,21 @@ fn test_edges_by_type_paged_basic() { } #[test] -fn test_edges_by_type_paged_roundtrip() { +fn test_edges_by_label_paged_roundtrip() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let n1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); for _ in 0..10 { engine - .upsert_edge(n1, n2, 3, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "LIKES", UpsertEdgeOptions::default()) .unwrap(); } @@ -1076,8 +1425,7 @@ fn test_edges_by_type_paged_roundtrip() { let mut cursor: Option = None; loop { let page = engine - .edges_by_type_paged( - 3, + .edges_by_label_paged("LIKES", &PageRequest { limit: Some(3), after: cursor, @@ -1091,13 +1439,13 @@ fn test_edges_by_type_paged_roundtrip() { } } - let mut all_unpaged = engine.edges_by_type(3).unwrap(); + let mut all_unpaged = engine.edges_by_label("LIKES").unwrap(); all_unpaged.sort(); assert_eq!(all_paged, all_unpaged); } #[test] -fn test_get_nodes_by_type_paged_hydrates_page_only() { +fn test_get_nodes_by_labels_paged_hydrates_page_only() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); @@ -1107,7 +1455,7 @@ fn test_get_nodes_by_type_paged_hydrates_page_only() { props.insert("idx".to_string(), PropValue::Int(i)); engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { props, @@ -1119,8 +1467,7 @@ fn test_get_nodes_by_type_paged_hydrates_page_only() { // Get first page of 3 hydrated records let page1 = engine - .get_nodes_by_type_paged( - 1, + .get_nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: None, @@ -1131,14 +1478,13 @@ fn test_get_nodes_by_type_paged_hydrates_page_only() { assert!(page1.next_cursor.is_some()); // Verify they're actual NodeRecords with properties for node in &page1.items { - assert_eq!(node.type_id, 1); + assert_eq!(node.labels.as_slice(), ["Person"]); assert!(node.props.contains_key("idx")); } // Get next page let page2 = engine - .get_nodes_by_type_paged( - 1, + .get_nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: page1.next_cursor, @@ -1155,27 +1501,61 @@ fn test_get_nodes_by_type_paged_hydrates_page_only() { } #[test] -fn test_get_edges_by_type_paged() { +fn test_get_nodes_by_labels_paged_multi_label_all() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let alice = engine + .upsert_node(&["Person", "Admin"], "alice", UpsertNodeOptions::default()) + .unwrap(); + let bob = engine + .upsert_node(&["Person", "Admin"], "bob", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_node("Person", "carol", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_node("Admin", "dave", UpsertNodeOptions::default()) + .unwrap(); + + let page = engine + .get_nodes_by_labels_paged( + &["Person", "Admin"], + &PageRequest { + limit: Some(2), + after: None, + }, + ) + .unwrap(); + let ids: Vec = page.items.iter().map(|node| node.id).collect(); + assert_eq!(ids, vec![alice, bob]); + assert_eq!(page.next_cursor, None); + assert_eq!(page.items[0].labels.as_slice(), ["Person", "Admin"]); + assert_eq!(page.items[1].labels.as_slice(), ["Person", "Admin"]); +} + +#[test] +fn test_get_edges_by_label_paged() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let n1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); for _ in 0..6 { engine - .upsert_edge(n1, n2, 7, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); } let page1 = engine - .get_edges_by_type_paged( - 7, + .get_edges_by_label_paged("FRIENDS_WITH", &PageRequest { limit: Some(2), after: None, @@ -1185,18 +1565,17 @@ fn test_get_edges_by_type_paged() { assert_eq!(page1.items.len(), 2); assert!(page1.next_cursor.is_some()); for edge in &page1.items { - assert_eq!(edge.type_id, 7); + assert_eq!(edge.label, "FRIENDS_WITH"); assert_eq!(edge.from, n1); assert_eq!(edge.to, n2); } // Round-trip - let mut all_paged: Vec = Vec::new(); + let mut all_paged: Vec = Vec::new(); let mut cursor: Option = None; loop { let page = engine - .get_edges_by_type_paged( - 7, + .get_edges_by_label_paged("FRIENDS_WITH", &PageRequest { limit: Some(2), after: cursor, @@ -1219,21 +1598,20 @@ fn test_paged_single_item_pages() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let id2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let id3 = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let mut expected = [id1, id2, id3]; expected.sort(); // Page 1-at-a-time let p1 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(1), after: None, @@ -1244,8 +1622,7 @@ fn test_paged_single_item_pages() { assert!(p1.next_cursor.is_some()); let p2 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(1), after: p1.next_cursor, @@ -1256,8 +1633,7 @@ fn test_paged_single_item_pages() { assert!(p2.next_cursor.is_some()); let p3 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(1), after: p2.next_cursor, @@ -1275,15 +1651,14 @@ fn test_paged_limit_larger_than_result_set() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let result = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(100), after: None, @@ -1303,13 +1678,12 @@ fn test_paged_limit_zero_returns_all() { for i in 0..5 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } let result = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(0), after: None, @@ -1328,20 +1702,19 @@ fn test_paged_cursor_on_deleted_id() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let id2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let id3 = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.delete_node(id2).unwrap(); // id2 is now a gap // Use deleted id2 as cursor. Should still work via binary search insertion point let result = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(10), after: Some(id2), @@ -1354,7 +1727,7 @@ fn test_paged_cursor_on_deleted_id() { assert_eq!(result.items, expected); } -// --- merge_type_ids_paged unit tests --- +// --- merge_record_ids_paged unit tests --- #[test] fn test_merge_paged_early_termination() { @@ -1369,7 +1742,7 @@ fn test_merge_paged_early_termination() { limit: Some(4), after: None, }; - let result = merge_type_ids_paged( + let result = merge_record_ids_paged( memtable.clone(), vec![seg1.clone(), seg2.clone()], &deleted, @@ -1383,7 +1756,7 @@ fn test_merge_paged_early_termination() { limit: Some(4), after: result.next_cursor, }; - let result2 = merge_type_ids_paged( + let result2 = merge_record_ids_paged( memtable.clone(), vec![seg1.clone(), seg2.clone()], &deleted, @@ -1397,7 +1770,7 @@ fn test_merge_paged_early_termination() { limit: Some(4), after: result2.next_cursor, }; - let result3 = merge_type_ids_paged(memtable, vec![seg1, seg2], &deleted, &page3); + let result3 = merge_record_ids_paged(memtable, vec![seg1, seg2], &deleted, &page3); assert_eq!(result3.items, vec![9, 10]); assert!(result3.next_cursor.is_none()); } @@ -1414,7 +1787,7 @@ fn test_merge_paged_cross_source_sorted_output() { limit: None, after: None, }; - let result = merge_type_ids_paged(memtable, vec![seg1, seg2], &deleted, &page); + let result = merge_record_ids_paged(memtable, vec![seg1, seg2], &deleted, &page); assert_eq!(result.items, vec![10, 15, 20, 30, 35, 40, 50, 55]); assert!(result.next_cursor.is_none()); @@ -1436,7 +1809,7 @@ fn test_merge_paged_dedup_across_sources() { limit: None, after: None, }; - let result = merge_type_ids_paged(memtable, vec![seg1, seg2], &deleted, &page); + let result = merge_record_ids_paged(memtable, vec![seg1, seg2], &deleted, &page); assert_eq!(result.items, vec![1, 2, 3, 4, 5]); } @@ -1452,7 +1825,7 @@ fn test_merge_paged_cursor_seek() { limit: Some(3), after: Some(5), }; - let result = merge_type_ids_paged(memtable, vec![seg1], &deleted, &page); + let result = merge_record_ids_paged(memtable, vec![seg1], &deleted, &page); assert_eq!(result.items, vec![6, 9, 10]); assert!(result.next_cursor.is_none()); } @@ -1468,14 +1841,14 @@ fn test_merge_paged_with_policies() { let mut keep_ids = Vec::new(); for i in 0..3 { let id = engine - .upsert_node(1, &format!("keep{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("keep{}", i), UpsertNodeOptions::default()) .unwrap(); keep_ids.push(id); } for i in 0..3 { engine .upsert_node( - 1, + "Person", &format!("prune{}", i), UpsertNodeOptions { weight: 0.1, @@ -1491,15 +1864,14 @@ fn test_merge_paged_with_policies() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); // Page through with limit=2, should only see the 3 high-weight nodes let p1 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(2), after: None, @@ -1510,8 +1882,7 @@ fn test_merge_paged_with_policies() { assert!(p1.next_cursor.is_some()); let p2 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(2), after: p1.next_cursor, @@ -1542,7 +1913,7 @@ fn test_find_nodes_memtable_only() { props.insert("color".to_string(), PropValue::String("red".to_string())); let a = engine .upsert_node( - 1, + "Person", "apple", UpsertNodeOptions { props: props.clone(), @@ -1556,7 +1927,7 @@ fn test_find_nodes_memtable_only() { props2.insert("color".to_string(), PropValue::String("red".to_string())); let b = engine .upsert_node( - 1, + "Person", "cherry", UpsertNodeOptions { props: props2, @@ -1570,7 +1941,7 @@ fn test_find_nodes_memtable_only() { props3.insert("color".to_string(), PropValue::String("green".to_string())); engine .upsert_node( - 1, + "Person", "lime", UpsertNodeOptions { props: props3, @@ -1581,22 +1952,22 @@ fn test_find_nodes_memtable_only() { .unwrap(); let mut reds = engine - .find_nodes(1, "color", &PropValue::String("red".to_string())) + .find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap(); reds.sort(); assert_eq!(reds, vec![a, b]); let greens = engine - .find_nodes(1, "color", &PropValue::String("green".to_string())) + .find_nodes("Person", "color", &PropValue::String("green".to_string())) .unwrap(); assert_eq!(greens.len(), 1); assert!(engine - .find_nodes(1, "color", &PropValue::String("blue".to_string())) + .find_nodes("Person", "color", &PropValue::String("blue".to_string())) .unwrap() .is_empty()); assert!(engine - .find_nodes(2, "color", &PropValue::String("red".to_string())) + .find_nodes("Company", "color", &PropValue::String("red".to_string())) .unwrap() .is_empty()); @@ -1615,7 +1986,7 @@ fn test_find_nodes_cross_source() { props.insert("color".to_string(), PropValue::String("red".to_string())); let a = engine .upsert_node( - 1, + "Person", "apple", UpsertNodeOptions { props, @@ -1631,7 +2002,7 @@ fn test_find_nodes_cross_source() { props2.insert("color".to_string(), PropValue::String("red".to_string())); let b = engine .upsert_node( - 1, + "Person", "cherry", UpsertNodeOptions { props: props2, @@ -1643,7 +2014,7 @@ fn test_find_nodes_cross_source() { // find_nodes should merge across memtable + segment let mut reds = engine - .find_nodes(1, "color", &PropValue::String("red".to_string())) + .find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap(); reds.sort(); assert_eq!(reds, vec![a, b]); @@ -1662,7 +2033,7 @@ fn test_find_nodes_excludes_deleted() { props.insert("color".to_string(), PropValue::String("red".to_string())); let a = engine .upsert_node( - 1, + "Person", "apple", UpsertNodeOptions { props: props.clone(), @@ -1673,7 +2044,7 @@ fn test_find_nodes_excludes_deleted() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "cherry", UpsertNodeOptions { props, @@ -1686,7 +2057,7 @@ fn test_find_nodes_excludes_deleted() { engine.delete_node(b).unwrap(); let reds = engine - .find_nodes(1, "color", &PropValue::String("red".to_string())) + .find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap(); assert_eq!(reds, vec![a]); @@ -1706,7 +2077,7 @@ fn test_find_nodes_survives_flush_and_reopen() { props.insert("lang".to_string(), PropValue::String("rust".to_string())); a = engine .upsert_node( - 1, + "Person", "overgraph", UpsertNodeOptions { props, @@ -1720,7 +2091,7 @@ fn test_find_nodes_survives_flush_and_reopen() { props2.insert("lang".to_string(), PropValue::String("python".to_string())); engine .upsert_node( - 1, + "Person", "other", UpsertNodeOptions { props: props2, @@ -1738,12 +2109,12 @@ fn test_find_nodes_survives_flush_and_reopen() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let results = engine - .find_nodes(1, "lang", &PropValue::String("rust".to_string())) + .find_nodes("Person", "lang", &PropValue::String("rust".to_string())) .unwrap(); assert_eq!(results, vec![a]); let py = engine - .find_nodes(1, "lang", &PropValue::String("python".to_string())) + .find_nodes("Person", "lang", &PropValue::String("python".to_string())) .unwrap(); assert_eq!(py.len(), 1); @@ -1765,7 +2136,7 @@ fn test_find_nodes_update_changes_index() { ); let a = engine .upsert_node( - 1, + "Person", "item", UpsertNodeOptions { props, @@ -1777,7 +2148,7 @@ fn test_find_nodes_update_changes_index() { assert_eq!( engine - .find_nodes(1, "status", &PropValue::String("active".to_string())) + .find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap(), vec![a] ); @@ -1790,7 +2161,7 @@ fn test_find_nodes_update_changes_index() { ); let a2 = engine .upsert_node( - 1, + "Person", "item", UpsertNodeOptions { props: props2, @@ -1802,12 +2173,12 @@ fn test_find_nodes_update_changes_index() { assert_eq!(a, a2); // same ID (dedup) assert!(engine - .find_nodes(1, "status", &PropValue::String("active".to_string())) + .find_nodes("Person", "status", &PropValue::String("active".to_string())) .unwrap() .is_empty()); assert_eq!( engine - .find_nodes(1, "status", &PropValue::String("inactive".to_string())) + .find_nodes("Person", "status", &PropValue::String("inactive".to_string())) .unwrap(), vec![a] ); @@ -1828,7 +2199,7 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { red_props.insert("color".to_string(), red.clone()); let a = engine .upsert_node( - 1, + "Person", "seg_update", UpsertNodeOptions { props: red_props.clone(), @@ -1838,7 +2209,7 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "seg_keep", UpsertNodeOptions { props: red_props.clone(), @@ -1853,7 +2224,7 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { assert_eq!( engine .upsert_node( - 1, + "Person", "seg_update", UpsertNodeOptions { props: blue_props, @@ -1865,7 +2236,7 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { ); let c = engine .upsert_node( - 1, + "Person", "imm_delete", UpsertNodeOptions { props: red_props.clone(), @@ -1877,7 +2248,7 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { let d = engine .upsert_node( - 1, + "Person", "active_keep", UpsertNodeOptions { props: red_props.clone(), @@ -1887,7 +2258,7 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { .unwrap(); engine .upsert_node( - 1, + "Person", "active_pruned", UpsertNodeOptions { props: red_props, @@ -1903,13 +2274,13 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); engine.reset_property_query_routes(); - let mut results = engine.find_nodes(1, "color", &red).unwrap(); + let mut results = engine.find_nodes("Person", "color", &red).unwrap(); results.sort_unstable(); assert_eq!(results, vec![b, d]); let routes = engine.property_query_route_snapshot(); @@ -1917,8 +2288,7 @@ fn test_find_nodes_fallback_routes_and_filters_latest_visible_records() { assert_eq!(routes.equality_index_lookup, 0); let page = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -1947,7 +2317,7 @@ fn test_find_nodes_building_declaration_still_uses_fallback() { props.insert("color".to_string(), red.clone()); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: props.clone(), @@ -1957,7 +2327,7 @@ fn test_find_nodes_building_declaration_still_uses_fallback() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props, @@ -1969,7 +2339,7 @@ fn test_find_nodes_building_declaration_still_uses_fallback() { let entry = SecondaryIndexManifestEntry { index_id: 1, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -1994,7 +2364,7 @@ fn test_find_nodes_building_declaration_still_uses_fallback() { assert_eq!(info.state, SecondaryIndexState::Building); engine.reset_property_query_routes(); - let mut results = engine.find_nodes(1, "color", &red).unwrap(); + let mut results = engine.find_nodes("Person", "color", &red).unwrap(); results.sort_unstable(); assert_eq!(results, vec![a, b]); @@ -2017,7 +2387,7 @@ fn test_find_nodes_ready_declaration_uses_index_lookup_across_sources() { seg_props.insert("color".to_string(), red.clone()); let seg_id = engine .upsert_node( - 1, + "Person", "seg", UpsertNodeOptions { props: seg_props, @@ -2031,7 +2401,7 @@ fn test_find_nodes_ready_declaration_uses_index_lookup_across_sources() { imm_props.insert("color".to_string(), red.clone()); let imm_id = engine .upsert_node( - 1, + "Person", "imm", UpsertNodeOptions { props: imm_props, @@ -2045,7 +2415,7 @@ fn test_find_nodes_ready_declaration_uses_index_lookup_across_sources() { active_props.insert("color".to_string(), red.clone()); let active_id = engine .upsert_node( - 1, + "Person", "active", UpsertNodeOptions { props: active_props, @@ -2055,27 +2425,26 @@ fn test_find_nodes_ready_declaration_uses_index_lookup_across_sources() { .unwrap(); let info = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); let ready = wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); assert_eq!(ready.index_id, info.index_id); engine.reset_property_query_routes(); - let results = engine.find_nodes(1, "color", &red).unwrap(); + let results = engine.find_nodes("Person", "color", &red).unwrap(); assert_eq!(results, vec![seg_id, imm_id, active_id]); let routes = engine.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 0); assert_eq!(routes.equality_index_lookup, 1); let all_page = engine - .find_nodes_paged(1, "color", &red, &PageRequest::default()) + .find_nodes_paged("Person", "color", &red, &PageRequest::default()) .unwrap(); assert_eq!(all_page.items, results); assert!(all_page.next_cursor.is_none()); let first_page = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2088,8 +2457,7 @@ fn test_find_nodes_ready_declaration_uses_index_lookup_across_sources() { assert_eq!(first_page.next_cursor, Some(imm_id)); let second_page = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2118,7 +2486,7 @@ fn test_find_nodes_ready_equality_index_matches_signed_zero_verifier_semantics() neg_zero_props.insert("temp".to_string(), PropValue::Float(-0.0)); let neg_zero = engine .upsert_node( - 1, + "Person", "temp-neg-zero", UpsertNodeOptions { props: neg_zero_props, @@ -2131,7 +2499,7 @@ fn test_find_nodes_ready_equality_index_matches_signed_zero_verifier_semantics() pos_zero_props.insert("temp".to_string(), PropValue::Float(0.0)); let pos_zero = engine .upsert_node( - 1, + "Person", "temp-pos-zero", UpsertNodeOptions { props: pos_zero_props, @@ -2144,7 +2512,7 @@ fn test_find_nodes_ready_equality_index_matches_signed_zero_verifier_semantics() non_zero_props.insert("temp".to_string(), PropValue::Float(1.0)); engine .upsert_node( - 1, + "Person", "temp-one", UpsertNodeOptions { props: non_zero_props, @@ -2155,7 +2523,7 @@ fn test_find_nodes_ready_equality_index_matches_signed_zero_verifier_semantics() engine.flush().unwrap(); let info = engine - .ensure_node_property_index(1, "temp", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "temp", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); wait_for_published_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); @@ -2163,7 +2531,7 @@ fn test_find_nodes_ready_equality_index_matches_signed_zero_verifier_semantics() engine.reset_property_query_routes(); assert_eq!( engine - .find_nodes(1, "temp", &PropValue::Float(-0.0)) + .find_nodes("Person", "temp", &PropValue::Float(-0.0)) .unwrap(), vec![neg_zero, pos_zero] ); @@ -2174,7 +2542,7 @@ fn test_find_nodes_ready_equality_index_matches_signed_zero_verifier_semantics() engine.reset_property_query_routes(); assert_eq!( engine - .find_nodes(1, "temp", &PropValue::Float(0.0)) + .find_nodes("Person", "temp", &PropValue::Float(0.0)) .unwrap(), vec![neg_zero, pos_zero] ); @@ -2198,7 +2566,7 @@ fn test_find_nodes_ready_declaration_suppresses_stale_and_collision_candidates() red_props.insert("color".to_string(), red.clone()); let node_id = engine .upsert_node( - 1, + "Person", "mutable", UpsertNodeOptions { props: red_props, @@ -2211,7 +2579,7 @@ fn test_find_nodes_ready_declaration_suppresses_stale_and_collision_candidates() blue_props.insert("color".to_string(), blue.clone()); let blue_id = engine .upsert_node( - 1, + "Person", "blue-only", UpsertNodeOptions { props: blue_props, @@ -2223,7 +2591,7 @@ fn test_find_nodes_ready_declaration_suppresses_stale_and_collision_candidates() engine.flush().unwrap(); let info = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); let ready = wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); assert_eq!(ready.index_id, info.index_id); @@ -2235,8 +2603,17 @@ fn test_find_nodes_ready_declaration_suppresses_stale_and_collision_candidates() let mut tampered_groups = std::collections::BTreeMap::new(); tampered_groups.insert(hash_prop_value(&red), vec![node_id, blue_id]); - crate::segment_writer::write_node_prop_eq_sidecar_to_path( - &crate::segment_writer::node_prop_eq_sidecar_path(&seg_dir, info.index_id), + let manifest = crate::manifest::load_manifest_readonly(&db_path) + .unwrap() + .unwrap(); + let entry = manifest + .secondary_indexes + .iter() + .find(|entry| entry.index_id == info.index_id) + .unwrap(); + crate::segment_writer::publish_node_prop_eq_sidecar_component( + &seg_dir, + entry, &tampered_groups, ) .unwrap(); @@ -2247,7 +2624,7 @@ fn test_find_nodes_ready_declaration_suppresses_stale_and_collision_candidates() assert_eq!( engine .upsert_node( - 1, + "Person", "mutable", UpsertNodeOptions { props: updated_props, @@ -2259,8 +2636,8 @@ fn test_find_nodes_ready_declaration_suppresses_stale_and_collision_candidates() ); engine.reset_property_query_routes(); - assert!(engine.find_nodes(1, "color", &red).unwrap().is_empty()); - assert_eq!(engine.find_nodes(1, "color", &blue).unwrap(), vec![node_id]); + assert!(engine.find_nodes("Person", "color", &red).unwrap().is_empty()); + assert_eq!(engine.find_nodes("Person", "color", &blue).unwrap(), vec![node_id]); let routes = engine.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 0); @@ -2281,7 +2658,7 @@ fn test_find_nodes_ready_declaration_respects_prune_policies() { let keep_id = engine .upsert_node( - 1, + "Person", "keep", UpsertNodeOptions { props: red_props.clone(), @@ -2293,7 +2670,7 @@ fn test_find_nodes_ready_declaration_respects_prune_policies() { let pruned_id = engine .upsert_node( - 1, + "Person", "pruned", UpsertNodeOptions { props: red_props, @@ -2309,28 +2686,27 @@ fn test_find_nodes_ready_declaration_respects_prune_policies() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let info = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); engine.reset_property_query_routes(); - assert_eq!(engine.find_nodes(1, "color", &red).unwrap(), vec![keep_id]); + assert_eq!(engine.find_nodes("Person", "color", &red).unwrap(), vec![keep_id]); let all_page = engine - .find_nodes_paged(1, "color", &red, &PageRequest::default()) + .find_nodes_paged("Person", "color", &red, &PageRequest::default()) .unwrap(); assert_eq!(all_page.items, vec![keep_id]); assert!(all_page.next_cursor.is_none()); let first_page = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2342,14 +2718,99 @@ fn test_find_nodes_ready_declaration_respects_prune_policies() { assert_eq!(first_page.items, vec![keep_id]); assert!(first_page.next_cursor.is_none()); + let exact_limit_page = engine + .find_nodes_paged("Person", + "color", + &red, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(exact_limit_page.items, vec![keep_id]); + assert!( + exact_limit_page.next_cursor.is_none(), + "ready equality pagination must not report a next page unless another verified node exists" + ); + let routes = engine.property_query_route_snapshot(); assert_eq!(routes.equality_scan_fallback, 0); - assert_eq!(routes.equality_index_lookup, 3); + assert_eq!(routes.equality_index_lookup, 4); assert_ne!(keep_id, pruned_id); engine.close().unwrap(); } +#[test] +fn test_find_nodes_ready_eq_cursor_no_false_next_after_stale_candidates() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let red = PropValue::String("red".to_string()); + let mut red_props = BTreeMap::new(); + red_props.insert("color".to_string(), red.clone()); + + let info = engine + .ensure_node_property_index("Employee", "color", SecondaryIndexKind::Equality) + .unwrap(); + wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + let keep_id = engine + .upsert_node( + &["Employee", "Current"], + "keep", + UpsertNodeOptions { + props: red_props.clone(), + ..Default::default() + }, + ) + .unwrap(); + let stale_ids = (0..5) + .map(|idx| { + engine + .upsert_node( + &["Employee", "Former"], + &format!("stale-{idx}"), + UpsertNodeOptions { + props: red_props.clone(), + ..Default::default() + }, + ) + .unwrap() + }) + .collect::>(); + engine.flush().unwrap(); + + for stale_id in stale_ids { + assert!(engine.remove_node_label(stale_id, "Employee").unwrap()); + } + + engine.reset_property_query_routes(); + let page = engine + .find_nodes_paged( + "Employee", + "color", + &red, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(page.items, vec![keep_id]); + assert!( + page.next_cursor.is_none(), + "ready equality pagination must not report a next page for stale label candidates" + ); + let routes = engine.property_query_route_snapshot(); + assert_eq!(routes.equality_scan_fallback, 0); + assert_eq!(routes.equality_index_lookup, 1); + + engine.close().unwrap(); +} + #[test] fn test_find_nodes_ready_declaration_keeps_revived_same_id_after_tombstone() { let dir = TempDir::new().unwrap(); @@ -2362,7 +2823,7 @@ fn test_find_nodes_ready_declaration_keeps_revived_same_id_after_tombstone() { let make_node = |updated_at: i64, key: &str| NodeRecord { id: 7, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: key.to_string(), props: red_props.clone(), created_at: updated_at, @@ -2373,34 +2834,30 @@ fn test_find_nodes_ready_declaration_keeps_revived_same_id_after_tombstone() { last_write_seq: 0, }; - engine - .write_op(&WalOp::UpsertNode(make_node(1_000, "older-segment"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(1_000, "older-segment"))) .unwrap(); engine.flush().unwrap(); - engine - .write_op(&WalOp::DeleteNode { + write_internal_wal_op(&engine, &WalOp::DeleteNode { id: 7, deleted_at: 2_000, }) .unwrap(); engine.flush().unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(3_000, "revived-active"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(3_000, "revived-active"))) .unwrap(); let info = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); engine.reset_property_query_routes(); - assert_eq!(engine.find_nodes(1, "color", &red).unwrap(), vec![7]); + assert_eq!(engine.find_nodes("Person", "color", &red).unwrap(), vec![7]); let page = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2420,7 +2877,7 @@ fn test_find_nodes_ready_declaration_keeps_revived_same_id_after_tombstone() { } #[test] -fn test_find_nodes_ready_declaration_filters_same_id_type_change() { +fn test_find_nodes_ready_declaration_filters_same_id_label_change() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); @@ -2429,9 +2886,9 @@ fn test_find_nodes_ready_declaration_filters_same_id_type_change() { let mut red_props = BTreeMap::new(); red_props.insert("color".to_string(), red.clone()); - let make_node = |type_id: u32, updated_at: i64, key: &str| NodeRecord { + let make_node = |label_id: u32, updated_at: i64, key: &str| NodeRecord { id: 9, - type_id, + label_ids: NodeLabelSet::single(label_id).unwrap(), key: key.to_string(), props: red_props.clone(), created_at: updated_at, @@ -2442,26 +2899,23 @@ fn test_find_nodes_ready_declaration_filters_same_id_type_change() { last_write_seq: 0, }; - engine - .write_op(&WalOp::UpsertNode(make_node(1, 1_000, "type1-segment"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(1, 1_000, "label1-segment"))) .unwrap(); engine.flush().unwrap(); - engine - .write_op(&WalOp::UpsertNode(make_node(2, 2_000, "type2-active"))) + write_internal_wal_op(&engine, &WalOp::UpsertNode(make_node(2, 2_000, "label2-active"))) .unwrap(); let info = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); engine.reset_property_query_routes(); - assert!(engine.find_nodes(1, "color", &red).unwrap().is_empty()); + assert!(engine.find_nodes("Person", "color", &red).unwrap().is_empty()); let page = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2494,7 +2948,7 @@ fn test_find_nodes_paged_basic() { props.insert("color".to_string(), PropValue::String("red".to_string())); let id = engine .upsert_node( - 1, + "Person", &format!("r{}", i), UpsertNodeOptions { props, @@ -2510,7 +2964,7 @@ fn test_find_nodes_paged_basic() { props.insert("color".to_string(), PropValue::String("blue".to_string())); engine .upsert_node( - 1, + "Person", &format!("b{}", i), UpsertNodeOptions { props, @@ -2525,8 +2979,7 @@ fn test_find_nodes_paged_basic() { // Page through 3 at a time let p1 = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2540,8 +2993,7 @@ fn test_find_nodes_paged_basic() { assert!(p1.next_cursor.is_some()); let p2 = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2555,8 +3007,7 @@ fn test_find_nodes_paged_basic() { assert!(p2.next_cursor.is_some()); let p3 = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2584,7 +3035,7 @@ fn test_find_nodes_paged_cross_source() { for i in 0..4 { engine .upsert_node( - 1, + "Person", &format!("seg{}", i), UpsertNodeOptions { props: props.clone(), @@ -2599,7 +3050,7 @@ fn test_find_nodes_paged_cross_source() { for i in 0..4 { engine .upsert_node( - 1, + "Person", &format!("mem{}", i), UpsertNodeOptions { props: props.clone(), @@ -2614,8 +3065,7 @@ fn test_find_nodes_paged_cross_source() { let mut cursor: Option = None; loop { let page = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2649,7 +3099,7 @@ fn test_find_nodes_paged_excludes_deleted() { let id1 = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: props.clone(), @@ -2659,7 +3109,7 @@ fn test_find_nodes_paged_excludes_deleted() { .unwrap(); let id2 = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props: props.clone(), @@ -2669,7 +3119,7 @@ fn test_find_nodes_paged_excludes_deleted() { .unwrap(); let id3 = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { props: props.clone(), @@ -2680,8 +3130,7 @@ fn test_find_nodes_paged_excludes_deleted() { engine.delete_node(id2).unwrap(); let result = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2708,7 +3157,7 @@ fn test_find_nodes_paged_with_policies() { engine .upsert_node( - 1, + "Person", "keep", UpsertNodeOptions { props: props.clone(), @@ -2718,7 +3167,7 @@ fn test_find_nodes_paged_with_policies() { .unwrap(); engine .upsert_node( - 1, + "Person", "prune", UpsertNodeOptions { props: props.clone(), @@ -2734,14 +3183,13 @@ fn test_find_nodes_paged_with_policies() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let result = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -2773,7 +3221,7 @@ fn test_find_nodes_range_fallback_orders_and_paginates() { props.insert("score".to_string(), PropValue::Int(score)); let id = engine .upsert_node( - 1, + "Person", key, UpsertNodeOptions { props, @@ -2787,7 +3235,7 @@ fn test_find_nodes_range_fallback_orders_and_paginates() { bad_props.insert("score".to_string(), PropValue::String("bad".to_string())); engine .upsert_node( - 1, + "Person", "bad", UpsertNodeOptions { props: bad_props, @@ -2797,8 +3245,7 @@ fn test_find_nodes_range_fallback_orders_and_paginates() { .unwrap(); let info = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -2809,8 +3256,7 @@ fn test_find_nodes_range_fallback_orders_and_paginates() { engine.reset_property_query_routes(); let lower_only = engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), None, @@ -2822,8 +3268,7 @@ fn test_find_nodes_range_fallback_orders_and_paginates() { ); let upper_only = engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", None, Some(&PropertyRangeBound::Included(PropValue::Int(20))), @@ -2835,8 +3280,7 @@ fn test_find_nodes_range_fallback_orders_and_paginates() { ); let page1 = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -2856,8 +3300,7 @@ fn test_find_nodes_range_fallback_orders_and_paginates() { ); let page2 = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -2888,7 +3331,7 @@ fn test_find_nodes_range_rejects_mixed_bound_variants_and_normalizes_zero() { props.insert("score".to_string(), PropValue::Float(value)); engine .upsert_node( - 1, + "Person", key, UpsertNodeOptions { props, @@ -2899,8 +3342,7 @@ fn test_find_nodes_range_rejects_mixed_bound_variants_and_normalizes_zero() { } let zeros = engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Float(-0.0))), Some(&PropertyRangeBound::Included(PropValue::Float(0.0))), @@ -2910,8 +3352,7 @@ fn test_find_nodes_range_rejects_mixed_bound_variants_and_normalizes_zero() { assert!(zeros[0] < zeros[1]); let err = engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(1))), Some(&PropertyRangeBound::Included(PropValue::Float(1.0))), @@ -2920,8 +3361,7 @@ fn test_find_nodes_range_rejects_mixed_bound_variants_and_normalizes_zero() { assert!(matches!(err, EngineError::InvalidOperation(_))); let err = engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Float(2.0))), Some(&PropertyRangeBound::Included(PropValue::Float(1.0))), @@ -2930,8 +3370,7 @@ fn test_find_nodes_range_rejects_mixed_bound_variants_and_normalizes_zero() { assert!(matches!(err, EngineError::InvalidOperation(_))); let err = engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Excluded(PropValue::Float(0.0))), Some(&PropertyRangeBound::Included(PropValue::Float(0.0))), @@ -2952,7 +3391,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() props_a.insert("score".to_string(), PropValue::Int(20)); let a = engine .upsert_node( - 1, + "Person", "seg_update", UpsertNodeOptions { props: props_a, @@ -2964,7 +3403,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() props_b.insert("score".to_string(), PropValue::Int(25)); let b = engine .upsert_node( - 1, + "Person", "seg_keep", UpsertNodeOptions { props: props_b, @@ -2979,7 +3418,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() assert_eq!( engine .upsert_node( - 1, + "Person", "seg_update", UpsertNodeOptions { props: props_a_new, @@ -2993,7 +3432,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() props_c.insert("score".to_string(), PropValue::Int(22)); let c = engine .upsert_node( - 1, + "Person", "imm_delete", UpsertNodeOptions { props: props_c, @@ -3007,7 +3446,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() props_d.insert("score".to_string(), PropValue::Int(21)); let d = engine .upsert_node( - 1, + "Person", "active_keep", UpsertNodeOptions { props: props_d, @@ -3019,7 +3458,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() props_pruned.insert("score".to_string(), PropValue::Int(23)); engine .upsert_node( - 1, + "Person", "active_pruned", UpsertNodeOptions { props: props_pruned, @@ -3035,15 +3474,14 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); engine.reset_property_query_routes(); let page1 = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -3056,8 +3494,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() assert_eq!(page1.items, vec![d]); let page2 = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -3080,7 +3517,7 @@ fn test_find_nodes_range_fallback_mixed_sources_filters_latest_visible_records() fn brute_force_range_oracle( engine: &DatabaseEngine, node_ids: &[u64], - type_id: u32, + label_id: u32, prop_key: &str, lower: Option<&PropertyRangeBound>, upper: Option<&PropertyRangeBound>, @@ -3092,7 +3529,7 @@ fn brute_force_range_oracle( let Some(node) = node.as_ref() else { continue; }; - if node.type_id != type_id { + if !node.label_ids.contains(label_id) { continue; } let Some(value) = node.props.get(prop_key) else { @@ -3120,7 +3557,7 @@ fn test_find_nodes_range_open_and_closed_intervals_match_in_fallback_and_ready_p seg_low_props.insert("score".to_string(), PropValue::Int(10)); let seg_low = engine .upsert_node( - 1, + "Person", "seg-low", UpsertNodeOptions { props: seg_low_props, @@ -3132,7 +3569,7 @@ fn test_find_nodes_range_open_and_closed_intervals_match_in_fallback_and_ready_p seg_high_props.insert("score".to_string(), PropValue::Int(30)); let seg_high = engine .upsert_node( - 1, + "Person", "seg-high", UpsertNodeOptions { props: seg_high_props, @@ -3146,7 +3583,7 @@ fn test_find_nodes_range_open_and_closed_intervals_match_in_fallback_and_ready_p imm_props.insert("score".to_string(), PropValue::Int(20)); let imm_20 = engine .upsert_node( - 1, + "Person", "imm-20", UpsertNodeOptions { props: imm_props, @@ -3160,7 +3597,7 @@ fn test_find_nodes_range_open_and_closed_intervals_match_in_fallback_and_ready_p active_props.insert("score".to_string(), PropValue::Int(20)); let active_20 = engine .upsert_node( - 1, + "Person", "active-20", UpsertNodeOptions { props: active_props, @@ -3206,7 +3643,7 @@ fn test_find_nodes_range_open_and_closed_intervals_match_in_fallback_and_ready_p for (lower, upper, expected) in &cases { assert_eq!( engine - .find_nodes_range(1, "score", lower.as_ref(), upper.as_ref()) + .find_nodes_range("Person", "score", lower.as_ref(), upper.as_ref()) .unwrap(), *expected ); @@ -3216,8 +3653,7 @@ fn test_find_nodes_range_open_and_closed_intervals_match_in_fallback_and_ready_p assert_eq!(routes.range_index_lookup, 0); let info = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -3230,7 +3666,7 @@ fn test_find_nodes_range_open_and_closed_intervals_match_in_fallback_and_ready_p for (lower, upper, expected) in &cases { assert_eq!( engine - .find_nodes_range(1, "score", lower.as_ref(), upper.as_ref()) + .find_nodes_range("Person", "score", lower.as_ref(), upper.as_ref()) .unwrap(), *expected ); @@ -3255,7 +3691,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "int-a", UpsertNodeOptions { props: int_a_props, @@ -3268,7 +3704,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() int_stale_props.insert("score_i".to_string(), PropValue::Int(15)); let int_stale = engine .upsert_node( - 1, + "Person", "int-stale", UpsertNodeOptions { props: int_stale_props, @@ -3281,7 +3717,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() uint_a_props.insert("score_u".to_string(), PropValue::UInt(1)); let uint_a = engine .upsert_node( - 1, + "Person", "uint-a", UpsertNodeOptions { props: uint_a_props, @@ -3295,7 +3731,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "float-a", UpsertNodeOptions { props: float_a_props, @@ -3308,7 +3744,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() float_stale_props.insert("score_f".to_string(), PropValue::Float(2.0)); let float_stale = engine .upsert_node( - 1, + "Person", "float-stale", UpsertNodeOptions { props: float_stale_props, @@ -3324,7 +3760,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "int-c", UpsertNodeOptions { props: int_c_props, @@ -3338,7 +3774,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "uint-b", UpsertNodeOptions { props: uint_b_props, @@ -3352,7 +3788,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "float-c", UpsertNodeOptions { props: float_c_props, @@ -3368,7 +3804,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() assert_eq!( engine .upsert_node( - 1, + "Person", "int-stale", UpsertNodeOptions { props: int_stale_new_props, @@ -3383,7 +3819,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "int-d", UpsertNodeOptions { props: int_d_props, @@ -3398,7 +3834,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "uint-c", UpsertNodeOptions { props: uint_c_props, @@ -3412,7 +3848,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() assert_eq!( engine .upsert_node( - 1, + "Person", "float-stale", UpsertNodeOptions { props: float_stale_new_props, @@ -3427,7 +3863,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() all_ids.push( engine .upsert_node( - 1, + "Person", "float-d", UpsertNodeOptions { props: float_d_props, @@ -3477,7 +3913,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() ); assert_eq!( engine - .find_nodes_range(1, prop_key, lower.as_ref(), upper.as_ref()) + .find_nodes_range("Person", prop_key, lower.as_ref(), upper.as_ref()) .unwrap(), oracle ); @@ -3488,7 +3924,7 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() assert_eq!(routes.range_index_lookup, 0); for (prop_key, kind, _, _) in &queries { - let info = engine.ensure_node_property_index(1, prop_key, kind.clone()).unwrap(); + let info = engine.ensure_node_property_index("Person", prop_key, kind.clone()).unwrap(); wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); } @@ -3496,14 +3932,13 @@ fn test_find_nodes_range_ready_parity_matches_bruteforce_oracle_across_domains() for ((prop_key, _, lower, upper), oracle) in queries.iter().zip(oracles.iter()) { assert_eq!( engine - .find_nodes_range(1, prop_key, lower.as_ref(), upper.as_ref()) + .find_nodes_range("Person", prop_key, lower.as_ref(), upper.as_ref()) .unwrap(), *oracle ); assert_eq!( engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", prop_key, lower.as_ref(), upper.as_ref(), @@ -3532,7 +3967,7 @@ fn test_find_nodes_range_ready_refills_segment_chunks_with_pruned_overrides() { props.insert("score".to_string(), PropValue::Int(score)); engine .upsert_node( - 1, + "Person", &format!("seg-{score:02}"), UpsertNodeOptions { props, @@ -3548,7 +3983,7 @@ fn test_find_nodes_range_ready_refills_segment_chunks_with_pruned_overrides() { props.insert("score".to_string(), PropValue::Int(score)); engine .upsert_node( - 1, + "Person", &format!("seg-{score:02}"), UpsertNodeOptions { props, @@ -3564,14 +3999,13 @@ fn test_find_nodes_range_ready_refills_segment_chunks_with_pruned_overrides() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let info = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -3581,16 +4015,15 @@ fn test_find_nodes_range_ready_refills_segment_chunks_with_pruned_overrides() { wait_for_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); let expected_page1: Vec = (61..=70) - .map(|score| engine.get_node_by_key(1, &format!("seg-{score:02}")).unwrap().unwrap().id) + .map(|score| engine.get_node_by_key("Person", &format!("seg-{score:02}")).unwrap().unwrap().id) .collect(); let expected_page2: Vec = (71..=80) - .map(|score| engine.get_node_by_key(1, &format!("seg-{score:02}")).unwrap().unwrap().id) + .map(|score| engine.get_node_by_key("Person", &format!("seg-{score:02}")).unwrap().unwrap().id) .collect(); engine.reset_property_query_routes(); let page1 = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(1))), Some(&PropertyRangeBound::Included(PropValue::Int(80))), @@ -3610,8 +4043,7 @@ fn test_find_nodes_range_ready_refills_segment_chunks_with_pruned_overrides() { ); let page2 = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(1))), Some(&PropertyRangeBound::Included(PropValue::Int(80))), @@ -3641,7 +4073,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { seg_props.insert("score".to_string(), PropValue::Int(30)); let seg_id = engine .upsert_node( - 1, + "Person", "seg-30", UpsertNodeOptions { props: seg_props, @@ -3655,7 +4087,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { imm_props_a.insert("score".to_string(), PropValue::Int(20)); let imm_a = engine .upsert_node( - 1, + "Person", "imm-20-a", UpsertNodeOptions { props: imm_props_a, @@ -3667,7 +4099,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { imm_props_b.insert("score".to_string(), PropValue::Int(20)); let imm_b = engine .upsert_node( - 1, + "Person", "imm-20-b", UpsertNodeOptions { props: imm_props_b, @@ -3681,7 +4113,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { active_props.insert("score".to_string(), PropValue::Int(25)); let active_25 = engine .upsert_node( - 1, + "Person", "active-25", UpsertNodeOptions { props: active_props, @@ -3693,7 +4125,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { active_props_20.insert("score".to_string(), PropValue::Int(20)); let active_20 = engine .upsert_node( - 1, + "Person", "active-20", UpsertNodeOptions { props: active_props_20, @@ -3703,8 +4135,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { .unwrap(); let info = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -3722,8 +4153,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { engine.reset_property_query_routes(); assert_eq!( engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -3733,8 +4163,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { ); let paged_all = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -3745,8 +4174,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { assert!(paged_all.next_cursor.is_none()); let first_page = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -3766,8 +4194,7 @@ fn test_find_nodes_range_ready_declaration_routes_and_orders_across_sources() { ); let second_page = engine - .find_nodes_range_paged( - 1, + .find_nodes_range_paged("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(20))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -3797,7 +4224,7 @@ fn test_find_nodes_range_ready_declaration_hides_stale_and_incompatible_older_ma props.insert("score".to_string(), PropValue::Int(20)); let mutable_id = engine .upsert_node( - 1, + "Person", "mutable", UpsertNodeOptions { props, @@ -3808,8 +4235,7 @@ fn test_find_nodes_range_ready_declaration_hides_stale_and_incompatible_older_ma engine.flush().unwrap(); let info = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, @@ -3823,7 +4249,7 @@ fn test_find_nodes_range_ready_declaration_hides_stale_and_incompatible_older_ma assert_eq!( engine .upsert_node( - 1, + "Person", "mutable", UpsertNodeOptions { props: incompatible_props, @@ -3837,7 +4263,7 @@ fn test_find_nodes_range_ready_declaration_hides_stale_and_incompatible_older_ma keep_props.insert("score".to_string(), PropValue::Int(25)); let keep_id = engine .upsert_node( - 1, + "Person", "keep", UpsertNodeOptions { props: keep_props, @@ -3849,8 +4275,7 @@ fn test_find_nodes_range_ready_declaration_hides_stale_and_incompatible_older_ma engine.reset_property_query_routes(); assert_eq!( engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "score", Some(&PropertyRangeBound::Included(PropValue::Int(0))), Some(&PropertyRangeBound::Included(PropValue::Int(30))), @@ -3875,7 +4300,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { count_props_a.insert("count".to_string(), PropValue::UInt(5)); let count_a = engine .upsert_node( - 1, + "Person", "count-a", UpsertNodeOptions { props: count_props_a, @@ -3887,7 +4312,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { count_props_b.insert("count".to_string(), PropValue::UInt(10)); let count_b = engine .upsert_node( - 1, + "Person", "count-b", UpsertNodeOptions { props: count_props_b, @@ -3899,7 +4324,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { incompatible_count_props.insert("count".to_string(), PropValue::Int(7)); engine .upsert_node( - 1, + "Person", "count-bad", UpsertNodeOptions { props: incompatible_count_props, @@ -3909,8 +4334,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { .unwrap(); let uint_info = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "count", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::UInt, @@ -3922,8 +4346,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { engine.reset_property_query_routes(); assert_eq!( engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "count", Some(&PropertyRangeBound::Included(PropValue::UInt(0))), Some(&PropertyRangeBound::Included(PropValue::UInt(10))), @@ -3939,7 +4362,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { temp_neg_zero.insert("temp".to_string(), PropValue::Float(-0.0)); let neg_zero = engine .upsert_node( - 1, + "Person", "temp-neg-zero", UpsertNodeOptions { props: temp_neg_zero, @@ -3951,7 +4374,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { temp_pos_zero.insert("temp".to_string(), PropValue::Float(0.0)); let pos_zero = engine .upsert_node( - 1, + "Person", "temp-pos-zero", UpsertNodeOptions { props: temp_pos_zero, @@ -3963,7 +4386,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { temp_one.insert("temp".to_string(), PropValue::Float(1.5)); let one = engine .upsert_node( - 1, + "Person", "temp-one", UpsertNodeOptions { props: temp_one, @@ -3975,7 +4398,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { temp_inf.insert("temp".to_string(), PropValue::Float(f64::INFINITY)); engine .upsert_node( - 1, + "Person", "temp-inf", UpsertNodeOptions { props: temp_inf, @@ -3985,8 +4408,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { .unwrap(); let float_info = engine - .ensure_node_property_index( - 1, + .ensure_node_property_index("Person", "temp", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Float, @@ -3998,8 +4420,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { engine.reset_property_query_routes(); assert_eq!( engine - .find_nodes_range( - 1, + .find_nodes_range("Person", "temp", Some(&PropertyRangeBound::Included(PropValue::Float(-0.0))), Some(&PropertyRangeBound::Included(PropValue::Float(1.5))), @@ -4015,7 +4436,7 @@ fn test_find_nodes_range_ready_domain_specific_uint_and_float() { } #[test] -fn test_nodes_by_type_paged_policy_refills_past_sparse_filtered_window() { +fn test_nodes_by_labels_paged_policy_refills_past_sparse_filtered_window() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); @@ -4025,7 +4446,7 @@ fn test_nodes_by_type_paged_policy_refills_past_sparse_filtered_window() { let weight = if i < 12 { 0.1 } else { 1.0 }; let id = engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { weight, @@ -4044,14 +4465,13 @@ fn test_nodes_by_type_paged_policy_refills_past_sparse_filtered_window() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let page1 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: None, @@ -4062,8 +4482,7 @@ fn test_nodes_by_type_paged_policy_refills_past_sparse_filtered_window() { assert!(page1.next_cursor.is_some()); let page2 = engine - .nodes_by_type_paged( - 1, + .nodes_by_labels_paged("Person", &PageRequest { limit: Some(3), after: page1.next_cursor, @@ -4088,7 +4507,7 @@ fn test_find_nodes_paged_policy_refills_past_sparse_filtered_window() { let weight = if i < 12 { 0.1 } else { 1.0 }; let id = engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { props, @@ -4108,14 +4527,13 @@ fn test_find_nodes_paged_policy_refills_past_sparse_filtered_window() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let page1 = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -4128,8 +4546,7 @@ fn test_find_nodes_paged_policy_refills_past_sparse_filtered_window() { assert!(page1.next_cursor.is_some()); let page2 = engine - .find_nodes_paged( - 1, + .find_nodes_paged("Person", "color", &red, &PageRequest { @@ -4142,6 +4559,70 @@ fn test_find_nodes_paged_policy_refills_past_sparse_filtered_window() { assert!(page2.next_cursor.is_none()); } +#[test] +fn test_find_nodes_paged_scan_fallback_cursor_requires_extra_verified_match() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let red = PropValue::String("red".to_string()); + + let mut props = BTreeMap::new(); + props.insert("color".to_string(), red.clone()); + let keep_id = engine + .upsert_node( + "Person", + "keep", + UpsertNodeOptions { + props: props.clone(), + weight: 1.0, + ..Default::default() + }, + ) + .unwrap(); + for i in 0..5 { + engine + .upsert_node( + "Person", + &format!("pruned-{i}"), + UpsertNodeOptions { + props: props.clone(), + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + } + + engine + .set_prune_policy( + "low_weight", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: None, + }, + ) + .unwrap(); + + let page = engine + .find_nodes_paged("Person", + "color", + &red, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(page.items, vec![keep_id]); + assert!( + page.next_cursor.is_none(), + "scan fallback pagination must not report a next page unless another verified node exists" + ); + + engine.close().unwrap(); +} + #[test] fn test_find_nodes_paged_default_returns_all() { let dir = TempDir::new().unwrap(); @@ -4155,7 +4636,7 @@ fn test_find_nodes_paged_default_returns_all() { for i in 0..5 { engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { props: props.clone(), @@ -4166,7 +4647,7 @@ fn test_find_nodes_paged_default_returns_all() { } let result = engine - .find_nodes_paged(1, "color", &red, &PageRequest::default()) + .find_nodes_paged("Person", "color", &red, &PageRequest::default()) .unwrap(); assert_eq!(result.items.len(), 5); assert!(result.next_cursor.is_none()); @@ -4182,7 +4663,7 @@ fn test_upsert_edge_default_temporal_fields() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4192,7 +4673,7 @@ fn test_upsert_edge_default_temporal_fields() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4201,7 +4682,7 @@ fn test_upsert_edge_default_temporal_fields() { ) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let edge = engine.get_edge(eid).unwrap().unwrap(); @@ -4220,7 +4701,7 @@ fn test_upsert_edge_custom_temporal_fields() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4230,7 +4711,7 @@ fn test_upsert_edge_custom_temporal_fields() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4242,7 +4723,7 @@ fn test_upsert_edge_custom_temporal_fields() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(1000), valid_to: Some(5000), @@ -4266,7 +4747,7 @@ fn test_temporal_fields_survive_flush_and_segment_read() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4276,7 +4757,7 @@ fn test_temporal_fields_survive_flush_and_segment_read() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4288,7 +4769,7 @@ fn test_temporal_fields_survive_flush_and_segment_read() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(2000), valid_to: Some(8000), @@ -4318,7 +4799,7 @@ fn test_temporal_fields_survive_wal_replay() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4328,7 +4809,7 @@ fn test_temporal_fields_survive_wal_replay() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4340,7 +4821,7 @@ fn test_temporal_fields_survive_wal_replay() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(3000), valid_to: Some(9000), @@ -4369,7 +4850,7 @@ fn test_batch_upsert_edges_temporal_fields() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4379,7 +4860,7 @@ fn test_batch_upsert_edges_temporal_fields() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4389,7 +4870,7 @@ fn test_batch_upsert_edges_temporal_fields() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -4402,7 +4883,7 @@ fn test_batch_upsert_edges_temporal_fields() { EdgeInput { from: a, to: b, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: Some(1000), @@ -4411,14 +4892,14 @@ fn test_batch_upsert_edges_temporal_fields() { EdgeInput { from: b, to: c, - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, // defaults }, ]; - let ids = engine.batch_upsert_edges(&inputs).unwrap(); + let ids = engine.batch_upsert_edges(inputs).unwrap(); let e1 = engine.get_edge(ids[0]).unwrap().unwrap(); assert_eq!(e1.valid_from, 1000); @@ -4439,7 +4920,7 @@ fn test_temporal_fields_survive_compaction() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4449,7 +4930,7 @@ fn test_temporal_fields_survive_compaction() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4461,7 +4942,7 @@ fn test_temporal_fields_survive_compaction() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(4000), valid_to: Some(7000), @@ -4475,7 +4956,7 @@ fn test_temporal_fields_survive_compaction() { // Add something to create segment 2 let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -4484,7 +4965,7 @@ fn test_temporal_fields_survive_compaction() { ) .unwrap(); engine - .upsert_edge(b, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -4510,7 +4991,7 @@ fn test_invalidate_edge_closes_validity_window() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4520,7 +5001,7 @@ fn test_invalidate_edge_closes_validity_window() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4529,7 +5010,7 @@ fn test_invalidate_edge_closes_validity_window() { ) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Edge should be valid initially @@ -4569,7 +5050,7 @@ fn test_invalidated_edge_hidden_from_neighbors() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4579,7 +5060,7 @@ fn test_invalidated_edge_hidden_from_neighbors() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4589,7 +5070,7 @@ fn test_invalidated_edge_hidden_from_neighbors() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -4598,10 +5079,10 @@ fn test_invalidated_edge_hidden_from_neighbors() { ) .unwrap(); let e_ab = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Both neighbors visible @@ -4627,7 +5108,7 @@ fn test_invalidated_edge_hidden_after_flush() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4637,7 +5118,7 @@ fn test_invalidated_edge_hidden_after_flush() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4646,7 +5127,7 @@ fn test_invalidated_edge_hidden_after_flush() { ) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Flush to segment, then invalidate (invalidation goes to memtable/WAL) @@ -4674,7 +5155,7 @@ fn test_invalidated_edge_survives_wal_replay() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -4684,7 +5165,7 @@ fn test_invalidated_edge_survives_wal_replay() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -4693,7 +5174,7 @@ fn test_invalidated_edge_survives_wal_replay() { ) .unwrap(); eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.invalidate_edge(eid, 1).unwrap(); engine.close().unwrap(); @@ -4720,13 +5201,13 @@ fn test_point_in_time_query_sees_valid_edges() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); // Edge a→b: valid from epoch 1000 to 5000 @@ -4734,7 +5215,7 @@ fn test_point_in_time_query_sees_valid_edges() { .upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(1000), valid_to: Some(5000), @@ -4747,7 +5228,7 @@ fn test_point_in_time_query_sees_valid_edges() { .upsert_edge( a, c, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(3000), valid_to: Some(8000), @@ -4827,10 +5308,10 @@ fn test_point_in_time_query_with_invalidated_edge() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); // Create edge with explicit validity window @@ -4838,7 +5319,7 @@ fn test_point_in_time_query_with_invalidated_edge() { .upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(1000), valid_to: Some(10000), @@ -4895,17 +5376,17 @@ fn test_point_in_time_query_after_flush() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine .upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(2000), valid_to: Some(8000), @@ -4948,13 +5429,13 @@ fn test_point_in_time_traverse_depth_two() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); // a→b valid 1000-5000, b→c valid 2000-6000 @@ -4962,7 +5443,7 @@ fn test_point_in_time_traverse_depth_two() { .upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(1000), valid_to: Some(5000), @@ -4974,7 +5455,7 @@ fn test_point_in_time_traverse_depth_two() { .upsert_edge( b, c, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(2000), valid_to: Some(6000), @@ -5007,13 +5488,13 @@ fn test_decay_scoring_orders_by_recency() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let hub = engine - .upsert_node(1, "hub", UpsertNodeOptions::default()) + .upsert_node("Person", "hub", UpsertNodeOptions::default()) .unwrap(); let old = engine - .upsert_node(1, "old", UpsertNodeOptions::default()) + .upsert_node("Person", "old", UpsertNodeOptions::default()) .unwrap(); let recent = engine - .upsert_node(1, "recent", UpsertNodeOptions::default()) + .upsert_node("Person", "recent", UpsertNodeOptions::default()) .unwrap(); let now = now_millis(); @@ -5026,7 +5507,7 @@ fn test_decay_scoring_orders_by_recency() { .upsert_edge( hub, old, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(one_day_ago), ..Default::default() @@ -5038,7 +5519,7 @@ fn test_decay_scoring_orders_by_recency() { .upsert_edge( hub, recent, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(one_hour_ago), ..Default::default() @@ -5076,13 +5557,13 @@ fn test_decay_scoring_with_different_base_weights() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let hub = engine - .upsert_node(1, "hub", UpsertNodeOptions::default()) + .upsert_node("Person", "hub", UpsertNodeOptions::default()) .unwrap(); let heavy_old = engine - .upsert_node(1, "heavy_old", UpsertNodeOptions::default()) + .upsert_node("Person", "heavy_old", UpsertNodeOptions::default()) .unwrap(); let light_new = engine - .upsert_node(1, "light_new", UpsertNodeOptions::default()) + .upsert_node("Person", "light_new", UpsertNodeOptions::default()) .unwrap(); let now = now_millis(); @@ -5094,7 +5575,7 @@ fn test_decay_scoring_with_different_base_weights() { .upsert_edge( hub, heavy_old, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 10.0, valid_from: Some(two_days_ago), @@ -5107,7 +5588,7 @@ fn test_decay_scoring_with_different_base_weights() { .upsert_edge( hub, light_new, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(one_hour_ago), ..Default::default() @@ -5140,17 +5621,17 @@ fn test_decay_zero_lambda_no_reorder() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine .upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 5.0, ..Default::default() @@ -5172,16 +5653,16 @@ fn test_decay_with_limit_returns_top_scored() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let hub = engine - .upsert_node(1, "hub", UpsertNodeOptions::default()) + .upsert_node("Person", "hub", UpsertNodeOptions::default()) .unwrap(); let n1 = engine - .upsert_node(1, "n1", UpsertNodeOptions::default()) + .upsert_node("Person", "n1", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "n2", UpsertNodeOptions::default()) + .upsert_node("Person", "n2", UpsertNodeOptions::default()) .unwrap(); let n3 = engine - .upsert_node(1, "n3", UpsertNodeOptions::default()) + .upsert_node("Person", "n3", UpsertNodeOptions::default()) .unwrap(); let now = now_millis(); @@ -5191,7 +5672,7 @@ fn test_decay_with_limit_returns_top_scored() { .upsert_edge( hub, n1, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(now - 72 * 3_600_000), ..Default::default() @@ -5202,7 +5683,7 @@ fn test_decay_with_limit_returns_top_scored() { .upsert_edge( hub, n2, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(now - 24 * 3_600_000), ..Default::default() @@ -5213,7 +5694,7 @@ fn test_decay_with_limit_returns_top_scored() { .upsert_edge( hub, n3, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(now - 3_600_000), ..Default::default() @@ -5245,13 +5726,13 @@ fn test_point_in_time_with_decay() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let hub = engine - .upsert_node(1, "hub", UpsertNodeOptions::default()) + .upsert_node("Person", "hub", UpsertNodeOptions::default()) .unwrap(); let n1 = engine - .upsert_node(1, "n1", UpsertNodeOptions::default()) + .upsert_node("Person", "n1", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "n2", UpsertNodeOptions::default()) + .upsert_node("Person", "n2", UpsertNodeOptions::default()) .unwrap(); // Edge 1: valid 1000-5000, updated_at=1000 @@ -5259,7 +5740,7 @@ fn test_point_in_time_with_decay() { .upsert_edge( hub, n1, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(1000), valid_to: Some(5000), @@ -5272,7 +5753,7 @@ fn test_point_in_time_with_decay() { .upsert_edge( hub, n2, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(2000), valid_to: Some(8000), @@ -5320,13 +5801,13 @@ fn test_decay_scoring_after_flush_segment_sourced() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let hub = engine - .upsert_node(1, "hub", UpsertNodeOptions::default()) + .upsert_node("Person", "hub", UpsertNodeOptions::default()) .unwrap(); let old = engine - .upsert_node(1, "old", UpsertNodeOptions::default()) + .upsert_node("Person", "old", UpsertNodeOptions::default()) .unwrap(); let recent = engine - .upsert_node(1, "recent", UpsertNodeOptions::default()) + .upsert_node("Person", "recent", UpsertNodeOptions::default()) .unwrap(); let now = now_millis(); @@ -5334,7 +5815,7 @@ fn test_decay_scoring_after_flush_segment_sourced() { .upsert_edge( hub, old, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(now - 48 * 3_600_000), ..Default::default() @@ -5345,7 +5826,7 @@ fn test_decay_scoring_after_flush_segment_sourced() { .upsert_edge( hub, recent, - 1, + "RELATES_TO", UpsertEdgeOptions { valid_from: Some(now - 3_600_000), ..Default::default() @@ -5380,10 +5861,10 @@ fn test_negative_decay_lambda_returns_error() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(a, a, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = engine.neighbors( @@ -5407,7 +5888,7 @@ fn test_temporal_adjacency_postings_survive_flush() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -5417,7 +5898,7 @@ fn test_temporal_adjacency_postings_survive_flush() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -5427,7 +5908,7 @@ fn test_temporal_adjacency_postings_survive_flush() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -5441,7 +5922,7 @@ fn test_temporal_adjacency_postings_survive_flush() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(1000), valid_to: Some(5000), @@ -5454,7 +5935,7 @@ fn test_temporal_adjacency_postings_survive_flush() { .upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(3000), valid_to: Some(9000), @@ -5519,7 +6000,7 @@ fn test_adjacency_hashmap_upsert_idempotent() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -5529,7 +6010,7 @@ fn test_adjacency_hashmap_upsert_idempotent() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -5541,13 +6022,13 @@ fn test_adjacency_hashmap_upsert_idempotent() { // Insert edge, then upsert it multiple times with different weights // With edge_uniqueness, (a, b, 10) deduplicates to the same edge ID. let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let eid2 = engine .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -5558,7 +6039,7 @@ fn test_adjacency_hashmap_upsert_idempotent() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 3.0, ..Default::default() @@ -5597,7 +6078,7 @@ fn test_compact_with_progress_reports_all_phases() { for i in 0..30 { node_ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5606,7 +6087,7 @@ fn test_compact_with_progress_reports_all_phases() { for i in 30..60 { node_ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5618,7 +6099,7 @@ fn test_compact_with_progress_reports_all_phases() { .upsert_edge( node_ids[i], node_ids[i + 1], - 1, + "RELATES_TO", UpsertEdgeOptions::default(), ) .unwrap(); @@ -5674,7 +6155,7 @@ fn test_compact_with_progress_cancel_during_tombstones() { for i in 0..20 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5682,7 +6163,7 @@ fn test_compact_with_progress_cancel_during_tombstones() { for i in 20..40 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5718,7 +6199,7 @@ fn test_compact_with_progress_cancel_during_merge_nodes() { for i in 0..20 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5726,7 +6207,7 @@ fn test_compact_with_progress_cancel_during_merge_nodes() { for i in 20..40 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5761,7 +6242,7 @@ fn test_compact_with_progress_cancel_during_merge_edges() { for i in 0..10 { node_ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5771,7 +6252,7 @@ fn test_compact_with_progress_cancel_during_merge_edges() { .upsert_edge( node_ids[i], node_ids[i + 1], - 1, + "RELATES_TO", UpsertEdgeOptions::default(), ) .unwrap(); @@ -5810,7 +6291,7 @@ fn test_compact_with_progress_cancel_before_write() { for i in 0..10 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5818,7 +6299,7 @@ fn test_compact_with_progress_cancel_before_write() { for i in 10..20 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5860,13 +6341,13 @@ fn test_compact_with_progress_records_processed_counts() { // 50 nodes in seg1, 50 nodes in seg2 for i in 0..50 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); for i in 50..100 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -5904,7 +6385,7 @@ fn test_compact_with_progress_tombstone_counts_all_examined() { for i in 0..50 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5952,7 +6433,7 @@ fn test_compact_no_callback_wrapper() { for i in 0..20 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -5960,7 +6441,7 @@ fn test_compact_no_callback_wrapper() { for i in 20..40 { ids.push( engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(), ); } @@ -6002,7 +6483,7 @@ fn test_get_node_by_key_found() { let engine = open_imm(&dir.path().join("db")); let id = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "Alice"), @@ -6010,9 +6491,9 @@ fn test_get_node_by_key_found() { }, ) .unwrap(); - let node = engine.get_node_by_key(1, "alice").unwrap().unwrap(); + let node = engine.get_node_by_key("Person", "alice").unwrap().unwrap(); assert_eq!(node.id, id); - assert_eq!(node.type_id, 1); + assert_eq!(node.labels.as_slice(), ["Person"]); assert_eq!(node.key, "alice"); assert_eq!( node.props.get("name"), @@ -6027,7 +6508,7 @@ fn test_get_node_by_key_not_found() { let engine = open_imm(&dir.path().join("db")); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "Alice"), @@ -6035,8 +6516,8 @@ fn test_get_node_by_key_not_found() { }, ) .unwrap(); - assert!(engine.get_node_by_key(1, "bob").unwrap().is_none()); - assert!(engine.get_node_by_key(2, "alice").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "bob").unwrap().is_none()); + assert!(engine.get_node_by_key("Company", "alice").unwrap().is_none()); engine.close().unwrap(); } @@ -6046,7 +6527,7 @@ fn test_get_node_by_key_after_flush() { let engine = open_imm(&dir.path().join("db")); let id = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "Alice"), @@ -6055,7 +6536,7 @@ fn test_get_node_by_key_after_flush() { ) .unwrap(); engine.flush().unwrap(); - let node = engine.get_node_by_key(1, "alice").unwrap().unwrap(); + let node = engine.get_node_by_key("Person", "alice").unwrap().unwrap(); assert_eq!(node.id, id); assert_eq!(node.key, "alice"); engine.close().unwrap(); @@ -6067,7 +6548,7 @@ fn test_get_node_by_key_after_compaction() { let engine = open_imm(&dir.path().join("db")); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "v1"), @@ -6078,7 +6559,7 @@ fn test_get_node_by_key_after_compaction() { engine.flush().unwrap(); let id2 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "v2"), @@ -6089,7 +6570,7 @@ fn test_get_node_by_key_after_compaction() { .unwrap(); engine.flush().unwrap(); engine.compact().unwrap(); - let node = engine.get_node_by_key(1, "alice").unwrap().unwrap(); + let node = engine.get_node_by_key("Person", "alice").unwrap().unwrap(); assert_eq!(node.id, id2); assert_eq!( node.props.get("name"), @@ -6105,7 +6586,7 @@ fn test_get_node_by_key_deleted() { let engine = open_imm(&dir.path().join("db")); let id = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "Alice"), @@ -6114,7 +6595,7 @@ fn test_get_node_by_key_deleted() { ) .unwrap(); engine.delete_node(id).unwrap(); - assert!(engine.get_node_by_key(1, "alice").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "alice").unwrap().is_none()); engine.close().unwrap(); } @@ -6124,7 +6605,7 @@ fn test_get_node_by_key_deleted_cross_source() { let engine = open_imm(&dir.path().join("db")); let id = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "Alice"), @@ -6134,7 +6615,7 @@ fn test_get_node_by_key_deleted_cross_source() { .unwrap(); engine.flush().unwrap(); engine.delete_node(id).unwrap(); - assert!(engine.get_node_by_key(1, "alice").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "alice").unwrap().is_none()); engine.close().unwrap(); } @@ -6144,7 +6625,7 @@ fn test_get_node_by_key_memtable_shadows_segment() { let engine = open_imm(&dir.path().join("db")); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "v1"), @@ -6155,7 +6636,7 @@ fn test_get_node_by_key_memtable_shadows_segment() { engine.flush().unwrap(); let id2 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "v2"), @@ -6164,7 +6645,7 @@ fn test_get_node_by_key_memtable_shadows_segment() { }, ) .unwrap(); - let node = engine.get_node_by_key(1, "alice").unwrap().unwrap(); + let node = engine.get_node_by_key("Person", "alice").unwrap().unwrap(); assert_eq!(node.id, id2); assert_eq!( node.props.get("name"), @@ -6180,16 +6661,16 @@ fn test_get_edge_by_triple_found() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let eid = engine .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { props: make_props("rel", "knows"), weight: 0.5, @@ -6197,11 +6678,11 @@ fn test_get_edge_by_triple_found() { }, ) .unwrap(); - let edge = engine.get_edge_by_triple(a, b, 10).unwrap().unwrap(); + let edge = engine.get_edge_by_triple(a, b, "KNOWS").unwrap().unwrap(); assert_eq!(edge.id, eid); assert_eq!(edge.from, a); assert_eq!(edge.to, b); - assert_eq!(edge.type_id, 10); + assert_eq!(edge.label, "KNOWS"); assert_eq!( edge.props.get("rel"), Some(&PropValue::String("knows".to_string())) @@ -6214,16 +6695,16 @@ fn test_get_edge_by_triple_not_found() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); - assert!(engine.get_edge_by_triple(a, b, 99).unwrap().is_none()); - assert!(engine.get_edge_by_triple(b, a, 10).unwrap().is_none()); + assert!(engine.get_edge_by_triple(a, b, "MISSING_EDGE_LABEL").unwrap().is_none()); + assert!(engine.get_edge_by_triple(b, a, "KNOWS").unwrap().is_none()); engine.close().unwrap(); } @@ -6232,16 +6713,16 @@ fn test_get_edge_by_triple_after_flush() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); - let edge = engine.get_edge_by_triple(a, b, 10).unwrap().unwrap(); + let edge = engine.get_edge_by_triple(a, b, "KNOWS").unwrap().unwrap(); assert_eq!(edge.id, eid); engine.close().unwrap(); } @@ -6251,16 +6732,16 @@ fn test_get_edge_by_triple_deleted() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.delete_edge(eid).unwrap(); - assert!(engine.get_edge_by_triple(a, b, 10).unwrap().is_none()); + assert!(engine.get_edge_by_triple(a, b, "KNOWS").unwrap().is_none()); engine.close().unwrap(); } @@ -6269,17 +6750,17 @@ fn test_get_edge_by_triple_deleted_cross_source() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); engine.delete_edge(eid).unwrap(); - assert!(engine.get_edge_by_triple(a, b, 10).unwrap().is_none()); + assert!(engine.get_edge_by_triple(a, b, "KNOWS").unwrap().is_none()); engine.close().unwrap(); } @@ -6295,16 +6776,16 @@ fn test_get_edge_by_triple_after_compaction() { }; let engine = DatabaseEngine::open(&dir.path().join("db"), &opts).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { props: make_props("v", "1"), ..Default::default() @@ -6316,7 +6797,7 @@ fn test_get_edge_by_triple_after_compaction() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { props: make_props("v", "2"), weight: 2.0, @@ -6326,7 +6807,7 @@ fn test_get_edge_by_triple_after_compaction() { .unwrap(); engine.flush().unwrap(); engine.compact().unwrap(); - let edge = engine.get_edge_by_triple(a, b, 10).unwrap().unwrap(); + let edge = engine.get_edge_by_triple(a, b, "KNOWS").unwrap().unwrap(); assert_eq!(edge.id, eid2); assert_eq!( edge.props.get("v"), @@ -6343,7 +6824,7 @@ fn test_get_nodes_bulk() { let engine = open_imm(&dir.path().join("db")); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("name", "A"), @@ -6353,7 +6834,7 @@ fn test_get_nodes_bulk() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props: make_props("name", "B"), @@ -6363,7 +6844,7 @@ fn test_get_nodes_bulk() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { props: make_props("name", "C"), @@ -6384,10 +6865,10 @@ fn test_get_nodes_bulk_mixed_found_missing() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.delete_node(b).unwrap(); let results = engine.get_nodes(&[a, b, 9999]).unwrap(); @@ -6403,11 +6884,11 @@ fn test_get_nodes_bulk_cross_source() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let results = engine.get_nodes(&[a, b]).unwrap(); assert_eq!(results[0].as_ref().unwrap().key, "a"); @@ -6429,19 +6910,19 @@ fn test_get_edges_bulk() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e2 = engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let results = engine.get_edges(&[e1, e2, 9999]).unwrap(); assert_eq!(results.len(), 3); @@ -6456,20 +6937,20 @@ fn test_get_edges_bulk_cross_source() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); let e2 = engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let results = engine.get_edges(&[e1, e2]).unwrap(); assert_eq!(results[0].as_ref().unwrap().from, a); @@ -6488,7 +6969,7 @@ fn test_get_nodes_bulk_multi_segment_interleaved() { // Segment 1: nodes 1, 2, 3 let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("seg", "1"), @@ -6498,7 +6979,7 @@ fn test_get_nodes_bulk_multi_segment_interleaved() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props: make_props("seg", "1"), @@ -6508,7 +6989,7 @@ fn test_get_nodes_bulk_multi_segment_interleaved() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { props: make_props("seg", "1"), @@ -6520,7 +7001,7 @@ fn test_get_nodes_bulk_multi_segment_interleaved() { // Segment 2: nodes 4, 5 let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { props: make_props("seg", "2"), @@ -6530,7 +7011,7 @@ fn test_get_nodes_bulk_multi_segment_interleaved() { .unwrap(); let e = engine .upsert_node( - 1, + "Person", "e", UpsertNodeOptions { props: make_props("seg", "2"), @@ -6558,15 +7039,15 @@ fn test_get_nodes_bulk_tombstone_in_newer_segment() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); // seg 1: a, b engine.delete_node(a).unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); // seg 2: tombstone(a), c @@ -6585,7 +7066,7 @@ fn test_get_nodes_bulk_memtable_shadows_segment() { let engine = open_imm(&dir.path().join("db")); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "old"), @@ -6596,7 +7077,7 @@ fn test_get_nodes_bulk_memtable_shadows_segment() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "new"), @@ -6621,7 +7102,7 @@ fn test_get_nodes_bulk_duplicate_ids() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let results = engine.get_nodes(&[a, a, a]).unwrap(); assert_eq!(results.len(), 3); @@ -6638,7 +7119,7 @@ fn test_get_nodes_bulk_duplicate_ids_in_segment() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); let results = engine.get_nodes(&[a, a, a]).unwrap(); @@ -6654,22 +7135,22 @@ fn test_get_edges_bulk_multi_segment_interleaved() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); // Segment 1 let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // Segment 2 let e2 = engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -6685,19 +7166,19 @@ fn test_get_edges_bulk_tombstone_cross_segment() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e2 = engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // seg 1: e1, e2 engine.delete_edge(e1).unwrap(); @@ -6715,7 +7196,7 @@ fn test_get_nodes_bulk_after_compaction() { let engine = open_imm(&dir.path().join("db")); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "1"), @@ -6725,7 +7206,7 @@ fn test_get_nodes_bulk_after_compaction() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props: make_props("v", "1"), @@ -6736,7 +7217,7 @@ fn test_get_nodes_bulk_after_compaction() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "2"), @@ -6747,7 +7228,7 @@ fn test_get_nodes_bulk_after_compaction() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { props: make_props("v", "1"), @@ -6776,7 +7257,7 @@ fn test_get_node_by_key_delete_then_recreate() { let engine = open_imm(&dir.path().join("db")); let id1 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("v", "1"), @@ -6789,7 +7270,7 @@ fn test_get_node_by_key_delete_then_recreate() { // Re-create with same key → gets a NEW id let id2 = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("v", "2"), @@ -6799,7 +7280,7 @@ fn test_get_node_by_key_delete_then_recreate() { ) .unwrap(); assert_ne!(id1, id2); - let node = engine.get_node_by_key(1, "alice").unwrap().unwrap(); + let node = engine.get_node_by_key("Person", "alice").unwrap().unwrap(); assert_eq!(node.id, id2); assert_eq!( node.props.get("v"), @@ -6816,16 +7297,16 @@ fn test_get_edge_by_triple_uniqueness_off_returns_latest() { // Default: edge_uniqueness = false let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let _e1 = engine .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { props: make_props("v", "1"), ..Default::default() @@ -6836,7 +7317,7 @@ fn test_get_edge_by_triple_uniqueness_off_returns_latest() { .upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { props: make_props("v", "2"), weight: 2.0, @@ -6845,7 +7326,7 @@ fn test_get_edge_by_triple_uniqueness_off_returns_latest() { ) .unwrap(); // With uniqueness off, both edges exist. Triple index maps to the latest. - let edge = engine.get_edge_by_triple(a, b, 10).unwrap().unwrap(); + let edge = engine.get_edge_by_triple(a, b, "KNOWS").unwrap().unwrap(); assert_eq!(edge.id, e2); assert_eq!( edge.props.get("v"), @@ -6863,19 +7344,19 @@ fn test_graph_patch_mixed_ops() { // Create some initial data let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // Patch: upsert a new node, a new edge a→b, invalidate e1 let patch = GraphPatch { upsert_nodes: vec![NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "c".to_string(), props: make_props("role", "new"), weight: 1.0, @@ -6885,7 +7366,7 @@ fn test_graph_patch_mixed_ops() { upsert_edges: vec![EdgeInput { from: a, to: b, - type_id: 2, + label: "WORKS_AT".to_string(), props: BTreeMap::new(), weight: 0.5, valid_from: None, @@ -6895,7 +7376,7 @@ fn test_graph_patch_mixed_ops() { delete_node_ids: vec![], delete_edge_ids: vec![], }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); // Verify results assert_eq!(result.node_ids.len(), 1); @@ -6925,7 +7406,7 @@ fn test_graph_patch_mixed_ops() { fn test_graph_patch_empty() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); - let result = engine.graph_patch(&GraphPatch::default()).unwrap(); + let result = engine.graph_patch(GraphPatch::default()).unwrap(); assert!(result.node_ids.is_empty()); assert!(result.edge_ids.is_empty()); engine.close().unwrap(); @@ -6939,7 +7420,7 @@ fn test_graph_patch_node_dedup() { // Pre-existing node let existing = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("v", "1"), @@ -6951,7 +7432,7 @@ fn test_graph_patch_node_dedup() { let patch = GraphPatch { upsert_nodes: vec![ NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "alice".to_string(), props: make_props("v", "2"), weight: 2.0, @@ -6959,7 +7440,7 @@ fn test_graph_patch_node_dedup() { sparse_vector: None, }, NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "alice".to_string(), props: make_props("v", "3"), weight: 3.0, @@ -6967,7 +7448,7 @@ fn test_graph_patch_node_dedup() { sparse_vector: None, }, NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "bob".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -6977,7 +7458,7 @@ fn test_graph_patch_node_dedup() { ], ..GraphPatch::default() }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); // alice deduped: both get the existing ID assert_eq!(result.node_ids[0], existing); @@ -7000,19 +7481,19 @@ fn test_graph_patch_delete_with_cascade() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e_ab = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e_bc = engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // Delete node b. Should cascade delete e_ab and e_bc @@ -7020,7 +7501,7 @@ fn test_graph_patch_delete_with_cascade() { delete_node_ids: vec![b], ..GraphPatch::default() }; - engine.graph_patch(&patch).unwrap(); + engine.graph_patch(patch).unwrap(); assert!(engine.get_node(b).unwrap().is_none()); assert!(engine.get_edge(e_ab).unwrap().is_none()); @@ -7037,20 +7518,20 @@ fn test_graph_patch_edge_delete() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let patch = GraphPatch { delete_edge_ids: vec![e], ..GraphPatch::default() }; - engine.graph_patch(&patch).unwrap(); + engine.graph_patch(patch).unwrap(); assert!(engine.get_edge(e).unwrap().is_none()); // Nodes survive @@ -7068,12 +7549,12 @@ fn test_graph_patch_ordering_upserts_before_deletes() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let patch = GraphPatch { upsert_nodes: vec![NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "a".to_string(), props: make_props("v", "updated"), weight: 2.0, @@ -7083,7 +7564,7 @@ fn test_graph_patch_ordering_upserts_before_deletes() { delete_node_ids: vec![a], ..GraphPatch::default() }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); assert_eq!(result.node_ids[0], a); // Delete wins, node should be gone @@ -7101,7 +7582,7 @@ fn test_graph_patch_invalidate_nonexistent_edge_skipped() { invalidate_edges: vec![(99999, 5000)], ..GraphPatch::default() }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); assert!(result.node_ids.is_empty()); assert!(result.edge_ids.is_empty()); engine.close().unwrap(); @@ -7116,18 +7597,18 @@ fn test_graph_patch_survives_wal_replay() { { let engine = open_imm(&db_path); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); invalidated_eid = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let patch = GraphPatch { upsert_nodes: vec![NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "c".to_string(), props: make_props("role", "new"), weight: 1.0, @@ -7137,7 +7618,7 @@ fn test_graph_patch_survives_wal_replay() { upsert_edges: vec![EdgeInput { from: a, to: b, - type_id: 5, + label: "OWNS".to_string(), props: BTreeMap::new(), weight: 0.5, valid_from: None, @@ -7147,7 +7628,7 @@ fn test_graph_patch_survives_wal_replay() { delete_edge_ids: vec![], delete_node_ids: vec![], }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); node_id = result.node_ids[0]; edge_id = result.edge_ids[0]; engine.close().unwrap(); @@ -7160,7 +7641,7 @@ fn test_graph_patch_survives_wal_replay() { assert_eq!(node.key, "c"); let edge = engine.get_edge(edge_id).unwrap().unwrap(); - assert_eq!(edge.type_id, 5); + assert_eq!(edge.label, "OWNS"); let inv_edge = engine.get_edge(invalidated_eid).unwrap().unwrap(); assert_eq!(inv_edge.valid_to, 2000); @@ -7188,7 +7669,7 @@ fn test_graph_patch_vectors_survive_wal_replay() { let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); let patch = GraphPatch { upsert_nodes: vec![NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "vector-c".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -7197,7 +7678,7 @@ fn test_graph_patch_vectors_survive_wal_replay() { }], ..GraphPatch::default() }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); node_id = result.node_ids[0]; let node = engine.get_node(node_id).unwrap().unwrap(); @@ -7223,7 +7704,7 @@ fn test_graph_patch_two_step_upsert_then_connect() { let patch1 = GraphPatch { upsert_nodes: vec![ NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "x".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -7231,7 +7712,7 @@ fn test_graph_patch_two_step_upsert_then_connect() { sparse_vector: None, }, NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "y".to_string(), props: BTreeMap::new(), weight: 1.0, @@ -7241,7 +7722,7 @@ fn test_graph_patch_two_step_upsert_then_connect() { ], ..GraphPatch::default() }; - let r1 = engine.graph_patch(&patch1).unwrap(); + let r1 = engine.graph_patch(patch1).unwrap(); let x = r1.node_ids[0]; let y = r1.node_ids[1]; @@ -7250,7 +7731,7 @@ fn test_graph_patch_two_step_upsert_then_connect() { upsert_edges: vec![EdgeInput { from: x, to: y, - type_id: 10, + label: "KNOWS".to_string(), props: make_props("rel", "friend"), weight: 1.0, valid_from: None, @@ -7258,12 +7739,12 @@ fn test_graph_patch_two_step_upsert_then_connect() { }], ..GraphPatch::default() }; - let r2 = engine.graph_patch(&patch2).unwrap(); + let r2 = engine.graph_patch(patch2).unwrap(); let edge = engine.get_edge(r2.edge_ids[0]).unwrap().unwrap(); assert_eq!(edge.from, x); assert_eq!(edge.to, y); - assert_eq!(edge.type_id, 10); + assert_eq!(edge.label, "KNOWS"); // Neighbors work let nbrs = engine.neighbors(x, &NeighborOptions::default()).unwrap(); @@ -7278,20 +7759,20 @@ fn test_graph_patch_after_flush() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // Patch against segment data let patch = GraphPatch { upsert_nodes: vec![NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "a".to_string(), props: make_props("v", "updated"), weight: 2.0, @@ -7301,7 +7782,7 @@ fn test_graph_patch_after_flush() { invalidate_edges: vec![(e, 500)], ..GraphPatch::default() }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); assert_eq!(result.node_ids[0], a); // deduped against segment let node = engine.get_node(a).unwrap().unwrap(); @@ -7323,13 +7804,13 @@ fn test_graph_patch_duplicate_edge_delete_safe() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let patch = GraphPatch { @@ -7337,7 +7818,7 @@ fn test_graph_patch_duplicate_edge_delete_safe() { delete_node_ids: vec![a], // cascade also deletes e ..GraphPatch::default() }; - engine.graph_patch(&patch).unwrap(); // should not panic + engine.graph_patch(patch).unwrap(); // should not panic assert!(engine.get_edge(e).unwrap().is_none()); assert!(engine.get_node(a).unwrap().is_none()); @@ -7351,16 +7832,16 @@ fn test_graph_patch_invalidate_pre_existing_edge() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = engine .upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { props: make_props("v", "original"), ..Default::default() @@ -7377,7 +7858,7 @@ fn test_graph_patch_invalidate_pre_existing_edge() { upsert_edges: vec![EdgeInput { from: a, to: b, - type_id: 1, + label: "RELATES_TO".to_string(), props: make_props("v", "updated"), weight: 2.0, valid_from: None, @@ -7386,7 +7867,7 @@ fn test_graph_patch_invalidate_pre_existing_edge() { invalidate_edges: vec![(e, 3000)], ..GraphPatch::default() }; - engine.graph_patch(&patch).unwrap(); + engine.graph_patch(patch).unwrap(); // The invalidation's UpsertEdge comes after the upsert's UpsertEdge in the ops vec, // so the invalidation's valid_to=3000 wins via last-write-wins in apply_op. @@ -7405,13 +7886,13 @@ fn test_delete_node_cascades_segment_edges() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // edge moves to segment @@ -7440,20 +7921,20 @@ fn test_delete_node_cascades_mixed_sources() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // e1 in segment let e2 = engine - .upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // e2 in memtable @@ -7470,13 +7951,13 @@ fn test_delete_node_cascades_incoming_segment_edges() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -7495,18 +7976,18 @@ fn test_graph_patch_delete_cascades_segment_edges() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .graph_patch(&GraphPatch { + .graph_patch(GraphPatch { delete_node_ids: vec![a], ..GraphPatch::default() }) @@ -7524,14 +8005,14 @@ fn test_prune_empty_policy_rejects() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let err = engine .prune(&PrunePolicy { max_age_ms: None, max_weight: None, - type_id: None, + label: None, }) .unwrap_err(); @@ -7542,19 +8023,19 @@ fn test_prune_empty_policy_rejects() { } #[test] -fn test_prune_type_id_only_rejects() { - // type_id alone without age or weight is rejected (safety) +fn test_prune_label_only_rejects() { + // A label alone without age or weight is rejected (safety). let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let err = engine .prune(&PrunePolicy { max_age_ms: None, max_weight: None, - type_id: Some(1), + label: Some("Person".to_string()), }) .unwrap_err(); @@ -7569,16 +8050,15 @@ fn test_prune_by_age_only() { // Insert nodes. They all get updated_at = now let a = engine - .upsert_node(1, "old", UpsertNodeOptions::default()) + .upsert_node("Person", "old", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "new", UpsertNodeOptions::default()) + .upsert_node("Person", "new", UpsertNodeOptions::default()) .unwrap(); // Hack: manually set "old" node to have an ancient updated_at via write_op - let old_node = engine.get_node(a).unwrap().unwrap(); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { + let old_node = internal_node_record(&engine, a).unwrap().unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { updated_at: 1000, // ancient timestamp ..old_node })) @@ -7589,7 +8069,7 @@ fn test_prune_by_age_only() { .prune(&PrunePolicy { max_age_ms: Some(1000), max_weight: None, - type_id: None, + label: None, }) .unwrap(); @@ -7606,7 +8086,7 @@ fn test_prune_by_weight_only() { let a = engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.1, @@ -7616,7 +8096,7 @@ fn test_prune_by_weight_only() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "mid", UpsertNodeOptions { weight: 0.5, @@ -7626,7 +8106,7 @@ fn test_prune_by_weight_only() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -7640,7 +8120,7 @@ fn test_prune_by_weight_only() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -7658,7 +8138,7 @@ fn test_prune_combo_age_and_weight() { let a = engine .upsert_node( - 1, + "Person", "old-low", UpsertNodeOptions { weight: 0.1, @@ -7668,7 +8148,7 @@ fn test_prune_combo_age_and_weight() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "old-high", UpsertNodeOptions { weight: 0.9, @@ -7678,7 +8158,7 @@ fn test_prune_combo_age_and_weight() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "new-low", UpsertNodeOptions { weight: 0.1, @@ -7688,16 +8168,14 @@ fn test_prune_combo_age_and_weight() { .unwrap(); // Make a and b old - let node_a = engine.get_node(a).unwrap().unwrap(); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { + let node_a = internal_node_record(&engine, a).unwrap().unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { updated_at: 1000, ..node_a })) .unwrap(); - let node_b = engine.get_node(b).unwrap().unwrap(); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { + let node_b = internal_node_record(&engine, b).unwrap().unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { updated_at: 1000, ..node_b })) @@ -7708,7 +8186,7 @@ fn test_prune_combo_age_and_weight() { .prune(&PrunePolicy { max_age_ms: Some(1000), max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -7720,14 +8198,14 @@ fn test_prune_combo_age_and_weight() { } #[test] -fn test_prune_type_scoped() { +fn test_prune_label_scoped() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine .upsert_node( - 1, - "type1-low", + "Person", + "label1-low", UpsertNodeOptions { weight: 0.1, ..Default::default() @@ -7736,8 +8214,8 @@ fn test_prune_type_scoped() { .unwrap(); let b = engine .upsert_node( - 2, - "type2-low", + "Company", + "label2-low", UpsertNodeOptions { weight: 0.1, ..Default::default() @@ -7745,18 +8223,18 @@ fn test_prune_type_scoped() { ) .unwrap(); - // Prune only type 1 with weight <= 0.5 + // Prune only Person-labeled nodes with weight <= 0.5. let result = engine .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(1), + label: Some("Person".to_string()), }) .unwrap(); assert_eq!(result.nodes_pruned, 1); - assert!(engine.get_node(a).unwrap().is_none()); // type 1, low → pruned - assert!(engine.get_node(b).unwrap().is_some()); // type 2 → not in scope + assert!(engine.get_node(a).unwrap().is_none()); // Person label, low -> pruned + assert!(engine.get_node(b).unwrap().is_some()); // Company label -> not in scope engine.close().unwrap(); } @@ -7767,7 +8245,7 @@ fn test_prune_cascade_deletes_edges() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -7777,7 +8255,7 @@ fn test_prune_cascade_deletes_edges() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -7787,7 +8265,7 @@ fn test_prune_cascade_deletes_edges() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.1, @@ -7796,13 +8274,13 @@ fn test_prune_cascade_deletes_edges() { ) .unwrap(); let e_ab = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e_bc = engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e_ca = engine - .upsert_edge(c, a, 1, UpsertEdgeOptions::default()) + .upsert_edge(c, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // Prune low-weight nodes (a and c) @@ -7810,7 +8288,7 @@ fn test_prune_cascade_deletes_edges() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -7833,7 +8311,7 @@ fn test_prune_shared_edge_dedup() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -7843,7 +8321,7 @@ fn test_prune_shared_edge_dedup() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.1, @@ -7852,14 +8330,14 @@ fn test_prune_shared_edge_dedup() { ) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = engine .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -7876,7 +8354,7 @@ fn test_prune_empty_result_no_match() { engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.9, @@ -7886,7 +8364,7 @@ fn test_prune_empty_result_no_match() { .unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.8, @@ -7900,7 +8378,7 @@ fn test_prune_empty_result_no_match() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.1), - type_id: None, + label: None, }) .unwrap(); @@ -7916,7 +8394,7 @@ fn test_prune_after_flush_segment_nodes() { let a = engine .upsert_node( - 1, + "Person", "seg-a", UpsertNodeOptions { weight: 0.1, @@ -7926,7 +8404,7 @@ fn test_prune_after_flush_segment_nodes() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "seg-b", UpsertNodeOptions { weight: 0.9, @@ -7935,7 +8413,7 @@ fn test_prune_after_flush_segment_nodes() { ) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -7944,7 +8422,7 @@ fn test_prune_after_flush_segment_nodes() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -7964,7 +8442,7 @@ fn test_prune_cross_source_memtable_and_segment() { // Node in segment let a = engine .upsert_node( - 1, + "Person", "in-seg", UpsertNodeOptions { weight: 0.1, @@ -7977,7 +8455,7 @@ fn test_prune_cross_source_memtable_and_segment() { // Node in memtable let b = engine .upsert_node( - 1, + "Person", "in-mem", UpsertNodeOptions { weight: 0.1, @@ -7988,14 +8466,14 @@ fn test_prune_cross_source_memtable_and_segment() { // Edge from memtable node to segment node let e = engine - .upsert_edge(b, a, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = engine .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -8015,7 +8493,7 @@ fn test_prune_cascade_edges_in_segment() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.9, @@ -8025,7 +8503,7 @@ fn test_prune_cascade_edges_in_segment() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.1, @@ -8034,14 +8512,14 @@ fn test_prune_cascade_edges_in_segment() { ) .unwrap(); let e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // Update b in memtable (still low weight) engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.1, @@ -8054,7 +8532,7 @@ fn test_prune_cascade_edges_in_segment() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -8076,7 +8554,7 @@ fn test_prune_survives_wal_replay() { let engine = open_imm(&db_path); a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -8086,7 +8564,7 @@ fn test_prune_survives_wal_replay() { .unwrap(); b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -8095,14 +8573,14 @@ fn test_prune_survives_wal_replay() { ) .unwrap(); e = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = engine .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); assert_eq!(result.nodes_pruned, 1); @@ -8126,7 +8604,7 @@ fn test_prune_weight_boundary() { let a = engine .upsert_node( - 1, + "Person", "exact", UpsertNodeOptions { weight: 0.5, @@ -8136,7 +8614,7 @@ fn test_prune_weight_boundary() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "above", UpsertNodeOptions { weight: 0.500001, @@ -8149,7 +8627,7 @@ fn test_prune_weight_boundary() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -8166,7 +8644,7 @@ fn test_prune_already_deleted_node_ignored() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -8181,7 +8659,7 @@ fn test_prune_already_deleted_node_ignored() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -8199,7 +8677,7 @@ fn test_prune_empty_db() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); @@ -8213,13 +8691,13 @@ fn test_prune_negative_age_rejected() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let result = engine.prune(&PrunePolicy { max_age_ms: Some(-100), max_weight: None, - type_id: None, + label: None, }); assert!(result.is_err()); @@ -8236,7 +8714,7 @@ fn test_prune_zero_age_rejected() { let result = engine.prune(&PrunePolicy { max_age_ms: Some(0), max_weight: None, - type_id: None, + label: None, }); assert!(result.is_err()); @@ -8244,48 +8722,46 @@ fn test_prune_zero_age_rejected() { } #[test] -fn test_prune_type_scoped_with_age() { +fn test_prune_label_scoped_with_age() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "t1-old", UpsertNodeOptions::default()) + .upsert_node("Person", "t1-old", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(2, "t2-old", UpsertNodeOptions::default()) + .upsert_node("Company", "t2-old", UpsertNodeOptions::default()) .unwrap(); let _c = engine - .upsert_node(1, "t1-new", UpsertNodeOptions::default()) + .upsert_node("Person", "t1-new", UpsertNodeOptions::default()) .unwrap(); // Make a and b old - let node_a = engine.get_node(a).unwrap().unwrap(); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { + let node_a = internal_node_record(&engine, a).unwrap().unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { updated_at: 1000, ..node_a })) .unwrap(); - let node_b = engine.get_node(b).unwrap().unwrap(); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { + let node_b = internal_node_record(&engine, b).unwrap().unwrap(); + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { updated_at: 1000, ..node_b })) .unwrap(); - // Prune old nodes of type 1 only + // Prune old Person-labeled nodes only. let result = engine .prune(&PrunePolicy { max_age_ms: Some(1000), max_weight: None, - type_id: Some(1), + label: Some("Person".to_string()), }) .unwrap(); assert_eq!(result.nodes_pruned, 1); // only t1-old - assert!(engine.get_node(a).unwrap().is_none()); // type 1 + old → pruned - assert!(engine.get_node(b).unwrap().is_some()); // type 2 → out of scope + assert!(engine.get_node(a).unwrap().is_none()); // Person label + old -> pruned + assert!(engine.get_node(b).unwrap().is_some()); // Company label -> out of scope engine.close().unwrap(); } @@ -8308,7 +8784,7 @@ fn test_set_and_list_prune_policies() { let policy = PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }; engine .set_prune_policy("low-weight", policy.clone()) @@ -8316,20 +8792,20 @@ fn test_set_and_list_prune_policies() { let list = engine.list_prune_policies().unwrap(); assert_eq!(list.len(), 1); - assert_eq!(list[0].0, "low-weight"); - assert_eq!(list[0].1.max_weight, Some(0.5)); + assert_eq!(list[0].name, "low-weight"); + assert_eq!(list[0].policy.max_weight, Some(0.5)); // Overwrite let policy2 = PrunePolicy { max_age_ms: Some(60_000), max_weight: None, - type_id: None, + label: None, }; engine.set_prune_policy("low-weight", policy2).unwrap(); let list = engine.list_prune_policies().unwrap(); assert_eq!(list.len(), 1); - assert_eq!(list[0].1.max_age_ms, Some(60_000)); - assert!(list[0].1.max_weight.is_none()); + assert_eq!(list[0].policy.max_age_ms, Some(60_000)); + assert!(list[0].policy.max_weight.is_none()); engine.close().unwrap(); } @@ -8348,7 +8824,7 @@ fn test_remove_prune_policy() { PrunePolicy { max_age_ms: None, max_weight: Some(0.3), - type_id: None, + label: None, }, ) .unwrap(); @@ -8377,18 +8853,18 @@ fn test_prune_policy_validation() { PrunePolicy { max_age_ms: None, max_weight: None, - type_id: None, + label: None, }, ); assert!(err.is_err()); - // type_id only rejected + // Label only rejected. let err = engine.set_prune_policy( "bad", PrunePolicy { max_age_ms: None, max_weight: None, - type_id: Some(1), + label: Some("Person".to_string()), }, ); assert!(err.is_err()); @@ -8399,7 +8875,7 @@ fn test_prune_policy_validation() { PrunePolicy { max_age_ms: Some(-1), max_weight: None, - type_id: None, + label: None, }, ); assert!(err.is_err()); @@ -8410,7 +8886,7 @@ fn test_prune_policy_validation() { PrunePolicy { max_age_ms: None, max_weight: Some(f32::NAN), - type_id: None, + label: None, }, ); assert!(err.is_err()); @@ -8421,7 +8897,7 @@ fn test_prune_policy_validation() { PrunePolicy { max_age_ms: None, max_weight: Some(-0.1), - type_id: None, + label: None, }, ); assert!(err.is_err()); @@ -8444,7 +8920,7 @@ fn test_prune_policy_survives_close_reopen() { PrunePolicy { max_age_ms: Some(30_000), max_weight: None, - type_id: None, + label: None, }, ) .unwrap(); @@ -8454,7 +8930,7 @@ fn test_prune_policy_survives_close_reopen() { PrunePolicy { max_age_ms: None, max_weight: Some(0.1), - type_id: Some(5), + label: Some("City".to_string()), }, ) .unwrap(); @@ -8466,11 +8942,11 @@ fn test_prune_policy_survives_close_reopen() { let list = engine.list_prune_policies().unwrap(); assert_eq!(list.len(), 2); // BTreeMap ordering: "age-rule" < "weight-rule" - assert_eq!(list[0].0, "age-rule"); - assert_eq!(list[0].1.max_age_ms, Some(30_000)); - assert_eq!(list[1].0, "weight-rule"); - assert_eq!(list[1].1.max_weight, Some(0.1)); - assert_eq!(list[1].1.type_id, Some(5)); + assert_eq!(list[0].name, "age-rule"); + assert_eq!(list[0].policy.max_age_ms, Some(30_000)); + assert_eq!(list[1].name, "weight-rule"); + assert_eq!(list[1].policy.max_weight, Some(0.1)); + assert_eq!(list[1].policy.label, Some("City".to_string())); engine.close().unwrap(); } @@ -8486,7 +8962,7 @@ fn test_compaction_auto_prune_by_weight() { // Create nodes with different weights let low = engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.1, @@ -8496,7 +8972,7 @@ fn test_compaction_auto_prune_by_weight() { .unwrap(); let high = engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -8510,7 +8986,7 @@ fn test_compaction_auto_prune_by_weight() { // IDs across segments, which forces the standard compaction path. let low2 = engine .upsert_node( - 1, + "Person", "low2", UpsertNodeOptions { weight: 0.2, @@ -8520,7 +8996,7 @@ fn test_compaction_auto_prune_by_weight() { .unwrap(); let high2 = engine .upsert_node( - 1, + "Person", "high2", UpsertNodeOptions { weight: 0.8, @@ -8530,7 +9006,7 @@ fn test_compaction_auto_prune_by_weight() { .unwrap(); engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -8547,7 +9023,7 @@ fn test_compaction_auto_prune_by_weight() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -8578,7 +9054,7 @@ fn test_compaction_auto_prune_cascade_edges() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -8588,7 +9064,7 @@ fn test_compaction_auto_prune_cascade_edges() { .unwrap(); // will be pruned let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -8598,7 +9074,7 @@ fn test_compaction_auto_prune_cascade_edges() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.9, @@ -8607,10 +9083,10 @@ fn test_compaction_auto_prune_cascade_edges() { ) .unwrap(); let e1 = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e2 = engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -8618,7 +9094,7 @@ fn test_compaction_auto_prune_cascade_edges() { // across segments, which forces the standard compaction path. let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.8, @@ -8628,7 +9104,7 @@ fn test_compaction_auto_prune_cascade_edges() { .unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -8644,7 +9120,7 @@ fn test_compaction_auto_prune_cascade_edges() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -8675,10 +9151,10 @@ fn test_compaction_multiple_policies_or_logic() { opts.compact_after_n_flushes = 0; let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - // Node that matches policy A (low weight) but not B (type 99) + // Node that matches policy A (low weight) but not B (MissingLabel) let n1 = engine .upsert_node( - 1, + "Person", "n1", UpsertNodeOptions { weight: 0.1, @@ -8686,10 +9162,10 @@ fn test_compaction_multiple_policies_or_logic() { }, ) .unwrap(); - // Node that matches policy B (type 99) but not A (high weight) + // Node that matches policy B (MissingLabel) but not A (high weight) let n2 = engine .upsert_node( - 99, + "MissingLabel", "n2", UpsertNodeOptions { weight: 0.9, @@ -8700,7 +9176,7 @@ fn test_compaction_multiple_policies_or_logic() { // Node that matches neither let n3 = engine .upsert_node( - 1, + "Person", "n3", UpsertNodeOptions { weight: 0.9, @@ -8713,7 +9189,7 @@ fn test_compaction_multiple_policies_or_logic() { // Update n3 in second segment to create overlapping IDs (forces standard path) engine .upsert_node( - 1, + "Person", "n3", UpsertNodeOptions { weight: 0.9, @@ -8730,24 +9206,24 @@ fn test_compaction_multiple_policies_or_logic() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); - // Policy B: prune all nodes of type 99 (by weight, use very high threshold) + // Policy B: prune all MissingLabel nodes (by weight, use very high threshold) engine .set_prune_policy( "type-99", PrunePolicy { max_age_ms: None, max_weight: Some(999.0), - type_id: Some(99), + label: Some("MissingLabel".to_string()), }, ) .unwrap(); let stats = engine.compact().unwrap().unwrap(); - assert_eq!(stats.nodes_auto_pruned, 2); // n1 (low weight) + n2 (type 99) + assert_eq!(stats.nodes_auto_pruned, 2); // n1 (low weight) + n2 (MissingLabel) assert!(engine.get_node(n1).unwrap().is_none()); assert!(engine.get_node(n2).unwrap().is_none()); @@ -8767,7 +9243,7 @@ fn test_compaction_no_policies_no_prune() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -8777,7 +9253,7 @@ fn test_compaction_no_policies_no_prune() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -8788,7 +9264,7 @@ fn test_compaction_no_policies_no_prune() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -8825,14 +9301,14 @@ fn test_removed_policy_no_longer_prunes() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -8843,7 +9319,7 @@ fn test_removed_policy_no_longer_prunes() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -8853,7 +9329,7 @@ fn test_removed_policy_no_longer_prunes() { .unwrap(); engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, @@ -8876,7 +9352,7 @@ fn test_removed_policy_no_longer_prunes() { } #[test] -fn test_compaction_type_scoped_policy() { +fn test_compaction_label_scoped_policy() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let mut opts = DbOptions::default(); @@ -8886,7 +9362,7 @@ fn test_compaction_type_scoped_policy() { let t1_low = engine .upsert_node( - 1, + "Person", "t1-low", UpsertNodeOptions { weight: 0.1, @@ -8896,7 +9372,7 @@ fn test_compaction_type_scoped_policy() { .unwrap(); let t2_low = engine .upsert_node( - 2, + "Company", "t2-low", UpsertNodeOptions { weight: 0.1, @@ -8906,7 +9382,7 @@ fn test_compaction_type_scoped_policy() { .unwrap(); let t1_high = engine .upsert_node( - 1, + "Person", "t1-high", UpsertNodeOptions { weight: 0.9, @@ -8918,7 +9394,7 @@ fn test_compaction_type_scoped_policy() { // Update t1_high in second segment to create overlapping IDs (forces standard path) engine .upsert_node( - 1, + "Person", "t1-high", UpsertNodeOptions { weight: 0.9, @@ -8928,14 +9404,14 @@ fn test_compaction_type_scoped_policy() { .unwrap(); engine.flush().unwrap(); - // Only prune type 1 with low weight + // Only prune Person-labeled nodes with low weight. engine .set_prune_policy( - "type1-low", + "label1-low", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(1), + label: Some("Person".to_string()), }, ) .unwrap(); @@ -8944,8 +9420,8 @@ fn test_compaction_type_scoped_policy() { assert_eq!(stats.nodes_auto_pruned, 1); // only t1_low assert!(engine.get_node(t1_low).unwrap().is_none()); - assert!(engine.get_node(t2_low).unwrap().is_some()); // type 2, out of scope - assert!(engine.get_node(t1_high).unwrap().is_some()); // type 1 but high weight + assert!(engine.get_node(t2_low).unwrap().is_some()); // Company label, out of scope + assert!(engine.get_node(t1_high).unwrap().is_some()); // Person label but high weight engine.close().unwrap(); } @@ -8962,7 +9438,7 @@ fn test_compaction_prune_stats_in_nodes_removed() { for i in 0..10 { engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { weight: 0.1, @@ -8975,7 +9451,7 @@ fn test_compaction_prune_stats_in_nodes_removed() { for i in 10..20 { engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { weight: 0.9, @@ -8987,7 +9463,7 @@ fn test_compaction_prune_stats_in_nodes_removed() { // Update n0 in second segment to create overlapping IDs (forces standard path) engine .upsert_node( - 1, + "Person", "n0", UpsertNodeOptions { weight: 0.1, @@ -9003,7 +9479,7 @@ fn test_compaction_prune_stats_in_nodes_removed() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9033,14 +9509,14 @@ fn test_manual_prune_unchanged_by_policies() { PrunePolicy { max_age_ms: None, max_weight: Some(0.0001), - type_id: None, + label: None, }, ) .unwrap(); let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -9050,7 +9526,7 @@ fn test_manual_prune_unchanged_by_policies() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -9064,7 +9540,7 @@ fn test_manual_prune_unchanged_by_policies() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.7), - type_id: None, + label: None, }) .unwrap(); @@ -9092,14 +9568,14 @@ fn test_bg_compaction_applies_prune_policies() { PrunePolicy { max_age_ms: None, max_weight: Some(0.3), - type_id: None, + label: None, }, ) .unwrap(); let low = engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.1, @@ -9109,7 +9585,7 @@ fn test_bg_compaction_applies_prune_policies() { .unwrap(); let high = engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -9122,7 +9598,7 @@ fn test_bg_compaction_applies_prune_policies() { // Second flush with overlapping ID to force standard path in bg compaction engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -9155,7 +9631,7 @@ fn test_read_time_policy_get_node() { let low = engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.2, @@ -9165,7 +9641,7 @@ fn test_read_time_policy_get_node() { .unwrap(); let high = engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -9185,7 +9661,7 @@ fn test_read_time_policy_get_node() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9208,7 +9684,7 @@ fn test_read_time_policy_get_node_by_key() { engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.2, @@ -9218,7 +9694,7 @@ fn test_read_time_policy_get_node_by_key() { .unwrap(); engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -9234,15 +9710,15 @@ fn test_read_time_policy_get_node_by_key() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); // Low hidden by policy - assert!(engine.get_node_by_key(1, "low").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "low").unwrap().is_none()); // High still visible - assert!(engine.get_node_by_key(1, "high").unwrap().is_some()); + assert!(engine.get_node_by_key("Person", "high").unwrap().is_some()); engine.close().unwrap(); } @@ -9258,7 +9734,7 @@ fn test_read_time_policy_get_nodes_batch() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.2, @@ -9268,7 +9744,7 @@ fn test_read_time_policy_get_nodes_batch() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -9278,7 +9754,7 @@ fn test_read_time_policy_get_nodes_batch() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.3, @@ -9293,7 +9769,7 @@ fn test_read_time_policy_get_nodes_batch() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9318,7 +9794,7 @@ fn test_read_time_policy_get_node_after_flush() { let low = engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.2, @@ -9328,7 +9804,7 @@ fn test_read_time_policy_get_node_after_flush() { .unwrap(); let high = engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -9344,7 +9820,7 @@ fn test_read_time_policy_get_node_after_flush() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9367,7 +9843,7 @@ fn test_read_time_policy_get_node_by_key_after_flush() { engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.2, @@ -9377,7 +9853,7 @@ fn test_read_time_policy_get_node_by_key_after_flush() { .unwrap(); engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 0.9, @@ -9393,13 +9869,13 @@ fn test_read_time_policy_get_node_by_key_after_flush() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); - assert!(engine.get_node_by_key(1, "low").unwrap().is_none()); - assert!(engine.get_node_by_key(1, "high").unwrap().is_some()); + assert!(engine.get_node_by_key("Person", "low").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "high").unwrap().is_some()); engine.close().unwrap(); } @@ -9419,7 +9895,7 @@ fn test_read_time_policy_upsert_dedup_unaffected() { let id1 = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.2, @@ -9435,7 +9911,7 @@ fn test_read_time_policy_upsert_dedup_unaffected() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9443,10 +9919,10 @@ fn test_read_time_policy_upsert_dedup_unaffected() { // Public read confirms it's hidden assert!(engine.get_node(id1).unwrap().is_none()); - // Upsert same (type_id, key). MUST reuse existing ID, not allocate new one + // Upsert same (label, key). MUST reuse existing ID, not allocate new one. let id2 = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.8, @@ -9477,7 +9953,7 @@ fn test_read_time_policy_upsert_dedup_after_flush() { let id1 = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.2, @@ -9493,7 +9969,7 @@ fn test_read_time_policy_upsert_dedup_after_flush() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9504,7 +9980,7 @@ fn test_read_time_policy_upsert_dedup_after_flush() { // Upsert must still find and reuse the existing ID from segment let id2 = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.8, @@ -9528,7 +10004,7 @@ fn test_read_time_policy_add_remove_takes_effect() { let id = engine .upsert_node( - 1, + "Person", "target", UpsertNodeOptions { weight: 0.3, @@ -9547,7 +10023,7 @@ fn test_read_time_policy_add_remove_takes_effect() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9561,7 +10037,7 @@ fn test_read_time_policy_add_remove_takes_effect() { } #[test] -fn test_read_time_policy_type_scoped() { +fn test_read_time_policy_label_scoped() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let mut opts = DbOptions::default(); @@ -9571,7 +10047,7 @@ fn test_read_time_policy_type_scoped() { let t1 = engine .upsert_node( - 1, + "Person", "t1-low", UpsertNodeOptions { weight: 0.2, @@ -9581,7 +10057,7 @@ fn test_read_time_policy_type_scoped() { .unwrap(); let t2 = engine .upsert_node( - 2, + "Company", "t2-low", UpsertNodeOptions { weight: 0.2, @@ -9590,19 +10066,19 @@ fn test_read_time_policy_type_scoped() { ) .unwrap(); - // Policy scoped to type_id=1 only + // Policy scoped to the Person label only. engine .set_prune_policy( "t1-only", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(1), + label: Some("Person".to_string()), }, ) .unwrap(); - // Type 1 node hidden, type 2 node still visible + // Person-labeled node hidden, Company-labeled node still visible. assert!(engine.get_node(t1).unwrap().is_none()); assert!(engine.get_node(t2).unwrap().is_some()); @@ -9621,7 +10097,7 @@ fn test_read_time_policy_no_policies_zero_overhead() { let id = engine .upsert_node( - 1, + "Person", "node", UpsertNodeOptions { weight: 0.1, @@ -9633,7 +10109,7 @@ fn test_read_time_policy_no_policies_zero_overhead() { // No policies registered, everything visible assert!(engine.list_prune_policies().unwrap().is_empty()); assert!(engine.get_node(id).unwrap().is_some()); - assert!(engine.get_node_by_key(1, "node").unwrap().is_some()); + assert!(engine.get_node_by_key("Person", "node").unwrap().is_some()); let batch = engine.get_nodes(&[id]).unwrap(); assert!(batch[0].is_some()); @@ -9652,63 +10128,63 @@ fn test_read_time_policy_multiple_policies_or() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.1, ..Default::default() }, ) - .unwrap(); // type 1, low weight + .unwrap(); // Person label, low weight let b = engine .upsert_node( - 2, + "Company", "b", UpsertNodeOptions { weight: 0.1, ..Default::default() }, ) - .unwrap(); // type 2, low weight + .unwrap(); // Company label, low weight let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.9, ..Default::default() }, ) - .unwrap(); // type 1, high weight + .unwrap(); // Person label, high weight - // Policy 1: type 1, weight <= 0.5 + // Policy 1: Person label, weight <= 0.5 engine .set_prune_policy( "p1", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(1), + label: Some("Person".to_string()), }, ) .unwrap(); - // Policy 2: type 2, weight <= 0.5 + // Policy 2: Company label, weight <= 0.5 engine .set_prune_policy( "p2", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(2), + label: Some("Company".to_string()), }, ) .unwrap(); - // a (type 1, 0.1): matches p1 → hidden + // a (Person, 0.1): matches p1 -> hidden assert!(engine.get_node(a).unwrap().is_none()); - // b (type 2, 0.1): matches p2 → hidden + // b (Company, 0.1): matches p2 -> hidden assert!(engine.get_node(b).unwrap().is_none()); - // c (type 1, 0.9): doesn't match p1 (weight too high), doesn't match p2 (wrong type) → visible + // c (Person, 0.9): doesn't match p1 (weight too high), doesn't match p2 (wrong label) -> visible assert!(engine.get_node(c).unwrap().is_some()); engine.close().unwrap(); @@ -9726,7 +10202,7 @@ fn test_read_time_policy_graph_patch_dedup_unaffected() { let id1 = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.2, @@ -9741,7 +10217,7 @@ fn test_read_time_policy_graph_patch_dedup_unaffected() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9749,7 +10225,7 @@ fn test_read_time_policy_graph_patch_dedup_unaffected() { // Use graph_patch to upsert same node. Must reuse ID let patch = GraphPatch { upsert_nodes: vec![NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: "node-a".to_string(), props: BTreeMap::new(), weight: 0.8, @@ -9761,7 +10237,7 @@ fn test_read_time_policy_graph_patch_dedup_unaffected() { delete_node_ids: Vec::new(), delete_edge_ids: Vec::new(), }; - let result = engine.graph_patch(&patch).unwrap(); + let result = engine.graph_patch(patch).unwrap(); assert_eq!( result.node_ids[0], id1, "graph_patch must reuse existing node ID" @@ -9781,7 +10257,7 @@ fn test_read_time_policy_neighbors() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.9, @@ -9791,7 +10267,7 @@ fn test_read_time_policy_neighbors() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.2, @@ -9801,7 +10277,7 @@ fn test_read_time_policy_neighbors() { .unwrap(); // will be excluded let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.8, @@ -9811,10 +10287,10 @@ fn test_read_time_policy_neighbors() { .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // No policy: both neighbors visible @@ -9828,7 +10304,7 @@ fn test_read_time_policy_neighbors() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9854,7 +10330,7 @@ fn test_read_time_policy_neighbors_limit() { let hub = engine .upsert_node( - 1, + "Person", "hub", UpsertNodeOptions { weight: 0.9, @@ -9867,7 +10343,7 @@ fn test_read_time_policy_neighbors_limit() { for i in 0..3 { let id = engine .upsert_node( - 1, + "Person", &format!("hi-{}", i), UpsertNodeOptions { weight: 0.8, @@ -9876,14 +10352,14 @@ fn test_read_time_policy_neighbors_limit() { ) .unwrap(); engine - .upsert_edge(hub, id, 1, UpsertEdgeOptions::default()) + .upsert_edge(hub, id, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); visible_ids.push(id); } for i in 0..2 { let id = engine .upsert_node( - 1, + "Person", &format!("lo-{}", i), UpsertNodeOptions { weight: 0.1, @@ -9892,7 +10368,7 @@ fn test_read_time_policy_neighbors_limit() { ) .unwrap(); engine - .upsert_edge(hub, id, 1, UpsertEdgeOptions::default()) + .upsert_edge(hub, id, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); } @@ -9902,7 +10378,7 @@ fn test_read_time_policy_neighbors_limit() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -9938,7 +10414,7 @@ fn test_read_time_policy_traverse_depth_two() { // a -> b -> c (c has low weight, should be excluded) let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.9, @@ -9948,7 +10424,7 @@ fn test_read_time_policy_traverse_depth_two() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -9958,7 +10434,7 @@ fn test_read_time_policy_traverse_depth_two() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.2, @@ -9968,7 +10444,7 @@ fn test_read_time_policy_traverse_depth_two() { .unwrap(); // will be excluded let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.9, @@ -9978,13 +10454,13 @@ fn test_read_time_policy_traverse_depth_two() { .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(b, d, 1, UpsertEdgeOptions::default()) + .upsert_edge(b, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine @@ -9993,7 +10469,7 @@ fn test_read_time_policy_traverse_depth_two() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -10018,7 +10494,7 @@ fn test_read_time_policy_top_k() { let hub = engine .upsert_node( - 1, + "Person", "hub", UpsertNodeOptions { weight: 0.9, @@ -10028,7 +10504,7 @@ fn test_read_time_policy_top_k() { .unwrap(); let hi = engine .upsert_node( - 1, + "Person", "hi", UpsertNodeOptions { weight: 0.8, @@ -10038,7 +10514,7 @@ fn test_read_time_policy_top_k() { .unwrap(); let lo = engine .upsert_node( - 1, + "Person", "lo", UpsertNodeOptions { weight: 0.2, @@ -10051,7 +10527,7 @@ fn test_read_time_policy_top_k() { .upsert_edge( hub, hi, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 5.0, ..Default::default() @@ -10062,7 +10538,7 @@ fn test_read_time_policy_top_k() { .upsert_edge( hub, lo, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 10.0, ..Default::default() @@ -10076,7 +10552,7 @@ fn test_read_time_policy_top_k() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -10102,7 +10578,7 @@ fn test_read_time_policy_extract_subgraph() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.9, @@ -10112,7 +10588,7 @@ fn test_read_time_policy_extract_subgraph() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.8, @@ -10122,7 +10598,7 @@ fn test_read_time_policy_extract_subgraph() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.2, @@ -10132,10 +10608,10 @@ fn test_read_time_policy_extract_subgraph() { .unwrap(); // excluded engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine @@ -10144,7 +10620,7 @@ fn test_read_time_policy_extract_subgraph() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -10161,7 +10637,7 @@ fn test_read_time_policy_extract_subgraph() { } #[test] -fn test_read_time_policy_nodes_by_type() { +fn test_read_time_policy_nodes_by_label_id() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let mut opts = DbOptions::default(); @@ -10171,7 +10647,7 @@ fn test_read_time_policy_nodes_by_type() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.2, @@ -10181,7 +10657,7 @@ fn test_read_time_policy_nodes_by_type() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -10196,12 +10672,12 @@ fn test_read_time_policy_nodes_by_type() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); - let ids = engine.nodes_by_type(1).unwrap(); + let ids = engine.nodes_by_labels("Person").unwrap(); assert!(!ids.contains(&a)); // excluded assert!(ids.contains(&b)); // visible @@ -10222,7 +10698,7 @@ fn test_read_time_policy_find_nodes() { engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: props.clone(), @@ -10233,7 +10709,7 @@ fn test_read_time_policy_find_nodes() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props: props.clone(), @@ -10249,13 +10725,13 @@ fn test_read_time_policy_find_nodes() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let ids = engine - .find_nodes(1, "color", &PropValue::String("red".to_string())) + .find_nodes("Person", "color", &PropValue::String("red".to_string())) .unwrap(); // Only b visible (a excluded by policy) assert_eq!(ids.len(), 1); @@ -10277,7 +10753,7 @@ fn test_read_time_policy_prune_still_works() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.2, @@ -10287,7 +10763,7 @@ fn test_read_time_policy_prune_still_works() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.9, @@ -10297,7 +10773,7 @@ fn test_read_time_policy_prune_still_works() { .unwrap(); // Edge between them for cascade testing engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); // Register policy that hides 'a' from reads @@ -10307,7 +10783,7 @@ fn test_read_time_policy_prune_still_works() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -10320,7 +10796,7 @@ fn test_read_time_policy_prune_still_works() { .prune(&PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }) .unwrap(); assert_eq!(result.nodes_pruned, 1); @@ -10334,6 +10810,197 @@ fn test_read_time_policy_prune_still_works() { engine.close().unwrap(); } +#[test] +fn test_multi_label_manual_prune_policy_membership_and_edge_cascade_once() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let mut opts = DbOptions::default(); + opts.wal_sync_mode = WalSyncMode::Immediate; + opts.compact_after_n_flushes = 0; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let prune = engine + .upsert_node( + &["PruneMemberA", "PruneMemberB"], + "prune", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + let keep_same_weight = engine + .upsert_node( + "PruneMemberA", + "keep-same-weight", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + let keep_other = engine + .upsert_node( + "PruneOther", + "keep-other", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + let edge = engine + .upsert_edge( + prune, + keep_other, + "PRUNE_MEMBER_EDGE", + UpsertEdgeOptions::default(), + ) + .unwrap(); + + let result = engine + .prune(&PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("PruneMemberB".to_string()), + }) + .unwrap(); + assert_eq!(result.nodes_pruned, 1); + assert_eq!(result.edges_pruned, 1); + assert!(engine.get_node(prune).unwrap().is_none()); + assert!(engine.get_edge(edge).unwrap().is_none()); + assert!(engine.get_node(keep_same_weight).unwrap().is_some()); + assert!(engine.get_node(keep_other).unwrap().is_some()); + engine.close().unwrap(); +} + +#[test] +fn test_multi_label_prune_stale_membership_suppressed_across_sources() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let mut opts = DbOptions::default(); + opts.wal_sync_mode = WalSyncMode::Immediate; + opts.compact_after_n_flushes = 0; + + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let policy = PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("StalePruneLabel".to_string()), + }; + + engine + .upsert_node( + &["StalePruneLabel", "StaleKeepLabel"], + "active", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let active_id = engine + .upsert_node( + "StaleKeepLabel", + "active", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(engine.prune(&policy).unwrap().nodes_pruned, 0); + assert!(engine.get_node(active_id).unwrap().is_some()); + + engine + .upsert_node( + &["StalePruneLabel", "StaleKeepLabel"], + "immutable", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let immutable_id = engine + .upsert_node( + "StaleKeepLabel", + "immutable", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + engine.freeze_memtable().unwrap(); + assert_eq!(engine.prune(&policy).unwrap().nodes_pruned, 0); + assert!(engine.get_node(immutable_id).unwrap().is_some()); + + engine + .upsert_node( + &["StalePruneLabel", "StaleKeepLabel"], + "flushed", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + let flushed_id = engine + .upsert_node( + "StaleKeepLabel", + "flushed", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + assert_eq!(engine.prune(&policy).unwrap().nodes_pruned, 0); + assert!(engine.get_node(flushed_id).unwrap().is_some()); + engine.close().unwrap(); + } + + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let policy = PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("StalePruneLabel".to_string()), + }; + assert_eq!(engine.prune(&policy).unwrap().nodes_pruned, 0); + assert!(engine + .get_node_by_key("StaleKeepLabel", "active") + .unwrap() + .is_some()); + assert!(engine + .get_node_by_key("StaleKeepLabel", "immutable") + .unwrap() + .is_some()); + assert!(engine + .get_node_by_key("StaleKeepLabel", "flushed") + .unwrap() + .is_some()); + let stats = engine + .compact() + .unwrap() + .expect("stale prune test must exercise compacted sources"); + assert!(stats.segments_merged > 1); + assert_eq!(engine.prune(&policy).unwrap().nodes_pruned, 0); + assert!(engine + .get_nodes_by_labels("StaleKeepLabel") + .unwrap() + .len() + >= 3); + engine.close().unwrap(); + } +} + #[test] fn test_read_time_policy_delete_node_cascade_unaffected() { // delete_node must cascade-delete ALL incident edges, even those to @@ -10347,7 +11014,7 @@ fn test_read_time_policy_delete_node_cascade_unaffected() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.9, @@ -10357,7 +11024,7 @@ fn test_read_time_policy_delete_node_cascade_unaffected() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.2, @@ -10366,7 +11033,7 @@ fn test_read_time_policy_delete_node_cascade_unaffected() { ) .unwrap(); // will be policy-excluded let edge_id = engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine @@ -10375,7 +11042,7 @@ fn test_read_time_policy_delete_node_cascade_unaffected() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -10400,7 +11067,7 @@ fn test_read_time_policy_neighbors_after_flush() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.9, @@ -10410,7 +11077,7 @@ fn test_read_time_policy_neighbors_after_flush() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.2, @@ -10420,7 +11087,7 @@ fn test_read_time_policy_neighbors_after_flush() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.8, @@ -10430,10 +11097,10 @@ fn test_read_time_policy_neighbors_after_flush() { .unwrap(); engine - .upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -10443,7 +11110,7 @@ fn test_read_time_policy_neighbors_after_flush() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -10471,7 +11138,7 @@ fn test_close_fast_basic() { let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); let id = engine - .upsert_node(1, "n1", UpsertNodeOptions::default()) + .upsert_node("Person", "n1", UpsertNodeOptions::default()) .unwrap(); engine.close_fast().unwrap(); @@ -10498,7 +11165,7 @@ fn test_close_fast_cancels_bg_compact() { for j in 0..100 { let key = format!("node_{}_{}", i, j); engine - .upsert_node(1, &key, UpsertNodeOptions::default()) + .upsert_node("Person", &key, UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -10515,8 +11182,8 @@ fn test_close_fast_cancels_bg_compact() { let engine2 = DatabaseEngine::open(dir.path(), &opts).unwrap(); let stats = engine2.stats().unwrap(); // All nodes should still be accessible - assert!(engine2.get_node_by_key(1, "node_0_0").unwrap().is_some()); - assert!(engine2.get_node_by_key(1, "node_2_99").unwrap().is_some()); + assert!(engine2.get_node_by_key("Person", "node_0_0").unwrap().is_some()); + assert!(engine2.get_node_by_key("Person", "node_2_99").unwrap().is_some()); // 3 segments still present (compaction was cancelled) assert!(stats.segment_count >= 1); // could be 3 or 1 if compaction finished fast engine2.close().unwrap(); @@ -10537,7 +11204,7 @@ fn test_close_fast_group_commit() { let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); let id = engine - .upsert_node(1, "gc_node", UpsertNodeOptions::default()) + .upsert_node("Person", "gc_node", UpsertNodeOptions::default()) .unwrap(); engine.close_fast().unwrap(); @@ -10611,13 +11278,13 @@ fn test_stats_segments_after_flush() { assert_eq!(engine.stats().unwrap().segment_count, 0); engine - .upsert_node(1, "n1", UpsertNodeOptions::default()) + .upsert_node("Person", "n1", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); assert_eq!(engine.stats().unwrap().segment_count, 1); engine - .upsert_node(1, "n2", UpsertNodeOptions::default()) + .upsert_node("Person", "n2", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); assert_eq!(engine.stats().unwrap().segment_count, 2); @@ -10636,13 +11303,13 @@ fn test_stats_tombstones() { let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); let n1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(n1, n2, 1, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); assert_eq!(engine.stats().unwrap().node_tombstone_count, 0); @@ -10676,11 +11343,11 @@ fn test_stats_last_compaction_ms() { // Create 2 segments and compact engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -10713,11 +11380,11 @@ fn test_stats_last_compaction_ms_bg() { // Create 2 segments engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -10753,7 +11420,7 @@ fn test_stats_pending_wal_bytes_group_commit() { // Write something. It should show as pending (sync interval hasn't fired) engine - .upsert_node(1, "buffered", UpsertNodeOptions::default()) + .upsert_node("Person", "buffered", UpsertNodeOptions::default()) .unwrap(); let stats = engine.stats().unwrap(); assert!(stats.pending_wal_bytes > 0, "should have buffered bytes"); @@ -10773,10 +11440,10 @@ fn test_stats_immutable_memtable_fields() { // Write some data engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let stats = engine.stats().unwrap(); @@ -10808,7 +11475,7 @@ fn test_stats_immutable_memtable_fields() { // Write more, freeze again engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.freeze_memtable().unwrap(); @@ -10850,20 +11517,20 @@ fn test_v3_planner_basic_winner_selection() { // Segment 1 (older): nodes with keys a, b, c engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); // Segment 2 (newer): update "a" with new weight, add "d" engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 2.0, @@ -10872,7 +11539,7 @@ fn test_v3_planner_basic_winner_selection() { ) .unwrap(); engine - .upsert_node(1, "d", UpsertNodeOptions::default()) + .upsert_node("Person", "d", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -10884,7 +11551,7 @@ fn test_v3_planner_basic_winner_selection() { assert_eq!(stats.nodes_kept, 4); // a, b, c, d // Verify the winner for "a" has the updated weight - let n = engine.get_node_by_key(1, "a").unwrap().unwrap(); + let n = engine.get_node_by_key("Person", "a").unwrap().unwrap(); assert_eq!(n.weight, 2.0); engine.close().unwrap(); @@ -10901,10 +11568,10 @@ fn test_v3_planner_tombstone_handling() { let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); let id1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let id2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -10932,19 +11599,19 @@ fn test_v3_planner_edge_cascade_on_tombstone() { let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); let n1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let n3 = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let e1 = engine - .upsert_edge(n1, n2, 1, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e2 = engine - .upsert_edge(n2, n3, 1, UpsertEdgeOptions::default()) + .upsert_edge(n2, n3, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -10975,7 +11642,7 @@ fn test_v3_planner_prune_policy_from_metadata() { // Create overlapping segments (overlapping IDs across segments) engine .upsert_node( - 1, + "Person", "low_weight", UpsertNodeOptions { weight: 0.1, @@ -10985,7 +11652,7 @@ fn test_v3_planner_prune_policy_from_metadata() { .unwrap(); engine .upsert_node( - 1, + "Person", "high_weight", UpsertNodeOptions { weight: 5.0, @@ -10997,7 +11664,7 @@ fn test_v3_planner_prune_policy_from_metadata() { // Update high_weight to create overlapping node ID across segments engine .upsert_node( - 1, + "Person", "high_weight", UpsertNodeOptions { weight: 5.0, @@ -11014,7 +11681,7 @@ fn test_v3_planner_prune_policy_from_metadata() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -11024,9 +11691,9 @@ fn test_v3_planner_prune_policy_from_metadata() { assert_eq!(stats.nodes_auto_pruned, 1); // Only high_weight node survives - let node = engine.get_node_by_key(1, "high_weight").unwrap(); + let node = engine.get_node_by_key("Person", "high_weight").unwrap(); assert!(node.is_some()); - let node = engine.get_node_by_key(1, "low_weight").unwrap(); + let node = engine.get_node_by_key("Person", "low_weight").unwrap(); assert!(node.is_none()); engine.close().unwrap(); @@ -11044,7 +11711,7 @@ fn test_v3_planner_prune_policy_edge_cascade() { let n1 = engine .upsert_node( - 1, + "Person", "keep", UpsertNodeOptions { weight: 5.0, @@ -11054,7 +11721,7 @@ fn test_v3_planner_prune_policy_edge_cascade() { .unwrap(); let n2 = engine .upsert_node( - 1, + "Person", "prune_me", UpsertNodeOptions { weight: 0.1, @@ -11064,7 +11731,7 @@ fn test_v3_planner_prune_policy_edge_cascade() { .unwrap(); let n3 = engine .upsert_node( - 1, + "Person", "also_keep", UpsertNodeOptions { weight: 5.0, @@ -11073,16 +11740,16 @@ fn test_v3_planner_prune_policy_edge_cascade() { ) .unwrap(); let _e1 = engine - .upsert_edge(n1, n2, 1, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let e2 = engine - .upsert_edge(n1, n3, 1, UpsertEdgeOptions::default()) + .upsert_edge(n1, n3, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); // Update a node to create overlapping IDs (forces V3 standard path) engine .upsert_node( - 1, + "Person", "keep", UpsertNodeOptions { weight: 5.0, @@ -11098,7 +11765,7 @@ fn test_v3_planner_prune_policy_edge_cascade() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -11124,10 +11791,10 @@ fn test_v3_planner_prune_policy_or_semantics() { }; let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); - // Type 1: low weight, type 2: low weight, type 3: safe + // Person: low weight, Company: low weight, Article: safe. engine .upsert_node( - 1, + "Person", "t1_low", UpsertNodeOptions { weight: 0.1, @@ -11137,7 +11804,7 @@ fn test_v3_planner_prune_policy_or_semantics() { .unwrap(); engine .upsert_node( - 1, + "Person", "t1_high", UpsertNodeOptions { weight: 5.0, @@ -11147,7 +11814,7 @@ fn test_v3_planner_prune_policy_or_semantics() { .unwrap(); engine .upsert_node( - 2, + "Company", "t2_low", UpsertNodeOptions { weight: 0.1, @@ -11157,7 +11824,7 @@ fn test_v3_planner_prune_policy_or_semantics() { .unwrap(); engine .upsert_node( - 3, + "Article", "t3_safe", UpsertNodeOptions { weight: 5.0, @@ -11169,7 +11836,7 @@ fn test_v3_planner_prune_policy_or_semantics() { // Update a node to create overlap (forces V3 standard path) engine .upsert_node( - 1, + "Person", "t1_high", UpsertNodeOptions { weight: 5.0, @@ -11182,22 +11849,22 @@ fn test_v3_planner_prune_policy_or_semantics() { // Policy A: prune type=1 with weight <= 0.5 engine .set_prune_policy( - "type1_low", + "label1_low", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(1), + label: Some("Person".to_string()), }, ) .unwrap(); // Policy B: prune type=2 with weight <= 0.5 engine .set_prune_policy( - "type2_low", + "label2_low", PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(2), + label: Some("Company".to_string()), }, ) .unwrap(); @@ -11222,7 +11889,7 @@ fn test_v3_planner_overlapping_multi_segment() { // Segment 1: nodes 1-50 for i in 0..50 { engine - .upsert_node(1, &format!("node_{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("node_{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -11231,7 +11898,7 @@ fn test_v3_planner_overlapping_multi_segment() { for i in 10..30 { engine .upsert_node( - 1, + "Person", &format!("node_{}", i), UpsertNodeOptions { weight: 2.0, @@ -11245,7 +11912,7 @@ fn test_v3_planner_overlapping_multi_segment() { // Segment 3: delete nodes 40-49 for i in 40..50 { let n = engine - .get_node_by_key(1, &format!("node_{}", i)) + .get_node_by_key("Person", &format!("node_{}", i)) .unwrap() .unwrap(); engine.delete_node(n.id).unwrap(); @@ -11259,7 +11926,7 @@ fn test_v3_planner_overlapping_multi_segment() { // Verify updated nodes have new weight for i in 10..30 { let n = engine - .get_node_by_key(1, &format!("node_{}", i)) + .get_node_by_key("Person", &format!("node_{}", i)) .unwrap() .unwrap(); assert_eq!(n.weight, 2.0, "node_{} should have updated weight", i); @@ -11267,7 +11934,7 @@ fn test_v3_planner_overlapping_multi_segment() { // Verify deleted nodes are gone for i in 40..50 { - let n = engine.get_node_by_key(1, &format!("node_{}", i)).unwrap(); + let n = engine.get_node_by_key("Person", &format!("node_{}", i)).unwrap(); assert!(n.is_none(), "node_{} should be deleted", i); } @@ -11286,16 +11953,16 @@ fn test_v3_compact_preserves_edges_across_segments() { // Segment 1: nodes let n1 = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); // Segment 2: edges let e1 = engine - .upsert_edge(n1, n2, 1, UpsertEdgeOptions::default()) + .upsert_edge(n1, n2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -11336,7 +12003,7 @@ fn test_v3_compact_reopen_durability() { for i in 0..100 { engine - .upsert_node(1, &format!("n{}", i), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{}", i), UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); @@ -11344,7 +12011,7 @@ fn test_v3_compact_reopen_durability() { for i in 50..100 { engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { weight: 2.0, @@ -11362,7 +12029,7 @@ fn test_v3_compact_reopen_durability() { let engine2 = DatabaseEngine::open(dir.path(), &opts).unwrap(); assert_eq!(engine2.segments_for_test().len(), 1); for i in 0..100 { - let n = engine2.get_node_by_key(1, &format!("n{}", i)).unwrap(); + let n = engine2.get_node_by_key("Person", &format!("n{}", i)).unwrap(); assert!(n.is_some(), "node n{} should exist after reopen", i); let n = n.unwrap(); let expected_weight = if i >= 50 { 2.0 } else { 1.0 }; @@ -11387,7 +12054,7 @@ fn test_vector_segment_flush_reopen_mixed_nodes() { let vector_node = engine .upsert_node( - 1, + "Person", "vector-node", UpsertNodeOptions { weight: 0.5, @@ -11399,7 +12066,7 @@ fn test_vector_segment_flush_reopen_mixed_nodes() { .unwrap(); let plain_node = engine .upsert_node( - 1, + "Person", "plain-node", UpsertNodeOptions { weight: 0.25, @@ -11429,7 +12096,7 @@ fn test_plain_segment_flush_reopen_v6_fast_path() { let node_id = engine .upsert_node( - 1, + "Person", "plain-node", UpsertNodeOptions { weight: 0.25, @@ -11445,16 +12112,23 @@ fn test_plain_segment_flush_reopen_v6_fast_path() { assert!(node.dense_vector.is_none()); assert!(node.sparse_vector.is_none()); - let seg_dir = crate::segment_writer::segment_dir(dir.path(), reopened.segments_for_test()[0].segment_id); - assert!(!seg_dir - .join(crate::segment_writer::NODE_VECTOR_META_FILENAME) - .exists()); - assert!(!seg_dir - .join(crate::segment_writer::NODE_DENSE_VECTOR_BLOB_FILENAME) - .exists()); - assert!(!seg_dir - .join(crate::segment_writer::NODE_SPARSE_VECTOR_BLOB_FILENAME) - .exists()); + let seg_dir = + crate::segment_writer::segment_dir(dir.path(), reopened.segments_for_test()[0].segment_id); + let manifest = crate::segment_components::decode_manifest_envelope( + &std::fs::read( + seg_dir.join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME), + ) + .unwrap(), + ) + .unwrap(); + assert!(manifest.components.iter().all(|record| { + !matches!( + record.kind, + crate::segment_components::SegmentComponentKind::NodeVectorMetadata + | crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob + | crate::segment_components::SegmentComponentKind::NodeSparseVectorBlob + ) + })); reopened.close().unwrap(); } @@ -11475,7 +12149,7 @@ fn test_vector_segments_survive_standard_compaction_reopen() { let node_id = engine .upsert_node( - 1, + "Person", "vector-node", UpsertNodeOptions { weight: 0.5, @@ -11488,7 +12162,7 @@ fn test_vector_segments_survive_standard_compaction_reopen() { engine .upsert_node( - 1, + "Person", "vector-node", UpsertNodeOptions { weight: 0.75, @@ -11500,7 +12174,7 @@ fn test_vector_segments_survive_standard_compaction_reopen() { .unwrap(); let sparse_only = engine .upsert_node( - 1, + "Person", "sparse-only", UpsertNodeOptions { weight: 0.9, @@ -11545,7 +12219,7 @@ fn test_standard_compaction_clears_stale_vector_payloads() { let node_id = engine .upsert_node( - 1, + "Person", "vector-node", UpsertNodeOptions { weight: 0.5, @@ -11559,7 +12233,7 @@ fn test_standard_compaction_clears_stale_vector_payloads() { engine .upsert_node( - 1, + "Person", "vector-node", UpsertNodeOptions { weight: 0.9, @@ -11577,16 +12251,23 @@ fn test_standard_compaction_clears_stale_vector_payloads() { assert!(node.dense_vector.is_none()); assert!(node.sparse_vector.is_none()); - let seg_dir = crate::segment_writer::segment_dir(dir.path(), reopened.segments_for_test()[0].segment_id); - assert!(!seg_dir - .join(crate::segment_writer::NODE_VECTOR_META_FILENAME) - .exists()); - assert!(!seg_dir - .join(crate::segment_writer::NODE_DENSE_VECTOR_BLOB_FILENAME) - .exists()); - assert!(!seg_dir - .join(crate::segment_writer::NODE_SPARSE_VECTOR_BLOB_FILENAME) - .exists()); + let seg_dir = + crate::segment_writer::segment_dir(dir.path(), reopened.segments_for_test()[0].segment_id); + let manifest = crate::segment_components::decode_manifest_envelope( + &std::fs::read( + seg_dir.join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME), + ) + .unwrap(), + ) + .unwrap(); + assert!(manifest.components.iter().all(|record| { + !matches!( + record.kind, + crate::segment_components::SegmentComponentKind::NodeVectorMetadata + | crate::segment_components::SegmentComponentKind::NodeDenseVectorBlob + | crate::segment_components::SegmentComponentKind::NodeSparseVectorBlob + ) + })); reopened.close().unwrap(); } @@ -11606,7 +12287,7 @@ fn test_vector_segments_survive_fast_merge_reopen() { let dense_node = engine .upsert_node( - 1, + "Person", "dense", UpsertNodeOptions { weight: 0.5, @@ -11619,7 +12300,7 @@ fn test_vector_segments_survive_fast_merge_reopen() { let sparse_node = engine .upsert_node( - 1, + "Person", "sparse", UpsertNodeOptions { weight: 0.7, @@ -11650,7 +12331,17 @@ fn test_vector_segments_survive_fast_merge_reopen() { fn dense_search_request( query: Vec, k: usize, - type_filter: Option>, + label_filter: Option>, + ef_search: Option, +) -> VectorSearchRequest { + dense_search_request_with_mode(query, k, label_filter, LabelMatchMode::Any, ef_search) +} + +fn dense_search_request_with_mode( + query: Vec, + k: usize, + label_filter: Option>, + label_match_mode: LabelMatchMode, ef_search: Option, ) -> VectorSearchRequest { VectorSearchRequest { @@ -11658,7 +12349,9 @@ fn dense_search_request( dense_query: Some(query), sparse_query: None, k, - type_filter, + label_filter: label_filter + .as_ref() + .map(|labels| read_node_label_filter(labels, label_match_mode)), ef_search, scope: None, dense_weight: None, @@ -11670,14 +12363,25 @@ fn dense_search_request( fn sparse_search_request( query: Vec<(u32, f32)>, k: usize, - type_filter: Option>, + label_filter: Option>, +) -> VectorSearchRequest { + sparse_search_request_with_mode(query, k, label_filter, LabelMatchMode::Any) +} + +fn sparse_search_request_with_mode( + query: Vec<(u32, f32)>, + k: usize, + label_filter: Option>, + label_match_mode: LabelMatchMode, ) -> VectorSearchRequest { VectorSearchRequest { mode: VectorSearchMode::Sparse, dense_query: None, sparse_query: Some(query), k, - type_filter, + label_filter: label_filter + .as_ref() + .map(|labels| read_node_label_filter(labels, label_match_mode)), ef_search: None, scope: None, dense_weight: None, @@ -11690,14 +12394,14 @@ fn vector_search_scope( start_node_id: u64, max_depth: u32, direction: Direction, - edge_type_filter: Option>, + edge_label_filter: Option>, at_epoch: Option, ) -> VectorSearchScope { VectorSearchScope { start_node_id, max_depth, direction, - edge_type_filter, + edge_label_filter: edge_label_filter.map(|edge_labels| read_filter_names(&edge_labels)), at_epoch, } } @@ -11705,7 +12409,25 @@ fn vector_search_scope( fn scoped_dense_search_request( query: Vec, k: usize, - type_filter: Option>, + label_filter: Option>, + ef_search: Option, + scope: VectorSearchScope, +) -> VectorSearchRequest { + scoped_dense_search_request_with_mode( + query, + k, + label_filter, + LabelMatchMode::Any, + ef_search, + scope, + ) +} + +fn scoped_dense_search_request_with_mode( + query: Vec, + k: usize, + label_filter: Option>, + label_match_mode: LabelMatchMode, ef_search: Option, scope: VectorSearchScope, ) -> VectorSearchRequest { @@ -11714,7 +12436,9 @@ fn scoped_dense_search_request( dense_query: Some(query), sparse_query: None, k, - type_filter, + label_filter: label_filter + .as_ref() + .map(|labels| read_node_label_filter(labels, label_match_mode)), ef_search, scope: Some(scope), dense_weight: None, @@ -11726,7 +12450,17 @@ fn scoped_dense_search_request( fn scoped_sparse_search_request( query: Vec<(u32, f32)>, k: usize, - type_filter: Option>, + label_filter: Option>, + scope: VectorSearchScope, +) -> VectorSearchRequest { + scoped_sparse_search_request_with_mode(query, k, label_filter, LabelMatchMode::Any, scope) +} + +fn scoped_sparse_search_request_with_mode( + query: Vec<(u32, f32)>, + k: usize, + label_filter: Option>, + label_match_mode: LabelMatchMode, scope: VectorSearchScope, ) -> VectorSearchRequest { VectorSearchRequest { @@ -11734,7 +12468,9 @@ fn scoped_sparse_search_request( dense_query: None, sparse_query: Some(query), k, - type_filter, + label_filter: label_filter + .as_ref() + .map(|labels| read_node_label_filter(labels, label_match_mode)), ef_search: None, scope: Some(scope), dense_weight: None, @@ -11867,7 +12603,7 @@ fn benchmark_clustered_sparse_inputs( (0..cluster_count) .flat_map(|cluster| { (0..points_per_cluster).map(move |member| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("sc{cluster}_n{member}"), props: BTreeMap::new(), weight: 1.0, @@ -11891,7 +12627,7 @@ fn benchmark_uniform_sparse_inputs( ) -> Vec { (0..count) .map(|index| NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("su{index}"), props: BTreeMap::new(), weight: 1.0, @@ -11914,7 +12650,7 @@ fn benchmark_sparse_multisegment_inputs_a( let mut inputs = Vec::with_capacity(count * 3); for i in 0..count { inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("shared_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -11928,7 +12664,7 @@ fn benchmark_sparse_multisegment_inputs_a( )), }); inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("stable_a_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -11942,8 +12678,8 @@ fn benchmark_sparse_multisegment_inputs_a( )), }); inputs.push(NodeInput { - type_id: 2, - key: format!("other_type_{i}"), + labels: vec!["Company".to_string()], + key: format!("other_label_{i}"), props: BTreeMap::new(), weight: 1.0, dense_vector: None, @@ -11970,7 +12706,7 @@ fn benchmark_sparse_multisegment_inputs_b( let shared = benchmark_clustered_sparse_vector(dimension_count, 3, i + 50_000, cluster_count, nnz); inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("shared_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -11978,7 +12714,7 @@ fn benchmark_sparse_multisegment_inputs_b( sparse_vector: Some(benchmark_scale_sparse_vector(&shared, 1.15)), }); inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("stable_b_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -12012,7 +12748,7 @@ fn benchmark_sparse_overlap_segment_inputs( nnz, ); inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("shared_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -12023,7 +12759,7 @@ fn benchmark_sparse_overlap_segment_inputs( )), }); inputs.push(NodeInput { - type_id: 1, + labels: vec!["Person".to_string()], key: format!("stable_{segment_index}_{i}"), props: BTreeMap::new(), weight: 1.0, @@ -12054,6 +12790,22 @@ fn assert_vector_hits_match(actual: &[VectorHit], expected: &[VectorHit]) { } } +fn rewrite_segment_component_payload_for_test(path: &Path, rewrite: impl FnOnce(&mut [u8])) { + let mut data = std::fs::read(path).unwrap(); + if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] + == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + rewrite(&mut data[start..end]); + } else { + rewrite(&mut data); + } + std::fs::write(path, data).unwrap(); +} + #[test] fn test_vector_search_dense_rejects_missing_query_and_wrong_dimension() { let dir = TempDir::new().unwrap(); @@ -12073,7 +12825,7 @@ fn test_vector_search_dense_rejects_missing_query_and_wrong_dimension() { dense_query: None, sparse_query: None, k: 5, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -12118,7 +12870,7 @@ fn test_vector_search_dense_empty_when_unconfigured_or_no_vectors() { let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); engine .upsert_node( - 1, + "Person", "plain", UpsertNodeOptions { weight: 0.5, @@ -12153,13 +12905,13 @@ fn test_vector_search_rejects_unimplemented_modes() { dense_query: Some(vec![1.0, 0.0]), sparse_query: None, k: 5, - type_filter: None, + label_filter: None, ef_search: None, scope: Some(VectorSearchScope { start_node_id: 1, max_depth: 1, direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, at_epoch: None, }), dense_weight: None, @@ -12175,7 +12927,7 @@ fn test_vector_search_rejects_unimplemented_modes() { dense_query: None, sparse_query: None, k: 5, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -12192,7 +12944,7 @@ fn test_vector_search_rejects_unimplemented_modes() { dense_query: None, sparse_query: None, k: 5, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight: None, @@ -12217,7 +12969,7 @@ fn test_vector_search_sparse_empty_when_no_sparse_vectors() { let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); engine .upsert_node( - 1, + "Person", "plain", UpsertNodeOptions { weight: 0.5, @@ -12252,7 +13004,7 @@ fn test_upsert_node_rejects_negative_sparse_weights() { let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let err = engine .upsert_node( - 1, + "Person", "bad-sparse", UpsertNodeOptions { weight: 0.5, @@ -12273,7 +13025,7 @@ fn test_vector_search_sparse_exact_ranking_and_query_canonicalization() { let segment_best = engine .upsert_node( - 1, + "Person", "segment-best", UpsertNodeOptions { weight: 0.5, @@ -12284,7 +13036,7 @@ fn test_vector_search_sparse_exact_ranking_and_query_canonicalization() { .unwrap(); let segment_mid = engine .upsert_node( - 1, + "Person", "segment-mid", UpsertNodeOptions { weight: 0.5, @@ -12297,7 +13049,7 @@ fn test_vector_search_sparse_exact_ranking_and_query_canonicalization() { let memtable_low = engine .upsert_node( - 1, + "Person", "memtable-low", UpsertNodeOptions { weight: 0.5, @@ -12326,7 +13078,7 @@ fn test_vector_search_sparse_zero_overlap_returns_empty() { let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -12344,13 +13096,13 @@ fn test_vector_search_sparse_zero_overlap_returns_empty() { } #[test] -fn test_vector_search_sparse_type_filter_deleted_and_policy_exclusion() { +fn test_vector_search_sparse_label_filter_deleted_and_policy_exclusion() { let dir = TempDir::new().unwrap(); let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let deleted = engine .upsert_node( - 1, + "Person", "deleted", UpsertNodeOptions { weight: 0.9, @@ -12361,7 +13113,7 @@ fn test_vector_search_sparse_type_filter_deleted_and_policy_exclusion() { .unwrap(); let kept = engine .upsert_node( - 1, + "Person", "kept", UpsertNodeOptions { weight: 0.9, @@ -12372,7 +13124,7 @@ fn test_vector_search_sparse_type_filter_deleted_and_policy_exclusion() { .unwrap(); let pruned = engine .upsert_node( - 1, + "Person", "pruned", UpsertNodeOptions { weight: 0.1, @@ -12381,10 +13133,10 @@ fn test_vector_search_sparse_type_filter_deleted_and_policy_exclusion() { }, ) .unwrap(); - let other_type = engine + let other_label = engine .upsert_node( - 2, - "other-type", + "Company", + "other-label", UpsertNodeOptions { weight: 0.9, sparse_vector: Some(vec![(3, 4.0)]), @@ -12400,44 +13152,127 @@ fn test_vector_search_sparse_type_filter_deleted_and_policy_exclusion() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let hits = engine - .vector_search(&sparse_search_request(vec![(3, 1.0)], 5, Some(vec![1]))) + .vector_search(&sparse_search_request(vec![(3, 1.0)], 5, Some(vec!["Person"]))) .unwrap(); let returned_ids: Vec = hits.iter().map(|hit| hit.node_id).collect(); assert_eq!(returned_ids, vec![kept]); assert!(!returned_ids.contains(&deleted)); assert!(!returned_ids.contains(&pruned)); - assert!(!returned_ids.contains(&other_type)); + assert!(!returned_ids.contains(&other_label)); } #[test] -fn test_vector_search_sparse_combines_shadowing_tombstones_type_filter_and_policy() { +fn test_vector_search_sparse_label_filter_supports_single_any_all_multi_label() { let dir = TempDir::new().unwrap(); - let opts = DbOptions { - compact_after_n_flushes: 0, - ..DbOptions::default() - }; - let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); - let stale_shared = engine + let person_employee = engine .upsert_node( - 1, - "shared", + &["Person", "Employee"], + "person-employee", UpsertNodeOptions { - weight: 0.9, - sparse_vector: Some(vec![(3, 1.0)]), + sparse_vector: Some(vec![(3, 3.0)]), ..Default::default() }, ) .unwrap(); - let deleted = engine + let employee = engine .upsert_node( - 1, + "Employee", + "employee", + UpsertNodeOptions { + sparse_vector: Some(vec![(3, 2.0)]), + ..Default::default() + }, + ) + .unwrap(); + let person = engine + .upsert_node( + "Person", + "person", + UpsertNodeOptions { + sparse_vector: Some(vec![(3, 1.0)]), + ..Default::default() + }, + ) + .unwrap(); + let company = engine + .upsert_node( + "Company", + "company", + UpsertNodeOptions { + sparse_vector: Some(vec![(3, 4.0)]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let single = engine + .vector_search(&sparse_search_request(vec![(3, 1.0)], 10, Some(vec!["Person"]))) + .unwrap(); + assert_eq!( + single.iter().map(|hit| hit.node_id).collect::>(), + vec![person_employee, person] + ); + + let any = engine + .vector_search(&sparse_search_request_with_mode( + vec![(3, 1.0)], + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::Any, + )) + .unwrap(); + assert_eq!( + any.iter().map(|hit| hit.node_id).collect::>(), + vec![person_employee, employee, person] + ); + assert!(!any.iter().any(|hit| hit.node_id == company)); + + let all = engine + .vector_search(&sparse_search_request_with_mode( + vec![(3, 1.0)], + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::All, + )) + .unwrap(); + assert_eq!(all.len(), 1); + assert_eq!(all[0].node_id, person_employee); + + engine.close().unwrap(); +} + +#[test] +fn test_vector_search_sparse_combines_shadowing_tombstones_label_filter_and_policy() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let stale_shared = engine + .upsert_node( + "Person", + "shared", + UpsertNodeOptions { + weight: 0.9, + sparse_vector: Some(vec![(3, 1.0)]), + ..Default::default() + }, + ) + .unwrap(); + let deleted = engine + .upsert_node( + "Person", "deleted", UpsertNodeOptions { weight: 0.9, @@ -12448,7 +13283,7 @@ fn test_vector_search_sparse_combines_shadowing_tombstones_type_filter_and_polic .unwrap(); let pruned = engine .upsert_node( - 1, + "Person", "pruned", UpsertNodeOptions { weight: 0.1, @@ -12457,10 +13292,10 @@ fn test_vector_search_sparse_combines_shadowing_tombstones_type_filter_and_polic }, ) .unwrap(); - let other_type = engine + let other_label = engine .upsert_node( - 2, - "other-type", + "Company", + "other-label", UpsertNodeOptions { weight: 0.9, sparse_vector: Some(vec![(3, 4.0)]), @@ -12472,7 +13307,7 @@ fn test_vector_search_sparse_combines_shadowing_tombstones_type_filter_and_polic let stable = engine .upsert_node( - 1, + "Person", "stable", UpsertNodeOptions { weight: 0.9, @@ -12490,14 +13325,14 @@ fn test_vector_search_sparse_combines_shadowing_tombstones_type_filter_and_polic PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let fresh_shared = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.9, @@ -12509,13 +13344,13 @@ fn test_vector_search_sparse_combines_shadowing_tombstones_type_filter_and_polic assert_eq!(stale_shared, fresh_shared); let hits = engine - .vector_search(&sparse_search_request(vec![(3, 1.0)], 5, Some(vec![1]))) + .vector_search(&sparse_search_request(vec![(3, 1.0)], 5, Some(vec!["Person"]))) .unwrap(); let returned_ids: Vec = hits.iter().map(|hit| hit.node_id).collect(); assert_eq!(returned_ids, vec![fresh_shared, stable]); assert!(!returned_ids.contains(&deleted)); assert!(!returned_ids.contains(&pruned)); - assert!(!returned_ids.contains(&other_type)); + assert!(!returned_ids.contains(&other_label)); } #[test] @@ -12525,7 +13360,7 @@ fn test_vector_search_sparse_flush_and_reopen_parity() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -12536,7 +13371,7 @@ fn test_vector_search_sparse_flush_and_reopen_parity() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -12561,6 +13396,170 @@ fn test_vector_search_sparse_flush_and_reopen_parity() { reopened.close().unwrap(); } +#[test] +fn test_vector_search_sparse_missing_postings_uses_exact_segment_fallback() { + let dir = TempDir::new().unwrap(); + let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); + + let best = engine + .upsert_node( + "Person", + "best", + UpsertNodeOptions { + sparse_vector: Some(vec![(2, 1.0), (7, 0.5)]), + ..Default::default() + }, + ) + .unwrap(); + let second = engine + .upsert_node( + "Person", + "second", + UpsertNodeOptions { + sparse_vector: Some(vec![(2, 0.5)]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let segment_id = engine.segments_for_test()[0].segment_id; + let seg_dir = crate::segment_writer::segment_dir(dir.path(), segment_id); + std::fs::remove_file(seg_dir.join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME)) + .unwrap(); + std::fs::remove_file(seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME)).unwrap(); + engine + .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + + let hits = engine + .vector_search(&sparse_search_request(vec![(2, 1.0)], 2, None)) + .unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.node_id).collect::>(), + vec![best, second] + ); + assert!((hits[0].score - 1.0).abs() < 1e-6); + assert!((hits[1].score - 0.5).abs() < 1e-6); +} + +#[test] +fn test_vector_search_sparse_invalid_postings_uses_exact_segment_fallback() { + let dir = TempDir::new().unwrap(); + let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); + + let best = engine + .upsert_node( + "Person", + "best", + UpsertNodeOptions { + sparse_vector: Some(vec![(2, 1.0), (7, 0.5)]), + ..Default::default() + }, + ) + .unwrap(); + let second = engine + .upsert_node( + "Person", + "second", + UpsertNodeOptions { + sparse_vector: Some(vec![(2, 0.5)]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let segment_id = engine.segments_for_test()[0].segment_id; + let seg_dir = crate::segment_writer::segment_dir(dir.path(), segment_id); + rewrite_segment_component_payload_for_test( + &seg_dir.join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME), + |index| { + index[20..24].copy_from_slice(&1u32.to_le_bytes()); + index[28..36].copy_from_slice(&12u64.to_le_bytes()); + }, + ); + engine + .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + let segment = engine.segments_for_test()[0].clone(); + assert!(segment.sparse_postings_available()); + + let hits = engine + .vector_search(&sparse_search_request(vec![(2, 1.0)], 2, None)) + .unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.node_id).collect::>(), + vec![best, second] + ); + assert!((hits[0].score - 1.0).abs() < 1e-6); + assert!((hits[1].score - 0.5).abs() < 1e-6); + assert!(!segment.sparse_postings_available()); + assert!(matches!( + segment.optional_component_availability_for_test( + crate::segment_components::SegmentComponentKind::SparsePostingIndex + ), + crate::segment_components::ComponentAvailability::CorruptIdentity { .. } + )); +} + +#[test] +fn test_vector_search_sparse_runtime_posting_error_latches_and_falls_back() { + let dir = TempDir::new().unwrap(); + let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); + + let best = engine + .upsert_node( + "Person", + "best", + UpsertNodeOptions { + sparse_vector: Some(vec![(2, 1.0), (7, 0.5)]), + ..Default::default() + }, + ) + .unwrap(); + let second = engine + .upsert_node( + "Person", + "second", + UpsertNodeOptions { + sparse_vector: Some(vec![(2, 0.5)]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let segment_id = engine.segments_for_test()[0].segment_id; + let seg_dir = crate::segment_writer::segment_dir(dir.path(), segment_id); + rewrite_segment_component_payload_for_test( + &seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME), + |postings| { + postings[8..12].copy_from_slice(&(-1.0f32).to_le_bytes()); + }, + ); + engine + .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + let segment = engine.segments_for_test()[0].clone(); + assert!(segment.sparse_postings_available()); + + let hits = engine + .vector_search(&sparse_search_request(vec![(2, 1.0)], 2, None)) + .unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.node_id).collect::>(), + vec![best, second] + ); + assert!(!segment.sparse_postings_available()); + assert!(matches!( + segment.optional_component_availability_for_test( + crate::segment_components::SegmentComponentKind::SparsePostingIndex + ), + crate::segment_components::ComponentAvailability::CorruptIdentity { .. } + )); +} + #[test] fn test_vector_search_hybrid_flush_and_reopen_parity() { let (dir, engine, _ids) = setup_hybrid_db(); @@ -12614,7 +13613,7 @@ fn test_vector_search_sparse_newer_segment_shadows_older_segment_candidate() { let stale = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.5, @@ -12625,7 +13624,7 @@ fn test_vector_search_sparse_newer_segment_shadows_older_segment_candidate() { .unwrap(); let stable = engine .upsert_node( - 1, + "Person", "stable", UpsertNodeOptions { weight: 0.5, @@ -12638,7 +13637,7 @@ fn test_vector_search_sparse_newer_segment_shadows_older_segment_candidate() { let fresh = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.5, @@ -12670,7 +13669,7 @@ fn test_vector_search_sparse_newer_non_match_hides_older_match() { let stale = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.5, @@ -12681,7 +13680,7 @@ fn test_vector_search_sparse_newer_non_match_hides_older_match() { .unwrap(); let stable = engine .upsert_node( - 1, + "Person", "stable", UpsertNodeOptions { weight: 0.5, @@ -12694,7 +13693,7 @@ fn test_vector_search_sparse_newer_non_match_hides_older_match() { let fresh = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.5, @@ -12723,7 +13722,7 @@ fn test_vector_search_sparse_standard_compaction_parity() { let stale = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.5, @@ -12734,7 +13733,7 @@ fn test_vector_search_sparse_standard_compaction_parity() { .unwrap(); let stable = engine .upsert_node( - 1, + "Person", "stable", UpsertNodeOptions { weight: 0.5, @@ -12747,7 +13746,7 @@ fn test_vector_search_sparse_standard_compaction_parity() { let fresh = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.9, @@ -12758,7 +13757,7 @@ fn test_vector_search_sparse_standard_compaction_parity() { .unwrap(); let deleted = engine .upsert_node( - 2, + "Company", "deleted", UpsertNodeOptions { weight: 0.4, @@ -12771,7 +13770,7 @@ fn test_vector_search_sparse_standard_compaction_parity() { engine.flush().unwrap(); engine.delete_node(deleted).unwrap(); - let request = sparse_search_request(vec![(2, 1.0), (5, 1.0)], 2, Some(vec![1])); + let request = sparse_search_request(vec![(2, 1.0), (5, 1.0)], 2, Some(vec!["Person"])); let before = engine.vector_search(&request).unwrap(); assert_eq!(before.len(), 2); assert_eq!(before[0].node_id, fresh); @@ -12799,7 +13798,7 @@ fn test_vector_search_sparse_fast_merge_compaction_parity() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -12810,7 +13809,7 @@ fn test_vector_search_sparse_fast_merge_compaction_parity() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -12823,7 +13822,7 @@ fn test_vector_search_sparse_fast_merge_compaction_parity() { let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -12834,7 +13833,7 @@ fn test_vector_search_sparse_fast_merge_compaction_parity() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.5, @@ -12892,7 +13891,7 @@ fn test_vector_search_sparse_background_compaction_parity_with_mixed_vectors() { ..Default::default() } }; - engine.upsert_node(1, &key, write).unwrap(); + engine.upsert_node("Person", &key, write).unwrap(); } engine.flush().unwrap(); } @@ -12925,7 +13924,7 @@ fn benchmark_vector_search_sparse_clustered_9216x12of4096() { benchmark_clustered_sparse_inputs(cluster_count, points_per_cluster, dimension_count, nnz); let flush_started = std::time::Instant::now(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs).unwrap(); engine.flush().unwrap(); let flush_ms = flush_started.elapsed().as_secs_f64() * 1_000.0; @@ -12964,7 +13963,7 @@ fn benchmark_vector_search_sparse_uniform_9216x12of4096() { let inputs = benchmark_uniform_sparse_inputs(9_216, dimension_count, nnz); let flush_started = std::time::Instant::now(); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs).unwrap(); engine.flush().unwrap(); let flush_ms = flush_started.elapsed().as_secs_f64() * 1_000.0; @@ -13015,12 +14014,12 @@ fn benchmark_vector_search_sparse_multisegment_filtered() { benchmark_sparse_multisegment_inputs_b(1_536, dimension_count, cluster_count, nnz); let started = std::time::Instant::now(); - engine.batch_upsert_nodes(&inputs_a).unwrap(); + engine.batch_upsert_nodes(inputs_a).unwrap(); engine.flush().unwrap(); flush_ms += started.elapsed().as_secs_f64() * 1_000.0; let started = std::time::Instant::now(); - engine.batch_upsert_nodes(&inputs_b).unwrap(); + engine.batch_upsert_nodes(inputs_b).unwrap(); engine.flush().unwrap(); flush_ms += started.elapsed().as_secs_f64() * 1_000.0; @@ -13030,7 +14029,7 @@ fn benchmark_vector_search_sparse_multisegment_filtered() { benchmark_clustered_sparse_query(dimension_count, 3, query_idx, cluster_count, nnz); let started = std::time::Instant::now(); let hits = engine - .vector_search(&sparse_search_request(query, 10, Some(vec![1]))) + .vector_search(&sparse_search_request(query, 10, Some(vec!["Person"]))) .unwrap(); search_micros.push(started.elapsed().as_secs_f64() * 1_000_000.0); assert!(hits.iter().all(|hit| hit.node_id > 0)); @@ -13069,7 +14068,7 @@ fn benchmark_sparse_flush_and_compaction_overlap() { cluster_count, nnz, ); - engine.batch_upsert_nodes(&inputs).unwrap(); + engine.batch_upsert_nodes(inputs).unwrap(); let started = std::time::Instant::now(); engine.flush().unwrap(); flush_durations_ms.push(started.elapsed().as_secs_f64() * 1_000.0); @@ -13078,7 +14077,7 @@ fn benchmark_sparse_flush_and_compaction_overlap() { let request = sparse_search_request( benchmark_clustered_sparse_query(dimension_count, 5, 7, cluster_count, nnz), 10, - Some(vec![1]), + Some(vec!["Person"]), ); let before = engine.vector_search(&request).unwrap(); @@ -13117,7 +14116,7 @@ fn test_vector_search_dense_memtable_shadows_segment_and_collapses_duplicates() let alpha = engine .upsert_node( - 1, + "Person", "alpha", UpsertNodeOptions { weight: 0.5, @@ -13128,7 +14127,7 @@ fn test_vector_search_dense_memtable_shadows_segment_and_collapses_duplicates() .unwrap(); let beta = engine .upsert_node( - 1, + "Person", "beta", UpsertNodeOptions { weight: 0.4, @@ -13141,7 +14140,7 @@ fn test_vector_search_dense_memtable_shadows_segment_and_collapses_duplicates() let alpha_updated = engine .upsert_node( - 1, + "Person", "alpha", UpsertNodeOptions { weight: 0.7, @@ -13162,7 +14161,228 @@ fn test_vector_search_dense_memtable_shadows_segment_and_collapses_duplicates() } #[test] -fn test_vector_search_dense_newer_segment_shadows_older_segment() { +fn test_vector_search_dense_newer_segment_shadows_older_segment() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let alpha = engine + .upsert_node( + "Person", + "alpha", + UpsertNodeOptions { + weight: 0.5, + dense_vector: Some(vec![0.0, 1.0]), + ..Default::default() + }, + ) + .unwrap(); + let beta = engine + .upsert_node( + "Person", + "beta", + UpsertNodeOptions { + weight: 0.4, + dense_vector: Some(vec![0.8, 0.2]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + engine + .upsert_node( + "Person", + "alpha", + UpsertNodeOptions { + weight: 0.9, + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let hits = engine + .vector_search(&dense_search_request(vec![1.0, 0.0], 2, None, None)) + .unwrap(); + assert_eq!(hits.len(), 2); + assert_eq!(hits[0].node_id, alpha); + assert_eq!(hits[1].node_id, beta); + assert!((hits[0].score - 1.0).abs() < 1e-6); +} + +#[test] +fn test_vector_search_dense_label_filter_and_deleted_node_exclusion() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let deleted = engine + .upsert_node( + "Person", + "deleted", + UpsertNodeOptions { + weight: 0.5, + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + let kept = engine + .upsert_node( + "Person", + "kept", + UpsertNodeOptions { + weight: 0.4, + dense_vector: Some(vec![0.9, 0.1]), + ..Default::default() + }, + ) + .unwrap(); + let other_label = engine + .upsert_node( + "Company", + "other-label", + UpsertNodeOptions { + weight: 0.3, + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + engine.delete_node(deleted).unwrap(); + + let hits = engine + .vector_search(&dense_search_request( + vec![1.0, 0.0], + 3, + Some(vec!["Person"]), + None, + )) + .unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].node_id, kept); + assert!(hits.iter().all(|hit| hit.node_id != deleted)); + assert!(hits.iter().all(|hit| hit.node_id != other_label)); +} + +#[test] +fn test_vector_search_dense_label_filter_supports_single_any_all_multi_label() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let person_employee = engine + .upsert_node( + &["Person", "Employee"], + "person-employee", + UpsertNodeOptions { + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + let person = engine + .upsert_node( + "Person", + "person", + UpsertNodeOptions { + dense_vector: Some(vec![0.8, 0.2]), + ..Default::default() + }, + ) + .unwrap(); + let employee = engine + .upsert_node( + "Employee", + "employee", + UpsertNodeOptions { + dense_vector: Some(vec![0.7, 0.3]), + ..Default::default() + }, + ) + .unwrap(); + let company = engine + .upsert_node( + "Company", + "company", + UpsertNodeOptions { + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let single = engine + .vector_search(&dense_search_request( + vec![1.0, 0.0], + 10, + Some(vec!["Person"]), + Some(8), + )) + .unwrap(); + assert_eq!( + single.iter().map(|hit| hit.node_id).collect::>(), + vec![person_employee, person] + ); + + let any = engine + .vector_search(&dense_search_request_with_mode( + vec![1.0, 0.0], + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::Any, + Some(8), + )) + .unwrap(); + assert_eq!( + any.iter().map(|hit| hit.node_id).collect::>(), + vec![person_employee, person, employee] + ); + assert!(!any.iter().any(|hit| hit.node_id == company)); + + let all = engine + .vector_search(&dense_search_request_with_mode( + vec![1.0, 0.0], + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::All, + Some(8), + )) + .unwrap(); + assert_eq!(all.len(), 1); + assert_eq!(all[0].node_id, person_employee); + + engine.close().unwrap(); +} + +#[test] +fn test_vector_search_scoped_label_filter_supports_any_all_multi_label() { let dir = TempDir::new().unwrap(); let opts = DbOptions { dense_vector: Some(DenseVectorConfig { @@ -13174,118 +14394,152 @@ fn test_vector_search_dense_newer_segment_shadows_older_segment() { }; let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); - let alpha = engine + let start = engine + .upsert_node("Anchor", "scope-label-start", UpsertNodeOptions::default()) + .unwrap(); + let person_employee = engine .upsert_node( - 1, - "alpha", + &["Person", "Employee"], + "scope-person-employee", UpsertNodeOptions { - weight: 0.5, - dense_vector: Some(vec![0.0, 1.0]), + dense_vector: Some(vec![1.0, 0.0]), + sparse_vector: Some(vec![(3, 4.0)]), ..Default::default() }, ) .unwrap(); - let beta = engine + let employee = engine .upsert_node( - 1, - "beta", + "Employee", + "scope-employee", UpsertNodeOptions { - weight: 0.4, dense_vector: Some(vec![0.8, 0.2]), + sparse_vector: Some(vec![(3, 3.0)]), ..Default::default() }, ) .unwrap(); - engine.flush().unwrap(); - - engine + let person = engine .upsert_node( - 1, - "alpha", + "Person", + "scope-person", UpsertNodeOptions { - weight: 0.9, - dense_vector: Some(vec![1.0, 0.0]), + dense_vector: Some(vec![0.7, 0.3]), + sparse_vector: Some(vec![(3, 2.0)]), ..Default::default() }, ) .unwrap(); - engine.flush().unwrap(); - - let hits = engine - .vector_search(&dense_search_request(vec![1.0, 0.0], 2, None, None)) - .unwrap(); - assert_eq!(hits.len(), 2); - assert_eq!(hits[0].node_id, alpha); - assert_eq!(hits[1].node_id, beta); - assert!((hits[0].score - 1.0).abs() < 1e-6); -} - -#[test] -fn test_vector_search_dense_type_filter_and_deleted_node_exclusion() { - let dir = TempDir::new().unwrap(); - let opts = DbOptions { - dense_vector: Some(DenseVectorConfig { - dimension: 2, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }), - ..DbOptions::default() - }; - let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); - - let deleted = engine + let company = engine .upsert_node( - 1, - "deleted", + "Company", + "scope-company", UpsertNodeOptions { - weight: 0.5, dense_vector: Some(vec![1.0, 0.0]), + sparse_vector: Some(vec![(3, 5.0)]), ..Default::default() }, ) .unwrap(); - let kept = engine + let unreachable = engine .upsert_node( - 1, - "kept", + &["Person", "Employee"], + "scope-label-unreachable", UpsertNodeOptions { - weight: 0.4, - dense_vector: Some(vec![0.9, 0.1]), + dense_vector: Some(vec![1.0, 0.0]), + sparse_vector: Some(vec![(3, 6.0)]), ..Default::default() }, ) .unwrap(); - let other_type = engine + let wrong_edge = engine .upsert_node( - 2, - "other-type", + &["Person", "Employee"], + "scope-label-wrong-edge", UpsertNodeOptions { - weight: 0.3, dense_vector: Some(vec![1.0, 0.0]), + sparse_vector: Some(vec![(3, 7.0)]), ..Default::default() }, ) .unwrap(); + + for node_id in [person_employee, employee, person, company] { + engine + .upsert_edge(start, node_id, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + engine + .upsert_edge(start, wrong_edge, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); engine.flush().unwrap(); - engine.delete_node(deleted).unwrap(); + let scope = vector_search_scope(start, 1, Direction::Outgoing, Some(vec!["KNOWS"]), None); + let dense_any = engine + .vector_search(&scoped_dense_search_request_with_mode( + vec![1.0, 0.0], + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::Any, + Some(8), + scope.clone(), + )) + .unwrap(); + assert_eq!( + dense_any.iter().map(|hit| hit.node_id).collect::>(), + vec![person_employee, employee, person] + ); + assert!(!dense_any.iter().any(|hit| hit.node_id == company)); + assert!(!dense_any.iter().any(|hit| hit.node_id == unreachable)); + assert!(!dense_any.iter().any(|hit| hit.node_id == wrong_edge)); - let hits = engine - .vector_search(&dense_search_request( + let dense_all = engine + .vector_search(&scoped_dense_search_request_with_mode( vec![1.0, 0.0], - 3, - Some(vec![1]), - None, + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::All, + Some(8), + scope.clone(), )) .unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].node_id, kept); - assert!(hits.iter().all(|hit| hit.node_id != deleted)); - assert!(hits.iter().all(|hit| hit.node_id != other_type)); + assert_eq!(dense_all.len(), 1); + assert_eq!(dense_all[0].node_id, person_employee); + + let sparse_any = engine + .vector_search(&scoped_sparse_search_request_with_mode( + vec![(3, 1.0)], + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::Any, + scope.clone(), + )) + .unwrap(); + assert_eq!( + sparse_any.iter().map(|hit| hit.node_id).collect::>(), + vec![person_employee, employee, person] + ); + assert!(!sparse_any.iter().any(|hit| hit.node_id == company)); + assert!(!sparse_any.iter().any(|hit| hit.node_id == unreachable)); + assert!(!sparse_any.iter().any(|hit| hit.node_id == wrong_edge)); + + let sparse_all = engine + .vector_search(&scoped_sparse_search_request_with_mode( + vec![(3, 1.0)], + 10, + Some(vec!["Person", "Employee"]), + LabelMatchMode::All, + scope, + )) + .unwrap(); + assert_eq!(sparse_all.len(), 1); + assert_eq!(sparse_all[0].node_id, person_employee); + + engine.close().unwrap(); } #[test] -fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy() { +fn test_vector_search_dense_combines_shadowing_tombstones_label_filter_and_policy() { let dir = TempDir::new().unwrap(); let opts = DbOptions { dense_vector: Some(DenseVectorConfig { @@ -13299,7 +14553,7 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy let shadowed = engine .upsert_node( - 1, + "Person", "shadowed", UpsertNodeOptions { weight: 0.9, @@ -13310,7 +14564,7 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy .unwrap(); let kept_a = engine .upsert_node( - 1, + "Person", "kept-a", UpsertNodeOptions { weight: 0.9, @@ -13321,7 +14575,7 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy .unwrap(); let kept_b = engine .upsert_node( - 1, + "Person", "kept-b", UpsertNodeOptions { weight: 0.8, @@ -13332,7 +14586,7 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy .unwrap(); let deleted = engine .upsert_node( - 1, + "Person", "deleted", UpsertNodeOptions { weight: 0.95, @@ -13343,7 +14597,7 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy .unwrap(); let pruned = engine .upsert_node( - 1, + "Person", "pruned", UpsertNodeOptions { weight: 0.1, @@ -13352,10 +14606,10 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy }, ) .unwrap(); - let other_type = engine + let other_label = engine .upsert_node( - 2, - "other-type", + "Company", + "other-label", UpsertNodeOptions { weight: 0.95, dense_vector: Some(vec![1.0, 0.0]), @@ -13372,14 +14626,14 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let shadowed_updated = engine .upsert_node( - 1, + "Person", "shadowed", UpsertNodeOptions { weight: 0.95, @@ -13394,7 +14648,7 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy .vector_search(&dense_search_request( vec![1.0, 0.0], 3, - Some(vec![1]), + Some(vec!["Person"]), Some(8), )) .unwrap(); @@ -13403,7 +14657,7 @@ fn test_vector_search_dense_combines_shadowing_tombstones_type_filter_and_policy assert_eq!(returned_ids, vec![shadowed, kept_a, kept_b]); assert!(!returned_ids.contains(&deleted)); assert!(!returned_ids.contains(&pruned)); - assert!(!returned_ids.contains(&other_type)); + assert!(!returned_ids.contains(&other_label)); } #[test] @@ -13426,7 +14680,7 @@ fn test_vector_search_dense_overfetch_recovers_visible_k() { for index in 0..9 { let node_id = engine .upsert_node( - 1, + "Person", &format!("stale-{index}"), UpsertNodeOptions { weight: 0.5, @@ -13439,7 +14693,7 @@ fn test_vector_search_dense_overfetch_recovers_visible_k() { } let visible_a = engine .upsert_node( - 1, + "Person", "visible-a", UpsertNodeOptions { weight: 0.5, @@ -13450,7 +14704,7 @@ fn test_vector_search_dense_overfetch_recovers_visible_k() { .unwrap(); let visible_b = engine .upsert_node( - 1, + "Person", "visible-b", UpsertNodeOptions { weight: 0.5, @@ -13464,7 +14718,7 @@ fn test_vector_search_dense_overfetch_recovers_visible_k() { for index in 0..stale_ids.len() { engine .upsert_node( - 1, + "Person", &format!("stale-{index}"), UpsertNodeOptions { weight: 0.8, @@ -13502,7 +14756,7 @@ fn test_vector_search_dense_exhausts_segments_before_returning_top_k() { let weaker = engine .upsert_node( - 1, + "Person", "older-weaker", UpsertNodeOptions { weight: 0.5, @@ -13516,7 +14770,7 @@ fn test_vector_search_dense_exhausts_segments_before_returning_top_k() { for index in 0..12 { engine .upsert_node( - 1, + "Person", &format!("shadowed-{index}"), UpsertNodeOptions { weight: 0.5, @@ -13528,7 +14782,7 @@ fn test_vector_search_dense_exhausts_segments_before_returning_top_k() { } let hidden_better = engine .upsert_node( - 1, + "Person", "hidden-better", UpsertNodeOptions { weight: 0.5, @@ -13542,7 +14796,7 @@ fn test_vector_search_dense_exhausts_segments_before_returning_top_k() { for index in 0..12 { engine .upsert_node( - 1, + "Person", &format!("shadowed-{index}"), UpsertNodeOptions { weight: 0.8, @@ -13576,7 +14830,7 @@ fn test_vector_search_dense_default_ef_search_matches_explicit_default() { for index in 0..128 { engine .upsert_node( - 1, + "Person", &format!("n{index}"), UpsertNodeOptions { weight: 0.5, @@ -13597,7 +14851,7 @@ fn test_vector_search_dense_default_ef_search_matches_explicit_default() { query, 10, None, - Some(crate::types::DEFAULT_DENSE_EF_SEARCH), + Some(DEFAULT_DENSE_EF_SEARCH), )) .unwrap(); @@ -13618,7 +14872,7 @@ fn test_vector_search_dense_supports_euclidean_and_dot_product_metrics() { let euclidean = DatabaseEngine::open(euclidean_dir.path(), &euclidean_opts).unwrap(); let near = euclidean .upsert_node( - 1, + "Person", "near", UpsertNodeOptions { weight: 0.5, @@ -13629,7 +14883,7 @@ fn test_vector_search_dense_supports_euclidean_and_dot_product_metrics() { .unwrap(); let far = euclidean .upsert_node( - 1, + "Person", "far", UpsertNodeOptions { weight: 0.5, @@ -13662,7 +14916,7 @@ fn test_vector_search_dense_supports_euclidean_and_dot_product_metrics() { let dot = DatabaseEngine::open(dot_dir.path(), &dot_opts).unwrap(); let lower = dot .upsert_node( - 1, + "Person", "lower", UpsertNodeOptions { weight: 0.5, @@ -13673,7 +14927,7 @@ fn test_vector_search_dense_supports_euclidean_and_dot_product_metrics() { .unwrap(); let higher = dot .upsert_node( - 1, + "Person", "higher", UpsertNodeOptions { weight: 0.5, @@ -13709,7 +14963,7 @@ fn test_vector_search_dense_small_graph_matches_exact_oracle() { let n1 = engine .upsert_node( - 1, + "Person", "n1", UpsertNodeOptions { weight: 0.5, @@ -13720,7 +14974,7 @@ fn test_vector_search_dense_small_graph_matches_exact_oracle() { .unwrap(); let n2 = engine .upsert_node( - 1, + "Person", "n2", UpsertNodeOptions { weight: 0.5, @@ -13731,7 +14985,7 @@ fn test_vector_search_dense_small_graph_matches_exact_oracle() { .unwrap(); let n3 = engine .upsert_node( - 1, + "Person", "n3", UpsertNodeOptions { weight: 0.5, @@ -13742,7 +14996,7 @@ fn test_vector_search_dense_small_graph_matches_exact_oracle() { .unwrap(); let n4 = engine .upsert_node( - 1, + "Person", "n4", UpsertNodeOptions { weight: 0.5, @@ -13787,6 +15041,181 @@ fn test_vector_search_dense_small_graph_matches_exact_oracle() { assert_eq!(hits, expected); } +#[test] +fn test_vector_search_dense_missing_hnsw_uses_exact_segment_fallback() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let best = engine + .upsert_node( + "Person", + "best", + UpsertNodeOptions { + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + let second = engine + .upsert_node( + "Person", + "second", + UpsertNodeOptions { + dense_vector: Some(vec![0.8, 0.2]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let segment_id = engine.segments_for_test()[0].segment_id; + let seg_dir = crate::segment_writer::segment_dir(dir.path(), segment_id); + std::fs::remove_file(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap(); + std::fs::remove_file(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME)).unwrap(); + engine + .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + + let hits = engine + .vector_search(&dense_search_request(vec![1.0, 0.0], 2, None, None)) + .unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.node_id).collect::>(), + vec![best, second] + ); +} + +#[test] +fn test_vector_search_dense_invalid_hnsw_uses_exact_segment_fallback() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let best = engine + .upsert_node( + "Person", + "best", + UpsertNodeOptions { + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + let second = engine + .upsert_node( + "Person", + "second", + UpsertNodeOptions { + dense_vector: Some(vec![0.7, 0.3]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let segment_id = engine.segments_for_test()[0].segment_id; + let seg_dir = crate::segment_writer::segment_dir(dir.path(), segment_id); + rewrite_segment_component_payload_for_test( + &seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), + |meta| { + meta[22..24].copy_from_slice(&(opts.dense_vector.as_ref().unwrap().hnsw.m + 1).to_le_bytes()); + }, + ); + engine + .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + + let hits = engine + .vector_search(&dense_search_request(vec![1.0, 0.0], 2, None, None)) + .unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.node_id).collect::>(), + vec![best, second] + ); +} + +#[test] +fn test_vector_search_dense_runtime_hnsw_error_latches_and_falls_back() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + let best = engine + .upsert_node( + "Person", + "best", + UpsertNodeOptions { + dense_vector: Some(vec![1.0, 0.0]), + ..Default::default() + }, + ) + .unwrap(); + let second = engine + .upsert_node( + "Person", + "second", + UpsertNodeOptions { + dense_vector: Some(vec![0.7, 0.3]), + ..Default::default() + }, + ) + .unwrap(); + engine.flush().unwrap(); + + let segment_id = engine.segments_for_test()[0].segment_id; + let seg_dir = crate::segment_writer::segment_dir(dir.path(), segment_id); + rewrite_segment_component_payload_for_test( + &seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), + |meta| { + let first_dense_offset = 36 + 8; + meta[first_dense_offset..first_dense_offset + 8] + .copy_from_slice(&u64::MAX.to_le_bytes()); + }, + ); + engine + .reopen_segment_reader_and_rebuild_sources_for_test(segment_id) + .unwrap(); + let segment = engine.segments_for_test()[0].clone(); + assert!(segment.dense_hnsw_header().is_some()); + + let hits = engine + .vector_search(&dense_search_request(vec![1.0, 0.0], 2, None, None)) + .unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.node_id).collect::>(), + vec![best, second] + ); + assert!(segment.dense_hnsw_header().is_none()); + assert!(matches!( + segment.optional_component_availability_for_test( + crate::segment_components::SegmentComponentKind::DenseHnswMetadata + ), + crate::segment_components::ComponentAvailability::CorruptIdentity { .. } + )); +} + #[test] fn test_vector_search_dense_standard_compaction_parity() { let dir = TempDir::new().unwrap(); @@ -13803,7 +15232,7 @@ fn test_vector_search_dense_standard_compaction_parity() { let stale = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.5, @@ -13814,7 +15243,7 @@ fn test_vector_search_dense_standard_compaction_parity() { .unwrap(); let stable = engine .upsert_node( - 1, + "Person", "stable", UpsertNodeOptions { weight: 0.5, @@ -13827,7 +15256,7 @@ fn test_vector_search_dense_standard_compaction_parity() { let fresh = engine .upsert_node( - 1, + "Person", "shared", UpsertNodeOptions { weight: 0.8, @@ -13838,7 +15267,7 @@ fn test_vector_search_dense_standard_compaction_parity() { .unwrap(); let deleted = engine .upsert_node( - 2, + "Company", "deleted", UpsertNodeOptions { weight: 0.4, @@ -13850,7 +15279,7 @@ fn test_vector_search_dense_standard_compaction_parity() { engine.flush().unwrap(); engine.delete_node(deleted).unwrap(); - let request = dense_search_request(vec![1.0, 0.0], 2, Some(vec![1]), Some(8)); + let request = dense_search_request(vec![1.0, 0.0], 2, Some(vec!["Person"]), Some(8)); let before = engine.vector_search(&request).unwrap(); assert_eq!(before.len(), 2); assert_eq!(before[0].node_id, fresh); @@ -13884,7 +15313,7 @@ fn test_vector_search_dense_fast_merge_compaction_parity() { let a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -13895,7 +15324,7 @@ fn test_vector_search_dense_fast_merge_compaction_parity() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.5, @@ -13908,7 +15337,7 @@ fn test_vector_search_dense_fast_merge_compaction_parity() { let c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { weight: 0.5, @@ -13919,7 +15348,7 @@ fn test_vector_search_dense_fast_merge_compaction_parity() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { weight: 0.5, @@ -13964,7 +15393,7 @@ fn test_vector_search_dense_background_compaction_parity() { for index in 0..16 { engine .upsert_node( - 1, + "Person", &format!("s{segment}_n{index}"), UpsertNodeOptions { weight: 0.5, @@ -14010,7 +15439,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an let start = engine .upsert_node( - 1, + "Person", "scope-start", UpsertNodeOptions { weight: 0.9, @@ -14021,7 +15450,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .unwrap(); let keep = engine .upsert_node( - 1, + "Person", "scope-keep", UpsertNodeOptions { weight: 0.9, @@ -14032,7 +15461,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .unwrap(); let shadowed = engine .upsert_node( - 1, + "Person", "scope-shadowed", UpsertNodeOptions { weight: 0.9, @@ -14043,7 +15472,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .unwrap(); let deleted = engine .upsert_node( - 1, + "Person", "scope-deleted", UpsertNodeOptions { weight: 0.9, @@ -14054,7 +15483,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .unwrap(); let pruned = engine .upsert_node( - 1, + "Person", "scope-pruned", UpsertNodeOptions { weight: 0.1, @@ -14063,10 +15492,10 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an }, ) .unwrap(); - let other_type = engine + let other_label = engine .upsert_node( - 2, - "scope-other-type", + "Company", + "scope-other-label", UpsertNodeOptions { weight: 0.95, dense_vector: Some(vec![1.0, 0.0]), @@ -14074,9 +15503,9 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an }, ) .unwrap(); - let wrong_edge_type = engine + let wrong_edge_label = engine .upsert_node( - 1, + "Person", "scope-wrong-edge", UpsertNodeOptions { weight: 0.9, @@ -14087,7 +15516,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .unwrap(); let temporal_old = engine .upsert_node( - 1, + "Person", "scope-temporal-old", UpsertNodeOptions { weight: 0.9, @@ -14098,7 +15527,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .unwrap(); let temporal_live = engine .upsert_node( - 1, + "Person", "scope-temporal-live", UpsertNodeOptions { weight: 0.9, @@ -14109,7 +15538,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .unwrap(); let unreachable = engine .upsert_node( - 1, + "Person", "scope-unreachable", UpsertNodeOptions { weight: 0.95, @@ -14119,12 +15548,12 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an ) .unwrap(); - for &node_id in &[keep, shadowed, deleted, pruned, other_type] { + for &node_id in &[keep, shadowed, deleted, pruned, other_label] { engine .upsert_edge( start, node_id, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(0), valid_to: Some(10_000), @@ -14136,8 +15565,8 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an engine .upsert_edge( start, - wrong_edge_type, - 20, + wrong_edge_label, + "REPORTS_TO", UpsertEdgeOptions { valid_from: Some(0), valid_to: Some(10_000), @@ -14149,7 +15578,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .upsert_edge( start, temporal_old, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(0), valid_to: Some(4_000), @@ -14161,7 +15590,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an .upsert_edge( start, temporal_live, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(4_500), valid_to: Some(9_000), @@ -14173,7 +15602,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an let shadowed_updated = engine .upsert_node( - 1, + "Person", "scope-shadowed", UpsertNodeOptions { weight: 0.95, @@ -14190,7 +15619,7 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -14198,9 +15627,9 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an let request = scoped_dense_search_request( vec![1.0, 0.0], 10, - Some(vec![1]), + Some(vec!["Person"]), Some(8), - vector_search_scope(start, 1, Direction::Outgoing, Some(vec![10]), Some(5_000)), + vector_search_scope(start, 1, Direction::Outgoing, Some(vec!["KNOWS"]), Some(5_000)), ); let hits = engine.vector_search(&request).unwrap(); let returned_ids: Vec = hits.iter().map(|hit| hit.node_id).collect(); @@ -14208,8 +15637,8 @@ fn test_vector_search_dense_scope_combines_start_edge_filters_temporal_policy_an assert_eq!(returned_ids, vec![shadowed, keep, temporal_live, start]); assert!(returned_ids.iter().all(|id| *id != deleted)); assert!(returned_ids.iter().all(|id| *id != pruned)); - assert!(returned_ids.iter().all(|id| *id != other_type)); - assert!(returned_ids.iter().all(|id| *id != wrong_edge_type)); + assert!(returned_ids.iter().all(|id| *id != other_label)); + assert!(returned_ids.iter().all(|id| *id != wrong_edge_label)); assert!(returned_ids.iter().all(|id| *id != temporal_old)); assert!(returned_ids.iter().all(|id| *id != unreachable)); } @@ -14231,13 +15660,13 @@ fn test_vector_search_dense_scope_large_reachable_set_excludes_better_unreachabl let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); let start = engine - .upsert_node(1, "dense-scope-start", UpsertNodeOptions::default()) + .upsert_node("Person", "dense-scope-start", UpsertNodeOptions::default()) .unwrap(); let mut reachable_ids = Vec::new(); for index in 0..2055 { let node_id = engine .upsert_node( - 1, + "Person", &format!("dense-scope-r-{index}"), UpsertNodeOptions { dense_vector: Some(vec![1.0 - index as f32 * 0.0001, index as f32 * 0.0001]), @@ -14246,7 +15675,7 @@ fn test_vector_search_dense_scope_large_reachable_set_excludes_better_unreachabl ) .unwrap(); engine - .upsert_edge(start, node_id, 10, UpsertEdgeOptions::default()) + .upsert_edge(start, node_id, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); reachable_ids.push(node_id); } @@ -14254,7 +15683,7 @@ fn test_vector_search_dense_scope_large_reachable_set_excludes_better_unreachabl for index in 0..16 { let node_id = engine .upsert_node( - 1, + "Person", &format!("dense-scope-u-{index}"), UpsertNodeOptions { dense_vector: Some(vec![1.0, index as f32 * 0.00001]), @@ -14271,7 +15700,7 @@ fn test_vector_search_dense_scope_large_reachable_set_excludes_better_unreachabl 5, None, Some(32), - vector_search_scope(start, 1, Direction::Outgoing, Some(vec![10]), None), + vector_search_scope(start, 1, Direction::Outgoing, Some(vec!["KNOWS"]), None), ); let hits = engine.vector_search(&request).unwrap(); let returned_ids: Vec = hits.iter().map(|hit| hit.node_id).collect(); @@ -14298,7 +15727,7 @@ fn test_vector_search_dense_scope_compaction_and_reopen_parity() { let start = engine .upsert_node( - 1, + "Person", "dense-scope-parity-start", UpsertNodeOptions { weight: 0.8, @@ -14309,7 +15738,7 @@ fn test_vector_search_dense_scope_compaction_and_reopen_parity() { .unwrap(); let stable = engine .upsert_node( - 1, + "Person", "dense-scope-stable", UpsertNodeOptions { weight: 0.9, @@ -14320,7 +15749,7 @@ fn test_vector_search_dense_scope_compaction_and_reopen_parity() { .unwrap(); let shared_old = engine .upsert_node( - 1, + "Person", "dense-scope-shared", UpsertNodeOptions { weight: 0.8, @@ -14331,7 +15760,7 @@ fn test_vector_search_dense_scope_compaction_and_reopen_parity() { .unwrap(); let deleted = engine .upsert_node( - 1, + "Person", "dense-scope-deleted", UpsertNodeOptions { weight: 0.95, @@ -14342,14 +15771,14 @@ fn test_vector_search_dense_scope_compaction_and_reopen_parity() { .unwrap(); for &node_id in &[stable, shared_old, deleted] { engine - .upsert_edge(start, node_id, 10, UpsertEdgeOptions::default()) + .upsert_edge(start, node_id, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } engine.flush().unwrap(); let shared_new = engine .upsert_node( - 1, + "Person", "dense-scope-shared", UpsertNodeOptions { weight: 0.95, @@ -14365,9 +15794,9 @@ fn test_vector_search_dense_scope_compaction_and_reopen_parity() { let request = scoped_dense_search_request( vec![1.0, 0.0], 3, - Some(vec![1]), + Some(vec!["Person"]), Some(8), - vector_search_scope(start, 1, Direction::Outgoing, Some(vec![10]), None), + vector_search_scope(start, 1, Direction::Outgoing, Some(vec!["KNOWS"]), None), ); let before = engine.vector_search(&request).unwrap(); let before_ids: Vec = before.iter().map(|hit| hit.node_id).collect(); @@ -14401,7 +15830,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { // Segment 1: A, B, C, D let a = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.9, @@ -14412,7 +15841,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "node-b", UpsertNodeOptions { weight: 0.9, @@ -14423,7 +15852,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "node-c", UpsertNodeOptions { weight: 0.9, @@ -14434,7 +15863,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "node-d", UpsertNodeOptions { weight: 0.9, @@ -14448,7 +15877,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { // Segment 2: shadow A with a different vector, add E let a2 = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.9, @@ -14460,7 +15889,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { assert_eq!(a, a2); let e = engine .upsert_node( - 1, + "Person", "node-e", UpsertNodeOptions { weight: 0.9, @@ -14475,7 +15904,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { engine.delete_node(b).unwrap(); let f = engine .upsert_node( - 1, + "Person", "node-f", UpsertNodeOptions { weight: 0.9, @@ -14486,10 +15915,10 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { .unwrap(); engine.flush().unwrap(); - // Segment 4: shadow C with a very different vector, add G (type_id=2) + // Segment 4: shadow C with a very different vector, add G with a non-matching label. let c2 = engine .upsert_node( - 1, + "Person", "node-c", UpsertNodeOptions { weight: 0.9, @@ -14501,7 +15930,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { assert_eq!(c, c2); let _g = engine .upsert_node( - 2, + "Company", "node-g", UpsertNodeOptions { weight: 0.9, @@ -14512,18 +15941,18 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { .unwrap(); engine.flush().unwrap(); - // Query: [1.0, 0.0], k=5, type_filter=[1] - // Surviving type_id=1 nodes with their newest vectors: + // Query: [1.0, 0.0], k=5, label_filter=["Person"] + // Surviving Person nodes with their newest vectors: // A: [0.5, 0.5] (shadowed in seg 2) // B: deleted (seg 3) // C: [0.1, 0.9] (shadowed in seg 4) // D: [0.3, 0.7] (original, seg 1) // E: [0.9, 0.1] (seg 2) // F: [0.95, 0.05] (seg 3) - // G: type_id=2 → filtered out + // G: Company → filtered out let query = vec![1.0, 0.0]; let hits = engine - .vector_search(&dense_search_request(query.clone(), 5, Some(vec![1]), None)) + .vector_search(&dense_search_request(query.clone(), 5, Some(vec!["Person"]), None)) .unwrap(); // Compute expected scores and sort by (score DESC, node_id ASC). @@ -14576,7 +16005,7 @@ fn test_vector_search_dense_four_segment_visibility_and_ordering() { #[test] fn test_vector_search_sparse_four_segment_visibility_and_ordering() { - // 4 segments exercising shadowing, tombstones, type filter, and exact score assertions. + // 4 segments exercising shadowing, tombstones, label filtering, and exact score assertions. // All 4 segments contain sparse data → sparse_segment_count >= 2 → parallel path exercised. let dir = TempDir::new().unwrap(); let opts = DbOptions { @@ -14588,7 +16017,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { // Segment 0 (oldest): A, B, C, D let a = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.9, @@ -14599,7 +16028,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { .unwrap(); let _b = engine .upsert_node( - 2, + "Company", "node-b", UpsertNodeOptions { weight: 0.9, @@ -14610,7 +16039,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { .unwrap(); let c = engine .upsert_node( - 1, + "Person", "node-c", UpsertNodeOptions { weight: 0.9, @@ -14621,7 +16050,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { .unwrap(); let d = engine .upsert_node( - 1, + "Person", "node-d", UpsertNodeOptions { weight: 0.9, @@ -14635,7 +16064,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { // Segment 1: shadow A with higher score, add E let a2 = engine .upsert_node( - 1, + "Person", "node-a", UpsertNodeOptions { weight: 0.9, @@ -14647,7 +16076,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { assert_eq!(a, a2); let e = engine .upsert_node( - 1, + "Person", "node-e", UpsertNodeOptions { weight: 0.9, @@ -14662,7 +16091,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { engine.delete_node(c).unwrap(); let f = engine .upsert_node( - 1, + "Person", "node-f", UpsertNodeOptions { weight: 0.9, @@ -14674,11 +16103,11 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { engine.flush().unwrap(); // Segment 3 (newest): shadow D with non-overlapping sparse vector, add G - // Same (type_id=1, key="node-d") so this shadows D's old entry. + // Same (label=Person, key="node-d") so this shadows D's old entry. // New vector [(99, 5.0)] has zero overlap with query dim 3 → score 0.0. let d2 = engine .upsert_node( - 1, + "Person", "node-d", UpsertNodeOptions { weight: 0.9, @@ -14690,7 +16119,7 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { assert_eq!(d, d2); let g = engine .upsert_node( - 1, + "Person", "node-g", UpsertNodeOptions { weight: 0.9, @@ -14701,17 +16130,17 @@ fn test_vector_search_sparse_four_segment_visibility_and_ordering() { .unwrap(); engine.flush().unwrap(); - // Query: sparse [(3, 1.0)], k=10, type_filter=[1] - // Surviving type_id=1 nodes with their newest sparse vectors: + // Query: sparse [(3, 1.0)], k=10, label_filter=["Person"] + // Surviving Person nodes with their newest sparse vectors: // A: [(3, 10.0)] (shadowed in seg 1) → score 10.0 - // B: type_id=2 → filtered out + // B: Company → filtered out // C: deleted (seg 2) → excluded // D: [(99, 5.0)] (shadowed in seg 3) → score 0.0, excluded // E: [(3, 0.5)] (seg 1) → score 0.5 // F: [(3, 1.5)] (seg 2) → score 1.5 // G: [(3, 0.8)] (seg 3) → score 0.8 let hits = engine - .vector_search(&sparse_search_request(vec![(3, 1.0)], 10, Some(vec![1]))) + .vector_search(&sparse_search_request(vec![(3, 1.0)], 10, Some(vec!["Person"]))) .unwrap(); let expected: Vec<(u64, f32)> = vec![(a, 10.0), (f, 1.5), (g, 0.8), (e, 0.5)]; @@ -14745,7 +16174,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a let start = engine .upsert_node( - 1, + "Person", "scope-start", UpsertNodeOptions { weight: 0.9, @@ -14756,7 +16185,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .unwrap(); let keep = engine .upsert_node( - 1, + "Person", "scope-keep", UpsertNodeOptions { weight: 0.9, @@ -14767,7 +16196,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .unwrap(); let shadowed = engine .upsert_node( - 1, + "Person", "scope-shadowed", UpsertNodeOptions { weight: 0.9, @@ -14778,7 +16207,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .unwrap(); let deleted = engine .upsert_node( - 1, + "Person", "scope-deleted", UpsertNodeOptions { weight: 0.9, @@ -14789,7 +16218,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .unwrap(); let pruned = engine .upsert_node( - 1, + "Person", "scope-pruned", UpsertNodeOptions { weight: 0.1, @@ -14798,10 +16227,10 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a }, ) .unwrap(); - let other_type = engine + let other_label = engine .upsert_node( - 2, - "scope-other-type", + "Company", + "scope-other-label", UpsertNodeOptions { weight: 0.95, sparse_vector: Some(vec![(3, 4.0)]), @@ -14809,9 +16238,9 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a }, ) .unwrap(); - let wrong_edge_type = engine + let wrong_edge_label = engine .upsert_node( - 1, + "Person", "scope-wrong-edge", UpsertNodeOptions { weight: 0.9, @@ -14822,7 +16251,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .unwrap(); let temporal_old = engine .upsert_node( - 1, + "Person", "scope-temporal-old", UpsertNodeOptions { weight: 0.9, @@ -14833,7 +16262,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .unwrap(); let temporal_live = engine .upsert_node( - 1, + "Person", "scope-temporal-live", UpsertNodeOptions { weight: 0.9, @@ -14844,7 +16273,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .unwrap(); let unreachable = engine .upsert_node( - 1, + "Person", "scope-unreachable", UpsertNodeOptions { weight: 0.95, @@ -14854,12 +16283,12 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a ) .unwrap(); - for &node_id in &[keep, shadowed, deleted, pruned, other_type] { + for &node_id in &[keep, shadowed, deleted, pruned, other_label] { engine .upsert_edge( start, node_id, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(0), valid_to: Some(10_000), @@ -14871,8 +16300,8 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a engine .upsert_edge( start, - wrong_edge_type, - 20, + wrong_edge_label, + "REPORTS_TO", UpsertEdgeOptions { valid_from: Some(0), valid_to: Some(10_000), @@ -14884,7 +16313,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .upsert_edge( start, temporal_old, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(0), valid_to: Some(4_000), @@ -14896,7 +16325,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a .upsert_edge( start, temporal_live, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(4_500), valid_to: Some(9_000), @@ -14908,7 +16337,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a let shadowed_updated = engine .upsert_node( - 1, + "Person", "scope-shadowed", UpsertNodeOptions { weight: 0.95, @@ -14925,7 +16354,7 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); @@ -14933,8 +16362,8 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a let request = scoped_sparse_search_request( vec![(3, 1.0)], 10, - Some(vec![1]), - vector_search_scope(start, 1, Direction::Outgoing, Some(vec![10]), Some(5_000)), + Some(vec!["Person"]), + vector_search_scope(start, 1, Direction::Outgoing, Some(vec!["KNOWS"]), Some(5_000)), ); let hits = engine.vector_search(&request).unwrap(); let returned_ids: Vec = hits.iter().map(|hit| hit.node_id).collect(); @@ -14942,8 +16371,8 @@ fn test_vector_search_sparse_scope_combines_start_edge_filters_temporal_policy_a assert_eq!(returned_ids, vec![shadowed, keep, temporal_live, start]); assert!(returned_ids.iter().all(|id| *id != deleted)); assert!(returned_ids.iter().all(|id| *id != pruned)); - assert!(returned_ids.iter().all(|id| *id != other_type)); - assert!(returned_ids.iter().all(|id| *id != wrong_edge_type)); + assert!(returned_ids.iter().all(|id| *id != other_label)); + assert!(returned_ids.iter().all(|id| *id != wrong_edge_label)); assert!(returned_ids.iter().all(|id| *id != temporal_old)); assert!(returned_ids.iter().all(|id| *id != unreachable)); } @@ -14954,13 +16383,13 @@ fn test_vector_search_sparse_scope_large_reachable_set_excludes_better_unreachab let engine = DatabaseEngine::open(dir.path(), &DbOptions::default()).unwrap(); let start = engine - .upsert_node(1, "sparse-scope-start", UpsertNodeOptions::default()) + .upsert_node("Person", "sparse-scope-start", UpsertNodeOptions::default()) .unwrap(); let mut reachable_ids = Vec::new(); for index in 0..2055 { let node_id = engine .upsert_node( - 1, + "Person", &format!("sparse-scope-r-{index}"), UpsertNodeOptions { sparse_vector: Some(vec![(1, 1.0 - index as f32 * 0.0001)]), @@ -14969,7 +16398,7 @@ fn test_vector_search_sparse_scope_large_reachable_set_excludes_better_unreachab ) .unwrap(); engine - .upsert_edge(start, node_id, 10, UpsertEdgeOptions::default()) + .upsert_edge(start, node_id, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); reachable_ids.push(node_id); } @@ -14977,7 +16406,7 @@ fn test_vector_search_sparse_scope_large_reachable_set_excludes_better_unreachab for index in 0..16 { let node_id = engine .upsert_node( - 1, + "Person", &format!("sparse-scope-u-{index}"), UpsertNodeOptions { sparse_vector: Some(vec![(1, 2.0 + index as f32)]), @@ -14993,7 +16422,7 @@ fn test_vector_search_sparse_scope_large_reachable_set_excludes_better_unreachab vec![(1, 1.0)], 5, None, - vector_search_scope(start, 1, Direction::Outgoing, Some(vec![10]), None), + vector_search_scope(start, 1, Direction::Outgoing, Some(vec!["KNOWS"]), None), ); let hits = engine.vector_search(&request).unwrap(); let returned_ids: Vec = hits.iter().map(|hit| hit.node_id).collect(); @@ -15015,7 +16444,7 @@ fn test_vector_search_sparse_scope_compaction_and_reopen_parity() { let start = engine .upsert_node( - 1, + "Person", "sparse-scope-parity-start", UpsertNodeOptions { weight: 0.8, @@ -15026,7 +16455,7 @@ fn test_vector_search_sparse_scope_compaction_and_reopen_parity() { .unwrap(); let stable = engine .upsert_node( - 1, + "Person", "sparse-scope-stable", UpsertNodeOptions { weight: 0.9, @@ -15037,7 +16466,7 @@ fn test_vector_search_sparse_scope_compaction_and_reopen_parity() { .unwrap(); let shared_old = engine .upsert_node( - 1, + "Person", "sparse-scope-shared", UpsertNodeOptions { weight: 0.8, @@ -15048,7 +16477,7 @@ fn test_vector_search_sparse_scope_compaction_and_reopen_parity() { .unwrap(); let deleted = engine .upsert_node( - 1, + "Person", "sparse-scope-deleted", UpsertNodeOptions { weight: 0.95, @@ -15059,14 +16488,14 @@ fn test_vector_search_sparse_scope_compaction_and_reopen_parity() { .unwrap(); for &node_id in &[stable, shared_old, deleted] { engine - .upsert_edge(start, node_id, 10, UpsertEdgeOptions::default()) + .upsert_edge(start, node_id, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); } engine.flush().unwrap(); let shared_new = engine .upsert_node( - 1, + "Person", "sparse-scope-shared", UpsertNodeOptions { weight: 0.95, @@ -15082,8 +16511,8 @@ fn test_vector_search_sparse_scope_compaction_and_reopen_parity() { let request = scoped_sparse_search_request( vec![(3, 1.0)], 3, - Some(vec![1]), - vector_search_scope(start, 1, Direction::Outgoing, Some(vec![10]), None), + Some(vec!["Person"]), + vector_search_scope(start, 1, Direction::Outgoing, Some(vec!["KNOWS"]), None), ); let before = engine.vector_search(&request).unwrap(); let before_ids: Vec = before.iter().map(|hit| hit.node_id).collect(); @@ -15129,7 +16558,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() let start = engine .upsert_node( - 1, + "Person", "scope-oracle-start", UpsertNodeOptions { weight: 0.9, @@ -15141,7 +16570,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .unwrap(); let reachable_a = engine .upsert_node( - 1, + "Person", "scope-oracle-a", UpsertNodeOptions { weight: 0.9, @@ -15153,7 +16582,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .unwrap(); let reachable_b_old = engine .upsert_node( - 1, + "Person", "scope-oracle-b", UpsertNodeOptions { weight: 0.8, @@ -15165,7 +16594,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .unwrap(); let reachable_deleted = engine .upsert_node( - 1, + "Person", "scope-oracle-deleted", UpsertNodeOptions { weight: 0.9, @@ -15175,10 +16604,10 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() }, ) .unwrap(); - let reachable_other_type = engine + let reachable_other_label = engine .upsert_node( - 2, - "scope-oracle-type2", + "Company", + "scope-oracle-company", UpsertNodeOptions { weight: 0.9, dense_vector: Some(vec![0.96, 0.04]), @@ -15189,7 +16618,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .unwrap(); let unreachable = engine .upsert_node( - 1, + "Person", "scope-oracle-unreachable", UpsertNodeOptions { weight: 0.9, @@ -15201,24 +16630,24 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .unwrap(); engine - .upsert_edge(start, reachable_a, 10, UpsertEdgeOptions::default()) + .upsert_edge(start, reachable_a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine .upsert_edge( reachable_a, reachable_b_old, - 10, + "KNOWS", UpsertEdgeOptions::default(), ) .unwrap(); engine - .upsert_edge(start, reachable_deleted, 10, UpsertEdgeOptions::default()) + .upsert_edge(start, reachable_deleted, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); engine .upsert_edge( start, - reachable_other_type, - 10, + reachable_other_label, + "KNOWS", UpsertEdgeOptions::default(), ) .unwrap(); @@ -15226,7 +16655,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() let reachable_b_new = engine .upsert_node( - 1, + "Person", "scope-oracle-b", UpsertNodeOptions { weight: 0.95, @@ -15240,14 +16669,14 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() engine.delete_node(reachable_deleted).unwrap(); assert!(engine.get_node(unreachable).unwrap().is_some()); - let scope = vector_search_scope(start, 2, Direction::Outgoing, Some(vec![10]), None); + let scope = vector_search_scope(start, 2, Direction::Outgoing, Some(vec!["KNOWS"]), None); let reachable_ids: std::collections::HashSet = engine .traverse( start, 2, &TraverseOptions { min_depth: 0, - edge_type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) @@ -15262,7 +16691,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() start, reachable_a, reachable_b_old, - reachable_other_type + reachable_other_label ]) ); @@ -15270,7 +16699,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .vector_search(&scoped_dense_search_request( vec![1.0, 0.0], 3, - Some(vec![1]), + Some(vec!["Person"]), Some(8), scope.clone(), )) @@ -15279,7 +16708,7 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .vector_search(&dense_search_request( vec![1.0, 0.0], 16, - Some(vec![1]), + Some(vec!["Person"]), Some(8), )) .unwrap(); @@ -15294,12 +16723,12 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() .vector_search(&scoped_sparse_search_request( vec![(1, 1.0)], 3, - Some(vec![1]), + Some(vec!["Person"]), scope, )) .unwrap(); let sparse_unscoped = engine - .vector_search(&sparse_search_request(vec![(1, 1.0)], 16, Some(vec![1]))) + .vector_search(&sparse_search_request(vec![(1, 1.0)], 16, Some(vec!["Person"]))) .unwrap(); let sparse_expected = filter_hits_by_scope(&sparse_unscoped, &reachable_ids, 3); assert_vector_hits_match(&sparse_scoped, &sparse_expected); @@ -15312,41 +16741,43 @@ fn test_vector_search_scope_matches_unscoped_results_filtered_by_reachable_ids() #[test] fn test_v3_matches_any_prune_policy_meta() { // Unit test for the metadata-based prune policy matcher - let policy_weight = PrunePolicy { + let label = |label_id| NodeLabelSet::single(label_id).unwrap(); + let labels = |label_ids: &[u32]| NodeLabelSet::from_canonical_ids(label_ids).unwrap(); + let policy_weight = ResolvedPrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label_id: None, }; - let policy_type_scoped = PrunePolicy { + let policy_label_scoped = ResolvedPrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: Some(1), + label_id: Some(1), }; - let policy_age = PrunePolicy { + let policy_age = ResolvedPrunePolicy { max_age_ms: Some(1000), max_weight: None, - type_id: None, + label_id: None, }; let now = 10_000i64; // Weight-only policy assert!(matches_any_prune_policy_meta( - 1, + &label(1), now, 0.1, std::slice::from_ref(&policy_weight), now )); assert!(matches_any_prune_policy_meta( - 1, + &label(1), now, 0.5, std::slice::from_ref(&policy_weight), now )); assert!(!matches_any_prune_policy_meta( - 1, + &label(1), now, 0.6, std::slice::from_ref(&policy_weight), @@ -15355,30 +16786,37 @@ fn test_v3_matches_any_prune_policy_meta() { // Type-scoped policy assert!(matches_any_prune_policy_meta( - 1, + &label(1), now, 0.1, - std::slice::from_ref(&policy_type_scoped), + std::slice::from_ref(&policy_label_scoped), now )); assert!(!matches_any_prune_policy_meta( - 2, + &label(2), now, 0.1, - std::slice::from_ref(&policy_type_scoped), + std::slice::from_ref(&policy_label_scoped), now )); // Wrong type + assert!(matches_any_prune_policy_meta( + &labels(&[1, 2]), + now, + 0.1, + std::slice::from_ref(&policy_label_scoped), + now + )); // Multi-label membership // Age-only policy: updated_at < now - max_age_ms = 10000 - 1000 = 9000 assert!(matches_any_prune_policy_meta( - 1, + &label(1), 8000, 1.0, std::slice::from_ref(&policy_age), now )); // Old enough assert!(!matches_any_prune_policy_meta( - 1, + &label(1), 9500, 1.0, std::slice::from_ref(&policy_age), @@ -15386,23 +16824,41 @@ fn test_v3_matches_any_prune_policy_meta() { )); // Too recent // OR across policies - let policies = vec![policy_type_scoped.clone(), policy_age.clone()]; + let policies = vec![policy_label_scoped.clone(), policy_age.clone()]; // Matches type-scoped (type=1, weight=0.1) - assert!(matches_any_prune_policy_meta(1, now, 0.1, &policies, now)); + assert!(matches_any_prune_policy_meta( + &label(1), + now, + 0.1, + &policies, + now + )); // Matches age (old enough) - assert!(matches_any_prune_policy_meta(5, 8000, 1.0, &policies, now)); + assert!(matches_any_prune_policy_meta( + &label(5), + 8000, + 1.0, + &policies, + now + )); // Matches neither - assert!(!matches_any_prune_policy_meta(5, now, 1.0, &policies, now)); + assert!(!matches_any_prune_policy_meta( + &label(5), + now, + 1.0, + &policies, + now + )); // AND within policy: both age AND weight must match - let policy_combo = PrunePolicy { + let policy_combo = ResolvedPrunePolicy { max_age_ms: Some(1000), max_weight: Some(0.5), - type_id: None, + label_id: None, }; // Old AND low weight → prune assert!(matches_any_prune_policy_meta( - 1, + &label(1), 8000, 0.1, std::slice::from_ref(&policy_combo), @@ -15410,7 +16866,7 @@ fn test_v3_matches_any_prune_policy_meta() { )); // Old but high weight → no prune assert!(!matches_any_prune_policy_meta( - 1, + &label(1), 8000, 1.0, std::slice::from_ref(&policy_combo), @@ -15418,15 +16874,499 @@ fn test_v3_matches_any_prune_policy_meta() { )); // Recent but low weight → no prune assert!(!matches_any_prune_policy_meta( - 1, + &label(1), 9500, 0.1, std::slice::from_ref(&policy_combo), now )); - // Empty policies → never match - assert!(!matches_any_prune_policy_meta(1, 0, 0.0, &[], now)); + // Empty policies → never match + assert!(!matches_any_prune_policy_meta(&label(1), 0, 0.0, &[], now)); +} + +fn make_compaction_test_node( + id: u64, + label_ids: &[u32], + key: &str, + updated_at: i64, + weight: f32, +) -> NodeRecord { + NodeRecord { + id, + label_ids: NodeLabelSet::from_canonical_ids(label_ids).unwrap(), + key: key.to_string(), + props: BTreeMap::new(), + created_at: 1000, + updated_at, + weight, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + } +} + +fn make_compaction_test_node_with_props( + id: u64, + label_ids: &[u32], + key: &str, + props: BTreeMap, + updated_at: i64, + weight: f32, +) -> NodeRecord { + NodeRecord { + props, + ..make_compaction_test_node(id, label_ids, key, updated_at, weight) + } +} + +fn write_compaction_test_segment( + seg_dir: &std::path::Path, + segment_id: u64, + ops: Vec, +) -> std::sync::Arc { + write_compaction_test_segment_with_secondary_indexes(seg_dir, segment_id, ops, &[]) +} + +fn write_compaction_test_segment_with_secondary_indexes( + seg_dir: &std::path::Path, + segment_id: u64, + ops: Vec, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> std::sync::Arc { + let mt = Memtable::new(); + for entry in secondary_indexes { + mt.register_secondary_index(entry); + } + for (idx, op) in ops.iter().enumerate() { + mt.apply_op(op, (idx + 1) as u64); + } + let degree_overlay = crate::degree_cache::DegreeOverlaySnapshot::empty(); + let info = crate::segment_writer::write_segment_with_degree_overlay_and_secondary_indexes( + seg_dir, + segment_id, + &mt, + None, + degree_overlay.as_ref(), + secondary_indexes, + ) + .unwrap(); + std::sync::Arc::new( + SegmentReader::open_with_info(seg_dir, &info, None, secondary_indexes).unwrap(), + ) +} + +fn compact_test_segments( + out_dir: &std::path::Path, + out_segment_id: u64, + segments: Vec>, + prune_policies: &[ResolvedPrunePolicy], +) -> (SegmentInfo, u64, u64, SegmentReader) { + compact_test_segments_with_secondary_indexes( + out_dir, + out_segment_id, + segments, + prune_policies, + &[], + ) +} + +fn compact_test_segments_with_secondary_indexes( + out_dir: &std::path::Path, + out_segment_id: u64, + segments: Vec>, + prune_policies: &[ResolvedPrunePolicy], + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> (SegmentInfo, u64, u64, SegmentReader) { + let has_tombstones = segments.iter().any(|segment| segment.has_tombstones()); + let cancel = std::sync::atomic::AtomicBool::new(false); + let (info, nodes_auto_pruned, edges_auto_pruned, _report) = bg_standard_merge( + &segments, + out_dir, + out_segment_id, + has_tombstones, + prune_policies, + None, + secondary_indexes, + &cancel, + ) + .unwrap(); + let reader = SegmentReader::open_with_info(out_dir, &info, None, secondary_indexes).unwrap(); + (info, nodes_auto_pruned, edges_auto_pruned, reader) +} + +#[test] +fn test_multi_label_v3_compaction_tombstone_and_replacement_memberships() { + let dir = TempDir::new().unwrap(); + let older_dir = dir.path().join("seg_0001"); + let newer_dir = dir.path().join("seg_0002"); + let out_dir = dir.path().join("seg_0003"); + + let older = write_compaction_test_segment( + &older_dir, + 1, + vec![ + WalOp::UpsertNode(make_compaction_test_node(1, &[1, 2], "survive", 100, 1.0)), + WalOp::UpsertNode(make_compaction_test_node(2, &[1, 2], "delete", 100, 1.0)), + WalOp::UpsertNode(make_compaction_test_node(3, &[1, 2], "replace", 100, 1.0)), + ], + ); + let newer = write_compaction_test_segment( + &newer_dir, + 2, + vec![ + WalOp::DeleteNode { + id: 2, + deleted_at: 200, + }, + WalOp::UpsertNode(make_compaction_test_node(3, &[2, 3], "replace", 300, 1.0)), + ], + ); + + let (info, nodes_auto_pruned, edges_auto_pruned, reader) = + compact_test_segments(&out_dir, 3, vec![newer, older], &[]); + assert_eq!(nodes_auto_pruned, 0); + assert_eq!(edges_auto_pruned, 0); + assert_eq!(info.node_count, 2); + + assert_eq!( + reader.get_node(1).unwrap().unwrap().label_ids.as_slice(), + &[1, 2] + ); + assert!(reader.get_node(2).unwrap().is_none()); + assert_eq!( + reader.get_node(3).unwrap().unwrap().label_ids.as_slice(), + &[2, 3] + ); + + assert_eq!( + reader.node_by_key(1, "survive").unwrap().map(|node| node.id), + Some(1) + ); + assert_eq!( + reader.node_by_key(2, "survive").unwrap().map(|node| node.id), + Some(1) + ); + assert!(reader.node_by_key(1, "delete").unwrap().is_none()); + assert!(reader.node_by_key(2, "delete").unwrap().is_none()); + assert!(reader.node_by_key(1, "replace").unwrap().is_none()); + assert_eq!( + reader.node_by_key(2, "replace").unwrap().map(|node| node.id), + Some(3) + ); + assert_eq!( + reader.node_by_key(3, "replace").unwrap().map(|node| node.id), + Some(3) + ); + + assert_eq!(reader.nodes_by_label_id(1).unwrap(), vec![1]); + assert_eq!(reader.nodes_by_label_id(2).unwrap(), vec![1, 3]); + assert_eq!(reader.nodes_by_label_id(3).unwrap(), vec![3]); + assert!(reader.nodes_by_time_range(1, 300, 300).unwrap().is_empty()); + assert_eq!(reader.nodes_by_time_range(2, 300, 300).unwrap(), vec![3]); + assert_eq!(reader.nodes_by_time_range(3, 300, 300).unwrap(), vec![3]); + + let reopened = SegmentReader::open_with_info(&out_dir, &info, None, &[]).unwrap(); + assert_eq!( + reopened.get_node(3).unwrap().unwrap().label_ids.as_slice(), + &[2, 3] + ); + assert_eq!(reopened.nodes_by_label_id(2).unwrap(), vec![1, 3]); + assert_eq!( + reopened + .node_by_key(3, "replace") + .unwrap() + .map(|node| node.id), + Some(3) + ); +} + +#[test] +fn test_multi_label_prune_policy_compaction_cascades_edges_by_membership() { + let dir = TempDir::new().unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); + + let source = write_compaction_test_segment( + &source_dir, + 1, + vec![ + WalOp::UpsertNode(make_compaction_test_node(1, &[1, 5], "prune", 100, 0.1)), + WalOp::UpsertNode(make_compaction_test_node(2, &[1], "keep", 100, 1.0)), + WalOp::UpsertEdge(EdgeRecord { + id: 10, + from: 1, + to: 2, + label_id: 10, + props: BTreeMap::new(), + created_at: 100, + updated_at: 100, + weight: 1.0, + valid_from: 0, + valid_to: i64::MAX, + last_write_seq: 0, + }), + ], + ); + let policies = [ResolvedPrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label_id: Some(5), + }]; + + let (info, nodes_auto_pruned, edges_auto_pruned, reader) = + compact_test_segments(&out_dir, 2, vec![source], &policies); + assert_eq!(nodes_auto_pruned, 1); + assert_eq!(edges_auto_pruned, 1); + assert_eq!(info.node_count, 1); + assert_eq!(info.edge_count, 0); + assert!(reader.get_node(1).unwrap().is_none()); + assert_eq!( + reader.get_node(2).unwrap().unwrap().label_ids.as_slice(), + &[1] + ); + assert!(reader.get_edge(10).unwrap().is_none()); + assert!(reader.node_by_key(5, "prune").unwrap().is_none()); + assert_eq!(reader.nodes_by_label_id(1).unwrap(), vec![2]); +} + +#[test] +fn test_multi_label_compaction_rebuilds_declared_sidecars_via_targeted_decode() { + let dir = TempDir::new().unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); + + let eq_entry = SecondaryIndexManifestEntry { + index_id: 301, + target: SecondaryIndexTarget::NodeProperty { + label_id: 2, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_entry = SecondaryIndexManifestEntry { + index_id: 302, + target: SecondaryIndexTarget::NodeProperty { + label_id: 3, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let indexes = vec![eq_entry.clone(), range_entry.clone()]; + + let source = write_compaction_test_segment_with_secondary_indexes( + &source_dir, + 1, + vec![ + WalOp::UpsertNode(make_compaction_test_node_with_props( + 1, + &[1, 2], + "eq-only", + BTreeMap::from([ + ("color".to_string(), PropValue::String("red".to_string())), + ("score".to_string(), PropValue::Int(10)), + ]), + 100, + 1.0, + )), + WalOp::UpsertNode(make_compaction_test_node_with_props( + 2, + &[2, 3], + "both", + BTreeMap::from([ + ("color".to_string(), PropValue::String("red".to_string())), + ("score".to_string(), PropValue::Int(20)), + ]), + 200, + 1.0, + )), + WalOp::UpsertNode(make_compaction_test_node_with_props( + 3, + &[3], + "range-only", + BTreeMap::from([ + ("color".to_string(), PropValue::String("blue".to_string())), + ("score".to_string(), PropValue::Int(30)), + ]), + 300, + 1.0, + )), + ], + &indexes, + ); + + let eq_sidecar = + crate::segment_writer::node_prop_eq_sidecar_path(&source_dir, eq_entry.index_id); + let range_sidecar = + crate::segment_writer::node_prop_range_sidecar_path(&source_dir, range_entry.index_id); + std::fs::remove_file(&eq_sidecar).unwrap(); + std::fs::remove_file(&range_sidecar).unwrap(); + + let (_info, nodes_auto_pruned, edges_auto_pruned, reader) = + compact_test_segments_with_secondary_indexes(&out_dir, 2, vec![source], &[], &indexes); + assert_eq!(nodes_auto_pruned, 0); + assert_eq!(edges_auto_pruned, 0); + + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + assert_eq!( + reader + .find_nodes_by_secondary_eq_index(eq_entry.index_id, red_hash) + .unwrap(), + vec![1, 2] + ); + + let score_20 = crate::memtable::encode_range_prop_value( + SecondaryIndexRangeDomain::Int, + &PropValue::Int(20), + ) + .unwrap(); + let score_30 = crate::memtable::encode_range_prop_value( + SecondaryIndexRangeDomain::Int, + &PropValue::Int(30), + ) + .unwrap(); + assert_eq!( + reader + .find_nodes_by_secondary_range_index_if_present( + range_entry.index_id, + None, + None, + None, + ) + .unwrap(), + Some(vec![(score_20, 2), (score_30, 3)]) + ); +} + +#[test] +fn test_multi_label_compaction_drops_stale_declared_sidecar_memberships() { + let dir = TempDir::new().unwrap(); + let older_dir = dir.path().join("seg_0001"); + let newer_dir = dir.path().join("seg_0002"); + let out_dir = dir.path().join("seg_0003"); + + let eq_entry = SecondaryIndexManifestEntry { + index_id: 401, + target: SecondaryIndexTarget::NodeProperty { + label_id: 2, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_entry = SecondaryIndexManifestEntry { + index_id: 402, + target: SecondaryIndexTarget::NodeProperty { + label_id: 2, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let indexes = vec![eq_entry.clone(), range_entry.clone()]; + + let older = write_compaction_test_segment_with_secondary_indexes( + &older_dir, + 1, + vec![ + WalOp::UpsertNode(make_compaction_test_node_with_props( + 1, + &[1, 2], + "replacement", + BTreeMap::from([ + ("color".to_string(), PropValue::String("red".to_string())), + ("score".to_string(), PropValue::Int(10)), + ]), + 100, + 1.0, + )), + WalOp::UpsertNode(make_compaction_test_node_with_props( + 2, + &[2, 3], + "deleted", + BTreeMap::from([ + ("color".to_string(), PropValue::String("red".to_string())), + ("score".to_string(), PropValue::Int(20)), + ]), + 100, + 1.0, + )), + ], + &indexes, + ); + let newer = write_compaction_test_segment_with_secondary_indexes( + &newer_dir, + 2, + vec![ + WalOp::UpsertNode(make_compaction_test_node_with_props( + 1, + &[4], + "replacement", + BTreeMap::from([ + ("color".to_string(), PropValue::String("red".to_string())), + ("score".to_string(), PropValue::Int(10)), + ]), + 300, + 1.0, + )), + WalOp::DeleteNode { + id: 2, + deleted_at: 300, + }, + ], + &indexes, + ); + + let (info, nodes_auto_pruned, edges_auto_pruned, reader) = + compact_test_segments_with_secondary_indexes( + &out_dir, + 3, + vec![newer, older], + &[], + &indexes, + ); + assert_eq!(nodes_auto_pruned, 0); + assert_eq!(edges_auto_pruned, 0); + assert_eq!(info.node_count, 1); + assert_eq!( + reader.get_node(1).unwrap().unwrap().label_ids.as_slice(), + &[4] + ); + assert!(reader.get_node(2).unwrap().is_none()); + + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + assert_eq!( + reader + .find_nodes_by_secondary_eq_index(eq_entry.index_id, red_hash) + .unwrap(), + Vec::::new() + ); + assert_eq!( + reader + .find_nodes_by_secondary_range_index_if_present( + range_entry.index_id, + None, + None, + None, + ) + .unwrap(), + Some(Vec::new()) + ); + assert!(reader.node_by_key(2, "replacement").unwrap().is_none()); + assert_eq!( + reader.node_by_key(4, "replacement").unwrap().map(|node| node.id), + Some(1) + ); } // ========================================================================= @@ -15446,7 +17386,7 @@ fn test_v3_tombstone_overlap_stress() { let mut props = BTreeMap::new(); props.insert("round".into(), PropValue::Int(round as i64)); db.upsert_node( - 1, + "Person", &key, UpsertNodeOptions { props, @@ -15457,7 +17397,7 @@ fn test_v3_tombstone_overlap_stress() { .unwrap(); } for i in 0..19u64 { - db.upsert_edge(i + 1, i + 2, 1, UpsertEdgeOptions::default()) + db.upsert_edge(i + 1, i + 2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); } db.flush().unwrap(); @@ -15500,7 +17440,7 @@ fn test_v3_policy_or_and_semantics() { let mut props = BTreeMap::new(); props.insert("name".into(), PropValue::String(format!("t1_{}", i))); db.upsert_node( - 1, + "Person", &format!("t1_{}", i), UpsertNodeOptions { props, @@ -15514,7 +17454,7 @@ fn test_v3_policy_or_and_semantics() { let mut props = BTreeMap::new(); props.insert("name".into(), PropValue::String(format!("t2_{}", i))); db.upsert_node( - 2, + "Company", &format!("t2_{}", i), UpsertNodeOptions { props, @@ -15528,7 +17468,7 @@ fn test_v3_policy_or_and_semantics() { // Touch a node to create a second segment (compaction needs ≥2). db.upsert_node( - 1, + "Person", "t1_0", UpsertNodeOptions { weight: 0.1, @@ -15543,7 +17483,7 @@ fn test_v3_policy_or_and_semantics() { PrunePolicy { max_age_ms: None, max_weight: Some(0.3), - type_id: Some(1), + label: Some("Person".to_string()), }, ) .unwrap(); @@ -15552,19 +17492,19 @@ fn test_v3_policy_or_and_semantics() { for i in 0..10u64 { assert!( - db.get_node_by_key(2, &format!("t2_{}", i)) + db.get_node_by_key("Company", &format!("t2_{}", i)) .unwrap() .is_some(), - "type2 node t2_{} should survive", + "Company-labeled node t2_{} should survive", i ); } for i in 5..10u64 { assert!( - db.get_node_by_key(1, &format!("t1_{}", i)) + db.get_node_by_key("Person", &format!("t1_{}", i)) .unwrap() .is_some(), - "type1 node t1_{} (w=0.8) should survive", + "Person-labeled node t1_{} (w=0.8) should survive", i ); } @@ -15580,7 +17520,7 @@ fn test_v3_edge_cascade_high_fanout() { let hub = db .upsert_node( - 1, + "Person", "hub", UpsertNodeOptions { weight: 0.5, @@ -15592,7 +17532,7 @@ fn test_v3_edge_cascade_high_fanout() { for i in 0..100u64 { let spoke = db .upsert_node( - 1, + "Person", &format!("spoke_{}", i), UpsertNodeOptions { weight: 0.5, @@ -15601,9 +17541,9 @@ fn test_v3_edge_cascade_high_fanout() { ) .unwrap(); spoke_ids.push(spoke); - db.upsert_edge(hub, spoke, 1, UpsertEdgeOptions::default()) + db.upsert_edge(hub, spoke, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(spoke, hub, 2, UpsertEdgeOptions::default()) + db.upsert_edge(spoke, hub, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); } db.flush().unwrap(); @@ -15659,7 +17599,7 @@ fn test_v3_deterministic_output() { let mut props = BTreeMap::new(); props.insert("idx".into(), PropValue::Int(i as i64)); db.upsert_node( - i as u32 % 3 + 1, + ["Person", "Company", "Article"][i as usize % 3], &format!("n_{}", i), UpsertNodeOptions { props, @@ -15673,7 +17613,7 @@ fn test_v3_deterministic_output() { db.upsert_edge( i + 1, (i + 5) % 50 + 1, - i as u32 % 2 + 1, + ["RELATES_TO", "WORKS_AT"][i as usize % 2], UpsertEdgeOptions::default(), ) .unwrap(); @@ -15683,7 +17623,7 @@ fn test_v3_deterministic_output() { let mut props = BTreeMap::new(); props.insert("idx".into(), PropValue::Int(i as i64 + 100)); db.upsert_node( - i as u32 % 3 + 1, + ["Person", "Company", "Article"][i as usize % 3], &format!("n_{}", i), UpsertNodeOptions { props, @@ -15712,7 +17652,7 @@ fn test_v3_deterministic_output() { ); if let (Some(a), Some(b)) = (n1, n2) { assert_eq!(a.props, b.props, "node {} props mismatch", i + 1); - assert_eq!(a.type_id, b.type_id, "node {} type_id mismatch", i + 1); + assert_eq!(a.labels, b.labels, "node {} labels mismatch", i + 1); assert_eq!(a.key, b.key, "node {} key mismatch", i + 1); } } @@ -15739,7 +17679,7 @@ fn test_v3_index_parity() { props2.clone() }; db.upsert_node( - i as u32 % 2 + 1, + ["Person", "Company"][i as usize % 2], &format!("key_{}", i), UpsertNodeOptions { props, @@ -15750,8 +17690,13 @@ fn test_v3_index_parity() { .unwrap(); } for i in 0..8u64 { - db.upsert_edge(i + 1, i + 2, i as u32 % 3 + 1, UpsertEdgeOptions::default()) - .unwrap(); + db.upsert_edge( + i + 1, + i + 2, + ["RELATES_TO", "WORKS_AT", "LIKES"][i as usize % 3], + UpsertEdgeOptions::default(), + ) + .unwrap(); } db.flush().unwrap(); @@ -15760,7 +17705,7 @@ fn test_v3_index_parity() { props.insert("color".into(), PropValue::String("green".into())); props.insert("updated".into(), PropValue::Bool(true)); db.upsert_node( - i as u32 % 2 + 1, + ["Person", "Company"][i as usize % 2], &format!("key_{}", i), UpsertNodeOptions { props, @@ -15771,7 +17716,7 @@ fn test_v3_index_parity() { .unwrap(); } for i in 5..10u64 { - db.upsert_edge(i + 1, 1, 2, UpsertEdgeOptions::default()) + db.upsert_edge(i + 1, 1, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); } db.flush().unwrap(); @@ -15780,7 +17725,7 @@ fn test_v3_index_parity() { let pre_nodes: Vec<_> = (1..=10).map(|i| db.get_node(i).unwrap()).collect(); let pre_key_lookups: Vec<_> = (0..10u32) .map(|i| { - db.get_node_by_key(i % 2 + 1, &format!("key_{}", i)) + db.get_node_by_key(["Person", "Company"][i as usize % 2], &format!("key_{}", i)) .unwrap() }) .collect(); @@ -15796,10 +17741,10 @@ fn test_v3_index_parity() { .unwrap() }) .collect(); - let pre_type1 = db.nodes_by_type(1).unwrap(); - let pre_type2 = db.nodes_by_type(2).unwrap(); + let pre_label1 = db.nodes_by_labels("Person").unwrap(); + let pre_label2 = db.nodes_by_labels("Company").unwrap(); let pre_find = db - .find_nodes(1, "color", &PropValue::String("green".into())) + .find_nodes("Person", "color", &PropValue::String("green".into())) .unwrap(); db.compact().unwrap(); @@ -15807,8 +17752,8 @@ fn test_v3_index_parity() { for (i, pre_node) in pre_nodes.iter().enumerate().take(10) { let post = db.get_node(i as u64 + 1).unwrap(); assert_eq!( - pre_node.as_ref().map(|n| (&n.props, n.type_id)), - post.as_ref().map(|n| (&n.props, n.type_id)), + pre_node.as_ref().map(|n| (&n.props, &n.labels)), + post.as_ref().map(|n| (&n.props, &n.labels)), "get_node({}) mismatch", i + 1 ); @@ -15816,7 +17761,7 @@ fn test_v3_index_parity() { for i in 0..10u32 { let post = db - .get_node_by_key(i % 2 + 1, &format!("key_{}", i)) + .get_node_by_key(["Person", "Company"][i as usize % 2], &format!("key_{}", i)) .unwrap(); assert_eq!( pre_key_lookups[i as usize].as_ref().map(|n| n.id), @@ -15841,22 +17786,22 @@ fn test_v3_index_parity() { assert_eq!(pre_ids, post_ids, "neighbors({}) edge set mismatch", i + 1); } - let post_type1 = db.nodes_by_type(1).unwrap(); + let post_label1 = db.nodes_by_labels("Person").unwrap(); assert_eq!( - pre_type1.len(), - post_type1.len(), - "nodes_by_type(1) mismatch" + pre_label1.len(), + post_label1.len(), + "nodes_by_labels(Person) mismatch" ); - let post_type2 = db.nodes_by_type(2).unwrap(); + let post_label2 = db.nodes_by_labels("Company").unwrap(); assert_eq!( - pre_type2.len(), - post_type2.len(), - "nodes_by_type(2) mismatch" + pre_label2.len(), + post_label2.len(), + "nodes_by_labels(Company) mismatch" ); let post_find = db - .find_nodes(1, "color", &PropValue::String("green".into())) + .find_nodes("Person", "color", &PropValue::String("green".into())) .unwrap(); let pre_find_ids: NodeIdSet = pre_find.iter().copied().collect(); let post_find_ids: NodeIdSet = post_find.iter().copied().collect(); @@ -15872,7 +17817,7 @@ fn test_v3_mixed_workload_stress() { for i in 0..30u64 { db.upsert_node( - 1, + "Person", &format!("mix_{}", i), UpsertNodeOptions { weight: 0.5, @@ -15882,7 +17827,7 @@ fn test_v3_mixed_workload_stress() { .unwrap(); } for i in 0..20u64 { - db.upsert_edge(i + 1, i + 2, 1, UpsertEdgeOptions::default()) + db.upsert_edge(i + 1, i + 2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); } db.flush().unwrap(); @@ -15894,7 +17839,7 @@ fn test_v3_mixed_workload_stress() { let mut props = BTreeMap::new(); props.insert("updated".into(), PropValue::Bool(true)); db.upsert_node( - 1, + "Person", &format!("mix_{}", i), UpsertNodeOptions { props, @@ -15911,7 +17856,7 @@ fn test_v3_mixed_workload_stress() { let mut props = BTreeMap::new(); props.insert("round3".into(), PropValue::Int(3)); db.upsert_node( - 1, + "Person", &format!("mix_{}", i), UpsertNodeOptions { props, @@ -15923,7 +17868,7 @@ fn test_v3_mixed_workload_stress() { } for i in 30..40u64 { db.upsert_node( - 2, + "Company", &format!("new_{}", i), UpsertNodeOptions { weight: 0.5, @@ -15946,14 +17891,14 @@ fn test_v3_mixed_workload_stress() { } for i in 0..5u64 { let n = db - .get_node_by_key(1, &format!("mix_{}", i)) + .get_node_by_key("Person", &format!("mix_{}", i)) .unwrap() .unwrap(); assert_eq!(n.props.get("round3"), Some(&PropValue::Int(3))); } for i in 30..40u64 { assert!( - db.get_node_by_key(2, &format!("new_{}", i)) + db.get_node_by_key("Company", &format!("new_{}", i)) .unwrap() .is_some(), "new node new_{} should exist", @@ -15971,7 +17916,7 @@ fn test_v3_cross_segment_edges() { for i in 0..20u64 { db.upsert_node( - 1, + "Person", &format!("cs_{}", i), UpsertNodeOptions { weight: 0.5, @@ -15983,14 +17928,14 @@ fn test_v3_cross_segment_edges() { db.flush().unwrap(); for i in 0..19u64 { - db.upsert_edge(i + 1, i + 2, 1, UpsertEdgeOptions::default()) + db.upsert_edge(i + 1, i + 2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); } - db.upsert_edge(1, 10, 2, UpsertEdgeOptions::default()) + db.upsert_edge(1, 10, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(5, 15, 2, UpsertEdgeOptions::default()) + db.upsert_edge(5, 15, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(10, 20, 2, UpsertEdgeOptions::default()) + db.upsert_edge(10, 20, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -16032,7 +17977,7 @@ fn test_v3_reopen_durability() { let mut props = BTreeMap::new(); props.insert("name".into(), PropValue::String(format!("durable_{}", i))); db.upsert_node( - i as u32 % 3 + 1, + ["Person", "Company", "Article"][i as usize % 3], &format!("dur_{}", i), UpsertNodeOptions { props, @@ -16046,7 +17991,7 @@ fn test_v3_reopen_durability() { db.upsert_edge( i + 1, (i + 3) % 50 + 1, - i as u32 % 2 + 1, + ["RELATES_TO", "WORKS_AT"][i as usize % 2], UpsertEdgeOptions::default(), ) .unwrap(); @@ -16057,7 +18002,7 @@ fn test_v3_reopen_durability() { let mut props = BTreeMap::new(); props.insert("name".into(), PropValue::String(format!("updated_{}", i))); db.upsert_node( - i as u32 % 3 + 1, + ["Person", "Company", "Article"][i as usize % 3], &format!("dur_{}", i), UpsertNodeOptions { props, @@ -16105,15 +18050,15 @@ fn test_v3_reopen_durability() { } assert!( - db.get_node_by_key(1, "dur_0").unwrap().is_some(), + db.get_node_by_key("Person", "dur_0").unwrap().is_some(), "key lookup should work" ); let nbrs = db.neighbors(1, &NeighborOptions::default()).unwrap(); assert!(!nbrs.is_empty(), "adjacency should work after reopen"); - let type1 = db.nodes_by_type(1).unwrap(); - assert!(!type1.is_empty(), "type query should work after reopen"); + let label1 = db.nodes_by_labels("Person").unwrap(); + assert!(!label1.is_empty(), "label query should work after reopen"); } /// Fast-merge path: verify metadata-driven indexes for non-overlapping segments. @@ -16129,7 +18074,7 @@ fn test_v3_fast_merge_index_parity() { let mut props = BTreeMap::new(); props.insert("seg".into(), PropValue::Int(seg as i64)); db.upsert_node( - (seg as u32 % 2) + 1, + ["Person", "Company"][seg as usize % 2], &format!("fm_{}_{}", seg, i), UpsertNodeOptions { props, @@ -16140,16 +18085,16 @@ fn test_v3_fast_merge_index_parity() { .unwrap(); } for i in 0..9u64 { - db.upsert_edge(base + i + 1, base + i + 2, 1, UpsertEdgeOptions::default()) + db.upsert_edge(base + i + 1, base + i + 2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); } db.flush().unwrap(); } let pre_nodes: Vec<_> = (1..=30).filter_map(|i| db.get_node(i).unwrap()).collect(); - let pre_key_0 = db.get_node_by_key(1, "fm_0_0").unwrap(); + let pre_key_0 = db.get_node_by_key("Person", "fm_0_0").unwrap(); let pre_nbrs_1 = db.neighbors(1, &NeighborOptions::default()).unwrap(); - let pre_type1 = db.nodes_by_type(1).unwrap(); + let pre_label1 = db.nodes_by_labels("Person").unwrap(); db.compact().unwrap(); @@ -16158,10 +18103,10 @@ fn test_v3_fast_merge_index_parity() { for (pre, post) in pre_nodes.iter().zip(post_nodes.iter()) { assert_eq!(pre.id, post.id); assert_eq!(pre.props, post.props); - assert_eq!(pre.type_id, post.type_id); + assert_eq!(pre.labels, post.labels); } - let post_key_0 = db.get_node_by_key(1, "fm_0_0").unwrap(); + let post_key_0 = db.get_node_by_key("Person", "fm_0_0").unwrap(); assert_eq!( pre_key_0.map(|n| n.id), post_key_0.map(|n| n.id), @@ -16171,16 +18116,16 @@ fn test_v3_fast_merge_index_parity() { let post_nbrs_1 = db.neighbors(1, &NeighborOptions::default()).unwrap(); assert_eq!(pre_nbrs_1.len(), post_nbrs_1.len(), "neighbors mismatch"); - let post_type1 = db.nodes_by_type(1).unwrap(); - assert_eq!(pre_type1.len(), post_type1.len(), "type query mismatch"); + let post_label1 = db.nodes_by_labels("Person").unwrap(); + assert_eq!(pre_label1.len(), post_label1.len(), "label query mismatch"); } // --- Timestamp range index tests --- -fn time_node(id: u64, type_id: u32, key: &str, updated_at: i64) -> NodeRecord { +fn time_node(id: u64, label_id: u32, key: &str, updated_at: i64) -> NodeRecord { NodeRecord { id, - type_id, + label_ids: NodeLabelSet::single(label_id).unwrap(), key: key.to_string(), props: BTreeMap::new(), created_at: updated_at - 100, @@ -16198,33 +18143,33 @@ fn test_time_range_memtable_only() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(3, 1, "c", 3000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(3, 1, "c", 3000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(4, 2, "d", 2500))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(4, 2, "d", 2500))) .unwrap(); // Exact range - let r = db.find_nodes_by_time_range(1, 1000, 3000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 1000, 3000).unwrap(); assert_eq!(r, vec![1, 2, 3]); // Partial range - let r = db.find_nodes_by_time_range(1, 1500, 2500).unwrap(); + let r = db.find_nodes_by_time_range("Person", 1500, 2500).unwrap(); assert_eq!(r, vec![2]); - // Type filter - let r = db.find_nodes_by_time_range(2, 2000, 3000).unwrap(); + // Label filter + let r = db.find_nodes_by_time_range("Company", 2000, 3000).unwrap(); assert_eq!(r, vec![4]); // No matches - let r = db.find_nodes_by_time_range(1, 4000, 5000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 4000, 5000).unwrap(); assert!(r.is_empty()); - // All within type 1 - let r = db.find_nodes_by_time_range(1, 0, i64::MAX).unwrap(); + // All within the Person label. + let r = db.find_nodes_by_time_range("Person", 0, i64::MAX).unwrap(); assert_eq!(r, vec![1, 2, 3]); db.close().unwrap(); @@ -16236,18 +18181,18 @@ fn test_time_range_across_flush() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); db.flush().unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(3, 1, "c", 3000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(3, 1, "c", 3000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(4, 1, "d", 1500))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(4, 1, "d", 1500))) .unwrap(); - let r = db.find_nodes_by_time_range(1, 1000, 2000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 1000, 2000).unwrap(); assert_eq!(r, vec![1, 2, 4]); db.close().unwrap(); @@ -16259,24 +18204,24 @@ fn test_time_range_survives_compaction() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); db.flush().unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(3, 1, "c", 3000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(3, 1, "c", 3000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(4, 1, "d", 4000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(4, 1, "d", 4000))) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); - let r = db.find_nodes_by_time_range(1, 1500, 3500).unwrap(); + let r = db.find_nodes_by_time_range("Person", 1500, 3500).unwrap(); assert_eq!(r, vec![2, 3]); - let r = db.find_nodes_by_time_range(1, 0, i64::MAX).unwrap(); + let r = db.find_nodes_by_time_range("Person", 0, i64::MAX).unwrap(); assert_eq!(r, vec![1, 2, 3, 4]); db.close().unwrap(); @@ -16288,17 +18233,17 @@ fn test_time_range_respects_tombstones() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(3, 1, "c", 3000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(3, 1, "c", 3000))) .unwrap(); db.flush().unwrap(); db.delete_node(2).unwrap(); - let r = db.find_nodes_by_time_range(1, 0, i64::MAX).unwrap(); + let r = db.find_nodes_by_time_range("Person", 0, i64::MAX).unwrap(); assert_eq!(r, vec![1, 3]); db.close().unwrap(); @@ -16310,29 +18255,29 @@ fn test_time_range_boundary_conditions() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(3, 1, "c", 3000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(3, 1, "c", 3000))) .unwrap(); // Inclusive boundaries - let r = db.find_nodes_by_time_range(1, 1000, 1000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 1000, 1000).unwrap(); assert_eq!(r, vec![1], "single-point range at lower bound"); - let r = db.find_nodes_by_time_range(1, 3000, 3000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 3000, 3000).unwrap(); assert_eq!(r, vec![3], "single-point range at upper bound"); - let r = db.find_nodes_by_time_range(1, 2000, 2000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 2000, 2000).unwrap(); assert_eq!(r, vec![2], "single-point range in middle"); // Empty range - let r = db.find_nodes_by_time_range(1, 1500, 1500).unwrap(); + let r = db.find_nodes_by_time_range("Person", 1500, 1500).unwrap(); assert!(r.is_empty(), "no nodes at this exact time"); // Inverted range - let r = db.find_nodes_by_time_range(1, 3000, 1000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 3000, 1000).unwrap(); assert!(r.is_empty(), "inverted range returns empty"); db.close().unwrap(); @@ -16345,7 +18290,7 @@ fn test_time_range_paged() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); for i in 1..=10u64 { - db.write_op(&WalOp::UpsertNode(time_node( + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node( i, 1, &format!("n{}", i), @@ -16356,8 +18301,7 @@ fn test_time_range_paged() { db.flush().unwrap(); let page1 = db - .find_nodes_by_time_range_paged( - 1, + .find_nodes_by_time_range_paged("Person", 1000, 10000, &PageRequest { @@ -16370,8 +18314,7 @@ fn test_time_range_paged() { assert!(page1.next_cursor.is_some()); let page2 = db - .find_nodes_by_time_range_paged( - 1, + .find_nodes_by_time_range_paged("Person", 1000, 10000, &PageRequest { @@ -16383,8 +18326,7 @@ fn test_time_range_paged() { assert_eq!(page2.items, vec![4, 5, 6]); let page3 = db - .find_nodes_by_time_range_paged( - 1, + .find_nodes_by_time_range_paged("Person", 1000, 10000, &PageRequest { @@ -16396,8 +18338,7 @@ fn test_time_range_paged() { assert_eq!(page3.items, vec![7, 8, 9]); let page4 = db - .find_nodes_by_time_range_paged( - 1, + .find_nodes_by_time_range_paged("Person", 1000, 10000, &PageRequest { @@ -16418,19 +18359,19 @@ fn test_time_range_upsert_updates_index() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - let r = db.find_nodes_by_time_range(1, 900, 1100).unwrap(); + let r = db.find_nodes_by_time_range("Person", 900, 1100).unwrap(); assert_eq!(r, vec![1]); // Update same node with new timestamp - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 5000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 5000))) .unwrap(); - let r = db.find_nodes_by_time_range(1, 900, 1100).unwrap(); + let r = db.find_nodes_by_time_range("Person", 900, 1100).unwrap(); assert!(r.is_empty(), "node should not appear at old timestamp"); - let r = db.find_nodes_by_time_range(1, 4900, 5100).unwrap(); + let r = db.find_nodes_by_time_range("Person", 4900, 5100).unwrap(); assert_eq!(r, vec![1], "node should appear at new timestamp"); db.close().unwrap(); @@ -16443,11 +18384,11 @@ fn test_time_range_survives_reopen() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(3, 1, "c", 3000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(3, 1, "c", 3000))) .unwrap(); db.flush().unwrap(); db.close().unwrap(); @@ -16455,9 +18396,9 @@ fn test_time_range_survives_reopen() { { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let r = db.find_nodes_by_time_range(1, 1500, 2500).unwrap(); + let r = db.find_nodes_by_time_range("Person", 1500, 2500).unwrap(); assert_eq!(r, vec![2]); - let r = db.find_nodes_by_time_range(1, 0, i64::MAX).unwrap(); + let r = db.find_nodes_by_time_range("Person", 0, i64::MAX).unwrap(); assert_eq!(r, vec![1, 2, 3]); db.close().unwrap(); } @@ -16469,15 +18410,15 @@ fn test_time_range_dedup_across_sources() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); db.flush().unwrap(); // Update same node in memtable (different time, same wide range) - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1500))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1500))) .unwrap(); - let r = db.find_nodes_by_time_range(1, 0, 2000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 0, 2000).unwrap(); assert_eq!(r, vec![1]); db.close().unwrap(); @@ -16489,13 +18430,13 @@ fn test_time_range_with_prune_policy() { let db_path = dir.path().join("testdb"); let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(NodeRecord { + write_internal_wal_op(&db, &WalOp::UpsertNode(NodeRecord { id: 3, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "c".to_string(), props: BTreeMap::new(), created_at: 2900, @@ -16512,12 +18453,12 @@ fn test_time_range_with_prune_policy() { PrunePolicy { max_age_ms: None, max_weight: Some(0.01), - type_id: None, + label: None, }, ) .unwrap(); - let r = db.find_nodes_by_time_range(1, 0, i64::MAX).unwrap(); + let r = db.find_nodes_by_time_range("Person", 0, i64::MAX).unwrap(); assert_eq!(r, vec![1, 2]); db.close().unwrap(); @@ -16533,24 +18474,24 @@ fn test_time_range_stale_segment_suppressed_by_newer_version() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Insert node at t=1000, flush to segment - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); db.flush().unwrap(); // Upsert same node with t=5000 (outside the query window) - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 5000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 5000))) .unwrap(); // Query [500, 2000]. Old segment has node at t=1000 (in range), // but current version is t=5000 (out of range). Must return empty. - let r = db.find_nodes_by_time_range(1, 500, 2000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 500, 2000).unwrap(); assert!( r.is_empty(), "stale segment entry must be suppressed by newer version" ); // Node should appear in a range that covers its current timestamp - let r = db.find_nodes_by_time_range(1, 4000, 6000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 4000, 6000).unwrap(); assert_eq!(r, vec![1]); db.close().unwrap(); @@ -16564,18 +18505,18 @@ fn test_time_range_stale_segment_suppressed_across_segments() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Segment 1: node at t=1000 - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); db.flush().unwrap(); // Segment 2: same node at t=5000 - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 5000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 5000))) .unwrap(); db.flush().unwrap(); // Query [500, 2000]. Segment 1 has t=1000 (in range), // but latest version (segment 2) has t=5000. Must return empty. - let r = db.find_nodes_by_time_range(1, 500, 2000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 500, 2000).unwrap(); assert!( r.is_empty(), "stale segment entry must be suppressed by newer segment version" @@ -16583,13 +18524,13 @@ fn test_time_range_stale_segment_suppressed_across_segments() { // After compaction, still correct db.compact().unwrap(); - let r = db.find_nodes_by_time_range(1, 500, 2000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 500, 2000).unwrap(); assert!( r.is_empty(), "stale entry must be suppressed after compaction too" ); - let r = db.find_nodes_by_time_range(1, 4000, 6000).unwrap(); + let r = db.find_nodes_by_time_range("Person", 4000, 6000).unwrap(); assert_eq!(r, vec![1]); db.close().unwrap(); @@ -16603,22 +18544,21 @@ fn test_time_range_paged_stale_suppressed() { let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Flush 3 nodes to segment - db.write_op(&WalOp::UpsertNode(time_node(1, 1, "a", 1000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(1, 1, "a", 1000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 2000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 2000))) .unwrap(); - db.write_op(&WalOp::UpsertNode(time_node(3, 1, "c", 3000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(3, 1, "c", 3000))) .unwrap(); db.flush().unwrap(); // Move node 2 outside the range - db.write_op(&WalOp::UpsertNode(time_node(2, 1, "b", 9000))) + write_internal_wal_op(&db, &WalOp::UpsertNode(time_node(2, 1, "b", 9000))) .unwrap(); // Paginated query [500, 4000] limit=10, should get [1, 3] (not [1, 2, 3]) let r = db - .find_nodes_by_time_range_paged( - 1, + .find_nodes_by_time_range_paged("Person", 500, 4000, &PageRequest { @@ -16641,9 +18581,9 @@ fn test_time_range_paged_policy_refills_past_sparse_filtered_window() { for i in 0..17u64 { let weight = if i < 12 { 0.1 } else { 1.0 }; - db.write_op(&WalOp::UpsertNode(NodeRecord { + write_internal_wal_op(&db, &WalOp::UpsertNode(NodeRecord { id: i + 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: format!("n{}", i), props: BTreeMap::new(), created_at: 1000 + i as i64, @@ -16664,14 +18604,13 @@ fn test_time_range_paged_policy_refills_past_sparse_filtered_window() { PrunePolicy { max_age_ms: None, max_weight: Some(0.5), - type_id: None, + label: None, }, ) .unwrap(); let page1 = db - .find_nodes_by_time_range_paged( - 1, + .find_nodes_by_time_range_paged("Person", 1000, 2000, &PageRequest { @@ -16684,8 +18623,7 @@ fn test_time_range_paged_policy_refills_past_sparse_filtered_window() { assert!(page1.next_cursor.is_some()); let page2 = db - .find_nodes_by_time_range_paged( - 1, + .find_nodes_by_time_range_paged("Person", 1000, 2000, &PageRequest { @@ -16700,6 +18638,116 @@ fn test_time_range_paged_policy_refills_past_sparse_filtered_window() { db.close().unwrap(); } +#[test] +fn test_time_range_paged_cursor_requires_extra_verified_match() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + for i in 0..6u64 { + let weight = if i == 0 { 1.0 } else { 0.1 }; + write_internal_wal_op(&db, &WalOp::UpsertNode(NodeRecord { + id: i + 1, + label_ids: NodeLabelSet::single(1).unwrap(), + key: format!("n{}", i), + props: BTreeMap::new(), + created_at: 900 + i as i64, + updated_at: 1000 + i as i64, + weight, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + })) + .unwrap(); + } + + db.set_prune_policy( + "low_weight", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: None, + }, + ) + .unwrap(); + + let page = db + .find_nodes_by_time_range_paged("Person", + 1000, + 2000, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(page.items, vec![1]); + assert!( + page.next_cursor.is_none(), + "time range pagination must not report a next page unless another verified node exists" + ); + + db.close().unwrap(); +} + +#[test] +fn test_find_nodes_by_time_range_cursor_no_false_next_after_stale_memberships() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let node = |id: u64, label_ids: &[u32], key: &str, updated_at: i64| NodeRecord { + id, + label_ids: NodeLabelSet::from_canonical_ids(label_ids).unwrap(), + key: key.to_string(), + props: BTreeMap::new(), + created_at: updated_at - 100, + updated_at, + weight: 1.0, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }; + + write_internal_wal_op(&db, &WalOp::UpsertNode(node(1, &[2], "keep", 1000))) + .unwrap(); + for id in 2..=6 { + write_internal_wal_op( + &db, + &WalOp::UpsertNode(node(id, &[2, 3], &format!("stale-{id}"), 1000 + id as i64)), + ) + .unwrap(); + } + db.flush().unwrap(); + + for id in 2..=6 { + write_internal_wal_op( + &db, + &WalOp::UpsertNode(node(id, &[3], &format!("stale-{id}"), 2000 + id as i64)), + ) + .unwrap(); + } + + let page = db + .find_nodes_by_time_range_paged( + "Company", + 1000, + 1010, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(page.items, vec![1]); + assert!( + page.next_cursor.is_none(), + "time range pagination must not report a next page for stale label memberships" + ); + + db.close().unwrap(); +} + // ---- PPR tests ---- fn approx_ppr_opts() -> PprOptions { @@ -16728,7 +18776,7 @@ fn test_ppr_single_node_no_edges() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let n1 = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let result = db .personalized_pagerank(&[n1], &PprOptions::default()) @@ -16751,17 +18799,17 @@ fn test_ppr_simple_chain() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let opts = PprOptions { @@ -16802,14 +18850,14 @@ fn test_ppr_cycle_converges() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, a, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let opts = PprOptions { @@ -16840,20 +18888,20 @@ fn test_ppr_weighted_edges() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.upsert_edge( a, c, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 9.0, ..Default::default() @@ -16890,27 +18938,27 @@ fn test_ppr_weighted_edges() { } #[test] -fn test_ppr_edge_type_filter() { - // A → B (type 1), A → C (type 2), seed = A - // Filter to type 1 only: only B should receive rank +fn test_ppr_edge_label_filter() { + // A → B (label 1), A → C (label 2), seed = A + // Filter to label 1 only: only B should receive rank let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 2, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); let opts = PprOptions { - edge_type_filter: Some(vec![1]), + edge_label_filter: Some(vec!["RELATES_TO".to_string()]), max_iterations: 100, ..PprOptions::default() }; @@ -16944,13 +18992,13 @@ fn test_ppr_max_results() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let center = db - .upsert_node(1, "center", UpsertNodeOptions::default()) + .upsert_node("Person", "center", UpsertNodeOptions::default()) .unwrap(); for i in 0..10 { let n = db - .upsert_node(1, &format!("n{i}"), UpsertNodeOptions::default()) + .upsert_node("Person", &format!("n{i}"), UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(center, n, 1, UpsertEdgeOptions::default()) + db.upsert_edge(center, n, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); } @@ -16970,17 +19018,17 @@ fn test_ppr_multiple_seeds() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let opts = PprOptions { @@ -17007,17 +19055,17 @@ fn test_ppr_respects_deleted_nodes() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.delete_node(b).unwrap(); @@ -17052,7 +19100,7 @@ fn test_ppr_deleted_seed_returns_empty() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); db.delete_node(a).unwrap(); @@ -17088,20 +19136,20 @@ fn test_ppr_across_flush() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); // Add more in memtable let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let opts = PprOptions { @@ -17122,20 +19170,20 @@ fn test_ppr_survives_compaction() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -17156,12 +19204,12 @@ fn test_ppr_duplicate_seeds() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let r1 = db @@ -17190,19 +19238,19 @@ fn test_ppr_known_values() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, a, 1, UpsertEdgeOptions::default()) + db.upsert_edge(c, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let d = 0.85_f64; @@ -17248,27 +19296,27 @@ fn test_ppr_approx_small_graph_matches_exact_order() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.upsert_edge( a, c, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 2.0, ..Default::default() }, ) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let exact = db @@ -17338,12 +19386,12 @@ fn test_ppr_approx_filters_deleted_seeds() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.delete_node(a).unwrap(); @@ -17354,28 +19402,28 @@ fn test_ppr_approx_filters_deleted_seeds() { } #[test] -fn test_ppr_approx_edge_type_filter() { +fn test_ppr_approx_edge_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 2, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); let result = db .personalized_pagerank( &[a], &PprOptions { - edge_type_filter: Some(vec![1]), + edge_label_filter: Some(vec!["RELATES_TO".to_string()]), ..approx_ppr_opts() }, ) @@ -17403,17 +19451,17 @@ fn test_ppr_approx_respects_deleted_nodes() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.delete_node(b).unwrap(); @@ -17440,19 +19488,19 @@ fn test_ppr_approx_across_flush() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = db.personalized_pagerank(&[a], &approx_ppr_opts()).unwrap(); @@ -17469,20 +19517,20 @@ fn test_ppr_approx_survives_compaction() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); - db.upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -17503,6 +19551,8 @@ fn test_export_empty_db() { let db = open_imm(dir.path()); let result = db.export_adjacency(&ExportOptions::default()).unwrap(); assert!(result.node_ids.is_empty()); + assert!(result.node_labels.is_empty()); + assert!(result.node_label_indexes.is_empty()); assert!(result.edges.is_empty()); db.close().unwrap(); } @@ -17512,17 +19562,132 @@ fn test_export_nodes_only() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(2, "b", UpsertNodeOptions::default()) + .upsert_node("Company", "b", UpsertNodeOptions::default()) .unwrap(); let result = db.export_adjacency(&ExportOptions::default()).unwrap(); - assert_eq!(result.node_ids.len(), 2); - assert!(result.node_ids.contains(&a)); - assert!(result.node_ids.contains(&b)); + assert_eq!(result.node_ids, vec![a, b]); + assert_eq!(result.node_labels, vec!["Person", "Company"]); + assert_eq!(result.node_label_indexes, vec![vec![0], vec![1]]); assert!(result.edges.is_empty()); + + let filtered = db + .export_adjacency(&ExportOptions { + node_label_filter: Some(read_node_label_filter(&["Company"], LabelMatchMode::Any)), + ..Default::default() + }) + .unwrap(); + assert_eq!(filtered.node_ids, vec![b]); + assert_eq!(filtered.node_labels, vec!["Company"]); + assert_eq!(filtered.node_label_indexes, vec![vec![0]]); + assert!(filtered.edges.is_empty()); + db.close().unwrap(); +} + +#[test] +fn test_export_node_labels_are_deterministic_for_multi_label_nodes() { + let dir = TempDir::new().unwrap(); + let db = open_imm(dir.path()); + db.ensure_node_label("Person").unwrap(); + db.ensure_node_label("Researcher").unwrap(); + db.ensure_node_label("Company").unwrap(); + + let ids = db + .batch_upsert_nodes(vec![ + NodeInput { + labels: vec!["Researcher".to_string(), "Person".to_string()], + key: "alice".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Company".to_string(), "Person".to_string()], + key: "ando".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + ]) + .unwrap(); + db.upsert_edge(ids[0], ids[1], "WORKS_AT", UpsertEdgeOptions::default()) + .unwrap(); + + let result = db.export_adjacency(&ExportOptions::default()).unwrap(); + assert_eq!(result.node_ids, ids); + assert_eq!( + result.node_labels, + vec![ + "Person".to_string(), + "Researcher".to_string(), + "Company".to_string() + ] + ); + assert_eq!(result.node_label_indexes, vec![vec![0, 1], vec![0, 2]]); + assert_eq!(result.edges.len(), 1); + assert_eq!(result.edges[0].from, ids[0]); + assert_eq!(result.edges[0].to, ids[1]); + db.close().unwrap(); +} + +#[test] +fn test_export_node_labels_stay_deterministic_after_flush_compact_reopen() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_imm(&db_path); + db.ensure_node_label("Person").unwrap(); + db.ensure_node_label("Researcher").unwrap(); + db.ensure_node_label("Company").unwrap(); + db.ensure_node_label("Mentor").unwrap(); + db.ensure_node_label("Reviewer").unwrap(); + + let ids = db + .batch_upsert_nodes(vec![ + NodeInput { + labels: vec!["Person".to_string(), "Researcher".to_string()], + key: "alice".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string(), "Company".to_string()], + key: "ando".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + ]) + .unwrap(); + db.flush().unwrap(); + + assert!(db.add_node_label(ids[0], "Mentor").unwrap()); + assert!(db.remove_node_label(ids[1], "Company").unwrap()); + assert!(db.add_node_label(ids[1], "Reviewer").unwrap()); + db.flush().unwrap(); + db.compact().unwrap(); db.close().unwrap(); + + let reopened = open_imm(&db_path); + let result = reopened.export_adjacency(&ExportOptions::default()).unwrap(); + assert_eq!(result.node_ids, ids); + assert_eq!( + result.node_labels, + vec![ + "Person".to_string(), + "Researcher".to_string(), + "Mentor".to_string(), + "Reviewer".to_string() + ] + ); + assert_eq!(result.node_label_indexes, vec![vec![0, 1, 2], vec![0, 3]]); + reopened.close().unwrap(); } #[test] @@ -17530,18 +19695,18 @@ fn test_export_full_graph() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -17551,14 +19716,14 @@ fn test_export_full_graph() { db.upsert_edge( b, c, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 3.0, ..Default::default() }, ) .unwrap(); - db.upsert_edge(c, a, 2, UpsertEdgeOptions::default()) + db.upsert_edge(c, a, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); let opts = ExportOptions { @@ -17569,73 +19734,112 @@ fn test_export_full_graph() { assert_eq!(result.node_ids.len(), 3); assert_eq!(result.edges.len(), 3); // Verify edge data - let ab = result.edges.iter().find(|e| e.0 == a && e.1 == b).unwrap(); - assert_eq!(ab.2, 1); - assert!((ab.3 - 2.0).abs() < 1e-6); - let ca = result.edges.iter().find(|e| e.0 == c && e.1 == a).unwrap(); - assert_eq!(ca.2, 2); + let ab = result + .edges + .iter() + .find(|e| e.from == a && e.to == b) + .unwrap(); + assert_eq!( + result.edge_labels[ab.edge_label_index as usize], + "RELATES_TO".to_string() + ); + assert!((ab.weight.unwrap() - 2.0).abs() < 1e-6); + let ca = result + .edges + .iter() + .find(|e| e.from == c && e.to == a) + .unwrap(); + assert_eq!( + result.edge_labels[ca.edge_label_index as usize], + "WORKS_AT".to_string() + ); db.close().unwrap(); } #[test] -fn test_export_node_type_filter() { +fn test_export_node_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(2, "b", UpsertNodeOptions::default()) + .upsert_node("Company", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) + .unwrap(); + let d = db + .upsert_node( + &["Person", "Employee"], + "d", + UpsertNodeOptions::default(), + ) + .unwrap(); + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, d, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - // Only type-1 nodes: a and c + // Any(Person) includes Person-only and multi-label Person+Employee nodes. let opts = ExportOptions { - node_type_filter: Some(vec![1]), + node_label_filter: Some(read_node_label_filter(&["Person"], LabelMatchMode::Any)), include_weights: true, ..Default::default() }; let result = db.export_adjacency(&opts).unwrap(); - assert_eq!(result.node_ids.len(), 2); + assert_eq!(result.node_ids.len(), 3); assert!(result.node_ids.contains(&a)); assert!(result.node_ids.contains(&c)); - // Edge a→b should be excluded (b is type-2, not in node set) - assert_eq!(result.edges.len(), 1); - assert_eq!(result.edges[0].0, a); - assert_eq!(result.edges[0].1, c); + assert!(result.node_ids.contains(&d)); + // Edge a->b should be excluded (b is Company-labeled, not in node set) + assert_eq!(result.edges.len(), 2); + assert!(result.edges.iter().any(|edge| edge.from == a && edge.to == c)); + assert!(result.edges.iter().any(|edge| edge.from == a && edge.to == d)); + + let all_opts = ExportOptions { + node_label_filter: Some(read_node_label_filter( + &["Person", "Employee"], + LabelMatchMode::All, + )), + include_weights: true, + ..Default::default() + }; + let all_result = db.export_adjacency(&all_opts).unwrap(); + assert_eq!(all_result.node_ids, vec![d]); + assert!(all_result.edges.is_empty()); db.close().unwrap(); } #[test] -fn test_export_edge_type_filter() { +fn test_export_edge_label_filter() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 2, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "WORKS_AT", UpsertEdgeOptions::default()) .unwrap(); - // Only edge type 2 + // Only the WORKS_AT edge label. let opts = ExportOptions { - edge_type_filter: Some(vec![2]), + edge_label_filter: Some(vec!["WORKS_AT".to_string()]), include_weights: true, ..Default::default() }; let result = db.export_adjacency(&opts).unwrap(); assert_eq!(result.edges.len(), 1); - assert_eq!(result.edges[0].2, 2); // type_id = 2 + assert_eq!( + result.edge_labels[result.edges[0].edge_label_index as usize], + "WORKS_AT".to_string() + ); db.close().unwrap(); } @@ -17644,15 +19848,15 @@ fn test_export_include_weights_false() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( a, b, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 5.0, ..Default::default() @@ -17666,7 +19870,7 @@ fn test_export_include_weights_false() { }; let result = db.export_adjacency(&opts).unwrap(); assert_eq!(result.edges.len(), 1); - assert_eq!(result.edges[0].3, 0.0); // weight zeroed out + assert_eq!(result.edges[0].weight, None); db.close().unwrap(); } @@ -17675,17 +19879,17 @@ fn test_export_respects_tombstones() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(a, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.delete_node(b).unwrap(); @@ -17699,7 +19903,7 @@ fn test_export_respects_tombstones() { assert!(!result.node_ids.contains(&b)); // Edge a→b should be gone (b is deleted) assert_eq!(result.edges.len(), 1); - assert_eq!(result.edges[0].1, c); + assert_eq!(result.edges[0].to, c); db.close().unwrap(); } @@ -17708,21 +19912,21 @@ fn test_export_across_flush() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); db.upsert_edge( b, c, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -17746,18 +19950,18 @@ fn test_export_survives_compaction() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); @@ -17778,7 +19982,7 @@ fn test_export_node_ids_sorted() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); for i in 0..10 { - db.upsert_node(1, &format!("n{i}"), UpsertNodeOptions::default()) + db.upsert_node("Person", &format!("n{i}"), UpsertNodeOptions::default()) .unwrap(); } let result = db.export_adjacency(&ExportOptions::default()).unwrap(); @@ -17797,20 +20001,20 @@ fn test_export_combined_filters() { let dir = TempDir::new().unwrap(); let db = open_imm(dir.path()); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(2, "c", UpsertNodeOptions::default()) + .upsert_node("Company", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.upsert_edge( a, b, - 2, + "WORKS_AT", UpsertEdgeOptions { weight: 2.0, ..Default::default() @@ -17820,7 +20024,7 @@ fn test_export_combined_filters() { db.upsert_edge( a, c, - 1, + "RELATES_TO", UpsertEdgeOptions { weight: 3.0, ..Default::default() @@ -17828,16 +20032,22 @@ fn test_export_combined_filters() { ) .unwrap(); - // node_type=1 + edge_type=1 → only edge a→b with type 1 + // Person node label + RELATES_TO edge label -> only edge a->b. let opts = ExportOptions { - node_type_filter: Some(vec![1]), - edge_type_filter: Some(vec![1]), + node_label_filter: Some(read_node_label_filter(&["Person"], LabelMatchMode::Any)), + edge_label_filter: Some(vec!["RELATES_TO".to_string()]), include_weights: true, }; let result = db.export_adjacency(&opts).unwrap(); - assert_eq!(result.node_ids.len(), 2); // a and b (type 1) + assert_eq!(result.node_ids.len(), 2); // a and b (Person-labeled) assert_eq!(result.edges.len(), 1); - assert_eq!(result.edges[0], (a, b, 1, 1.0)); + assert_eq!(result.edges[0].from, a); + assert_eq!(result.edges[0].to, b); + assert_eq!( + result.edge_labels[result.edges[0].edge_label_index as usize], + "RELATES_TO".to_string() + ); + assert_eq!(result.edges[0].weight, Some(1.0)); db.close().unwrap(); } @@ -17849,17 +20059,17 @@ fn test_ppr_low_damping_seed_dominates() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = db @@ -17893,19 +20103,19 @@ fn test_ppr_high_damping_spreads_rank() { let dir = TempDir::new().unwrap(); let db = open_imm(&dir.path().join("db")); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(a, b, 1, UpsertEdgeOptions::default()) + db.upsert_edge(a, b, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(b, c, 1, UpsertEdgeOptions::default()) + db.upsert_edge(b, c, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); - db.upsert_edge(c, a, 1, UpsertEdgeOptions::default()) + db.upsert_edge(c, a, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); let result = db @@ -17936,7 +20146,7 @@ fn test_ppr_high_damping_spreads_rank() { } // ========================================================================== -// Hybrid fusion search tests (Phase 19e CP11) +// Hybrid fusion search tests // ========================================================================== fn hybrid_search_request( @@ -17952,7 +20162,7 @@ fn hybrid_search_request( dense_query, sparse_query, k, - type_filter: None, + label_filter: None, ef_search: None, scope: None, dense_weight, @@ -17980,7 +20190,7 @@ fn setup_hybrid_db() -> (TempDir, DatabaseEngine, [u64; 5]) { // Node 1: dense rank #1 (high first component), sparse rank #4 let id1 = engine .upsert_node( - 1, + "Person", "n1", UpsertNodeOptions { dense_vector: Some(vec![0.95, 0.05, 0.05, 0.05]), @@ -17993,7 +20203,7 @@ fn setup_hybrid_db() -> (TempDir, DatabaseEngine, [u64; 5]) { // Node 2: dense rank #4, sparse rank #1 (highest sparse score) let id2 = engine .upsert_node( - 1, + "Person", "n2", UpsertNodeOptions { dense_vector: Some(vec![0.3, 0.5, 0.5, 0.5]), @@ -18006,7 +20216,7 @@ fn setup_hybrid_db() -> (TempDir, DatabaseEngine, [u64; 5]) { // Node 3: dense rank #2, sparse rank #2. Balanced, should rise in fusion. let id3 = engine .upsert_node( - 1, + "Person", "n3", UpsertNodeOptions { dense_vector: Some(vec![0.85, 0.1, 0.1, 0.1]), @@ -18019,7 +20229,7 @@ fn setup_hybrid_db() -> (TempDir, DatabaseEngine, [u64; 5]) { // Node 4: dense rank #3, sparse rank #3 let id4 = engine .upsert_node( - 1, + "Person", "n4", UpsertNodeOptions { dense_vector: Some(vec![0.6, 0.3, 0.3, 0.3]), @@ -18032,7 +20242,7 @@ fn setup_hybrid_db() -> (TempDir, DatabaseEngine, [u64; 5]) { // Node 5: dense rank #5, sparse rank #5. Worst in both. let id5 = engine .upsert_node( - 1, + "Person", "n5", UpsertNodeOptions { dense_vector: Some(vec![0.1, 0.4, 0.6, 0.6]), @@ -18298,7 +20508,7 @@ fn test_hybrid_weighted_score_fusion_equal_dense_scores() { // All same dense vector, different sparse scores. let id_a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0]), @@ -18309,7 +20519,7 @@ fn test_hybrid_weighted_score_fusion_equal_dense_scores() { .unwrap(); let id_b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0]), @@ -18320,7 +20530,7 @@ fn test_hybrid_weighted_score_fusion_equal_dense_scores() { .unwrap(); let id_c = engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0]), @@ -18369,7 +20579,7 @@ fn test_hybrid_partial_overlap() { // Node 1: dense only (high similarity) engine .upsert_node( - 1, + "Person", "dense_only", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0]), @@ -18381,7 +20591,7 @@ fn test_hybrid_partial_overlap() { // Node 2: sparse only (high score) engine .upsert_node( - 1, + "Person", "sparse_only", UpsertNodeOptions { sparse_vector: Some(vec![(0, 1.0)]), @@ -18393,7 +20603,7 @@ fn test_hybrid_partial_overlap() { // Node 3: both (moderate in each) engine .upsert_node( - 1, + "Person", "both", UpsertNodeOptions { dense_vector: Some(vec![0.7, 0.7]), @@ -18442,7 +20652,7 @@ fn test_hybrid_with_scope() { for i in 0..4u64 { let id = engine .upsert_node( - 1, + "Person", &format!("n{}", i), UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0]), @@ -18454,10 +20664,10 @@ fn test_hybrid_with_scope() { ids.push(id); } engine - .upsert_edge(ids[0], ids[1], 1, UpsertEdgeOptions::default()) + .upsert_edge(ids[0], ids[1], "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(ids[1], ids[2], 1, UpsertEdgeOptions::default()) + .upsert_edge(ids[1], ids[2], "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); engine.flush().unwrap(); @@ -18474,7 +20684,7 @@ fn test_hybrid_with_scope() { start_node_id: ids[0], max_depth: 1, direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, at_epoch: None, }); @@ -18500,7 +20710,7 @@ fn test_hybrid_with_scope() { } #[test] -fn test_hybrid_type_filter() { +fn test_hybrid_label_filter() { let dir = TempDir::new().unwrap(); let opts = DbOptions { dense_vector: Some(DenseVectorConfig { @@ -18512,10 +20722,10 @@ fn test_hybrid_type_filter() { }; let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); - // Type 1 = "article", type 2 = "comment". + // Person label = "article", Company label = "comment". let id_article = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { dense_vector: Some(vec![1.0, 0.0]), @@ -18527,7 +20737,7 @@ fn test_hybrid_type_filter() { engine .upsert_node( - 2, + "Company", "b", UpsertNodeOptions { dense_vector: Some(vec![0.9, 0.1]), @@ -18539,7 +20749,7 @@ fn test_hybrid_type_filter() { engine.flush().unwrap(); - // Filter to type_id=1 only (the article). + // Filter to the Article label only. let mut req = hybrid_search_request( Some(vec![1.0, 0.0]), Some(vec![(0, 1.0)]), @@ -18548,7 +20758,7 @@ fn test_hybrid_type_filter() { None, None, ); - req.type_filter = Some(vec![1]); + req.label_filter = Some(read_node_label_filter(&["Person"], LabelMatchMode::Any)); let results = engine.vector_search(&req).unwrap(); assert_eq!(results.len(), 1); @@ -18734,7 +20944,7 @@ fn test_hybrid_accuracy_oracle_20_nodes() { let dense_norm: Vec = node.dense.iter().map(|v| v / norm).collect(); let id = engine .upsert_node( - 1, + "Person", node.key, UpsertNodeOptions { dense_vector: Some(dense_norm), @@ -18968,7 +21178,7 @@ fn test_get_nodes_by_keys_basic() { let engine = open_imm(&dir.path().join("db")); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("name", "A"), @@ -18978,7 +21188,7 @@ fn test_get_nodes_by_keys_basic() { .unwrap(); engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { props: make_props("name", "B"), @@ -18988,7 +21198,7 @@ fn test_get_nodes_by_keys_basic() { .unwrap(); engine .upsert_node( - 2, + "Company", "charlie", UpsertNodeOptions { props: make_props("name", "C"), @@ -18997,8 +21207,8 @@ fn test_get_nodes_by_keys_basic() { ) .unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "alice"), (1, "bob"), (2, "charlie")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "alice"), ("Person", "bob"), ("Company", "charlie")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert_eq!(results.len(), 3); assert_eq!(results[0].as_ref().unwrap().key, "alice"); assert_eq!(results[1].as_ref().unwrap().key, "bob"); @@ -19011,15 +21221,15 @@ fn test_get_nodes_by_keys_mixed_found_missing() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); engine.delete_node(b).unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "alice"), (1, "bob"), (1, "nonexistent")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "alice"), ("Person", "bob"), ("Person", "nonexistent")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert_eq!(results.len(), 3); assert!(results[0].is_some()); assert!(results[1].is_none()); // deleted @@ -19032,15 +21242,15 @@ fn test_get_nodes_by_keys_cross_source() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "alice"), (1, "bob")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "alice"), ("Person", "bob")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert_eq!(results[0].as_ref().unwrap().key, "alice"); assert_eq!(results[1].as_ref().unwrap().key, "bob"); engine.close().unwrap(); @@ -19050,8 +21260,8 @@ fn test_get_nodes_by_keys_cross_source() { fn test_get_nodes_by_keys_empty() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); - let keys: Vec<(u32, &str)> = vec![]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert!(results.is_empty()); engine.close().unwrap(); } @@ -19062,7 +21272,7 @@ fn test_get_nodes_by_keys_multi_segment() { let engine = open_imm(&dir.path().join("db")); engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("seg", "1"), @@ -19072,7 +21282,7 @@ fn test_get_nodes_by_keys_multi_segment() { .unwrap(); engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { props: make_props("seg", "1"), @@ -19083,7 +21293,7 @@ fn test_get_nodes_by_keys_multi_segment() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "c", UpsertNodeOptions { props: make_props("seg", "2"), @@ -19093,7 +21303,7 @@ fn test_get_nodes_by_keys_multi_segment() { .unwrap(); engine .upsert_node( - 1, + "Person", "d", UpsertNodeOptions { props: make_props("seg", "2"), @@ -19103,8 +21313,8 @@ fn test_get_nodes_by_keys_multi_segment() { .unwrap(); engine.flush().unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "d"), (1, "a"), (1, "c"), (1, "b")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "d"), ("Person", "a"), ("Person", "c"), ("Person", "b")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert_eq!(results.len(), 4); assert_eq!(results[0].as_ref().unwrap().key, "d"); assert_eq!(results[1].as_ref().unwrap().key, "a"); @@ -19118,20 +21328,20 @@ fn test_get_nodes_by_keys_tombstone_in_newer_segment() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); // seg 1: a, b engine.delete_node(a).unwrap(); engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); // seg 2: tombstone(a), c - let keys: Vec<(u32, &str)> = vec![(1, "a"), (1, "b"), (1, "c")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "a"), ("Person", "b"), ("Person", "c")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert!(results[0].is_none()); // a tombstoned in seg 2 assert_eq!(results[1].as_ref().unwrap().key, "b"); assert_eq!(results[2].as_ref().unwrap().key, "c"); @@ -19144,7 +21354,7 @@ fn test_get_nodes_by_keys_memtable_shadows_segment() { let engine = open_imm(&dir.path().join("db")); engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "old"), @@ -19155,7 +21365,7 @@ fn test_get_nodes_by_keys_memtable_shadows_segment() { engine.flush().unwrap(); engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "new"), @@ -19165,8 +21375,8 @@ fn test_get_nodes_by_keys_memtable_shadows_segment() { ) .unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "a")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "a")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); let node = results[0].as_ref().unwrap(); assert_eq!( node.props.get("v"), @@ -19181,11 +21391,11 @@ fn test_get_nodes_by_keys_duplicate_keys() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "a"), (1, "a"), (1, "a")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "a"), ("Person", "a"), ("Person", "a")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert_eq!(results.len(), 3); assert_eq!(results[0].as_ref().unwrap().key, "a"); assert_eq!(results[1].as_ref().unwrap().key, "a"); @@ -19207,19 +21417,19 @@ fn test_get_nodes_by_keys_after_compaction() { ) .unwrap(); engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); // triggers compaction - let keys: Vec<(u32, &str)> = vec![(1, "a"), (1, "b"), (1, "c")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "a"), ("Person", "b"), ("Person", "c")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert_eq!(results.len(), 3); assert!(results.iter().all(|r| r.is_some())); engine.close().unwrap(); @@ -19231,7 +21441,7 @@ fn test_get_nodes_by_keys_delete_then_recreate() { let engine = open_imm(&dir.path().join("db")); let old_id = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "old"), @@ -19243,7 +21453,7 @@ fn test_get_nodes_by_keys_delete_then_recreate() { engine.delete_node(old_id).unwrap(); let new_id = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "new"), @@ -19253,8 +21463,8 @@ fn test_get_nodes_by_keys_delete_then_recreate() { .unwrap(); assert_ne!(old_id, new_id); - let keys: Vec<(u32, &str)> = vec![(1, "a")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "a")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); let node = results[0].as_ref().unwrap(); assert_eq!(node.id, new_id); assert_eq!( @@ -19265,12 +21475,12 @@ fn test_get_nodes_by_keys_delete_then_recreate() { } #[test] -fn test_get_nodes_by_keys_different_type_ids() { +fn test_get_nodes_by_keys_different_label_ids() { let dir = TempDir::new().unwrap(); let engine = open_imm(&dir.path().join("db")); engine .upsert_node( - 1, + "Person", "x", UpsertNodeOptions { props: make_props("t", "1"), @@ -19280,7 +21490,7 @@ fn test_get_nodes_by_keys_different_type_ids() { .unwrap(); engine .upsert_node( - 2, + "Company", "x", UpsertNodeOptions { props: make_props("t", "2"), @@ -19289,13 +21499,13 @@ fn test_get_nodes_by_keys_different_type_ids() { ) .unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "x"), (2, "x")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "x"), ("Company", "x")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert_eq!(results.len(), 2); let n1 = results[0].as_ref().unwrap(); let n2 = results[1].as_ref().unwrap(); - assert_eq!(n1.type_id, 1); - assert_eq!(n2.type_id, 2); + assert_eq!(n1.labels.as_slice(), ["Person"]); + assert_eq!(n2.labels.as_slice(), ["Company"]); assert_eq!(n1.props.get("t"), Some(&PropValue::String("1".to_string()))); assert_eq!(n2.props.get("t"), Some(&PropValue::String("2".to_string()))); engine.close().unwrap(); @@ -19317,7 +21527,7 @@ fn test_get_nodes_by_keys_policy_filtering() { // Use weight-based policy so there's no timing dependency engine .upsert_node( - 1, + "Person", "low", UpsertNodeOptions { weight: 0.05, @@ -19327,7 +21537,7 @@ fn test_get_nodes_by_keys_policy_filtering() { .unwrap(); engine .upsert_node( - 1, + "Person", "high", UpsertNodeOptions { weight: 5.0, @@ -19341,15 +21551,15 @@ fn test_get_nodes_by_keys_policy_filtering() { .set_prune_policy( "low_weight", PrunePolicy { - type_id: Some(1), + label: Some("Person".to_string()), max_age_ms: None, max_weight: Some(0.1), }, ) .unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "low"), (1, "high")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "low"), ("Person", "high")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert!(results[0].is_none()); // excluded by policy (weight <= 0.1) assert!(results[1].is_some()); engine.close().unwrap(); @@ -19364,7 +21574,7 @@ fn test_get_nodes_by_keys_tombstone_prevents_fallthrough() { // seg1: "a" → old node let old_id = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "seg1"), @@ -19376,7 +21586,7 @@ fn test_get_nodes_by_keys_tombstone_prevents_fallthrough() { // seg2: "a" → updated node (same key, same node_id via upsert) engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "seg2"), @@ -19388,8 +21598,8 @@ fn test_get_nodes_by_keys_tombstone_prevents_fallthrough() { // memtable: delete the node engine.delete_node(old_id).unwrap(); - let keys: Vec<(u32, &str)> = vec![(1, "a")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "a")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert!( results[0].is_none(), "tombstoned node must not fall through to older segment" @@ -19407,7 +21617,7 @@ fn test_get_nodes_by_keys_immutable_tombstone_shadows_older_immutable() { // Create node, freeze → immutable 1 (oldest, has the record) let id = engine - .upsert_node(1, "doomed", UpsertNodeOptions::default()) + .upsert_node("Person", "doomed", UpsertNodeOptions::default()) .unwrap(); engine.freeze_memtable().unwrap(); @@ -19417,13 +21627,13 @@ fn test_get_nodes_by_keys_immutable_tombstone_shadows_older_immutable() { // Scalar path (known correct) assert!( - engine.get_node_by_key(1, "doomed").unwrap().is_none(), + engine.get_node_by_key("Person", "doomed").unwrap().is_none(), "scalar get_node_by_key must hide node tombstoned in newer immutable" ); // Batch path (the regression target) - let keys: Vec<(u32, &str)> = vec![(1, "doomed")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "doomed")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert!( results[0].is_none(), "batch get_nodes_by_keys must hide node tombstoned in newer immutable" @@ -19444,7 +21654,7 @@ fn test_get_nodes_by_keys_delete_recreate_delete_cross_segment() { // seg1: "a" → old_id let old_id = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "old"), @@ -19458,7 +21668,7 @@ fn test_get_nodes_by_keys_delete_recreate_delete_cross_segment() { engine.delete_node(old_id).unwrap(); let new_id = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { props: make_props("v", "new"), @@ -19475,13 +21685,13 @@ fn test_get_nodes_by_keys_delete_recreate_delete_cross_segment() { // Scalar path assert!( - engine.get_node_by_key(1, "a").unwrap().is_none(), + engine.get_node_by_key("Person", "a").unwrap().is_none(), "scalar must return None for delete-recreate-delete across segments" ); // Batch path - let keys: Vec<(u32, &str)> = vec![(1, "a")]; - let results = engine.get_nodes_by_keys(&keys).unwrap(); + let keys: Vec<(&str, &str)> = vec![("Person", "a")]; + let results = engine.get_nodes_by_keys(&read_node_key_queries(&keys)).unwrap(); assert!( results[0].is_none(), "batch must return None for delete-recreate-delete across segments" diff --git a/src/engine/tests/txn.rs b/src/engine/tests/txn.rs index a294ba4..133be1b 100644 --- a/src/engine/tests/txn.rs +++ b/src/engine/tests/txn.rs @@ -16,30 +16,92 @@ fn test_write_txn_staged_reads_and_rollback_leave_no_trace() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let mut txn = engine.begin_write_txn().unwrap(); let alice = txn - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let staged = txn.get_node(alice).unwrap().unwrap(); assert_eq!(staged.id, None); + assert_eq!(staged.labels, vec!["Person".to_string()]); assert_eq!(staged.created_at, None); assert_eq!(staged.updated_at, None); assert_eq!(staged.key, "alice"); - assert!(engine.get_node_by_key(1, "alice").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "alice").unwrap().is_none()); txn.rollback().unwrap(); assert!(matches!( - txn.upsert_node(1, "after", UpsertNodeOptions::default()), + txn.upsert_node("Person", "after", UpsertNodeOptions::default()), Err(EngineError::TxnClosed) )); - assert!(engine.get_node_by_key(1, "alice").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "alice").unwrap().is_none()); engine.close().unwrap(); } let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert!(reopened.get_node_by_key(1, "alice").unwrap().is_none()); + assert!(reopened.get_node_by_key("Person", "alice").unwrap().is_none()); reopened.close().unwrap(); } +#[test] +fn test_write_txn_rollback_does_not_create_named_tokens() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut txn = engine.begin_write_txn().unwrap(); + let left = txn + .upsert_node("TxnRollbackNode", "left", UpsertNodeOptions::default()) + .unwrap(); + let right = txn + .upsert_node("TxnRollbackNode", "right", UpsertNodeOptions::default()) + .unwrap(); + txn.upsert_edge( + left, + right, + "TXN_ROLLBACK_EDGE", + UpsertEdgeOptions::default(), + ) + .unwrap(); + assert!(txn + .get_node_by_key("TxnRollbackNode", "left") + .unwrap() + .is_some()); + + txn.rollback().unwrap(); + assert_eq!(engine.get_node_label_id("TxnRollbackNode").unwrap(), None); + assert_eq!(engine.get_edge_label_id("TXN_ROLLBACK_EDGE").unwrap(), None); + assert!(engine + .get_node_by_key("TxnRollbackNode", "left") + .unwrap() + .is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_write_txn_unknown_read_only_lookups_do_not_create_tokens() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let a = engine + .upsert_node("KnownTxnNode", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("KnownTxnNode", "b", UpsertNodeOptions::default()) + .unwrap(); + + let txn = engine.begin_write_txn().unwrap(); + assert!(txn + .get_node_by_key("MissingTxnNode", "a") + .unwrap() + .is_none()); + assert!(txn + .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "MISSING_TXN_EDGE") + .unwrap() + .is_none()); + assert_eq!(engine.get_node_label_id("MissingTxnNode").unwrap(), None); + assert_eq!(engine.get_edge_label_id("MISSING_TXN_EDGE").unwrap(), None); + engine.close().unwrap(); +} + #[test] fn test_write_txn_lifecycle_closed_db_and_finished_txn_rules() { let dir = TempDir::new().unwrap(); @@ -53,7 +115,7 @@ fn test_write_txn_lifecycle_closed_db_and_finished_txn_rules() { let mut commit_txn = engine.begin_write_txn().unwrap(); commit_txn - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); commit_txn.commit().unwrap(); assert!(matches!(commit_txn.commit(), Err(EngineError::TxnClosed))); @@ -74,13 +136,13 @@ fn test_write_txn_read_own_writes_update_and_delete_views() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let alice_id = engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); let alice_ref = txn .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 2.0, @@ -93,18 +155,18 @@ fn test_write_txn_read_own_writes_update_and_delete_views() { assert!(updated.created_at.is_some()); assert_eq!(updated.updated_at, None); assert!((updated.weight - 2.0).abs() < f32::EPSILON); - assert_eq!(txn.get_node_by_key(1, "alice").unwrap().unwrap().id, Some(alice_id)); + assert_eq!(txn.get_node_by_key("Person", "alice").unwrap().unwrap().id, Some(alice_id)); txn.delete_node(alice_ref).unwrap(); assert!(txn.get_node(TxnNodeRef::Id(alice_id)).unwrap().is_none()); - assert!(txn.get_node_by_key(1, "alice").unwrap().is_none()); + assert!(txn.get_node_by_key("Person", "alice").unwrap().is_none()); let staged = txn - .upsert_node(1, "staged", UpsertNodeOptions::default()) + .upsert_node("Person", "staged", UpsertNodeOptions::default()) .unwrap(); assert!(txn.get_node(staged.clone()).unwrap().is_some()); txn.delete_node(staged).unwrap(); - assert!(txn.get_node_by_key(1, "staged").unwrap().is_none()); + assert!(txn.get_node_by_key("Person", "staged").unwrap().is_none()); engine.close().unwrap(); } @@ -116,23 +178,24 @@ fn test_write_txn_commit_create_and_connect_local_refs() { let mut txn = engine.begin_write_txn().unwrap(); let alice = txn - .upsert_node_as("alice", 1, "alice", UpsertNodeOptions::default()) + .upsert_node_as("alice", "Person", "alice", UpsertNodeOptions::default()) .unwrap(); let bob = txn - .upsert_node_as("bob", 1, "bob", UpsertNodeOptions::default()) + .upsert_node_as("bob", "Person", "bob", UpsertNodeOptions::default()) .unwrap(); let edge = txn .upsert_edge_as( "knows", alice.clone(), bob.clone(), - 7, + "FRIENDS_WITH", UpsertEdgeOptions::default(), ) .unwrap(); let staged_edge = txn.get_edge(edge.clone()).unwrap().unwrap(); assert_eq!(staged_edge.id, None); + assert_eq!(staged_edge.label, "FRIENDS_WITH".to_string()); assert_eq!(staged_edge.from, alice); assert_eq!(staged_edge.to, bob); @@ -162,9 +225,9 @@ fn test_write_txn_commit_create_and_connect_local_refs() { ( committed_edge.from, committed_edge.to, - committed_edge.type_id + committed_edge.label ), - (alice_id, bob_id, 7) + (alice_id, bob_id, "FRIENDS_WITH".to_string()) ); engine.close().unwrap(); @@ -173,6 +236,77 @@ fn test_write_txn_commit_create_and_connect_local_refs() { reopened.close().unwrap(); } +#[test] +fn test_write_txn_first_use_tokens_precede_dependent_wal_ops() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let mut txn = engine.begin_write_txn().unwrap(); + let alice = txn + .upsert_node("TxnWalPerson", "alice", UpsertNodeOptions::default()) + .unwrap(); + let bob = txn + .upsert_node("TxnWalPerson", "bob", UpsertNodeOptions::default()) + .unwrap(); + txn.upsert_edge(alice, bob, "TXN_WAL_KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let result = txn.commit().unwrap(); + let alice_id = result.node_ids[0]; + let edge_id = result.edge_ids[0]; + + let ops = WalReader::read_generation(&db_path, 0).unwrap(); + let node_token_pos = ops + .iter() + .position(|(_, op)| { + matches!( + op, + WalOp::EnsureNodeLabel { label, .. } if label == "TxnWalPerson" + ) + }) + .unwrap(); + let first_node_pos = ops + .iter() + .position(|(_, op)| matches!(op, WalOp::UpsertNode(node) if node.id == alice_id)) + .unwrap(); + let edge_token_pos = ops + .iter() + .position(|(_, op)| { + matches!( + op, + WalOp::EnsureEdgeLabel { label, .. } if label == "TXN_WAL_KNOWS" + ) + }) + .unwrap(); + let edge_pos = ops + .iter() + .position(|(_, op)| matches!(op, WalOp::UpsertEdge(edge) if edge.id == edge_id)) + .unwrap(); + assert!(node_token_pos < first_node_pos); + assert!(edge_token_pos < edge_pos); + + drop(engine); + let reopened = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert_eq!( + reopened + .get_node(alice_id) + .unwrap() + .unwrap() + .labels + .as_slice(), + ["TxnWalPerson"] + ); + assert_eq!( + reopened.get_edge(edge_id).unwrap().unwrap().label, + "TXN_WAL_KNOWS" + ); + reopened.close().unwrap(); +} + #[test] fn test_write_txn_stage_intents_assigns_unaliased_slots_atomically() { let dir = TempDir::new().unwrap(); @@ -183,13 +317,13 @@ fn test_write_txn_stage_intents_assigns_unaliased_slots_atomically() { txn.stage_intents(vec![ TxnIntent::UpsertNode { alias: None, - type_id: 1, + labels: vec!["Person".to_string()], key: "alice".into(), options: UpsertNodeOptions::default(), }, TxnIntent::UpsertNode { alias: None, - type_id: 1, + labels: vec!["Person".to_string()], key: "bob".into(), options: UpsertNodeOptions::default(), }, @@ -197,7 +331,7 @@ fn test_write_txn_stage_intents_assigns_unaliased_slots_atomically() { alias: None, from: TxnNodeRef::Local(TxnLocalRef::Slot(0)), to: TxnNodeRef::Local(TxnLocalRef::Slot(1)), - type_id: 7, + label: "FRIENDS_WITH".to_string(), options: UpsertEdgeOptions::default(), }, ]) @@ -226,14 +360,14 @@ fn test_write_txn_stage_intents_assigns_unaliased_slots_atomically() { alias: None, from: TxnNodeRef::Local(TxnLocalRef::Slot(9)), to: TxnNodeRef::Local(TxnLocalRef::Slot(10)), - type_id: 9, + label: "RELATED_TO".to_string(), options: UpsertEdgeOptions::default(), }]), Err(EngineError::InvalidOperation(_)) )); assert_eq!( failed - .upsert_node(1, "after-failed-stage", UpsertNodeOptions::default()) + .upsert_node("Person", "after-failed-stage", UpsertNodeOptions::default()) .unwrap(), TxnNodeRef::Local(TxnLocalRef::Slot(0)) ); @@ -246,31 +380,31 @@ fn test_write_txn_overlay_normalizes_mixed_endpoint_refs() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let alice_id = engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let bob_id = engine - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); let alice = txn - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let bob = txn - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); let edge = txn - .upsert_edge(alice.clone(), bob.clone(), 9, UpsertEdgeOptions::default()) + .upsert_edge(alice.clone(), bob.clone(), "RELATED_TO", UpsertEdgeOptions::default()) .unwrap(); assert_eq!( txn.get_edge_by_triple( TxnNodeRef::Key { - type_id: 1, + label: "Person".to_string(), key: "alice".into(), }, TxnNodeRef::Id(bob_id), - 9, + "RELATED_TO", ) .unwrap() .unwrap() @@ -281,21 +415,21 @@ fn test_write_txn_overlay_normalizes_mixed_endpoint_refs() { }) ); assert!(txn - .get_edge_by_triple(TxnNodeRef::Id(alice_id), TxnNodeRef::Id(bob_id), 9) + .get_edge_by_triple(TxnNodeRef::Id(alice_id), TxnNodeRef::Id(bob_id), "RELATED_TO") .unwrap() .is_some()); txn.invalidate_edge( TxnEdgeRef::Triple { from: TxnNodeRef::Key { - type_id: 1, + label: "Person".to_string(), key: "alice".into(), }, to: TxnNodeRef::Key { - type_id: 1, + label: "Person".to_string(), key: "bob".into(), }, - type_id: 9, + label: "RELATED_TO".to_string(), }, 123, ) @@ -305,13 +439,200 @@ fn test_write_txn_overlay_normalizes_mixed_endpoint_refs() { txn.delete_edge(TxnEdgeRef::Triple { from: TxnNodeRef::Id(alice_id), to: TxnNodeRef::Id(bob_id), - type_id: 9, + label: "RELATED_TO".to_string(), }) .unwrap(); assert!(txn.get_edge(edge).unwrap().is_none()); engine.close().unwrap(); } +#[test] +fn test_write_txn_multi_label_overlay_label_diff_keeps_node_identity() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open( + &db_path, + &DbOptions { + edge_uniqueness: true, + ..Default::default() + }, + ) + .unwrap(); + + let mut txn = engine.begin_write_txn().unwrap(); + let node = txn + .upsert_node( + &["TxnOverlayA", "TxnOverlayB"], + "same", + UpsertNodeOptions::default(), + ) + .unwrap(); + let peer = txn + .upsert_node("TxnOverlayPeer", "peer", UpsertNodeOptions::default()) + .unwrap(); + let edge = txn + .upsert_edge( + TxnNodeRef::Key { + label: "TxnOverlayB".to_string(), + key: "same".to_string(), + }, + peer.clone(), + "TXN_OVERLAY_EDGE", + UpsertEdgeOptions::default(), + ) + .unwrap(); + + assert!(txn.add_node_label(node.clone(), "TxnOverlayC").unwrap()); + assert!(txn.remove_node_label(node.clone(), "TxnOverlayB").unwrap()); + assert!(txn + .get_node_by_key("TxnOverlayB", "same") + .unwrap() + .is_none()); + assert_eq!( + txn.get_node_by_key("TxnOverlayC", "same") + .unwrap() + .unwrap() + .local, + match node.clone() { + TxnNodeRef::Local(local) => Some(local), + _ => None, + } + ); + assert!(txn.get_edge(edge.clone()).unwrap().is_some()); + assert!(txn + .get_edge_by_triple( + TxnNodeRef::Key { + label: "TxnOverlayC".to_string(), + key: "same".to_string(), + }, + peer.clone(), + "TXN_OVERLAY_EDGE", + ) + .unwrap() + .is_some()); + + let replacement = txn + .upsert_node( + "TxnOverlayB", + "same", + UpsertNodeOptions { + weight: 9.0, + ..Default::default() + }, + ) + .unwrap(); + let original = txn.get_node(node.clone()).unwrap().unwrap(); + let replacement_view = txn.get_node(replacement).unwrap().unwrap(); + assert_eq!( + original.labels, + vec!["TxnOverlayA".to_string(), "TxnOverlayC".to_string()] + ); + assert_eq!(replacement_view.labels, vec!["TxnOverlayB".to_string()]); + assert!((replacement_view.weight - 9.0).abs() < f32::EPSILON); + assert!(txn.get_edge(edge).unwrap().is_some()); + assert!(txn + .get_edge_by_triple( + TxnNodeRef::Key { + label: "TxnOverlayB".to_string(), + key: "same".to_string(), + }, + peer.clone(), + "TXN_OVERLAY_EDGE", + ) + .unwrap() + .is_none()); + + let result = txn.commit().unwrap(); + let original_id = engine + .get_node_by_key("TxnOverlayA", "same") + .unwrap() + .unwrap() + .id; + let replacement_id = engine + .get_node_by_key("TxnOverlayB", "same") + .unwrap() + .unwrap() + .id; + assert_ne!(original_id, replacement_id); + assert_eq!( + engine + .get_node_by_key("TxnOverlayC", "same") + .unwrap() + .unwrap() + .id, + original_id + ); + assert_eq!( + engine + .get_edge(result.edge_ids[0]) + .unwrap() + .unwrap() + .from, + original_id + ); + engine.close().unwrap(); +} + +#[test] +fn test_write_txn_multi_label_delete_cascades_staged_edge_after_label_changes() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open( + &db_path, + &DbOptions { + edge_uniqueness: true, + ..Default::default() + }, + ) + .unwrap(); + + let mut txn = engine.begin_write_txn().unwrap(); + let node = txn + .upsert_node( + &["TxnCascadeA", "TxnCascadeB"], + "node", + UpsertNodeOptions::default(), + ) + .unwrap(); + let peer = txn + .upsert_node("TxnCascadePeer", "peer", UpsertNodeOptions::default()) + .unwrap(); + let edge = txn + .upsert_edge( + TxnNodeRef::Key { + label: "TxnCascadeB".to_string(), + key: "node".to_string(), + }, + peer.clone(), + "TXN_CASCADE_EDGE", + UpsertEdgeOptions::default(), + ) + .unwrap(); + + assert!(txn.add_node_label(node.clone(), "TxnCascadeC").unwrap()); + assert!(txn.remove_node_label(node.clone(), "TxnCascadeB").unwrap()); + txn.delete_node(TxnNodeRef::Key { + label: "TxnCascadeC".to_string(), + key: "node".to_string(), + }) + .unwrap(); + + assert!(txn.get_edge(edge).unwrap().is_none()); + assert!(txn + .get_edge_by_triple( + TxnNodeRef::Key { + label: "TxnCascadeA".to_string(), + key: "node".to_string(), + }, + peer, + "TXN_CASCADE_EDGE", + ) + .unwrap() + .is_none()); + txn.rollback().unwrap(); + engine.close().unwrap(); +} + #[test] fn test_write_txn_overlay_latest_opinion_updates_all_locals() { let dir = TempDir::new().unwrap(); @@ -325,16 +646,16 @@ fn test_write_txn_overlay_latest_opinion_updates_all_locals() { ) .unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); let first_node = txn .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 1.0, @@ -344,7 +665,7 @@ fn test_write_txn_overlay_latest_opinion_updates_all_locals() { .unwrap(); let second_node = txn .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 2.0, @@ -356,7 +677,7 @@ fn test_write_txn_overlay_latest_opinion_updates_all_locals() { txn.delete_node(second_node.clone()).unwrap(); assert!(txn.get_node(first_node.clone()).unwrap().is_none()); txn.upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 3.0, @@ -369,14 +690,14 @@ fn test_write_txn_overlay_latest_opinion_updates_all_locals() { assert!((revived.weight - 3.0).abs() < f32::EPSILON); let first_edge = txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 1.0; options }) .unwrap(); let second_edge = txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 2.0; options @@ -406,23 +727,23 @@ fn test_write_txn_delete_then_reupsert_preserves_committed_identity_in_reads() { ) .unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let edge_id = engine - .upsert_edge(a, b, 7, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); let staged_a = txn - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); txn.delete_node(staged_a).unwrap(); let revived_a = txn .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 4.0, @@ -435,11 +756,11 @@ fn test_write_txn_delete_then_reupsert_preserves_committed_identity_in_reads() { assert!(revived_a_view.created_at.is_some()); let staged_edge = txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, UpsertEdgeOptions::default()) + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); txn.delete_edge(staged_edge).unwrap(); let revived_edge = txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, UpsertEdgeOptions::default()) + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); let revived_edge_view = txn.get_edge(revived_edge).unwrap().unwrap(); assert_eq!(revived_edge_view.id, Some(edge_id)); @@ -453,22 +774,22 @@ fn test_write_txn_nonunique_same_triple_edges_keep_local_identity() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); let first = txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 1.0; options }) .unwrap(); let second = txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 2.0; options @@ -477,7 +798,7 @@ fn test_write_txn_nonunique_same_triple_edges_keep_local_identity() { assert!((txn.get_edge(first.clone()).unwrap().unwrap().weight - 1.0).abs() < f32::EPSILON); let latest = txn - .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7) + .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH") .unwrap() .unwrap(); assert_eq!(latest.local, edge_local(second.clone())); @@ -487,7 +808,7 @@ fn test_write_txn_nonunique_same_triple_edges_keep_local_identity() { assert!(txn.get_edge(first.clone()).unwrap().is_some()); assert!(txn.get_edge(second).unwrap().is_none()); assert!(txn - .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7) + .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH") .unwrap() .is_none()); @@ -495,7 +816,7 @@ fn test_write_txn_nonunique_same_triple_edges_keep_local_identity() { assert_eq!(result.edge_ids.len(), 2); assert!(engine.get_edge(result.edge_ids[0]).unwrap().is_some()); assert!(engine.get_edge(result.edge_ids[1]).unwrap().is_none()); - assert!(engine.get_edge_by_triple(a, b, 7).unwrap().is_none()); + assert!(engine.get_edge_by_triple(a, b, "FRIENDS_WITH").unwrap().is_none()); engine.close().unwrap(); } @@ -505,22 +826,22 @@ fn test_write_txn_nonunique_old_same_triple_delete_or_invalidate_keeps_latest() let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let mut delete_txn = engine.begin_write_txn().unwrap(); let first = delete_txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 1.0; options }) .unwrap(); let second = delete_txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 2.0; options @@ -529,7 +850,7 @@ fn test_write_txn_nonunique_old_same_triple_delete_or_invalidate_keeps_latest() delete_txn.delete_edge(first.clone()).unwrap(); assert!(delete_txn.get_edge(first).unwrap().is_none()); let latest = delete_txn - .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7) + .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH") .unwrap() .unwrap(); assert_eq!(latest.local, edge_local(second.clone())); @@ -537,20 +858,20 @@ fn test_write_txn_nonunique_old_same_triple_delete_or_invalidate_keeps_latest() let delete_result = delete_txn.commit().unwrap(); assert!(engine.get_edge(delete_result.edge_ids[0]).unwrap().is_none()); assert_eq!( - engine.get_edge_by_triple(a, b, 7).unwrap().unwrap().id, + engine.get_edge_by_triple(a, b, "FRIENDS_WITH").unwrap().unwrap().id, delete_result.edge_ids[1] ); let mut invalidate_txn = engine.begin_write_txn().unwrap(); let third = invalidate_txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 8, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "COLLABORATES_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 3.0; options }) .unwrap(); let fourth = invalidate_txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 8, { + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "COLLABORATES_WITH", { let mut options = UpsertEdgeOptions::default(); options.weight = 4.0; options @@ -566,26 +887,26 @@ fn test_write_txn_nonunique_old_same_triple_delete_or_invalidate_keeps_latest() Some(123) ); let latest = invalidate_txn - .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 8) + .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "COLLABORATES_WITH") .unwrap() .unwrap(); assert_eq!(latest.local, edge_local(fourth)); assert!((latest.weight - 4.0).abs() < f32::EPSILON); let invalidate_result = invalidate_txn.commit().unwrap(); assert_eq!( - engine.get_edge_by_triple(a, b, 8).unwrap().unwrap().id, + engine.get_edge_by_triple(a, b, "COLLABORATES_WITH").unwrap().unwrap().id, invalidate_result.edge_ids[1] ); let regular_first = engine - .upsert_edge(a, b, 9, { + .upsert_edge(a, b, "RELATED_TO", { let mut options = UpsertEdgeOptions::default(); options.weight = 5.0; options }) .unwrap(); let regular_second = engine - .upsert_edge(a, b, 9, { + .upsert_edge(a, b, "RELATED_TO", { let mut options = UpsertEdgeOptions::default(); options.weight = 6.0; options @@ -593,12 +914,12 @@ fn test_write_txn_nonunique_old_same_triple_delete_or_invalidate_keeps_latest() .unwrap(); engine.invalidate_edge(regular_first, 456).unwrap(); assert_eq!( - engine.get_edge_by_triple(a, b, 9).unwrap().unwrap().id, + engine.get_edge_by_triple(a, b, "RELATED_TO").unwrap().unwrap().id, regular_second ); engine.delete_edge(regular_first).unwrap(); assert_eq!( - engine.get_edge_by_triple(a, b, 9).unwrap().unwrap().id, + engine.get_edge_by_triple(a, b, "RELATED_TO").unwrap().unwrap().id, regular_second ); engine.close().unwrap(); @@ -610,45 +931,45 @@ fn test_write_txn_delete_node_reupsert_does_not_revive_incident_edges() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let committed_edge = engine - .upsert_edge(a, b, 7, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); let mut committed_txn = engine.begin_write_txn().unwrap(); committed_txn.delete_node(TxnNodeRef::Id(a)).unwrap(); committed_txn - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); assert!(committed_txn .get_edge(TxnEdgeRef::Id(committed_edge)) .unwrap() .is_none()); assert!(committed_txn - .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7) + .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH") .unwrap() .is_none()); let revived = committed_txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, UpsertEdgeOptions::default()) + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); assert!(committed_txn.get_edge(revived).unwrap().is_some()); committed_txn.rollback().unwrap(); let mut staged_txn = engine.begin_write_txn().unwrap(); let staged_edge = staged_txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 8, UpsertEdgeOptions::default()) + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "COLLABORATES_WITH", UpsertEdgeOptions::default()) .unwrap(); staged_txn.delete_node(TxnNodeRef::Id(a)).unwrap(); staged_txn - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); assert!(staged_txn.get_edge(staged_edge).unwrap().is_none()); let revived = staged_txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 8, UpsertEdgeOptions::default()) + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "COLLABORATES_WITH", UpsertEdgeOptions::default()) .unwrap(); assert!(staged_txn.get_edge(revived).unwrap().is_some()); engine.close().unwrap(); @@ -660,19 +981,19 @@ fn test_write_txn_deleted_endpoint_triple_read_returns_none() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_edge(a, b, 7, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); txn.delete_node(TxnNodeRef::Id(a)).unwrap(); assert!(txn - .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7) + .get_edge_by_triple(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH") .unwrap() .is_none()); engine.close().unwrap(); @@ -686,10 +1007,10 @@ fn test_write_txn_snapshot_read_does_not_see_later_commit() { let txn = engine.begin_write_txn().unwrap(); engine - .upsert_node(1, "later", UpsertNodeOptions::default()) + .upsert_node("Person", "later", UpsertNodeOptions::default()) .unwrap(); - assert!(txn.get_node_by_key(1, "later").unwrap().is_none()); + assert!(txn.get_node_by_key("Person", "later").unwrap().is_none()); engine.close().unwrap(); } @@ -700,10 +1021,10 @@ fn test_write_txn_same_key_insert_conflict_has_no_wal_or_id_leak() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let mut txn = engine.begin_write_txn().unwrap(); - txn.upsert_node(1, "alice", UpsertNodeOptions::default()) + txn.upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let seq_before = engine.engine_seq_for_test(); let next_before = engine.next_node_id().unwrap(); @@ -715,6 +1036,133 @@ fn test_write_txn_same_key_insert_conflict_has_no_wal_or_id_leak() { engine.close().unwrap(); } +#[test] +fn test_write_txn_conflict_does_not_publish_staged_new_label_tokens() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + engine + .upsert_node("TxnStable", "conflict", UpsertNodeOptions::default()) + .unwrap(); + + let mut txn = engine.begin_write_txn().unwrap(); + let left = txn + .upsert_node("TxnNoLeakNode", "left", UpsertNodeOptions::default()) + .unwrap(); + let right = txn + .upsert_node("TxnNoLeakNode", "right", UpsertNodeOptions::default()) + .unwrap(); + txn.upsert_edge(left, right, "TXN_NO_LEAK_EDGE", UpsertEdgeOptions::default()) + .unwrap(); + txn.upsert_node( + "TxnStable", + "conflict", + UpsertNodeOptions { + weight: 2.0, + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_node( + "TxnStable", + "conflict", + UpsertNodeOptions { + weight: 3.0, + ..Default::default() + }, + ) + .unwrap(); + let seq_before = engine.engine_seq_for_test(); + let next_node_before = engine.next_node_id().unwrap(); + let next_edge_before = engine.next_edge_id().unwrap(); + + let err = txn.commit().unwrap_err(); + assert!(matches!(err, EngineError::TxnConflict(_))); + assert_eq!(engine.engine_seq_for_test(), seq_before); + assert_eq!(engine.next_node_id().unwrap(), next_node_before); + assert_eq!(engine.next_edge_id().unwrap(), next_edge_before); + assert_eq!(engine.get_node_label_id("TxnNoLeakNode").unwrap(), None); + assert_eq!(engine.get_edge_label_id("TXN_NO_LEAK_EDGE").unwrap(), None); + assert!(engine + .get_node_by_key("TxnNoLeakNode", "left") + .unwrap() + .is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_write_txn_multi_label_key_conflict_and_patch_token_atomicity() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut txn = engine.begin_write_txn().unwrap(); + txn.upsert_node( + &["TxnConflictNewA", "TxnConflictNewB"], + "same", + UpsertNodeOptions::default(), + ) + .unwrap(); + engine + .upsert_node("TxnConflictNewB", "same", UpsertNodeOptions::default()) + .unwrap(); + let err = txn.commit().unwrap_err(); + assert!(matches!(err, EngineError::TxnConflict(_))); + assert_eq!(engine.get_node_label_id("TxnConflictNewA").unwrap(), None); + + let stable = engine + .upsert_node("TxnPatchStable", "node", UpsertNodeOptions::default()) + .unwrap(); + + let mut noop_txn = engine.begin_write_txn().unwrap(); + assert!(!noop_txn + .add_node_label(TxnNodeRef::Id(stable), "TxnPatchStable") + .unwrap()); + assert!(!noop_txn + .remove_node_label(TxnNodeRef::Id(stable), "TxnPatchUnknownNoToken") + .unwrap()); + let noop_result = noop_txn.commit().unwrap(); + assert!(noop_result.node_ids.is_empty()); + assert!(noop_result.edge_ids.is_empty()); + assert_eq!( + engine.get_node_label_id("TxnPatchUnknownNoToken").unwrap(), + None + ); + + let mut patch_txn = engine.begin_write_txn().unwrap(); + assert!(patch_txn + .add_node_label(TxnNodeRef::Id(stable), "TxnPatchNoLeak") + .unwrap()); + engine + .upsert_node( + "TxnPatchStable", + "node", + UpsertNodeOptions { + weight: 2.0, + ..Default::default() + }, + ) + .unwrap(); + let err = patch_txn.commit().unwrap_err(); + assert!(matches!(err, EngineError::TxnConflict(_))); + assert_eq!(engine.get_node_label_id("TxnPatchNoLeak").unwrap(), None); + assert_eq!( + engine.get_node(stable).unwrap().unwrap().labels, + vec!["TxnPatchStable".to_string()] + ); + + let mut rollback_txn = engine.begin_write_txn().unwrap(); + let solo = rollback_txn + .upsert_node("TxnRollbackSolo", "solo", UpsertNodeOptions::default()) + .unwrap(); + let err = rollback_txn.remove_node_label(solo, "TxnRollbackSolo").unwrap_err(); + assert!(err.to_string().contains("last node label")); + rollback_txn.rollback().unwrap(); + assert_eq!(engine.get_node_label_id("TxnRollbackSolo").unwrap(), None); + engine.close().unwrap(); +} + #[test] fn test_write_txn_local_create_connect_conflicts_when_node_key_appears_after_begin() { let dir = TempDir::new().unwrap(); @@ -723,15 +1171,15 @@ fn test_write_txn_local_create_connect_conflicts_when_node_key_appears_after_beg let mut txn = engine.begin_write_txn().unwrap(); let alice = txn - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let bob = txn - .upsert_node(1, "bob", UpsertNodeOptions::default()) + .upsert_node("Person", "bob", UpsertNodeOptions::default()) .unwrap(); - txn.upsert_edge(alice, bob, 7, UpsertEdgeOptions::default()) + txn.upsert_edge(alice, bob, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let seq_before = engine.engine_seq_for_test(); let next_node_before = engine.next_node_id().unwrap(); @@ -752,12 +1200,12 @@ fn test_write_txn_later_node_conflict_after_tentative_create_does_not_leak_count let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let mut txn = engine.begin_write_txn().unwrap(); - txn.upsert_node(1, "tentative", UpsertNodeOptions::default()) + txn.upsert_node("Person", "tentative", UpsertNodeOptions::default()) .unwrap(); - txn.upsert_node(1, "conflict", UpsertNodeOptions::default()) + txn.upsert_node("Person", "conflict", UpsertNodeOptions::default()) .unwrap(); engine - .upsert_node(1, "conflict", UpsertNodeOptions::default()) + .upsert_node("Person", "conflict", UpsertNodeOptions::default()) .unwrap(); let seq_before = engine.engine_seq_for_test(); let next_node_before = engine.next_node_id().unwrap(); @@ -766,7 +1214,7 @@ fn test_write_txn_later_node_conflict_after_tentative_create_does_not_leak_count assert!(matches!(err, EngineError::TxnConflict(_))); assert_eq!(engine.engine_seq_for_test(), seq_before); assert_eq!(engine.next_node_id().unwrap(), next_node_before); - assert!(engine.get_node_by_key(1, "tentative").unwrap().is_none()); + assert!(engine.get_node_by_key("Person", "tentative").unwrap().is_none()); engine.close().unwrap(); } @@ -776,22 +1224,22 @@ fn test_write_txn_later_edge_conflict_after_tentative_create_does_not_leak_count let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = engine - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); - txn.upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, UpsertEdgeOptions::default()) + txn.upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); - txn.upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(c), 8, UpsertEdgeOptions::default()) + txn.upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(c), "COLLABORATES_WITH", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, c, 8, UpsertEdgeOptions::default()) + .upsert_edge(a, c, "COLLABORATES_WITH", UpsertEdgeOptions::default()) .unwrap(); let seq_before = engine.engine_seq_for_test(); let next_edge_before = engine.next_edge_id().unwrap(); @@ -800,7 +1248,7 @@ fn test_write_txn_later_edge_conflict_after_tentative_create_does_not_leak_count assert!(matches!(err, EngineError::TxnConflict(_))); assert_eq!(engine.engine_seq_for_test(), seq_before); assert_eq!(engine.next_edge_id().unwrap(), next_edge_before); - assert!(engine.get_edge_by_triple(a, b, 7).unwrap().is_none()); + assert!(engine.get_edge_by_triple(a, b, "FRIENDS_WITH").unwrap().is_none()); engine.close().unwrap(); } @@ -810,12 +1258,12 @@ fn test_write_txn_key_delete_conflict_when_target_deleted_after_begin() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id = engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); txn.delete_node(TxnNodeRef::Key { - type_id: 1, + label: "Person".to_string(), key: "alice".into(), }) .unwrap(); @@ -834,23 +1282,23 @@ fn test_write_txn_local_connect_conflicts_when_edge_triple_appears_after_begin() let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); let a_ref = txn - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b_ref = txn - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - txn.upsert_edge(a_ref, b_ref, 7, UpsertEdgeOptions::default()) + txn.upsert_edge(a_ref, b_ref, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, b, 7, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); let seq_before = engine.engine_seq_for_test(); let next_node_before = engine.next_node_id().unwrap(); @@ -871,11 +1319,11 @@ fn test_write_txn_same_node_update_conflicts() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id = engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); txn.upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 2.0, @@ -885,7 +1333,7 @@ fn test_write_txn_same_node_update_conflicts() { .unwrap(); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 3.0, @@ -906,17 +1354,17 @@ fn test_write_txn_same_triple_edge_conflicts() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); - txn.upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, UpsertEdgeOptions::default()) + txn.upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); engine - .upsert_edge(a, b, 7, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); let err = txn.commit().unwrap_err(); @@ -930,20 +1378,20 @@ fn test_write_txn_edge_triple_delete_conflict_when_deleted_after_begin() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let edge_id = engine - .upsert_edge(a, b, 7, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); txn.delete_edge(TxnEdgeRef::Triple { from: TxnNodeRef::Id(a), to: TxnNodeRef::Id(b), - type_id: 7, + label: "FRIENDS_WITH".to_string(), }) .unwrap(); engine.delete_edge(edge_id).unwrap(); @@ -960,13 +1408,13 @@ fn test_write_txn_delete_update_race_conflicts() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id = engine - .upsert_node(1, "alice", UpsertNodeOptions::default()) + .upsert_node("Person", "alice", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); txn.delete_node(TxnNodeRef::Id(id)).unwrap(); engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { weight: 4.0, @@ -987,10 +1435,10 @@ fn test_write_txn_delete_node_conflicts_on_future_incident_edge() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let mut txn = engine.begin_write_txn().unwrap(); @@ -999,7 +1447,7 @@ fn test_write_txn_delete_node_conflicts_on_future_incident_edge() { .upsert_edge( a, b, - 7, + "FRIENDS_WITH", UpsertEdgeOptions { valid_from: Some(i64::MAX / 2), ..Default::default() @@ -1021,10 +1469,10 @@ fn test_write_txn_deleted_endpoint_rejected_and_deleted_edge_not_resurrected() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let a = engine - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = engine - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let mut invalid = engine.begin_write_txn().unwrap(); @@ -1033,7 +1481,7 @@ fn test_write_txn_deleted_endpoint_rejected_and_deleted_edge_not_resurrected() { invalid.upsert_edge( TxnNodeRef::Id(a), TxnNodeRef::Id(b), - 7, + "FRIENDS_WITH", UpsertEdgeOptions::default() ), Err(EngineError::InvalidOperation(_)) @@ -1041,7 +1489,7 @@ fn test_write_txn_deleted_endpoint_rejected_and_deleted_edge_not_resurrected() { let mut txn = engine.begin_write_txn().unwrap(); let edge = txn - .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), 7, UpsertEdgeOptions::default()) + .upsert_edge(TxnNodeRef::Id(a), TxnNodeRef::Id(b), "FRIENDS_WITH", UpsertEdgeOptions::default()) .unwrap(); txn.delete_edge(edge.clone()).unwrap(); txn.invalidate_edge(edge, 123).unwrap(); @@ -1057,13 +1505,13 @@ fn test_write_txn_mixed_implicit_explicit_independent_success_after_flush() { let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let base = engine - .upsert_node(1, "base", UpsertNodeOptions::default()) + .upsert_node("Person", "base", UpsertNodeOptions::default()) .unwrap(); engine.flush().unwrap(); let mut txn = engine.begin_write_txn().unwrap(); txn.upsert_node( - 1, + "Person", "base", UpsertNodeOptions { weight: 2.0, @@ -1072,7 +1520,7 @@ fn test_write_txn_mixed_implicit_explicit_independent_success_after_flush() { ) .unwrap(); let other = engine - .upsert_node(1, "other", UpsertNodeOptions::default()) + .upsert_node("Person", "other", UpsertNodeOptions::default()) .unwrap(); assert!(txn.commit().is_ok()); assert!((engine.get_node(base).unwrap().unwrap().weight - 2.0).abs() < f32::EPSILON); diff --git a/src/engine/tests/wal_atomic.rs b/src/engine/tests/wal_atomic.rs new file mode 100644 index 0000000..52d3451 --- /dev/null +++ b/src/engine/tests/wal_atomic.rs @@ -0,0 +1,1281 @@ +use std::io::{Read, Seek, SeekFrom, Write}; + +#[derive(Clone, Copy)] +enum WalTailDamage { + MissingCommit, + CorruptCommit, +} + +fn wal_frame_bounds(path: &Path) -> Vec<(u64, u64)> { + let mut file = std::fs::File::open(path).unwrap(); + let mut pos = 8u64; + file.seek(SeekFrom::Start(pos)).unwrap(); + let mut frames = Vec::new(); + loop { + let mut len_buf = [0u8; 4]; + match file.read_exact(&mut len_buf) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(err) => panic!("failed to read WAL frame length: {err}"), + } + let payload_len = u32::from_le_bytes(len_buf) as u64; + let frame_start = pos; + let frame_end = frame_start + 4 + 4 + payload_len; + frames.push((frame_start, frame_end)); + pos = frame_end; + file.seek(SeekFrom::Start(pos)).unwrap(); + } + frames +} + +fn truncate_wal_after_frame(db_path: &Path, frame_count: usize) { + let path = wal_generation_path(db_path, 0); + let frames = wal_frame_bounds(&path); + let new_len = if frame_count == 0 { + 8 + } else { + frames + .get(frame_count - 1) + .map(|(_, end)| *end) + .expect("requested WAL frame must exist") + }; + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_len(new_len) + .unwrap(); +} + +fn corrupt_last_wal_frame_crc(db_path: &Path) { + let path = wal_generation_path(db_path, 0); + let frames = wal_frame_bounds(&path); + let (last_start, _) = *frames.last().expect("WAL must contain a frame"); + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start(last_start + 4)).unwrap(); + let mut crc_buf = [0u8; 4]; + file.read_exact(&mut crc_buf).unwrap(); + let crc = u32::from_le_bytes(crc_buf) ^ 0xFFFF_FFFF; + file.seek(SeekFrom::Start(last_start + 4)).unwrap(); + file.write_all(&crc.to_le_bytes()).unwrap(); + file.sync_all().unwrap(); +} + +fn truncate_last_wal_byte(db_path: &Path) { + let path = wal_generation_path(db_path, 0); + let len = std::fs::metadata(&path).unwrap().len(); + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_len(len - 1) + .unwrap(); +} + +fn append_raw_wal_frame(db_path: &Path, seq: u64, walop_bytes: &[u8]) { + let path = wal_generation_path(db_path, 0); + let seq_bytes = seq.to_le_bytes(); + let total_payload = 8 + walop_bytes.len(); + let len = total_payload as u32; + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&seq_bytes); + hasher.update(walop_bytes); + let crc = hasher.finalize(); + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(path) + .unwrap(); + file.write_all(&len.to_le_bytes()).unwrap(); + file.write_all(&crc.to_le_bytes()).unwrap(); + file.write_all(&seq_bytes).unwrap(); + file.write_all(walop_bytes).unwrap(); + file.sync_all().unwrap(); +} + +fn damage_last_atomic_commit(db_path: &Path, damage: WalTailDamage) { + match damage { + WalTailDamage::MissingCommit => { + let frame_count = wal_frame_bounds(&wal_generation_path(db_path, 0)).len(); + truncate_wal_after_frame(db_path, frame_count - 1); + } + WalTailDamage::CorruptCommit => corrupt_last_wal_frame_crc(db_path), + } +} + +fn write_manifest_with_catalog( + db_path: &Path, + node_labels: &[(&str, u32)], + edge_labels: &[(&str, u32)], +) { + std::fs::create_dir_all(db_path).unwrap(); + let mut manifest = default_manifest(); + for &(label, label_id) in node_labels { + manifest.node_label_tokens.insert(label.to_string(), label_id); + manifest.next_node_label_id = manifest.next_node_label_id.max(label_id + 1); + } + for &(label, label_id) in edge_labels { + manifest.edge_label_tokens.insert(label.to_string(), label_id); + manifest.next_edge_label_id = manifest.next_edge_label_id.max(label_id + 1); + } + write_manifest(db_path, &manifest).unwrap(); +} + +fn write_atomic_ops(db_path: &Path, ops: &[(u64, WalOp)]) { + let mut writer = WalWriter::open_generation(db_path, 0).unwrap(); + writer.append_batch(ops).unwrap(); + writer.sync().unwrap(); +} + +fn append_atomic_ops(writer: &mut WalWriter, ops: &[(u64, WalOp)]) { + writer.append_batch(ops).unwrap(); +} + +fn node_op(id: u64, label_id: u32, key: &str) -> WalOp { + WalOp::UpsertNode(NodeRecord { + id, + label_ids: NodeLabelSet::single(label_id).unwrap(), + key: key.to_string(), + props: BTreeMap::new(), + created_at: id as i64, + updated_at: id as i64, + weight: 1.0, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }) +} + +fn edge_op(id: u64, from: u64, to: u64, label_id: u32) -> WalOp { + WalOp::UpsertEdge(EdgeRecord { + id, + from, + to, + label_id: label_id, + props: BTreeMap::new(), + created_at: id as i64, + updated_at: id as i64, + weight: 1.0, + valid_from: 0, + valid_to: i64::MAX, + last_write_seq: 0, + }) +} + +fn node_input(label: &str, key: &str) -> NodeInput { + NodeInput { + labels: vec![label.to_string()], + key: key.to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + } +} + +fn edge_input(from: u64, to: u64, label: &str) -> EdgeInput { + EdgeInput { + from, + to, + label: label.to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + } +} + +fn assert_single_atomic_batch(records: &[(u64, WalOp)], first_seq: u64, op_count: u32) { + assert_eq!(records.len(), op_count as usize + 2); + assert!(matches!( + records.first().map(|(_, op)| op), + Some(WalOp::BeginAtomicBatch { + first_seq: observed_first_seq, + op_count: observed_op_count, + }) if *observed_first_seq == first_seq && *observed_op_count == op_count + )); + assert!(matches!( + records.last().map(|(_, op)| op), + Some(WalOp::CommitAtomicBatch { + first_seq: observed_first_seq, + op_count: observed_op_count, + }) if *observed_first_seq == first_seq && *observed_op_count == op_count + )); +} + +fn assert_last_atomic_batch(records: &[(u64, WalOp)], op_count: u32) { + let tail_len = op_count as usize + 2; + assert!(records.len() >= tail_len, "records: {records:?}"); + let tail = &records[records.len() - tail_len..]; + assert!(matches!( + tail.first().map(|(_, op)| op), + Some(WalOp::BeginAtomicBatch { + op_count: observed_op_count, + .. + }) if *observed_op_count == op_count + )); + assert!(matches!( + tail.last().map(|(_, op)| op), + Some(WalOp::CommitAtomicBatch { + op_count: observed_op_count, + .. + }) if *observed_op_count == op_count + )); +} + +#[test] +fn test_atomic_batch_append_emits_begin_ops_commit() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[("Person", 1)], &[]); + + write_atomic_ops( + &db_path, + &[(1, node_op(1, 1, "a")), (2, node_op(2, 1, "b"))], + ); + + let records = WalReader::read_generation(&db_path, 0).unwrap(); + assert_single_atomic_batch(&records, 1, 2); + assert!(matches!(&records[1].1, WalOp::UpsertNode(node) if node.id == 1)); + assert!(matches!(&records[2].1, WalOp::UpsertNode(node) if node.id == 2)); +} + +#[test] +fn test_representative_public_multi_op_apis_emit_atomic_markers() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + engine.ensure_node_label("Person").unwrap(); + engine.ensure_edge_label("KNOWS").unwrap(); + + let ids = engine + .batch_upsert_nodes(vec![node_input("Person", "batch-a"), node_input("Person", "batch-b")]) + .unwrap(); + assert_last_atomic_batch(&WalReader::read_generation(&db_path, 0).unwrap(), 2); + + engine + .batch_upsert_edges(vec![ + edge_input(ids[0], ids[1], "KNOWS"), + edge_input(ids[1], ids[0], "KNOWS"), + ]) + .unwrap(); + assert_last_atomic_batch(&WalReader::read_generation(&db_path, 0).unwrap(), 2); + + engine + .graph_patch(GraphPatch { + upsert_nodes: vec![node_input("Person", "patch-a"), node_input("Person", "patch-b")], + ..Default::default() + }) + .unwrap(); + assert_last_atomic_batch(&WalReader::read_generation(&db_path, 0).unwrap(), 2); + + let mut txn = engine.begin_write_txn().unwrap(); + txn.upsert_node("Person", "txn-a", UpsertNodeOptions::default()) + .unwrap(); + txn.upsert_node("Person", "txn-b", UpsertNodeOptions::default()) + .unwrap(); + txn.commit().unwrap(); + assert_last_atomic_batch(&WalReader::read_generation(&db_path, 0).unwrap(), 2); + + let cascade_a = engine + .upsert_node("Person", "cascade-a", UpsertNodeOptions::default()) + .unwrap(); + let cascade_b = engine + .upsert_node("Person", "cascade-b", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_edge(cascade_a, cascade_b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.delete_node(cascade_a).unwrap(); + assert_last_atomic_batch(&WalReader::read_generation(&db_path, 0).unwrap(), 2); + + let keep = engine + .upsert_node("Person", "prune-keep", UpsertNodeOptions::default()) + .unwrap(); + let prune = engine + .upsert_node( + "Person", + "prune-delete", + UpsertNodeOptions { + weight: 0.1, + ..Default::default() + }, + ) + .unwrap(); + engine + .upsert_edge(keep, prune, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .prune(&PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("Person".to_string()), + }) + .unwrap(); + assert_last_atomic_batch(&WalReader::read_generation(&db_path, 0).unwrap(), 2); + engine.close().unwrap(); +} + +#[test] +fn test_existing_label_upsert_node_emits_no_atomic_markers() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + engine.ensure_node_label("Person").unwrap(); + let id = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + + let records = WalReader::read_generation(&db_path, 0).unwrap(); + assert_no_atomic_batch_markers(&records); + assert!(matches!(records.last().map(|(_, op)| op), Some(WalOp::UpsertNode(node)) if node.id == id)); + engine.close().unwrap(); +} + +#[test] +fn test_existing_label_upsert_edge_emits_no_atomic_markers() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + engine.ensure_node_label("Person").unwrap(); + engine.ensure_edge_label("KNOWS").unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let edge_id = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let records = WalReader::read_generation(&db_path, 0).unwrap(); + assert_no_atomic_batch_markers(&records); + assert!(matches!(records.last().map(|(_, op)| op), Some(WalOp::UpsertEdge(edge)) if edge.id == edge_id)); + engine.close().unwrap(); +} + +#[test] +fn test_single_op_ensure_delete_and_invalidate_paths_emit_no_atomic_markers() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + engine.ensure_node_label("Person").unwrap(); + engine.ensure_edge_label("KNOWS").unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let orphan = engine + .upsert_node("Person", "orphan", UpsertNodeOptions::default()) + .unwrap(); + let edge_id = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.invalidate_edge(edge_id, 123).unwrap(); + engine.delete_edge(edge_id).unwrap(); + engine.delete_node(orphan).unwrap(); + + let records = WalReader::read_generation(&db_path, 0).unwrap(); + assert_no_atomic_batch_markers(&records); + assert!(records + .iter() + .any(|(_, op)| matches!(op, WalOp::UpsertEdge(edge) if edge.id == edge_id && edge.valid_to == 123))); + assert!(records + .iter() + .any(|(_, op)| matches!(op, WalOp::DeleteEdge { id, .. } if *id == edge_id))); + assert!(records + .iter() + .any(|(_, op)| matches!(op, WalOp::DeleteNode { id, .. } if *id == orphan))); + engine.close().unwrap(); +} + +#[test] +fn test_explicit_label_ensures_replay_as_standalone_token_mutations() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureEdgeLabel { + label: "KNOWS".to_string(), + label_id: 1, + }, + 2, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + let records = WalReader::read_generation(&db_path, 0).unwrap(); + assert_no_atomic_batch_markers(&records); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Person").unwrap(), Some(1)); + assert_eq!(engine.get_edge_label_id("KNOWS").unwrap(), Some(1)); + assert!(engine.get_node(1).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_first_use_upsert_node_emits_atomic_batch_and_replays_fully() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("api_wal_atomic"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let id = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let records = WalReader::read_generation(&db_path, 0).unwrap(); + assert_single_atomic_batch(&records, 1, 2); + assert!(matches!(&records[1].1, WalOp::EnsureNodeLabel { label, .. } if label == "Person")); + assert!(matches!(&records[2].1, WalOp::UpsertNode(node) if node.id == id)); + engine.close().unwrap(); + + let replay_path = dir.path().join("manual_node_replay"); + write_manifest_with_catalog(&replay_path, &[], &[]); + write_atomic_ops( + &replay_path, + &[ + ( + 1, + WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + ), + (2, node_op(1, 1, "alice")), + ], + ); + let replayed = DatabaseEngine::open(&replay_path, &DbOptions::default()).unwrap(); + assert_eq!(replayed.get_node_label_id("Person").unwrap(), Some(1)); + assert_eq!( + replayed.get_node_by_key("Person", "alice").unwrap().unwrap().id, + 1 + ); + replayed.close().unwrap(); +} + +#[test] +fn test_first_use_upsert_edge_emits_atomic_batch_and_replays_fully() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("api_wal_atomic"); + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + engine.ensure_node_label("Person").unwrap(); + let a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let edge_id = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let records = WalReader::read_generation(&db_path, 0).unwrap(); + let tail = &records[records.len() - 4..]; + assert_single_atomic_batch(tail, 4, 2); + assert!(matches!(&tail[1].1, WalOp::EnsureEdgeLabel { label, .. } if label == "KNOWS")); + assert!(matches!(&tail[2].1, WalOp::UpsertEdge(edge) if edge.id == edge_id)); + engine.close().unwrap(); + + let replay_path = dir.path().join("manual_edge_replay"); + write_manifest_with_catalog(&replay_path, &[("Person", 1)], &[]); + let mut writer = WalWriter::open_generation(&replay_path, 0).unwrap(); + writer.append(&node_op(1, 1, "a"), 1).unwrap(); + writer.append(&node_op(2, 1, "b"), 2).unwrap(); + append_atomic_ops( + &mut writer, + &[ + ( + 3, + WalOp::EnsureEdgeLabel { + label: "KNOWS".to_string(), + label_id: 1, + }, + ), + (4, edge_op(10, 1, 2, 1)), + ], + ); + writer.sync().unwrap(); + drop(writer); + let replayed = DatabaseEngine::open(&replay_path, &DbOptions::default()).unwrap(); + assert_eq!(replayed.get_edge_label_id("KNOWS").unwrap(), Some(1)); + assert_eq!(replayed.get_edge(10).unwrap().unwrap().label, "KNOWS"); + replayed.close().unwrap(); +} + +fn assert_first_use_node_tail_discard(damage: WalTailDamage) { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + write_atomic_ops( + &db_path, + &[ + ( + 1, + WalOp::EnsureNodeLabel { + label: "TailNode".to_string(), + label_id: 1, + }, + ), + (2, node_op(1, 1, "tail")), + ], + ); + damage_last_atomic_commit(&db_path, damage); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("TailNode").unwrap(), None); + assert!(engine.get_node(1).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_missing_commit_discards_first_use_node_batch() { + assert_first_use_node_tail_discard(WalTailDamage::MissingCommit); +} + +#[test] +fn test_corrupt_commit_discards_first_use_node_batch() { + assert_first_use_node_tail_discard(WalTailDamage::CorruptCommit); +} + +fn assert_first_use_edge_tail_discard(damage: WalTailDamage) { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[("Person", 1)], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer.append(&node_op(1, 1, "a"), 1).unwrap(); + writer.append(&node_op(2, 1, "b"), 2).unwrap(); + append_atomic_ops( + &mut writer, + &[ + ( + 3, + WalOp::EnsureEdgeLabel { + label: "TAIL_EDGE".to_string(), + label_id: 1, + }, + ), + (4, edge_op(10, 1, 2, 1)), + ], + ); + writer.sync().unwrap(); + drop(writer); + damage_last_atomic_commit(&db_path, damage); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_edge_label_id("TAIL_EDGE").unwrap(), None); + assert!(engine.get_edge(10).unwrap().is_none()); + assert!(engine.get_node(1).unwrap().is_some()); + engine.close().unwrap(); +} + +#[test] +fn test_missing_commit_discards_first_use_edge_batch() { + assert_first_use_edge_tail_discard(WalTailDamage::MissingCommit); +} + +#[test] +fn test_corrupt_commit_discards_first_use_edge_batch() { + assert_first_use_edge_tail_discard(WalTailDamage::CorruptCommit); +} + +#[test] +fn test_batch_upsert_nodes_complete_replay_and_torn_tail_discard() { + let dir = TempDir::new().unwrap(); + let complete_path = dir.path().join("batch_nodes_complete"); + write_manifest_with_catalog(&complete_path, &[("Person", 1)], &[]); + write_atomic_ops( + &complete_path, + &[(1, node_op(1, 1, "a")), (2, node_op(2, 1, "b"))], + ); + let engine = DatabaseEngine::open(&complete_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 2); + engine.close().unwrap(); + + let torn_path = dir.path().join("batch_nodes_torn"); + write_manifest_with_catalog(&torn_path, &[("Person", 1)], &[]); + write_atomic_ops( + &torn_path, + &[(1, node_op(1, 1, "a")), (2, node_op(2, 1, "b"))], + ); + damage_last_atomic_commit(&torn_path, WalTailDamage::MissingCommit); + let engine = DatabaseEngine::open(&torn_path, &DbOptions::default()).unwrap(); + assert!(engine.get_nodes_by_labels("Person").unwrap().is_empty()); + engine.close().unwrap(); +} + +#[test] +fn test_batch_upsert_edges_complete_replay_and_torn_tail_discard() { + let dir = TempDir::new().unwrap(); + let complete_path = dir.path().join("batch_edges_complete"); + write_manifest_with_catalog(&complete_path, &[], &[("KNOWS", 1)]); + write_atomic_ops( + &complete_path, + &[(1, edge_op(10, 1, 2, 1)), (2, edge_op(11, 2, 3, 1))], + ); + let engine = DatabaseEngine::open(&complete_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_edges_by_label("KNOWS").unwrap().len(), 2); + engine.close().unwrap(); + + let torn_path = dir.path().join("batch_edges_torn"); + write_manifest_with_catalog(&torn_path, &[], &[("KNOWS", 1)]); + write_atomic_ops( + &torn_path, + &[(1, edge_op(10, 1, 2, 1)), (2, edge_op(11, 2, 3, 1))], + ); + damage_last_atomic_commit(&torn_path, WalTailDamage::CorruptCommit); + let engine = DatabaseEngine::open(&torn_path, &DbOptions::default()).unwrap(); + assert!(engine.get_edges_by_label("KNOWS").unwrap().is_empty()); + engine.close().unwrap(); +} + +#[test] +fn test_atomic_batch_replay_tracks_intra_batch_edge_state_for_degree_deltas() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("batch_edge_degree"); + write_manifest_with_catalog(&db_path, &[("Person", 1)], &[("KNOWS", 1)]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer.append(&node_op(1, 1, "a"), 1).unwrap(); + writer.append(&node_op(2, 1, "b"), 2).unwrap(); + writer.append(&node_op(3, 1, "c"), 3).unwrap(); + append_atomic_ops( + &mut writer, + &[(4, edge_op(10, 1, 2, 1)), (5, edge_op(10, 1, 3, 1))], + ); + writer.sync().unwrap(); + drop(writer); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let edge = engine.get_edge(10).unwrap().expect("edge should replay"); + assert_eq!(edge.from, 1); + assert_eq!(edge.to, 3); + assert_eq!(engine.degree(1, &DegreeOptions::default()).unwrap(), 1); + assert_eq!( + engine + .degree( + 2, + &DegreeOptions { + direction: Direction::Incoming, + ..Default::default() + }, + ) + .unwrap(), + 0 + ); + assert_eq!( + engine + .degree( + 3, + &DegreeOptions { + direction: Direction::Incoming, + ..Default::default() + }, + ) + .unwrap(), + 1 + ); + engine.close().unwrap(); +} + +#[test] +fn test_graph_patch_complete_replay_and_torn_tail_discard() { + let dir = TempDir::new().unwrap(); + let complete_path = dir.path().join("graph_patch_complete"); + write_manifest_with_catalog(&complete_path, &[("Person", 1)], &[("KNOWS", 1)]); + write_atomic_ops( + &complete_path, + &[ + (1, node_op(1, 1, "a")), + (2, node_op(2, 1, "b")), + (3, edge_op(10, 1, 2, 1)), + ], + ); + let engine = DatabaseEngine::open(&complete_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 2); + assert!(engine.get_edge(10).unwrap().is_some()); + engine.close().unwrap(); + + let torn_path = dir.path().join("graph_patch_torn"); + write_manifest_with_catalog(&torn_path, &[("Person", 1)], &[("KNOWS", 1)]); + write_atomic_ops( + &torn_path, + &[ + (1, node_op(1, 1, "a")), + (2, node_op(2, 1, "b")), + (3, edge_op(10, 1, 2, 1)), + ], + ); + damage_last_atomic_commit(&torn_path, WalTailDamage::MissingCommit); + let engine = DatabaseEngine::open(&torn_path, &DbOptions::default()).unwrap(); + assert!(engine.get_nodes_by_labels("Person").unwrap().is_empty()); + assert!(engine.get_edge(10).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_write_transaction_commit_complete_replay_and_torn_tail_discard() { + let dir = TempDir::new().unwrap(); + let complete_path = dir.path().join("txn_complete"); + write_manifest_with_catalog(&complete_path, &[("TxnNode", 1)], &[("TXN_EDGE", 1)]); + write_atomic_ops( + &complete_path, + &[ + (1, node_op(1, 1, "txn-a")), + (2, node_op(2, 1, "txn-b")), + (3, edge_op(10, 1, 2, 1)), + ], + ); + let engine = DatabaseEngine::open(&complete_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_nodes_by_labels("TxnNode").unwrap().len(), 2); + assert!(engine.get_edge(10).unwrap().is_some()); + engine.close().unwrap(); + + let torn_path = dir.path().join("txn_torn"); + write_manifest_with_catalog(&torn_path, &[("TxnNode", 1)], &[("TXN_EDGE", 1)]); + write_atomic_ops( + &torn_path, + &[ + (1, node_op(1, 1, "txn-a")), + (2, node_op(2, 1, "txn-b")), + (3, edge_op(10, 1, 2, 1)), + ], + ); + damage_last_atomic_commit(&torn_path, WalTailDamage::CorruptCommit); + let engine = DatabaseEngine::open(&torn_path, &DbOptions::default()).unwrap(); + assert!(engine.get_nodes_by_labels("TxnNode").unwrap().is_empty()); + assert!(engine.get_edge(10).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_cascade_delete_node_complete_replay_and_torn_tail_discard() { + let dir = TempDir::new().unwrap(); + let complete_path = dir.path().join("cascade_complete"); + write_manifest_with_catalog(&complete_path, &[("Person", 1)], &[("KNOWS", 1)]); + let mut writer = WalWriter::open_generation(&complete_path, 0).unwrap(); + writer.append(&node_op(1, 1, "a"), 1).unwrap(); + writer.append(&node_op(2, 1, "b"), 2).unwrap(); + writer.append(&edge_op(10, 1, 2, 1), 3).unwrap(); + append_atomic_ops( + &mut writer, + &[ + ( + 4, + WalOp::DeleteEdge { + id: 10, + deleted_at: 4, + }, + ), + ( + 5, + WalOp::DeleteNode { + id: 1, + deleted_at: 5, + }, + ), + ], + ); + writer.sync().unwrap(); + drop(writer); + let engine = DatabaseEngine::open(&complete_path, &DbOptions::default()).unwrap(); + assert!(engine.get_node(1).unwrap().is_none()); + assert!(engine.get_edge(10).unwrap().is_none()); + assert!(engine.get_node(2).unwrap().is_some()); + engine.close().unwrap(); + + let torn_path = dir.path().join("cascade_torn"); + write_manifest_with_catalog(&torn_path, &[("Person", 1)], &[("KNOWS", 1)]); + let mut writer = WalWriter::open_generation(&torn_path, 0).unwrap(); + writer.append(&node_op(1, 1, "a"), 1).unwrap(); + writer.append(&node_op(2, 1, "b"), 2).unwrap(); + writer.append(&edge_op(10, 1, 2, 1), 3).unwrap(); + append_atomic_ops( + &mut writer, + &[ + ( + 4, + WalOp::DeleteEdge { + id: 10, + deleted_at: 4, + }, + ), + ( + 5, + WalOp::DeleteNode { + id: 1, + deleted_at: 5, + }, + ), + ], + ); + writer.sync().unwrap(); + drop(writer); + damage_last_atomic_commit(&torn_path, WalTailDamage::MissingCommit); + let engine = DatabaseEngine::open(&torn_path, &DbOptions::default()).unwrap(); + assert!(engine.get_node(1).unwrap().is_some()); + assert!(engine.get_edge(10).unwrap().is_some()); + engine.close().unwrap(); +} + +#[test] +fn test_prune_complete_replay_and_torn_tail_discard() { + let dir = TempDir::new().unwrap(); + let complete_path = dir.path().join("prune_complete"); + write_manifest_with_catalog(&complete_path, &[("Person", 1)], &[("KNOWS", 1)]); + let mut writer = WalWriter::open_generation(&complete_path, 0).unwrap(); + writer.append(&node_op(1, 1, "keep"), 1).unwrap(); + writer.append(&node_op(2, 1, "prune"), 2).unwrap(); + writer.append(&edge_op(10, 1, 2, 1), 3).unwrap(); + append_atomic_ops( + &mut writer, + &[ + ( + 4, + WalOp::DeleteEdge { + id: 10, + deleted_at: 4, + }, + ), + ( + 5, + WalOp::DeleteNode { + id: 2, + deleted_at: 5, + }, + ), + ], + ); + writer.sync().unwrap(); + drop(writer); + let engine = DatabaseEngine::open(&complete_path, &DbOptions::default()).unwrap(); + assert!(engine.get_node(1).unwrap().is_some()); + assert!(engine.get_node(2).unwrap().is_none()); + assert!(engine.get_edge(10).unwrap().is_none()); + engine.close().unwrap(); + + let torn_path = dir.path().join("prune_torn"); + write_manifest_with_catalog(&torn_path, &[("Person", 1)], &[("KNOWS", 1)]); + let mut writer = WalWriter::open_generation(&torn_path, 0).unwrap(); + writer.append(&node_op(1, 1, "keep"), 1).unwrap(); + writer.append(&node_op(2, 1, "prune"), 2).unwrap(); + writer.append(&edge_op(10, 1, 2, 1), 3).unwrap(); + append_atomic_ops( + &mut writer, + &[ + ( + 4, + WalOp::DeleteEdge { + id: 10, + deleted_at: 4, + }, + ), + ( + 5, + WalOp::DeleteNode { + id: 2, + deleted_at: 5, + }, + ), + ], + ); + writer.sync().unwrap(); + drop(writer); + damage_last_atomic_commit(&torn_path, WalTailDamage::CorruptCommit); + let engine = DatabaseEngine::open(&torn_path, &DbOptions::default()).unwrap(); + assert!(engine.get_node(2).unwrap().is_some()); + assert!(engine.get_edge(10).unwrap().is_some()); + engine.close().unwrap(); +} + +#[test] +fn test_committed_batch_before_corrupt_tail_batch_is_preserved() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[("Person", 1)], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + append_atomic_ops( + &mut writer, + &[(1, node_op(1, 1, "kept-a")), (2, node_op(2, 1, "kept-b"))], + ); + append_atomic_ops( + &mut writer, + &[(3, node_op(3, 1, "lost-a")), (4, node_op(4, 1, "lost-b"))], + ); + writer.sync().unwrap(); + drop(writer); + damage_last_atomic_commit(&db_path, WalTailDamage::CorruptCommit); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert!(engine.get_node(1).unwrap().is_some()); + assert!(engine.get_node(2).unwrap().is_some()); + assert!(engine.get_node(3).unwrap().is_none()); + assert!(engine.get_node(4).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_mismatched_commit_discards_open_batch() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::BeginAtomicBatch { + first_seq: 1, + op_count: 2, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Mismatch".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer.append(&node_op(1, 1, "lost"), 2).unwrap(); + writer + .append( + &WalOp::CommitAtomicBatch { + first_seq: 99, + op_count: 2, + }, + 1, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Mismatch").unwrap(), None); + assert!(engine.get_node(1).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_commit_without_begin_preserves_earlier_standalone_records() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer.append(&node_op(1, 1, "kept"), 2).unwrap(); + writer + .append( + &WalOp::CommitAtomicBatch { + first_seq: 3, + op_count: 2, + }, + 3, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Person").unwrap(), Some(1)); + assert!(engine.get_node(1).unwrap().is_some()); + engine.close().unwrap(); +} + +#[test] +fn test_corrupt_marker_frame_keeps_prior_standalone_records_and_discards_tail() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Person".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer.append(&node_op(1, 1, "kept"), 2).unwrap(); + writer.sync().unwrap(); + drop(writer); + + let mut corrupt_marker = vec![OpTag::BeginAtomicBatch as u8]; + corrupt_marker.extend_from_slice(&3u64.to_le_bytes()); + append_raw_wal_frame(&db_path, 3, &corrupt_marker); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Person").unwrap(), Some(1)); + assert!(engine.get_node(1).unwrap().is_some()); + engine.close().unwrap(); +} + +#[test] +fn test_truncated_commit_marker_discards_open_batch() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + write_atomic_ops( + &db_path, + &[ + ( + 1, + WalOp::EnsureNodeLabel { + label: "Truncated".to_string(), + label_id: 1, + }, + ), + (2, node_op(1, 1, "lost")), + ], + ); + truncate_last_wal_byte(&db_path); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Truncated").unwrap(), None); + assert!(engine.get_node(1).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_nested_begin_discards_open_batch_without_applying_partial_ops() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Base".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::BeginAtomicBatch { + first_seq: 2, + op_count: 2, + }, + 2, + ) + .unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Lost".to_string(), + label_id: 2, + }, + 2, + ) + .unwrap(); + writer + .append( + &WalOp::BeginAtomicBatch { + first_seq: 3, + op_count: 2, + }, + 3, + ) + .unwrap(); + writer.append(&node_op(2, 2, "lost"), 3).unwrap(); + writer.sync().unwrap(); + drop(writer); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Base").unwrap(), Some(1)); + assert_eq!(engine.get_node_label_id("Lost").unwrap(), None); + assert!(engine.get_node(2).unwrap().is_none()); + engine.close().unwrap(); +} + +#[test] +fn test_huge_atomic_op_count_begin_discards_tail_without_preallocating() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Base".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::BeginAtomicBatch { + first_seq: 2, + op_count: u32::MAX, + }, + 2, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Base").unwrap(), Some(1)); + engine.close().unwrap(); +} + +#[test] +fn test_malformed_enclosed_record_discards_open_batch_without_failing_open() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + let mut writer = WalWriter::open_generation(&db_path, 0).unwrap(); + writer + .append( + &WalOp::EnsureNodeLabel { + label: "Base".to_string(), + label_id: 1, + }, + 1, + ) + .unwrap(); + writer + .append( + &WalOp::BeginAtomicBatch { + first_seq: 2, + op_count: 2, + }, + 2, + ) + .unwrap(); + writer.sync().unwrap(); + drop(writer); + append_raw_wal_frame(&db_path, 2, &[OpTag::EnsureNodeLabel as u8]); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert_eq!(engine.get_node_label_id("Base").unwrap(), Some(1)); + engine.close().unwrap(); +} + +#[test] +fn test_recovered_incomplete_atomic_tail_is_truncated_before_later_appends() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + write_atomic_ops( + &db_path, + &[ + ( + 1, + WalOp::EnsureNodeLabel { + label: "Lost".to_string(), + label_id: 1, + }, + ), + (2, node_op(1, 1, "lost")), + ], + ); + damage_last_atomic_commit(&db_path, WalTailDamage::MissingCommit); + + let opts = DbOptions { + wal_sync_mode: WalSyncMode::Immediate, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert_eq!(engine.get_node_label_id("Lost").unwrap(), None); + assert!(engine.get_node(1).unwrap().is_none()); + + let survivor = engine + .upsert_node("Survivor", "kept", UpsertNodeOptions::default()) + .unwrap(); + engine.close_fast().unwrap(); + + let reopened = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert_eq!(reopened.get_node_label_id("Lost").unwrap(), None); + assert!(reopened.get_node_by_key("Lost", "lost").unwrap().is_none()); + assert_eq!( + reopened + .get_node(survivor) + .unwrap() + .unwrap() + .labels + .as_slice(), + ["Survivor"] + ); + reopened.close().unwrap(); +} + +#[test] +fn test_committed_batch_with_invalid_enclosed_op_fails_without_token_manifest_leak() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("wal_atomic"); + write_manifest_with_catalog(&db_path, &[], &[]); + write_atomic_ops( + &db_path, + &[ + ( + 1, + WalOp::EnsureNodeLabel { + label: "Valid".to_string(), + label_id: 1, + }, + ), + (2, node_op(1, 99, "missing-token")), + ], + ); + + assert!(matches!( + DatabaseEngine::open(&db_path, &DbOptions::default()), + Err(EngineError::CorruptWal(_)) + )); + let manifest = load_manifest_readonly(&db_path).unwrap().unwrap(); + assert!(!manifest.node_label_tokens.contains_key("Valid")); +} diff --git a/src/engine/tests/write.rs b/src/engine/tests/write.rs index 4a977fc..dd39b7d 100644 --- a/src/engine/tests/write.rs +++ b/src/engine/tests/write.rs @@ -10,9 +10,9 @@ let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id1 = engine - .upsert_node(1, "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .upsert_node("Person", "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) .unwrap(); - let id2 = engine.upsert_node(1, "bob", UpsertNodeOptions { weight: 0.6, ..Default::default() }).unwrap(); + let id2 = engine.upsert_node("Person", "bob", UpsertNodeOptions { weight: 0.6, ..Default::default() }).unwrap(); assert_ne!(id1, id2); assert_eq!(engine.node_count().unwrap(), 2); @@ -31,13 +31,13 @@ let mut props_v1 = BTreeMap::new(); props_v1.insert("version".to_string(), PropValue::Int(1)); - let id1 = engine.upsert_node(1, "alice", UpsertNodeOptions { props: props_v1, weight: 0.5, ..Default::default() }).unwrap(); + let id1 = engine.upsert_node("Person", "alice", UpsertNodeOptions { props: props_v1, weight: 0.5, ..Default::default() }).unwrap(); let mut props_v2 = BTreeMap::new(); props_v2.insert("version".to_string(), PropValue::Int(2)); - let id2 = engine.upsert_node(1, "alice", UpsertNodeOptions { props: props_v2, weight: 0.9, ..Default::default() }).unwrap(); + let id2 = engine.upsert_node("Person", "alice", UpsertNodeOptions { props: props_v2, weight: 0.9, ..Default::default() }).unwrap(); - // Same (type_id, key) → same ID, updated fields + // Same (label_id, key) → same ID, updated fields assert_eq!(id1, id2); assert_eq!(engine.node_count().unwrap(), 1); @@ -54,1070 +54,2657 @@ let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let node_id = engine.upsert_node(1, "alice", UpsertNodeOptions::default()).unwrap(); + let node_id = engine.upsert_node("Person", "alice", UpsertNodeOptions::default()).unwrap(); let node = engine.get_node(node_id).unwrap().unwrap(); assert!((node.weight - 1.0).abs() < f32::EPSILON); engine.close().unwrap(); } - #[test] - fn test_upsert_node_with_vectors_survives_restart() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let opts = DbOptions { - dense_vector: Some(DenseVectorConfig { - dimension: 3, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }), - ..DbOptions::default() - }; - - let node_id; - { - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - node_id = engine - .upsert_node( - 1, - "alice", - UpsertNodeOptions { - weight: 0.5, - dense_vector: Some(vec![0.1, 0.2, 0.3]), - sparse_vector: Some(vec![(9, 0.0), (4, 1.0), (2, 2.0), (4, 0.5), (2, 0.0)]), - ..Default::default() - }, - ) - .unwrap(); - - let node = engine.get_node(node_id).unwrap().unwrap(); - assert_eq!(node.dense_vector, Some(vec![0.1, 0.2, 0.3])); - assert_eq!(node.sparse_vector, Some(vec![(2, 2.0), (4, 1.5)])); - engine.close().unwrap(); + fn node_input_with_labels( + labels: &[&str], + key: &str, + props: BTreeMap, + ) -> NodeInput { + NodeInput { + labels: labels.iter().map(|label| (*label).to_string()).collect(), + key: key.to_string(), + props, + weight: 1.0, + dense_vector: None, + sparse_vector: None, } + } - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - let node = engine.get_node(node_id).unwrap().unwrap(); - assert_eq!(node.dense_vector, Some(vec![0.1, 0.2, 0.3])); - assert_eq!(node.sparse_vector, Some(vec![(2, 2.0), (4, 1.5)])); - engine.close().unwrap(); + fn score_props(score: i64) -> BTreeMap { + let mut props = BTreeMap::new(); + props.insert("score".to_string(), PropValue::Int(score)); + props } #[test] - fn test_upsert_node_dense_vector_requires_config() { + fn test_multi_label_upsert_replaces_and_queries_memberships() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let err = engine + engine + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Equality) + .unwrap(); + engine + .ensure_node_property_index("Employee", "score", SecondaryIndexKind::Equality) + .unwrap(); + engine + .ensure_node_property_index( + "Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + engine + .ensure_node_property_index( + "Employee", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + + let id = engine .upsert_node( - 1, + &["Person", "Employee"], "alice", UpsertNodeOptions { - weight: 0.5, - dense_vector: Some(vec![0.1, 0.2, 0.3]), + props: score_props(7), ..Default::default() }, ) - .unwrap_err(); - assert!(matches!(err, EngineError::InvalidOperation(_))); - engine.close().unwrap(); - } - - #[test] - fn test_upsert_node_rejects_wrong_dense_dimension() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let opts = DbOptions { - dense_vector: Some(DenseVectorConfig { - dimension: 2, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }), - ..DbOptions::default() - }; - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + .unwrap(); + let node = engine.get_node(id).unwrap().unwrap(); + assert_eq!(node.labels, vec!["Person".to_string(), "Employee".to_string()]); + assert_eq!( + engine.get_node_by_key("Person", "alice").unwrap().unwrap().id, + id + ); + assert_eq!( + engine.get_node_by_key("Employee", "alice").unwrap().unwrap().id, + id + ); + assert_eq!(engine.nodes_by_labels("Person").unwrap(), vec![id]); + assert_eq!(engine.nodes_by_labels("Employee").unwrap(), vec![id]); + assert_eq!(engine.count_nodes_by_labels("Person").unwrap(), 1); + assert_eq!(engine.count_nodes_by_labels("Employee").unwrap(), 1); + assert_eq!( + engine + .get_nodes_by_keys(&[ + NodeKeyQuery { + label: "Employee".to_string(), + key: "alice".to_string(), + }, + NodeKeyQuery { + label: "Missing".to_string(), + key: "alice".to_string(), + }, + NodeKeyQuery { + label: "Person".to_string(), + key: "alice".to_string(), + }, + ]) + .unwrap() + .into_iter() + .map(|node| node.map(|node| node.id)) + .collect::>(), + vec![Some(id), None, Some(id)] + ); + assert_eq!( + engine + .find_nodes("Person", "score", &PropValue::Int(7)) + .unwrap(), + vec![id] + ); + assert_eq!( + engine + .find_nodes("Employee", "score", &PropValue::Int(7)) + .unwrap(), + vec![id] + ); + assert_eq!( + engine + .find_nodes_range( + "Employee", + "score", + Some(&PropertyRangeBound::Included(PropValue::Int(7))), + Some(&PropertyRangeBound::Included(PropValue::Int(7))), + ) + .unwrap(), + vec![id] + ); - let err = engine + let updated = engine .upsert_node( - 1, + &["Person"], "alice", UpsertNodeOptions { - weight: 0.5, - dense_vector: Some(vec![0.1, 0.2, 0.3]), + props: score_props(9), ..Default::default() }, ) - .unwrap_err(); - assert!(matches!(err, EngineError::InvalidOperation(_))); + .unwrap(); + assert_eq!(updated, id); + assert_eq!(engine.get_node(id).unwrap().unwrap().labels, vec!["Person".to_string()]); + assert!(engine.get_node_by_key("Employee", "alice").unwrap().is_none()); + assert!(engine.nodes_by_labels("Employee").unwrap().is_empty()); + assert_eq!(engine.count_nodes_by_labels("Employee").unwrap(), 0); + assert!(engine + .find_nodes("Employee", "score", &PropValue::Int(7)) + .unwrap() + .is_empty()); + assert_eq!( + engine + .find_nodes("Person", "score", &PropValue::Int(9)) + .unwrap(), + vec![id] + ); + engine.close().unwrap(); } #[test] - fn test_write_op_normalizes_node_vectors() { + fn test_flexible_label_inputs_and_batch_label_vectors() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let opts = DbOptions { - dense_vector: Some(DenseVectorConfig { - dimension: 2, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }), - ..DbOptions::default() - }; - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - engine - .write_op(&WalOp::UpsertNode(NodeRecord { - id: 1, - type_id: 1, - key: "manual".to_string(), - props: BTreeMap::new(), - created_at: 100, - updated_at: 101, - weight: 0.5, - dense_vector: Some(vec![0.1, 0.2]), - sparse_vector: Some(vec![(5, 0.0), (3, 1.0), (3, 2.0)]), - last_write_seq: 0, - })) + let owned_label = String::from("Person"); + let id = engine + .upsert_node(owned_label.clone(), "alice", UpsertNodeOptions::default()) .unwrap(); + assert_eq!( + engine + .upsert_node(&owned_label, "alice", UpsertNodeOptions::default()) + .unwrap(), + id + ); + + let labels = vec!["Person".to_string(), "Employee".to_string()]; + assert_eq!( + engine + .upsert_node(labels, "alice", UpsertNodeOptions::default()) + .unwrap(), + id + ); + assert_eq!( + engine.get_node(id).unwrap().unwrap().labels, + vec!["Person".to_string(), "Employee".to_string()] + ); + + let array_labels = ["Person".to_string(), "Reviewer".to_string()]; + let bob = engine + .upsert_node(&array_labels, "bob", UpsertNodeOptions::default()) + .unwrap(); + assert_eq!( + engine.get_node(bob).unwrap().unwrap().labels, + vec!["Person".to_string(), "Reviewer".to_string()] + ); + + let batch_ids = engine + .batch_upsert_nodes(vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "carol".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string(), "Employee".to_string()], + key: "dana".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string(), "Contractor".to_string()], + key: "erin".to_string(), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }, + ]) + .unwrap(); + assert_eq!( + engine.get_node(batch_ids[0]).unwrap().unwrap().labels, + vec!["Person".to_string()] + ); + assert_eq!( + engine.get_node(batch_ids[1]).unwrap().unwrap().labels, + vec!["Person".to_string(), "Employee".to_string()] + ); + assert_eq!( + engine.get_node(batch_ids[2]).unwrap().unwrap().labels, + vec!["Person".to_string(), "Contractor".to_string()] + ); + + let mut txn = engine.begin_write_txn().unwrap(); + txn.upsert_node_as( + "frank", + vec!["Person".to_string(), "Employee".to_string()], + "frank", + UpsertNodeOptions::default(), + ) + .unwrap(); + txn.commit().unwrap(); + + let frank = engine.get_node_by_key("Employee", "frank").unwrap().unwrap(); + assert_eq!( + frank.labels, + vec!["Person".to_string(), "Employee".to_string()] + ); - let node = engine.get_node(1).unwrap().unwrap(); - assert_eq!(node.dense_vector, Some(vec![0.1, 0.2])); - assert_eq!(node.sparse_vector, Some(vec![(3, 3.0)])); engine.close().unwrap(); } #[test] - fn test_batch_upsert_nodes_with_vectors_survives_restart() { + fn test_multi_label_conflicts_and_token_staging_are_atomic() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let opts = DbOptions { - dense_vector: Some(DenseVectorConfig { - dimension: 3, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }), - ..DbOptions::default() - }; - - let alice_id; - let bob_id; - { - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - let ids = engine - .batch_upsert_nodes(&[ - NodeInput { - type_id: 1, - key: "alice".to_string(), - props: BTreeMap::new(), - weight: 0.5, - dense_vector: Some(vec![0.1, 0.2, 0.3]), - sparse_vector: Some(vec![ - (9, 0.0), - (4, 1.0), - (2, 2.0), - (4, 0.25), - ]), - }, - NodeInput { - type_id: 1, - key: "bob".to_string(), - props: BTreeMap::new(), - weight: 0.7, - dense_vector: None, - sparse_vector: None, - }, - ]) - .unwrap(); - - alice_id = ids[0]; - bob_id = ids[1]; + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let alice = engine.get_node(alice_id).unwrap().unwrap(); - assert_eq!(alice.dense_vector, Some(vec![0.1, 0.2, 0.3])); - assert_eq!(alice.sparse_vector, Some(vec![(2, 2.0), (4, 1.25)])); + let left = engine + .upsert_node("ConflictA", "shared", UpsertNodeOptions::default()) + .unwrap(); + let right = engine + .upsert_node("ConflictB", "shared", UpsertNodeOptions::default()) + .unwrap(); - let bob = engine.get_node(bob_id).unwrap().unwrap(); - assert!(bob.dense_vector.is_none()); - assert!(bob.sparse_vector.is_none()); - engine.close().unwrap(); - } + let err = engine + .upsert_node( + &["ConflictA", "ConflictB", "ConflictNew"], + "shared", + UpsertNodeOptions::default(), + ) + .unwrap_err(); + assert!(err.to_string().contains("node key conflict")); + assert_eq!(engine.get_node_label_id("ConflictNew").unwrap(), None); + assert_eq!( + engine.get_node_by_key("ConflictA", "shared").unwrap().unwrap().id, + left + ); + assert_eq!( + engine.get_node_by_key("ConflictB", "shared").unwrap().unwrap().id, + right + ); + + let duplicate = engine + .upsert_node( + &["DuplicateLabel", "DuplicateLabel"], + "dup", + UpsertNodeOptions::default(), + ) + .unwrap_err(); + assert!(duplicate.to_string().contains("duplicate label")); + assert_eq!(engine.get_node_label_id("DuplicateLabel").unwrap(), None); + + let batch_err = engine + .batch_upsert_nodes(vec![ + node_input_with_labels(&["BatchA"], "shared", BTreeMap::new()), + node_input_with_labels(&["BatchB"], "shared", BTreeMap::new()), + node_input_with_labels(&["BatchA", "BatchB", "BatchNew"], "shared", BTreeMap::new()), + ]) + .unwrap_err(); + assert!(batch_err.to_string().contains("node key conflict")); + assert_eq!(engine.get_node_label_id("BatchA").unwrap(), None); + assert_eq!(engine.get_node_label_id("BatchB").unwrap(), None); + assert_eq!(engine.get_node_label_id("BatchNew").unwrap(), None); + assert_eq!(engine.node_count().unwrap(), 2); - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - let alice = engine.get_node(alice_id).unwrap().unwrap(); - assert_eq!(alice.dense_vector, Some(vec![0.1, 0.2, 0.3])); - assert_eq!(alice.sparse_vector, Some(vec![(2, 2.0), (4, 1.25)])); + let batch_ids = engine + .batch_upsert_nodes(vec![ + node_input_with_labels(&["BatchOkA", "BatchOkB"], "ok", BTreeMap::new()), + node_input_with_labels(&["BatchOkB"], "other", BTreeMap::new()), + ]) + .unwrap(); + assert_eq!(batch_ids.len(), 2); + assert_eq!( + engine.get_node_by_key("BatchOkA", "ok").unwrap().unwrap().id, + batch_ids[0] + ); + assert_eq!( + engine.get_node_by_key("BatchOkB", "ok").unwrap().unwrap().id, + batch_ids[0] + ); + assert_eq!( + engine + .get_node_by_key("BatchOkB", "other") + .unwrap() + .unwrap() + .id, + batch_ids[1] + ); - let bob = engine.get_node(bob_id).unwrap().unwrap(); - assert!(bob.dense_vector.is_none()); - assert!(bob.sparse_vector.is_none()); engine.close().unwrap(); } #[test] - fn test_upsert_node_different_types_same_key() { + fn test_graph_patch_multi_label_conflict_rolls_back_tokens_and_nodes() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let id1 = engine - .upsert_node(1, "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + engine + .upsert_node("PatchA", "shared", UpsertNodeOptions::default()) .unwrap(); - let id2 = engine - .upsert_node(2, "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + engine + .upsert_node("PatchB", "shared", UpsertNodeOptions::default()) .unwrap(); - - // Different type_id → different nodes - assert_ne!(id1, id2); + let err = engine + .graph_patch(GraphPatch { + upsert_nodes: vec![node_input_with_labels( + &["PatchA", "PatchB", "PatchNew"], + "shared", + BTreeMap::new(), + )], + ..Default::default() + }) + .unwrap_err(); + assert!(err.to_string().contains("node key conflict")); + assert_eq!(engine.get_node_label_id("PatchNew").unwrap(), None); assert_eq!(engine.node_count().unwrap(), 2); + assert!(engine.get_node_by_key("PatchNew", "shared").unwrap().is_none()); engine.close().unwrap(); } #[test] - fn test_upsert_node_id_counter_monotonic() { + fn test_stale_label_memberships_are_suppressed_after_flush_reopen_and_compact() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let opts = DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }; - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let id; + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + id = engine + .upsert_node(&["FreshA", "FreshB"], "k", UpsertNodeOptions::default()) + .unwrap(); + engine.flush().unwrap(); - let mut ids = Vec::new(); - for i in 0..10 { - ids.push( + assert_eq!( engine - .upsert_node(1, &format!("node:{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .upsert_node(&["FreshA"], "k", UpsertNodeOptions::default()) .unwrap(), + id ); - } + assert!(engine.get_node_by_key("FreshB", "k").unwrap().is_none()); + assert!(engine.nodes_by_labels("FreshB").unwrap().is_empty()); + assert_eq!(engine.count_nodes_by_labels("FreshB").unwrap(), 0); + + engine.flush().unwrap(); + assert!(engine.get_node_by_key("FreshB", "k").unwrap().is_none()); + assert!(engine.nodes_by_labels("FreshB").unwrap().is_empty()); + + let mut kept_ids = Vec::new(); + for index in 0..8 { + let node_id = engine + .upsert_node( + &["FreshPageA", "FreshPageB"], + &format!("page-{index}"), + UpsertNodeOptions::default(), + ) + .unwrap(); + if index >= 5 { + kept_ids.push(node_id); + } + } + engine.flush().unwrap(); + for index in 0..5 { + engine + .upsert_node( + &["FreshPageA"], + &format!("page-{index}"), + UpsertNodeOptions::default(), + ) + .unwrap(); + } - // All IDs should be unique and monotonically increasing - for i in 1..ids.len() { - assert!(ids[i] > ids[i - 1]); + let label_page = engine + .nodes_by_labels_paged( + "FreshPageB", + &PageRequest { + limit: Some(2), + after: None, + }, + ) + .unwrap(); + assert_eq!(label_page.items, kept_ids[..2].to_vec()); + let label_page_2 = engine + .nodes_by_labels_paged( + "FreshPageB", + &PageRequest { + limit: Some(2), + after: label_page.next_cursor, + }, + ) + .unwrap(); + assert_eq!(label_page_2.items, kept_ids[2..].to_vec()); + assert!(label_page_2.next_cursor.is_none()); + + let time_page = engine + .find_nodes_by_time_range_paged( + "FreshPageB", + i64::MIN, + i64::MAX, + &PageRequest { + limit: Some(2), + after: None, + }, + ) + .unwrap(); + assert_eq!(time_page.items, kept_ids[..2].to_vec()); + let time_page_2 = engine + .find_nodes_by_time_range_paged( + "FreshPageB", + i64::MIN, + i64::MAX, + &PageRequest { + limit: Some(2), + after: time_page.next_cursor, + }, + ) + .unwrap(); + assert_eq!(time_page_2.items, kept_ids[2..].to_vec()); + assert!(time_page_2.next_cursor.is_none()); + + engine.close().unwrap(); } - engine.close().unwrap(); - } + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + assert_eq!( + engine.get_node_by_key("FreshA", "k").unwrap().unwrap().id, + id + ); + assert!(engine.get_node_by_key("FreshB", "k").unwrap().is_none()); + assert!(engine.nodes_by_labels("FreshB").unwrap().is_empty()); + engine.compact().unwrap(); + assert_eq!( + engine.get_node_by_key("FreshA", "k").unwrap().unwrap().id, + id + ); + assert!(engine.get_node_by_key("FreshB", "k").unwrap().is_none()); + assert!(engine.nodes_by_labels("FreshB").unwrap().is_empty()); + engine.close().unwrap(); + } + } #[test] - fn test_upsert_edge_new() { + fn test_stale_label_memberships_are_suppressed_from_immutable_memtable() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let n1 = engine - .upsert_node(1, "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + engine + .ensure_node_property_index("ImmFreshB", "score", SecondaryIndexKind::Equality) .unwrap(); - let n2 = engine.upsert_node(1, "bob", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - - let e1 = engine - .upsert_edge(n1, n2, 10, UpsertEdgeOptions::default()) + engine + .ensure_node_property_index( + "ImmFreshB", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + let id = engine + .upsert_node( + &["ImmFreshA", "ImmFreshB"], + "k", + UpsertNodeOptions { + props: score_props(42), + ..Default::default() + }, + ) .unwrap(); + engine.freeze_memtable().unwrap(); - assert_eq!(engine.edge_count().unwrap(), 1); - let edge = engine.get_edge(e1).unwrap().unwrap(); - assert_eq!(edge.from, n1); - assert_eq!(edge.to, n2); + assert_eq!( + engine + .upsert_node( + &["ImmFreshA"], + "k", + UpsertNodeOptions { + props: score_props(43), + ..Default::default() + }, + ) + .unwrap(), + id + ); + + assert_eq!( + engine.get_node_by_key("ImmFreshA", "k").unwrap().unwrap().id, + id + ); + assert!(engine.get_node_by_key("ImmFreshB", "k").unwrap().is_none()); + assert_eq!( + engine + .get_nodes_by_keys(&[ + NodeKeyQuery { + label: "ImmFreshB".to_string(), + key: "k".to_string(), + }, + NodeKeyQuery { + label: "ImmFreshA".to_string(), + key: "k".to_string(), + }, + ]) + .unwrap() + .into_iter() + .map(|node| node.map(|node| node.id)) + .collect::>(), + vec![None, Some(id)] + ); + assert!(engine.nodes_by_labels("ImmFreshB").unwrap().is_empty()); + assert_eq!(engine.count_nodes_by_labels("ImmFreshB").unwrap(), 0); + assert!(engine + .find_nodes("ImmFreshB", "score", &PropValue::Int(42)) + .unwrap() + .is_empty()); + assert!(engine + .find_nodes_range( + "ImmFreshB", + "score", + Some(&PropertyRangeBound::Included(PropValue::Int(42))), + Some(&PropertyRangeBound::Included(PropValue::Int(42))), + ) + .unwrap() + .is_empty()); + assert!(engine + .find_nodes_by_time_range("ImmFreshB", i64::MIN, i64::MAX) + .unwrap() + .is_empty()); engine.close().unwrap(); } #[test] - fn test_upsert_edge_without_uniqueness_creates_duplicates() { + fn test_add_remove_node_label_semantics() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - - // Default: edge_uniqueness = false let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let e1 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions::default()) - .unwrap(); - let e2 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions::default()) + let id = engine + .upsert_node("PatchPerson", "alice", UpsertNodeOptions::default()) .unwrap(); + let initial_updated_at = engine.get_node(id).unwrap().unwrap().updated_at; + std::thread::sleep(std::time::Duration::from_millis(10)); + assert!(engine.add_node_label(id, "PatchEmployee").unwrap()); + let after_add = engine.get_node(id).unwrap().unwrap(); + assert!(after_add.updated_at > initial_updated_at); + assert!(engine + .find_nodes_by_time_range("PatchPerson", initial_updated_at, initial_updated_at) + .unwrap() + .is_empty()); + assert_eq!( + engine + .find_nodes_by_time_range("PatchPerson", after_add.updated_at, after_add.updated_at) + .unwrap(), + vec![id] + ); + assert_eq!( + engine + .find_nodes_by_time_range( + "PatchEmployee", + after_add.updated_at, + after_add.updated_at, + ) + .unwrap(), + vec![id] + ); + assert!(!engine.add_node_label(id, "PatchEmployee").unwrap()); + assert_eq!( + engine + .get_node_by_key("PatchEmployee", "alice") + .unwrap() + .unwrap() + .id, + id + ); + assert_eq!( + engine.get_node(id).unwrap().unwrap().labels, + vec!["PatchPerson".to_string(), "PatchEmployee".to_string()] + ); - // Without uniqueness: creates separate edges - assert_ne!(e1, e2); - assert_eq!(engine.edge_count().unwrap(), 2); + engine + .upsert_node("PatchContractor", "alice", UpsertNodeOptions::default()) + .unwrap(); + let err = engine.add_node_label(id, "PatchContractor").unwrap_err(); + assert!(err.to_string().contains("node key conflict")); + + assert!(!engine.remove_node_label(id, "MissingButValid").unwrap()); + std::thread::sleep(std::time::Duration::from_millis(10)); + assert!(engine.remove_node_label(id, "PatchEmployee").unwrap()); + let after_remove = engine.get_node(id).unwrap().unwrap(); + assert!(after_remove.updated_at > after_add.updated_at); + assert!(engine + .find_nodes_by_time_range("PatchPerson", after_add.updated_at, after_add.updated_at) + .unwrap() + .is_empty()); + assert_eq!( + engine + .find_nodes_by_time_range( + "PatchPerson", + after_remove.updated_at, + after_remove.updated_at, + ) + .unwrap(), + vec![id] + ); + assert!(engine + .find_nodes_by_time_range("PatchEmployee", after_add.updated_at, after_add.updated_at) + .unwrap() + .is_empty()); + assert!(engine + .get_node_by_key("PatchEmployee", "alice") + .unwrap() + .is_none()); + assert!(!engine.remove_node_label(id, "PatchEmployee").unwrap()); + let err = engine.remove_node_label(id, "PatchPerson").unwrap_err(); + assert!(err.to_string().contains("last node label")); + + let full_labels = [ + "Full0", "Full1", "Full2", "Full3", "Full4", "Full5", "Full6", "Full7", "Full8", + "Full9", + ]; + let full_id = engine + .upsert_node(&full_labels[..], "full", UpsertNodeOptions::default()) + .unwrap(); + let label_count = engine.list_node_labels().unwrap().len(); + let err = engine.add_node_label(full_id, "FullNew").unwrap_err(); + assert!(err.to_string().contains("at most 10 labels")); + assert_eq!(engine.get_node_label_id("FullNew").unwrap(), None); + assert_eq!(engine.list_node_labels().unwrap().len(), label_count); engine.close().unwrap(); } #[test] - fn test_upsert_edge_with_uniqueness_dedup() { + fn test_write_txn_multi_label_upsert_and_patch_semantics() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let opts = DbOptions { - edge_uniqueness: true, - ..DbOptions::default() - }; - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - - let e1 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) - .unwrap(); - let e2 = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions { weight: 0.9, ..Default::default() }) + let mut txn = engine.begin_write_txn().unwrap(); + let alice = txn + .upsert_node(&["TxnPerson", "TxnEmployee"], "alice", UpsertNodeOptions::default()) .unwrap(); + let staged = txn.get_node(alice.clone()).unwrap().unwrap(); + assert_eq!( + staged.labels, + vec!["TxnPerson".to_string(), "TxnEmployee".to_string()] + ); + assert_eq!( + txn.get_node_by_key("TxnEmployee", "alice") + .unwrap() + .unwrap() + .local, + match alice.clone() { + TxnNodeRef::Local(local) => Some(local), + _ => None, + } + ); + assert!(txn.add_node_label(alice.clone(), "TxnManager").unwrap()); + assert!(!txn.add_node_label(alice.clone(), "TxnManager").unwrap()); + assert!(txn.remove_node_label(alice.clone(), "TxnEmployee").unwrap()); + assert!(txn + .get_node_by_key("TxnEmployee", "alice") + .unwrap() + .is_none()); + let committed = txn.commit().unwrap(); + let id = committed.node_ids[0]; + assert_eq!( + engine.get_node(id).unwrap().unwrap().labels, + vec!["TxnPerson".to_string(), "TxnManager".to_string()] + ); + assert_eq!( + engine.get_node_by_key("TxnManager", "alice").unwrap().unwrap().id, + id + ); + assert!(engine + .get_node_by_key("TxnEmployee", "alice") + .unwrap() + .is_none()); - // With uniqueness: same triple → same ID, updated weight - assert_eq!(e1, e2); - assert_eq!(engine.edge_count().unwrap(), 1); - assert!((engine.get_edge(e1).unwrap().unwrap().weight - 0.9).abs() < f32::EPSILON); - - // Different triple → new edge - let e3 = engine - .upsert_edge(1, 2, 20, UpsertEdgeOptions::default()) + let mut conflict_txn = engine.begin_write_txn().unwrap(); + conflict_txn + .upsert_node("TxnConflictA", "same", UpsertNodeOptions::default()) .unwrap(); - assert_ne!(e1, e3); - assert_eq!(engine.edge_count().unwrap(), 2); + conflict_txn + .upsert_node("TxnConflictB", "same", UpsertNodeOptions::default()) + .unwrap(); + assert!(matches!( + conflict_txn.upsert_node( + &["TxnConflictA", "TxnConflictB"], + "same", + UpsertNodeOptions::default(), + ), + Err(EngineError::InvalidOperation(_)) + )); + conflict_txn.rollback().unwrap(); + assert_eq!(engine.get_node_label_id("TxnConflictA").unwrap(), None); + assert_eq!(engine.get_node_label_id("TxnConflictB").unwrap(), None); + + let mut invalid_key_txn = engine.begin_write_txn().unwrap(); + let too_long_key = "k".repeat(u16::MAX as usize + 1); + let err = invalid_key_txn + .upsert_node("TxnInvalidKey", &too_long_key, UpsertNodeOptions::default()) + .unwrap_err(); + assert!(err.to_string().contains("node key must be at most")); + assert_eq!(engine.get_node_label_id("TxnInvalidKey").unwrap(), None); engine.close().unwrap(); } #[test] - fn test_batch_upsert_nodes() { + fn test_upsert_node_with_vectors_survives_restart() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let inputs: Vec = (0..1000) - .map(|i| NodeInput { - type_id: 1, - key: format!("node:{}", i), - props: BTreeMap::new(), - weight: 0.5, - dense_vector: None, - sparse_vector: None, - }) - .collect(); - - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); - assert_eq!(ids.len(), 1000); - assert_eq!(engine.node_count().unwrap(), 1000); + let node_id; + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + node_id = engine + .upsert_node( + "Person", + "alice", + UpsertNodeOptions { + weight: 0.5, + dense_vector: Some(vec![0.1, 0.2, 0.3]), + sparse_vector: Some(vec![(9, 0.0), (4, 1.0), (2, 2.0), (4, 0.5), (2, 0.0)]), + ..Default::default() + }, + ) + .unwrap(); - // All queryable - for (i, &id) in ids.iter().enumerate() { - let node = engine.get_node(id).unwrap().unwrap(); - assert_eq!(node.key, format!("node:{}", i)); + let node = engine.get_node(node_id).unwrap().unwrap(); + assert_eq!(node.dense_vector, Some(vec![0.1, 0.2, 0.3])); + assert_eq!(node.sparse_vector, Some(vec![(2, 2.0), (4, 1.5)])); + engine.close().unwrap(); } + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let node = engine.get_node(node_id).unwrap().unwrap(); + assert_eq!(node.dense_vector, Some(vec![0.1, 0.2, 0.3])); + assert_eq!(node.sparse_vector, Some(vec![(2, 2.0), (4, 1.5)])); engine.close().unwrap(); } #[test] - fn test_batch_upsert_nodes_with_dedup() { + fn test_upsert_node_dense_vector_requires_config() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // Pre-insert a node - let pre_id = engine - .upsert_node(1, "existing", UpsertNodeOptions { weight: 0.5, ..Default::default() }) - .unwrap(); - - // Batch with duplicate key and one that matches pre-existing - let inputs = vec![ - NodeInput { - type_id: 1, - key: "new1".into(), - props: BTreeMap::new(), - weight: 0.5, - dense_vector: None, - sparse_vector: None, - }, - NodeInput { - type_id: 1, - key: "existing".into(), - props: BTreeMap::new(), - weight: 0.9, - dense_vector: None, - sparse_vector: None, - }, - NodeInput { - type_id: 1, - key: "new1".into(), - props: BTreeMap::new(), - weight: 0.8, - dense_vector: None, - sparse_vector: None, - }, // dup within batch - ]; - - let ids = engine.batch_upsert_nodes(&inputs).unwrap(); - assert_eq!(ids.len(), 3); - assert_eq!(ids[1], pre_id); // "existing" reuses pre-existing ID - assert_eq!(ids[0], ids[2]); // "new1" appears twice → same ID - assert_eq!(engine.node_count().unwrap(), 2); // "existing" + "new1" - + let err = engine + .upsert_node( + "Person", + "alice", + UpsertNodeOptions { + weight: 0.5, + dense_vector: Some(vec![0.1, 0.2, 0.3]), + ..Default::default() + }, + ) + .unwrap_err(); + assert!(matches!(err, EngineError::InvalidOperation(_))); engine.close().unwrap(); } #[test] - fn test_batch_upsert_edges() { + fn test_upsert_node_rejects_wrong_dense_dimension() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - - let inputs: Vec = (0..100) - .map(|i| EdgeInput { - from: i, - to: i + 1, - type_id: 10, - props: BTreeMap::new(), - weight: 1.0, - valid_from: None, - valid_to: None, - }) - .collect(); - - let ids = engine.batch_upsert_edges(&inputs).unwrap(); - assert_eq!(ids.len(), 100); - assert_eq!(engine.edge_count().unwrap(), 100); - + let err = engine + .upsert_node( + "Person", + "alice", + UpsertNodeOptions { + weight: 0.5, + dense_vector: Some(vec![0.1, 0.2, 0.3]), + ..Default::default() + }, + ) + .unwrap_err(); + assert!(matches!(err, EngineError::InvalidOperation(_))); engine.close().unwrap(); } #[test] - fn test_upsert_survives_restart() { + fn test_write_op_normalizes_node_vectors() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 2, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let label_id = engine.ensure_node_label("ManualVectorNode").unwrap(); - let (id1, id2, eid); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - id1 = engine - .upsert_node(1, "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) - .unwrap(); - id2 = engine.upsert_node(1, "bob", UpsertNodeOptions { weight: 0.6, ..Default::default() }).unwrap(); - eid = engine - .upsert_edge(id1, id2, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine.close().unwrap(); - } - - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // close() flushes to segments; verify via cross-source lookup - assert_eq!(engine.get_nodes_by_type(1).unwrap().len(), 2); - assert_eq!(engine.get_node(id1).unwrap().unwrap().key, "alice"); - assert_eq!(engine.get_node(id2).unwrap().unwrap().key, "bob"); - assert_eq!(engine.get_edge(eid).unwrap().unwrap().from, id1); + write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { + id: 1, + label_ids: NodeLabelSet::single(label_id).unwrap(), + key: "manual".to_string(), + props: BTreeMap::new(), + created_at: 100, + updated_at: 101, + weight: 0.5, + dense_vector: Some(vec![0.1, 0.2]), + sparse_vector: Some(vec![(5, 0.0), (3, 1.0), (3, 2.0)]), + last_write_seq: 0, + })) + .unwrap(); - // Upsert dedup should still work after close-flush + reopen - let id1_again = engine - .upsert_node(1, "alice", UpsertNodeOptions { weight: 0.99, ..Default::default() }) - .unwrap(); - assert_eq!(id1_again, id1); + let node = engine.get_node(1).unwrap().unwrap(); + assert_eq!(node.dense_vector, Some(vec![0.1, 0.2])); + assert_eq!(node.sparse_vector, Some(vec![(3, 3.0)])); + engine.close().unwrap(); + } - // New allocations should not reuse old IDs - let id3 = engine - .upsert_node(1, "charlie", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + #[test] + fn test_batch_upsert_nodes_with_vectors_survives_restart() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..DbOptions::default() + }; + + let alice_id; + let bob_id; + { + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let ids = engine + .batch_upsert_nodes(vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "alice".to_string(), + props: BTreeMap::new(), + weight: 0.5, + dense_vector: Some(vec![0.1, 0.2, 0.3]), + sparse_vector: Some(vec![ + (9, 0.0), + (4, 1.0), + (2, 2.0), + (4, 0.25), + ]), + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "bob".to_string(), + props: BTreeMap::new(), + weight: 0.7, + dense_vector: None, + sparse_vector: None, + }, + ]) .unwrap(); - assert!(id3 > id2); + alice_id = ids[0]; + bob_id = ids[1]; + + let alice = engine.get_node(alice_id).unwrap().unwrap(); + assert_eq!(alice.dense_vector, Some(vec![0.1, 0.2, 0.3])); + assert_eq!(alice.sparse_vector, Some(vec![(2, 2.0), (4, 1.25)])); + + let bob = engine.get_node(bob_id).unwrap().unwrap(); + assert!(bob.dense_vector.is_none()); + assert!(bob.sparse_vector.is_none()); engine.close().unwrap(); } + + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + let alice = engine.get_node(alice_id).unwrap().unwrap(); + assert_eq!(alice.dense_vector, Some(vec![0.1, 0.2, 0.3])); + assert_eq!(alice.sparse_vector, Some(vec![(2, 2.0), (4, 1.25)])); + + let bob = engine.get_node(bob_id).unwrap().unwrap(); + assert!(bob.dense_vector.is_none()); + assert!(bob.sparse_vector.is_none()); + engine.close().unwrap(); } #[test] - fn test_upsert_node_preserves_created_at() { + fn test_upsert_node_different_labels_same_key() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id1 = engine - .upsert_node(1, "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .upsert_node("Person", "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) .unwrap(); - let created_at_v1 = engine.get_node(id1).unwrap().unwrap().created_at; - - // Small delay not needed, just upsert again. created_at must be preserved let id2 = engine - .upsert_node(1, "alice", UpsertNodeOptions { weight: 0.9, ..Default::default() }) + .upsert_node("Company", "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) .unwrap(); - assert_eq!(id1, id2); - let node = engine.get_node(id1).unwrap().unwrap(); - assert_eq!(node.created_at, created_at_v1); - assert!(node.updated_at >= created_at_v1); + // Different label-key memberships produce different nodes. + assert_ne!(id1, id2); + assert_eq!(engine.node_count().unwrap(), 2); engine.close().unwrap(); } #[test] - fn test_batch_upsert_edges_with_uniqueness() { + fn test_upsert_node_id_counter_monotonic() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let opts = DbOptions { - edge_uniqueness: true, - ..DbOptions::default() - }; - let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); - - // Pre-insert an edge - let pre_id = engine - .upsert_edge(1, 2, 10, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) - .unwrap(); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // Batch with: duplicate within batch + match against pre-existing - let inputs = vec![ - EdgeInput { - from: 3, - to: 4, - type_id: 10, - props: BTreeMap::new(), - weight: 0.5, - valid_from: None, - valid_to: None, - }, - EdgeInput { - from: 1, - to: 2, - type_id: 10, - props: BTreeMap::new(), - weight: 0.9, - valid_from: None, - valid_to: None, - }, // matches pre-existing - EdgeInput { - from: 3, - to: 4, - type_id: 10, - props: BTreeMap::new(), - weight: 0.8, - valid_from: None, - valid_to: None, - }, // dup within batch - ]; + let mut ids = Vec::new(); + for i in 0..10 { + ids.push( + engine + .upsert_node("Person", &format!("node:{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(), + ); + } - let ids = engine.batch_upsert_edges(&inputs).unwrap(); - assert_eq!(ids.len(), 3); - assert_eq!(ids[1], pre_id); // reuses pre-existing ID - assert_eq!(ids[0], ids[2]); // within-batch dedup - assert_eq!(engine.edge_count().unwrap(), 2); // pre-existing + one new + // All IDs should be unique and monotonically increasing + for i in 1..ids.len() { + assert!(ids[i] > ids[i - 1]); + } engine.close().unwrap(); } #[test] - fn test_id_counters_survive_restart() { + fn test_upsert_edge_new() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let last_node_id; - let last_edge_id; - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - for i in 0..10 { - engine - .upsert_node(1, &format!("n:{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) - .unwrap(); - } - for i in 0..5 { - engine - .upsert_edge(i, i + 1, 10, UpsertEdgeOptions::default()) - .unwrap(); - } - last_node_id = engine.next_node_id().unwrap(); - last_edge_id = engine.next_edge_id().unwrap(); - engine.close().unwrap(); - } + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert!(engine.next_node_id().unwrap() >= last_node_id); - assert!(engine.next_edge_id().unwrap() >= last_edge_id); - engine.close().unwrap(); - } - } + let n1 = engine + .upsert_node("Person", "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + let n2 = engine.upsert_node("Person", "bob", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - // --- Adjacency, neighbors, delete tests --- + let e1 = engine + .upsert_edge(n1, n2, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + assert_eq!(engine.edge_count().unwrap(), 1); + let edge = engine.get_edge(e1).unwrap().unwrap(); + assert_eq!(edge.from, n1); + assert_eq!(edge.to, n2); + + engine.close().unwrap(); + } #[test] - fn test_neighbors_outgoing() { + fn test_upsert_edge_without_uniqueness_creates_duplicates() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + // Default: edge_uniqueness = false let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let e1 = engine + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + let e2 = engine + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + // Without uniqueness: creates separate edges + assert_ne!(e1, e2); + assert_eq!(engine.edge_count().unwrap(), 2); - engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + engine.close().unwrap(); + } + + #[test] + fn test_upsert_edge_with_uniqueness_dedup() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let opts = DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + let e1 = engine + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) .unwrap(); - engine - .upsert_edge(a, c, 20, UpsertEdgeOptions { weight: 0.8, ..Default::default() }) + let e2 = engine + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions { weight: 0.9, ..Default::default() }) .unwrap(); - let out = engine - .neighbors(a, &NeighborOptions::default()) + // With uniqueness: same triple → same ID, updated weight + assert_eq!(e1, e2); + assert_eq!(engine.edge_count().unwrap(), 1); + assert!((engine.get_edge(e1).unwrap().unwrap().weight - 0.9).abs() < f32::EPSILON); + + // Different triple → new edge + let e3 = engine + .upsert_edge(1, 2, "REPORTS_TO", UpsertEdgeOptions::default()) .unwrap(); - assert_eq!(out.len(), 2); - let neighbor_ids: Vec = out.iter().map(|e| e.node_id).collect(); - assert!(neighbor_ids.contains(&b)); - assert!(neighbor_ids.contains(&c)); + assert_ne!(e1, e3); + assert_eq!(engine.edge_count().unwrap(), 2); engine.close().unwrap(); } #[test] - fn test_neighbors_incoming() { + fn test_batch_upsert_nodes() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let inputs: Vec = (0..1000) + .map(|i| NodeInput { + labels: vec!["Person".to_string()], + key: format!("node:{}", i), + props: BTreeMap::new(), + weight: 0.5, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + + let ids = engine.batch_upsert_nodes(inputs).unwrap(); + assert_eq!(ids.len(), 1000); + assert_eq!(engine.node_count().unwrap(), 1000); + + // All queryable + for (i, &id) in ids.iter().enumerate() { + let node = engine.get_node(id).unwrap().unwrap(); + assert_eq!(node.key, format!("node:{}", i)); + } + + engine.close().unwrap(); + } + + #[test] + fn test_batch_upsert_nodes_with_dedup() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + // Pre-insert a node + let pre_id = engine + .upsert_node("Person", "existing", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + + // Batch with duplicate key and one that matches pre-existing + let inputs = vec![ + NodeInput { + labels: vec!["Person".to_string()], + key: "new1".into(), + props: BTreeMap::new(), + weight: 0.5, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "existing".into(), + props: BTreeMap::new(), + weight: 0.9, + dense_vector: None, + sparse_vector: None, + }, + NodeInput { + labels: vec!["Person".to_string()], + key: "new1".into(), + props: BTreeMap::new(), + weight: 0.8, + dense_vector: None, + sparse_vector: None, + }, // dup within batch + ]; + + let ids = engine.batch_upsert_nodes(inputs).unwrap(); + assert_eq!(ids.len(), 3); + assert_eq!(ids[1], pre_id); // "existing" reuses pre-existing ID + assert_eq!(ids[0], ids[2]); // "new1" appears twice → same ID + assert_eq!(engine.node_count().unwrap(), 2); // "existing" + "new1" + + engine.close().unwrap(); + } + + #[test] + fn test_batch_upsert_edges() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let inputs: Vec = (0..100) + .map(|i| EdgeInput { + from: i, + to: i + 1, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 1.0, + valid_from: None, + valid_to: None, + }) + .collect(); + + let ids = engine.batch_upsert_edges(inputs).unwrap(); + assert_eq!(ids.len(), 100); + assert_eq!(engine.edge_count().unwrap(), 100); + + engine.close().unwrap(); + } + + #[test] + fn test_upsert_survives_restart() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let (id1, id2, eid); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + id1 = engine + .upsert_node("Person", "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + id2 = engine.upsert_node("Person", "bob", UpsertNodeOptions { weight: 0.6, ..Default::default() }).unwrap(); + eid = engine + .upsert_edge(id1, id2, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.close().unwrap(); + } + + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + // close() flushes to segments; verify via cross-source lookup + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 2); + assert_eq!(engine.get_node(id1).unwrap().unwrap().key, "alice"); + assert_eq!(engine.get_node(id2).unwrap().unwrap().key, "bob"); + assert_eq!(engine.get_edge(eid).unwrap().unwrap().from, id1); + + // Upsert dedup should still work after close-flush + reopen + let id1_again = engine + .upsert_node("Person", "alice", UpsertNodeOptions { weight: 0.99, ..Default::default() }) + .unwrap(); + assert_eq!(id1_again, id1); + + // New allocations should not reuse old IDs + let id3 = engine + .upsert_node("Person", "charlie", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + assert!(id3 > id2); + + engine.close().unwrap(); + } + } + + #[test] + fn test_upsert_node_preserves_created_at() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let id1 = engine + .upsert_node("Person", "alice", UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + let created_at_v1 = engine.get_node(id1).unwrap().unwrap().created_at; + + // Small delay not needed, just upsert again. created_at must be preserved + let id2 = engine + .upsert_node("Person", "alice", UpsertNodeOptions { weight: 0.9, ..Default::default() }) + .unwrap(); + assert_eq!(id1, id2); + + let node = engine.get_node(id1).unwrap().unwrap(); + assert_eq!(node.created_at, created_at_v1); + assert!(node.updated_at >= created_at_v1); + + engine.close().unwrap(); + } + + #[test] + fn test_batch_upsert_edges_with_uniqueness() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let opts = DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }; + let engine = DatabaseEngine::open(&db_path, &opts).unwrap(); + + // Pre-insert an edge + let pre_id = engine + .upsert_edge(1, 2, "KNOWS", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + + // Batch with: duplicate within batch + match against pre-existing + let inputs = vec![ + EdgeInput { + from: 3, + to: 4, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 0.5, + valid_from: None, + valid_to: None, + }, + EdgeInput { + from: 1, + to: 2, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 0.9, + valid_from: None, + valid_to: None, + }, // matches pre-existing + EdgeInput { + from: 3, + to: 4, + label: "KNOWS".to_string(), + props: BTreeMap::new(), + weight: 0.8, + valid_from: None, + valid_to: None, + }, // dup within batch + ]; + + let ids = engine.batch_upsert_edges(inputs).unwrap(); + assert_eq!(ids.len(), 3); + assert_eq!(ids[1], pre_id); // reuses pre-existing ID + assert_eq!(ids[0], ids[2]); // within-batch dedup + assert_eq!(engine.edge_count().unwrap(), 2); // pre-existing + one new + + engine.close().unwrap(); + } + + #[test] + fn test_id_counters_survive_restart() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let last_node_id; + let last_edge_id; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + for i in 0..10 { + engine + .upsert_node("Person", &format!("n:{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + } + for i in 0..5 { + engine + .upsert_edge(i, i + 1, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + last_node_id = engine.next_node_id().unwrap(); + last_edge_id = engine.next_edge_id().unwrap(); + engine.close().unwrap(); + } + + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert!(engine.next_node_id().unwrap() >= last_node_id); + assert!(engine.next_edge_id().unwrap() >= last_edge_id); + engine.close().unwrap(); + } + } + + // --- Adjacency, neighbors, delete tests --- + + #[test] + fn test_neighbors_outgoing() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + + engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions { weight: 0.8, ..Default::default() }) + .unwrap(); + + let out = engine + .neighbors(a, &NeighborOptions::default()) + .unwrap(); + assert_eq!(out.len(), 2); + let neighbor_ids: Vec = out.iter().map(|e| e.node_id).collect(); + assert!(neighbor_ids.contains(&b)); + assert!(neighbor_ids.contains(&c)); + + engine.close().unwrap(); + } + + #[test] + fn test_neighbors_incoming() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + + engine + .upsert_edge(a, c, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(b, c, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let inc = engine + .neighbors(c, &NeighborOptions { direction: Direction::Incoming, ..Default::default() }) + .unwrap(); + assert_eq!(inc.len(), 2); + let neighbor_ids: Vec = inc.iter().map(|e| e.node_id).collect(); + assert!(neighbor_ids.contains(&a)); + assert!(neighbor_ids.contains(&b)); + + engine.close().unwrap(); + } + + #[test] + fn test_neighbors_with_label_filter() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + + engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); // KNOWS + engine + .upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions::default()) + .unwrap(); // REPORTS_TO + + let labeled = engine + .neighbors(a, &NeighborOptions { edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }) + .unwrap(); + assert_eq!(labeled.len(), 1); + assert_eq!(labeled[0].node_id, b); + + engine.close().unwrap(); + } + + #[test] + fn test_neighbors_with_limit() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let hub = engine.upsert_node("Person", "hub", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + for i in 0..10 { + let n = engine + .upsert_node("Person", &format!("spoke:{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + engine + .upsert_edge(hub, n, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + } + + let limited = engine + .neighbors(hub, &NeighborOptions { limit: Some(3), ..Default::default() }) + .unwrap(); + assert_eq!(limited.len(), 3); + + engine.close().unwrap(); + } + + #[test] + fn test_delete_node_via_api() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + engine.delete_node(b).unwrap(); + + assert!(engine.get_node(b).unwrap().is_none()); + assert_eq!(engine.node_count().unwrap(), 1); + + // b excluded from a's neighbors (node tombstone filtering) + let out = engine + .neighbors(a, &NeighborOptions::default()) + .unwrap(); + assert!(out.is_empty()); + + engine.close().unwrap(); + } + + #[test] + fn test_delete_edge_via_api() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let eid = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + engine.delete_edge(eid).unwrap(); + + assert!(engine.get_edge(eid).unwrap().is_none()); + assert_eq!(engine.edge_count().unwrap(), 0); + assert!(engine + .neighbors(a, &NeighborOptions::default()) + .unwrap() + .is_empty()); + + engine.close().unwrap(); + } + + #[test] + fn test_delete_survives_restart() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let (a, b, eid); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + eid = engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine.delete_node(b).unwrap(); + engine.delete_edge(eid).unwrap(); + engine.close().unwrap(); + } + + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + assert!(engine.get_node(b).unwrap().is_none()); + assert!(engine.get_edge(eid).unwrap().is_none()); + // close() flushes to segments; use cross-source counts + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 1); + // Verify deleted edge not visible + assert!(engine + .neighbors(a, &NeighborOptions::default()) + .unwrap() + .is_empty()); + engine.close().unwrap(); + } + } + + #[test] + fn test_neighbors_survive_restart() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let (a, b, c); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + a = engine.upsert_node("Person", "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + b = engine.upsert_node("Person", "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + c = engine.upsert_node("Person", "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + engine + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + engine + .upsert_edge(a, c, "REPORTS_TO", UpsertEdgeOptions { weight: 0.8, ..Default::default() }) + .unwrap(); + engine + .upsert_edge(b, c, "KNOWS", UpsertEdgeOptions { weight: 0.5, ..Default::default() }) + .unwrap(); + engine.close().unwrap(); + } + + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + // a → b, c + let out_a = engine + .neighbors(a, &NeighborOptions::default()) + .unwrap(); + assert_eq!(out_a.len(), 2); + // b → c + let out_b = engine + .neighbors(b, &NeighborOptions::default()) + .unwrap(); + assert_eq!(out_b.len(), 1); + assert_eq!(out_b[0].node_id, c); + // c ← a, b + let inc_c = engine + .neighbors(c, &NeighborOptions { direction: Direction::Incoming, ..Default::default() }) + .unwrap(); + assert_eq!(inc_c.len(), 2); + engine.close().unwrap(); + } + } + + #[test] + fn test_node_property_index_ensure_drop_list_and_conflicting_range_domains() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let eq = engine + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(eq.state, SecondaryIndexState::Building); + + let eq_again = engine + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(eq_again.index_id, eq.index_id); + + let range = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + assert_eq!(range.state, SecondaryIndexState::Building); + + let indexes = engine.list_node_property_indexes().unwrap(); + assert_eq!(indexes.len(), 2); + assert_eq!(indexes[0].prop_key, "color"); + assert_eq!(indexes[1].prop_key, "score"); + + let err = engine + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Float, + }, + ) + .unwrap_err(); + assert!(matches!(err, EngineError::InvalidOperation(_))); + + assert!(engine + .drop_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap()); + assert!(!engine + .drop_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap()); + + let indexes = engine.list_node_property_indexes().unwrap(); + assert_eq!(indexes.len(), 1); + assert_eq!(indexes[0].index_id, range.index_id); + + engine.close().unwrap(); + } + + #[test] + fn test_node_property_index_retry_failed_clears_error_and_preserves_id() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let created = engine + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + engine.shutdown_secondary_index_worker(); + + engine + .with_runtime_manifest_write(|manifest| { + let entry = manifest + .secondary_indexes + .iter_mut() + .find(|entry| entry.index_id == created.index_id) + .unwrap(); + entry.state = SecondaryIndexState::Failed; + entry.last_error = Some("boom".to_string()); + Ok(()) + }) + .unwrap(); + engine.rebuild_secondary_index_catalog().unwrap(); + + let retried = engine + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(retried.index_id, created.index_id); + assert_eq!(retried.state, SecondaryIndexState::Building); + assert!(retried.last_error.is_none()); + + engine.close().unwrap(); + } + + #[test] + fn test_ensure_node_property_index_seeds_active_and_immutable_memtables() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut frozen_props = BTreeMap::new(); + frozen_props.insert("status".to_string(), PropValue::String("active".to_string())); + frozen_props.insert("age".to_string(), PropValue::Int(30)); + let frozen_id = engine + .upsert_node( + "Person", + "frozen", + UpsertNodeOptions { + props: frozen_props, + ..Default::default() + }, + ) + .unwrap(); + engine.freeze_memtable().unwrap(); + + let mut active_props = BTreeMap::new(); + active_props.insert("status".to_string(), PropValue::String("active".to_string())); + active_props.insert("age".to_string(), PropValue::Int(35)); + let active_id = engine + .upsert_node( + "Person", + "active", + UpsertNodeOptions { + props: active_props, + ..Default::default() + }, + ) + .unwrap(); + + let mut bad_props = BTreeMap::new(); + bad_props.insert("status".to_string(), PropValue::String("active".to_string())); + bad_props.insert("age".to_string(), PropValue::String("old".to_string())); + let bad_id = engine + .upsert_node( + "Person", + "bad", + UpsertNodeOptions { + props: bad_props, + ..Default::default() + }, + ) + .unwrap(); + + let eq = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let range = engine + .ensure_node_property_index("Person", + "age", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + + let status_hash = hash_prop_value(&PropValue::String("active".to_string())); + let active_memtable = engine.active_memtable(); + let active_eq_state = active_memtable.secondary_eq_state(); + let active_eq_ids = active_eq_state + .get(&eq.index_id) + .unwrap() + .get(&status_hash) + .unwrap(); + assert!(active_eq_ids.contains(&active_id)); + assert!(active_eq_ids.contains(&bad_id)); + + let frozen_memtable = engine.immutable_memtable(0); + let frozen_eq_state = frozen_memtable.secondary_eq_state(); + let frozen_eq_ids = frozen_eq_state + .get(&eq.index_id) + .unwrap() + .get(&status_hash) + .unwrap(); + assert!(frozen_eq_ids.contains(&frozen_id)); + + let active_memtable = engine.active_memtable(); + let active_range_state = active_memtable.secondary_range_state(); + let active_range = active_range_state + .get(&range.index_id) + .unwrap(); + assert!(active_range.contains(&(35u64 ^ (1u64 << 63), active_id))); + assert!(!active_range.iter().any(|&(_, node_id)| node_id == bad_id)); + + let frozen_memtable = engine.immutable_memtable(0); + let frozen_range_state = frozen_memtable.secondary_range_state(); + let frozen_range = frozen_range_state + .get(&range.index_id) + .unwrap(); + assert!(frozen_range.contains(&(30u64 ^ (1u64 << 63), frozen_id))); + + engine.close().unwrap(); + } + + #[test] + fn test_secondary_index_seeding_refreshes_immutable_memtable_bytes_cache() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let mut props = BTreeMap::new(); + props.insert("status".to_string(), PropValue::String("active".to_string())); + props.insert("age".to_string(), PropValue::Int(30)); + engine + .upsert_node( + "Person", + "frozen", + UpsertNodeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); + engine.freeze_memtable().unwrap(); + + let before = engine.stats().unwrap().immutable_memtable_bytes; + let info = engine + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let after = engine.stats().unwrap().immutable_memtable_bytes; + let actual_after: usize = (0..engine.immutable_epoch_count()) + .map(|idx| engine.immutable_memtable(idx).estimated_size()) + .sum(); + assert_eq!(after, actual_after); + assert!(after >= before); + + engine + .drop_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + let after_drop = engine.stats().unwrap().immutable_memtable_bytes; + let actual_after_drop: usize = (0..engine.immutable_epoch_count()) + .map(|idx| engine.immutable_memtable(idx).estimated_size()) + .sum(); + assert_eq!(after_drop, actual_after_drop); + assert!(engine + .list_node_property_indexes().unwrap() + .iter() + .all(|entry| entry.index_id != info.index_id)); + + engine.close().unwrap(); + } + + // --- Edge Property Index Declaration Tests --- + + #[test] + fn test_edge_property_index_ensure_drop_list_and_conflicting_range_domains() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let eq = engine + .ensure_edge_property_index("RELATES_TO", "label", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(eq.state, SecondaryIndexState::Building); + + let eq_again = engine + .ensure_edge_property_index("RELATES_TO", "label", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(eq_again.index_id, eq.index_id); + + let range = engine + .ensure_edge_property_index("RELATES_TO", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + assert_eq!(range.state, SecondaryIndexState::Building); + + let indexes = engine.list_edge_property_indexes().unwrap(); + assert_eq!(indexes.len(), 2); + assert_eq!(indexes[0].prop_key, "label"); + assert_eq!(indexes[1].prop_key, "score"); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let err = engine + .ensure_edge_property_index("RELATES_TO", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Float, + }, + ) + .unwrap_err(); + assert!(matches!(err, EngineError::InvalidOperation(_))); - engine - .upsert_edge(a, c, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine - .upsert_edge(b, c, 10, UpsertEdgeOptions::default()) - .unwrap(); + assert!(engine + .drop_edge_property_index("RELATES_TO", "label", SecondaryIndexKind::Equality) + .unwrap()); + assert!(!engine + .drop_edge_property_index("RELATES_TO", "label", SecondaryIndexKind::Equality) + .unwrap()); - let inc = engine - .neighbors(c, &NeighborOptions { direction: Direction::Incoming, ..Default::default() }) - .unwrap(); - assert_eq!(inc.len(), 2); - let neighbor_ids: Vec = inc.iter().map(|e| e.node_id).collect(); - assert!(neighbor_ids.contains(&a)); - assert!(neighbor_ids.contains(&b)); + let indexes = engine.list_edge_property_indexes().unwrap(); + assert_eq!(indexes.len(), 1); + assert_eq!(indexes[0].index_id, range.index_id); engine.close().unwrap(); } #[test] - fn test_neighbors_with_type_filter() { + fn test_edge_property_index_retry_failed_clears_error_and_preserves_id() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); + let created = engine + .ensure_edge_property_index("RELATES_TO", "label", SecondaryIndexKind::Equality) + .unwrap(); + engine.shutdown_secondary_index_worker(); engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) - .unwrap(); // type 10 - engine - .upsert_edge(a, c, 20, UpsertEdgeOptions::default()) - .unwrap(); // type 20 + .with_runtime_manifest_write(|manifest| { + let entry = manifest + .secondary_indexes + .iter_mut() + .find(|entry| entry.index_id == created.index_id) + .unwrap(); + entry.state = SecondaryIndexState::Failed; + entry.last_error = Some("boom".to_string()); + Ok(()) + }) + .unwrap(); + engine.rebuild_secondary_index_catalog().unwrap(); - let typed = engine - .neighbors(a, &NeighborOptions { type_filter: Some(vec![10]), ..Default::default() }) + let retried = engine + .ensure_edge_property_index("RELATES_TO", "label", SecondaryIndexKind::Equality) .unwrap(); - assert_eq!(typed.len(), 1); - assert_eq!(typed[0].node_id, b); + assert_eq!(retried.index_id, created.index_id); + assert_eq!(retried.state, SecondaryIndexState::Building); + assert!(retried.last_error.is_none()); engine.close().unwrap(); } #[test] - fn test_neighbors_with_limit() { + fn test_ensure_edge_property_index_seeds_active_and_immutable_memtables() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let hub = engine.upsert_node(1, "hub", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - for i in 0..10 { - let n = engine - .upsert_node(1, &format!("spoke:{}", i), UpsertNodeOptions { weight: 0.5, ..Default::default() }) - .unwrap(); - engine - .upsert_edge(hub, n, 10, UpsertEdgeOptions::default()) - .unwrap(); - } - - let limited = engine - .neighbors(hub, &NeighborOptions { limit: Some(3), ..Default::default() }) + let node_a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let node_b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - assert_eq!(limited.len(), 3); - engine.close().unwrap(); - } + let mut frozen_props = BTreeMap::new(); + frozen_props.insert("status".to_string(), PropValue::String("active".to_string())); + frozen_props.insert("score".to_string(), PropValue::Int(30)); + let frozen_edge_id = engine + .upsert_edge( + node_a, + node_b, + "RELATES_TO", + UpsertEdgeOptions { + props: frozen_props, + ..Default::default() + }, + ) + .unwrap(); + engine.freeze_memtable().unwrap(); - #[test] - fn test_delete_node_via_api() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); + let node_c = engine + .upsert_node("Person", "c", UpsertNodeOptions::default()) + .unwrap(); + let mut active_props = BTreeMap::new(); + active_props.insert("status".to_string(), PropValue::String("active".to_string())); + active_props.insert("score".to_string(), PropValue::Int(50)); + let active_edge_id = engine + .upsert_edge( + node_a, + node_c, + "RELATES_TO", + UpsertEdgeOptions { + props: active_props, + ..Default::default() + }, + ) + .unwrap(); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let eq = engine + .ensure_edge_property_index("RELATES_TO", "status", SecondaryIndexKind::Equality) + .unwrap(); + let range = engine + .ensure_edge_property_index("RELATES_TO", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let status_hash = hash_prop_value(&PropValue::String("active".to_string())); + let active_memtable = engine.active_memtable(); + let active_eq_state = active_memtable.secondary_eq_state(); + let active_eq_ids = active_eq_state + .get(&eq.index_id) + .unwrap() + .get(&status_hash) .unwrap(); + assert!(active_eq_ids.contains(&active_edge_id)); - engine.delete_node(b).unwrap(); + let frozen_memtable = engine.immutable_memtable(0); + let frozen_eq_state = frozen_memtable.secondary_eq_state(); + let frozen_eq_ids = frozen_eq_state + .get(&eq.index_id) + .unwrap() + .get(&status_hash) + .unwrap(); + assert!(frozen_eq_ids.contains(&frozen_edge_id)); - assert!(engine.get_node(b).unwrap().is_none()); - assert_eq!(engine.node_count().unwrap(), 1); + let active_range_state = active_memtable.secondary_range_state(); + let active_range = active_range_state.get(&range.index_id).unwrap(); + assert!(active_range.contains(&(50u64 ^ (1u64 << 63), active_edge_id))); - // b excluded from a's neighbors (node tombstone filtering) - let out = engine - .neighbors(a, &NeighborOptions::default()) - .unwrap(); - assert!(out.is_empty()); + let frozen_range_state = frozen_memtable.secondary_range_state(); + let frozen_range = frozen_range_state.get(&range.index_id).unwrap(); + assert!(frozen_range.contains(&(30u64 ^ (1u64 << 63), frozen_edge_id))); engine.close().unwrap(); } #[test] - fn test_delete_edge_via_api() { + fn test_edge_property_index_foreground_maintenance() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open( + &db_path, + &DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }, + ) + .unwrap(); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let eq = engine + .ensure_edge_property_index("RELATES_TO", "color", SecondaryIndexKind::Equality) + .unwrap(); + let range = engine + .ensure_edge_property_index("RELATES_TO", + "weight", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); - let a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - let eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + let node_a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let node_b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - engine.delete_edge(eid).unwrap(); + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + props.insert("weight".to_string(), PropValue::Int(10)); + let edge_id = engine + .upsert_edge( + node_a, + node_b, + "RELATES_TO", + UpsertEdgeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); - assert!(engine.get_edge(eid).unwrap().is_none()); - assert_eq!(engine.edge_count().unwrap(), 0); - assert!(engine - .neighbors(a, &NeighborOptions::default()) + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let mem = engine.active_memtable(); + let eq_state = mem.secondary_eq_state(); + assert!(eq_state + .get(&eq.index_id) .unwrap() - .is_empty()); + .get(&red_hash) + .unwrap() + .contains(&edge_id)); + let range_state = mem.secondary_range_state(); + assert!(range_state + .get(&range.index_id) + .unwrap() + .contains(&(10u64 ^ (1u64 << 63), edge_id))); - engine.close().unwrap(); - } + let mut updated_props = BTreeMap::new(); + updated_props.insert("color".to_string(), PropValue::String("blue".to_string())); + updated_props.insert("weight".to_string(), PropValue::Int(20)); + engine + .upsert_edge( + node_a, + node_b, + "RELATES_TO", + UpsertEdgeOptions { + props: updated_props, + ..Default::default() + }, + ) + .unwrap(); - #[test] - fn test_delete_survives_restart() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); + let blue_hash = hash_prop_value(&PropValue::String("blue".to_string())); + let mem = engine.active_memtable(); + let eq_state = mem.secondary_eq_state(); + let red_ids = eq_state + .get(&eq.index_id) + .unwrap() + .get(&red_hash); + assert!(red_ids.is_none() || !red_ids.unwrap().contains(&edge_id)); + assert!(eq_state + .get(&eq.index_id) + .unwrap() + .get(&blue_hash) + .unwrap() + .contains(&edge_id)); + let range_state = mem.secondary_range_state(); + assert!(range_state + .get(&range.index_id) + .unwrap() + .contains(&(20u64 ^ (1u64 << 63), edge_id))); + assert!(!range_state + .get(&range.index_id) + .unwrap() + .contains(&(10u64 ^ (1u64 << 63), edge_id))); - let (a, b, eid); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - eid = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine.delete_node(b).unwrap(); - engine.delete_edge(eid).unwrap(); - engine.close().unwrap(); - } + engine.delete_edge(edge_id).unwrap(); + let mem = engine.active_memtable(); + let eq_state = mem.secondary_eq_state(); + let blue_gone = eq_state + .get(&eq.index_id) + .and_then(|groups| groups.get(&blue_hash)) + .is_none_or(|ids| !ids.contains(&edge_id)); + assert!(blue_gone); + let range_gone = mem + .secondary_range_state() + .get(&range.index_id) + .is_none_or(|entries| !entries.contains(&(20u64 ^ (1u64 << 63), edge_id))); + assert!(range_gone); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - assert!(engine.get_node(b).unwrap().is_none()); - assert!(engine.get_edge(eid).unwrap().is_none()); - // close() flushes to segments; use cross-source counts - assert_eq!(engine.get_nodes_by_type(1).unwrap().len(), 1); - // Verify deleted edge not visible - assert!(engine - .neighbors(a, &NeighborOptions::default()) - .unwrap() - .is_empty()); - engine.close().unwrap(); - } + engine.close().unwrap(); } #[test] - fn test_neighbors_survive_restart() { + fn test_node_and_edge_property_indexes_coexist() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open( + &db_path, + &DbOptions { + edge_uniqueness: true, + ..DbOptions::default() + }, + ) + .unwrap(); - let (a, b, c); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - a = engine.upsert_node(1, "a", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - b = engine.upsert_node(1, "b", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - c = engine.upsert_node(1, "c", UpsertNodeOptions { weight: 0.5, ..Default::default() }).unwrap(); - engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) - .unwrap(); - engine - .upsert_edge(a, c, 20, UpsertEdgeOptions { weight: 0.8, ..Default::default() }) - .unwrap(); - engine - .upsert_edge(b, c, 10, UpsertEdgeOptions { weight: 0.5, ..Default::default() }) - .unwrap(); - engine.close().unwrap(); - } + let node_eq = engine + .ensure_node_property_index("Person", "score", SecondaryIndexKind::Equality) + .unwrap(); + let edge_eq = engine + .ensure_edge_property_index("RELATES_TO", "score", SecondaryIndexKind::Equality) + .unwrap(); + assert_ne!(node_eq.index_id, edge_eq.index_id); - { - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // a → b, c - let out_a = engine - .neighbors(a, &NeighborOptions::default()) - .unwrap(); - assert_eq!(out_a.len(), 2); - // b → c - let out_b = engine - .neighbors(b, &NeighborOptions::default()) - .unwrap(); - assert_eq!(out_b.len(), 1); - assert_eq!(out_b[0].node_id, c); - // c ← a, b - let inc_c = engine - .neighbors(c, &NeighborOptions { direction: Direction::Incoming, ..Default::default() }) - .unwrap(); - assert_eq!(inc_c.len(), 2); - engine.close().unwrap(); - } - } + let node_indexes = engine.list_node_property_indexes().unwrap(); + assert_eq!(node_indexes.len(), 1); + assert_eq!(node_indexes[0].index_id, node_eq.index_id); - #[test] - fn test_node_property_index_ensure_drop_list_and_conflicting_range_domains() { - let dir = TempDir::new().unwrap(); - let db_path = dir.path().join("testdb"); - let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let edge_indexes = engine.list_edge_property_indexes().unwrap(); + assert_eq!(edge_indexes.len(), 1); + assert_eq!(edge_indexes[0].index_id, edge_eq.index_id); - let eq = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + let node_a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - assert_eq!(eq.state, SecondaryIndexState::Building); - - let eq_again = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + let node_b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - assert_eq!(eq_again.index_id, eq.index_id); - - let range = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Int, + let mut props = BTreeMap::new(); + props.insert("score".to_string(), PropValue::Int(42)); + let node_id = engine + .upsert_node( + "Person", + "x", + UpsertNodeOptions { + props: props.clone(), + ..Default::default() + }, + ) + .unwrap(); + let edge_id = engine + .upsert_edge( + node_a, + node_b, + "RELATES_TO", + UpsertEdgeOptions { + props, + ..Default::default() }, ) .unwrap(); - assert_eq!(range.state, SecondaryIndexState::Building); - let indexes = engine.list_node_property_indexes().unwrap(); - assert_eq!(indexes.len(), 2); - assert_eq!(indexes[0].prop_key, "color"); - assert_eq!(indexes[1].prop_key, "score"); + let score_hash = hash_prop_value(&PropValue::Int(42)); + let mem = engine.active_memtable(); + let eq_state = mem.secondary_eq_state(); - let err = engine - .ensure_node_property_index( - 1, - "score", - SecondaryIndexKind::Range { - domain: SecondaryIndexRangeDomain::Float, - }, - ) - .unwrap_err(); - assert!(matches!(err, EngineError::InvalidOperation(_))); + let node_ids = eq_state + .get(&node_eq.index_id) + .unwrap() + .get(&score_hash) + .unwrap(); + assert!(node_ids.contains(&node_id)); + assert!(!node_ids.contains(&edge_id)); + + let edge_ids = eq_state + .get(&edge_eq.index_id) + .unwrap() + .get(&score_hash) + .unwrap(); + assert!(edge_ids.contains(&edge_id)); + assert!(!edge_ids.contains(&node_id)); assert!(engine - .drop_node_property_index(1, "color", SecondaryIndexKind::Equality) - .unwrap()); - assert!(!engine - .drop_node_property_index(1, "color", SecondaryIndexKind::Equality) + .drop_edge_property_index("RELATES_TO", "score", SecondaryIndexKind::Equality) .unwrap()); - - let indexes = engine.list_node_property_indexes().unwrap(); - assert_eq!(indexes.len(), 1); - assert_eq!(indexes[0].index_id, range.index_id); + let node_indexes = engine.list_node_property_indexes().unwrap(); + assert_eq!(node_indexes.len(), 1); engine.close().unwrap(); } #[test] - fn test_node_property_index_retry_failed_clears_error_and_preserves_id() { + fn test_edge_property_index_shared_id_sequence() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let created = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) + let node_idx = engine + .ensure_node_property_index("Person", "x", SecondaryIndexKind::Equality) .unwrap(); - engine.shutdown_secondary_index_worker(); - - engine - .with_runtime_manifest_write(|manifest| { - let entry = manifest - .secondary_indexes - .iter_mut() - .find(|entry| entry.index_id == created.index_id) - .unwrap(); - entry.state = SecondaryIndexState::Failed; - entry.last_error = Some("boom".to_string()); - Ok(()) - }) + let edge_idx = engine + .ensure_edge_property_index("RELATES_TO", "x", SecondaryIndexKind::Equality) .unwrap(); - engine.rebuild_secondary_index_catalog().unwrap(); + assert_eq!(edge_idx.index_id, node_idx.index_id + 1); - let retried = engine - .ensure_node_property_index(1, "color", SecondaryIndexKind::Equality) - .unwrap(); - assert_eq!(retried.index_id, created.index_id); - assert_eq!(retried.state, SecondaryIndexState::Building); - assert!(retried.last_error.is_none()); + engine.close().unwrap(); + } + + #[test] + fn test_edge_property_index_persists_across_reopen() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let (color_index_id, weight_index_id) = { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let color_info = engine + .ensure_edge_property_index("RELATES_TO", "color", SecondaryIndexKind::Equality) + .unwrap(); + let weight_info = engine + .ensure_edge_property_index("RELATES_TO", + "weight", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Float, + }, + ) + .unwrap(); + let index_ids = (color_info.index_id, weight_info.index_id); + engine.close().unwrap(); + index_ids + }; + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let color = wait_for_edge_property_index_state( + &engine, + color_index_id, + SecondaryIndexState::Ready, + ); + let weight = wait_for_edge_property_index_state( + &engine, + weight_index_id, + SecondaryIndexState::Ready, + ); + let indexes = engine.list_edge_property_indexes().unwrap(); + assert_eq!(indexes.len(), 2); + assert_eq!(color.prop_key, "color"); + assert!(matches!(color.kind, SecondaryIndexKind::Equality)); + assert_eq!(color.state, SecondaryIndexState::Ready); + assert_eq!(weight.prop_key, "weight"); + assert!(matches!( + weight.kind, + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Float + } + )); + assert_eq!(weight.state, SecondaryIndexState::Ready); engine.close().unwrap(); } #[test] - fn test_ensure_node_property_index_seeds_active_and_immutable_memtables() { + fn test_edge_property_background_build_writes_sidecar_and_publishes_ready() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut frozen_props = BTreeMap::new(); - frozen_props.insert("status".to_string(), PropValue::String("active".to_string())); - frozen_props.insert("age".to_string(), PropValue::Int(30)); - let frozen_id = engine - .upsert_node( - 1, - "frozen", - UpsertNodeOptions { - props: frozen_props, - ..Default::default() - }, - ) + let node_a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - engine.freeze_memtable().unwrap(); - - let mut active_props = BTreeMap::new(); - active_props.insert("status".to_string(), PropValue::String("active".to_string())); - active_props.insert("age".to_string(), PropValue::Int(35)); - let active_id = engine - .upsert_node( - 1, - "active", - UpsertNodeOptions { - props: active_props, + let node_b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + engine + .upsert_edge( + node_a, + node_b, + "RELATES_TO", + UpsertEdgeOptions { + props, ..Default::default() }, ) .unwrap(); + let segment_info = engine.flush().unwrap().expect("segment should flush"); - let mut bad_props = BTreeMap::new(); - bad_props.insert("status".to_string(), PropValue::String("active".to_string())); - bad_props.insert("age".to_string(), PropValue::String("old".to_string())); - let bad_id = engine - .upsert_node( - 1, - "bad", - UpsertNodeOptions { - props: bad_props, - ..Default::default() - }, - ) + let info = engine + .ensure_edge_property_index("RELATES_TO", "color", SecondaryIndexKind::Equality) .unwrap(); + assert_eq!(info.state, SecondaryIndexState::Building); + + let seg_dir = db_path + .join("segments") + .join(format!("seg_{:04}", segment_info.id)); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + if let Ok(bytes) = std::fs::read( + seg_dir.join(crate::segment_components::SEGMENT_COMPONENT_MANIFEST_FILENAME), + ) { + let manifest = + crate::segment_components::decode_manifest_envelope(&bytes).unwrap(); + let has_edge_sidecar = manifest.components.iter().any(|record| { + record.kind + == crate::segment_components::SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: info.index_id, + } + }); + if has_edge_sidecar { + break; + } + } + assert!( + std::time::Instant::now() < deadline, + "edge sidecar component record was not background-built" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } - let eq = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let sidecar_dir = seg_dir.join("secondary_indexes"); + let sidecar_exists = std::fs::read_dir(&sidecar_dir) + .unwrap() + .flatten() + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("edge_prop_eq_")) + }); + assert!(sidecar_exists); + + wait_for_edge_property_index_state(&engine, info.index_id, SecondaryIndexState::Ready); + + engine.close().unwrap(); + } + + #[test] + fn test_node_property_background_build_writes_targeted_planner_stats() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + for (key, color, score) in [("a", "red", 10), ("b", "red", 20), ("c", "blue", 30)] { + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String(color.to_string())); + props.insert("score".to_string(), PropValue::Int(score)); + engine + .upsert_node( + "Person", + key, + UpsertNodeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); + } + engine.flush().unwrap().expect("segment should flush"); + assert!(engine.segments_for_test()[0] + .planner_stats() + .unwrap() + .equality_index_stats + .is_empty()); + + let color = engine + .ensure_node_property_index("Person", "color", SecondaryIndexKind::Equality) .unwrap(); - let range = engine - .ensure_node_property_index( - 1, - "age", + let score = engine + .ensure_node_property_index("Person", + "score", SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, }, ) .unwrap(); + assert_eq!(color.state, SecondaryIndexState::Building); + assert_eq!(score.state, SecondaryIndexState::Building); + wait_for_property_index_state( + &engine, + color.index_id, + SecondaryIndexState::Ready, + ); + wait_for_property_index_state( + &engine, + score.index_id, + SecondaryIndexState::Ready, + ); + wait_for_published_property_index_state( + &engine, + color.index_id, + SecondaryIndexState::Ready, + ); + wait_for_published_property_index_state( + &engine, + score.index_id, + SecondaryIndexState::Ready, + ); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let reader = engine.segments_for_test()[0].clone(); + let stats = reader.planner_stats().expect("targeted stats should load"); + let equality = stats + .equality_index_stats + .iter() + .find(|stats| stats.index_id == color.index_id); + let range = stats + .range_index_stats + .iter() + .find(|stats| stats.index_id == score.index_id); + if let (Some(equality), Some(range)) = (equality, range) { + assert_eq!( + stats.build_kind, + crate::planner_stats::PlannerStatsBuildKind::SecondaryIndexRefresh + ); + assert_eq!(equality.total_postings, 3); + assert_eq!(equality.value_group_count, 2); + assert!(equality.sidecar_present_at_build); + assert_eq!(range.total_entries, 3); + assert!(range.sidecar_present_at_build); + break; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for node targeted planner stats; stats: {:?}", + stats + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } - let status_hash = hash_prop_value(&PropValue::String("active".to_string())); - let active_memtable = engine.active_memtable(); - let active_eq_state = active_memtable.secondary_eq_state(); - let active_eq_ids = active_eq_state - .get(&eq.index_id) - .unwrap() - .get(&status_hash) - .unwrap(); - assert!(active_eq_ids.contains(&active_id)); - assert!(active_eq_ids.contains(&bad_id)); + engine.close().unwrap(); + } - let frozen_memtable = engine.immutable_memtable(0); - let frozen_eq_state = frozen_memtable.secondary_eq_state(); - let frozen_eq_ids = frozen_eq_state - .get(&eq.index_id) - .unwrap() - .get(&status_hash) - .unwrap(); - assert!(frozen_eq_ids.contains(&frozen_id)); + #[test] + fn test_edge_property_background_build_writes_targeted_planner_stats() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let active_memtable = engine.active_memtable(); - let active_range_state = active_memtable.secondary_range_state(); - let active_range = active_range_state - .get(&range.index_id) + let node_a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); - assert!(active_range.contains(&(35u64 ^ (1u64 << 63), active_id))); - assert!(!active_range.iter().any(|&(_, node_id)| node_id == bad_id)); + let node_b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + for (color, score) in [("red", 10), ("red", 20), ("blue", 30)] { + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String(color.to_string())); + props.insert("score".to_string(), PropValue::Int(score)); + engine + .upsert_edge( + node_a, + node_b, + "RELATES_TO", + UpsertEdgeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); + } + engine.flush().unwrap().expect("segment should flush"); + assert!(engine.segments_for_test()[0] + .planner_stats() + .unwrap() + .equality_index_stats + .is_empty()); - let frozen_memtable = engine.immutable_memtable(0); - let frozen_range_state = frozen_memtable.secondary_range_state(); - let frozen_range = frozen_range_state - .get(&range.index_id) + let color = engine + .ensure_edge_property_index("RELATES_TO", "color", SecondaryIndexKind::Equality) .unwrap(); - assert!(frozen_range.contains(&(30u64 ^ (1u64 << 63), frozen_id))); + let score = engine + .ensure_edge_property_index("RELATES_TO", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + assert_eq!(color.state, SecondaryIndexState::Building); + assert_eq!(score.state, SecondaryIndexState::Building); + wait_for_edge_property_index_state( + &engine, + color.index_id, + SecondaryIndexState::Ready, + ); + wait_for_edge_property_index_state( + &engine, + score.index_id, + SecondaryIndexState::Ready, + ); + wait_for_published_property_index_state( + &engine, + color.index_id, + SecondaryIndexState::Ready, + ); + wait_for_published_property_index_state( + &engine, + score.index_id, + SecondaryIndexState::Ready, + ); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let reader = engine.segments_for_test()[0].clone(); + let stats = reader.planner_stats().expect("targeted stats should load"); + let equality = stats + .equality_index_stats + .iter() + .find(|stats| stats.index_id == color.index_id); + let range = stats + .range_index_stats + .iter() + .find(|stats| stats.index_id == score.index_id); + if let (Some(equality), Some(range)) = (equality, range) { + assert_eq!( + stats.build_kind, + crate::planner_stats::PlannerStatsBuildKind::SecondaryIndexRefresh + ); + assert_eq!(equality.total_postings, 3); + assert_eq!(equality.value_group_count, 2); + assert!(equality.sidecar_present_at_build); + assert_eq!(range.total_entries, 3); + assert!(range.sidecar_present_at_build); + break; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for edge targeted planner stats; stats: {:?}", + stats + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } engine.close().unwrap(); } #[test] - fn test_secondary_index_seeding_refreshes_immutable_memtable_bytes_cache() { + fn test_node_edge_range_domain_conflict_scoped_to_target() { let dir = TempDir::new().unwrap(); let db_path = dir.path().join("testdb"); let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - let mut props = BTreeMap::new(); - props.insert("status".to_string(), PropValue::String("active".to_string())); - props.insert("age".to_string(), PropValue::Int(30)); engine - .upsert_node( - 1, - "frozen", - UpsertNodeOptions { - props, - ..Default::default() + .ensure_node_property_index("Person", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, }, ) .unwrap(); - engine.freeze_memtable().unwrap(); - let before = engine.stats().unwrap().immutable_memtable_bytes; - let info = engine - .ensure_node_property_index(1, "status", SecondaryIndexKind::Equality) + let edge_range = engine + .ensure_edge_property_index("RELATES_TO", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Float, + }, + ) .unwrap(); - let after = engine.stats().unwrap().immutable_memtable_bytes; - let actual_after: usize = (0..engine.immutable_epoch_count()) - .map(|idx| engine.immutable_memtable(idx).estimated_size()) - .sum(); - assert_eq!(after, actual_after); - assert!(after >= before); + assert_eq!(edge_range.state, SecondaryIndexState::Building); - engine - .drop_node_property_index(1, "status", SecondaryIndexKind::Equality) - .unwrap(); - let after_drop = engine.stats().unwrap().immutable_memtable_bytes; - let actual_after_drop: usize = (0..engine.immutable_epoch_count()) - .map(|idx| engine.immutable_memtable(idx).estimated_size()) - .sum(); - assert_eq!(after_drop, actual_after_drop); - assert!(engine - .list_node_property_indexes().unwrap() - .iter() - .all(|entry| entry.index_id != info.index_id)); + engine.close().unwrap(); + } + + #[test] + fn test_edge_property_index_wal_recovery_rebuilds_memtable_state() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + + let eq_index_id; + let range_index_id; + let edge_id; + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let eq = engine + .ensure_edge_property_index("RELATES_TO", "color", SecondaryIndexKind::Equality) + .unwrap(); + let range = engine + .ensure_edge_property_index("RELATES_TO", + "weight", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + eq_index_id = eq.index_id; + range_index_id = range.index_id; + + let node_a = engine + .upsert_node("Person", "a", UpsertNodeOptions::default()) + .unwrap(); + let node_b = engine + .upsert_node("Person", "b", UpsertNodeOptions::default()) + .unwrap(); + + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + props.insert("weight".to_string(), PropValue::Int(42)); + edge_id = engine + .upsert_edge( + node_a, + node_b, + "RELATES_TO", + UpsertEdgeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); + + engine.sync().unwrap(); + // close_fast skips the memtable flush so edges remain in the WAL + // and will be replayed into the memtable on reopen. + engine.close_fast().unwrap(); + } + + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let indexes = engine.list_edge_property_indexes().unwrap(); + assert_eq!(indexes.len(), 2); + + let mem = engine.active_memtable(); + let eq_state = mem.secondary_eq_state(); + let color_hash = hash_prop_value(&PropValue::String("red".to_string())); + let eq_ids = eq_state + .get(&eq_index_id) + .expect("eq index should exist after WAL recovery") + .get(&color_hash) + .expect("color hash group should exist after WAL recovery"); + assert!(eq_ids.contains(&edge_id)); + + let range_state = mem.secondary_range_state(); + let range_entries = range_state + .get(&range_index_id) + .expect("range index should exist after WAL recovery"); + assert!(range_entries.contains(&(42u64 ^ (1u64 << 63), edge_id))); engine.close().unwrap(); } diff --git a/src/engine/txn.rs b/src/engine/txn.rs index 5ff35fa..34ccde1 100644 --- a/src/engine/txn.rs +++ b/src/engine/txn.rs @@ -15,13 +15,15 @@ pub(crate) struct StagedTxnIntent { #[derive(Debug, Clone, PartialEq, Eq, Hash)] enum TxnEndpointKey { Id(u64), - Key(u32, String), + Local(TxnLocalRef), + Key(String, String), } #[derive(Debug, Clone)] enum NodeOverlayOpinion { Live(TxnNodeView), Deleted(Option), + RemovedLabel, } #[derive(Debug, Clone)] @@ -36,14 +38,14 @@ struct TxnOverlay { node_aliases: HashSet, edge_aliases: HashSet, nodes_by_local: HashMap, - node_key_locals: HashMap<(u32, String), Vec>, + node_key_locals: HashMap<(String, String), Vec>, nodes_by_id: NodeIdMap, - nodes_by_key: HashMap<(u32, String), NodeOverlayOpinion>, + nodes_by_key: HashMap<(String, String), NodeOverlayOpinion>, deleted_node_ids_seen: NodeIdSet, edges_by_local: HashMap, - edge_triple_locals: HashMap<(TxnEndpointKey, TxnEndpointKey, u32), Vec>, + edge_triple_locals: HashMap<(TxnEndpointKey, TxnEndpointKey, String), Vec>, edges_by_id: NodeIdMap, - edges_by_triple: HashMap<(TxnEndpointKey, TxnEndpointKey, u32), EdgeOverlayOpinion>, + edges_by_triple: HashMap<(TxnEndpointKey, TxnEndpointKey, String), EdgeOverlayOpinion>, } /// Explicit write transaction handle. @@ -78,16 +80,19 @@ impl DatabaseEngine { } impl WriteTxn { - pub fn upsert_node( + pub fn upsert_node( &mut self, - type_id: u32, + labels: L, key: &str, options: UpsertNodeOptions, - ) -> Result { + ) -> Result + where + L: IntoNodeLabels, + { let local = self.next_slot_ref()?; let intent = TxnIntent::UpsertNode { alias: None, - type_id, + labels: labels.into_node_labels(), key: key.to_string(), options, }; @@ -96,17 +101,20 @@ impl WriteTxn { Ok(TxnNodeRef::Local(local)) } - pub fn upsert_node_as( + pub fn upsert_node_as( &mut self, alias: &str, - type_id: u32, + labels: L, key: &str, options: UpsertNodeOptions, - ) -> Result { + ) -> Result + where + L: IntoNodeLabels, + { let local = TxnLocalRef::Alias(alias.to_string()); let intent = TxnIntent::UpsertNode { alias: Some(alias.to_string()), - type_id, + labels: labels.into_node_labels(), key: key.to_string(), options, }; @@ -114,11 +122,80 @@ impl WriteTxn { Ok(TxnNodeRef::Local(local)) } + pub fn add_node_label(&mut self, target: TxnNodeRef, label: &str) -> Result { + self.ensure_open()?; + validate_label_token_name(label)?; + let Some(view) = self.get_node(target)? else { + return Err(EngineError::InvalidOperation( + "transaction node target does not exist".to_string(), + )); + }; + if view.labels.iter().any(|existing| existing == label) { + return Ok(false); + } + let mut labels = view.labels.clone(); + labels.push(label.to_string()); + let intent = TxnIntent::UpsertNode { + alias: None, + labels, + key: view.key.clone(), + options: UpsertNodeOptions { + props: view.props, + weight: view.weight, + dense_vector: view.dense_vector, + sparse_vector: view.sparse_vector, + }, + }; + self.append_entry(intent, view.local.clone(), None)?; + Ok(true) + } + + pub fn remove_node_label( + &mut self, + target: TxnNodeRef, + label: &str, + ) -> Result { + self.ensure_open()?; + validate_label_token_name(label)?; + let Some(view) = self.get_node(target)? else { + return Err(EngineError::InvalidOperation( + "transaction node target does not exist".to_string(), + )); + }; + if !view.labels.iter().any(|existing| existing == label) { + return Ok(false); + } + if view.labels.len() == 1 { + return Err(EngineError::InvalidOperation( + "cannot remove the last node label".to_string(), + )); + } + let labels = view + .labels + .iter() + .filter(|existing| existing.as_str() != label) + .cloned() + .collect(); + let intent = TxnIntent::UpsertNode { + alias: None, + labels, + key: view.key.clone(), + options: UpsertNodeOptions { + props: view.props, + weight: view.weight, + dense_vector: view.dense_vector, + sparse_vector: view.sparse_vector, + }, + }; + self.append_entry(intent, view.local.clone(), None)?; + Ok(true) + } + pub fn upsert_edge( &mut self, from: TxnNodeRef, to: TxnNodeRef, - type_id: u32, + label: &str, options: UpsertEdgeOptions, ) -> Result { let local = self.next_slot_ref()?; @@ -126,7 +203,7 @@ impl WriteTxn { alias: None, from, to, - type_id, + label: label.to_string(), options, }; self.append_entry(intent, None, Some(local.clone()))?; @@ -139,7 +216,7 @@ impl WriteTxn { alias: &str, from: TxnNodeRef, to: TxnNodeRef, - type_id: u32, + label: &str, options: UpsertEdgeOptions, ) -> Result { let local = TxnLocalRef::Alias(alias.to_string()); @@ -147,7 +224,7 @@ impl WriteTxn { alias: Some(alias.to_string()), from, to, - type_id, + label: label.to_string(), options, }; self.append_entry(intent, None, Some(local.clone()))?; @@ -243,13 +320,13 @@ impl WriteTxn { pub fn get_node_by_key( &self, - type_id: u32, + label: &str, key: &str, ) -> Result, EngineError> { self.ensure_open()?; self.overlay .get_node(&self.snapshot, &TxnNodeRef::Key { - type_id, + label: label.to_string(), key: key.to_string(), }) } @@ -258,12 +335,16 @@ impl WriteTxn { &self, from: TxnNodeRef, to: TxnNodeRef, - type_id: u32, + label: &str, ) -> Result, EngineError> { self.ensure_open()?; self.overlay.get_edge( &self.snapshot, - &TxnEdgeRef::Triple { from, to, type_id }, + &TxnEdgeRef::Triple { + from, + to, + label: label.to_string(), + }, ) } @@ -371,22 +452,22 @@ impl TxnOverlay { match intent { TxnIntent::UpsertNode { alias, - type_id, + labels, key, options, - } => self.apply_upsert_node(snapshot, alias, *type_id, key, options, produced_node), + } => self.apply_upsert_node(snapshot, alias, labels, key, options, produced_node), TxnIntent::UpsertEdge { alias, from, to, - type_id, + label, options, } => self.apply_upsert_edge( snapshot, alias, from, to, - *type_id, + label, options, produced_edge, ), @@ -402,11 +483,13 @@ impl TxnOverlay { &mut self, snapshot: &ReadView, alias: &Option, - type_id: u32, + labels: &[String], key: &str, options: &UpsertNodeOptions, produced: Option, ) -> Result<(), EngineError> { + let validated_labels = ValidatedNodeLabelList::new(labels.iter().map(String::as_str))?; + validate_node_key_for_write(key)?; if let Some(alias) = alias { if self.node_aliases.contains(alias) { return Err(EngineError::InvalidOperation(format!( @@ -416,15 +499,71 @@ impl TxnOverlay { } } - let existing = match self.nodes_by_key.get(&(type_id, key.to_string())) { - Some(NodeOverlayOpinion::Live(view)) => Some(view.clone()), - Some(NodeOverlayOpinion::Deleted(view)) => view.clone(), - None => snapshot.get_node_by_key(type_id, key)?.map(node_to_txn_view), - }; + let mut existing: Option = None; + for &label in validated_labels.as_slice() { + let node_key = (label.to_string(), key.to_string()); + let candidate = match self.nodes_by_key.get(&node_key) { + Some(NodeOverlayOpinion::Live(view)) => Some(view.clone()), + Some(NodeOverlayOpinion::Deleted(view)) => view.clone(), + Some(NodeOverlayOpinion::RemovedLabel) => None, + None => match snapshot.label_catalog.resolve_node_label_for_read(label)? { + Some(label_id) => snapshot + .get_node_by_label_key(label_id, key)? + .map(|node| { + node_to_txn_view_with_resolved_label( + node, + label_id, + label.to_string(), + snapshot.label_catalog.as_ref(), + ) + }) + .transpose()?, + None => None, + }, + }; + let Some(candidate) = candidate else { + continue; + }; + match existing.as_ref() { + Some(winner) if !txn_node_views_match(winner, &candidate) => { + return Err(EngineError::InvalidOperation(format!( + "node key conflict for key '{}': requested label memberships resolve to different transaction nodes", + key + ))); + } + None => existing = Some(candidate), + _ => {} + } + } + + let labels: Vec = validated_labels + .as_slice() + .iter() + .map(|label| (*label).to_string()) + .collect(); + let related_locals = existing + .as_ref() + .map(|view| self.node_locals_for_view(view)) + .unwrap_or_default(); + if let Some(existing_view) = existing.as_ref() { + let removed_labels: Vec = existing_view + .labels + .iter() + .filter(|label| !labels.iter().any(|new_label| new_label == *label)) + .cloned() + .collect(); + for old_label in removed_labels { + let key = (old_label, key.to_string()); + self.nodes_by_key + .insert(key.clone(), NodeOverlayOpinion::RemovedLabel); + self.node_key_locals.remove(&key); + } + } + let view = TxnNodeView { id: existing.as_ref().and_then(|node| node.id), local: produced.clone(), - type_id, + labels: labels.clone(), key: key.to_string(), props: options.props.clone(), created_at: existing.and_then(|node| node.created_at), @@ -436,7 +575,7 @@ impl TxnOverlay { if let Some(alias) = alias { self.node_aliases.insert(alias.clone()); } - self.insert_node_live(view, produced); + self.insert_node_live_with_locals(view, produced, related_locals); Ok(()) } @@ -447,10 +586,11 @@ impl TxnOverlay { alias: &Option, from: &TxnNodeRef, to: &TxnNodeRef, - type_id: u32, + label: &str, options: &UpsertEdgeOptions, produced: Option, ) -> Result<(), EngineError> { + validate_label_token_name(label)?; if let Some(alias) = alias { if self.edge_aliases.contains(alias) { return Err(EngineError::InvalidOperation(format!( @@ -462,7 +602,9 @@ impl TxnOverlay { let from_key = self.endpoint_key(snapshot, from)?; let to_key = self.endpoint_key(snapshot, to)?; - let triple_key = (from_key, to_key, type_id); + let triple_key = (from_key, to_key, label.to_string()); + let from_ref = self.canonical_node_ref(snapshot, from)?; + let to_ref = self.canonical_node_ref(snapshot, to)?; let existing = if self.edge_uniqueness { match self.edges_by_triple.get(&triple_key) { Some(EdgeOverlayOpinion::Live(view)) => Some(view.clone()), @@ -471,9 +613,21 @@ impl TxnOverlay { let from_id = self.committed_node_id(snapshot, from)?; let to_id = self.committed_node_id(snapshot, to)?; match (from_id, to_id) { - (Some(from_id), Some(to_id)) => snapshot - .get_edge_by_triple(from_id, to_id, type_id)? - .map(edge_to_txn_view), + (Some(from_id), Some(to_id)) => { + match snapshot.label_catalog.resolve_edge_label_for_read(label)? { + Some(label_id) => snapshot + .get_edge_by_triple(from_id, to_id, label_id)? + .map(|edge| { + edge_to_txn_view_with_resolved_label( + edge, + label_id, + label.to_string(), + ) + }) + .transpose()?, + None => None, + } + } _ => None, } } @@ -486,9 +640,9 @@ impl TxnOverlay { let view = TxnEdgeView { id: existing.as_ref().and_then(|edge| edge.id), local: produced.clone(), - from: from.clone(), - to: to.clone(), - type_id, + from: from_ref, + to: to_ref, + label: label.to_string(), props: options.props.clone(), created_at, updated_at: None, @@ -548,6 +702,7 @@ impl TxnOverlay { TxnNodeRef::Local(local) => match self.nodes_by_local.get(local) { Some(NodeOverlayOpinion::Live(view)) => Ok(Some(view.clone())), Some(NodeOverlayOpinion::Deleted(_)) => Ok(None), + Some(NodeOverlayOpinion::RemovedLabel) => Ok(None), None => Err(EngineError::InvalidOperation(format!( "unknown transaction node local ref {:?}", local @@ -556,15 +711,38 @@ impl TxnOverlay { TxnNodeRef::Id(id) => match self.nodes_by_id.get(id) { Some(NodeOverlayOpinion::Live(view)) => Ok(Some(view.clone())), Some(NodeOverlayOpinion::Deleted(_)) => Ok(None), - None => Ok(snapshot.get_node(*id)?.map(node_to_txn_view)), + Some(NodeOverlayOpinion::RemovedLabel) => Ok(None), + None => { + snapshot + .get_node(*id)? + .map(|node| node_to_txn_view(node, snapshot.label_catalog.as_ref())) + .transpose() + } }, - TxnNodeRef::Key { type_id, key } => { - match self.nodes_by_key.get(&(*type_id, key.clone())) { + TxnNodeRef::Key { label, key } => { + validate_label_token_name(label)?; + match self.nodes_by_key.get(&(label.clone(), key.clone())) { Some(NodeOverlayOpinion::Live(view)) => Ok(Some(view.clone())), Some(NodeOverlayOpinion::Deleted(_)) => Ok(None), - None => Ok(snapshot - .get_node_by_key(*type_id, key)? - .map(node_to_txn_view)), + Some(NodeOverlayOpinion::RemovedLabel) => Ok(None), + None => { + let Some(label_id) = + snapshot.label_catalog.resolve_node_label_for_read(label)? + else { + return Ok(None); + }; + snapshot + .get_node_by_label_key(label_id, key)? + .map(|node| { + node_to_txn_view_with_resolved_label( + node, + label_id, + label.clone(), + snapshot.label_catalog.as_ref(), + ) + }) + .transpose() + } } } } @@ -589,14 +767,20 @@ impl TxnOverlay { } }, TxnEdgeRef::Id(id) => self.edges_by_id.get(id), - TxnEdgeRef::Triple { from, to, type_id } => { + TxnEdgeRef::Triple { + from, + to, + label, + } => { + validate_label_token_name(label)?; let Some(from_key) = self.read_endpoint_key(snapshot, from)? else { return Ok(None); }; let Some(to_key) = self.read_endpoint_key(snapshot, to)? else { return Ok(None); }; - self.edges_by_triple.get(&(from_key, to_key, *type_id)) + self.edges_by_triple + .get(&(from_key, to_key, label.clone())) } }; match overlay { @@ -614,19 +798,33 @@ impl TxnOverlay { let edge = match target { TxnEdgeRef::Local(_) => return Ok(None), TxnEdgeRef::Id(id) => snapshot.get_edge(*id)?, - TxnEdgeRef::Triple { from, to, type_id } => { + TxnEdgeRef::Triple { + from, + to, + label, + } => { + validate_label_token_name(label)?; let Some(from_id) = self.committed_node_id(snapshot, from)? else { return Ok(None); }; let Some(to_id) = self.committed_node_id(snapshot, to)? else { return Ok(None); }; - snapshot.get_edge_by_triple(from_id, to_id, *type_id)? + let Some(label_id) = snapshot + .label_catalog + .resolve_edge_label_for_read(label)? + else { + return Ok(None); + }; + snapshot.get_edge_by_triple(from_id, to_id, label_id)? } }; match edge { Some(edge) if !self.committed_edge_endpoint_deleted(&edge) => { - Ok(Some(edge_to_txn_view(edge))) + Ok(Some(edge_to_txn_view( + edge, + snapshot.label_catalog.as_ref(), + )?)) } _ => Ok(None), } @@ -680,20 +878,32 @@ impl TxnOverlay { Some(NodeOverlayOpinion::Deleted(_)) => Ok(None), _ => Ok(Some(TxnEndpointKey::Id(*id))), }, - TxnNodeRef::Key { type_id, key } => { - match self.nodes_by_key.get(&(*type_id, key.clone())) { + TxnNodeRef::Key { label, key } => { + validate_label_token_name(label)?; + match self.nodes_by_key.get(&(label.clone(), key.clone())) { Some(NodeOverlayOpinion::Live(view)) => { if let Some(id) = view.id { Ok(Some(TxnEndpointKey::Id(id))) } else { - Ok(Some(TxnEndpointKey::Key(*type_id, key.clone()))) + Ok(Some(self.uncommitted_endpoint_key(view, label, key))) } } Some(NodeOverlayOpinion::Deleted(_)) => Ok(None), - None => match snapshot.get_node_by_key(*type_id, key)? { - Some(node) => Ok(Some(TxnEndpointKey::Id(node.id))), - None => Ok(Some(TxnEndpointKey::Key(*type_id, key.clone()))), - }, + Some(NodeOverlayOpinion::RemovedLabel) => { + Ok(Some(TxnEndpointKey::Key(label.clone(), key.clone()))) + } + None => { + match snapshot.label_catalog.resolve_node_label_for_read(label)? { + Some(label_id) => match snapshot.get_node_by_label_key(label_id, key)? { + Some(node) => Ok(Some(TxnEndpointKey::Id(node.id))), + None => Ok(Some(TxnEndpointKey::Key( + label.clone(), + key.clone(), + ))), + }, + None => Ok(Some(TxnEndpointKey::Key(label.clone(), key.clone()))), + } + } } } TxnNodeRef::Local(local) => match self.nodes_by_local.get(local) { @@ -701,10 +911,15 @@ impl TxnOverlay { if let Some(id) = view.id { Ok(Some(TxnEndpointKey::Id(id))) } else { - Ok(Some(TxnEndpointKey::Key(view.type_id, view.key.clone()))) + Ok(Some(self.uncommitted_endpoint_key( + view, + txn_node_view_fallback_label(view)?, + &view.key, + ))) } } Some(NodeOverlayOpinion::Deleted(_)) => Ok(None), + Some(NodeOverlayOpinion::RemovedLabel) => Ok(None), None => Err(EngineError::InvalidOperation(format!( "unknown transaction node local ref {:?}", local @@ -726,22 +941,31 @@ impl TxnOverlay { ))), _ => Ok(TxnEndpointKey::Id(*id)), }, - TxnNodeRef::Key { type_id, key } => { - match self.nodes_by_key.get(&(*type_id, key.clone())) { + TxnNodeRef::Key { label, key } => { + validate_label_token_name(label)?; + match self.nodes_by_key.get(&(label.clone(), key.clone())) { Some(NodeOverlayOpinion::Live(view)) => { if let Some(id) = view.id { Ok(TxnEndpointKey::Id(id)) } else { - Ok(TxnEndpointKey::Key(*type_id, key.clone())) + Ok(self.uncommitted_endpoint_key(view, label, key)) } } Some(NodeOverlayOpinion::Deleted(_)) => Err(EngineError::InvalidOperation( - format!("transaction node key ({}, {}) is deleted", type_id, key), + format!("transaction node key ({}, {}) is deleted", label, key), )), - None => match snapshot.get_node_by_key(*type_id, key)? { - Some(node) => Ok(TxnEndpointKey::Id(node.id)), - None => Ok(TxnEndpointKey::Key(*type_id, key.clone())), - }, + Some(NodeOverlayOpinion::RemovedLabel) => { + Ok(TxnEndpointKey::Key(label.clone(), key.clone())) + } + None => { + match snapshot.label_catalog.resolve_node_label_for_read(label)? { + Some(label_id) => match snapshot.get_node_by_label_key(label_id, key)? { + Some(node) => Ok(TxnEndpointKey::Id(node.id)), + None => Ok(TxnEndpointKey::Key(label.clone(), key.clone())), + }, + None => Ok(TxnEndpointKey::Key(label.clone(), key.clone())), + } + } } } TxnNodeRef::Local(local) => match self.nodes_by_local.get(local) { @@ -749,13 +973,20 @@ impl TxnOverlay { if let Some(id) = view.id { Ok(TxnEndpointKey::Id(id)) } else { - Ok(TxnEndpointKey::Key(view.type_id, view.key.clone())) + Ok(self.uncommitted_endpoint_key( + view, + txn_node_view_fallback_label(view)?, + &view.key, + )) } } Some(NodeOverlayOpinion::Deleted(_)) => Err(EngineError::InvalidOperation(format!( "transaction node local ref {:?} is deleted", local ))), + Some(NodeOverlayOpinion::RemovedLabel) => Err(EngineError::InvalidOperation( + format!("transaction node local ref {:?} is deleted", local), + )), None => Err(EngineError::InvalidOperation(format!( "unknown transaction node local ref {:?}", local @@ -775,28 +1006,112 @@ impl TxnOverlay { } } - fn insert_node_live(&mut self, view: TxnNodeView, local: Option) { - let key = (view.type_id, view.key.clone()); - if let Some(local) = local { - let locals = self.node_key_locals.entry(key.clone()).or_default(); - if !locals.contains(&local) { - locals.push(local); - } + fn insert_node_live_with_locals( + &mut self, + view: TxnNodeView, + local: Option, + related_locals: Vec, + ) { + let mut locals_to_track = related_locals; + if let Some(local) = local.as_ref() { + push_distinct_txn_local(&mut locals_to_track, local.clone()); } + for label in &view.labels { + let key = (label.clone(), view.key.clone()); + for local in &locals_to_track { + let locals = self.node_key_locals.entry(key.clone()).or_default(); + if !locals.contains(local) { + locals.push(local.clone()); + } + } + let opinion = NodeOverlayOpinion::Live(view.clone()); + self.nodes_by_key.insert(key.clone(), opinion.clone()); + self.set_node_locals_for_key(&key, opinion); + } let opinion = NodeOverlayOpinion::Live(view.clone()); - self.nodes_by_key.insert(key.clone(), opinion.clone()); if let Some(id) = view.id { - self.nodes_by_id.insert(id, opinion); + self.nodes_by_id.insert(id, opinion.clone()); + } + for local in locals_to_track { + self.nodes_by_local + .insert(local.clone(), node_opinion_for_local(&opinion, &local)); + } + } + + fn node_locals_for_view(&self, view: &TxnNodeView) -> Vec { + let mut locals = Vec::new(); + if let Some(local) = &view.local { + push_distinct_txn_local(&mut locals, local.clone()); + } + for label in &view.labels { + let key = (label.clone(), view.key.clone()); + if let Some(known_locals) = self.node_key_locals.get(&key) { + for local in known_locals { + push_distinct_txn_local(&mut locals, local.clone()); + } + } + } + locals + } + + fn uncommitted_endpoint_key( + &self, + view: &TxnNodeView, + fallback_label: &str, + fallback_key: &str, + ) -> TxnEndpointKey { + if let Some(local) = self.canonical_uncommitted_node_local(view) { + TxnEndpointKey::Local(local) + } else { + TxnEndpointKey::Key(fallback_label.to_string(), fallback_key.to_string()) + } + } + + fn canonical_uncommitted_node_local(&self, view: &TxnNodeView) -> Option { + for label in &view.labels { + let key = (label.clone(), view.key.clone()); + let Some(locals) = self.node_key_locals.get(&key) else { + continue; + }; + for local in locals { + let Some(NodeOverlayOpinion::Live(local_view)) = self.nodes_by_local.get(local) + else { + continue; + }; + if local_view.id.is_none() + && local_view.key == view.key + && txn_label_sets_equal(&local_view.labels, &view.labels) + { + return Some(local.clone()); + } + } + } + view.local.clone() + } + + fn canonical_node_ref( + &self, + snapshot: &ReadView, + target: &TxnNodeRef, + ) -> Result { + let Some(view) = self.get_node(snapshot, target)? else { + return Ok(target.clone()); + }; + if let Some(id) = view.id { + Ok(TxnNodeRef::Id(id)) + } else if let Some(local) = self.canonical_uncommitted_node_local(&view) { + Ok(TxnNodeRef::Local(local)) + } else { + Ok(target.clone()) } - self.set_node_locals_for_key(&key, NodeOverlayOpinion::Live(view)); } fn insert_edge_live_with_key( &mut self, view: TxnEdgeView, local: Option, - triple_key: (TxnEndpointKey, TxnEndpointKey, u32), + triple_key: (TxnEndpointKey, TxnEndpointKey, String), ) { if self.edge_uniqueness { if let Some(local) = local.as_ref() { @@ -828,8 +1143,8 @@ impl TxnOverlay { ) -> Result<(), EngineError> { let from_key = self.endpoint_key(snapshot, &view.from)?; let to_key = self.endpoint_key(snapshot, &view.to)?; - let type_id = view.type_id; - let triple_key = (from_key, to_key, type_id); + let label = view.label.clone(); + let triple_key = (from_key, to_key, label); let opinion = EdgeOverlayOpinion::Live(view.clone()); if self.edge_uniqueness { @@ -854,7 +1169,7 @@ impl TxnOverlay { fn insert_edge_triple_delete_if_current( &mut self, - triple_key: (TxnEndpointKey, TxnEndpointKey, u32), + triple_key: (TxnEndpointKey, TxnEndpointKey, String), opinion: EdgeOverlayOpinion, deleted_local: Option<&TxnLocalRef>, deleted_id: Option, @@ -873,7 +1188,7 @@ impl TxnOverlay { fn edge_triple_matches_target( &self, - triple_key: &(TxnEndpointKey, TxnEndpointKey, u32), + triple_key: &(TxnEndpointKey, TxnEndpointKey, String), target_local: Option<&TxnLocalRef>, target_id: Option, ) -> bool { @@ -885,7 +1200,7 @@ impl TxnOverlay { fn track_edge_local_for_triple( &mut self, - triple_key: (TxnEndpointKey, TxnEndpointKey, u32), + triple_key: (TxnEndpointKey, TxnEndpointKey, String), local: &TxnLocalRef, ) { if self.edge_uniqueness { @@ -909,27 +1224,37 @@ impl TxnOverlay { TxnNodeRef::Id(id) => { self.nodes_by_id.insert(*id, opinion.clone()); } - TxnNodeRef::Key { type_id, key } => { + TxnNodeRef::Key { label, key } => { self.nodes_by_key - .insert((*type_id, key.clone()), opinion.clone()); + .insert((label.clone(), key.clone()), opinion.clone()); } } if let Some(id) = deleted_id { self.deleted_node_ids_seen.insert(id); } if let Some(view) = existing.as_ref() { - let key = (view.type_id, view.key.clone()); - self.nodes_by_key.insert(key.clone(), opinion.clone()); + let keys: Vec<(String, String)> = view + .labels + .iter() + .map(|label| (label.clone(), view.key.clone())) + .collect(); + for key in &keys { + self.nodes_by_key.insert(key.clone(), opinion.clone()); + } if let Some(id) = view.id { self.nodes_by_id.insert(id, opinion.clone()); } if let Some(local) = &view.local { - let locals = self.node_key_locals.entry(key.clone()).or_default(); - if !locals.contains(local) { - locals.push(local.clone()); + for key in &keys { + let locals = self.node_key_locals.entry(key.clone()).or_default(); + if !locals.contains(local) { + locals.push(local.clone()); + } } } - self.set_node_locals_for_key(&key, opinion); + for key in keys { + self.set_node_locals_for_key(&key, opinion.clone()); + } } } @@ -961,7 +1286,7 @@ impl TxnOverlay { (None, None) => TxnEdgeRef::Triple { from: view.from.clone(), to: view.to.clone(), - type_id: view.type_id, + label: view.label.clone(), }, }; self.mark_edge_deleted(snapshot, &target, Some(view))?; @@ -995,33 +1320,41 @@ impl TxnOverlay { TxnNodeRef::Id(id) => Some(*id), _ => None, }); - let deleted_key = existing - .map(|view| (view.type_id, view.key.as_str())) - .or(match target { - TxnNodeRef::Key { type_id, key } => Some((*type_id, key.as_str())), - _ => None, - }); + let deleted_keys = deleted_node_keys(existing, target); match candidate { TxnNodeRef::Id(id) => Ok(deleted_id == Some(*id)), - TxnNodeRef::Key { type_id, key } => { - if deleted_key == Some((*type_id, key.as_str())) { + TxnNodeRef::Key { label, key } => { + validate_label_token_name(label)?; + if deleted_keys + .iter() + .any(|(deleted_label, deleted_key)| { + deleted_label == label && deleted_key == key + }) + { return Ok(true); } let Some(id) = deleted_id else { return Ok(false); }; + let Some(label_id) = snapshot.label_catalog.resolve_node_label_for_read(label)? else { + return Ok(false); + }; Ok(snapshot - .get_node_by_key(*type_id, key)? + .get_node_by_label_key(label_id, key)? .is_some_and(|node| node.id == id)) } TxnNodeRef::Local(local) => match self.nodes_by_local.get(local) { Some(NodeOverlayOpinion::Live(view)) | Some(NodeOverlayOpinion::Deleted(Some(view))) => { Ok(view.id.is_some_and(|id| deleted_id == Some(id)) - || deleted_key == Some((view.type_id, view.key.as_str()))) + || deleted_keys.iter().any(|(label, key)| { + view.key == *key + && view.labels.iter().any(|existing| existing == label) + })) } Some(NodeOverlayOpinion::Deleted(None)) => Ok(false), + Some(NodeOverlayOpinion::RemovedLabel) => Ok(false), None => Err(EngineError::InvalidOperation(format!( "unknown transaction node local ref {:?}", local @@ -1044,13 +1377,18 @@ impl TxnOverlay { TxnEdgeRef::Id(id) => { self.edges_by_id.insert(*id, opinion.clone()); } - TxnEdgeRef::Triple { from, to, type_id } => { + TxnEdgeRef::Triple { + from, + to, + label, + } => { + validate_label_token_name(label)?; if let (Some(from_key), Some(to_key)) = ( self.read_endpoint_key(snapshot, from)?, self.read_endpoint_key(snapshot, to)?, ) { self.insert_edge_triple_delete_if_current( - (from_key, to_key, *type_id), + (from_key, to_key, label.clone()), opinion.clone(), None, None, @@ -1064,7 +1402,7 @@ impl TxnOverlay { } let from_key = self.endpoint_key(snapshot, &view.from)?; let to_key = self.endpoint_key(snapshot, &view.to)?; - let triple_key = (from_key, to_key, view.type_id); + let triple_key = (from_key, to_key, view.label.clone()); if let Some(local) = &view.local { self.edges_by_local .insert(local.clone(), edge_opinion_for_local(&opinion, local)); @@ -1082,7 +1420,7 @@ impl TxnOverlay { fn set_node_locals_for_key( &mut self, - key: &(u32, String), + key: &(String, String), opinion: NodeOverlayOpinion, ) { let Some(locals) = self.node_key_locals.get(key).cloned() else { @@ -1096,7 +1434,7 @@ impl TxnOverlay { fn set_edge_locals_for_triple( &mut self, - triple_key: &(TxnEndpointKey, TxnEndpointKey, u32), + triple_key: &(TxnEndpointKey, TxnEndpointKey, String), opinion: EdgeOverlayOpinion, ) { let Some(locals) = self.edge_triple_locals.get(triple_key).cloned() else { @@ -1124,6 +1462,7 @@ fn node_opinion_for_local( view.local = Some(local.clone()); view })), + NodeOverlayOpinion::RemovedLabel => NodeOverlayOpinion::RemovedLabel, } } @@ -1159,34 +1498,148 @@ fn edge_opinion_matches_target( } } +fn push_distinct_txn_local(locals: &mut Vec, local: TxnLocalRef) { + if !locals.contains(&local) { + locals.push(local); + } +} + +fn txn_label_sets_equal(left: &[String], right: &[String]) -> bool { + left.len() == right.len() && left.iter().all(|label| right.iter().any(|other| other == label)) +} + +fn txn_node_view_fallback_label(view: &TxnNodeView) -> Result<&str, EngineError> { + view.labels.first().map(String::as_str).ok_or_else(|| { + EngineError::InvalidOperation(format!( + "transaction node view for key '{}' has no labels", + view.key + )) + }) +} + +fn deleted_node_keys( + existing: Option<&TxnNodeView>, + target: &TxnNodeRef, +) -> Vec<(String, String)> { + if let Some(view) = existing { + return view + .labels + .iter() + .map(|label| (label.clone(), view.key.clone())) + .collect(); + } + match target { + TxnNodeRef::Key { label, key } => vec![(label.clone(), key.clone())], + _ => Vec::new(), + } +} + +fn push_distinct_txn_name<'a>( + name: &'a str, + names: &mut Vec<&'a str>, + seen: &mut HashSet<&'a str>, +) { + if seen.insert(name) { + names.push(name); + } +} + +fn collect_txn_intent_read_label_names<'a>( + intent: &'a TxnIntent, + node_labels: &mut Vec<&'a str>, + seen_node_labels: &mut HashSet<&'a str>, + edge_labels: &mut Vec<&'a str>, + seen_edge_labels: &mut HashSet<&'a str>, +) { + match intent { + TxnIntent::UpsertNode { .. } => {} + TxnIntent::UpsertEdge { from, to, .. } => { + collect_txn_node_ref_read_label_names(from, node_labels, seen_node_labels); + collect_txn_node_ref_read_label_names(to, node_labels, seen_node_labels); + } + TxnIntent::DeleteNode { target } => { + collect_txn_node_ref_read_label_names(target, node_labels, seen_node_labels); + } + TxnIntent::DeleteEdge { target } | TxnIntent::InvalidateEdge { target, .. } => { + collect_txn_edge_ref_read_label_names( + target, + node_labels, + seen_node_labels, + edge_labels, + seen_edge_labels, + ); + } + } +} + +fn collect_txn_node_ref_read_label_names<'a>( + target: &'a TxnNodeRef, + node_labels: &mut Vec<&'a str>, + seen_node_labels: &mut HashSet<&'a str>, +) { + match target { + TxnNodeRef::Key { label, .. } => { + push_distinct_txn_name(label, node_labels, seen_node_labels); + } + TxnNodeRef::Id(_) | TxnNodeRef::Local(_) => {} + } +} + +fn collect_txn_edge_ref_read_label_names<'a>( + target: &'a TxnEdgeRef, + node_labels: &mut Vec<&'a str>, + seen_node_labels: &mut HashSet<&'a str>, + edge_labels: &mut Vec<&'a str>, + seen_edge_labels: &mut HashSet<&'a str>, +) { + match target { + TxnEdgeRef::Triple { + from, + to, + label, + } => { + collect_txn_node_ref_read_label_names(from, node_labels, seen_node_labels); + collect_txn_node_ref_read_label_names(to, node_labels, seen_node_labels); + push_distinct_txn_name(label, edge_labels, seen_edge_labels); + } + TxnEdgeRef::Id(_) | TxnEdgeRef::Local(_) => {} + } +} + fn collect_txn_intent_cache_targets( intent: &TxnIntent, + label_resolution: &TxnLabelResolution, node_keys: &mut HashSet<(u32, String)>, node_ids: &mut NodeIdSet, edge_ids: &mut NodeIdSet, ) { match intent { - TxnIntent::UpsertNode { type_id, key, .. } => { - node_keys.insert((*type_id, key.clone())); + TxnIntent::UpsertNode { labels, key, .. } => { + for label in labels { + if let Some(label_id) = label_resolution.node_label_id(label) { + node_keys.insert((label_id, key.clone())); + } + } } TxnIntent::UpsertEdge { from, to, .. } => { - collect_txn_node_ref_cache_targets(from, node_keys, node_ids); - collect_txn_node_ref_cache_targets(to, node_keys, node_ids); + collect_txn_node_ref_cache_targets(from, label_resolution, node_keys, node_ids); + collect_txn_node_ref_cache_targets(to, label_resolution, node_keys, node_ids); } TxnIntent::DeleteNode { target } => { - collect_txn_node_ref_cache_targets(target, node_keys, node_ids); + collect_txn_node_ref_cache_targets(target, label_resolution, node_keys, node_ids); } TxnIntent::DeleteEdge { target } => { - collect_txn_edge_ref_cache_targets(target, node_keys, node_ids, edge_ids); + collect_txn_edge_ref_cache_targets(target, label_resolution, node_keys, node_ids, edge_ids); } TxnIntent::InvalidateEdge { target, .. } => { - collect_txn_edge_ref_cache_targets(target, node_keys, node_ids, edge_ids); + collect_txn_edge_ref_cache_targets(target, label_resolution, node_keys, node_ids, edge_ids); } } } fn collect_txn_node_ref_cache_targets( target: &TxnNodeRef, + label_resolution: &TxnLabelResolution, node_keys: &mut HashSet<(u32, String)>, node_ids: &mut NodeIdSet, ) { @@ -1194,8 +1647,10 @@ fn collect_txn_node_ref_cache_targets( TxnNodeRef::Id(id) => { node_ids.insert(*id); } - TxnNodeRef::Key { type_id, key } => { - node_keys.insert((*type_id, key.clone())); + TxnNodeRef::Key { label, key } => { + if let Some(label_id) = label_resolution.node_label_id(label) { + node_keys.insert((label_id, key.clone())); + } } TxnNodeRef::Local(_) => {} } @@ -1203,6 +1658,7 @@ fn collect_txn_node_ref_cache_targets( fn collect_txn_edge_ref_cache_targets( target: &TxnEdgeRef, + label_resolution: &TxnLabelResolution, node_keys: &mut HashSet<(u32, String)>, node_ids: &mut NodeIdSet, edge_ids: &mut NodeIdSet, @@ -1212,18 +1668,22 @@ fn collect_txn_edge_ref_cache_targets( edge_ids.insert(*id); } TxnEdgeRef::Triple { from, to, .. } => { - collect_txn_node_ref_cache_targets(from, node_keys, node_ids); - collect_txn_node_ref_cache_targets(to, node_keys, node_ids); + collect_txn_node_ref_cache_targets(from, label_resolution, node_keys, node_ids); + collect_txn_node_ref_cache_targets(to, label_resolution, node_keys, node_ids); } TxnEdgeRef::Local(_) => {} } } -fn node_to_txn_view(node: NodeRecord) -> TxnNodeView { - TxnNodeView { +fn node_to_txn_view( + node: NodeRecord, + catalog: &ReadLabelCatalogSnapshot, +) -> Result { + let labels = txn_labels_from_record(&node, catalog)?; + Ok(TxnNodeView { id: Some(node.id), local: None, - type_id: node.type_id, + labels, key: node.key, props: node.props, created_at: Some(node.created_at), @@ -1231,23 +1691,114 @@ fn node_to_txn_view(node: NodeRecord) -> TxnNodeView { weight: node.weight, dense_vector: node.dense_vector, sparse_vector: node.sparse_vector, + }) +} + +fn node_to_txn_view_with_resolved_label( + node: NodeRecord, + expected_label_id: u32, + label: String, + catalog: &ReadLabelCatalogSnapshot, +) -> Result { + if !node.label_ids.contains(expected_label_id) { + return Err(EngineError::InvalidOperation(format!( + "node record {} resolved by label '{}' expected label_id {} but found {:?}", + node.id, label, expected_label_id, node.label_ids + ))); + } + let labels = txn_labels_from_record(&node, catalog)?; + Ok(TxnNodeView { + id: Some(node.id), + local: None, + labels, + key: node.key, + props: node.props, + created_at: Some(node.created_at), + updated_at: Some(node.updated_at), + weight: node.weight, + dense_vector: node.dense_vector, + sparse_vector: node.sparse_vector, + }) +} + +fn txn_labels_from_record( + node: &NodeRecord, + catalog: &ReadLabelCatalogSnapshot, +) -> Result, EngineError> { + node.label_ids + .as_slice() + .iter() + .map(|&label_id| { + catalog.node_label(label_id).map(str::to_string).ok_or_else(|| { + EngineError::InvalidOperation(format!( + "node record {} references missing node label_id {}", + node.id, label_id + )) + }) + }) + .collect() +} + +fn txn_node_views_match(left: &TxnNodeView, right: &TxnNodeView) -> bool { + match (left.id, right.id) { + (Some(left), Some(right)) => left == right, + (None, None) => left.local.is_some() && left.local == right.local, + _ => false, } } -fn edge_to_txn_view(edge: EdgeRecord) -> TxnEdgeView { - TxnEdgeView { +fn edge_to_txn_view( + edge: EdgeRecord, + catalog: &ReadLabelCatalogSnapshot, +) -> Result { + let label = catalog + .edge_label(edge.label_id) + .ok_or_else(|| { + EngineError::InvalidOperation(format!( + "edge record {} references missing edge-label label_id {}", + edge.id, edge.label_id + )) + })? + .to_string(); + Ok(TxnEdgeView { id: Some(edge.id), local: None, from: TxnNodeRef::Id(edge.from), to: TxnNodeRef::Id(edge.to), - type_id: edge.type_id, + label, props: edge.props, created_at: Some(edge.created_at), updated_at: Some(edge.updated_at), weight: edge.weight, valid_from: Some(edge.valid_from), valid_to: Some(edge.valid_to), - } + }) +} + +fn edge_to_txn_view_with_resolved_label( + edge: EdgeRecord, + expected_label_id: u32, + label: String, +) -> Result { + if edge.label_id != expected_label_id { + return Err(EngineError::InvalidOperation(format!( + "edge record {} resolved by edge label '{}' expected label_id {} but found {}", + edge.id, label, expected_label_id, edge.label_id + ))); + } + Ok(TxnEdgeView { + id: Some(edge.id), + local: None, + from: TxnNodeRef::Id(edge.from), + to: TxnNodeRef::Id(edge.to), + label, + props: edge.props, + created_at: Some(edge.created_at), + updated_at: Some(edge.updated_at), + weight: edge.weight, + valid_from: Some(edge.valid_from), + valid_to: Some(edge.valid_to), + }) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1274,6 +1825,8 @@ struct PlannedTxnState { local_node_ids: BTreeMap, local_edge_ids: BTreeMap, nodes_by_key: HashMap<(u32, String), (u64, i64)>, + removed_node_keys: HashSet<(u32, String)>, + node_records_by_id: NodeIdMap, edges_by_triple: HashMap<(u64, u64, u32), (u64, i64)>, edge_id_to_triple: NodeIdMap<(u64, u64, u32)>, edge_records_by_id: NodeIdMap, @@ -1321,43 +1874,128 @@ struct TxnPlanningCache { edge_opinions_by_id: NodeIdMap, } +#[derive(Default)] +struct TxnLabelResolution { + node_labels: HashMap>, + edge_labels: HashMap>, +} + +impl TxnLabelResolution { + fn node_label_id(&self, label: &str) -> Option { + self.node_labels.get(label).and_then(|id| *id) + } + + fn edge_label_id(&self, label: &str) -> Option { + self.edge_labels.get(label).and_then(|id| *id) + } + + fn insert_node_label(&mut self, label: String, label_id: Option) { + self.node_labels.entry(label).or_insert(label_id); + } + + fn insert_edge_label(&mut self, label: String, label_id: Option) { + self.edge_labels.entry(label).or_insert(label_id); + } +} + impl EngineCore { fn plan_txn_commit(&mut self, request: &TxnCommitRequest) -> Result { let now = now_millis(); - let mut ops = Vec::new(); + let (label_resolution, mut ops, label_catalog_changed) = self.resolve_txn_label_names(request)?; let mut state = PlannedTxnState::default(); - let mut cache = self.build_txn_planning_cache(request)?; + let mut cache = self.build_txn_planning_cache(request, &label_resolution)?; let mut next_node_id = self.next_node_id; let mut next_edge_id = self.next_edge_id; for entry in &request.entries { match &entry.intent { TxnIntent::UpsertNode { - type_id, + labels, key, options, .. } => { - let (id, created_at) = if let Some(&(id, created_at)) = - state.nodes_by_key.get(&(*type_id, key.clone())) - { - (id, created_at) - } else { - self.validate_node_key_conflict(request, &mut cache, *type_id, key)?; - match self.cached_current_node_key(&mut cache, *type_id, key)? { - Some(node) => (node.id, node.created_at), - None => { - let id = next_node_id; - next_node_id = next_node_id.checked_add(1).ok_or_else(|| { - EngineError::InvalidOperation("node id counter overflow".into()) - })?; - (id, now) + validate_node_key_for_write(key)?; + let validated_labels = + ValidatedNodeLabelList::new(labels.iter().map(String::as_str))?; + let mut label_ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + for (idx, &label) in validated_labels.as_slice().iter().enumerate() { + label_ids[idx] = label_resolution.node_label_id(label).ok_or_else(|| { + EngineError::InvalidOperation(format!( + "transaction node label '{}' was not resolved for commit", + label + )) + })?; + } + let label_ids = + NodeLabelSet::from_label_ids(label_ids[..validated_labels.len()].iter().copied())?; + + let mut winner: Option<(u64, i64, Option)> = None; + for &label_id in label_ids.as_slice() { + let key_tuple = (label_id, key.clone()); + let candidate = if let Some(&(id, created_at)) = + state.nodes_by_key.get(&key_tuple) + { + Some((id, created_at, state.node_records_by_id.get(&id).cloned())) + } else if state.removed_node_keys.contains(&key_tuple) { + None + } else { + self.validate_node_key_conflict(request, &mut cache, label_id, key)?; + let current = self.cached_current_node_key(&mut cache, label_id, key)?; + let node = match current { + Some(node) => Some(node), + None => self.cached_begin_node_key( + request, + &mut cache, + label_id, + key, + )?, + }; + node.map(|node| (node.id, node.created_at, Some(node))) + }; + let Some(candidate) = candidate else { + continue; + }; + match winner.as_ref() { + Some((winner_id, _, _)) if *winner_id != candidate.0 => { + return Err(node_key_conflict_error(key, *winner_id, candidate.0)); } + None => winner = Some(candidate), + _ => {} + } + } + + let (id, created_at, previous_record) = match winner { + Some(existing) => existing, + None => { + let id = next_node_id; + next_node_id = next_node_id.checked_add(1).ok_or_else(|| { + EngineError::InvalidOperation("node id counter overflow".into()) + })?; + (id, now, None) } }; - state - .nodes_by_key - .insert((*type_id, key.clone()), (id, created_at)); + + let previous_labels = state + .node_records_by_id + .get(&id) + .map(|node| node.label_ids) + .or_else(|| previous_record.as_ref().map(|node| node.label_ids)); + if let Some(previous_labels) = previous_labels { + for &old_label_id in previous_labels.as_slice() { + if !label_ids.contains(old_label_id) { + let key_tuple = (old_label_id, key.clone()); + state.nodes_by_key.remove(&key_tuple); + state.removed_node_keys.insert(key_tuple); + } + } + } + + for &label_id in label_ids.as_slice() { + let key_tuple = (label_id, key.clone()); + state.removed_node_keys.remove(&key_tuple); + state.nodes_by_key.insert(key_tuple, (id, created_at)); + } state.deleted_node_ids.remove(&id); if let Some(local) = &entry.produced_node { state.local_node_ids.insert(local.clone(), id); @@ -1367,9 +2005,9 @@ impl EngineCore { options.dense_vector.as_ref(), options.sparse_vector.as_ref(), )?; - ops.push(WalOp::UpsertNode(NodeRecord { + let node = NodeRecord { id, - type_id: *type_id, + label_ids, key: key.clone(), props: options.props.clone(), created_at, @@ -1378,29 +2016,37 @@ impl EngineCore { dense_vector, sparse_vector, last_write_seq: 0, - })); + }; + state.node_records_by_id.insert(id, node.clone()); + ops.push(WalOp::UpsertNode(node)); state.node_ids.push(id); } TxnIntent::UpsertEdge { from, to, - type_id, + label, options, .. } => { + let label_id = label_resolution.edge_label_id(label).ok_or_else(|| { + EngineError::InvalidOperation(format!( + "transaction edge label '{}' was not resolved for commit", + label + )) + })?; let from_id = - self.resolve_node_ref_required(from, &state, request, &mut cache)?; + self.resolve_node_ref_required(from, &state, request, &label_resolution, &mut cache)?; let to_id = - self.resolve_node_ref_required(to, &state, request, &mut cache)?; + self.resolve_node_ref_required(to, &state, request, &label_resolution, &mut cache)?; self.validate_node_id_conflict(&mut cache, from_id, request.snapshot_seq)?; self.validate_node_id_conflict(&mut cache, to_id, request.snapshot_seq)?; - let triple = (from_id, to_id, *type_id); + let triple = (from_id, to_id, label_id); let (id, created_at) = if self.edge_uniqueness { if let Some(&(id, created_at)) = state.edges_by_triple.get(&triple) { (id, created_at) } else { - self.validate_edge_triple_conflict(request, &mut cache, from_id, to_id, *type_id)?; - match self.cached_current_edge_triple(&mut cache, from_id, to_id, *type_id)? { + self.validate_edge_triple_conflict(request, &mut cache, from_id, to_id, label_id)?; + match self.cached_current_edge_triple(&mut cache, from_id, to_id, label_id)? { Some(edge) => (edge.id, edge.created_at), None => { let id = next_edge_id; @@ -1414,7 +2060,7 @@ impl EngineCore { } } } else { - self.validate_edge_triple_conflict(request, &mut cache, from_id, to_id, *type_id)?; + self.validate_edge_triple_conflict(request, &mut cache, from_id, to_id, label_id)?; let id = next_edge_id; next_edge_id = next_edge_id.checked_add(1).ok_or_else(|| { EngineError::InvalidOperation("edge id counter overflow".into()) @@ -1430,7 +2076,7 @@ impl EngineCore { id, from: from_id, to: to_id, - type_id: *type_id, + label_id: label_id, props: options.props.clone(), created_at, updated_at: now, @@ -1446,7 +2092,7 @@ impl EngineCore { } TxnIntent::DeleteNode { target } => { let Some(id) = - self.resolve_node_ref_optional(target, &state, request, &mut cache)? + self.resolve_node_ref_optional(target, &state, request, &label_resolution, &mut cache)? else { continue; }; @@ -1487,7 +2133,7 @@ impl EngineCore { } TxnIntent::DeleteEdge { target } => { let Some(id) = - self.resolve_edge_ref_optional(target, &state, request, &mut cache)? + self.resolve_edge_ref_optional(target, &state, request, &label_resolution, &mut cache)? else { continue; }; @@ -1505,7 +2151,7 @@ impl EngineCore { } TxnIntent::InvalidateEdge { target, valid_to } => { let Some(id) = - self.resolve_edge_ref_optional(target, &state, request, &mut cache)? + self.resolve_edge_ref_optional(target, &state, request, &label_resolution, &mut cache)? else { continue; }; @@ -1518,7 +2164,7 @@ impl EngineCore { valid_to: *valid_to, ..edge }; - let triple = (updated.from, updated.to, updated.type_id); + let triple = (updated.from, updated.to, updated.label_id); set_planned_edge_triple_if_current_or_absent( &mut state, triple, @@ -1536,9 +2182,9 @@ impl EngineCore { valid_to: *valid_to, ..edge }; - let triple = (updated.from, updated.to, updated.type_id); + let triple = (updated.from, updated.to, updated.label_id); let current_triple = - self.cached_current_edge_triple(&mut cache, updated.from, updated.to, updated.type_id)?; + self.cached_current_edge_triple(&mut cache, updated.from, updated.to, updated.label_id)?; if current_triple.as_ref().is_none_or(|edge| edge.id == id) { state .edges_by_triple @@ -1568,12 +2214,93 @@ impl EngineCore { }), auto_flush: true, track_ids: false, + label_catalog_changed, }) } + fn resolve_txn_label_names( + &self, + request: &TxnCommitRequest, + ) -> Result<(TxnLabelResolution, Vec, bool), EngineError> { + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let mut resolution = TxnLabelResolution::default(); + + let mut write_node_labels = Vec::new(); + let mut seen_write_node_labels = HashSet::new(); + let mut write_edge_labels = Vec::new(); + let mut seen_write_edge_labels = HashSet::new(); + + for entry in &request.entries { + match &entry.intent { + TxnIntent::UpsertNode { labels, key, .. } => { + validate_node_key_for_write(key)?; + let labels = ValidatedNodeLabelList::new(labels.iter().map(String::as_str))?; + for &label in labels.as_slice() { + if seen_write_node_labels.insert(label) { + write_node_labels.push(label); + } + } + } + TxnIntent::UpsertEdge { label, .. } => { + if seen_write_edge_labels.insert(label.as_str()) { + write_edge_labels.push(label.as_str()); + } + } + TxnIntent::DeleteNode { .. } + | TxnIntent::DeleteEdge { .. } + | TxnIntent::InvalidateEdge { .. } => {} + } + } + + for label in write_node_labels { + let label_id = label_plan.resolve_node_label_for_write(label)?; + resolution.insert_node_label(label.to_string(), Some(label_id)); + } + for label in write_edge_labels { + let label_id = label_plan.resolve_edge_label_for_write(label)?; + resolution.insert_edge_label(label.to_string(), Some(label_id)); + } + + let mut read_node_labels = Vec::new(); + let mut seen_read_node_labels = HashSet::new(); + let mut read_edge_labels = Vec::new(); + let mut seen_read_edge_labels = HashSet::new(); + for entry in &request.entries { + collect_txn_intent_read_label_names( + &entry.intent, + &mut read_node_labels, + &mut seen_read_node_labels, + &mut read_edge_labels, + &mut seen_read_edge_labels, + ); + } + + for label in read_node_labels { + if resolution.node_labels.contains_key(label) { + continue; + } + let label_id = resolve_node_label_for_read(&catalog, label)?; + resolution.insert_node_label(label.to_string(), label_id); + } + for label in read_edge_labels { + if resolution.edge_labels.contains_key(label) { + continue; + } + let label_id = resolve_edge_label_for_read(&catalog, label)?; + resolution.insert_edge_label(label.to_string(), label_id); + } + + let token_op_count = label_plan.token_op_count(); + let mut ops = Vec::with_capacity(token_op_count + request.entries.len()); + label_plan.push_token_ops(&mut ops); + Ok((resolution, ops, token_op_count > 0)) + } + fn build_txn_planning_cache( &self, request: &TxnCommitRequest, + label_resolution: &TxnLabelResolution, ) -> Result { let mut node_keys = HashSet::new(); let mut node_ids = NodeIdSet::default(); @@ -1581,6 +2308,7 @@ impl EngineCore { for entry in &request.entries { collect_txn_intent_cache_targets( &entry.intent, + label_resolution, &mut node_keys, &mut node_ids, &mut edge_ids, @@ -1592,15 +2320,15 @@ impl EngineCore { if !node_keys.is_empty() { let key_refs: Vec<(u32, &str)> = node_keys .iter() - .map(|(type_id, key)| (*type_id, key.as_str())) + .map(|(label_id, key)| (*label_id, key.as_str())) .collect(); - let begin_nodes = request.snapshot.get_nodes_by_keys_raw(&key_refs)?; - let current_nodes = self.get_nodes_by_keys_raw(&key_refs)?; - for ((type_id, key), node) in node_keys.iter().cloned().zip(begin_nodes) { - cache.begin_node_keys.insert((type_id, key), node); + let begin_nodes = request.snapshot.get_nodes_by_label_keys_raw(&key_refs)?; + let current_nodes = self.get_nodes_by_label_keys_raw(&key_refs)?; + for ((label_id, key), node) in node_keys.iter().cloned().zip(begin_nodes) { + cache.begin_node_keys.insert((label_id, key), node); } - for ((type_id, key), node) in node_keys.into_iter().zip(current_nodes) { - cache.current_node_keys.insert((type_id, key), node); + for ((label_id, key), node) in node_keys.into_iter().zip(current_nodes) { + cache.current_node_keys.insert((label_id, key), node); } } @@ -1623,12 +2351,12 @@ impl EngineCore { &self, request: &TxnCommitRequest, cache: &mut TxnPlanningCache, - type_id: u32, + label_id: u32, key: &str, ) -> Result, EngineError> { - let cache_key = (type_id, key.to_string()); + let cache_key = (label_id, key.to_string()); if !cache.begin_node_keys.contains_key(&cache_key) { - let node = request.snapshot.get_node_by_key_raw(type_id, key)?; + let node = request.snapshot.get_node_by_label_key_raw(label_id, key)?; cache.begin_node_keys.insert(cache_key.clone(), node); } Ok(cache.begin_node_keys.get(&cache_key).cloned().flatten()) @@ -1637,12 +2365,12 @@ impl EngineCore { fn cached_current_node_key( &self, cache: &mut TxnPlanningCache, - type_id: u32, + label_id: u32, key: &str, ) -> Result, EngineError> { - let cache_key = (type_id, key.to_string()); + let cache_key = (label_id, key.to_string()); if !cache.current_node_keys.contains_key(&cache_key) { - let node = self.get_node_by_key_raw(type_id, key)?; + let node = self.get_node_by_label_key_raw(label_id, key)?; cache.current_node_keys.insert(cache_key.clone(), node); } Ok(cache.current_node_keys.get(&cache_key).cloned().flatten()) @@ -1654,13 +2382,13 @@ impl EngineCore { cache: &mut TxnPlanningCache, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result, EngineError> { - let key = (from, to, type_id); + let key = (from, to, label_id); if let std::collections::hash_map::Entry::Vacant(entry) = cache.begin_edge_triples.entry(key) { - let edge = request.snapshot.get_edge_by_triple(from, to, type_id)?; + let edge = request.snapshot.get_edge_by_triple(from, to, label_id)?; entry.insert(edge); } Ok(cache.begin_edge_triples.get(&key).cloned().flatten()) @@ -1671,13 +2399,13 @@ impl EngineCore { cache: &mut TxnPlanningCache, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result, EngineError> { - let key = (from, to, type_id); + let key = (from, to, label_id); if let std::collections::hash_map::Entry::Vacant(entry) = cache.current_edge_triples.entry(key) { - let edge = self.get_edge_by_triple(from, to, type_id)?; + let edge = self.get_edge_by_triple(from, to, label_id)?; entry.insert(edge); } Ok(cache.current_edge_triples.get(&key).cloned().flatten()) @@ -1728,17 +2456,17 @@ impl EngineCore { &self, request: &TxnCommitRequest, cache: &mut TxnPlanningCache, - type_id: u32, + label_id: u32, key: &str, ) -> Result<(), EngineError> { - let current = self.cached_current_node_key(cache, type_id, key)?; + let current = self.cached_current_node_key(cache, label_id, key)?; if let Some(current) = current.as_ref() { if current.last_write_seq <= request.snapshot_seq { return Ok(()); } } - let begin = self.cached_begin_node_key(request, cache, type_id, key)?; + let begin = self.cached_begin_node_key(request, cache, label_id, key)?; match (begin, current) { (Some(begin), Some(current)) if begin.id == current.id => { self.validate_node_id_conflict(cache, begin.id, request.snapshot_seq) @@ -1747,14 +2475,14 @@ impl EngineCore { self.validate_node_id_conflict(cache, begin.id, request.snapshot_seq)?; Err(EngineError::TxnConflict(format!( "node key ({}, {}) changed after transaction begin", - type_id, key + label_id, key ))) } (None, Some(current)) => { if current.last_write_seq > request.snapshot_seq { Err(EngineError::TxnConflict(format!( "node key ({}, {}) appeared after transaction begin", - type_id, key + label_id, key ))) } else { Ok(()) @@ -1767,7 +2495,7 @@ impl EngineCore { fn incident_edges_for_txn_delete( &self, node_id: u64, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let tombstones = self.collect_tombstones(); let deleted_edges = &tombstones.1; let mut results = self @@ -1811,16 +2539,16 @@ impl EngineCore { cache: &mut TxnPlanningCache, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result<(), EngineError> { - let current = self.cached_current_edge_triple(cache, from, to, type_id)?; + let current = self.cached_current_edge_triple(cache, from, to, label_id)?; if let Some(current) = current.as_ref() { if current.last_write_seq <= request.snapshot_seq { return Ok(()); } } - let begin = self.cached_begin_edge_triple(request, cache, from, to, type_id)?; + let begin = self.cached_begin_edge_triple(request, cache, from, to, label_id)?; match (begin, current) { (Some(begin), Some(current)) if begin.id == current.id => { self.validate_edge_id_conflict(cache, begin.id, request.snapshot_seq) @@ -1829,14 +2557,14 @@ impl EngineCore { self.validate_edge_id_conflict(cache, begin.id, request.snapshot_seq)?; Err(EngineError::TxnConflict(format!( "edge triple ({}, {}, {}) changed after transaction begin", - from, to, type_id + from, to, label_id ))) } (None, Some(current)) => { if current.last_write_seq > request.snapshot_seq { Err(EngineError::TxnConflict(format!( "edge triple ({}, {}, {}) appeared after transaction begin", - from, to, type_id + from, to, label_id ))) } else { Ok(()) @@ -1891,10 +2619,11 @@ impl EngineCore { target: &TxnNodeRef, state: &PlannedTxnState, request: &TxnCommitRequest, + label_resolution: &TxnLabelResolution, cache: &mut TxnPlanningCache, ) -> Result { let id = self - .resolve_node_ref_optional(target, state, request, cache)? + .resolve_node_ref_optional(target, state, request, label_resolution, cache)? .ok_or_else(|| { EngineError::InvalidOperation(format!( "transaction node ref {:?} does not resolve to an existing or staged node", @@ -1915,6 +2644,7 @@ impl EngineCore { target: &TxnNodeRef, state: &PlannedTxnState, request: &TxnCommitRequest, + label_resolution: &TxnLabelResolution, cache: &mut TxnPlanningCache, ) -> Result, EngineError> { match target { @@ -1929,19 +2659,25 @@ impl EngineCore { "unknown transaction node local ref {:?}", local )) - }), - TxnNodeRef::Key { type_id, key } => { - if let Some(&(id, _)) = state.nodes_by_key.get(&(*type_id, key.clone())) { + }), + TxnNodeRef::Key { label, key } => { + let Some(label_id) = label_resolution.node_label_id(label) else { + return Ok(None); + }; + let key_tuple = (label_id, key.clone()); + if let Some(&(id, _)) = state.nodes_by_key.get(&key_tuple) { Ok(Some(id)) + } else if state.removed_node_keys.contains(&key_tuple) { + Ok(None) } else { let current = self - .cached_current_node_key(cache, *type_id, key)? + .cached_current_node_key(cache, label_id, key)? .map(|node| node.id); if current.is_some() { Ok(current) } else { Ok(self - .cached_begin_node_key(request, cache, *type_id, key)? + .cached_begin_node_key(request, cache, label_id, key)? .map(|node| node.id)) } } @@ -1954,6 +2690,7 @@ impl EngineCore { target: &TxnEdgeRef, state: &PlannedTxnState, request: &TxnCommitRequest, + label_resolution: &TxnLabelResolution, cache: &mut TxnPlanningCache, ) -> Result, EngineError> { match target { @@ -1969,25 +2706,35 @@ impl EngineCore { local )) }), - TxnEdgeRef::Triple { from, to, type_id } => { - let Some(from_id) = self.resolve_node_ref_optional(from, state, request, cache)? + TxnEdgeRef::Triple { + from, + to, + label, + } => { + let Some(label_id) = label_resolution.edge_label_id(label) else { + return Ok(None); + }; + let Some(from_id) = + self.resolve_node_ref_optional(from, state, request, label_resolution, cache)? else { return Ok(None); }; - let Some(to_id) = self.resolve_node_ref_optional(to, state, request, cache)? else { + let Some(to_id) = + self.resolve_node_ref_optional(to, state, request, label_resolution, cache)? + else { return Ok(None); }; - if let Some(&(id, _)) = state.edges_by_triple.get(&(from_id, to_id, *type_id)) { + if let Some(&(id, _)) = state.edges_by_triple.get(&(from_id, to_id, label_id)) { Ok(Some(id)) } else { let current = self - .cached_current_edge_triple(cache, from_id, to_id, *type_id)? + .cached_current_edge_triple(cache, from_id, to_id, label_id)? .map(|edge| edge.id); if current.is_some() { Ok(current) } else { Ok(self - .cached_begin_edge_triple(request, cache, from_id, to_id, *type_id)? + .cached_begin_edge_triple(request, cache, from_id, to_id, label_id)? .map(|edge| edge.id)) } } diff --git a/src/engine/write.rs b/src/engine/write.rs index 0dfddad..0f7bb66 100644 --- a/src/engine/write.rs +++ b/src/engine/write.rs @@ -1,7 +1,346 @@ // Write operations: upsert, delete, batch, patch, prune. // This file is include!()'d into mod.rs. All items share the engine module scope. +type TokenCreationList = Vec<(String, u32)>; +type TokenCreationSet = (TokenCreationList, TokenCreationList); + +struct LabelResolutionPlan<'a> { + catalog: &'a RuntimeLabelCatalog, + node_labels_to_create: Vec<(String, u32)>, + edge_labels_to_create: Vec<(String, u32)>, + new_node_label_to_id: BTreeMap, + new_edge_label_to_id: BTreeMap, + next_node_label_id: u32, + next_edge_label_id: u32, + #[cfg(test)] + node_label_resolve_calls: usize, + #[cfg(test)] + edge_label_resolve_calls: usize, +} + +impl<'a> LabelResolutionPlan<'a> { + fn from_catalog(catalog: &'a RuntimeLabelCatalog) -> Self { + Self { + catalog, + node_labels_to_create: Vec::new(), + edge_labels_to_create: Vec::new(), + new_node_label_to_id: BTreeMap::new(), + new_edge_label_to_id: BTreeMap::new(), + next_node_label_id: catalog.next_node_label_id, + next_edge_label_id: catalog.next_edge_label_id, + #[cfg(test)] + node_label_resolve_calls: 0, + #[cfg(test)] + edge_label_resolve_calls: 0, + } + } + + fn resolve_node_label_for_write(&mut self, label: &str) -> Result { + #[cfg(test)] + { + self.node_label_resolve_calls += 1; + } + if let Some(&label_id) = self.catalog.node_label_to_id.get(label) { + return Ok(label_id); + } + if let Some(&label_id) = self.new_node_label_to_id.get(label) { + return Ok(label_id); + } + validate_label_token_name(label)?; + if self.next_node_label_id == u32::MAX { + return Err(EngineError::InvalidOperation( + "node label token ID space exhausted".to_string(), + )); + } + let label_id = self.next_node_label_id; + self.next_node_label_id += 1; + let label = label.to_string(); + self.new_node_label_to_id.insert(label.clone(), label_id); + self.node_labels_to_create.push((label, label_id)); + Ok(label_id) + } + + fn resolve_edge_label_for_write(&mut self, label: &str) -> Result { + #[cfg(test)] + { + self.edge_label_resolve_calls += 1; + } + if let Some(&label_id) = self.catalog.edge_label_to_id.get(label) { + return Ok(label_id); + } + if let Some(&label_id) = self.new_edge_label_to_id.get(label) { + return Ok(label_id); + } + validate_label_token_name(label)?; + if self.next_edge_label_id == u32::MAX { + return Err(EngineError::InvalidOperation( + "edge-label token ID space exhausted".to_string(), + )); + } + let label_id = self.next_edge_label_id; + self.next_edge_label_id += 1; + let label = label.to_string(); + self.new_edge_label_to_id.insert(label.clone(), label_id); + self.edge_labels_to_create.push((label, label_id)); + Ok(label_id) + } + + #[cfg(test)] + fn resolve_node_label_ids_for_request<'request, I>( + &mut self, + labels: I, + ) -> Result, EngineError> + where + I: IntoIterator, + { + let labels = labels.into_iter(); + let (min_len, _) = labels.size_hint(); + let mut resolved = HashMap::with_capacity(min_len); + let mut ids = Vec::with_capacity(min_len); + for label in labels { + let label_id = if let Some(&label_id) = resolved.get(label) { + label_id + } else { + let label_id = self.resolve_node_label_for_write(label)?; + resolved.insert(label, label_id); + label_id + }; + ids.push(label_id); + } + Ok(ids) + } + + fn resolve_validated_node_label_set_for_write( + &mut self, + labels: &ValidatedNodeLabelList<'_>, + ) -> Result { + let mut ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + for (idx, &label) in labels.as_slice().iter().enumerate() { + ids[idx] = self.resolve_node_label_for_write(label)?; + } + NodeLabelSet::from_label_ids(ids[..labels.len()].iter().copied()) + } + + fn resolve_validated_node_label_sets_for_request( + &mut self, + requests: &[ValidatedNodeLabelList<'_>], + ) -> Result, EngineError> { + let mut resolved = HashMap::new(); + let mut label_sets = Vec::with_capacity(requests.len()); + for labels in requests { + let mut ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + for (idx, &label) in labels.as_slice().iter().enumerate() { + let label_id = match resolved.entry(label) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let label_id = self.resolve_node_label_for_write(label)?; + entry.insert(label_id); + label_id + } + }; + ids[idx] = label_id; + } + label_sets.push(NodeLabelSet::from_label_ids( + ids[..labels.len()].iter().copied(), + )?); + } + Ok(label_sets) + } + + fn resolve_edge_label_ids_for_request<'request, I>( + &mut self, + edge_labels: I, + ) -> Result, EngineError> + where + I: IntoIterator, + { + let edge_labels = edge_labels.into_iter(); + let (min_len, _) = edge_labels.size_hint(); + let mut resolved = HashMap::with_capacity(min_len); + let mut ids = Vec::with_capacity(min_len); + for label in edge_labels { + let label_id = if let Some(&label_id) = resolved.get(label) { + label_id + } else { + let label_id = self.resolve_edge_label_for_write(label)?; + resolved.insert(label, label_id); + label_id + }; + ids.push(label_id); + } + Ok(ids) + } + + fn token_op_count(&self) -> usize { + self.node_labels_to_create.len() + self.edge_labels_to_create.len() + } + + fn push_token_ops(&self, ops: &mut Vec) { + for (label, label_id) in &self.node_labels_to_create { + ops.push(WalOp::EnsureNodeLabel { + label: label.clone(), + label_id: *label_id, + }); + } + for (label, label_id) in &self.edge_labels_to_create { + ops.push(WalOp::EnsureEdgeLabel { + label: label.clone(), + label_id: *label_id, + }); + } + } + + fn token_creations(&self) -> TokenCreationSet { + ( + self.node_labels_to_create.clone(), + self.edge_labels_to_create.clone(), + ) + } +} + +fn validate_prune_policy(policy: &PrunePolicy) -> Result<(), EngineError> { + if let Some(label) = policy.label.as_deref() { + validate_label_token_name(label)?; + } + if policy.max_age_ms.is_none() && policy.max_weight.is_none() { + return Err(EngineError::InvalidOperation( + "Prune policy must set at least max_age_ms or max_weight".to_string(), + )); + } + if let Some(age) = policy.max_age_ms { + if age <= 0 { + return Err(EngineError::InvalidOperation( + "max_age_ms must be positive".to_string(), + )); + } + } + if let Some(w) = policy.max_weight { + if w.is_nan() || w < 0.0 { + return Err(EngineError::InvalidOperation( + "max_weight must be non-negative and not NaN".to_string(), + )); + } + } + Ok(()) +} + +fn validate_node_key_for_write(key: &str) -> Result<(), EngineError> { + if key.len() > u16::MAX as usize { + return Err(EngineError::InvalidOperation(format!( + "node key must be at most {} UTF-8 bytes, got {}", + u16::MAX, + key.len() + ))); + } + Ok(()) +} + +fn node_key_conflict_error(key: &str, existing: u64, conflicting: u64) -> EngineError { + EngineError::InvalidOperation(format!( + "node key conflict for key '{key}': requested label memberships resolve to node IDs {existing} and {conflicting}" + )) +} + +fn stage_node_label_token_in_manifest( + manifest: &mut ManifestState, + label: &str, + label_id: u32, +) -> Result<(), EngineError> { + validate_label_token_name(label)?; + if let Some(existing_id) = manifest.node_label_tokens.get(label) { + if *existing_id != label_id { + return Err(EngineError::ManifestError(format!( + "node label token conflict: label '{label}' already has label_id {existing_id}, not {label_id}" + ))); + } + return Ok(()); + } + if let Some((existing_label, _)) = manifest + .node_label_tokens + .iter() + .find(|(_, existing_id)| **existing_id == label_id) + { + return Err(EngineError::ManifestError(format!( + "node label token conflict: label_id {label_id} is assigned to both '{existing_label}' and '{label}'" + ))); + } + manifest + .node_label_tokens + .insert(label.to_string(), label_id); + manifest.next_node_label_id = manifest + .next_node_label_id + .max(label_id.saturating_add(1)); + Ok(()) +} + +fn stage_edge_label_token_in_manifest( + manifest: &mut ManifestState, + label: &str, + label_id: u32, +) -> Result<(), EngineError> { + validate_label_token_name(label)?; + if let Some(existing_id) = manifest.edge_label_tokens.get(label) { + if *existing_id != label_id { + return Err(EngineError::ManifestError(format!( + "edge-label token conflict: edge label '{label}' already has label_id {existing_id}, not {label_id}" + ))); + } + return Ok(()); + } + if let Some((existing_edge_label, _)) = manifest + .edge_label_tokens + .iter() + .find(|(_, existing_id)| **existing_id == label_id) + { + return Err(EngineError::ManifestError(format!( + "edge-label token conflict: label_id {label_id} is assigned to both '{existing_edge_label}' and '{label}'" + ))); + } + manifest + .edge_label_tokens + .insert(label.to_string(), label_id); + manifest.next_edge_label_id = manifest + .next_edge_label_id + .max(label_id.saturating_add(1)); + Ok(()) +} + +fn stage_label_tokens_in_manifest( + manifest: &mut ManifestState, + node_labels: &[(String, u32)], + edge_labels: &[(String, u32)], +) -> Result<(), EngineError> { + for (label, label_id) in node_labels { + stage_node_label_token_in_manifest(manifest, label, *label_id)?; + } + for (label, label_id) in edge_labels { + stage_edge_label_token_in_manifest(manifest, label, *label_id)?; + } + Ok(()) +} + impl EngineCore { + fn apply_manifest_token_creations( + &mut self, + node_labels: &[(String, u32)], + edge_labels: &[(String, u32)], + ) -> Result<(), EngineError> { + if node_labels.is_empty() && edge_labels.is_empty() { + return Ok(()); + } + { + let mut catalog = self.label_catalog.write().unwrap(); + for (label, label_id) in node_labels { + catalog.apply_node_label(label.clone(), *label_id, None)?; + } + for (label, label_id) in edge_labels { + catalog.apply_edge_label(label.clone(), *label_id, None)?; + } + catalog.apply_to_manifest(&mut self.manifest); + } + Ok(()) + } + fn commit_core_write_plan( &mut self, plan: CoreWritePlan, @@ -15,7 +354,11 @@ impl EngineCore { _ => self.append_and_apply_normalized(&plan.ops)?, } if !plan.ops.is_empty() { - publish_impact = PublishImpact::SnapshotOnly; + publish_impact = if plan.label_catalog_changed { + PublishImpact::SnapshotWithLabelCatalog + } else { + PublishImpact::SnapshotOnly + }; } if plan.track_ids { @@ -36,19 +379,28 @@ impl EngineCore { (result, publish_impact) } - fn plan_core_write(&mut self, request: &CoreWriteRequest) -> Result { + fn plan_core_write( + &mut self, + request: &CoreWriteRequest, + ) -> Result { match request { + CoreWriteRequest::EnsureNodeLabel { label } => self.plan_ensure_node_label(label), + CoreWriteRequest::EnsureEdgeLabel { label } => self.plan_ensure_edge_label(label), CoreWriteRequest::UpsertNode { - type_id, + labels, key, options, - } => self.plan_upsert_node(*type_id, key, options), + } => self.plan_upsert_node(labels, key, options), + CoreWriteRequest::AddNodeLabel { id, label } => self.plan_add_node_label(*id, label), + CoreWriteRequest::RemoveNodeLabel { id, label } => { + self.plan_remove_node_label(*id, label) + } CoreWriteRequest::UpsertEdge { from, to, - type_id, + label, options, - } => self.plan_upsert_edge(*from, *to, *type_id, options), + } => self.plan_upsert_edge(*from, *to, label, options), CoreWriteRequest::BatchUpsertNodes { inputs } => self.plan_batch_upsert_nodes(inputs), CoreWriteRequest::BatchUpsertEdges { inputs } => self.plan_batch_upsert_edges(inputs), CoreWriteRequest::DeleteNode { id } => self.plan_delete_node(*id), @@ -67,6 +419,8 @@ impl EngineCore { | CoreWriteRequest::RemovePrunePolicy { .. } | CoreWriteRequest::EnsureNodePropertyIndex { .. } | CoreWriteRequest::DropNodePropertyIndex { .. } + | CoreWriteRequest::EnsureEdgePropertyIndex { .. } + | CoreWriteRequest::DropEdgePropertyIndex { .. } | CoreWriteRequest::ApplySecondaryIndexReadFollowup { .. } | CoreWriteRequest::Sync | CoreWriteRequest::Flush @@ -78,20 +432,73 @@ impl EngineCore { } } + fn plan_ensure_node_label(&mut self, label: &str) -> Result { + let (label_id, should_create) = self + .label_catalog + .read() + .unwrap() + .reserve_node_label(label)?; + let ops = if should_create { + vec![WalOp::EnsureNodeLabel { + label: label.to_string(), + label_id, + }] + } else { + Vec::new() + }; + Ok(CoreWritePlan { + ops, + reply: CoreWriteReply::U32(label_id), + auto_flush: false, + track_ids: false, + label_catalog_changed: should_create, + }) + } + + fn plan_ensure_edge_label(&mut self, label: &str) -> Result { + let (label_id, should_create) = self + .label_catalog + .read() + .unwrap() + .reserve_edge_label(label)?; + let ops = if should_create { + vec![WalOp::EnsureEdgeLabel { + label: label.to_string(), + label_id: label_id, + }] + } else { + Vec::new() + }; + Ok(CoreWritePlan { + ops, + reply: CoreWriteReply::U32(label_id), + auto_flush: false, + track_ids: false, + label_catalog_changed: should_create, + }) + } + fn plan_upsert_node( &mut self, - type_id: u32, + labels: &[String], key: &str, options: &UpsertNodeOptions, ) -> Result { - let now = now_millis(); + let validated_labels = ValidatedNodeLabelList::new(labels.iter().map(String::as_str))?; + validate_node_key_for_write(key)?; let (dense_vector, sparse_vector) = normalize_node_vectors_for_write( self.manifest.dense_vector.as_ref(), options.dense_vector.as_ref(), options.sparse_vector.as_ref(), )?; + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let label_ids = label_plan.resolve_validated_node_label_set_for_write(&validated_labels)?; + let token_op_count = label_plan.token_op_count(); + let label_catalog_changed = token_op_count > 0; + let now = now_millis(); - let (id, created_at) = match self.find_existing_node(type_id, key)? { + let (id, created_at) = match self.find_existing_node_for_label_set(label_ids, key)? { Some((id, created_at)) => (id, created_at), None => { let id = self.next_node_id; @@ -103,7 +510,7 @@ impl EngineCore { let node = NodeRecord { id, - type_id, + label_ids, key: key.to_string(), props: options.props.clone(), created_at, @@ -114,11 +521,260 @@ impl EngineCore { last_write_seq: 0, }; + let mut ops = Vec::with_capacity(token_op_count + 1); + label_plan.push_token_ops(&mut ops); + drop(label_plan); + drop(catalog); + ops.push(WalOp::UpsertNode(node)); + Ok(CoreWritePlan { - ops: vec![WalOp::UpsertNode(node)], + ops, reply: CoreWriteReply::U64(id), auto_flush: true, track_ids: false, + label_catalog_changed, + }) + } + + fn plan_node_upsert_records( + &mut self, + inputs: &[NodeInput], + label_sets: &[NodeLabelSet], + normalized_vectors: Vec<(Option, Option)>, + now: i64, + ) -> Result<(Vec, Vec, u64), EngineError> { + let mut committed_keys: HashMap<(u32, String), NodeRecord> = HashMap::new(); + let mut committed_nodes_by_id: NodeIdMap = NodeIdMap::default(); + if !inputs.is_empty() { + let mut distinct_keys = Vec::new(); + let mut seen_keys = HashSet::new(); + for (input, label_set) in inputs.iter().zip(label_sets.iter().copied()) { + for &label_id in label_set.as_slice() { + if seen_keys.insert((label_id, input.key.as_str())) { + distinct_keys.push((label_id, input.key.as_str())); + } + } + } + let existing_nodes = self.get_nodes_by_label_keys_raw(&distinct_keys)?; + for ((label_id, key), existing) in distinct_keys.into_iter().zip(existing_nodes) { + if let Some(node) = existing { + committed_nodes_by_id + .entry(node.id) + .or_insert_with(|| node.clone()); + committed_keys.insert((label_id, key.to_string()), node); + } + } + } + + let mut batch_keys: HashMap<(u32, String), (u64, i64)> = HashMap::new(); + let mut removed_keys: HashSet<(u32, String)> = HashSet::new(); + let mut staged_label_sets: NodeIdMap = NodeIdMap::default(); + let mut next_node_id = self.next_node_id; + let mut records = Vec::with_capacity(inputs.len()); + let mut ids = Vec::with_capacity(inputs.len()); + + for ((input, &label_set), (dense_vector, sparse_vector)) in inputs + .iter() + .zip(label_sets.iter()) + .zip(normalized_vectors.into_iter()) + { + let mut winner: Option<(u64, i64)> = None; + for &label_id in label_set.as_slice() { + let key_tuple = (label_id, input.key.clone()); + let membership = batch_keys + .get(&key_tuple) + .copied() + .or_else(|| { + if removed_keys.contains(&key_tuple) { + None + } else { + committed_keys + .get(&key_tuple) + .map(|node| (node.id, node.created_at)) + } + }); + if let Some((id, created_at)) = membership { + match winner { + Some((winner_id, _)) if winner_id != id => { + return Err(node_key_conflict_error(&input.key, winner_id, id)); + } + None => winner = Some((id, created_at)), + _ => {} + } + } + } + + let (id, created_at) = match winner { + Some(existing) => existing, + None => { + let id = next_node_id; + next_node_id = next_node_id.checked_add(1).ok_or_else(|| { + EngineError::InvalidOperation("node id counter overflow".into()) + })?; + (id, now) + } + }; + + let previous_labels = staged_label_sets + .get(&id) + .copied() + .or_else(|| committed_nodes_by_id.get(&id).map(|node| node.label_ids)); + if let Some(previous_labels) = previous_labels { + for &old_label_id in previous_labels.as_slice() { + if !label_set.contains(old_label_id) { + let key_tuple = (old_label_id, input.key.clone()); + batch_keys.remove(&key_tuple); + removed_keys.insert(key_tuple); + } + } + } + for &new_label_id in label_set.as_slice() { + let key_tuple = (new_label_id, input.key.clone()); + removed_keys.remove(&key_tuple); + batch_keys.insert(key_tuple, (id, created_at)); + } + staged_label_sets.insert(id, label_set); + + records.push(NodeRecord { + id, + label_ids: label_set, + key: input.key.clone(), + props: input.props.clone(), + created_at, + updated_at: now, + weight: input.weight, + dense_vector, + sparse_vector, + last_write_seq: 0, + }); + ids.push(id); + } + + Ok((records, ids, next_node_id)) + } + + fn plan_add_node_label( + &mut self, + id: u64, + label: &str, + ) -> Result { + validate_label_token_name(label)?; + let current = self.get_nodes_raw(&[id])?.into_iter().next().flatten().ok_or_else(|| { + EngineError::InvalidOperation(format!("node {id} does not exist")) + })?; + + let catalog = self.label_catalog.read().unwrap(); + let existing_label_id = catalog.node_label_to_id.get(label).copied(); + if existing_label_id.is_some_and(|label_id| current.label_ids.contains(label_id)) { + return Ok(CoreWritePlan { + ops: Vec::new(), + reply: CoreWriteReply::Bool(false), + auto_flush: false, + track_ids: false, + label_catalog_changed: false, + }); + } + if current.label_ids.len() == MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::InvalidOperation(format!( + "node label set must contain at most {} labels", + MAX_NODE_LABELS_PER_NODE + ))); + } + + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let label_id = match existing_label_id { + Some(label_id) => label_id, + None => label_plan.resolve_node_label_for_write(label)?, + }; + let token_op_count = label_plan.token_op_count(); + let label_catalog_changed = token_op_count > 0; + + if let Some((existing_id, _)) = self.find_existing_node(label_id, ¤t.key)? { + if existing_id != current.id { + return Err(node_key_conflict_error(¤t.key, current.id, existing_id)); + } + } + + let mut next_labels = current.label_ids.as_slice().to_vec(); + next_labels.push(label_id); + let label_ids = NodeLabelSet::from_label_ids(next_labels)?; + let mut ops = Vec::with_capacity(token_op_count + 1); + label_plan.push_token_ops(&mut ops); + drop(label_plan); + drop(catalog); + ops.push(WalOp::UpsertNode(NodeRecord { + updated_at: now_millis(), + label_ids, + ..current + })); + + Ok(CoreWritePlan { + ops, + reply: CoreWriteReply::Bool(true), + auto_flush: true, + track_ids: false, + label_catalog_changed, + }) + } + + fn plan_remove_node_label( + &mut self, + id: u64, + label: &str, + ) -> Result { + validate_label_token_name(label)?; + let current = self.get_nodes_raw(&[id])?.into_iter().next().flatten().ok_or_else(|| { + EngineError::InvalidOperation(format!("node {id} does not exist")) + })?; + + let label_id = { + let catalog = self.label_catalog.read().unwrap(); + match resolve_node_label_for_read(&catalog, label)? { + Some(label_id) => label_id, + None => { + return Ok(CoreWritePlan { + ops: Vec::new(), + reply: CoreWriteReply::Bool(false), + auto_flush: false, + track_ids: false, + label_catalog_changed: false, + }); + } + } + }; + + if !current.label_ids.contains(label_id) { + return Ok(CoreWritePlan { + ops: Vec::new(), + reply: CoreWriteReply::Bool(false), + auto_flush: false, + track_ids: false, + label_catalog_changed: false, + }); + } + if current.label_ids.len() == 1 { + return Err(EngineError::InvalidOperation( + "cannot remove the last node label".to_string(), + )); + } + + let next_labels = current + .label_ids + .as_slice() + .iter() + .copied() + .filter(|&existing| existing != label_id); + let label_ids = NodeLabelSet::from_label_ids(next_labels)?; + Ok(CoreWritePlan { + ops: vec![WalOp::UpsertNode(NodeRecord { + updated_at: now_millis(), + label_ids, + ..current + })], + reply: CoreWriteReply::Bool(true), + auto_flush: true, + track_ids: false, + label_catalog_changed: false, }) } @@ -126,13 +782,18 @@ impl EngineCore { &mut self, from: u64, to: u64, - type_id: u32, + label: &str, options: &UpsertEdgeOptions, ) -> Result { + let (label_id, should_create_token) = self + .label_catalog + .read() + .unwrap() + .reserve_edge_label(label)?; let now = now_millis(); let (id, created_at) = if self.edge_uniqueness { - match self.find_existing_edge(from, to, type_id)? { + match self.find_existing_edge(from, to, label_id)? { Some((id, created_at)) => (id, created_at), None => { let id = self.next_edge_id; @@ -152,7 +813,7 @@ impl EngineCore { id, from, to, - type_id, + label_id: label_id, props: options.props.clone(), created_at, updated_at: now, @@ -162,81 +823,113 @@ impl EngineCore { last_write_seq: 0, }; + let mut ops = Vec::with_capacity(1 + usize::from(should_create_token)); + if should_create_token { + ops.push(WalOp::EnsureEdgeLabel { + label: label.to_string(), + label_id: label_id, + }); + } + ops.push(WalOp::UpsertEdge(edge)); + Ok(CoreWritePlan { - ops: vec![WalOp::UpsertEdge(edge)], + ops, reply: CoreWriteReply::U64(id), auto_flush: true, track_ids: false, + label_catalog_changed: should_create_token, }) } - fn plan_batch_upsert_nodes(&mut self, inputs: &[NodeInput]) -> Result { - let now = now_millis(); - let mut ops = Vec::with_capacity(inputs.len()); - let mut ids = Vec::with_capacity(inputs.len()); - let mut batch_keys: HashMap<(u32, String), (u64, i64)> = HashMap::new(); - + fn plan_batch_upsert_nodes( + &mut self, + inputs: &[NodeInput], + ) -> Result { + let mut validated_labels = Vec::with_capacity(inputs.len()); + let mut normalized_vectors = Vec::with_capacity(inputs.len()); for input in inputs { - let (dense_vector, sparse_vector) = normalize_node_vectors_for_write( + validated_labels.push(ValidatedNodeLabelList::new( + input.labels.iter().map(String::as_str), + )?); + validate_node_key_for_write(&input.key)?; + normalized_vectors.push(normalize_node_vectors_for_write( self.manifest.dense_vector.as_ref(), input.dense_vector.as_ref(), input.sparse_vector.as_ref(), - )?; - let key_tuple = (input.type_id, input.key.clone()); - - let (id, created_at) = if let Some(&(id, created_at)) = batch_keys.get(&key_tuple) { - (id, created_at) - } else if let Some((id, created_at)) = - self.find_existing_node(input.type_id, &input.key)? - { - (id, created_at) - } else { - let id = self.next_node_id; - self.next_node_id += 1; - self.update_next_node_id_seen(); - (id, now) - }; - - batch_keys.insert(key_tuple, (id, created_at)); + )?); + } - ops.push(WalOp::UpsertNode(NodeRecord { - id, - type_id: input.type_id, - key: input.key.clone(), - props: input.props.clone(), - created_at, - updated_at: now, - weight: input.weight, - dense_vector, - sparse_vector, - last_write_seq: 0, - })); - ids.push(id); + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let label_sets = + label_plan.resolve_validated_node_label_sets_for_request(&validated_labels)?; + let now = now_millis(); + let token_op_count = label_plan.token_op_count(); + let label_catalog_changed = token_op_count > 0; + let mut ops = Vec::with_capacity(token_op_count + inputs.len()); + label_plan.push_token_ops(&mut ops); + drop(label_plan); + drop(catalog); + let (records, ids, next_node_id) = + self.plan_node_upsert_records(inputs, &label_sets, normalized_vectors, now)?; + if next_node_id != self.next_node_id { + self.next_node_id = next_node_id; + self.update_next_node_id_seen(); } + ops.extend(records.into_iter().map(WalOp::UpsertNode)); Ok(CoreWritePlan { ops, reply: CoreWriteReply::VecU64(ids), auto_flush: true, track_ids: false, + label_catalog_changed, }) } - fn plan_batch_upsert_edges(&mut self, inputs: &[EdgeInput]) -> Result { + fn plan_batch_upsert_edges( + &mut self, + inputs: &[EdgeInput], + ) -> Result { + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let label_ids = label_plan.resolve_edge_label_ids_for_request( + inputs.iter().map(|input| input.label.as_str()), + )?; let now = now_millis(); - let mut ops = Vec::with_capacity(inputs.len()); + let token_op_count = label_plan.token_op_count(); + let label_catalog_changed = token_op_count > 0; + let mut ops = Vec::with_capacity(token_op_count + inputs.len()); + label_plan.push_token_ops(&mut ops); + drop(label_plan); + drop(catalog); let mut ids = Vec::with_capacity(inputs.len()); + let mut committed_triples: HashMap<(u64, u64, u32), (u64, i64)> = HashMap::new(); + if self.edge_uniqueness && !inputs.is_empty() { + let mut distinct_triples = Vec::new(); + let mut seen_triples = HashSet::new(); + for (input, &label_id) in inputs.iter().zip(label_ids.iter()) { + let triple = (input.from, input.to, label_id); + if seen_triples.insert(triple) { + distinct_triples.push(triple); + } + } + let existing_edges = self.find_existing_edges_batch(&distinct_triples)?; + for (triple, existing) in distinct_triples.into_iter().zip(existing_edges) { + if let Some((id, created_at)) = existing { + committed_triples.insert(triple, (id, created_at)); + } + } + } let mut batch_triples: HashMap<(u64, u64, u32), (u64, i64)> = HashMap::new(); - for input in inputs { - let triple = (input.from, input.to, input.type_id); + for (input, label_id) in inputs.iter().zip(label_ids.iter().copied()) { + let triple = (input.from, input.to, label_id); let (id, created_at) = if self.edge_uniqueness { if let Some(&(id, created_at)) = batch_triples.get(&triple) { (id, created_at) - } else if let Some((id, created_at)) = - self.find_existing_edge(input.from, input.to, input.type_id)? - { + } else if let Some(&(id, created_at)) = committed_triples.get(&triple) { (id, created_at) } else { let id = self.next_edge_id; @@ -259,7 +952,7 @@ impl EngineCore { id, from: input.from, to: input.to, - type_id: input.type_id, + label_id: label_id, props: input.props.clone(), created_at, updated_at: now, @@ -276,6 +969,7 @@ impl EngineCore { reply: CoreWriteReply::VecU64(ids), auto_flush: true, track_ids: false, + label_catalog_changed, }) } @@ -299,6 +993,7 @@ impl EngineCore { reply: CoreWriteReply::Unit, auto_flush: true, track_ids: false, + label_catalog_changed: false, }) } @@ -311,10 +1006,15 @@ impl EngineCore { reply: CoreWriteReply::Unit, auto_flush: true, track_ids: false, + label_catalog_changed: false, }) } - fn plan_invalidate_edge(&mut self, id: u64, valid_to: i64) -> Result { + fn plan_invalidate_edge( + &mut self, + id: u64, + valid_to: i64, + ) -> Result { let edge = match self.get_edge(id)? { Some(edge) => edge, None => { @@ -323,6 +1023,7 @@ impl EngineCore { reply: CoreWriteReply::OptionEdge(None), auto_flush: true, track_ids: false, + label_catalog_changed: false, }); } }; @@ -338,17 +1039,23 @@ impl EngineCore { reply: CoreWriteReply::OptionEdge(Some(updated)), auto_flush: true, track_ids: false, + label_catalog_changed: false, }) } #[cfg(test)] fn plan_write_op(&mut self, op: &WalOp) -> Result { let normalized = normalize_wal_op_for_write(self.manifest.dense_vector.as_ref(), op)?; + self.validate_wal_op_label_tokens_for_write(&normalized)?; Ok(CoreWritePlan { ops: vec![normalized], reply: CoreWriteReply::Unit, auto_flush: false, track_ids: true, + label_catalog_changed: matches!( + op, + WalOp::EnsureNodeLabel { .. } | WalOp::EnsureEdgeLabel { .. } + ), }) } @@ -358,67 +1065,99 @@ impl EngineCore { .iter() .map(|op| normalize_wal_op_for_write(self.manifest.dense_vector.as_ref(), op)) .collect::>()?; + for op in &normalized_ops { + self.validate_wal_op_label_tokens_for_write(op)?; + } Ok(CoreWritePlan { ops: normalized_ops, reply: CoreWriteReply::Unit, auto_flush: false, track_ids: true, - }) - } - - fn plan_graph_patch(&mut self, patch: &GraphPatch) -> Result { - let now = now_millis(); - let mut ops: Vec = Vec::new(); - - let mut node_ids = Vec::with_capacity(patch.upsert_nodes.len()); - let mut batch_keys: HashMap<(u32, String), (u64, i64)> = HashMap::new(); - - for input in &patch.upsert_nodes { - let (dense_vector, sparse_vector) = normalize_node_vectors_for_write( - self.manifest.dense_vector.as_ref(), - input.dense_vector.as_ref(), - input.sparse_vector.as_ref(), - )?; - let key_tuple = (input.type_id, input.key.clone()); - let (id, created_at) = if let Some(&(id, created_at)) = batch_keys.get(&key_tuple) { - (id, created_at) - } else if let Some((id, created_at)) = - self.find_existing_node(input.type_id, &input.key)? - { - (id, created_at) - } else { - let id = self.next_node_id; - self.next_node_id += 1; - self.update_next_node_id_seen(); - (id, now) - }; - batch_keys.insert(key_tuple, (id, created_at)); - ops.push(WalOp::UpsertNode(NodeRecord { - id, - type_id: input.type_id, - key: input.key.clone(), - props: input.props.clone(), - created_at, - updated_at: now, - weight: input.weight, - dense_vector, - sparse_vector, - last_write_seq: 0, - })); - node_ids.push(id); + label_catalog_changed: ops.iter().any(|op| { + matches!( + op, + WalOp::EnsureNodeLabel { .. } | WalOp::EnsureEdgeLabel { .. } + ) + }), + }) + } + + fn plan_graph_patch(&mut self, patch: &GraphPatch) -> Result { + let mut validated_node_labels = Vec::with_capacity(patch.upsert_nodes.len()); + let mut normalized_node_vectors = Vec::with_capacity(patch.upsert_nodes.len()); + for input in &patch.upsert_nodes { + validated_node_labels.push(ValidatedNodeLabelList::new( + input.labels.iter().map(String::as_str), + )?); + validate_node_key_for_write(&input.key)?; + normalized_node_vectors.push(normalize_node_vectors_for_write( + self.manifest.dense_vector.as_ref(), + input.dense_vector.as_ref(), + input.sparse_vector.as_ref(), + )?); } + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let node_label_sets = + label_plan.resolve_validated_node_label_sets_for_request(&validated_node_labels)?; + let edge_label_ids = label_plan.resolve_edge_label_ids_for_request( + patch + .upsert_edges + .iter() + .map(|input| input.label.as_str()), + )?; + let now = now_millis(); + let token_op_count = label_plan.token_op_count(); + let label_catalog_changed = token_op_count > 0; + let mut ops: Vec = Vec::with_capacity( + token_op_count + + patch.upsert_nodes.len() + + patch.upsert_edges.len() + + patch.invalidate_edges.len() + + patch.delete_edge_ids.len() + + patch.delete_node_ids.len(), + ); + label_plan.push_token_ops(&mut ops); + drop(label_plan); + drop(catalog); + + let (node_records, node_ids, next_node_id) = self.plan_node_upsert_records( + &patch.upsert_nodes, + &node_label_sets, + normalized_node_vectors, + now, + )?; + ops.extend(node_records.into_iter().map(WalOp::UpsertNode)); + let mut edge_ids = Vec::with_capacity(patch.upsert_edges.len()); + let mut committed_triples: HashMap<(u64, u64, u32), (u64, i64)> = HashMap::new(); + if self.edge_uniqueness && !patch.upsert_edges.is_empty() { + let mut distinct_triples = Vec::new(); + let mut seen_triples = HashSet::new(); + for (input, &label_id) in patch.upsert_edges.iter().zip(edge_label_ids.iter()) { + let triple = (input.from, input.to, label_id); + if seen_triples.insert(triple) { + distinct_triples.push(triple); + } + } + let existing_edges = self.find_existing_edges_batch(&distinct_triples)?; + for (triple, existing) in distinct_triples.into_iter().zip(existing_edges) { + if let Some((id, created_at)) = existing { + committed_triples.insert(triple, (id, created_at)); + } + } + } let mut batch_triples: HashMap<(u64, u64, u32), (u64, i64)> = HashMap::new(); + let mut staged_edge_op_idx: HashMap = HashMap::new(); + let mut staged_incident_edges: HashMap> = HashMap::new(); - for input in &patch.upsert_edges { - let triple = (input.from, input.to, input.type_id); + for (input, label_id) in patch.upsert_edges.iter().zip(edge_label_ids.iter().copied()) { + let triple = (input.from, input.to, label_id); let (id, created_at) = if self.edge_uniqueness { if let Some(&(id, created_at)) = batch_triples.get(&triple) { (id, created_at) - } else if let Some((id, created_at)) = - self.find_existing_edge(input.from, input.to, input.type_id)? - { + } else if let Some(&(id, created_at)) = committed_triples.get(&triple) { (id, created_at) } else { let id = self.next_edge_id; @@ -435,11 +1174,11 @@ impl EngineCore { if self.edge_uniqueness { batch_triples.insert(triple, (id, created_at)); } - ops.push(WalOp::UpsertEdge(EdgeRecord { + let edge = EdgeRecord { id, from: input.from, to: input.to, - type_id: input.type_id, + label_id: label_id, props: input.props.clone(), created_at, updated_at: now, @@ -447,29 +1186,81 @@ impl EngineCore { valid_from: input.valid_from.unwrap_or(created_at), valid_to: input.valid_to.unwrap_or(i64::MAX), last_write_seq: 0, - })); + }; + staged_incident_edges.entry(edge.from).or_default().push(id); + if edge.to != edge.from { + staged_incident_edges.entry(edge.to).or_default().push(id); + } + let op_idx = ops.len(); + ops.push(WalOp::UpsertEdge(edge)); + staged_edge_op_idx.insert(id, op_idx); edge_ids.push(id); } if !patch.invalidate_edges.is_empty() { - let inv_ids: Vec = patch.invalidate_edges.iter().map(|&(id, _)| id).collect(); - let inv_edges = self.get_edges(&inv_ids)?; - for (&(_, valid_to), opt_edge) in patch.invalidate_edges.iter().zip(inv_edges) { - if let Some(edge) = opt_edge { - ops.push(WalOp::UpsertEdge(EdgeRecord { - updated_at: now, - valid_to, - ..edge - })); + let mut inv_lookup_ids = Vec::new(); + let mut inv_lookup_positions = HashMap::new(); + for &(id, _) in &patch.invalidate_edges { + if !staged_edge_op_idx.contains_key(&id) + && !inv_lookup_positions.contains_key(&id) + { + inv_lookup_positions.insert(id, inv_lookup_ids.len()); + inv_lookup_ids.push(id); + } + } + let committed_inv_edges = self.get_edges(&inv_lookup_ids)?; + + for &(id, valid_to) in &patch.invalidate_edges { + if let Some(&op_idx) = staged_edge_op_idx.get(&id) { + let mut edge = match ops.get(op_idx) { + Some(WalOp::UpsertEdge(edge)) => edge.clone(), + _ => { + return Err(EngineError::InvalidOperation( + "staged edge overlay pointed at a non-edge WAL op".into(), + )); + } + }; + edge.updated_at = now; + edge.valid_to = valid_to; + let updated_op_idx = ops.len(); + ops.push(WalOp::UpsertEdge(edge)); + staged_edge_op_idx.insert(id, updated_op_idx); + continue; + } + + if let Some(&idx) = inv_lookup_positions.get(&id) { + if let Some(edge) = committed_inv_edges[idx].as_ref() { + let updated = EdgeRecord { + updated_at: now, + valid_to, + ..edge.clone() + }; + staged_incident_edges + .entry(updated.from) + .or_default() + .push(updated.id); + if updated.to != updated.from { + staged_incident_edges + .entry(updated.to) + .or_default() + .push(updated.id); + } + let op_idx = ops.len(); + ops.push(WalOp::UpsertEdge(updated)); + staged_edge_op_idx.insert(id, op_idx); + } } } } + let mut deleted_edge_ids = HashSet::new(); for &eid in &patch.delete_edge_ids { - ops.push(WalOp::DeleteEdge { - id: eid, - deleted_at: now, - }); + if deleted_edge_ids.insert(eid) { + ops.push(WalOp::DeleteEdge { + id: eid, + deleted_at: now, + }); + } } let patch_tombstones = if patch.delete_node_ids.is_empty() { @@ -481,10 +1272,22 @@ impl EngineCore { let ts = patch_tombstones.as_ref().map(|(dn, de)| (dn, de)); let incident = self.neighbors_raw(nid, Direction::Both, None, 0, None, None, ts)?; for entry in &incident { - ops.push(WalOp::DeleteEdge { - id: entry.edge_id, - deleted_at: now, - }); + if deleted_edge_ids.insert(entry.edge_id) { + ops.push(WalOp::DeleteEdge { + id: entry.edge_id, + deleted_at: now, + }); + } + } + if let Some(staged_incident) = staged_incident_edges.get(&nid) { + for &eid in staged_incident { + if deleted_edge_ids.insert(eid) { + ops.push(WalOp::DeleteEdge { + id: eid, + deleted_at: now, + }); + } + } } ops.push(WalOp::DeleteNode { id: nid, @@ -492,43 +1295,56 @@ impl EngineCore { }); } + if next_node_id != self.next_node_id { + self.next_node_id = next_node_id; + self.update_next_node_id_seen(); + } + Ok(CoreWritePlan { ops, reply: CoreWriteReply::PatchResult(PatchResult { node_ids, edge_ids }), auto_flush: true, track_ids: false, + label_catalog_changed, }) } fn plan_prune(&mut self, policy: &PrunePolicy) -> Result { - if policy.max_age_ms.is_none() && policy.max_weight.is_none() { - return Err(EngineError::InvalidOperation( - "Prune policy must set at least max_age_ms or max_weight".to_string(), - )); - } - if let Some(age) = policy.max_age_ms { - if age <= 0 { - return Err(EngineError::InvalidOperation( - "max_age_ms must be positive".to_string(), - )); - } - } + validate_prune_policy(policy)?; + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let label_id = policy + .label + .as_deref() + .map(|label| label_plan.resolve_node_label_for_write(label)) + .transpose()?; + let token_op_count = label_plan.token_op_count(); + let label_catalog_changed = token_op_count > 0; + let mut ops = Vec::with_capacity(token_op_count); + label_plan.push_token_ops(&mut ops); + drop(label_plan); + drop(catalog); + let resolved_policy = ResolvedPrunePolicy { + max_age_ms: policy.max_age_ms, + max_weight: policy.max_weight, + label_id, + }; let now = now_millis(); - let targets = self.collect_prune_targets(policy, now)?; + let targets = self.collect_prune_targets(&resolved_policy, now)?; if targets.is_empty() { return Ok(CoreWritePlan { - ops: Vec::new(), + ops, reply: CoreWriteReply::PruneResult(PruneResult { nodes_pruned: 0, edges_pruned: 0, }), auto_flush: true, track_ids: false, + label_catalog_changed, }); } - let mut ops = Vec::new(); let mut edges_seen = NodeIdSet::default(); let prune_tombstones = self.collect_tombstones(); @@ -564,104 +1380,85 @@ impl EngineCore { }), auto_flush: true, track_ids: false, + label_catalog_changed, }) } /// Collect node IDs matching the prune policy by scanning memtable + segments. - /// When `type_id` is set, uses the type index for efficiency. + /// When `label_id` is set, uses the label posting index for efficiency. /// Uses raw (unfiltered) reads. Prune must see ALL nodes, including those /// hidden by registered policies, to ensure correct deletion. fn collect_prune_targets( &self, - policy: &PrunePolicy, + policy: &ResolvedPrunePolicy, now: i64, ) -> Result, EngineError> { let age_cutoff = policy.max_age_ms.map(|age| now - age); - if let Some(type_id) = policy.type_id { - // Use the type index (raw). Must see all nodes including policy-excluded ones - let ids = self.nodes_by_type_raw(type_id)?; - let nodes = self.get_nodes_raw(&ids)?; - let targets = ids - .iter() - .zip(nodes) - .filter_map(|(&id, opt)| { - opt.filter(|n| Self::matches_prune_criteria(n, age_cutoff, policy.max_weight)) - .map(|_| id) - }) - .collect(); - Ok(targets) + if let Some(label_id) = policy.label_id { + // Use the label posting index (raw). Must see all nodes including policy-excluded ones. + // Latest visibility is verified from metadata below. + let ids = self.nodes_by_label_id_raw(label_id)?; + self.collect_prune_targets_from_candidates(ids, policy, age_cutoff) } else { - // Scan all nodes: active memtable first, then immutable memtables - // (newest-first), then segments (newest-first), dedup by ID. - // Flat tombstone set is safe here: monotonic ID allocation guarantees - // tombstoned IDs are never re-upserted, so source precedence doesn't matter. - let mut deleted = self.memtable.collect_deleted_nodes_at(u64::MAX); - for epoch in &self.immutable_epochs { - deleted.extend(epoch.memtable.collect_deleted_nodes_at(u64::MAX)); - } - for seg in &self.segments { - deleted.extend(seg.deleted_node_ids()); - } - + // Scan all sources for candidate IDs without hydrating NodeRecords, then + // latest-verify each candidate through SourceList visibility metadata. let mut seen = NodeIdSet::default(); - let mut targets = Vec::new(); + let mut candidates = Vec::new(); // Active memtable nodes (freshest) - let _ = self.memtable.for_each_visible_node_at(u64::MAX, &mut |node| { - if !deleted.contains(&node.id) - && seen.insert(node.id) - && Self::matches_prune_criteria(node, age_cutoff, policy.max_weight) - { - targets.push(node.id); + for node_id in self.memtable.visible_node_ids_at(self.engine_seq) { + if seen.insert(node_id) { + candidates.push(node_id); } - ControlFlow::Continue(()) - }); + } // Immutable memtable nodes (newest-first) for epoch in &self.immutable_epochs { - let _ = epoch.memtable.for_each_visible_node_at(u64::MAX, &mut |node| { - if !deleted.contains(&node.id) - && seen.insert(node.id) - && Self::matches_prune_criteria(node, age_cutoff, policy.max_weight) - { - targets.push(node.id); + for node_id in epoch.memtable.visible_node_ids_at(self.engine_seq) { + if seen.insert(node_id) { + candidates.push(node_id); } - ControlFlow::Continue(()) - }); + } } - // Segment nodes (newest segments first, skip already-seen) + // Segment node metadata (newest segments first, skip already-seen) for seg in &self.segments { - for node in seg.all_nodes()? { - if !deleted.contains(&node.id) && seen.insert(node.id) - && Self::matches_prune_criteria(&node, age_cutoff, policy.max_weight) { - targets.push(node.id); - } + for index in 0..seg.node_meta_count() as usize { + let meta = seg.node_meta_at(index)?; + if seen.insert(meta.node_id) { + candidates.push(meta.node_id); + } } } - Ok(targets) + self.collect_prune_targets_from_candidates(candidates, policy, age_cutoff) } } - /// Check whether a node matches the prune criteria (AND logic). - fn matches_prune_criteria( - node: &NodeRecord, + fn collect_prune_targets_from_candidates( + &self, + candidates: Vec, + policy: &ResolvedPrunePolicy, age_cutoff: Option, - max_weight: Option, - ) -> bool { - if let Some(cutoff) = age_cutoff { - if node.updated_at >= cutoff { - return false; // Too recent, does not match - } - } - if let Some(max_w) = max_weight { - if node.weight > max_w { - return false; // Weight too high, does not match + ) -> Result, EngineError> { + let visibility = self.sources().find_node_visibility_meta(&candidates)?; + let mut targets = Vec::new(); + for (&node_id, state) in candidates.iter().zip(visibility.iter()) { + if let NodeVisibilityState::Live(meta) = state { + if matches_prune_cutoff( + &meta.label_ids, + meta.updated_at, + meta.weight, + age_cutoff, + policy.max_weight, + policy.label_id, + ) { + targets.push(node_id); + } } } - true + Ok(targets) } // --- Named prune policies (compaction-filter auto-prune) --- @@ -674,45 +1471,35 @@ impl EngineCore { name: &str, policy: PrunePolicy, ) -> Result { - // Validate: at least one substantive filter - if policy.max_age_ms.is_none() && policy.max_weight.is_none() { - return Err(EngineError::InvalidOperation( - "Prune policy must set at least max_age_ms or max_weight".to_string(), - )); - } - if let Some(age) = policy.max_age_ms { - if age <= 0 { - return Err(EngineError::InvalidOperation( - "max_age_ms must be positive".to_string(), - )); - } - } - if let Some(w) = policy.max_weight { - if w.is_nan() || w < 0.0 { - return Err(EngineError::InvalidOperation( - "max_weight must be non-negative and not NaN".to_string(), - )); - } + validate_prune_policy(&policy)?; + + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let mut node_labels_to_stage = Vec::new(); + if let Some(label) = policy.label.as_deref() { + let label_id = label_plan.resolve_node_label_for_write(label)?; + node_labels_to_stage.push((label.to_string(), label_id)); } + let (node_labels_to_create, edge_labels_to_create) = label_plan.token_creations(); + drop(label_plan); + drop(catalog); if self .manifest .prune_policies .get(name) - .is_some_and(|existing| { - existing.max_age_ms == policy.max_age_ms - && existing.max_weight == policy.max_weight - && existing.type_id == policy.type_id - }) + .is_some_and(|existing| existing == &policy) { return Ok(PublishImpact::NoPublish); } let name = name.to_string(); self.with_runtime_manifest_write(|manifest| { + stage_label_tokens_in_manifest(manifest, &node_labels_to_stage, &[])?; manifest.prune_policies.insert(name, policy); Ok(()) })?; + self.apply_manifest_token_creations(&node_labels_to_create, &edge_labels_to_create)?; Ok(PublishImpact::RebuildSources) } @@ -739,30 +1526,79 @@ impl EngineCore { } /// List all registered prune policies. - pub fn list_prune_policies(&self) -> Vec<(String, PrunePolicy)> { + pub fn list_prune_policies(&self) -> Result, EngineError> { + let catalog = self.label_catalog.read().unwrap(); self.manifest .prune_policies .iter() - .map(|(k, v)| (k.clone(), v.clone())) + .map(|(name, policy)| { + let resolved = resolve_manifest_prune_policy(policy, &catalog)?; + let policy = public_prune_policy_from_resolved(&resolved, &catalog)?; + Ok(PrunePolicyInfo { + name: name.clone(), + policy, + }) + }) .collect() } - fn node_property_index_info(entry: &SecondaryIndexManifestEntry) -> NodePropertyIndexInfo { - match &entry.target { - SecondaryIndexTarget::NodeProperty { type_id, prop_key } => NodePropertyIndexInfo { + fn node_property_index_info( + entry: &SecondaryIndexManifestEntry, + catalog: &impl LabelCatalogLookup, + ) -> Result { + Ok(match &entry.target { + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => NodePropertyIndexInfo { index_id: entry.index_id, - type_id: *type_id, + label: catalog + .node_label(*label_id) + .map(str::to_string) + .ok_or_else(|| { + EngineError::ManifestError(format!( + "node property index {} references missing node label label_id {}", + entry.index_id, label_id + )) + })?, prop_key: prop_key.clone(), kind: entry.kind.clone(), state: entry.state, last_error: entry.last_error.clone(), }, - } + SecondaryIndexTarget::EdgeProperty { .. } => { + unreachable!("node_property_index_info called with EdgeProperty target") + } + }) + } + + fn edge_property_index_info( + entry: &SecondaryIndexManifestEntry, + catalog: &impl LabelCatalogLookup, + ) -> Result { + Ok(match &entry.target { + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => EdgePropertyIndexInfo { + index_id: entry.index_id, + label: catalog + .edge_label(*label_id) + .map(str::to_string) + .ok_or_else(|| { + EngineError::ManifestError(format!( + "edge property index {} references missing edge-label label_id {}", + entry.index_id, label_id + )) + })?, + prop_key: prop_key.clone(), + kind: entry.kind.clone(), + state: entry.state, + last_error: entry.last_error.clone(), + }, + SecondaryIndexTarget::NodeProperty { .. } => { + unreachable!("edge_property_index_info called with NodeProperty target") + } + }) } pub fn ensure_node_property_index( &mut self, - type_id: u32, + label: &str, prop_key: &str, kind: SecondaryIndexKind, ) -> Result<(NodePropertyIndexInfo, PublishImpact), EngineError> { @@ -772,22 +1608,33 @@ impl EngineCore { Retry, } + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let label_id = label_plan.resolve_node_label_for_write(label)?; + let node_labels_to_stage = vec![(label.to_string(), label_id)]; + let (node_labels_to_create, edge_labels_to_create) = label_plan.token_creations(); + drop(label_plan); + drop(catalog); let prop_key = prop_key.to_string(); let (entry, outcome) = self.with_runtime_manifest_write(|manifest| { + stage_label_tokens_in_manifest(manifest, &node_labels_to_stage, &[])?; if matches!(&kind, SecondaryIndexKind::Range { .. }) { for existing in &manifest.secondary_indexes { let SecondaryIndexTarget::NodeProperty { - type_id: existing_type_id, + label_id: existing_label_id, prop_key: existing_prop_key, - } = &existing.target; - if *existing_type_id == type_id + } = &existing.target + else { + continue; + }; + if *existing_label_id == label_id && existing_prop_key == &prop_key && matches!(existing.kind, SecondaryIndexKind::Range { .. }) && existing.kind != kind { return Err(EngineError::InvalidOperation(format!( "property index ({}, {}) already has a range declaration with a different domain", - type_id, prop_key + label_id, prop_key ))); } } @@ -796,7 +1643,7 @@ impl EngineCore { if let Some(existing) = manifest.secondary_indexes.iter_mut().find(|entry| { entry.target == SecondaryIndexTarget::NodeProperty { - type_id, + label_id, prop_key: prop_key.clone(), } && entry.kind == kind @@ -812,7 +1659,7 @@ impl EngineCore { let entry = SecondaryIndexManifestEntry { index_id: manifest.next_secondary_index_id, target: SecondaryIndexTarget::NodeProperty { - type_id, + label_id, prop_key: prop_key.clone(), }, kind: kind.clone(), @@ -823,6 +1670,7 @@ impl EngineCore { manifest.secondary_indexes.push(entry.clone()); Ok((entry, EnsureOutcome::New)) })?; + self.apply_manifest_token_creations(&node_labels_to_create, &edge_labels_to_create)?; let publish_impact = match outcome { EnsureOutcome::Existing => PublishImpact::NoPublish, @@ -845,21 +1693,32 @@ impl EngineCore { } }; - Ok((Self::node_property_index_info(&entry), publish_impact)) + let catalog = self.label_catalog.read().unwrap(); + Ok(( + Self::node_property_index_info(&entry, &*catalog)?, + publish_impact, + )) } pub fn drop_node_property_index( &mut self, - type_id: u32, + label: &str, prop_key: &str, kind: SecondaryIndexKind, ) -> Result<(bool, PublishImpact), EngineError> { + let label_id = { + let catalog = self.label_catalog.read().unwrap(); + let Some(label_id) = resolve_node_label_for_read(&catalog, label)? else { + return Ok((false, PublishImpact::NoPublish)); + }; + label_id + }; let prop_key = prop_key.to_string(); let removed = self.with_runtime_manifest_write(|manifest| { let idx = manifest.secondary_indexes.iter().position(|entry| { entry.target == SecondaryIndexTarget::NodeProperty { - type_id, + label_id, prop_key: prop_key.clone(), } && entry.kind == kind @@ -873,38 +1732,310 @@ impl EngineCore { self.rebuild_secondary_index_catalog()?; self.remove_secondary_index_entry_from_memtables(entry.index_id)?; - self.enqueue_secondary_index_job(SecondaryIndexJob::DropCleanup { - index_id: entry.index_id, - }); + self.enqueue_secondary_index_job(SecondaryIndexJob::DropCleanup { entry }); + Ok((true, PublishImpact::RebuildSources)) + } + + pub fn ensure_edge_property_index( + &mut self, + label: &str, + prop_key: &str, + kind: SecondaryIndexKind, + ) -> Result<(EdgePropertyIndexInfo, PublishImpact), EngineError> { + enum EnsureOutcome { + Existing, + New, + Retry, + } + + let catalog = self.label_catalog.read().unwrap(); + let mut label_plan = LabelResolutionPlan::from_catalog(&catalog); + let label_id = label_plan.resolve_edge_label_for_write(label)?; + let edge_labels_to_stage = vec![(label.to_string(), label_id)]; + let (node_labels_to_create, edge_labels_to_create) = label_plan.token_creations(); + drop(label_plan); + drop(catalog); + let prop_key = prop_key.to_string(); + let (entry, outcome) = self.with_runtime_manifest_write(|manifest| { + stage_label_tokens_in_manifest(manifest, &[], &edge_labels_to_stage)?; + if matches!(&kind, SecondaryIndexKind::Range { .. }) { + for existing in &manifest.secondary_indexes { + let SecondaryIndexTarget::EdgeProperty { + label_id: existing_label_id, + prop_key: existing_prop_key, + } = &existing.target + else { + continue; + }; + if *existing_label_id == label_id + && existing_prop_key == &prop_key + && matches!(existing.kind, SecondaryIndexKind::Range { .. }) + && existing.kind != kind + { + return Err(EngineError::InvalidOperation(format!( + "edge property index ({}, {}) already has a range declaration with a different domain", + label_id, prop_key + ))); + } + } + } + + if let Some(existing) = manifest.secondary_indexes.iter_mut().find(|entry| { + entry.target + == SecondaryIndexTarget::EdgeProperty { + label_id: label_id, + prop_key: prop_key.clone(), + } + && entry.kind == kind + }) { + if existing.state == SecondaryIndexState::Failed { + existing.state = SecondaryIndexState::Building; + existing.last_error = None; + return Ok((existing.clone(), EnsureOutcome::Retry)); + } + return Ok((existing.clone(), EnsureOutcome::Existing)); + } + + let entry = SecondaryIndexManifestEntry { + index_id: manifest.next_secondary_index_id, + target: SecondaryIndexTarget::EdgeProperty { + label_id: label_id, + prop_key: prop_key.clone(), + }, + kind: kind.clone(), + state: SecondaryIndexState::Building, + last_error: None, + }; + manifest.next_secondary_index_id = manifest.next_secondary_index_id.saturating_add(1); + manifest.secondary_indexes.push(entry.clone()); + Ok((entry, EnsureOutcome::New)) + })?; + self.apply_manifest_token_creations(&node_labels_to_create, &edge_labels_to_create)?; + + let publish_impact = match outcome { + EnsureOutcome::Existing => PublishImpact::NoPublish, + EnsureOutcome::New => { + self.rebuild_secondary_index_catalog()?; + self.seed_secondary_index_entry(&entry)?; + self.enqueue_secondary_index_job(SecondaryIndexJob::Build { + index_id: entry.index_id, + }); + PublishImpact::RebuildSources + } + EnsureOutcome::Retry => { + self.rebuild_secondary_index_catalog()?; + self.remove_secondary_index_entry_from_memtables(entry.index_id)?; + self.seed_secondary_index_entry(&entry)?; + self.enqueue_secondary_index_job(SecondaryIndexJob::Build { + index_id: entry.index_id, + }); + PublishImpact::RebuildSources + } + }; + + let catalog = self.label_catalog.read().unwrap(); + Ok(( + Self::edge_property_index_info(&entry, &*catalog)?, + publish_impact, + )) + } + + pub fn drop_edge_property_index( + &mut self, + label: &str, + prop_key: &str, + kind: SecondaryIndexKind, + ) -> Result<(bool, PublishImpact), EngineError> { + let label_id = { + let catalog = self.label_catalog.read().unwrap(); + let Some(label_id) = resolve_edge_label_for_read(&catalog, label)? else { + return Ok((false, PublishImpact::NoPublish)); + }; + label_id + }; + let prop_key = prop_key.to_string(); + let removed = self.with_runtime_manifest_write(|manifest| { + let idx = manifest.secondary_indexes.iter().position(|entry| { + entry.target + == SecondaryIndexTarget::EdgeProperty { + label_id: label_id, + prop_key: prop_key.clone(), + } + && entry.kind == kind + }); + Ok(idx.map(|idx| manifest.secondary_indexes.remove(idx))) + })?; + + let Some(entry) = removed else { + return Ok((false, PublishImpact::NoPublish)); + }; + + self.rebuild_secondary_index_catalog()?; + self.remove_secondary_index_entry_from_memtables(entry.index_id)?; + self.enqueue_secondary_index_job(SecondaryIndexJob::DropCleanup { entry }); Ok((true, PublishImpact::RebuildSources)) } + fn require_node_label_token_for_numeric_stub(&self, label_id: u32) -> Result<(), EngineError> { + if self + .label_catalog + .read() + .unwrap() + .node_id_to_label + .contains_key(&label_id) + { + Ok(()) + } else { + Err(EngineError::InvalidOperation(format!( + "numeric node label_id {label_id} is not present in the node label catalog; call ensure_node_label first" + ))) + } + } + + fn require_edge_label_token_for_numeric_stub(&self, label_id: u32) -> Result<(), EngineError> { + if self + .label_catalog + .read() + .unwrap() + .edge_id_to_label + .contains_key(&label_id) + { + Ok(()) + } else { + Err(EngineError::InvalidOperation(format!( + "numeric edge-label label_id {label_id} is not present in the edge-label catalog; call ensure_edge_label first" + ))) + } + } + + #[allow(dead_code)] + fn validate_wal_op_label_tokens_for_write(&self, op: &WalOp) -> Result<(), EngineError> { + match op { + WalOp::UpsertNode(node) => { + for &label_id in node.label_ids.as_slice() { + self.require_node_label_token_for_numeric_stub(label_id)?; + } + Ok(()) + } + WalOp::UpsertEdge(edge) => self.require_edge_label_token_for_numeric_stub(edge.label_id), + WalOp::EnsureNodeLabel { label, label_id } => { + validate_label_token_name(label)?; + let catalog = self.label_catalog.read().unwrap(); + if let Some(existing_id) = catalog.node_label_to_id.get(label) { + if *existing_id != *label_id { + return Err(EngineError::InvalidOperation(format!( + "node label token conflict: label '{label}' already has label_id {existing_id}, not {label_id}" + ))); + } + return Ok(()); + } + if catalog.node_id_to_label.contains_key(label_id) { + return Err(EngineError::InvalidOperation(format!( + "node label token conflict: label_id {label_id} is already assigned" + ))); + } + Ok(()) + } + WalOp::EnsureEdgeLabel { label, label_id } => { + validate_label_token_name(label)?; + let catalog = self.label_catalog.read().unwrap(); + if let Some(existing_id) = catalog.edge_label_to_id.get(label) { + if *existing_id != *label_id { + return Err(EngineError::InvalidOperation(format!( + "edge-label token conflict: edge label '{label}' already has label_id {existing_id}, not {label_id}" + ))); + } + return Ok(()); + } + if catalog.edge_id_to_label.contains_key(label_id) { + return Err(EngineError::InvalidOperation(format!( + "edge-label token conflict: label_id {label_id} is already assigned" + ))); + } + Ok(()) + } + WalOp::DeleteNode { .. } | WalOp::DeleteEdge { .. } => Ok(()), + WalOp::BeginAtomicBatch { .. } | WalOp::CommitAtomicBatch { .. } => Err( + EngineError::InvalidOperation( + "WAL atomic batch markers cannot be submitted as write ops".into(), + ), + ), + } + } + // --- Segment-aware dedup lookups (for upsert) --- - /// Look up a node by (type_id, key) across memtable + segments. + /// Look up a node by (label_id, key) across memtable + segments. /// Used by upsert_node for dedup. Uses raw (unfiltered) lookup to prevent /// policy-excluded nodes from being treated as "not found" (which would /// allocate a duplicate ID, causing silent data corruption). fn find_existing_node( &self, - type_id: u32, + label_id: u32, key: &str, ) -> Result, EngineError> { Ok(self - .get_node_by_key_raw(type_id, key)? + .get_node_by_label_key_raw(label_id, key)? .map(|n| (n.id, n.created_at))) } - /// Look up an edge by (from, to, type_id) across memtable + segments. + fn find_existing_node_for_label_set( + &self, + label_ids: NodeLabelSet, + key: &str, + ) -> Result, EngineError> { + let key_lookups: Vec<(u32, &str)> = label_ids + .as_slice() + .iter() + .map(|&label_id| (label_id, key)) + .collect(); + let existing = self.find_existing_nodes_batch(&key_lookups)?; + let mut winner: Option<(u64, i64)> = None; + for node in existing.into_iter().flatten() { + match winner { + Some((winner_id, _)) if winner_id != node.0 => { + return Err(node_key_conflict_error(key, winner_id, node.0)); + } + None => winner = Some(node), + _ => {} + } + } + Ok(winner) + } + + fn find_existing_nodes_batch( + &self, + keys: &[(u32, &str)], + ) -> Result>, EngineError> { + Ok(self + .get_nodes_by_label_keys_raw(keys)? + .into_iter() + .map(|node| node.map(|n| (n.id, n.created_at))) + .collect()) + } + + /// Look up an edge by (from, to, label_id) across memtable + segments. /// Used by upsert_edge for uniqueness enforcement. Delegates to public get_edge_by_triple. fn find_existing_edge( &self, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result, EngineError> { Ok(self - .get_edge_by_triple(from, to, type_id)? + .get_edge_by_triple(from, to, label_id)? .map(|e| (e.id, e.created_at))) } + + fn find_existing_edges_batch( + &self, + triples: &[(u64, u64, u32)], + ) -> Result>, EngineError> { + Ok(self + .sources() + .find_edges_by_triples(triples)? + .into_iter() + .map(|edge| edge.map(|e| (e.id, e.created_at))) + .collect()) + } } diff --git a/src/lib.rs b/src/lib.rs index e1fe4c9..49c4889 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ //! //! An absurdly fast embedded graph database. Pure Rust, sub-microsecond reads. //! -//! OverGraph stores typed nodes and edges with schemaless properties (MessagePack), +//! OverGraph stores labeled nodes and edges with schemaless properties (MessagePack), //! temporal validity windows, exponential decay scoring, and automatic retention //! policies. It runs inside your process with no separate server or network calls. //! @@ -13,7 +13,7 @@ //! //! use std::path::Path; //! let mut db = DatabaseEngine::open(Path::new("./my-db"), &DbOptions::default()).unwrap(); -//! let id = db.upsert_node(1, "user:alice", UpsertNodeOptions::default()).unwrap(); +//! let id = db.upsert_node("User", "alice", UpsertNodeOptions::default()).unwrap(); //! let neighbors = db.neighbors(id, &NeighborOptions { limit: Some(50), ..Default::default() }).unwrap(); //! db.close().unwrap(); //! ``` @@ -33,35 +33,287 @@ pub mod engine; pub mod error; pub mod types; -// Internal modules: accessible within the workspace (connectors, CLI binaries) -// but hidden from public documentation since they are implementation details. +// Internal modules stay crate-private so public Rust callers cannot depend on +// storage, WAL, segment, or planner implementation details. #[doc(hidden)] pub(crate) mod degree_cache; #[doc(hidden)] -pub mod dense_hnsw; +pub(crate) mod dense_hnsw; #[doc(hidden)] -pub mod encoding; +pub(crate) mod edge_metadata; +#[doc(hidden)] +pub(crate) mod encoding; +// Diagnostic exception: `overgraph-inspect` uses the read-only manifest loader, +// and `DatabaseEngine::manifest()` remains an explicit introspection surface. #[doc(hidden)] pub mod manifest; #[doc(hidden)] -pub mod memtable; +pub(crate) mod memtable; #[doc(hidden)] pub(crate) mod parallel; #[doc(hidden)] pub(crate) mod planner_stats; #[doc(hidden)] -pub mod segment_reader; +pub(crate) mod scrub; +#[doc(hidden)] +pub(crate) mod segment_components; +#[doc(hidden)] +pub(crate) mod segment_reader; #[doc(hidden)] -pub mod segment_writer; +pub(crate) mod segment_writer; #[doc(hidden)] -pub mod source_list; +pub(crate) mod source_list; #[doc(hidden)] -pub mod sparse_postings; +pub(crate) mod sparse_postings; #[doc(hidden)] -pub mod wal; +pub(crate) mod wal; #[doc(hidden)] -pub mod wal_sync; +pub(crate) mod wal_sync; pub use engine::{DatabaseEngine, WriteTxn}; pub use error::EngineError; -pub use types::*; +pub use types::{ + canonicalize_sparse_vector, canonicalize_sparse_vector_owned, hash_prop_key, hash_prop_value, + validate_dense_vector, validate_dense_vector_config, AdjacencyExport, AllShortestPathsOptions, + CompactionPhase, CompactionProgress, CompactionStats, ComponentOptions, ComponentScrubFinding, + DbOptions, DbStats, DegreeOptions, DenseMetric, DenseVector, DenseVectorConfig, Direction, + EdgeFilterExpr, EdgeInput, EdgeLabelInfo, EdgePattern, EdgePropertyIndexInfo, EdgeQuery, + EdgeQueryOrder, EdgeView, ExportEdge, ExportOptions, FusionMode, GraphPatch, GraphPatternQuery, + HnswConfig, IntoNodeLabels, IsConnectedOptions, LabelMatchMode, ManifestState, NeighborEntry, + NeighborOptions, NodeFilterExpr, NodeIdBuildHasher, NodeIdHasher, NodeIdMap, NodeIdSet, + NodeInput, NodeKeyQuery, NodeLabelFilter, NodeLabelInfo, NodePattern, NodePropertyIndexInfo, + NodeQuery, NodeQueryOrder, NodeView, PageRequest, PageResult, PatchResult, PatternOrder, + PprAlgorithm, PprApproxMeta, PprOptions, PprResult, PropValue, PropertyRangeBound, + PropertyRangeCursor, PropertyRangePageRequest, PropertyRangePageResult, PrunePolicy, + PrunePolicyInfo, PruneResult, QueryEdgeIdsResult, QueryEdgesResult, QueryMatch, + QueryNodeIdsResult, QueryNodesResult, QueryPatternResult, QueryPlan, QueryPlanKind, + QueryPlanNode, QueryPlanNote, QueryPlanPublicInputs, QueryPlanPublicName, QueryPlanWarning, + ScoringMode, ScrubFindingType, ScrubReport, SecondaryIndexKind, SecondaryIndexManifestEntry, + SecondaryIndexRangeDomain, SecondaryIndexState, SecondaryIndexTarget, SegmentInfo, + SegmentScrubResult, ShortestPath, ShortestPathOptions, SparseVector, Subgraph, SubgraphOptions, + TombstoneEntry, TopKOptions, TraversalCursor, TraversalHit, TraversalPageResult, + TraverseOptions, TxnCommitResult, TxnEdgeRef, TxnEdgeView, TxnIntent, TxnLocalRef, TxnNodeRef, + TxnNodeView, UpsertEdgeOptions, UpsertNodeOptions, VectorHit, VectorSearchMode, + VectorSearchRequest, VectorSearchScope, WalSyncMode, DEFAULT_DENSE_EF_SEARCH, +}; + +#[cfg(test)] +mod public_api_boundary_tests { + fn source(path: &str) -> String { + std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(path)) + .unwrap() + } + + fn rust_source_paths() -> Vec { + fn collect(dir: &std::path::Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + collect(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } + } + + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut paths = Vec::new(); + for dir in ["src", "tests", "benches"] { + collect(&root.join(dir), &mut paths); + } + paths + } + + fn assert_files_do_not_contain(paths: &[std::path::PathBuf], patterns: &[String]) { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + for path in paths { + let contents = std::fs::read_to_string(path).unwrap(); + let display = path.strip_prefix(root).unwrap_or(path).display(); + for pattern in patterns { + assert!( + !contents.contains(pattern), + "`{pattern}` must not remain in Rust active edge-label source ({display})" + ); + } + } + } + + #[test] + fn internal_numeric_records_are_not_publicly_exported() { + let lib = source("src/lib.rs"); + let types = source("src/types.rs"); + + assert!( + !lib.contains(concat!("pub use types", "::*")), + "public API must explicitly re-export stable DTOs and not glob-export internal records" + ); + for forbidden in [ + concat!("pub struct ", "NodeRecord"), + concat!("pub struct ", "EdgeRecord"), + ] { + assert!( + !types.contains(forbidden), + "`{forbidden}` would expose internal numeric label/type records" + ); + } + for required in [ + concat!("pub(crate) struct ", "NodeRecord"), + concat!("pub(crate) struct ", "EdgeRecord"), + ] { + assert!( + types.contains(required), + "`{required}` must remain the internal storage/WAL record boundary" + ); + } + } + + #[test] + fn rust_active_edge_label_id_vocabulary_has_no_backend_type_terms() { + let paths = rust_source_paths(); + assert_files_do_not_contain( + &paths, + &[ + concat!("EDGE", "_TYPE").to_string(), + concat!("Edge", "Type").to_string(), + concat!("edge", "_type").to_string(), + concat!("edge ", "type").to_string(), + concat!("edge", "-", "type").to_string(), + concat!("edges_by", "_type").to_string(), + concat!("visible_edges_by", "_type").to_string(), + concat!("type", "_edge_index").to_string(), + concat!("type", "_ids").to_string(), + concat!("type", "Id").to_string(), + concat!("type", " IDs").to_string(), + concat!("distinct ", "type").to_string(), + concat!("these ", "types").to_string(), + concat!("filtered", "_types").to_string(), + concat!("filtered", "_type", "_labels").to_string(), + concat!("Type ", "filter works").to_string(), + concat!("let ", "typed").to_string(), + concat!(":", "type", ":").to_string(), + concat!(":", "types", ":{").to_string(), + ], + ); + + let segment_reader = source("src/segment_reader.rs"); + for pattern in [ + concat!("entry", "_type"), + concat!("let e", "_type"), + concat!("match e", "_type"), + ] { + assert!( + !segment_reader.contains(pattern), + "`{pattern}` must not remain in segment edge label readers" + ); + } + } + + #[test] + fn implementation_modules_are_not_public_api() { + let lib = source("src/lib.rs"); + for forbidden in [ + concat!("pub mod ", "dense_hnsw;"), + concat!("pub mod ", "encoding;"), + concat!("pub mod ", "memtable;"), + concat!("pub mod ", "segment_reader;"), + concat!("pub mod ", "segment_writer;"), + concat!("pub mod ", "source_list;"), + concat!("pub mod ", "sparse_postings;"), + concat!("pub mod ", "wal;"), + concat!("pub mod ", "wal_sync;"), + ] { + assert!( + !lib.contains(forbidden), + "`{forbidden}` would expose implementation internals as Rust public API" + ); + } + } + + #[test] + fn manifest_module_public_surface_stays_read_only_diagnostic_only() { + let manifest = source("src/manifest.rs"); + for forbidden in [ + concat!("pub fn ", "write_manifest"), + concat!("pub fn ", "load_manifest("), + concat!("pub fn ", "default_manifest"), + ] { + assert!( + !manifest.contains(forbidden), + "`{forbidden}` must stay crate-private; manifest diagnostics expose read-only loading only" + ); + } + assert!( + manifest.contains(concat!("pub fn ", "load_manifest_readonly")), + "the inspect binary relies on the explicit read-only diagnostic manifest loader" + ); + } + + #[test] + fn rust_public_edge_vocabulary_uses_labels() { + let lib = source("src/lib.rs"); + let types = source("src/types.rs"); + let engine = source("src/engine/mod.rs"); + let read = source("src/engine/read.rs"); + let write = source("src/engine/write.rs"); + let txn = source("src/engine/txn.rs"); + let manifest = source("src/manifest.rs"); + + for forbidden in [ + concat!("Edge", "TypeInfo"), + concat!("pub edge", "_", "type:"), + concat!("pub edge", "_", "type", "_filter:"), + concat!("pub edge", "_", "type", "_index:"), + concat!("pub fn ensure_edge", "_", "type"), + concat!("pub fn get_edge", "_", "type("), + concat!("pub fn list_edge", "_", "types"), + concat!("pub fn ", "edges_by_label_id"), + concat!("pub fn ", "get_edges_by_label_id"), + concat!("pub fn ", "count_edges_by_label_id"), + ] { + assert!( + !lib.contains(forbidden) + && !types.contains(forbidden) + && !engine.contains(forbidden), + "`{forbidden}` must not remain in the Rust public edge-label API" + ); + } + + for forbidden in [ + concat!("edge ", "type"), + concat!("edge", "-", "type"), + concat!("edge ", "type token"), + concat!("edge ", "type catalog"), + concat!("resolved by edge ", "type"), + concat!("transaction edge ", "type"), + ] { + assert!( + !types.contains(forbidden) + && !engine.contains(forbidden) + && !read.contains(forbidden) + && !write.contains(forbidden) + && !txn.contains(forbidden) + && !manifest.contains(forbidden), + "`{forbidden}` must not remain in public-facing Rust edge-label diagnostics or docs" + ); + } + + for required in [ + concat!("Edge", "LabelInfo"), + concat!("pub label: String"), + concat!("pub edge_label_filter: Option>"), + concat!("pub edge_label_index: u32"), + concat!("pub fn ensure_edge_label"), + concat!("pub fn get_edge_label_id"), + concat!("pub fn get_edge_label("), + concat!("pub fn list_edge_labels"), + concat!("pub fn edges_by_label"), + concat!("pub fn get_edges_by_label"), + concat!("pub fn count_edges_by_label"), + ] { + assert!( + lib.contains(required) || types.contains(required) || engine.contains(required), + "`{required}` should exist in the Rust public edge-label API" + ); + } + } +} diff --git a/src/manifest.rs b/src/manifest.rs index 51167e5..d9b471c 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -1,5 +1,6 @@ use crate::error::EngineError; -use crate::types::ManifestState; +use crate::segment_writer::SEGMENT_FORMAT_VERSION; +use crate::types::{validate_label_token_name, ManifestState, LABEL_TOKEN_SCHEMA_VERSION}; use std::fs; use std::io::Write; use std::path::Path; @@ -13,7 +14,7 @@ const MANIFEST_PREV: &str = "manifest.prev"; /// 2. If manifest.current exists, rename to manifest.prev /// 3. Rename manifest.tmp → manifest.current /// 4. fsync the directory to make the rename durable -pub fn write_manifest(db_dir: &Path, state: &ManifestState) -> Result<(), EngineError> { +pub(crate) fn write_manifest(db_dir: &Path, state: &ManifestState) -> Result<(), EngineError> { let tmp_path = db_dir.join(MANIFEST_TMP); let current_path = db_dir.join(MANIFEST_CURRENT); let prev_path = db_dir.join(MANIFEST_PREV); @@ -47,7 +48,7 @@ pub fn write_manifest(db_dir: &Path, state: &ManifestState) -> Result<(), Engine /// 1. manifest.current (normal path) /// 2. manifest.tmp (crash between rename steps; tmp has the newest state) /// 3. manifest.prev (fallback if current is corrupt) -pub fn load_manifest(db_dir: &Path) -> Result, EngineError> { +pub(crate) fn load_manifest(db_dir: &Path) -> Result, EngineError> { let current_path = db_dir.join(MANIFEST_CURRENT); let tmp_path = db_dir.join(MANIFEST_TMP); let prev_path = db_dir.join(MANIFEST_PREV); @@ -82,7 +83,10 @@ fn try_load_manifest_file(path: &Path) -> Result, EngineEr let content = fs::read_to_string(path)?; match serde_json::from_str::(&content) { - Ok(state) => Ok(Some(state)), + Ok(state) => { + validate_manifest_identity(&state)?; + Ok(Some(state)) + } Err(e) => { eprintln!("warning: corrupt manifest at {}: {}", path.display(), e); Ok(None) @@ -90,6 +94,79 @@ fn try_load_manifest_file(path: &Path) -> Result, EngineEr } } +fn validate_manifest_identity(state: &ManifestState) -> Result<(), EngineError> { + validate_label_token_manifest(state)?; + for segment in &state.segments { + if segment.segment_format_version != SEGMENT_FORMAT_VERSION + || segment.segment_data_id == [0; 32] + { + return Err(EngineError::ManifestError(format!( + "unsupported segment manifest entry for segment {}; rebuild the database", + segment.id + ))); + } + } + Ok(()) +} + +fn validate_label_token_manifest(state: &ManifestState) -> Result<(), EngineError> { + if state.label_token_schema_version == 0 { + return Err(EngineError::ManifestError( + "database manifest is missing label token schema; recreate the database".to_string(), + )); + } + if state.label_token_schema_version != LABEL_TOKEN_SCHEMA_VERSION { + return Err(EngineError::ManifestError(format!( + "unsupported label token schema version: expected {}, got {}", + LABEL_TOKEN_SCHEMA_VERSION, state.label_token_schema_version + ))); + } + validate_token_namespace( + "node label", + &state.node_label_tokens, + state.next_node_label_id, + )?; + validate_token_namespace( + "edge label", + &state.edge_label_tokens, + state.next_edge_label_id, + )?; + Ok(()) +} + +fn validate_token_namespace( + namespace: &str, + tokens: &std::collections::BTreeMap, + next_id: u32, +) -> Result<(), EngineError> { + let mut ids = std::collections::BTreeMap::new(); + let mut max_id = 0u32; + for (name, &label_id) in tokens { + if let Err(error) = validate_label_token_name(name) { + return Err(EngineError::ManifestError(format!( + "{namespace} token name '{name}' is invalid: {error}" + ))); + } + if label_id == 0 { + return Err(EngineError::ManifestError(format!( + "{namespace} token '{name}' uses reserved label_id 0" + ))); + } + if let Some(existing_name) = ids.insert(label_id, name) { + return Err(EngineError::ManifestError(format!( + "{namespace} token conflict: label_id {label_id} is assigned to both '{existing_name}' and '{name}'" + ))); + } + max_id = max_id.max(label_id); + } + if next_id <= max_id { + return Err(EngineError::ManifestError(format!( + "{namespace} next token id {next_id} must be greater than max assigned id {max_id}" + ))); + } + Ok(()) +} + /// fsync a directory to make rename operations durable. /// No-op on Windows. NTFS doesn't support directory fsync via File::open(). fn fsync_dir(dir: &Path) -> Result<(), EngineError> { @@ -103,8 +180,12 @@ fn fsync_dir(dir: &Path) -> Result<(), EngineError> { Ok(()) } -/// Read-only manifest load. Same priority chain as `load_manifest` but never -/// writes to disk. Safe to call on a live or crashed database without side effects. +/// Diagnostic read-only manifest load. +/// +/// This uses the same priority chain as `load_manifest` but never writes to +/// disk, so diagnostic tooling can inspect a live or crashed database without +/// side effects. The returned `ManifestState` is a raw manifest view and may +/// include internal numeric token IDs; ordinary graph APIs use names instead. pub fn load_manifest_readonly(db_dir: &Path) -> Result, EngineError> { let current_path = db_dir.join(MANIFEST_CURRENT); let tmp_path = db_dir.join(MANIFEST_TMP); @@ -123,9 +204,14 @@ pub fn load_manifest_readonly(db_dir: &Path) -> Result, En } /// Create a fresh default manifest state. -pub fn default_manifest() -> ManifestState { +pub(crate) fn default_manifest() -> ManifestState { ManifestState { version: 1, + label_token_schema_version: LABEL_TOKEN_SCHEMA_VERSION, + node_label_tokens: std::collections::BTreeMap::new(), + edge_label_tokens: std::collections::BTreeMap::new(), + next_node_label_id: 1, + next_edge_label_id: 1, segments: Vec::new(), next_node_id: 1, next_edge_id: 1, @@ -155,6 +241,8 @@ mod tests { id: 1, node_count: 100, edge_count: 200, + segment_format_version: 10, + segment_data_id: [1; 32], }], next_node_id: 101, next_edge_id: 201, @@ -253,6 +341,11 @@ mod tests { fn test_default_manifest() { let m = default_manifest(); assert_eq!(m.version, 1); + assert_eq!(m.label_token_schema_version, LABEL_TOKEN_SCHEMA_VERSION); + assert!(m.node_label_tokens.is_empty()); + assert!(m.edge_label_tokens.is_empty()); + assert_eq!(m.next_node_label_id, 1); + assert_eq!(m.next_edge_label_id, 1); assert!(m.segments.is_empty()); assert_eq!(m.next_node_id, 1); assert_eq!(m.next_edge_id, 1); @@ -267,6 +360,11 @@ mod tests { let dir = TempDir::new().unwrap(); let legacy_json = r#"{ "version": 1, + "label_token_schema_version": 1, + "node_label_tokens": {}, + "edge_label_tokens": {}, + "next_node_label_id": 1, + "next_edge_label_id": 1, "segments": [], "next_node_id": 10, "next_edge_id": 20, @@ -323,6 +421,11 @@ mod tests { let dir = TempDir::new().unwrap(); let legacy_json = r#"{ "version": 1, + "label_token_schema_version": 1, + "node_label_tokens": {}, + "edge_label_tokens": {}, + "next_node_label_id": 1, + "next_edge_label_id": 1, "segments": [], "next_node_id": 10, "next_edge_id": 20, @@ -338,4 +441,138 @@ mod tests { assert!(loaded.secondary_indexes.is_empty()); assert_eq!(loaded.next_secondary_index_id, 0); } + + #[test] + fn test_load_manifest_missing_label_token_schema_rejected() { + let dir = TempDir::new().unwrap(); + let legacy_json = r#"{ + "version": 1, + "segments": [], + "next_node_id": 10, + "next_edge_id": 20, + "prune_policies": {} +}"#; + fs::write(dir.path().join(MANIFEST_CURRENT), legacy_json).unwrap(); + + let err = load_manifest(dir.path()).unwrap_err(); + assert!(err.to_string().contains("missing label token schema")); + } + + #[test] + fn test_load_manifest_rejects_label_token_reverse_conflict() { + let dir = TempDir::new().unwrap(); + let mut state = default_manifest(); + state.node_label_tokens.insert("Person".to_string(), 1); + state.node_label_tokens.insert("Company".to_string(), 1); + state.next_node_label_id = 2; + write_manifest(dir.path(), &state).unwrap(); + + let current_path = dir.path().join(MANIFEST_CURRENT); + let err = try_load_manifest_file(¤t_path).unwrap_err(); + assert!(err.to_string().contains("token conflict")); + } + + #[test] + fn test_load_manifest_rejects_label_token_next_id_not_above_max() { + let dir = TempDir::new().unwrap(); + let mut state = default_manifest(); + state.edge_label_tokens.insert("KNOWS".to_string(), 3); + state.next_edge_label_id = 3; + write_manifest(dir.path(), &state).unwrap(); + + let current_path = dir.path().join(MANIFEST_CURRENT); + let err = try_load_manifest_file(¤t_path).unwrap_err(); + assert!(err + .to_string() + .contains("must be greater than max assigned")); + } + + #[test] + fn test_load_manifest_rejects_invalid_label_token_names() { + let dir = TempDir::new().unwrap(); + let mut state = default_manifest(); + state.node_label_tokens.insert(" Person".to_string(), 1); + state.next_node_label_id = 2; + write_manifest(dir.path(), &state).unwrap(); + + let current_path = dir.path().join(MANIFEST_CURRENT); + let err = try_load_manifest_file(¤t_path).unwrap_err(); + assert!(err.to_string().contains("token name")); + assert!(err.to_string().contains("invalid")); + } + + #[test] + fn test_manifest_round_trip_node_and_edge_secondary_indexes() { + let dir = TempDir::new().unwrap(); + let state = ManifestState { + next_secondary_index_id: 4, + secondary_indexes: vec![ + crate::types::SecondaryIndexManifestEntry { + index_id: 0, + target: crate::types::SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: crate::types::SecondaryIndexKind::Equality, + state: crate::types::SecondaryIndexState::Building, + last_error: None, + }, + crate::types::SecondaryIndexManifestEntry { + index_id: 1, + target: crate::types::SecondaryIndexTarget::EdgeProperty { + label_id: 2, + prop_key: "weight".to_string(), + }, + kind: crate::types::SecondaryIndexKind::Range { + domain: crate::types::SecondaryIndexRangeDomain::Float, + }, + state: crate::types::SecondaryIndexState::Building, + last_error: None, + }, + crate::types::SecondaryIndexManifestEntry { + index_id: 2, + target: crate::types::SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "label".to_string(), + }, + kind: crate::types::SecondaryIndexKind::Equality, + state: crate::types::SecondaryIndexState::Ready, + last_error: None, + }, + ], + ..default_manifest() + }; + write_manifest(dir.path(), &state).unwrap(); + + let raw_manifest = fs::read_to_string(dir.path().join(MANIFEST_CURRENT)).unwrap(); + assert!(raw_manifest.contains("\"label_id\"")); + assert!(!raw_manifest.contains(concat!("\"", "type", "_id\""))); + + let loaded = load_manifest(dir.path()).unwrap().unwrap(); + assert_eq!(loaded.secondary_indexes.len(), 3); + assert_eq!(loaded.next_secondary_index_id, 4); + assert!(matches!( + &loaded.secondary_indexes[0].target, + crate::types::SecondaryIndexTarget::NodeProperty { label_id: 1, .. } + )); + assert!(matches!( + &loaded.secondary_indexes[1].target, + crate::types::SecondaryIndexTarget::EdgeProperty { label_id: 2, .. } + )); + assert_eq!(loaded.secondary_indexes[1].index_id, 1); + assert!(matches!( + loaded.secondary_indexes[1].kind, + crate::types::SecondaryIndexKind::Range { + domain: crate::types::SecondaryIndexRangeDomain::Float + } + )); + assert!(matches!( + &loaded.secondary_indexes[2].target, + crate::types::SecondaryIndexTarget::EdgeProperty { label_id: 1, .. } + )); + assert_eq!( + loaded.secondary_indexes[2].state, + crate::types::SecondaryIndexState::Ready + ); + } } diff --git a/src/memtable.rs b/src/memtable.rs index 79f44fa..856ff56 100644 --- a/src/memtable.rs +++ b/src/memtable.rs @@ -1,8 +1,26 @@ +use crate::edge_metadata::EdgeMetadataCandidate; +#[cfg(test)] +use crate::edge_metadata::{i64_matches_bounds, weight_matches_bounds, RangeBoundFlags}; use crate::types::*; use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::ops::ControlFlow; +use std::ops::{Bound, ControlFlow}; +#[cfg(test)] +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::RwLock; +#[cfg(test)] +static ENDPOINT_CURSOR_ENTRIES_VISITED_FOR_TEST: AtomicUsize = AtomicUsize::new(0); + +#[cfg(test)] +pub(crate) fn reset_endpoint_cursor_entries_visited_for_test() { + ENDPOINT_CURSOR_ENTRIES_VISITED_FOR_TEST.store(0, Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn endpoint_cursor_entries_visited_for_test() -> usize { + ENDPOINT_CURSOR_ENTRIES_VISITED_FOR_TEST.load(Ordering::Relaxed) +} + fn encode_signed_range_key(value: i64) -> u64 { (value as u64) ^ (1u64 << 63) } @@ -40,7 +58,7 @@ pub(crate) fn encode_range_prop_value( #[derive(Debug, Clone)] pub struct AdjEntry { pub edge_id: u64, - pub type_id: u32, + pub label_id: u32, pub neighbor_id: u64, pub weight: f32, pub valid_from: i64, @@ -107,6 +125,12 @@ type SecondaryEqMemberState = HashMap>>; type SecondaryEqState = HashMap; type SecondaryRangeState = HashMap>>; +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct MemtableEndpointCountEstimate { + pub(crate) count: usize, + pub(crate) exact: bool, +} + fn apply_size_delta(total: &mut usize, before: usize, after: usize) { if after >= before { *total += after - before; @@ -143,7 +167,8 @@ fn estimate_adj_slot(slot: &MembershipSlot) -> usize { fn estimate_secondary_decl_entry(entry: &SecondaryIndexManifestEntry) -> usize { let prop_key_len = match &entry.target { - SecondaryIndexTarget::NodeProperty { prop_key, .. } => prop_key.len(), + SecondaryIndexTarget::NodeProperty { prop_key, .. } + | SecondaryIndexTarget::EdgeProperty { prop_key, .. } => prop_key.len(), }; 96 + prop_key_len + entry.last_error.as_ref().map(|msg| msg.len()).unwrap_or(0) } @@ -188,12 +213,19 @@ struct MemtableState { edge_triple_index: HashMap<(u64, u64, u32), LookupSlot>, adj_out: NodeIdMap>>, adj_in: NodeIdMap>>, - type_node_index: HashMap>>, - type_edge_index: HashMap>>, + ordered_edge_ids: BTreeMap>, + ordered_label_edge_index: BTreeMap<(u32, u64), MembershipSlot<()>>, + ordered_adj_out: NodeIdMap>>, + ordered_adj_in: NodeIdMap>>, + label_node_index: HashMap>>, + label_edge_index: HashMap>>, time_node_index: BTreeMap<(u32, i64, u64), MembershipSlot<()>>, secondary_index_declarations: HashMap, secondary_eq_by_prop: HashMap>>, secondary_range_by_prop: HashMap>>, + secondary_edge_eq_by_prop: HashMap>>, + secondary_edge_range_by_prop: + HashMap>>, secondary_eq_state: SecondaryEqState, secondary_range_state: SecondaryRangeState, } @@ -242,11 +274,11 @@ fn current_adj_map( result } -fn current_type_index( +fn current_label_membership_index( source: &HashMap>>, ) -> HashMap { let mut result = HashMap::new(); - for (&type_id, members) in source { + for (&target_label_id, members) in source { let mut visible = NodeIdSet::default(); for (&member_id, slot) in members { if slot_option_current(slot).is_some() { @@ -254,7 +286,7 @@ fn current_type_index( } } if !visible.is_empty() { - result.insert(type_id, visible); + result.insert(target_label_id, visible); } } result @@ -342,8 +374,8 @@ impl MemtableState { } } - fn set_node_key(&mut self, type_id: u32, key: &str, value: Option, write_seq: u64) { - let by_key = self.node_key_index.entry(type_id).or_default(); + fn set_node_key(&mut self, label_id: u32, key: &str, value: Option, write_seq: u64) { + let by_key = self.node_key_index.entry(label_id).or_default(); if let Some(slot) = by_key.get_mut(key) { let before = estimate_lookup_slot(slot); slot.replace(write_seq, value); @@ -360,11 +392,11 @@ impl MemtableState { &mut self, from: u64, to: u64, - type_id: u32, + label_id: u32, value: Option, write_seq: u64, ) { - if let Some(slot) = self.edge_triple_index.get_mut(&(from, to, type_id)) { + if let Some(slot) = self.edge_triple_index.get_mut(&(from, to, label_id)) { let before = estimate_lookup_slot(slot); slot.replace(write_seq, value); let after = estimate_lookup_slot(slot); @@ -372,7 +404,7 @@ impl MemtableState { } else { let slot = VersionedSlot::new(write_seq, value); self.estimated_bytes += estimate_lookup_slot(&slot); - self.edge_triple_index.insert((from, to, type_id), slot); + self.edge_triple_index.insert((from, to, label_id), slot); } } @@ -397,15 +429,36 @@ impl MemtableState { } } - fn set_type_slot( + fn set_ordered_adj_slot( + map: &mut NodeIdMap>>, + owner_id: u64, + member_id: u64, + value: Option, + write_seq: u64, + ) -> (usize, usize) { + let members = map.entry(owner_id).or_default(); + if let Some(slot) = members.get_mut(&member_id) { + let before = estimate_adj_slot(slot); + slot.replace(write_seq, value); + let after = estimate_adj_slot(slot); + (before, after) + } else { + let slot = VersionedSlot::new(write_seq, value); + let after = estimate_adj_slot(&slot); + members.insert(member_id, slot); + (0, after) + } + } + + fn set_label_membership_slot( map: &mut HashMap>>, - type_id: u32, + target_label_id: u32, member_id: u64, present: bool, write_seq: u64, ) -> (usize, usize) { let value = present.then_some(()); - let members = map.entry(type_id).or_default(); + let members = map.entry(target_label_id).or_default(); if let Some(slot) = members.get_mut(&member_id) { let before = estimate_membership_slot(slot); slot.replace(write_seq, value); @@ -460,6 +513,41 @@ impl MemtableState { } } + fn set_ordered_edge_slot(&mut self, edge_id: u64, present: bool, write_seq: u64) { + let value = present.then_some(()); + if let Some(slot) = self.ordered_edge_ids.get_mut(&edge_id) { + let before = estimate_membership_slot(slot); + slot.replace(write_seq, value); + let after = estimate_membership_slot(slot); + apply_size_delta(&mut self.estimated_bytes, before, after); + } else { + let slot = VersionedSlot::new(write_seq, value); + self.estimated_bytes += estimate_membership_slot(&slot); + self.ordered_edge_ids.insert(edge_id, slot); + } + } + + fn set_ordered_edge_label_slot( + &mut self, + label_id: u32, + edge_id: u64, + present: bool, + write_seq: u64, + ) { + let value = present.then_some(()); + let key = (label_id, edge_id); + if let Some(slot) = self.ordered_label_edge_index.get_mut(&key) { + let before = estimate_membership_slot(slot); + slot.replace(write_seq, value); + let after = estimate_membership_slot(slot); + apply_size_delta(&mut self.estimated_bytes, before, after); + } else { + let slot = VersionedSlot::new(write_seq, value); + self.estimated_bytes += estimate_membership_slot(&slot); + self.ordered_label_edge_index.insert(key, slot); + } + } + fn set_secondary_eq_slot_in( state: &mut SecondaryEqState, index_id: u64, @@ -510,10 +598,16 @@ impl MemtableState { } } - fn set_node_type_slot(&mut self, type_id: u32, member_id: u64, present: bool, write_seq: u64) { - let (before, after) = Self::set_type_slot( - &mut self.type_node_index, - type_id, + fn set_node_label_slot( + &mut self, + label_id: u32, + member_id: u64, + present: bool, + write_seq: u64, + ) { + let (before, after) = Self::set_label_membership_slot( + &mut self.label_node_index, + label_id, member_id, present, write_seq, @@ -521,10 +615,16 @@ impl MemtableState { apply_size_delta(&mut self.estimated_bytes, before, after); } - fn set_edge_type_slot(&mut self, type_id: u32, member_id: u64, present: bool, write_seq: u64) { - let (before, after) = Self::set_type_slot( - &mut self.type_edge_index, - type_id, + fn set_edge_label_slot( + &mut self, + label_id: u32, + member_id: u64, + present: bool, + write_seq: u64, + ) { + let (before, after) = Self::set_label_membership_slot( + &mut self.label_edge_index, + label_id, member_id, present, write_seq, @@ -539,8 +639,21 @@ impl MemtableState { value: Option, write_seq: u64, ) { - let (before, after) = - Self::set_adj_slot(&mut self.adj_out, owner_id, member_id, value, write_seq); + let (before, after) = Self::set_adj_slot( + &mut self.adj_out, + owner_id, + member_id, + value.clone(), + write_seq, + ); + apply_size_delta(&mut self.estimated_bytes, before, after); + let (before, after) = Self::set_ordered_adj_slot( + &mut self.ordered_adj_out, + owner_id, + member_id, + value, + write_seq, + ); apply_size_delta(&mut self.estimated_bytes, before, after); } @@ -551,8 +664,21 @@ impl MemtableState { value: Option, write_seq: u64, ) { - let (before, after) = - Self::set_adj_slot(&mut self.adj_in, owner_id, member_id, value, write_seq); + let (before, after) = Self::set_adj_slot( + &mut self.adj_in, + owner_id, + member_id, + value.clone(), + write_seq, + ); + apply_size_delta(&mut self.estimated_bytes, before, after); + let (before, after) = Self::set_ordered_adj_slot( + &mut self.ordered_adj_in, + owner_id, + member_id, + value, + write_seq, + ); apply_size_delta(&mut self.estimated_bytes, before, after); } @@ -640,8 +766,28 @@ impl MemtableState { .values() .map(|entries| entries.values().map(estimate_adj_slot).sum::()) .sum(); - let type_idx_size: usize = self - .type_node_index + let ordered_edge_size: usize = self + .ordered_edge_ids + .values() + .map(estimate_membership_slot) + .sum::() + + self + .ordered_label_edge_index + .values() + .map(estimate_membership_slot) + .sum::(); + let ordered_adj_size: usize = self + .ordered_adj_out + .values() + .map(|entries| entries.values().map(estimate_adj_slot).sum::()) + .sum::() + + self + .ordered_adj_in + .values() + .map(|entries| entries.values().map(estimate_adj_slot).sum::()) + .sum::(); + let label_idx_size: usize = self + .label_node_index .values() .map(|members| { members @@ -651,7 +797,7 @@ impl MemtableState { }) .sum::() + self - .type_edge_index + .label_edge_index .values() .map(|members| { members @@ -694,6 +840,30 @@ impl MemtableState { .sum::() }) .sum(); + let secondary_edge_eq_lookup_size: usize = self + .secondary_edge_eq_by_prop + .values() + .map(|by_prop| { + by_prop + .iter() + .map(|(prop_key, index_ids)| { + estimate_secondary_eq_lookup_entry(prop_key, index_ids) + }) + .sum::() + }) + .sum(); + let secondary_edge_range_lookup_size: usize = self + .secondary_edge_range_by_prop + .values() + .map(|by_prop| { + by_prop + .iter() + .map(|(prop_key, indexes)| { + estimate_secondary_range_lookup_entry(prop_key, indexes) + }) + .sum::() + }) + .sum(); let secondary_eq_state_size: usize = self .secondary_eq_state .values() @@ -726,11 +896,15 @@ impl MemtableState { + edge_triple_size + adj_out_size + adj_in_size - + type_idx_size + + ordered_edge_size + + ordered_adj_size + + label_idx_size + time_idx_size + secondary_decl_size + secondary_eq_lookup_size + secondary_range_lookup_size + + secondary_edge_eq_lookup_size + + secondary_edge_range_lookup_size + secondary_eq_state_size + secondary_range_state_size } @@ -743,9 +917,9 @@ impl MemtableState { record_current(self.edges.get(&id)?) } - fn current_edge_triple_id(&self, from: u64, to: u64, type_id: u32) -> Option { + fn current_edge_triple_id(&self, from: u64, to: u64, label_id: u32) -> Option { self.edge_triple_index - .get(&(from, to, type_id)) + .get(&(from, to, label_id)) .and_then(slot_option_current) .copied() } @@ -808,99 +982,49 @@ impl MemtableState { }) } - fn add_secondary_index_entries_for_node(&mut self, node: &NodeRecord, write_seq: u64) { - let eq_by_prop = &self.secondary_eq_by_prop; - let range_by_prop = &self.secondary_range_by_prop; - let mut eq_actions = Vec::new(); - let mut range_actions = Vec::new(); - for (prop_key, prop_value) in &node.props { - if let Some(index_ids) = eq_by_prop - .get(&node.type_id) - .and_then(|by_prop| by_prop.get(prop_key.as_str())) - { - let value_hash = hash_prop_value(prop_value); - for &index_id in index_ids { - eq_actions.push((index_id, value_hash)); - } - } - - if let Some(indexes) = range_by_prop - .get(&node.type_id) - .and_then(|by_prop| by_prop.get(prop_key.as_str())) - { - for &(index_id, domain) in indexes { - if let Some(encoded) = encode_range_prop_value(domain, prop_value) { - range_actions.push((index_id, encoded)); - } - } - } - } - for (index_id, value_hash) in eq_actions { - self.set_secondary_eq_slot(index_id, value_hash, node.id, true, write_seq); - } - for (index_id, encoded) in range_actions { - self.set_secondary_range_slot(index_id, encoded, node.id, true, write_seq); + fn collect_secondary_index_entries_for_node_label( + &self, + label_id: u32, + node_id: u64, + props: &BTreeMap, + present: bool, + eq_actions: &mut Vec<(u64, u64, u64, bool)>, + range_actions: &mut Vec<(u64, u64, u64, bool)>, + ) { + let eq_by_prop = self.secondary_eq_by_prop.get(&label_id); + let range_by_prop = self.secondary_range_by_prop.get(&label_id); + if eq_by_prop.is_none() && range_by_prop.is_none() { + return; } - } - fn remove_secondary_index_entries_for_node(&mut self, node: &NodeRecord, write_seq: u64) { - let eq_by_prop = &self.secondary_eq_by_prop; - let range_by_prop = &self.secondary_range_by_prop; - let mut eq_actions = Vec::new(); - let mut range_actions = Vec::new(); - for (prop_key, prop_value) in &node.props { - if let Some(index_ids) = eq_by_prop - .get(&node.type_id) - .and_then(|by_prop| by_prop.get(prop_key.as_str())) - { + for (prop_key, prop_value) in props { + if let Some(index_ids) = eq_by_prop.and_then(|by_prop| by_prop.get(prop_key.as_str())) { let value_hash = hash_prop_value(prop_value); for &index_id in index_ids { - eq_actions.push((index_id, value_hash)); + eq_actions.push((index_id, value_hash, node_id, present)); } } - if let Some(indexes) = range_by_prop - .get(&node.type_id) - .and_then(|by_prop| by_prop.get(prop_key.as_str())) + if let Some(indexes) = range_by_prop.and_then(|by_prop| by_prop.get(prop_key.as_str())) { for &(index_id, domain) in indexes { if let Some(encoded) = encode_range_prop_value(domain, prop_value) { - range_actions.push((index_id, encoded)); + range_actions.push((index_id, encoded, node_id, present)); } } } } - for (index_id, value_hash) in eq_actions { - self.set_secondary_eq_slot(index_id, value_hash, node.id, false, write_seq); - } - for (index_id, encoded) in range_actions { - self.set_secondary_range_slot(index_id, encoded, node.id, false, write_seq); - } } - fn sync_secondary_index_entries_for_node_upsert( - &mut self, - old_node: Option<&NodeRecord>, + fn collect_secondary_index_updates_for_node_label( + &self, + label_id: u32, + old_node: &NodeRecord, new_node: &NodeRecord, - write_seq: u64, + eq_actions: &mut Vec<(u64, u64, u64, bool)>, + range_actions: &mut Vec<(u64, u64, u64, bool)>, ) { - let Some(old_node) = old_node else { - self.add_secondary_index_entries_for_node(new_node, write_seq); - return; - }; - - if old_node.type_id != new_node.type_id { - self.remove_secondary_index_entries_for_node(old_node, write_seq); - self.add_secondary_index_entries_for_node(new_node, write_seq); - return; - } - - let eq_by_prop = &self.secondary_eq_by_prop; - let range_by_prop = &self.secondary_range_by_prop; - let mut eq_actions = Vec::new(); - let mut range_actions = Vec::new(); - - if let Some(by_prop) = eq_by_prop.get(&new_node.type_id) { + if let Some(by_prop) = self.secondary_eq_by_prop.get(&label_id) { for (prop_key, index_ids) in by_prop { let old_value = old_node.props.get(prop_key.as_str()); let new_value = new_node.props.get(prop_key.as_str()); @@ -922,7 +1046,7 @@ impl MemtableState { } } - if let Some(by_prop) = range_by_prop.get(&new_node.type_id) { + if let Some(by_prop) = self.secondary_range_by_prop.get(&label_id) { for (prop_key, indexes) in by_prop { let old_value = old_node.props.get(prop_key.as_str()); let new_value = new_node.props.get(prop_key.as_str()); @@ -943,6 +1067,24 @@ impl MemtableState { } } } + } + + fn add_secondary_index_entries_for_node(&mut self, node: &NodeRecord, write_seq: u64) { + if self.secondary_eq_by_prop.is_empty() && self.secondary_range_by_prop.is_empty() { + return; + } + let mut eq_actions = Vec::new(); + let mut range_actions = Vec::new(); + for &label_id in node.label_ids.as_slice() { + self.collect_secondary_index_entries_for_node_label( + label_id, + node.id, + &node.props, + true, + &mut eq_actions, + &mut range_actions, + ); + } for (index_id, value_hash, node_id, present) in eq_actions { self.set_secondary_eq_slot(index_id, value_hash, node_id, present, write_seq); } @@ -950,89 +1092,338 @@ impl MemtableState { self.set_secondary_range_slot(index_id, encoded, node_id, present, write_seq); } } -} - -/// In-memory graph state. The current head and optional per-entry history live -/// under one memtable-wide `RwLock`, so active and frozen epochs share the same -/// authoritative MVCC substrate. -pub struct Memtable { - state: RwLock, -} -impl Clone for Memtable { - fn clone(&self) -> Self { - Self { - state: RwLock::new(self.state.read().unwrap().clone()), + fn remove_secondary_index_entries_for_node(&mut self, node: &NodeRecord, write_seq: u64) { + if self.secondary_eq_by_prop.is_empty() && self.secondary_range_by_prop.is_empty() { + return; + } + let mut eq_actions = Vec::new(); + let mut range_actions = Vec::new(); + for &label_id in node.label_ids.as_slice() { + self.collect_secondary_index_entries_for_node_label( + label_id, + node.id, + &node.props, + false, + &mut eq_actions, + &mut range_actions, + ); + } + for (index_id, value_hash, node_id, present) in eq_actions { + self.set_secondary_eq_slot(index_id, value_hash, node_id, present, write_seq); + } + for (index_id, encoded, node_id, present) in range_actions { + self.set_secondary_range_slot(index_id, encoded, node_id, present, write_seq); } } -} - -impl Default for Memtable { - fn default() -> Self { - Self::new() - } -} -impl Memtable { - pub fn new() -> Self { - Self { - state: RwLock::new(MemtableState::default()), + fn sync_secondary_index_entries_for_node_upsert( + &mut self, + old_node: Option<&NodeRecord>, + new_node: &NodeRecord, + write_seq: u64, + ) { + if self.secondary_eq_by_prop.is_empty() && self.secondary_range_by_prop.is_empty() { + return; } - } - pub(crate) fn apply_op(&self, op: &WalOp, last_write_seq: u64) { - let mut state = self.state.write().unwrap(); - match op { - WalOp::UpsertNode(node) => { - let old_node = state.current_node(node.id).cloned(); - let was_deleted = state.node_deleted_at(node.id, u64::MAX); - if let Some(old) = old_node.as_ref() { - if old.type_id != node.type_id || old.key != node.key { - state.set_node_key(old.type_id, &old.key, None, last_write_seq); - } - if old.type_id != node.type_id { - state.set_node_type_slot(old.type_id, old.id, false, last_write_seq); - } - if old.type_id != node.type_id || old.updated_at != node.updated_at { - state.set_time_slot( - (old.type_id, old.updated_at, old.id), - false, - last_write_seq, - ); - } - } + let Some(old_node) = old_node else { + self.add_secondary_index_entries_for_node(new_node, write_seq); + return; + }; - let mut stored = node.clone(); - stored.last_write_seq = last_write_seq; - state.set_node_key(node.type_id, &node.key, Some(node.id), last_write_seq); - state.set_node_type_slot(node.type_id, node.id, true, last_write_seq); - state.set_time_slot( - (node.type_id, node.updated_at, node.id), - true, - last_write_seq, - ); - state.sync_secondary_index_entries_for_node_upsert( - old_node.as_ref(), - &stored, - last_write_seq, + let mut eq_actions = Vec::new(); + let mut range_actions = Vec::new(); + + for &old_label_id in old_node.label_ids.as_slice() { + if !new_node.label_ids.contains(old_label_id) { + self.collect_secondary_index_entries_for_node_label( + old_label_id, + old_node.id, + &old_node.props, + false, + &mut eq_actions, + &mut range_actions, ); - if was_deleted { - state.set_node_tombstone_slot(node.id, false, last_write_seq); - } - state.set_node_state(node.id, RecordState::Live(stored), last_write_seq); } - WalOp::UpsertEdge(edge) => { - let old_edge = state.current_edge(edge.id).cloned(); - let was_deleted = state.edge_deleted_at(edge.id, u64::MAX); + } + + for &new_label_id in new_node.label_ids.as_slice() { + if old_node.label_ids.contains(new_label_id) { + self.collect_secondary_index_updates_for_node_label( + new_label_id, + old_node, + new_node, + &mut eq_actions, + &mut range_actions, + ); + } else { + self.collect_secondary_index_entries_for_node_label( + new_label_id, + new_node.id, + &new_node.props, + true, + &mut eq_actions, + &mut range_actions, + ); + } + } + for (index_id, value_hash, node_id, present) in eq_actions { + self.set_secondary_eq_slot(index_id, value_hash, node_id, present, write_seq); + } + for (index_id, encoded, node_id, present) in range_actions { + self.set_secondary_range_slot(index_id, encoded, node_id, present, write_seq); + } + } + + fn add_secondary_index_entries_for_edge(&mut self, edge: &EdgeRecord, write_seq: u64) { + if self.secondary_edge_eq_by_prop.is_empty() && self.secondary_edge_range_by_prop.is_empty() + { + return; + } + let eq_by_prop = self.secondary_edge_eq_by_prop.get(&edge.label_id); + let range_by_prop = self.secondary_edge_range_by_prop.get(&edge.label_id); + if eq_by_prop.is_none() && range_by_prop.is_none() { + return; + } + let mut eq_actions = Vec::new(); + let mut range_actions = Vec::new(); + for (prop_key, prop_value) in &edge.props { + if let Some(index_ids) = eq_by_prop.and_then(|by_prop| by_prop.get(prop_key.as_str())) { + let value_hash = hash_prop_value(prop_value); + for &index_id in index_ids { + eq_actions.push((index_id, value_hash)); + } + } + + if let Some(indexes) = range_by_prop.and_then(|by_prop| by_prop.get(prop_key.as_str())) + { + for &(index_id, domain) in indexes { + if let Some(encoded) = encode_range_prop_value(domain, prop_value) { + range_actions.push((index_id, encoded)); + } + } + } + } + for (index_id, value_hash) in eq_actions { + self.set_secondary_eq_slot(index_id, value_hash, edge.id, true, write_seq); + } + for (index_id, encoded) in range_actions { + self.set_secondary_range_slot(index_id, encoded, edge.id, true, write_seq); + } + } + + fn remove_secondary_index_entries_for_edge(&mut self, edge: &EdgeRecord, write_seq: u64) { + if self.secondary_edge_eq_by_prop.is_empty() && self.secondary_edge_range_by_prop.is_empty() + { + return; + } + let eq_by_prop = self.secondary_edge_eq_by_prop.get(&edge.label_id); + let range_by_prop = self.secondary_edge_range_by_prop.get(&edge.label_id); + if eq_by_prop.is_none() && range_by_prop.is_none() { + return; + } + let mut eq_actions = Vec::new(); + let mut range_actions = Vec::new(); + for (prop_key, prop_value) in &edge.props { + if let Some(index_ids) = eq_by_prop.and_then(|by_prop| by_prop.get(prop_key.as_str())) { + let value_hash = hash_prop_value(prop_value); + for &index_id in index_ids { + eq_actions.push((index_id, value_hash)); + } + } + + if let Some(indexes) = range_by_prop.and_then(|by_prop| by_prop.get(prop_key.as_str())) + { + for &(index_id, domain) in indexes { + if let Some(encoded) = encode_range_prop_value(domain, prop_value) { + range_actions.push((index_id, encoded)); + } + } + } + } + for (index_id, value_hash) in eq_actions { + self.set_secondary_eq_slot(index_id, value_hash, edge.id, false, write_seq); + } + for (index_id, encoded) in range_actions { + self.set_secondary_range_slot(index_id, encoded, edge.id, false, write_seq); + } + } + + fn sync_secondary_index_entries_for_edge_upsert( + &mut self, + old_edge: Option<&EdgeRecord>, + new_edge: &EdgeRecord, + write_seq: u64, + ) { + if self.secondary_edge_eq_by_prop.is_empty() && self.secondary_edge_range_by_prop.is_empty() + { + return; + } + + let Some(old_edge) = old_edge else { + self.add_secondary_index_entries_for_edge(new_edge, write_seq); + return; + }; + + if old_edge.label_id != new_edge.label_id { + self.remove_secondary_index_entries_for_edge(old_edge, write_seq); + self.add_secondary_index_entries_for_edge(new_edge, write_seq); + return; + } + + let eq_by_prop = self.secondary_edge_eq_by_prop.get(&new_edge.label_id); + let range_by_prop = self.secondary_edge_range_by_prop.get(&new_edge.label_id); + if eq_by_prop.is_none() && range_by_prop.is_none() { + return; + } + let mut eq_actions = Vec::new(); + let mut range_actions = Vec::new(); + + if let Some(by_prop) = eq_by_prop { + for (prop_key, index_ids) in by_prop { + let old_value = old_edge.props.get(prop_key.as_str()); + let new_value = new_edge.props.get(prop_key.as_str()); + if old_value == new_value { + continue; + } + if let Some(old_value) = old_value { + let value_hash = hash_prop_value(old_value); + for &index_id in index_ids { + eq_actions.push((index_id, value_hash, old_edge.id, false)); + } + } + if let Some(new_value) = new_value { + let value_hash = hash_prop_value(new_value); + for &index_id in index_ids { + eq_actions.push((index_id, value_hash, new_edge.id, true)); + } + } + } + } + + if let Some(by_prop) = range_by_prop { + for (prop_key, indexes) in by_prop { + let old_value = old_edge.props.get(prop_key.as_str()); + let new_value = new_edge.props.get(prop_key.as_str()); + if old_value == new_value { + continue; + } + for &(index_id, domain) in indexes { + if let Some(encoded) = + old_value.and_then(|value| encode_range_prop_value(domain, value)) + { + range_actions.push((index_id, encoded, old_edge.id, false)); + } + if let Some(encoded) = + new_value.and_then(|value| encode_range_prop_value(domain, value)) + { + range_actions.push((index_id, encoded, new_edge.id, true)); + } + } + } + } + for (index_id, value_hash, edge_id, present) in eq_actions { + self.set_secondary_eq_slot(index_id, value_hash, edge_id, present, write_seq); + } + for (index_id, encoded, edge_id, present) in range_actions { + self.set_secondary_range_slot(index_id, encoded, edge_id, present, write_seq); + } + } +} + +/// In-memory graph state. The current head and optional per-entry history live +/// under one memtable-wide `RwLock`, so active and frozen epochs share the same +/// authoritative MVCC substrate. +pub struct Memtable { + state: RwLock, +} + +impl Clone for Memtable { + fn clone(&self) -> Self { + Self { + state: RwLock::new(self.state.read().unwrap().clone()), + } + } +} + +impl Default for Memtable { + fn default() -> Self { + Self::new() + } +} + +impl Memtable { + pub fn new() -> Self { + Self { + state: RwLock::new(MemtableState::default()), + } + } + + pub(crate) fn apply_op(&self, op: &WalOp, last_write_seq: u64) { + let mut state = self.state.write().unwrap(); + match op { + WalOp::UpsertNode(node) => { + let old_node = state.current_node(node.id).cloned(); + let was_deleted = state.node_deleted_at(node.id, u64::MAX); + if let Some(old) = old_node.as_ref() { + for &old_label_id in old.label_ids.as_slice() { + let label_removed = !node.label_ids.contains(old_label_id); + if label_removed || old.key != node.key { + state.set_node_key(old_label_id, &old.key, None, last_write_seq); + } + if label_removed { + state.set_node_label_slot(old_label_id, old.id, false, last_write_seq); + } + if label_removed || old.updated_at != node.updated_at { + state.set_time_slot( + (old_label_id, old.updated_at, old.id), + false, + last_write_seq, + ); + } + } + } + + let mut stored = node.clone(); + stored.last_write_seq = last_write_seq; + for &new_label_id in node.label_ids.as_slice() { + state.set_node_key(new_label_id, &node.key, Some(node.id), last_write_seq); + state.set_node_label_slot(new_label_id, node.id, true, last_write_seq); + state.set_time_slot( + (new_label_id, node.updated_at, node.id), + true, + last_write_seq, + ); + } + state.sync_secondary_index_entries_for_node_upsert( + old_node.as_ref(), + &stored, + last_write_seq, + ); + if was_deleted { + state.set_node_tombstone_slot(node.id, false, last_write_seq); + } + state.set_node_state(node.id, RecordState::Live(stored), last_write_seq); + } + WalOp::UpsertEdge(edge) => { + let old_edge = state.current_edge(edge.id).cloned(); + let was_deleted = state.edge_deleted_at(edge.id, u64::MAX); if let Some(old) = old_edge.as_ref() { - if (old.from != edge.from || old.to != edge.to || old.type_id != edge.type_id) - && state.current_edge_triple_id(old.from, old.to, old.type_id) + if (old.from != edge.from || old.to != edge.to || old.label_id != edge.label_id) + && state.current_edge_triple_id(old.from, old.to, old.label_id) == Some(old.id) { - state.set_edge_triple(old.from, old.to, old.type_id, None, last_write_seq); + state.set_edge_triple(old.from, old.to, old.label_id, None, last_write_seq); } - if old.type_id != edge.type_id { - state.set_edge_type_slot(old.type_id, old.id, false, last_write_seq); + if old.label_id != edge.label_id { + state.set_edge_label_slot(old.label_id, old.id, false, last_write_seq); + state.set_ordered_edge_label_slot( + old.label_id, + old.id, + false, + last_write_seq, + ); } if old.from != edge.from || old.to != edge.to { state.set_adj_out_slot(old.from, old.id, None, last_write_seq); @@ -1042,7 +1433,7 @@ impl Memtable { let adj_out_entry = AdjEntry { edge_id: edge.id, - type_id: edge.type_id, + label_id: edge.label_id, neighbor_id: edge.to, weight: edge.weight, valid_from: edge.valid_from, @@ -1050,7 +1441,7 @@ impl Memtable { }; let adj_in_entry = AdjEntry { edge_id: edge.id, - type_id: edge.type_id, + label_id: edge.label_id, neighbor_id: edge.from, weight: edge.weight, valid_from: edge.valid_from, @@ -1058,12 +1449,12 @@ impl Memtable { }; let current_triple_id = - state.current_edge_triple_id(edge.from, edge.to, edge.type_id); + state.current_edge_triple_id(edge.from, edge.to, edge.label_id); let should_update_triple = match old_edge.as_ref() { Some(old) if old.from == edge.from && old.to == edge.to - && old.type_id == edge.type_id => + && old.label_id == edge.label_id => { current_triple_id.is_none_or(|current_id| current_id == edge.id) } @@ -1073,17 +1464,24 @@ impl Memtable { state.set_edge_triple( edge.from, edge.to, - edge.type_id, + edge.label_id, Some(edge.id), last_write_seq, ); } state.set_adj_out_slot(edge.from, edge.id, Some(adj_out_entry), last_write_seq); state.set_adj_in_slot(edge.to, edge.id, Some(adj_in_entry), last_write_seq); - state.set_edge_type_slot(edge.type_id, edge.id, true, last_write_seq); + state.set_edge_label_slot(edge.label_id, edge.id, true, last_write_seq); + state.set_ordered_edge_label_slot(edge.label_id, edge.id, true, last_write_seq); + state.set_ordered_edge_slot(edge.id, true, last_write_seq); let mut stored = edge.clone(); stored.last_write_seq = last_write_seq; + state.sync_secondary_index_entries_for_edge_upsert( + old_edge.as_ref(), + &stored, + last_write_seq, + ); if was_deleted { state.set_edge_tombstone_slot(edge.id, false, last_write_seq); } @@ -1091,13 +1489,15 @@ impl Memtable { } WalOp::DeleteNode { id, deleted_at } => { if let Some(node) = state.current_node(*id).cloned() { - state.set_node_key(node.type_id, &node.key, None, last_write_seq); - state.set_node_type_slot(node.type_id, node.id, false, last_write_seq); - state.set_time_slot( - (node.type_id, node.updated_at, node.id), - false, - last_write_seq, - ); + for &label_id in node.label_ids.as_slice() { + state.set_node_key(label_id, &node.key, None, last_write_seq); + state.set_node_label_slot(label_id, node.id, false, last_write_seq); + state.set_time_slot( + (label_id, node.updated_at, node.id), + false, + last_write_seq, + ); + } state.remove_secondary_index_entries_for_node(&node, last_write_seq); } state.set_node_state( @@ -1112,18 +1512,27 @@ impl Memtable { } WalOp::DeleteEdge { id, deleted_at } => { if let Some(edge) = state.current_edge(*id).cloned() { - if state.current_edge_triple_id(edge.from, edge.to, edge.type_id) == Some(*id) { + if state.current_edge_triple_id(edge.from, edge.to, edge.label_id) == Some(*id) + { state.set_edge_triple( edge.from, edge.to, - edge.type_id, + edge.label_id, None, last_write_seq, ); } - state.set_edge_type_slot(edge.type_id, edge.id, false, last_write_seq); + state.set_edge_label_slot(edge.label_id, edge.id, false, last_write_seq); + state.set_ordered_edge_label_slot( + edge.label_id, + edge.id, + false, + last_write_seq, + ); state.set_adj_out_slot(edge.from, edge.id, None, last_write_seq); state.set_adj_in_slot(edge.to, edge.id, None, last_write_seq); + state.set_ordered_edge_slot(edge.id, false, last_write_seq); + state.remove_secondary_index_entries_for_edge(&edge, last_write_seq); } state.set_edge_state( *id, @@ -1135,6 +1544,10 @@ impl Memtable { ); state.set_edge_tombstone_slot(*id, true, last_write_seq); } + WalOp::EnsureNodeLabel { .. } + | WalOp::EnsureEdgeLabel { .. } + | WalOp::BeginAtomicBatch { .. } + | WalOp::CommitAtomicBatch { .. } => {} } } @@ -1151,10 +1564,10 @@ impl Memtable { .insert(entry.index_id, entry.clone()); state.estimated_bytes += estimate_secondary_decl_entry(entry); match &entry.target { - SecondaryIndexTarget::NodeProperty { type_id, prop_key } => match &entry.kind { + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => match &entry.kind { SecondaryIndexKind::Equality => { let (lookup_before, lookup_after) = { - let by_prop = state.secondary_eq_by_prop.entry(*type_id).or_default(); + let by_prop = state.secondary_eq_by_prop.entry(*label_id).or_default(); match by_prop.get_mut(prop_key) { Some(index_ids) => { let before = @@ -1179,7 +1592,7 @@ impl Memtable { let Some(node) = record_current(slot) else { continue; }; - if node.type_id != *type_id { + if !node.label_ids.contains(*label_id) { continue; } let Some(prop_value) = node.props.get(prop_key) else { @@ -1199,7 +1612,7 @@ impl Memtable { } SecondaryIndexKind::Range { domain } => { let (lookup_before, lookup_after) = { - let by_prop = state.secondary_range_by_prop.entry(*type_id).or_default(); + let by_prop = state.secondary_range_by_prop.entry(*label_id).or_default(); match by_prop.get_mut(prop_key) { Some(indexes) => { let before = @@ -1228,7 +1641,7 @@ impl Memtable { let Some(node) = record_current(slot) else { continue; }; - if node.type_id != *type_id { + if !node.label_ids.contains(*label_id) { continue; } let Some(prop_value) = node.props.get(prop_key) else { @@ -1250,6 +1663,111 @@ impl Memtable { } } }, + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => match &entry.kind { + SecondaryIndexKind::Equality => { + let (lookup_before, lookup_after) = { + let by_prop = state + .secondary_edge_eq_by_prop + .entry(*label_id) + .or_default(); + match by_prop.get_mut(prop_key) { + Some(index_ids) => { + let before = + estimate_secondary_eq_lookup_entry(prop_key, index_ids); + index_ids.push(entry.index_id); + let after = estimate_secondary_eq_lookup_entry(prop_key, index_ids); + (before, after) + } + None => { + let index_ids = vec![entry.index_id]; + let after = + estimate_secondary_eq_lookup_entry(prop_key, &index_ids); + by_prop.insert(prop_key.clone(), index_ids); + (0, after) + } + } + }; + apply_size_delta(&mut state.estimated_bytes, lookup_before, lookup_after); + state.secondary_eq_state.entry(entry.index_id).or_default(); + let mut seeded = Vec::new(); + for slot in state.edges.values() { + let Some(edge) = record_current(slot) else { + continue; + }; + if edge.label_id != *label_id { + continue; + } + let Some(prop_value) = edge.props.get(prop_key) else { + continue; + }; + seeded.push((hash_prop_value(prop_value), edge.id, edge.last_write_seq)); + } + for (value_hash, edge_id, write_seq) in seeded { + state.set_secondary_eq_slot( + entry.index_id, + value_hash, + edge_id, + true, + write_seq, + ); + } + } + SecondaryIndexKind::Range { domain } => { + let (lookup_before, lookup_after) = { + let by_prop = state + .secondary_edge_range_by_prop + .entry(*label_id) + .or_default(); + match by_prop.get_mut(prop_key) { + Some(indexes) => { + let before = + estimate_secondary_range_lookup_entry(prop_key, indexes); + indexes.push((entry.index_id, *domain)); + let after = + estimate_secondary_range_lookup_entry(prop_key, indexes); + (before, after) + } + None => { + let indexes = vec![(entry.index_id, *domain)]; + let after = + estimate_secondary_range_lookup_entry(prop_key, &indexes); + by_prop.insert(prop_key.clone(), indexes); + (0, after) + } + } + }; + apply_size_delta(&mut state.estimated_bytes, lookup_before, lookup_after); + state + .secondary_range_state + .entry(entry.index_id) + .or_default(); + let mut seeded = Vec::new(); + for slot in state.edges.values() { + let Some(edge) = record_current(slot) else { + continue; + }; + if edge.label_id != *label_id { + continue; + } + let Some(prop_value) = edge.props.get(prop_key) else { + continue; + }; + let Some(encoded) = encode_range_prop_value(*domain, prop_value) else { + continue; + }; + seeded.push((encoded, edge.id, edge.last_write_seq)); + } + for (encoded, edge_id, write_seq) in seeded { + state.set_secondary_range_slot( + entry.index_id, + encoded, + edge_id, + true, + write_seq, + ); + } + } + }, } } @@ -1262,12 +1780,12 @@ impl Memtable { .estimated_bytes .saturating_sub(estimate_secondary_decl_entry(&entry)); match entry.target { - SecondaryIndexTarget::NodeProperty { type_id, prop_key } => match entry.kind { + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => match entry.kind { SecondaryIndexKind::Equality => { - let (lookup_before, lookup_after, remove_type_entry) = { + let (lookup_before, lookup_after, remove_label_entry) = { let mut delta = (0, 0); - let mut remove_type_entry = false; - if let Some(by_prop) = state.secondary_eq_by_prop.get_mut(&type_id) { + let mut remove_label_entry = false; + if let Some(by_prop) = state.secondary_eq_by_prop.get_mut(&label_id) { let mut remove_prop_entry = false; if let Some(index_ids) = by_prop.get_mut(&prop_key) { delta.0 = estimate_secondary_eq_lookup_entry(&prop_key, index_ids); @@ -1282,13 +1800,13 @@ impl Memtable { if remove_prop_entry { by_prop.remove(&prop_key); } - remove_type_entry = by_prop.is_empty(); + remove_label_entry = by_prop.is_empty(); } - (delta.0, delta.1, remove_type_entry) + (delta.0, delta.1, remove_label_entry) }; apply_size_delta(&mut state.estimated_bytes, lookup_before, lookup_after); - if remove_type_entry { - state.secondary_eq_by_prop.remove(&type_id); + if remove_label_entry { + state.secondary_eq_by_prop.remove(&label_id); } if let Some(groups) = state.secondary_eq_state.remove(&index_id) { state.estimated_bytes = state @@ -1297,10 +1815,10 @@ impl Memtable { } } SecondaryIndexKind::Range { domain } => { - let (lookup_before, lookup_after, remove_type_entry) = { + let (lookup_before, lookup_after, remove_label_entry) = { let mut delta = (0, 0); - let mut remove_type_entry = false; - if let Some(by_prop) = state.secondary_range_by_prop.get_mut(&type_id) { + let mut remove_label_entry = false; + if let Some(by_prop) = state.secondary_range_by_prop.get_mut(&label_id) { let mut remove_prop_entry = false; if let Some(indexes) = by_prop.get_mut(&prop_key) { delta.0 = estimate_secondary_range_lookup_entry(&prop_key, indexes); @@ -1317,13 +1835,84 @@ impl Memtable { if remove_prop_entry { by_prop.remove(&prop_key); } - remove_type_entry = by_prop.is_empty(); + remove_label_entry = by_prop.is_empty(); } - (delta.0, delta.1, remove_type_entry) + (delta.0, delta.1, remove_label_entry) }; apply_size_delta(&mut state.estimated_bytes, lookup_before, lookup_after); - if remove_type_entry { - state.secondary_range_by_prop.remove(&type_id); + if remove_label_entry { + state.secondary_range_by_prop.remove(&label_id); + } + if let Some(entries) = state.secondary_range_state.remove(&index_id) { + state.estimated_bytes = state + .estimated_bytes + .saturating_sub(estimate_secondary_range_state_entries(&entries)); + } + } + }, + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => match entry.kind { + SecondaryIndexKind::Equality => { + let (lookup_before, lookup_after, remove_label_entry) = { + let mut delta = (0, 0); + let mut remove_label_entry = false; + if let Some(by_prop) = state.secondary_edge_eq_by_prop.get_mut(&label_id) { + let mut remove_prop_entry = false; + if let Some(index_ids) = by_prop.get_mut(&prop_key) { + delta.0 = estimate_secondary_eq_lookup_entry(&prop_key, index_ids); + index_ids.retain(|&id| id != index_id); + if index_ids.is_empty() { + remove_prop_entry = true; + } else { + delta.1 = + estimate_secondary_eq_lookup_entry(&prop_key, index_ids); + } + } + if remove_prop_entry { + by_prop.remove(&prop_key); + } + remove_label_entry = by_prop.is_empty(); + } + (delta.0, delta.1, remove_label_entry) + }; + apply_size_delta(&mut state.estimated_bytes, lookup_before, lookup_after); + if remove_label_entry { + state.secondary_edge_eq_by_prop.remove(&label_id); + } + if let Some(groups) = state.secondary_eq_state.remove(&index_id) { + state.estimated_bytes = state + .estimated_bytes + .saturating_sub(estimate_secondary_eq_state_groups(&groups)); + } + } + SecondaryIndexKind::Range { domain } => { + let (lookup_before, lookup_after, remove_label_entry) = { + let mut delta = (0, 0); + let mut remove_label_entry = false; + if let Some(by_prop) = state.secondary_edge_range_by_prop.get_mut(&label_id) + { + let mut remove_prop_entry = false; + if let Some(indexes) = by_prop.get_mut(&prop_key) { + delta.0 = estimate_secondary_range_lookup_entry(&prop_key, indexes); + indexes.retain(|&(id, existing_domain)| { + id != index_id || existing_domain != domain + }); + if indexes.is_empty() { + remove_prop_entry = true; + } else { + delta.1 = + estimate_secondary_range_lookup_entry(&prop_key, indexes); + } + } + if remove_prop_entry { + by_prop.remove(&prop_key); + } + remove_label_entry = by_prop.is_empty(); + } + (delta.0, delta.1, remove_label_entry) + }; + apply_size_delta(&mut state.estimated_bytes, lookup_before, lookup_after); + if remove_label_entry { + state.secondary_edge_range_by_prop.remove(&label_id); } if let Some(entries) = state.secondary_range_state.remove(&index_id) { state.estimated_bytes = state @@ -1361,29 +1950,128 @@ impl Memtable { remaining.push(id); } } - } + } + + pub(crate) fn get_edge_at(&self, id: u64, snapshot_seq: u64) -> Option { + let state = self.state.read().unwrap(); + state.edge_at(id, snapshot_seq).cloned() + } + + pub(crate) fn edge_properties_at( + &self, + id: u64, + prop_keys: &[String], + snapshot_seq: u64, + ) -> Option> { + let state = self.state.read().unwrap(); + let edge = state.edge_at(id, snapshot_seq)?; + let mut props = BTreeMap::new(); + for key in prop_keys { + if let Some(value) = edge.props.get(key) { + props.insert(key.clone(), value.clone()); + } + } + Some(props) + } + + pub(crate) fn get_edge_core_at( + &self, + id: u64, + snapshot_seq: u64, + ) -> Option<(u64, u64, i64, i64, f32, i64, i64)> { + let state = self.state.read().unwrap(); + let edge = state.edge_at(id, snapshot_seq)?; + Some(( + edge.from, + edge.to, + edge.created_at, + edge.updated_at, + edge.weight, + edge.valid_from, + edge.valid_to, + )) + } + + pub(crate) fn get_edge_metadata_at( + &self, + id: u64, + snapshot_seq: u64, + ) -> Option { + let state = self.state.read().unwrap(); + state + .edge_at(id, snapshot_seq) + .map(EdgeMetadataCandidate::from_edge) + } + + pub(crate) fn batch_get_node_visibility_meta_at( + &self, + lookups: &[(usize, u64)], + snapshot_seq: u64, + results: &mut [NodeVisibilityState], + ) -> Vec<(usize, u64)> { + if lookups.is_empty() { + return Vec::new(); + } + + let state = self.state.read().unwrap(); + let mut cache = + NodeIdMap::with_capacity_and_hasher(lookups.len(), NodeIdBuildHasher::default()); + let mut remaining = Vec::with_capacity(lookups.len()); + + for &(orig_idx, id) in lookups { + let visibility = match cache.get(&id).copied() { + Some(state) => state, + None => { + let state = match state + .nodes + .get(&id) + .and_then(|slot| record_at(slot, snapshot_seq)) + { + Some(RecordState::Live(node)) => { + NodeVisibilityState::Live(NodeVisibilityMeta { + label_ids: node.label_ids, + updated_at: node.updated_at, + weight: node.weight, + }) + } + Some(RecordState::Tombstone(_)) => NodeVisibilityState::Deleted, + None if state.node_deleted_at(id, snapshot_seq) => { + NodeVisibilityState::Deleted + } + None => NodeVisibilityState::Missing, + }; + cache.insert(id, state); + state + } + }; + + match visibility { + NodeVisibilityState::Live(_) | NodeVisibilityState::Deleted => { + results[orig_idx] = visibility; + } + NodeVisibilityState::Missing => remaining.push((orig_idx, id)), + } + } - pub(crate) fn get_edge_at(&self, id: u64, snapshot_seq: u64) -> Option { - let state = self.state.read().unwrap(); - state.edge_at(id, snapshot_seq).cloned() + remaining } - pub(crate) fn get_edge_core_at( + pub(crate) fn edge_visibility_state_at( &self, id: u64, snapshot_seq: u64, - ) -> Option<(u64, u64, i64, i64, f32, i64, i64)> { + ) -> EdgeVisibilityState { let state = self.state.read().unwrap(); - let edge = state.edge_at(id, snapshot_seq)?; - Some(( - edge.from, - edge.to, - edge.created_at, - edge.updated_at, - edge.weight, - edge.valid_from, - edge.valid_to, - )) + match state + .edges + .get(&id) + .and_then(|slot| record_at(slot, snapshot_seq)) + { + Some(RecordState::Live(_)) => EdgeVisibilityState::Live, + Some(RecordState::Tombstone(_)) => EdgeVisibilityState::Deleted, + None if state.edge_deleted_at(id, snapshot_seq) => EdgeVisibilityState::Deleted, + None => EdgeVisibilityState::Missing, + } } pub(crate) fn batch_get_nodes_at( @@ -1524,12 +2212,13 @@ impl Memtable { pub(crate) fn node_by_key_at( &self, - type_id: u32, + label_id: u32, key: &str, snapshot_seq: u64, ) -> Option { let state = self.state.read().unwrap(); - let node_id = *slot_option_at(state.node_key_index.get(&type_id)?.get(key)?, snapshot_seq)?; + let node_id = + *slot_option_at(state.node_key_index.get(&label_id)?.get(key)?, snapshot_seq)?; state.node_at(node_id, snapshot_seq).cloned() } @@ -1537,17 +2226,68 @@ impl Memtable { &self, from: u64, to: u64, - type_id: u32, + label_id: u32, snapshot_seq: u64, ) -> Option { let state = self.state.read().unwrap(); let edge_id = *slot_option_at( - state.edge_triple_index.get(&(from, to, type_id))?, + state.edge_triple_index.get(&(from, to, label_id))?, snapshot_seq, )?; state.edge_at(edge_id, snapshot_seq).cloned() } + pub(crate) fn batch_edges_by_triples_at( + &self, + lookups: &[(usize, u64, u64, u32)], + snapshot_seq: u64, + results: &mut [Option], + ) -> Vec<(usize, u64, u64, u32)> { + #[derive(Clone, Copy)] + enum CachedLookup { + Live(usize), + Miss, + } + + if lookups.is_empty() { + return Vec::new(); + } + + let state = self.state.read().unwrap(); + let mut cache = HashMap::with_capacity(lookups.len()); + let mut remaining = Vec::with_capacity(lookups.len()); + + for &(orig_idx, from, to, label_id) in lookups { + let triple = (from, to, label_id); + match cache.get(&triple).copied() { + Some(CachedLookup::Live(cached_idx)) => { + results[orig_idx] = results[cached_idx].clone(); + } + Some(CachedLookup::Miss) => remaining.push((orig_idx, from, to, label_id)), + None => { + let outcome = match state + .edge_triple_index + .get(&triple) + .and_then(|slot| slot_option_at(slot, snapshot_seq)) + .and_then(|edge_id| state.edge_at(*edge_id, snapshot_seq)) + { + Some(edge) => { + results[orig_idx] = Some(edge.clone()); + CachedLookup::Live(orig_idx) + } + None => { + remaining.push((orig_idx, from, to, label_id)); + CachedLookup::Miss + } + }; + cache.insert(triple, outcome); + } + } + } + + remaining + } + pub(crate) fn for_each_visible_node_at( &self, snapshot_seq: u64, @@ -1568,10 +2308,10 @@ impl Memtable { ControlFlow::Continue(()) } - pub(crate) fn visible_nodes_by_type(&self, type_id: u32, snapshot_seq: u64) -> Vec { + pub(crate) fn visible_nodes_by_label_id(&self, label_id: u32, snapshot_seq: u64) -> Vec { let state = self.state.read().unwrap(); let mut ids = Vec::new(); - if let Some(members) = state.type_node_index.get(&type_id) { + if let Some(members) = state.label_node_index.get(&label_id) { for (&node_id, slot) in members { if slot_option_visible(slot, snapshot_seq) { ids.push(node_id); @@ -1603,11 +2343,15 @@ impl Memtable { .count() } - pub(crate) fn visible_nodes_by_type_count(&self, type_id: u32, snapshot_seq: u64) -> usize { + pub(crate) fn visible_nodes_by_label_id_count( + &self, + label_id: u32, + snapshot_seq: u64, + ) -> usize { let state = self.state.read().unwrap(); state - .type_node_index - .get(&type_id) + .label_node_index + .get(&label_id) .map(|members| { members .values() @@ -1617,10 +2361,10 @@ impl Memtable { .unwrap_or(0) } - pub(crate) fn visible_edges_by_type(&self, type_id: u32, snapshot_seq: u64) -> Vec { + pub(crate) fn visible_edges_by_label_id(&self, label_id: u32, snapshot_seq: u64) -> Vec { let state = self.state.read().unwrap(); let mut ids = Vec::new(); - if let Some(members) = state.type_edge_index.get(&type_id) { + if let Some(members) = state.label_edge_index.get(&label_id) { for (&edge_id, slot) in members { if slot_option_visible(slot, snapshot_seq) { ids.push(edge_id); @@ -1631,9 +2375,190 @@ impl Memtable { ids } + pub(crate) fn visible_edges_by_label_id_count( + &self, + label_id: u32, + snapshot_seq: u64, + ) -> usize { + let state = self.state.read().unwrap(); + state + .ordered_label_edge_index + .range((label_id, 0)..=(label_id, u64::MAX)) + .filter(|(_, slot)| slot_option_visible(slot, snapshot_seq)) + .count() + } + + #[cfg(test)] + pub(crate) fn visible_edge_ids_at(&self, snapshot_seq: u64) -> Vec { + let state = self.state.read().unwrap(); + let mut ids = Vec::new(); + for (&edge_id, slot) in &state.edges { + if matches!(record_at(slot, snapshot_seq), Some(RecordState::Live(_))) { + ids.push(edge_id); + } + } + ids.sort_unstable(); + ids + } + + pub(crate) fn next_visible_edge_id_after( + &self, + snapshot_seq: u64, + after: Option, + ) -> Option { + let state = self.state.read().unwrap(); + let start = after.map_or(Bound::Unbounded, Bound::Excluded); + for (&edge_id, slot) in state.ordered_edge_ids.range((start, Bound::Unbounded)) { + if slot_option_visible(slot, snapshot_seq) { + return Some(edge_id); + } + } + None + } + + pub(crate) fn next_visible_edge_by_label_id_after( + &self, + label_id: u32, + snapshot_seq: u64, + after: Option, + ) -> Option { + let state = self.state.read().unwrap(); + let start_key = (label_id, after.unwrap_or(0)); + let start = match after { + Some(_) => Bound::Excluded(start_key), + None => Bound::Included(start_key), + }; + let end = Bound::Included((label_id, u64::MAX)); + for (&(_, edge_id), slot) in state.ordered_label_edge_index.range((start, end)) { + if slot_option_visible(slot, snapshot_seq) { + return Some(edge_id); + } + } + None + } + + pub(crate) fn edge_ids_by_triple_at( + &self, + from: u64, + to: u64, + label_id: u32, + snapshot_seq: u64, + ) -> Vec { + let state = self.state.read().unwrap(); + let mut ids = Vec::new(); + if let Some(entries) = state.adj_out.get(&from) { + for (&edge_id, slot) in entries { + let Some(entry) = slot_option_at(slot, snapshot_seq) else { + continue; + }; + if entry.neighbor_id == to && entry.label_id == label_id { + ids.push(edge_id); + } + } + } + ids.sort_unstable(); + ids + } + + #[cfg(test)] + pub(crate) fn edge_metadata_scan_ids_at( + &self, + snapshot_seq: u64, + mut predicate: F, + ) -> Vec + where + F: FnMut(EdgeMetadataCandidate) -> bool, + { + let state = self.state.read().unwrap(); + let mut ids = Vec::new(); + for (&edge_id, slot) in &state.edges { + let Some(RecordState::Live(edge)) = record_at(slot, snapshot_seq) else { + continue; + }; + let meta = EdgeMetadataCandidate::from_edge(edge); + if predicate(meta) { + ids.push(edge_id); + } + } + ids.sort_unstable(); + ids + } + + pub(crate) fn for_each_edge_metadata_at( + &self, + snapshot_seq: u64, + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(EdgeMetadataCandidate) -> ControlFlow<()>, + { + let state = self.state.read().unwrap(); + for slot in state.edges.values() { + let Some(RecordState::Live(edge)) = record_at(slot, snapshot_seq) else { + continue; + }; + if callback(EdgeMetadataCandidate::from_edge(edge)).is_break() { + return ControlFlow::Break(()); + } + } + ControlFlow::Continue(()) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_weight_range_at( + &self, + label_id: Option, + bounds: RangeBoundFlags, + snapshot_seq: u64, + ) -> Vec { + self.edge_metadata_scan_ids_at(snapshot_seq, |meta| { + label_id.is_none_or(|target| meta.label_id == target) + && weight_matches_bounds(meta.weight, bounds) + }) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_updated_at_range_at( + &self, + label_id: Option, + bounds: RangeBoundFlags, + snapshot_seq: u64, + ) -> Vec { + self.edge_metadata_scan_ids_at(snapshot_seq, |meta| { + label_id.is_none_or(|target| meta.label_id == target) + && i64_matches_bounds(meta.updated_at, bounds) + }) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_valid_from_range_at( + &self, + label_id: Option, + bounds: RangeBoundFlags, + snapshot_seq: u64, + ) -> Vec { + self.edge_metadata_scan_ids_at(snapshot_seq, |meta| { + label_id.is_none_or(|target| meta.label_id == target) + && i64_matches_bounds(meta.valid_from, bounds) + }) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_valid_to_range_at( + &self, + label_id: Option, + bounds: RangeBoundFlags, + snapshot_seq: u64, + ) -> Vec { + self.edge_metadata_scan_ids_at(snapshot_seq, |meta| { + label_id.is_none_or(|target| meta.label_id == target) + && i64_matches_bounds(meta.valid_to, bounds) + }) + } + pub(crate) fn visible_nodes_by_time_range( &self, - type_id: u32, + label_id: u32, from_ms: i64, to_ms: i64, snapshot_seq: u64, @@ -1643,13 +2568,13 @@ impl Memtable { } let state = self.state.read().unwrap(); use std::ops::Bound; - let start = (type_id, from_ms, 0u64); - let end = (type_id, to_ms, u64::MAX); + let start = (label_id, from_ms, 0u64); + let end = (label_id, to_ms, u64::MAX); let mut ids = state .time_node_index .range((Bound::Included(start), Bound::Included(end))) - .filter_map(|(&(entry_type, _, node_id), slot)| { - (entry_type == type_id && slot_option_visible(slot, snapshot_seq)) + .filter_map(|(&(entry_label_id, _, node_id), slot)| { + (entry_label_id == label_id && slot_option_visible(slot, snapshot_seq)) .then_some(node_id) }) .collect::>(); @@ -1659,7 +2584,7 @@ impl Memtable { pub(crate) fn for_each_visible_node_by_time_range_at( &self, - type_id: u32, + label_id: u32, from_ms: i64, to_ms: i64, snapshot_seq: u64, @@ -1673,13 +2598,13 @@ impl Memtable { } let state = self.state.read().unwrap(); use std::ops::Bound; - let start = (type_id, from_ms, 0u64); - let end = (type_id, to_ms, u64::MAX); - for (&(entry_type, _, node_id), slot) in state + let start = (label_id, from_ms, 0u64); + let end = (label_id, to_ms, u64::MAX); + for (&(entry_label_id, _, node_id), slot) in state .time_node_index .range((Bound::Included(start), Bound::Included(end))) { - if entry_type != type_id || !slot_option_visible(slot, snapshot_seq) { + if entry_label_id != label_id || !slot_option_visible(slot, snapshot_seq) { continue; } if callback(node_id).is_break() { @@ -1693,10 +2618,10 @@ impl Memtable { &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, limit: usize, snapshot_seq: u64, - ) -> Vec { + ) -> Vec { let state = self.state.read().unwrap(); if state.node_deleted_at(node_id, snapshot_seq) { return Vec::new(); @@ -1706,7 +2631,7 @@ impl Memtable { let mut self_loop_edge_ids = NodeIdSet::default(); let mut collect = |map: Option<&NodeIdMap>>, dedupe_self_loops: bool, - results: &mut Vec| { + results: &mut Vec| { let Some(map) = map else { return; }; @@ -1717,7 +2642,7 @@ impl Memtable { let Some(entry) = slot_option_at(slot, snapshot_seq) else { continue; }; - if type_filter.is_some_and(|types| !types.contains(&entry.type_id)) { + if label_filter_ids.is_some_and(|label_ids| !label_ids.contains(&entry.label_id)) { continue; } if state.node_deleted_at(entry.neighbor_id, snapshot_seq) { @@ -1730,10 +2655,10 @@ impl Memtable { } else if entry.neighbor_id == node_id { self_loop_edge_ids.insert(entry.edge_id); } - results.push(NeighborEntry { + results.push(NeighborRecord { node_id: entry.neighbor_id, edge_id: entry.edge_id, - edge_type_id: entry.type_id, + edge_label_id: entry.label_id, weight: entry.weight, valid_from: entry.valid_from, valid_to: entry.valid_to, @@ -1761,9 +2686,9 @@ impl Memtable { &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, snapshot_seq: u64, - ) -> Vec { + ) -> Vec { let state = self.state.read().unwrap(); if state.node_deleted_at(node_id, snapshot_seq) { return Vec::new(); @@ -1773,7 +2698,7 @@ impl Memtable { let mut self_loop_edge_ids = NodeIdSet::default(); let mut collect = |map: Option<&NodeIdMap>>, dedupe_self_loops: bool, - results: &mut Vec| { + results: &mut Vec| { let Some(map) = map else { return; }; @@ -1781,7 +2706,7 @@ impl Memtable { let Some(entry) = slot_option_at(slot, snapshot_seq) else { continue; }; - if type_filter.is_some_and(|types| !types.contains(&entry.type_id)) { + if label_filter_ids.is_some_and(|label_ids| !label_ids.contains(&entry.label_id)) { continue; } if dedupe_self_loops && entry.neighbor_id == node_id { @@ -1791,10 +2716,10 @@ impl Memtable { } else if entry.neighbor_id == node_id { self_loop_edge_ids.insert(entry.edge_id); } - results.push(NeighborEntry { + results.push(NeighborRecord { node_id: entry.neighbor_id, edge_id: entry.edge_id, - edge_type_id: entry.type_id, + edge_label_id: entry.label_id, weight: entry.weight, valid_from: entry.valid_from, valid_to: entry.valid_to, @@ -1822,7 +2747,7 @@ impl Memtable { &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, snapshot_seq: u64, callback: &mut F, ) -> ControlFlow<()> @@ -1844,7 +2769,7 @@ impl Memtable { let Some(entry) = slot_option_at(slot, snapshot_seq) else { continue; }; - if type_filter.is_some_and(|types| !types.contains(&entry.type_id)) { + if label_filter_ids.is_some_and(|label_ids| !label_ids.contains(&entry.label_id)) { continue; } if state.node_deleted_at(entry.neighbor_id, snapshot_seq) { @@ -1882,19 +2807,148 @@ impl Memtable { } } - pub(crate) fn visible_types(&self, snapshot_seq: u64) -> Vec { + fn next_visible_adj_edge_after( + state: &MemtableState, + node_id: u64, + outgoing: bool, + label_filter_ids: Option<&[u32]>, + snapshot_seq: u64, + after: Option, + ) -> Option { + if state.node_deleted_at(node_id, snapshot_seq) { + return None; + } + let source = if outgoing { + &state.ordered_adj_out + } else { + &state.ordered_adj_in + }; + let entries = source.get(&node_id)?; + let start = after.map_or(Bound::Unbounded, Bound::Excluded); + for (&edge_id, slot) in entries.range((start, Bound::Unbounded)) { + #[cfg(test)] + ENDPOINT_CURSOR_ENTRIES_VISITED_FOR_TEST.fetch_add(1, Ordering::Relaxed); + let Some(entry) = slot_option_at(slot, snapshot_seq) else { + continue; + }; + if label_filter_ids.is_some_and(|label_ids| !label_ids.contains(&entry.label_id)) { + continue; + } + if state.node_deleted_at(entry.neighbor_id, snapshot_seq) { + continue; + } + return Some(edge_id); + } + None + } + + fn visible_adj_edge_count_estimate( + state: &MemtableState, + node_id: u64, + outgoing: bool, + label_filter_ids: Option<&[u32]>, + snapshot_seq: u64, + ) -> MemtableEndpointCountEstimate { + if label_filter_ids.is_some_and(<[u32]>::is_empty) + || state.node_deleted_at(node_id, snapshot_seq) + { + return MemtableEndpointCountEstimate { + count: 0, + exact: true, + }; + } + let source = if outgoing { + &state.ordered_adj_out + } else { + &state.ordered_adj_in + }; + let Some(entries) = source.get(&node_id) else { + return MemtableEndpointCountEstimate { + count: 0, + exact: true, + }; + }; + MemtableEndpointCountEstimate { + count: entries.len(), + exact: entries.is_empty(), + } + } + + pub(crate) fn next_visible_edge_from_endpoint_after( + &self, + node_id: u64, + label_filter_ids: Option<&[u32]>, + snapshot_seq: u64, + after: Option, + ) -> Option { + let state = self.state.read().unwrap(); + Self::next_visible_adj_edge_after( + &state, + node_id, + true, + label_filter_ids, + snapshot_seq, + after, + ) + } + + pub(crate) fn next_visible_edge_to_endpoint_after( + &self, + node_id: u64, + label_filter_ids: Option<&[u32]>, + snapshot_seq: u64, + after: Option, + ) -> Option { + let state = self.state.read().unwrap(); + Self::next_visible_adj_edge_after( + &state, + node_id, + false, + label_filter_ids, + snapshot_seq, + after, + ) + } + + pub(crate) fn visible_edges_from_endpoint_count_estimate( + &self, + node_id: u64, + label_filter_ids: Option<&[u32]>, + snapshot_seq: u64, + ) -> MemtableEndpointCountEstimate { + let state = self.state.read().unwrap(); + Self::visible_adj_edge_count_estimate(&state, node_id, true, label_filter_ids, snapshot_seq) + } + + pub(crate) fn visible_edges_to_endpoint_count_estimate( + &self, + node_id: u64, + label_filter_ids: Option<&[u32]>, + snapshot_seq: u64, + ) -> MemtableEndpointCountEstimate { + let state = self.state.read().unwrap(); + Self::visible_adj_edge_count_estimate( + &state, + node_id, + false, + label_filter_ids, + snapshot_seq, + ) + } + + pub(crate) fn visible_node_label_ids(&self, snapshot_seq: u64) -> Vec { let state = self.state.read().unwrap(); - let mut types = Vec::new(); - for (&type_id, members) in &state.type_node_index { + let mut label_ids = Vec::new(); + for (&label_id, members) in &state.label_node_index { if members .values() .any(|slot| slot_option_visible(slot, snapshot_seq)) { - types.push(type_id); + label_ids.push(label_id); } } - types.sort_unstable(); - types + label_ids.sort_unstable(); + label_ids } pub(crate) fn find_secondary_eq_nodes_at( @@ -1944,7 +2998,71 @@ impl Memtable { ids } - pub(crate) fn find_secondary_eq_nodes_by_hash_at_limited( + pub(crate) fn find_secondary_eq_nodes_by_hash_at_limited( + &self, + index_id: u64, + value_hash: u64, + snapshot_seq: u64, + max_ids: Option, + ) -> Vec { + let state = self.state.read().unwrap(); + let Some(groups) = state.secondary_eq_state.get(&index_id) else { + return Vec::new(); + }; + let Some(group) = groups.get(&value_hash) else { + return Vec::new(); + }; + + let mut ids = Vec::new(); + for (&node_id, slot) in group { + if !slot_option_visible(slot, snapshot_seq) { + continue; + } + ids.push(node_id); + if max_ids.is_some_and(|max_ids| ids.len() >= max_ids) { + break; + } + } + ids.sort_unstable(); + ids + } + + pub(crate) fn secondary_eq_node_count_at( + &self, + index_id: u64, + prop_key: &str, + prop_value: &PropValue, + snapshot_seq: u64, + ) -> usize { + let state = self.state.read().unwrap(); + let value_hash = hash_prop_value(prop_value); + let Some(groups) = state.secondary_eq_state.get(&index_id) else { + return 0; + }; + let Some(group) = groups.get(&value_hash) else { + return 0; + }; + + let mut count = 0; + for (&node_id, slot) in group { + if !slot_option_visible(slot, snapshot_seq) { + continue; + } + let Some(node) = state.node_at(node_id, snapshot_seq) else { + continue; + }; + if node + .props + .get(prop_key) + .is_some_and(|value| value == prop_value) + { + count += 1; + } + } + count + } + + pub(crate) fn find_secondary_eq_edges_by_hash_at_limited( &self, index_id: u64, value_hash: u64, @@ -1960,11 +3078,11 @@ impl Memtable { }; let mut ids = Vec::new(); - for (&node_id, slot) in group { + for (&edge_id, slot) in group { if !slot_option_visible(slot, snapshot_seq) { continue; } - ids.push(node_id); + ids.push(edge_id); if max_ids.is_some_and(|max_ids| ids.len() >= max_ids) { break; } @@ -1973,7 +3091,7 @@ impl Memtable { ids } - pub(crate) fn secondary_eq_node_count_at( + pub(crate) fn secondary_eq_edge_count_at( &self, index_id: u64, prop_key: &str, @@ -1990,14 +3108,14 @@ impl Memtable { }; let mut count = 0; - for (&node_id, slot) in group { + for (&edge_id, slot) in group { if !slot_option_visible(slot, snapshot_seq) { continue; } - let Some(node) = state.node_at(node_id, snapshot_seq) else { + let Some(edge) = state.edge_at(edge_id, snapshot_seq) else { continue; }; - if node + if edge .props .get(prop_key) .is_some_and(|value| value == prop_value) @@ -2150,43 +3268,27 @@ impl Memtable { deleted } - /// Current-head helpers below keep the existing memtable-facing API shape - /// for writer planning, flush, stats, and tests. - pub fn get_node(&self, id: u64) -> Option { - self.get_node_at(id, u64::MAX) - } - - pub fn get_edge(&self, id: u64) -> Option { - self.get_edge_at(id, u64::MAX) - } - - pub fn node_by_key(&self, type_id: u32, key: &str) -> Option { - self.node_by_key_at(type_id, key, u64::MAX) - } - - pub fn edge_by_triple(&self, from: u64, to: u64, type_id: u32) -> Option { - self.edge_by_triple_at(from, to, type_id, u64::MAX) - } - - pub fn neighbors( + #[allow(dead_code)] + pub(crate) fn neighbors( &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, limit: usize, - ) -> Vec { - self.neighbors_at(node_id, direction, type_filter, limit, u64::MAX) + ) -> Vec { + self.neighbors_at(node_id, direction, label_filter_ids, limit, u64::MAX) } - pub fn neighbors_batch( + #[allow(dead_code)] + pub(crate) fn neighbors_batch( &self, node_ids: &[u64], direction: Direction, - type_filter: Option<&[u32]>, - ) -> NodeIdMap> { + label_filter_ids: Option<&[u32]>, + ) -> NodeIdMap> { let mut results = NodeIdMap::default(); for &nid in node_ids { - let entries = self.neighbors(nid, direction, type_filter, 0); + let entries = self.neighbors(nid, direction, label_filter_ids, 0); if !entries.is_empty() { results.insert(nid, entries); } @@ -2198,12 +3300,12 @@ impl Memtable { &self, node_ids: &[u64], direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, snapshot_seq: u64, - ) -> NodeIdMap> { + ) -> NodeIdMap> { let mut results = NodeIdMap::default(); for &nid in node_ids { - let entries = self.neighbors_at(nid, direction, type_filter, 0, snapshot_seq); + let entries = self.neighbors_at(nid, direction, label_filter_ids, 0, snapshot_seq); if !entries.is_empty() { results.insert(nid, entries); } @@ -2211,41 +3313,6 @@ impl Memtable { results } - pub fn for_each_adj_entry( - &self, - node_id: u64, - direction: Direction, - type_filter: Option<&[u32]>, - callback: &mut F, - ) -> ControlFlow<()> - where - F: FnMut(u64, u64, f32, i64, i64) -> ControlFlow<()>, - { - self.for_each_adj_entry_at(node_id, direction, type_filter, u64::MAX, callback) - } - - pub fn incident_edge_ids(&self, node_id: u64) -> Vec { - let state = self.state.read().unwrap(); - let mut ids = Vec::new(); - if let Some(map) = state.adj_out.get(&node_id) { - for (&edge_id, slot) in map { - if slot_option_current(slot).is_some() { - ids.push(edge_id); - } - } - } - if let Some(map) = state.adj_in.get(&node_id) { - for (&edge_id, slot) in map { - if slot_option_current(slot).is_some() { - ids.push(edge_id); - } - } - } - ids.sort_unstable(); - ids.dedup(); - ids - } - pub fn node_count(&self) -> usize { let state = self.state.read().unwrap(); state @@ -2310,22 +3377,14 @@ impl Memtable { current_adj_map(&state.adj_in) } - pub fn nodes_by_type(&self, type_id: u32) -> Vec { - self.visible_nodes_by_type(type_id, u64::MAX) - } - - pub fn edges_by_type(&self, type_id: u32) -> Vec { - self.visible_edges_by_type(type_id, u64::MAX) - } - - pub fn type_node_index(&self) -> HashMap { + pub fn label_node_index(&self) -> HashMap { let state = self.state.read().unwrap(); - current_type_index(&state.type_node_index) + current_label_membership_index(&state.label_node_index) } - pub fn type_edge_index(&self) -> HashMap { + pub fn label_edge_index(&self) -> HashMap { let state = self.state.read().unwrap(); - current_type_index(&state.type_edge_index) + current_label_membership_index(&state.label_edge_index) } pub fn secondary_index_declarations(&self) -> HashMap { @@ -2348,30 +3407,6 @@ impl Memtable { current_time_index(&state.time_node_index) } - pub fn nodes_by_time_range(&self, type_id: u32, from_ms: i64, to_ms: i64) -> Vec { - self.visible_nodes_by_time_range(type_id, from_ms, to_ms, u64::MAX) - } - - pub fn find_nodes(&self, type_id: u32, prop_key: &str, prop_value: &PropValue) -> Vec { - self.visible_nodes_by_type(type_id, u64::MAX) - .into_iter() - .filter(|id| { - self.get_node(*id) - .and_then(|node| node.props.get(prop_key).cloned()) - .is_some_and(|value| value == *prop_value) - }) - .collect() - } - - pub fn find_secondary_eq_nodes( - &self, - index_id: u64, - prop_key: &str, - prop_value: &PropValue, - ) -> Vec { - self.find_secondary_eq_nodes_at(index_id, prop_key, prop_value, u64::MAX) - } - fn estimate_node_record(node: &NodeRecord) -> usize { let dense_bytes = node .dense_vector @@ -2433,8 +3468,8 @@ impl Memtable { #[cfg(test)] impl Memtable { - fn type_node_index_key_count(&self) -> usize { - self.type_node_index().len() + fn label_node_index_key_count(&self) -> usize { + self.label_node_index().len() } fn node_key_index_key_count(&self) -> usize { @@ -2460,10 +3495,10 @@ mod tests { use super::*; use std::collections::BTreeMap; - fn make_node(id: u64, type_id: u32, key: &str) -> NodeRecord { + fn make_node(id: u64, label_id: u32, key: &str) -> NodeRecord { NodeRecord { id, - type_id, + label_ids: NodeLabelSet::single(label_id).unwrap(), key: key.to_string(), props: BTreeMap::new(), created_at: 1000, @@ -2475,19 +3510,34 @@ mod tests { } } - fn make_node_at(id: u64, type_id: u32, key: &str, updated_at: i64) -> NodeRecord { + fn make_node_at(id: u64, label_id: u32, key: &str, updated_at: i64) -> NodeRecord { NodeRecord { updated_at, - ..make_node(id, type_id, key) + ..make_node(id, label_id, key) + } + } + + fn make_node_with_labels(id: u64, label_ids: &[u32], key: &str, updated_at: i64) -> NodeRecord { + NodeRecord { + id, + label_ids: NodeLabelSet::from_canonical_ids(label_ids).unwrap(), + key: key.to_string(), + props: BTreeMap::new(), + created_at: 1000, + updated_at, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, } } - fn make_edge(id: u64, from: u64, to: u64, type_id: u32) -> EdgeRecord { + fn make_edge(id: u64, from: u64, to: u64, label_id: u32) -> EdgeRecord { EdgeRecord { id, from, to, - type_id, + label_id: label_id, props: BTreeMap::new(), created_at: 2000, updated_at: 2001, @@ -2500,13 +3550,26 @@ mod tests { fn make_node_with_props( id: u64, - type_id: u32, + label_id: u32, + key: &str, + props: BTreeMap, + ) -> NodeRecord { + NodeRecord { + props, + ..make_node(id, label_id, key) + } + } + + fn make_node_with_labels_and_props( + id: u64, + label_ids: &[u32], key: &str, props: BTreeMap, + updated_at: i64, ) -> NodeRecord { NodeRecord { props, - ..make_node(id, type_id, key) + ..make_node_with_labels(id, label_ids, key, updated_at) } } @@ -2516,10 +3579,98 @@ mod tests { mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); mt.apply_op(&WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), 2); - assert_eq!(mt.get_node(1).unwrap().key, "alice"); - assert_eq!(mt.get_edge(1).unwrap().from, 1); - assert_eq!(mt.node_by_key(1, "alice").unwrap().id, 1); - assert_eq!(mt.edge_by_triple(1, 2, 10).unwrap().id, 1); + assert_eq!(mt.get_node_at(1, u64::MAX).unwrap().key, "alice"); + assert_eq!(mt.get_edge_at(1, u64::MAX).unwrap().from, 1); + assert_eq!(mt.node_by_key_at(1, "alice", u64::MAX).unwrap().id, 1); + assert_eq!(mt.edge_by_triple_at(1, 2, 10, u64::MAX).unwrap().id, 1); + } + + #[test] + fn edge_metadata_source_helpers_return_visible_ids() { + let mt = Memtable::new(); + let mut edge_a = make_edge(10, 1, 2, 5); + edge_a.weight = -0.0; + edge_a.updated_at = 100; + edge_a.valid_from = 10; + edge_a.valid_to = 100; + let mut edge_b = make_edge(11, 1, 3, 5); + edge_b.weight = 0.0; + edge_b.updated_at = 200; + edge_b.valid_from = 20; + edge_b.valid_to = 200; + let mut edge_c = make_edge(12, 4, 1, 6); + edge_c.weight = f32::NAN; + edge_c.updated_at = 300; + + mt.apply_op(&WalOp::UpsertEdge(edge_a), 1); + mt.apply_op(&WalOp::UpsertEdge(edge_b), 2); + mt.apply_op(&WalOp::UpsertEdge(edge_c), 3); + mt.apply_op( + &WalOp::DeleteEdge { + id: 12, + deleted_at: 400, + }, + 4, + ); + + assert_eq!(mt.visible_edge_ids_at(3), vec![10, 11, 12]); + assert_eq!(mt.visible_edge_ids_at(4), vec![10, 11]); + assert_eq!(mt.visible_edges_by_label_id(5, 4), vec![10, 11]); + let mut outgoing_ids = mt + .neighbors_batch_at(&[1], Direction::Outgoing, Some(&[5]), 4) + .into_values() + .flatten() + .map(|entry| entry.edge_id) + .collect::>(); + outgoing_ids.sort_unstable(); + assert_eq!(outgoing_ids, vec![10, 11]); + let mut both_ids = mt + .neighbors_batch_at(&[1], Direction::Both, None, 4) + .into_values() + .flatten() + .map(|entry| entry.edge_id) + .collect::>(); + both_ids.sort_unstable(); + both_ids.dedup(); + assert_eq!(both_ids, vec![10, 11]); + assert_eq!(mt.edge_ids_by_triple_at(1, 2, 5, 4), vec![10]); + assert_eq!( + mt.edge_ids_by_weight_range_at( + Some(5), + RangeBoundFlags::inclusive(Some(0.0), Some(0.0)), + 4, + ), + vec![10, 11] + ); + assert_eq!( + mt.edge_ids_by_updated_at_range_at( + Some(5), + RangeBoundFlags::inclusive(Some(150), Some(250)), + 4, + ), + vec![11] + ); + assert_eq!( + mt.edge_ids_by_valid_from_range_at( + None, + RangeBoundFlags::inclusive(Some(0), Some(15)), + 4, + ), + vec![10] + ); + assert_eq!( + mt.edge_ids_by_valid_to_range_at( + None, + RangeBoundFlags { + lower: Some(100), + lower_inclusive: false, + upper: None, + upper_inclusive: true, + }, + 4, + ), + vec![11] + ); } #[test] @@ -2531,7 +3682,7 @@ mod tests { assert_eq!(mt.get_node_at(1, 10).unwrap().key, "alice"); assert_eq!(mt.get_node_at(1, 19).unwrap().key, "alice"); assert_eq!(mt.get_node_at(1, 20).unwrap().key, "alice_v2"); - assert_eq!(mt.get_node(1).unwrap().key, "alice_v2"); + assert_eq!(mt.get_node_at(1, u64::MAX).unwrap().key, "alice_v2"); } #[test] @@ -2546,7 +3697,7 @@ mod tests { 6, ); - assert!(mt.get_node(1).is_none()); + assert!(mt.get_node_at(1, u64::MAX).is_none()); assert_eq!(mt.get_node_at(1, 5).unwrap().key, "alice"); assert!(mt.get_node_at(1, 6).is_none()); assert!(mt.is_node_deleted_at(1, 6)); @@ -2613,11 +3764,11 @@ mod tests { assert_eq!(mt.node_by_key_at(1, "alice", 1).unwrap().id, 1); assert!(mt.node_by_key_at(1, "alice", 2).is_none()); assert_eq!(mt.node_by_key_at(1, "alice", 3).unwrap().id, 2); - assert_eq!(mt.node_by_key(1, "alice").unwrap().id, 2); + assert_eq!(mt.node_by_key_at(1, "alice", u64::MAX).unwrap().id, 2); } #[test] - fn adjacency_and_type_memberships_are_snapshot_aware() { + fn adjacency_and_label_memberships_are_snapshot_aware() { let mt = Memtable::new(); mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 2); @@ -2630,8 +3781,8 @@ mod tests { 4, ); - assert_eq!(mt.visible_edges_by_type(10, 3), vec![1]); - assert!(mt.visible_edges_by_type(10, 4).is_empty()); + assert_eq!(mt.visible_edges_by_label_id(10, 3), vec![1]); + assert!(mt.visible_edges_by_label_id(10, 4).is_empty()); let before_delete = mt.neighbors_at(1, Direction::Outgoing, None, 0, 3); assert_eq!(before_delete.len(), 1); assert_eq!(before_delete[0].node_id, 2); @@ -2651,6 +3802,111 @@ mod tests { assert_eq!(mt.visible_nodes_by_time_range(1, 150, 250, 2), vec![1]); } + #[test] + fn multi_label_node_memberships_are_maintained_by_label() { + let mt = Memtable::new(); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(1, &[1, 2, 5], "alice", 100)), + 1, + ); + + for label_id in [1, 2, 5] { + assert_eq!( + mt.node_by_key_at(label_id, "alice", 1).map(|node| node.id), + Some(1) + ); + assert_eq!(mt.visible_nodes_by_label_id(label_id, 1), vec![1]); + assert_eq!( + mt.visible_nodes_by_time_range(label_id, 100, 100, 1), + vec![1] + ); + } + assert!(mt.node_by_key_at(9, "alice", 1).is_none()); + assert!(mt.visible_nodes_by_label_id(9, 1).is_empty()); + + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(1, &[2, 5, 9], "alice", 200)), + 2, + ); + + assert_eq!( + mt.node_by_key_at(1, "alice", 1).map(|node| node.id), + Some(1) + ); + assert!(mt.node_by_key_at(1, "alice", 2).is_none()); + assert!(mt.visible_nodes_by_label_id(1, 2).is_empty()); + assert!(mt.visible_nodes_by_time_range(1, 100, 100, 2).is_empty()); + + for label_id in [2, 5, 9] { + assert_eq!( + mt.node_by_key_at(label_id, "alice", 2).map(|node| node.id), + Some(1) + ); + assert_eq!(mt.visible_nodes_by_label_id(label_id, 2), vec![1]); + assert!(mt + .visible_nodes_by_time_range(label_id, 100, 100, 2) + .is_empty()); + assert_eq!( + mt.visible_nodes_by_time_range(label_id, 200, 200, 2), + vec![1] + ); + } + + mt.apply_op( + &WalOp::DeleteNode { + id: 1, + deleted_at: 300, + }, + 3, + ); + for label_id in [2, 5, 9] { + assert!(mt.node_by_key_at(label_id, "alice", 3).is_none()); + assert!(mt.visible_nodes_by_label_id(label_id, 3).is_empty()); + assert!(mt + .visible_nodes_by_time_range(label_id, 200, 200, 3) + .is_empty()); + } + } + + #[test] + fn multi_label_key_replacement_updates_all_label_memberships() { + let mt = Memtable::new(); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(1, &[1, 2], "alice", 100)), + 1, + ); + + for label_id in [1, 2] { + assert_eq!( + mt.node_by_key_at(label_id, "alice", 1).map(|node| node.id), + Some(1) + ); + } + + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(1, &[2, 3], "alice2", 100)), + 2, + ); + + for label_id in [1, 2] { + assert!( + mt.node_by_key_at(label_id, "alice", 2).is_none(), + "old key remained visible for label {label_id}" + ); + } + assert!( + mt.node_by_key_at(1, "alice2", 2).is_none(), + "new key became visible for removed label" + ); + for label_id in [2, 3] { + assert_eq!( + mt.node_by_key_at(label_id, "alice2", 2).map(|node| node.id), + Some(1), + "new key missing for label {label_id}" + ); + } + } + #[test] fn secondary_eq_membership_history_is_snapshot_correct() { let mt = Memtable::new(); @@ -2659,7 +3915,7 @@ mod tests { let entry = SecondaryIndexManifestEntry { index_id: 10, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "name".into(), }, kind: SecondaryIndexKind::Equality, @@ -2692,6 +3948,149 @@ mod tests { ); } + #[test] + fn multi_label_secondary_memberships_are_maintained_by_declared_label() { + let mt = Memtable::new(); + let eq_label_1 = SecondaryIndexManifestEntry { + index_id: 10, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "color".into(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let eq_label_2 = SecondaryIndexManifestEntry { + index_id: 11, + target: SecondaryIndexTarget::NodeProperty { + label_id: 2, + prop_key: "color".into(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_label_2 = SecondaryIndexManifestEntry { + index_id: 12, + target: SecondaryIndexTarget::NodeProperty { + label_id: 2, + prop_key: "score".into(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_label_3 = SecondaryIndexManifestEntry { + index_id: 13, + target: SecondaryIndexTarget::NodeProperty { + label_id: 3, + prop_key: "score".into(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + for entry in [&eq_label_1, &eq_label_2, &range_label_2, &range_label_3] { + mt.register_secondary_index(entry); + } + + let mut props = BTreeMap::new(); + props.insert("color".into(), PropValue::String("red".into())); + props.insert("score".into(), PropValue::Int(42)); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels_and_props( + 1, + &[1, 2, 3], + "item", + props, + 100, + )), + 1, + ); + + assert_eq!( + mt.find_secondary_eq_nodes_at(10, "color", &PropValue::String("red".into()), 1), + vec![1] + ); + assert_eq!( + mt.find_secondary_eq_nodes_at(11, "color", &PropValue::String("red".into()), 1), + vec![1] + ); + let encoded_42 = + encode_range_prop_value(SecondaryIndexRangeDomain::Int, &PropValue::Int(42)).unwrap(); + assert_eq!( + mt.visible_secondary_range_entries( + 12, + Some((encoded_42, true)), + Some((encoded_42, true)), + None, + 1 + ), + vec![(encoded_42, 1)] + ); + assert_eq!( + mt.visible_secondary_range_entries( + 13, + Some((encoded_42, true)), + Some((encoded_42, true)), + None, + 1 + ), + vec![(encoded_42, 1)] + ); + + let mut updated_props = BTreeMap::new(); + updated_props.insert("color".into(), PropValue::String("blue".into())); + updated_props.insert("score".into(), PropValue::Int(50)); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels_and_props( + 1, + &[2], + "item", + updated_props, + 200, + )), + 2, + ); + + assert!(mt + .find_secondary_eq_nodes_at(10, "color", &PropValue::String("red".into()), 2) + .is_empty()); + assert!(mt + .find_secondary_eq_nodes_at(11, "color", &PropValue::String("red".into()), 2) + .is_empty()); + assert_eq!( + mt.find_secondary_eq_nodes_at(11, "color", &PropValue::String("blue".into()), 2), + vec![1] + ); + let encoded_50 = + encode_range_prop_value(SecondaryIndexRangeDomain::Int, &PropValue::Int(50)).unwrap(); + assert!(mt + .visible_secondary_range_entries( + 13, + Some((encoded_42, true)), + Some((encoded_42, true)), + None, + 2 + ) + .is_empty()); + assert_eq!( + mt.visible_secondary_range_entries( + 12, + Some((encoded_50, true)), + Some((encoded_50, true)), + None, + 2 + ), + vec![(encoded_50, 1)] + ); + } + #[test] fn same_write_seq_replace_overwrites_head_in_place() { let mut slot = VersionedSlot::new(1, 10u64); @@ -2710,7 +4109,7 @@ mod tests { let eq_entry = SecondaryIndexManifestEntry { index_id: 10, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "name".into(), }, kind: SecondaryIndexKind::Equality, @@ -2720,7 +4119,7 @@ mod tests { let range_entry = SecondaryIndexManifestEntry { index_id: 11, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "age".into(), }, kind: SecondaryIndexKind::Range { @@ -2832,7 +4231,7 @@ mod tests { let eq_entry = SecondaryIndexManifestEntry { index_id: 10, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "name".into(), }, kind: SecondaryIndexKind::Equality, @@ -2842,7 +4241,7 @@ mod tests { let range_entry = SecondaryIndexManifestEntry { index_id: 11, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "age".into(), }, kind: SecondaryIndexKind::Range { @@ -2875,9 +4274,9 @@ mod tests { ); assert_eq!(mt.node_key_index_key_count(), 0); - assert_eq!(mt.type_node_index_key_count(), 0); + assert_eq!(mt.label_node_index_key_count(), 0); assert_eq!(mt.time_node_index_len(), 0); - assert!(mt.get_node(1).is_none()); + assert!(mt.get_node_at(1, u64::MAX).is_none()); assert_eq!(mt.max_node_id(), 1); } } diff --git a/src/planner_stats.rs b/src/planner_stats.rs index 46c7435..458ed74 100644 --- a/src/planner_stats.rs +++ b/src/planner_stats.rs @@ -1,34 +1,47 @@ use crate::error::EngineError; use crate::memtable::encode_range_prop_value; +#[cfg(test)] +use crate::segment_components::{ + decode_identity_header, COMPONENT_IDENTITY_HEADER_LEN, COMPONENT_IDENTITY_HEADER_MAGIC, +}; use crate::segment_reader::SegmentReader; -use crate::segment_writer::{CompactEdgeMeta, CompactNodeMeta}; +use crate::segment_writer::{ + publish_planner_stats_component_payload_from_latest, CompactEdgeMeta, CompactNodeMeta, +}; use crate::types::{ hash_prop_value, EdgeRecord, NodeIdMap, NodeRecord, PropValue, SecondaryIndexKind, SecondaryIndexManifestEntry, SecondaryIndexRangeDomain, SecondaryIndexState, - SecondaryIndexTarget, + SecondaryIndexTarget, MAX_NODE_LABELS_PER_NODE, }; use crc32fast::Hasher as Crc32Hasher; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; -use std::fs::{self, File}; -use std::io::{Read, Write}; +#[cfg(test)] +use std::fs; +#[cfg(test)] +use std::fs::File; +#[cfg(test)] +use std::io::Read; +#[cfg(test)] +use std::io::Write; use std::path::Path; use std::sync::Arc; pub(crate) const PLANNER_STATS_FILENAME: &str = "planner_stats.dat"; +#[cfg(test)] const PLANNER_STATS_TMP_FILENAME: &str = "planner_stats.tmp"; const PLANNER_STATS_MAGIC: [u8; 8] = *b"OGPST01\0"; pub(crate) const PLANNER_STATS_FORMAT_VERSION: u32 = 1; const PLANNER_STATS_ENVELOPE_LEN: usize = 8 + 4 + 8 + 4 + 4; -pub(crate) const PLANNER_STATS_MAX_PROPERTY_KEYS_PER_TYPE: usize = 256; -const PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_TYPE: usize = 1024; +pub(crate) const PLANNER_STATS_MAX_PROPERTY_KEYS_PER_LABEL: usize = 256; +const PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_LABEL: usize = 1024; pub(crate) const PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY: usize = 32; pub(crate) const PLANNER_STATS_MAX_DISTINCT_TRACKED_VALUES: usize = 4096; pub(crate) const PLANNER_STATS_RANGE_BUCKETS: usize = 64; pub(crate) const PLANNER_STATS_TIMESTAMP_BUCKETS: usize = 64; pub(crate) const PLANNER_STATS_NODE_ID_SAMPLE_SIZE: usize = 1024; -pub(crate) const PLANNER_STATS_TOP_HUBS_PER_EDGE_TYPE: usize = 32; +pub(crate) const PLANNER_STATS_TOP_HUBS_PER_EDGE_LABEL: usize = 32; pub(crate) const PLANNER_STATS_SOFT_SIDECAR_BYTES: usize = 16 * 1024 * 1024; pub(crate) const PLANNER_STATS_HARD_SIDECAR_BYTES: usize = 64 * 1024 * 1024; pub(crate) const PLANNER_STATS_HARD_CANDIDATE_CAP: usize = 65_536; @@ -125,6 +138,15 @@ pub(crate) enum PlannerStatsDeclaredIndexKind { Range, } +#[derive( + Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, +)] +pub(crate) enum PlannerStatsDeclaredIndexTarget { + #[default] + NodeProperty, + EdgeProperty, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum DeclaredIndexRuntimeCoverageState { Available, @@ -137,6 +159,7 @@ pub(crate) enum DeclaredIndexRuntimeCoverageState { pub(crate) struct DeclaredIndexRuntimeCoverageKey { pub segment_id: u64, pub index_id: u64, + pub target: PlannerStatsDeclaredIndexTarget, pub kind: PlannerStatsDeclaredIndexKind, } @@ -155,9 +178,7 @@ impl DeclaredIndexRuntimeCoverage { if entry.state != SecondaryIndexState::Ready { continue; } - if ready_node_property_target(entry).is_none() { - continue; - } + let target = planner_stats_declared_index_target(entry); let kind = match entry.kind { SecondaryIndexKind::Equality => PlannerStatsDeclaredIndexKind::Equality, SecondaryIndexKind::Range { .. } => PlannerStatsDeclaredIndexKind::Range, @@ -166,8 +187,13 @@ impl DeclaredIndexRuntimeCoverage { coverage.insert( segment.segment_id, entry.index_id, + target, kind, - segment.declared_index_runtime_coverage_state(entry.index_id, kind), + segment.declared_index_runtime_coverage_state_for_target( + entry.index_id, + target, + kind, + ), ); } } @@ -178,6 +204,7 @@ impl DeclaredIndexRuntimeCoverage { &mut self, segment_id: u64, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, kind: PlannerStatsDeclaredIndexKind, state: DeclaredIndexRuntimeCoverageState, ) { @@ -185,6 +212,7 @@ impl DeclaredIndexRuntimeCoverage { DeclaredIndexRuntimeCoverageKey { segment_id, index_id, + target, kind, }, state, @@ -195,12 +223,14 @@ impl DeclaredIndexRuntimeCoverage { &self, segment_id: u64, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, kind: PlannerStatsDeclaredIndexKind, ) -> DeclaredIndexRuntimeCoverageState { self.states .get(&DeclaredIndexRuntimeCoverageKey { segment_id, index_id, + target, kind, }) .copied() @@ -211,9 +241,11 @@ impl DeclaredIndexRuntimeCoverage { &self, segment_id: u64, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, kind: PlannerStatsDeclaredIndexKind, ) -> bool { - self.state(segment_id, index_id, kind) == DeclaredIndexRuntimeCoverageState::Available + self.state(segment_id, index_id, target, kind) + == DeclaredIndexRuntimeCoverageState::Available } pub(crate) fn entry_count(&self) -> usize { @@ -224,8 +256,10 @@ impl DeclaredIndexRuntimeCoverage { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct DeclaredIndexStatsFingerprint { pub index_id: u64, + #[serde(default)] + pub target: PlannerStatsDeclaredIndexTarget, pub kind: PlannerStatsDeclaredIndexKind, - pub type_id: u32, + pub target_label_id: u32, pub prop_key: String, pub range_domain: Option, } @@ -260,7 +294,7 @@ pub(crate) struct SegmentPlannerStatsV1 { pub general_property_sampled_node_count: u64, pub general_property_sampled_raw_bytes: u64, pub general_property_budget_exhausted: bool, - pub type_stats: Vec, + pub node_label_stats: Vec, pub timestamp_stats: Vec, pub property_stats: Vec, pub equality_index_stats: Vec, @@ -269,9 +303,46 @@ pub(crate) struct SegmentPlannerStatsV1 { pub node_id_sample: Vec, } +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct StatsCorePartial { + pub node_count: u64, + pub edge_count: u64, + pub truncated: bool, + pub general_property_stats_complete: bool, + pub general_property_sampled_node_count: u64, + pub general_property_sampled_raw_bytes: u64, + pub general_property_budget_exhausted: bool, + pub node_label_stats: Vec, + pub timestamp_stats: Vec, + pub property_stats: Vec, + pub adjacency_stats: Vec, + pub node_id_sample: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct DeclaredIndexStatsEvidence { + pub equality_index_stats: Vec, + pub range_index_stats: Vec, +} + +impl DeclaredIndexStatsEvidence { + pub(crate) fn sort(&mut self) { + self.equality_index_stats + .sort_by_key(|stats| stats.index_id); + self.range_index_stats.sort_by_key(|stats| stats.index_id); + } + + pub(crate) fn extend(&mut self, mut other: DeclaredIndexStatsEvidence) { + self.equality_index_stats + .append(&mut other.equality_index_stats); + self.range_index_stats.append(&mut other.range_index_stats); + self.sort(); + } +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub(crate) struct TypePlannerStats { - pub type_id: u32, +pub(crate) struct NodeLabelPlannerStats { + pub label_id: u32, pub node_count: u64, pub min_node_id: Option, pub max_node_id: Option, @@ -281,7 +352,7 @@ pub(crate) struct TypePlannerStats { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub(crate) struct TimestampPlannerStats { - pub type_id: u32, + pub label_id: u32, pub count: u64, pub min_ms: i64, pub max_ms: i64, @@ -296,7 +367,7 @@ pub(crate) struct TimestampBucket { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub(crate) struct PropertyPlannerStats { - pub type_id: u32, + pub label_id: u32, pub prop_key: String, pub tracked_reason: PropertyStatsTrackedReason, pub present_count: u64, @@ -333,7 +404,7 @@ pub(crate) struct RangeValueSummary { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub(crate) struct EqualityIndexPlannerStats { pub index_id: u64, - pub type_id: u32, + pub target_label_id: u32, pub prop_key: String, pub total_postings: u64, pub value_group_count: u64, @@ -345,7 +416,7 @@ pub(crate) struct EqualityIndexPlannerStats { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub(crate) struct RangeIndexPlannerStats { pub index_id: u64, - pub type_id: u32, + pub target_label_id: u32, pub prop_key: String, pub domain: SecondaryIndexRangeDomain, pub total_entries: u64, @@ -364,7 +435,7 @@ pub(crate) struct RangeBucket { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub(crate) struct AdjacencyPlannerStats { pub direction: PlannerStatsDirection, - pub edge_type_id: Option, + pub edge_label_id: Option, pub source_node_count: u64, pub total_edges: u64, pub min_fanout: u32, @@ -490,10 +561,10 @@ pub(crate) struct PlannerStatsView { pub missing_segment_stats: usize, pub unavailable_segment_stats: usize, pub full_rollup: FullRollupStats, - pub type_coverage: PlannerStatsFamilyCoverage, + pub node_label_coverage: PlannerStatsFamilyCoverage, pub timestamp_coverage: PlannerStatsFamilyCoverage, pub property_rollups: BTreeMap<(u32, String), PropertyRollupStats>, - pub type_rollups: BTreeMap, + pub node_label_rollups: BTreeMap, pub timestamp_rollups: BTreeMap, pub equality_index_rollups: BTreeMap, pub range_index_rollups: BTreeMap, @@ -509,8 +580,8 @@ pub(crate) struct FullRollupStats { } #[derive(Clone, Debug, Default)] -pub(crate) struct TypeRollupStats { - pub type_id: u32, +pub(crate) struct NodeLabelRollupStats { + pub label_id: u32, pub node_count: u64, pub min_node_id: Option, pub max_node_id: Option, @@ -520,7 +591,7 @@ pub(crate) struct TypeRollupStats { #[derive(Clone, Debug, Default)] pub(crate) struct TimestampRollupStats { - pub type_id: u32, + pub label_id: u32, pub count: u64, pub min_ms: Option, pub max_ms: Option, @@ -538,7 +609,7 @@ struct TimestampSegmentRollupStats { #[derive(Clone, Debug, Default)] pub(crate) struct PropertyRollupStats { - pub type_id: u32, + pub label_id: u32, pub prop_key: String, pub present_count: u64, pub null_count: u64, @@ -549,7 +620,7 @@ pub(crate) struct PropertyRollupStats { #[derive(Clone, Debug, Default)] pub(crate) struct EqualityIndexRollupStats { pub index_id: u64, - pub type_id: u32, + pub target_label_id: u32, pub prop_key: String, pub total_postings: u64, pub value_group_count: u64, @@ -570,7 +641,7 @@ struct EqualitySegmentRollupStats { #[derive(Clone, Debug)] pub(crate) struct RangeIndexRollupStats { pub index_id: u64, - pub type_id: u32, + pub target_label_id: u32, pub prop_key: String, pub domain: SecondaryIndexRangeDomain, pub total_entries: u64, @@ -592,7 +663,7 @@ impl Default for RangeIndexRollupStats { fn default() -> Self { Self { index_id: 0, - type_id: 0, + target_label_id: 0, prop_key: String::new(), domain: SecondaryIndexRangeDomain::Int, total_entries: 0, @@ -607,7 +678,7 @@ impl Default for RangeIndexRollupStats { #[derive(Clone, Debug, Default)] pub(crate) struct AdjacencyRollupStats { pub direction: PlannerStatsDirection, - pub edge_type_id: Option, + pub edge_label_id: Option, pub source_node_count: u64, pub total_edges: u64, pub max_fanout: u32, @@ -643,9 +714,9 @@ impl PlannerStatsView { view } - pub(crate) fn type_node_count(&self, type_id: u32) -> u64 { - self.type_rollups - .get(&type_id) + pub(crate) fn node_label_count(&self, label_id: u32) -> u64 { + self.node_label_rollups + .get(&label_id) .map_or(0, |rollup| rollup.node_count) } @@ -692,15 +763,16 @@ impl PlannerStatsView { pub(crate) fn timestamp_estimate( &self, - type_id: u32, + label_id: u32, lower_ms: i64, upper_ms: i64, ) -> Option { - let Some(rollup) = self.timestamp_rollups.get(&type_id) else { - if self.type_node_count(type_id) == 0 && self.type_coverage.covered_count() > 0 { + let Some(rollup) = self.timestamp_rollups.get(&label_id) else { + if self.node_label_count(label_id) == 0 && self.node_label_coverage.covered_count() > 0 + { return Some(PlannerStatsValueEstimate { count: 0, - exact: !self.type_coverage.has_uncovered(), + exact: !self.node_label_coverage.has_uncovered(), }); } return None; @@ -728,9 +800,9 @@ impl PlannerStatsView { Some(PlannerStatsValueEstimate { count, exact }) } - pub(crate) fn timestamp_covers_segment(&self, type_id: u32, segment_id: u64) -> bool { - self.timestamp_rollups.get(&type_id).map_or_else( - || self.type_node_count(type_id) == 0 && self.type_coverage.covers(segment_id), + pub(crate) fn timestamp_covers_segment(&self, label_id: u32, segment_id: u64) -> bool { + self.timestamp_rollups.get(&label_id).map_or_else( + || self.node_label_count(label_id) == 0 && self.node_label_coverage.covers(segment_id), |rollup| rollup.coverage.covers(segment_id), ) } @@ -756,37 +828,37 @@ impl PlannerStatsView { "full stats coverage exceeds segment count" ); debug_assert!( - self.type_coverage.covered_count() <= self.segment_count, - "type stats coverage exceeds segment count" + self.node_label_coverage.covered_count() <= self.segment_count, + "node label stats coverage exceeds segment count" ); debug_assert!( self.timestamp_coverage.covered_count() <= self.segment_count, "timestamp stats coverage exceeds segment count" ); - for (type_id, rollup) in &self.type_rollups { - debug_assert_eq!(*type_id, rollup.type_id); + for (label_id, rollup) in &self.node_label_rollups { + debug_assert_eq!(*label_id, rollup.label_id); } - for (type_id, rollup) in &self.timestamp_rollups { - debug_assert_eq!(*type_id, rollup.type_id); + for (label_id, rollup) in &self.timestamp_rollups { + debug_assert_eq!(*label_id, rollup.label_id); } - for ((type_id, prop_key), rollup) in &self.property_rollups { - debug_assert_eq!(*type_id, rollup.type_id); + for ((label_id, prop_key), rollup) in &self.property_rollups { + debug_assert_eq!(*label_id, rollup.label_id); debug_assert_eq!(prop_key, &rollup.prop_key); } for (index_id, rollup) in &self.equality_index_rollups { debug_assert_eq!(*index_id, rollup.index_id); debug_assert!( !rollup.prop_key.is_empty(), - "equality rollup for type {} must have property key", - rollup.type_id + "equality rollup for target label {} must have property key", + rollup.target_label_id ); } for (index_id, rollup) in &self.range_index_rollups { debug_assert_eq!(*index_id, rollup.index_id); debug_assert!( !rollup.prop_key.is_empty(), - "range rollup for type {} must have property key", - rollup.type_id + "range rollup for target label {} must have property key", + rollup.target_label_id ); match rollup.domain { SecondaryIndexRangeDomain::Int @@ -794,9 +866,9 @@ impl PlannerStatsView { | SecondaryIndexRangeDomain::Float => {} } } - for ((direction, edge_type_id), rollup) in &self.adjacency_rollups { + for ((direction, edge_label_id), rollup) in &self.adjacency_rollups { debug_assert_eq!(*direction, rollup.direction); - debug_assert_eq!(*edge_type_id, rollup.edge_type_id); + debug_assert_eq!(*edge_label_id, rollup.edge_label_id); } } } @@ -956,18 +1028,20 @@ struct AdjacencyRollupBuilder { #[derive(Clone)] struct EqualityIndexDeclaration { - type_id: u32, + target: PlannerStatsDeclaredIndexTarget, + target_label_id: u32, prop_key: String, } #[derive(Clone)] struct RangeIndexDeclaration { - type_id: u32, + target: PlannerStatsDeclaredIndexTarget, + target_label_id: u32, prop_key: String, domain: SecondaryIndexRangeDomain, } -type DeclaredIndexFingerprintSet = BTreeSet<(u64, u8, u32, String, u8)>; +type DeclaredIndexFingerprintSet = BTreeSet<(u64, u8, u8, u32, String, u8)>; #[cfg(test)] fn build_planner_stats_view_from_snapshots( @@ -992,10 +1066,10 @@ fn all_available_runtime_coverage_for_snapshots( ) -> DeclaredIndexRuntimeCoverage { let mut coverage = DeclaredIndexRuntimeCoverage::default(); for entry in secondary_indexes { - if entry.state != SecondaryIndexState::Ready || ready_node_property_target(entry).is_none() - { + if entry.state != SecondaryIndexState::Ready { continue; } + let target = planner_stats_declared_index_target(entry); let kind = match entry.kind { SecondaryIndexKind::Equality => PlannerStatsDeclaredIndexKind::Equality, SecondaryIndexKind::Range { .. } => PlannerStatsDeclaredIndexKind::Range, @@ -1004,6 +1078,7 @@ fn all_available_runtime_coverage_for_snapshots( coverage.insert( segment.segment_id, entry.index_id, + target, kind, DeclaredIndexRuntimeCoverageState::Available, ); @@ -1024,13 +1099,13 @@ fn build_planner_stats_view_from_snapshots_with_runtime_coverage( .collect::>() .into(); let mut full_coverage = CoverageBuilder::new(all_segment_ids.clone()); - let mut type_coverage = CoverageBuilder::new(all_segment_ids.clone()); + let mut node_label_coverage = CoverageBuilder::new(all_segment_ids.clone()); let mut timestamp_coverage = CoverageBuilder::new(all_segment_ids.clone()); let mut full_rollup = FullRollupStats::default(); - let mut type_rollups: BTreeMap = BTreeMap::new(); + let mut node_label_rollups: BTreeMap = BTreeMap::new(); let mut timestamp_rollups: BTreeMap = BTreeMap::new(); - let mut segment_type_ids: BTreeMap> = BTreeMap::new(); - let mut segment_timestamp_type_ids: BTreeMap> = BTreeMap::new(); + let mut segment_label_ids: BTreeMap> = BTreeMap::new(); + let mut segment_timestamp_label_ids: BTreeMap> = BTreeMap::new(); let mut property_builders: BTreeMap<(u32, String), PropertyRollupBuilder> = BTreeMap::new(); let equality_declarations = ready_equality_declarations(secondary_indexes); let range_declarations = ready_range_declarations(secondary_indexes); @@ -1051,30 +1126,30 @@ fn build_planner_stats_view_from_snapshots_with_runtime_coverage( PlannerStatsAvailability::Available(stats) => { available_segment_stats += 1; full_coverage.mark_covered(segment.segment_id); - type_coverage.mark_covered(segment.segment_id); + node_label_coverage.mark_covered(segment.segment_id); timestamp_coverage.mark_covered(segment.segment_id); full_rollup.node_count = full_rollup.node_count.saturating_add(stats.node_count); full_rollup.edge_count = full_rollup.edge_count.saturating_add(stats.edge_count); if stats.general_property_stats_complete { complete_property_segment_ids.insert(segment.segment_id); } - segment_type_ids.insert( + segment_label_ids.insert( segment.segment_id, stats - .type_stats + .node_label_stats .iter() - .map(|type_stats| type_stats.type_id) + .map(|node_label_stats| node_label_stats.label_id) .collect(), ); - segment_timestamp_type_ids.insert( + segment_timestamp_label_ids.insert( segment.segment_id, stats .timestamp_stats .iter() - .map(|timestamp| timestamp.type_id) + .map(|timestamp| timestamp.label_id) .collect(), ); - add_type_rollups(&mut type_rollups, stats); + add_node_label_rollups(&mut node_label_rollups, stats); add_timestamp_rollups(&mut timestamp_rollups, segment.segment_id, stats); add_property_rollups( &mut property_builders, @@ -1118,14 +1193,14 @@ fn build_planner_stats_view_from_snapshots_with_runtime_coverage( } full_rollup.coverage = full_coverage.finish(); - let type_coverage = type_coverage.finish(); + let node_label_coverage = node_label_coverage.finish(); let timestamp_coverage = timestamp_coverage.finish(); finalize_timestamp_rollup_coverage( &mut timestamp_rollups, - &type_rollups, + &node_label_rollups, all_segment_ids.clone(), - &segment_type_ids, - &segment_timestamp_type_ids, + &segment_label_ids, + &segment_timestamp_label_ids, ); let property_rollups = property_builders @@ -1170,10 +1245,10 @@ fn build_planner_stats_view_from_snapshots_with_runtime_coverage( missing_segment_stats, unavailable_segment_stats, full_rollup, - type_coverage, + node_label_coverage, timestamp_coverage, property_rollups, - type_rollups, + node_label_rollups, timestamp_rollups, equality_index_rollups, range_index_rollups, @@ -1314,7 +1389,7 @@ fn ready_equality_declarations( ) -> BTreeMap { let mut declarations = BTreeMap::new(); for entry in secondary_indexes { - let Some((type_id, prop_key)) = ready_node_property_target(entry) else { + let Some((target, target_label_id, prop_key)) = ready_property_target(entry) else { continue; }; if !matches!(entry.kind, SecondaryIndexKind::Equality) { @@ -1322,7 +1397,11 @@ fn ready_equality_declarations( } declarations.insert( entry.index_id, - EqualityIndexDeclaration { type_id, prop_key }, + EqualityIndexDeclaration { + target, + target_label_id, + prop_key, + }, ); } declarations @@ -1333,7 +1412,7 @@ fn ready_range_declarations( ) -> BTreeMap { let mut declarations = BTreeMap::new(); for entry in secondary_indexes { - let Some((type_id, prop_key)) = ready_node_property_target(entry) else { + let Some((target, target_label_id, prop_key)) = ready_property_target(entry) else { continue; }; let SecondaryIndexKind::Range { domain } = entry.kind else { @@ -1342,7 +1421,8 @@ fn ready_range_declarations( declarations.insert( entry.index_id, RangeIndexDeclaration { - type_id, + target, + target_label_id, prop_key, domain, }, @@ -1362,7 +1442,7 @@ fn equality_rollup_builders( EqualityRollupBuilder { stats: EqualityIndexRollupStats { index_id: *index_id, - type_id: declaration.type_id, + target_label_id: declaration.target_label_id, prop_key: declaration.prop_key.clone(), ..Default::default() }, @@ -1384,7 +1464,7 @@ fn range_rollup_builders( RangeRollupBuilder { stats: RangeIndexRollupStats { index_id: *index_id, - type_id: declaration.type_id, + target_label_id: declaration.target_label_id, prop_key: declaration.prop_key.clone(), domain: declaration.domain, ..Default::default() @@ -1396,35 +1476,46 @@ fn range_rollup_builders( builders } -fn ready_node_property_target(entry: &SecondaryIndexManifestEntry) -> Option<(u32, String)> { +fn ready_property_target( + entry: &SecondaryIndexManifestEntry, +) -> Option<(PlannerStatsDeclaredIndexTarget, u32, String)> { if entry.state != crate::types::SecondaryIndexState::Ready { return None; } match &entry.target { - SecondaryIndexTarget::NodeProperty { type_id, prop_key } => { - Some((*type_id, prop_key.clone())) - } + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => Some(( + PlannerStatsDeclaredIndexTarget::NodeProperty, + *label_id, + prop_key.clone(), + )), + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => Some(( + PlannerStatsDeclaredIndexTarget::EdgeProperty, + *label_id, + prop_key.clone(), + )), } } -fn add_type_rollups( - type_rollups: &mut BTreeMap, +fn add_node_label_rollups( + node_label_rollups: &mut BTreeMap, stats: &SegmentPlannerStatsV1, ) { - for type_stats in &stats.type_stats { - let rollup = type_rollups - .entry(type_stats.type_id) - .or_insert_with(|| TypeRollupStats { - type_id: type_stats.type_id, + for node_label_stats in &stats.node_label_stats { + let rollup = node_label_rollups + .entry(node_label_stats.label_id) + .or_insert_with(|| NodeLabelRollupStats { + label_id: node_label_stats.label_id, ..Default::default() }); - rollup.node_count = rollup.node_count.saturating_add(type_stats.node_count); - rollup.min_node_id = min_option(rollup.min_node_id, type_stats.min_node_id); - rollup.max_node_id = max_option(rollup.max_node_id, type_stats.max_node_id); + rollup.node_count = rollup + .node_count + .saturating_add(node_label_stats.node_count); + rollup.min_node_id = min_option(rollup.min_node_id, node_label_stats.min_node_id); + rollup.max_node_id = max_option(rollup.max_node_id, node_label_stats.max_node_id); rollup.min_updated_at_ms = - min_option(rollup.min_updated_at_ms, type_stats.min_updated_at_ms); + min_option(rollup.min_updated_at_ms, node_label_stats.min_updated_at_ms); rollup.max_updated_at_ms = - max_option(rollup.max_updated_at_ms, type_stats.max_updated_at_ms); + max_option(rollup.max_updated_at_ms, node_label_stats.max_updated_at_ms); } } @@ -1435,9 +1526,9 @@ fn add_timestamp_rollups( ) { for timestamp in &stats.timestamp_stats { let rollup = timestamp_rollups - .entry(timestamp.type_id) + .entry(timestamp.label_id) .or_insert_with(|| TimestampRollupStats { - type_id: timestamp.type_id, + label_id: timestamp.label_id, ..Default::default() }); rollup.count = rollup.count.saturating_add(timestamp.count); @@ -1457,31 +1548,31 @@ fn add_timestamp_rollups( fn finalize_timestamp_rollup_coverage( timestamp_rollups: &mut BTreeMap, - type_rollups: &BTreeMap, + node_label_rollups: &BTreeMap, all_segment_ids: Arc<[u64]>, - segment_type_ids: &BTreeMap>, - segment_timestamp_type_ids: &BTreeMap>, + segment_label_ids: &BTreeMap>, + segment_timestamp_label_ids: &BTreeMap>, ) { - let timestamp_type_ids: Vec = type_rollups + let timestamp_label_ids: Vec = node_label_rollups .keys() .chain(timestamp_rollups.keys()) .copied() .collect::>() .into_iter() .collect(); - for type_id in timestamp_type_ids { + for label_id in timestamp_label_ids { let mut coverage = CoverageBuilder::new(all_segment_ids.clone()); for segment_id in all_segment_ids.iter().copied() { - let Some(segment_types) = segment_type_ids.get(&segment_id) else { + let Some(segment_labels) = segment_label_ids.get(&segment_id) else { continue; }; - if !segment_types.contains(&type_id) { + if !segment_labels.contains(&label_id) { coverage.mark_covered(segment_id); continue; } - if segment_timestamp_type_ids + if segment_timestamp_label_ids .get(&segment_id) - .is_some_and(|timestamps| timestamps.contains(&type_id)) + .is_some_and(|timestamps| timestamps.contains(&label_id)) { coverage.mark_covered(segment_id); } else { @@ -1489,9 +1580,9 @@ fn finalize_timestamp_rollup_coverage( } } let rollup = timestamp_rollups - .entry(type_id) + .entry(label_id) .or_insert_with(|| TimestampRollupStats { - type_id, + label_id, ..Default::default() }); rollup.coverage = coverage.finish(); @@ -1505,12 +1596,12 @@ fn add_property_rollups( stats: &SegmentPlannerStatsV1, ) { for property in &stats.property_stats { - let key = (property.type_id, property.prop_key.clone()); + let key = (property.label_id, property.prop_key.clone()); let builder = property_builders .entry(key) .or_insert_with(|| PropertyRollupBuilder { stats: PropertyRollupStats { - type_id: property.type_id, + label_id: property.label_id, prop_key: property.prop_key.clone(), ..Default::default() }, @@ -1556,6 +1647,7 @@ fn add_equality_rollups( || !runtime_coverage.is_available( segment_id, index_stats.index_id, + declaration.target, PlannerStatsDeclaredIndexKind::Equality, ) { @@ -1613,6 +1705,7 @@ fn add_range_rollups( || !runtime_coverage.is_available( segment_id, index_stats.index_id, + declaration.target, PlannerStatsDeclaredIndexKind::Range, ) { @@ -1645,13 +1738,13 @@ fn add_adjacency_rollups( stats: &SegmentPlannerStatsV1, ) { for adjacency in &stats.adjacency_stats { - let key = (adjacency.direction, adjacency.edge_type_id); + let key = (adjacency.direction, adjacency.edge_label_id); let builder = adjacency_builders .entry(key) .or_insert_with(|| AdjacencyRollupBuilder { stats: AdjacencyRollupStats { direction: adjacency.direction, - edge_type_id: adjacency.edge_type_id, + edge_label_id: adjacency.edge_label_id, ..Default::default() }, coverage: CoverageBuilder::new(all_segment_ids.clone()), @@ -1692,7 +1785,7 @@ fn merge_adjacency_top_hubs( .cmp(&a.count) .then_with(|| a.node_id.cmp(&b.node_id)) }); - merged.truncate(PLANNER_STATS_TOP_HUBS_PER_EDGE_TYPE); + merged.truncate(PLANNER_STATS_TOP_HUBS_PER_EDGE_LABEL); *current = merged; } @@ -1701,13 +1794,16 @@ fn declared_equality_block_matches( declaration: &EqualityIndexDeclaration, declared_fingerprints: &DeclaredIndexFingerprintSet, ) -> bool { - if block.type_id != declaration.type_id || block.prop_key != declaration.prop_key { + if block.target_label_id != declaration.target_label_id + || block.prop_key != declaration.prop_key + { return false; } declared_fingerprints.contains(&declared_index_key( block.index_id, + declaration.target, PlannerStatsDeclaredIndexKind::Equality, - declaration.type_id, + declaration.target_label_id, &declaration.prop_key, None, )) @@ -1718,7 +1814,7 @@ fn declared_range_block_matches( declaration: &RangeIndexDeclaration, declared_fingerprints: &DeclaredIndexFingerprintSet, ) -> bool { - if block.type_id != declaration.type_id + if block.target_label_id != declaration.target_label_id || block.prop_key != declaration.prop_key || block.domain != declaration.domain { @@ -1726,8 +1822,9 @@ fn declared_range_block_matches( } declared_fingerprints.contains(&declared_index_key( block.index_id, + declaration.target, PlannerStatsDeclaredIndexKind::Range, - declaration.type_id, + declaration.target_label_id, &declaration.prop_key, Some(declaration.domain), )) @@ -1740,8 +1837,9 @@ fn declared_index_fingerprint_set(stats: &SegmentPlannerStatsV1) -> DeclaredInde .map(|declared| { declared_index_key( declared.index_id, + declared.target, declared.kind, - declared.type_id, + declared.target_label_id, &declared.prop_key, declared.range_domain, ) @@ -1751,15 +1849,17 @@ fn declared_index_fingerprint_set(stats: &SegmentPlannerStatsV1) -> DeclaredInde fn declared_index_key( index_id: u64, + target: PlannerStatsDeclaredIndexTarget, kind: PlannerStatsDeclaredIndexKind, - type_id: u32, + target_label_id: u32, prop_key: &str, range_domain: Option, -) -> (u64, u8, u32, String, u8) { +) -> (u64, u8, u8, u32, String, u8) { ( index_id, + declared_index_target_rank(target), declared_index_kind_rank(kind), - type_id, + target_label_id, prop_key.to_string(), range_domain_rank(range_domain), ) @@ -1964,7 +2064,7 @@ fn estimate_i64_histogram( } #[derive(Default)] -struct TypeAccumulator { +struct NodeLabelAccumulator { node_count: u64, min_node_id: Option, max_node_id: Option, @@ -1975,7 +2075,7 @@ struct TypeAccumulator { #[derive(Clone)] struct PropertyAccumulator { - type_id: u32, + label_id: u32, prop_key: String, tracked_reason: PropertyStatsTrackedReason, present_count: u64, @@ -1989,9 +2089,9 @@ struct PropertyAccumulator { } impl PropertyAccumulator { - fn new(type_id: u32, prop_key: String, tracked_reason: PropertyStatsTrackedReason) -> Self { + fn new(label_id: u32, prop_key: String, tracked_reason: PropertyStatsTrackedReason) -> Self { Self { - type_id, + label_id, prop_key, tracked_reason, present_count: 0, @@ -2084,7 +2184,7 @@ impl PropertyAccumulator { let top_values = top_value_frequencies(self.value_counts, PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY); PropertyPlannerStats { - type_id: self.type_id, + label_id: self.label_id, prop_key: self.prop_key, tracked_reason: self.tracked_reason, present_count: self.present_count, @@ -2163,6 +2263,7 @@ impl ValueKindCounts { } } +#[cfg(test)] pub(crate) fn read_planner_stats_sidecar( seg_dir: &Path, expected_segment_id: u64, @@ -2184,40 +2285,31 @@ pub(crate) fn read_planner_stats_sidecar( } } -pub(crate) fn write_flush_planner_stats_sidecar_best_effort( - seg_dir: &Path, - segment_id: u64, - nodes: &NodeIdMap, - edges: &NodeIdMap, - secondary_indexes: &[SecondaryIndexManifestEntry], -) { - let result = build_flush_stats(segment_id, seg_dir, nodes, edges, secondary_indexes) - .and_then(|stats| write_planner_stats_sidecar_atomic(seg_dir, stats).map(|_| ())); - if result.is_err() { - cleanup_stats_tmp(seg_dir); +pub(crate) fn read_planner_stats_payload( + data: &[u8], + expected_segment_id: u64, + expected_node_count: u64, + expected_edge_count: u64, +) -> PlannerStatsAvailability { + if data.is_empty() { + return PlannerStatsAvailability::Missing; } -} - -pub(crate) fn write_compaction_planner_stats_sidecar_best_effort( - seg_dir: &Path, - segment_id: u64, - segments: &[Arc], - node_metas: &[CompactNodeMeta], - edge_metas: &[CompactEdgeMeta], - secondary_indexes: &[SecondaryIndexManifestEntry], -) { - let result = build_compaction_stats( - segment_id, - seg_dir, - segments, - node_metas, - edge_metas, - secondary_indexes, - PlannerStatsBuildPolicy::compaction(), - ) - .and_then(|stats| write_planner_stats_sidecar_atomic(seg_dir, stats).map(|_| ())); - if result.is_err() { - cleanup_stats_tmp(seg_dir); + if data.len() > PLANNER_STATS_HARD_SIDECAR_BYTES { + return PlannerStatsAvailability::Unavailable { + reason: format!( + "planner stats sidecar exceeds hard cap: {} bytes", + data.len() + ), + }; + } + match decode_planner_stats_envelope( + data, + expected_segment_id, + expected_node_count, + expected_edge_count, + ) { + Ok(stats) => PlannerStatsAvailability::Available(Box::new(stats)), + Err(reason) => PlannerStatsAvailability::Unavailable { reason }, } } @@ -2252,7 +2344,7 @@ pub(crate) fn write_targeted_secondary_index_planner_stats_sidecar( let target_equality_stats = if matches!(target_index.kind, SecondaryIndexKind::Equality) { let mut stats = - build_equality_index_stats_from_sidecars(seg_dir, std::slice::from_ref(target_index))?; + build_equality_index_stats_from_segment(segment, std::slice::from_ref(target_index))?; let Some(stats) = stats.pop() else { return Ok(PlannerStatsWriteOutcome::SkippedTargetUnavailable); }; @@ -2265,7 +2357,7 @@ pub(crate) fn write_targeted_secondary_index_planner_stats_sidecar( }; let target_range_stats = if matches!(target_index.kind, SecondaryIndexKind::Range { .. }) { let mut stats = - build_range_index_stats_from_sidecars(seg_dir, std::slice::from_ref(target_index))?; + build_range_index_stats_from_segment(segment, std::slice::from_ref(target_index))?; let Some(stats) = stats.pop() else { return Ok(PlannerStatsWriteOutcome::SkippedTargetUnavailable); }; @@ -2277,44 +2369,34 @@ pub(crate) fn write_targeted_secondary_index_planner_stats_sidecar( None }; - let declared = declared_index_fingerprints(&ready_indexes); - let declaration_fingerprint = declaration_fingerprint(&declared); - let mut stats = match read_planner_stats_sidecar( + let written = publish_planner_stats_component_payload_from_latest( seg_dir, - segment.segment_id, - segment.node_count(), - segment.edge_count(), - ) { - PlannerStatsAvailability::Available(stats) => { - let mut stats = *stats; - retain_current_declared_index_stats(&mut stats, &ready_indexes, target_index.index_id); - stats - } - PlannerStatsAvailability::Missing | PlannerStatsAvailability::Unavailable { .. } => { - build_minimal_targeted_refresh_stats(segment)? - } - }; - - stats.build_kind = PlannerStatsBuildKind::SecondaryIndexRefresh; - stats.built_at_ms = 0; - stats.declared_indexes = declared; - stats.declaration_fingerprint = declaration_fingerprint; - stats.truncated |= !stats.general_property_stats_complete; - - if let Some(equality) = target_equality_stats { - stats.equality_index_stats.push(equality); - stats - .equality_index_stats - .sort_by_key(|index_stats| index_stats.index_id); - } - if let Some(range) = target_range_stats { - stats.range_index_stats.push(range); - stats - .range_index_stats - .sort_by_key(|index_stats| index_stats.index_id); + &ready_indexes, + |current_payload, segment_id, node_count, edge_count| { + let base_stats = current_payload + .and_then(|payload| { + match read_planner_stats_payload(payload, segment_id, node_count, edge_count) { + PlannerStatsAvailability::Available(stats) => Some(stats.as_ref().clone()), + PlannerStatsAvailability::Missing + | PlannerStatsAvailability::Unavailable { .. } => None, + } + }) + .map(Ok) + .unwrap_or_else(|| build_minimal_targeted_refresh_stats(segment))?; + let stats = merge_targeted_declared_index_stats( + base_stats, + &ready_indexes, + target_index.index_id, + target_equality_stats, + target_range_stats, + ); + planner_stats_sidecar_payload(stats) + }, + )?; + if !written { + return Ok(PlannerStatsWriteOutcome::SkippedOversize); } - - write_planner_stats_sidecar_atomic_cleanup_on_error(seg_dir, stats) + Ok(PlannerStatsWriteOutcome::Written) } pub(crate) fn planner_stats_declaration_fingerprint_for_entry( @@ -2323,49 +2405,45 @@ pub(crate) fn planner_stats_declaration_fingerprint_for_entry( declaration_fingerprint(&declared_index_fingerprints(std::slice::from_ref(entry))) } -pub(crate) fn build_flush_stats( - segment_id: u64, - seg_dir: &Path, +pub(crate) fn build_flush_stats_core_partial( nodes: &NodeIdMap, edges: &NodeIdMap, secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result { +) -> Result { let policy = PlannerStatsBuildPolicy::flush(); - let declared = declared_index_fingerprints(secondary_indexes); let declared_property_reasons = declared_property_reasons(secondary_indexes); - let mut type_accs = BTreeMap::new(); + let mut label_accs = BTreeMap::new(); let mut property_candidates = BTreeMap::new(); let mut sorted_nodes: Vec<&NodeRecord> = nodes.values().collect(); sorted_nodes.sort_unstable_by_key(|node| node.id); for node in &sorted_nodes { - observe_type(&mut type_accs, node.id, node.type_id, node.updated_at); - if policy.allow_general_property_decode { - observe_general_property_candidates( - &mut property_candidates, - &declared_property_reasons, - node.type_id, - &node.props, - ); + for &label_id in node.label_ids.as_slice() { + observe_label(&mut label_accs, node.id, label_id, node.updated_at); + if policy.allow_general_property_decode { + observe_general_property_candidates( + &mut property_candidates, + &declared_property_reasons, + label_id, + &node.props, + ); + } } } let mut property_accs = - seed_property_accumulators(&declared_property_reasons, property_candidates, &type_accs); + seed_property_accumulators(&declared_property_reasons, property_candidates, &label_accs); if policy.allow_general_property_decode { for node in &sorted_nodes { - observe_selected_node_properties(&mut property_accs, node.type_id, &node.props); + for &label_id in node.label_ids.as_slice() { + observe_selected_node_properties(&mut property_accs, label_id, &node.props); + } } } let mut sorted_edges: Vec<&EdgeRecord> = edges.values().collect(); sorted_edges.sort_unstable_by_key(|edge| edge.id); - let stats = SegmentPlannerStatsV1 { - format_version: PLANNER_STATS_FORMAT_VERSION, - segment_id, - build_kind: PlannerStatsBuildKind::Flush, - built_at_ms: 0, - declaration_fingerprint: declaration_fingerprint(&declared), - declared_indexes: declared, + let timestamp_stats = finalize_timestamp_stats_from_label_accs(&label_accs); + Ok(StatsCorePartial { node_count: sorted_nodes.len() as u64, edge_count: sorted_edges.len() as u64, truncated: false, @@ -2373,31 +2451,78 @@ pub(crate) fn build_flush_stats( general_property_sampled_node_count: sorted_nodes.len() as u64, general_property_sampled_raw_bytes: 0, general_property_budget_exhausted: false, - type_stats: finalize_type_stats(type_accs), - timestamp_stats: finalize_timestamp_stats(&sorted_nodes), + node_label_stats: finalize_node_label_stats(label_accs), + timestamp_stats, property_stats: finalize_property_stats(property_accs), - equality_index_stats: build_equality_index_stats_from_sidecars(seg_dir, secondary_indexes)?, - range_index_stats: build_range_index_stats_from_sidecars(seg_dir, secondary_indexes)?, adjacency_stats: build_adjacency_stats_from_edges(sorted_edges.iter().copied()), node_id_sample: node_id_sample(sorted_nodes.iter().map(|node| node.id)), - }; - Ok(stats) + }) +} + +pub(crate) fn assemble_flush_stats_from_partials( + segment_id: u64, + secondary_indexes: &[SecondaryIndexManifestEntry], + core: StatsCorePartial, + declared_evidence: DeclaredIndexStatsEvidence, +) -> SegmentPlannerStatsV1 { + assemble_stats_from_partials( + segment_id, + PlannerStatsBuildKind::Flush, + secondary_indexes, + core, + declared_evidence, + ) } -fn build_compaction_stats( +#[cfg(test)] +pub(crate) fn build_flush_stats( segment_id: u64, seg_dir: &Path, + nodes: &NodeIdMap, + edges: &NodeIdMap, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result { + let core = build_flush_stats_core_partial(nodes, edges, secondary_indexes)?; + let declared_evidence = DeclaredIndexStatsEvidence { + equality_index_stats: build_equality_index_stats_from_sidecars(seg_dir, secondary_indexes)?, + range_index_stats: build_range_index_stats_from_sidecars(seg_dir, secondary_indexes)?, + }; + Ok(assemble_flush_stats_from_partials( + segment_id, + secondary_indexes, + core, + declared_evidence, + )) +} + +pub(crate) fn build_compaction_stats_core_partial( + segments: &[Arc], + node_metas: &[CompactNodeMeta], + edge_metas: &[CompactEdgeMeta], + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result { + build_compaction_stats_core_partial_with_policy( + segments, + node_metas, + edge_metas, + secondary_indexes, + PlannerStatsBuildPolicy::compaction(), + ) +} + +fn build_compaction_stats_core_partial_with_policy( segments: &[Arc], node_metas: &[CompactNodeMeta], edge_metas: &[CompactEdgeMeta], secondary_indexes: &[SecondaryIndexManifestEntry], policy: PlannerStatsBuildPolicy, -) -> Result { - let declared = declared_index_fingerprints(secondary_indexes); +) -> Result { let declared_property_reasons = declared_property_reasons(secondary_indexes); - let mut type_accs = BTreeMap::new(); + let mut label_accs = BTreeMap::new(); for meta in node_metas { - observe_type(&mut type_accs, meta.node_id, meta.type_id, meta.updated_at); + for &label_id in meta.label_ids.as_slice() { + observe_label(&mut label_accs, meta.node_id, label_id, meta.updated_at); + } } let mut property_candidates = BTreeMap::new(); @@ -2421,33 +2546,32 @@ fn build_compaction_stats( meta.src_data_offset, meta.node_id, )?; - observe_general_property_candidates( - &mut property_candidates, - &declared_property_reasons, - meta.type_id, - &props, - ); - sampled_props.push((meta.type_id, props)); + for &label_id in meta.label_ids.as_slice() { + observe_general_property_candidates( + &mut property_candidates, + &declared_property_reasons, + label_id, + &props, + ); + } + sampled_props.push((meta.label_ids, props)); sampled_node_count += 1; sampled_raw_bytes = next_bytes; } } let mut property_accs = - seed_property_accumulators(&declared_property_reasons, property_candidates, &type_accs); - for (type_id, props) in &sampled_props { - observe_selected_node_properties(&mut property_accs, *type_id, props); + seed_property_accumulators(&declared_property_reasons, property_candidates, &label_accs); + for (label_ids, props) in &sampled_props { + for &label_id in label_ids.as_slice() { + observe_selected_node_properties(&mut property_accs, label_id, props); + } } let general_property_stats_complete = sampled_node_count == node_metas.len() as u64 && !budget_exhausted; let edge_refs = edge_metas.iter().map(EdgeMetaRef::from); - Ok(SegmentPlannerStatsV1 { - format_version: PLANNER_STATS_FORMAT_VERSION, - segment_id, - build_kind: PlannerStatsBuildKind::Compaction, - built_at_ms: 0, - declaration_fingerprint: declaration_fingerprint(&declared), - declared_indexes: declared, + let timestamp_stats = finalize_timestamp_stats_from_label_accs(&label_accs); + Ok(StatsCorePartial { node_count: node_metas.len() as u64, edge_count: edge_metas.len() as u64, truncated: !general_property_stats_complete, @@ -2455,114 +2579,311 @@ fn build_compaction_stats( general_property_sampled_node_count: sampled_node_count, general_property_sampled_raw_bytes: sampled_raw_bytes, general_property_budget_exhausted: budget_exhausted, - type_stats: finalize_type_stats(type_accs), - timestamp_stats: finalize_timestamp_stats_from_meta(node_metas), + node_label_stats: finalize_node_label_stats(label_accs), + timestamp_stats, property_stats: finalize_property_stats(property_accs), - equality_index_stats: build_equality_index_stats_from_sidecars(seg_dir, secondary_indexes)?, - range_index_stats: build_range_index_stats_from_sidecars(seg_dir, secondary_indexes)?, adjacency_stats: build_adjacency_stats_from_edge_meta(edge_refs), node_id_sample: node_id_sample(node_metas.iter().map(|meta| meta.node_id)), }) } +pub(crate) fn assemble_compaction_stats_from_partials( + segment_id: u64, + secondary_indexes: &[SecondaryIndexManifestEntry], + core: StatsCorePartial, + declared_evidence: DeclaredIndexStatsEvidence, +) -> SegmentPlannerStatsV1 { + assemble_stats_from_partials( + segment_id, + PlannerStatsBuildKind::Compaction, + secondary_indexes, + core, + declared_evidence, + ) +} + +#[cfg(test)] +pub(crate) fn build_compaction_stats( + segment_id: u64, + seg_dir: &Path, + segments: &[Arc], + node_metas: &[CompactNodeMeta], + edge_metas: &[CompactEdgeMeta], + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result { + let core = + build_compaction_stats_core_partial(segments, node_metas, edge_metas, secondary_indexes)?; + let declared_evidence = DeclaredIndexStatsEvidence { + equality_index_stats: build_equality_index_stats_from_sidecars(seg_dir, secondary_indexes)?, + range_index_stats: build_range_index_stats_from_sidecars(seg_dir, secondary_indexes)?, + }; + Ok(assemble_compaction_stats_from_partials( + segment_id, + secondary_indexes, + core, + declared_evidence, + )) +} + +fn assemble_stats_from_partials( + segment_id: u64, + build_kind: PlannerStatsBuildKind, + secondary_indexes: &[SecondaryIndexManifestEntry], + core: StatsCorePartial, + mut declared_evidence: DeclaredIndexStatsEvidence, +) -> SegmentPlannerStatsV1 { + declared_evidence.sort(); + let declared = declared_index_fingerprints(secondary_indexes); + SegmentPlannerStatsV1 { + format_version: PLANNER_STATS_FORMAT_VERSION, + segment_id, + build_kind, + built_at_ms: 0, + declaration_fingerprint: declaration_fingerprint(&declared), + declared_indexes: declared, + node_count: core.node_count, + edge_count: core.edge_count, + truncated: core.truncated, + general_property_stats_complete: core.general_property_stats_complete, + general_property_sampled_node_count: core.general_property_sampled_node_count, + general_property_sampled_raw_bytes: core.general_property_sampled_raw_bytes, + general_property_budget_exhausted: core.general_property_budget_exhausted, + node_label_stats: core.node_label_stats, + timestamp_stats: core.timestamp_stats, + property_stats: core.property_stats, + equality_index_stats: declared_evidence.equality_index_stats, + range_index_stats: declared_evidence.range_index_stats, + adjacency_stats: core.adjacency_stats, + node_id_sample: core.node_id_sample, + } +} + +#[cfg(test)] fn build_equality_index_stats_from_sidecars( seg_dir: &Path, secondary_indexes: &[SecondaryIndexManifestEntry], ) -> Result, EngineError> { let mut result = Vec::new(); for entry in secondary_indexes { - if !matches!(entry.kind, SecondaryIndexKind::Equality) { + if entry.state != SecondaryIndexState::Ready + || !matches!(entry.kind, SecondaryIndexKind::Equality) + { continue; } - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; - let path = seg_dir - .join("secondary_indexes") - .join(format!("node_prop_eq_{}.dat", entry.index_id)); + let file_name = match &entry.target { + SecondaryIndexTarget::NodeProperty { .. } => { + format!("node_prop_eq_{}.dat", entry.index_id) + } + SecondaryIndexTarget::EdgeProperty { .. } => { + format!("edge_prop_eq_{}.dat", entry.index_id) + } + }; + let path = seg_dir.join("secondary_indexes").join(file_name); let groups = read_secondary_eq_group_counts(&path)?; let sidecar_present_at_build = groups.is_some(); let groups = groups.unwrap_or_default(); - let mut value_counts = BTreeMap::new(); - let mut total_postings = 0u64; - let mut max_group_postings = 0u64; - for (&value_hash, &count) in &groups { - total_postings += count; - max_group_postings = max_group_postings.max(count); - value_counts.insert(value_hash, count); - } - result.push(EqualityIndexPlannerStats { - index_id: entry.index_id, - type_id: *type_id, - prop_key: prop_key.clone(), - total_postings, - value_group_count: groups.len() as u64, - max_group_postings, - top_value_hashes: top_value_frequencies( - value_counts, - PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY, - ), + result.push(equality_index_stats_from_group_counts( + entry, + &groups, sidecar_present_at_build, - }); + )); } result.sort_by_key(|stats| stats.index_id); Ok(result) } -fn build_range_index_stats_from_sidecars( - seg_dir: &Path, +fn build_equality_index_stats_from_segment( + segment: &SegmentReader, secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result, EngineError> { +) -> Result, EngineError> { let mut result = Vec::new(); for entry in secondary_indexes { - let SecondaryIndexKind::Range { domain } = entry.kind else { + if entry.state != SecondaryIndexState::Ready + || !matches!(entry.kind, SecondaryIndexKind::Equality) + { continue; - }; - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; - let path = seg_dir - .join("secondary_indexes") - .join(format!("node_prop_range_{}.dat", entry.index_id)); - let encoded_values = read_secondary_range_encoded_values(&path)?; - let sidecar_present_at_build = encoded_values.is_some(); - let mut encoded_values = encoded_values.unwrap_or_default(); - encoded_values.sort_unstable(); - let min_encoded = encoded_values.first().copied(); - let max_encoded = encoded_values.last().copied(); - let buckets = range_buckets(&encoded_values, PLANNER_STATS_RANGE_BUCKETS); - result.push(RangeIndexPlannerStats { - index_id: entry.index_id, - type_id: *type_id, - prop_key: prop_key.clone(), - domain, - total_entries: encoded_values.len() as u64, - min_encoded, - max_encoded, - buckets, + } + let mut groups = BTreeMap::new(); + let sidecar_present_at_build = + segment.for_each_declared_secondary_eq_group(entry, |value_hash, ids| { + groups.insert(value_hash, ids.len() as u64); + Ok(()) + })?; + result.push(equality_index_stats_from_group_counts( + entry, + &groups, sidecar_present_at_build, - }); + )); } result.sort_by_key(|stats| stats.index_id); Ok(result) } -fn ready_planner_stats_indexes( - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Vec { - let mut indexes: Vec<_> = secondary_indexes +pub(crate) fn equality_index_stats_from_written_groups( + entry: &SecondaryIndexManifestEntry, + groups: &BTreeMap>, +) -> EqualityIndexPlannerStats { + let group_counts: BTreeMap = groups .iter() - .filter(|entry| entry.state == SecondaryIndexState::Ready) - .cloned() + .map(|(&value_hash, ids)| (value_hash, ids.len() as u64)) .collect(); - indexes.sort_by_key(|entry| entry.index_id); - indexes + equality_index_stats_from_group_counts(entry, &group_counts, true) } -fn planner_stats_declaration_matches( - left: &SecondaryIndexManifestEntry, - right: &SecondaryIndexManifestEntry, -) -> bool { - left.index_id == right.index_id - && left.kind == right.kind - && left.target == right.target - && left.state == SecondaryIndexState::Ready +pub(crate) fn equality_index_stats_from_group_counts( + entry: &SecondaryIndexManifestEntry, + groups: &BTreeMap, + sidecar_present_at_build: bool, +) -> EqualityIndexPlannerStats { + let (target_label_id, prop_key) = match &entry.target { + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => (*label_id, prop_key), + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => (*label_id, prop_key), + }; + let mut value_counts = BTreeMap::new(); + let mut total_postings = 0u64; + let mut max_group_postings = 0u64; + for (&value_hash, &count) in groups { + total_postings += count; + max_group_postings = max_group_postings.max(count); + value_counts.insert(value_hash, count); + } + EqualityIndexPlannerStats { + index_id: entry.index_id, + target_label_id, + prop_key: prop_key.clone(), + total_postings, + value_group_count: groups.len() as u64, + max_group_postings, + top_value_hashes: top_value_frequencies( + value_counts, + PLANNER_STATS_MAX_HEAVY_HITTERS_PER_KEY, + ), + sidecar_present_at_build, + } +} + +#[cfg(test)] +fn build_range_index_stats_from_sidecars( + seg_dir: &Path, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result, EngineError> { + let mut result = Vec::new(); + for entry in secondary_indexes { + if entry.state != SecondaryIndexState::Ready { + continue; + } + let SecondaryIndexKind::Range { .. } = entry.kind else { + continue; + }; + let file_name = match &entry.target { + SecondaryIndexTarget::NodeProperty { .. } => { + format!("node_prop_range_{}.dat", entry.index_id) + } + SecondaryIndexTarget::EdgeProperty { .. } => { + format!("edge_prop_range_{}.dat", entry.index_id) + } + }; + let path = seg_dir.join("secondary_indexes").join(file_name); + let encoded_values = read_secondary_range_encoded_values(&path)?; + let sidecar_present_at_build = encoded_values.is_some(); + result.push(range_index_stats_from_encoded_values( + entry, + &encoded_values.unwrap_or_default(), + sidecar_present_at_build, + )); + } + result.sort_by_key(|stats| stats.index_id); + Ok(result) +} + +fn build_range_index_stats_from_segment( + segment: &SegmentReader, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result, EngineError> { + let mut result = Vec::new(); + for entry in secondary_indexes { + if entry.state != SecondaryIndexState::Ready { + continue; + } + let SecondaryIndexKind::Range { .. } = entry.kind else { + continue; + }; + let mut encoded_values = Vec::new(); + let sidecar_present_at_build = segment.for_each_declared_secondary_range_entry( + entry, + |encoded_value, _record_id| { + encoded_values.push(encoded_value); + Ok(()) + }, + )?; + result.push(range_index_stats_from_encoded_values( + entry, + &encoded_values, + sidecar_present_at_build, + )); + } + result.sort_by_key(|stats| stats.index_id); + Ok(result) +} + +pub(crate) fn range_index_stats_from_written_entries( + entry: &SecondaryIndexManifestEntry, + entries: &[(u64, u64)], +) -> RangeIndexPlannerStats { + let encoded_values: Vec = entries + .iter() + .map(|(encoded_value, _node_id)| *encoded_value) + .collect(); + range_index_stats_from_encoded_values(entry, &encoded_values, true) +} + +pub(crate) fn range_index_stats_from_encoded_values( + entry: &SecondaryIndexManifestEntry, + encoded_values: &[u64], + sidecar_present_at_build: bool, +) -> RangeIndexPlannerStats { + let SecondaryIndexKind::Range { domain } = entry.kind else { + unreachable!("range stats require a range secondary index") + }; + let (target_label_id, prop_key) = match &entry.target { + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => (*label_id, prop_key), + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => (*label_id, prop_key), + }; + let mut encoded_values = encoded_values.to_vec(); + encoded_values.sort_unstable(); + RangeIndexPlannerStats { + index_id: entry.index_id, + target_label_id, + prop_key: prop_key.clone(), + domain, + total_entries: encoded_values.len() as u64, + min_encoded: encoded_values.first().copied(), + max_encoded: encoded_values.last().copied(), + buckets: range_buckets(&encoded_values, PLANNER_STATS_RANGE_BUCKETS), + sidecar_present_at_build, + } +} + +fn ready_planner_stats_indexes( + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Vec { + let mut indexes: Vec<_> = secondary_indexes + .iter() + .filter(|entry| entry.state == SecondaryIndexState::Ready) + .cloned() + .collect(); + indexes.sort_by_key(|entry| entry.index_id); + indexes +} + +fn planner_stats_declaration_matches( + left: &SecondaryIndexManifestEntry, + right: &SecondaryIndexManifestEntry, +) -> bool { + left.index_id == right.index_id + && left.kind == right.kind + && left.target == right.target + && left.state == SecondaryIndexState::Ready && right.state == SecondaryIndexState::Ready } @@ -2577,10 +2898,10 @@ fn retain_current_declared_index_stats( && ready_indexes.iter().any(|entry| { matches!(entry.kind, SecondaryIndexKind::Equality) && entry.index_id == block.index_id - && matches!( + && secondary_index_target_matches_stats( &entry.target, - SecondaryIndexTarget::NodeProperty { type_id, prop_key } - if *type_id == block.type_id && prop_key == &block.prop_key + block.target_label_id, + &block.prop_key, ) }) }); @@ -2590,40 +2911,75 @@ fn retain_current_declared_index_stats( && ready_indexes.iter().any(|entry| { matches!(entry.kind, SecondaryIndexKind::Range { domain } if domain == block.domain) && entry.index_id == block.index_id - && matches!( + && secondary_index_target_matches_stats( &entry.target, - SecondaryIndexTarget::NodeProperty { type_id, prop_key } - if *type_id == block.type_id && prop_key == &block.prop_key + block.target_label_id, + &block.prop_key, ) }) }); } +fn merge_targeted_declared_index_stats( + mut stats: SegmentPlannerStatsV1, + ready_indexes: &[SecondaryIndexManifestEntry], + target_index_id: u64, + target_equality_stats: Option, + target_range_stats: Option, +) -> SegmentPlannerStatsV1 { + let declared = declared_index_fingerprints(ready_indexes); + let declaration_fingerprint = declaration_fingerprint(&declared); + retain_current_declared_index_stats(&mut stats, ready_indexes, target_index_id); + + stats.build_kind = PlannerStatsBuildKind::SecondaryIndexRefresh; + stats.built_at_ms = 0; + stats.declared_indexes = declared; + stats.declaration_fingerprint = declaration_fingerprint; + stats.truncated |= !stats.general_property_stats_complete; + + if let Some(equality) = target_equality_stats { + stats.equality_index_stats.push(equality); + } + if let Some(range) = target_range_stats { + stats.range_index_stats.push(range); + } + stats + .equality_index_stats + .sort_by_key(|index_stats| index_stats.index_id); + stats + .range_index_stats + .sort_by_key(|index_stats| index_stats.index_id); + stats +} + +fn secondary_index_target_matches_stats( + target: &SecondaryIndexTarget, + target_label_id: u32, + prop_key: &str, +) -> bool { + match target { + SecondaryIndexTarget::NodeProperty { + label_id: expected_label_id, + prop_key: target_prop_key, + } + | SecondaryIndexTarget::EdgeProperty { + label_id: expected_label_id, + prop_key: target_prop_key, + } => *expected_label_id == target_label_id && target_prop_key == prop_key, + } +} + fn build_minimal_targeted_refresh_stats( segment: &SegmentReader, ) -> Result { - let mut type_accs = BTreeMap::new(); - let mut timestamp_groups: BTreeMap> = BTreeMap::new(); + let mut label_accs = BTreeMap::new(); let mut node_ids = Vec::with_capacity(segment.node_meta_count() as usize); for index in 0..segment.node_meta_count() as usize { - let ( - node_id, - _data_offset, - _data_len, - type_id, - updated_at, - _weight, - _key_len, - _prop_hash_offset, - _prop_hash_count, - _last_write_seq, - ) = segment.node_meta_at(index)?; - observe_type(&mut type_accs, node_id, type_id, updated_at); - timestamp_groups - .entry(type_id) - .or_default() - .push(updated_at); - node_ids.push(node_id); + let meta = segment.node_meta_at(index)?; + for &label_id in meta.label_ids.as_slice() { + observe_label(&mut label_accs, meta.node_id, label_id, meta.updated_at); + } + node_ids.push(meta.node_id); } let mut edge_refs = Vec::with_capacity(segment.edge_meta_count() as usize); @@ -2634,16 +2990,17 @@ fn build_minimal_targeted_refresh_stats( _data_len, from, to, - type_id, + label_id, _updated_at, _weight, _valid_from, _valid_to, _last_write_seq, ) = segment.edge_meta_at(index)?; - edge_refs.push(EdgeMetaRef { type_id, from, to }); + edge_refs.push(EdgeMetaRef { label_id, from, to }); } + let timestamp_stats = finalize_timestamp_stats_from_label_accs(&label_accs); Ok(SegmentPlannerStatsV1 { format_version: PLANNER_STATS_FORMAT_VERSION, segment_id: segment.segment_id, @@ -2658,8 +3015,8 @@ fn build_minimal_targeted_refresh_stats( general_property_sampled_node_count: 0, general_property_sampled_raw_bytes: 0, general_property_budget_exhausted: segment.node_count() > 0, - type_stats: finalize_type_stats(type_accs), - timestamp_stats: finalize_timestamp_groups(timestamp_groups), + node_label_stats: finalize_node_label_stats(label_accs), + timestamp_stats, property_stats: Vec::new(), equality_index_stats: Vec::new(), range_index_stats: Vec::new(), @@ -2668,16 +3025,12 @@ fn build_minimal_targeted_refresh_stats( }) } +#[cfg(test)] pub(crate) fn write_planner_stats_sidecar_atomic( seg_dir: &Path, stats: SegmentPlannerStatsV1, ) -> Result { - let Some(payload) = serialize_stats_with_limits( - stats, - PLANNER_STATS_SOFT_SIDECAR_BYTES, - PLANNER_STATS_HARD_SIDECAR_BYTES, - )? - else { + let Some(payload) = planner_stats_sidecar_payload(stats)? else { cleanup_stats_tmp(seg_dir); return Ok(PlannerStatsWriteOutcome::SkippedOversize); }; @@ -2693,6 +3046,7 @@ pub(crate) fn write_planner_stats_sidecar_atomic( Ok(PlannerStatsWriteOutcome::Written) } +#[cfg(test)] fn write_planner_stats_sidecar_atomic_cleanup_on_error( seg_dir: &Path, stats: SegmentPlannerStatsV1, @@ -2704,6 +3058,16 @@ fn write_planner_stats_sidecar_atomic_cleanup_on_error( result } +pub(crate) fn planner_stats_sidecar_payload( + stats: SegmentPlannerStatsV1, +) -> Result>, EngineError> { + serialize_stats_with_limits( + stats, + PLANNER_STATS_SOFT_SIDECAR_BYTES, + PLANNER_STATS_HARD_SIDECAR_BYTES, + ) +} + fn serialize_stats_with_limits( mut stats: SegmentPlannerStatsV1, soft_limit: usize, @@ -2826,11 +3190,13 @@ fn encode_enveloped_stats(stats: &SegmentPlannerStatsV1) -> Result, Engi Ok(data) } +#[cfg(test)] enum PlannerStatsReadFailure { Missing, Unavailable(String), } +#[cfg(test)] fn read_planner_stats_file( path: &Path, expected_segment_id: u64, @@ -2848,16 +3214,17 @@ fn read_planner_stats_file( .metadata() .map_err(|error| PlannerStatsReadFailure::Unavailable(error.to_string()))? .len(); - if file_len > PLANNER_STATS_HARD_SIDECAR_BYTES as u64 { + if file_len > (PLANNER_STATS_HARD_SIDECAR_BYTES + COMPONENT_IDENTITY_HEADER_LEN) as u64 { return Err(PlannerStatsReadFailure::Unavailable(format!( "planner stats sidecar exceeds hard cap: {} bytes", file_len ))); } let mut data = Vec::with_capacity(file_len as usize); - file.take((PLANNER_STATS_HARD_SIDECAR_BYTES + 1) as u64) + file.take(file_len.saturating_add(1)) .read_to_end(&mut data) .map_err(|error| PlannerStatsReadFailure::Unavailable(error.to_string()))?; + let data = planner_stats_payload_slice(&data)?; if data.len() > PLANNER_STATS_HARD_SIDECAR_BYTES { return Err(PlannerStatsReadFailure::Unavailable(format!( "planner stats sidecar exceeds hard cap: {} bytes", @@ -2865,7 +3232,7 @@ fn read_planner_stats_file( ))); } decode_planner_stats_envelope( - &data, + data, expected_segment_id, expected_node_count, expected_edge_count, @@ -2873,6 +3240,34 @@ fn read_planner_stats_file( .map_err(PlannerStatsReadFailure::Unavailable) } +#[cfg(test)] +fn planner_stats_payload_slice(data: &[u8]) -> Result<&[u8], PlannerStatsReadFailure> { + if data.len() >= COMPONENT_IDENTITY_HEADER_LEN + && data[0..COMPONENT_IDENTITY_HEADER_MAGIC.len()] == COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = decode_identity_header(data) + .map_err(|error| PlannerStatsReadFailure::Unavailable(error.to_string()))?; + let end = header + .payload_offset + .checked_add(header.payload_len) + .ok_or_else(|| { + PlannerStatsReadFailure::Unavailable( + "planner stats identity payload range overflow".into(), + ) + })?; + if end > data.len() as u64 { + return Err(PlannerStatsReadFailure::Unavailable(format!( + "planner stats identity payload range [{}, {}) exceeds file length {}", + header.payload_offset, + end, + data.len() + ))); + } + return Ok(&data[header.payload_offset as usize..end as usize]); + } + Ok(data) +} + fn decode_planner_stats_envelope( data: &[u8], expected_segment_id: u64, @@ -2970,76 +3365,92 @@ fn validate_stats_payload( { return Err("planner stats node id sample is not sorted".to_string()); } - let type_counts = validate_type_stats(stats)?; - validate_timestamp_stats(stats, &type_counts)?; - validate_property_stats(stats, &type_counts)?; - validate_declared_index_stats(stats, &type_counts)?; + let label_counts = validate_node_label_stats(stats)?; + validate_timestamp_stats(stats, &label_counts)?; + validate_property_stats(stats, &label_counts)?; + validate_declared_index_stats(stats, &label_counts)?; validate_adjacency_stats(stats)?; Ok(()) } -fn validate_type_stats(stats: &SegmentPlannerStatsV1) -> Result, String> { - let mut type_counts = BTreeMap::new(); +fn validate_node_label_stats(stats: &SegmentPlannerStatsV1) -> Result, String> { + let mut label_counts = BTreeMap::new(); let mut total = 0u64; - for type_stat in &stats.type_stats { - if type_stat.node_count == 0 { + for label_stat in &stats.node_label_stats { + if label_stat.node_count == 0 { return Err(format!( - "planner stats type {} has zero node count", - type_stat.type_id + "planner stats label {} has zero node count", + label_stat.label_id )); } - if type_counts - .insert(type_stat.type_id, type_stat.node_count) + if label_stat.node_count > stats.node_count { + return Err(format!( + "planner stats label {} node count {} exceeds segment node count {}", + label_stat.label_id, label_stat.node_count, stats.node_count + )); + } + if label_counts + .insert(label_stat.label_id, label_stat.node_count) .is_some() { return Err(format!( - "planner stats type {} appears more than once", - type_stat.type_id + "planner stats label {} appears more than once", + label_stat.label_id )); } - total = checked_add_count(total, type_stat.node_count, "type node counts")?; + total = checked_add_count(total, label_stat.node_count, "node label counts")?; validate_ordered_option_pair( - type_stat.min_node_id, - type_stat.max_node_id, - "type node id bounds", + label_stat.min_node_id, + label_stat.max_node_id, + "node label id bounds", )?; validate_ordered_option_pair( - type_stat.min_updated_at_ms, - type_stat.max_updated_at_ms, - "type updated-at bounds", + label_stat.min_updated_at_ms, + label_stat.max_updated_at_ms, + "node label updated-at bounds", )?; } - if total != stats.node_count { + if stats.node_label_stats.len() <= 1 && total != stats.node_count { return Err(format!( - "planner stats type counts sum to {}, expected {}", + "planner stats label counts sum to {}, expected {} for single-label stats", total, stats.node_count )); } - Ok(type_counts) + let max_memberships = stats + .node_count + .checked_mul(MAX_NODE_LABELS_PER_NODE as u64) + .ok_or_else(|| "planner stats label count bound overflow".to_string())?; + if total < stats.node_count || total > max_memberships { + return Err(format!( + "planner stats label counts sum to {}, expected between {} and {}", + total, stats.node_count, max_memberships + )); + } + Ok(label_counts) } fn validate_timestamp_stats( stats: &SegmentPlannerStatsV1, - type_counts: &BTreeMap, + label_counts: &BTreeMap, ) -> Result<(), String> { let mut seen = BTreeMap::new(); for timestamp in &stats.timestamp_stats { - let Some(type_count) = type_counts.get(×tamp.type_id) else { + let Some(label_count) = label_counts.get(×tamp.label_id) else { return Err(format!( - "planner stats timestamp section references unknown type {}", - timestamp.type_id + "planner stats timestamp section references unknown label {}", + timestamp.label_id )); }; - if seen.insert(timestamp.type_id, ()).is_some() { + if seen.insert(timestamp.label_id, ()).is_some() { return Err(format!( - "planner stats timestamp section repeats type {}", - timestamp.type_id + "planner stats timestamp section repeats label {}", + timestamp.label_id )); } - if timestamp.count != *type_count { + if timestamp.count != *label_count { return Err(format!( - "planner stats timestamp count for type {} is {}, expected {}", - timestamp.type_id, timestamp.count, type_count + "planner stats timestamp count for label {} is {}, expected {}", + timestamp.label_id, timestamp.count, label_count )); } if timestamp.min_ms > timestamp.max_ms { @@ -3052,26 +3463,26 @@ fn validate_timestamp_stats( fn validate_property_stats( stats: &SegmentPlannerStatsV1, - type_counts: &BTreeMap, + label_counts: &BTreeMap, ) -> Result<(), String> { let mut seen = BTreeMap::new(); for prop in &stats.property_stats { - let Some(type_count) = type_counts.get(&prop.type_id) else { + let Some(label_count) = label_counts.get(&prop.label_id) else { return Err(format!( - "planner stats property {} references unknown type {}", - prop.prop_key, prop.type_id + "planner stats property {} references unknown label {}", + prop.prop_key, prop.label_id )); }; - let key = (prop.type_id, prop.prop_key.as_str()); + let key = (prop.label_id, prop.prop_key.as_str()); if seen.insert(key, ()).is_some() { return Err(format!( - "planner stats property {} for type {} appears more than once", - prop.prop_key, prop.type_id + "planner stats property {} for label {} appears more than once", + prop.prop_key, prop.label_id )); } - if prop.present_count > *type_count { + if prop.present_count > *label_count { return Err(format!( - "planner stats property {} present count exceeds type count", + "planner stats property {} present count exceeds label count", prop.prop_key )); } @@ -3131,9 +3542,10 @@ fn validate_property_stats( fn validate_declared_index_stats( stats: &SegmentPlannerStatsV1, - type_counts: &BTreeMap, + label_counts: &BTreeMap, ) -> Result<(), String> { let declared = declared_index_map(stats)?; + let edge_label_counts = edge_label_counts_from_adjacency_stats(stats); let mut equality_seen = BTreeMap::new(); for equality in &stats.equality_index_stats { let Some(declared_index) = declared.get(&equality.index_id) else { @@ -3143,7 +3555,7 @@ fn validate_declared_index_stats( )); }; if declared_index.kind != PlannerStatsDeclaredIndexKind::Equality - || declared_index.type_id != equality.type_id + || declared_index.target_label_id != equality.target_label_id || declared_index.prop_key != equality.prop_key { return Err(format!( @@ -3157,10 +3569,11 @@ fn validate_declared_index_stats( equality.index_id )); } - let type_count = *type_counts.get(&equality.type_id).unwrap_or(&0); - if equality.total_postings > type_count { + let target_count = + declared_index_target_count(declared_index, label_counts, &edge_label_counts); + if equality.total_postings > target_count { return Err(format!( - "planner stats equality index {} postings exceed type count", + "planner stats equality index {} postings exceed target count", equality.index_id )); } @@ -3192,7 +3605,7 @@ fn validate_declared_index_stats( )); }; if declared_index.kind != PlannerStatsDeclaredIndexKind::Range - || declared_index.type_id != range.type_id + || declared_index.target_label_id != range.target_label_id || declared_index.prop_key != range.prop_key || declared_index.range_domain != Some(range.domain) { @@ -3207,10 +3620,11 @@ fn validate_declared_index_stats( range.index_id )); } - let type_count = *type_counts.get(&range.type_id).unwrap_or(&0); - if range.total_entries > type_count { + let target_count = + declared_index_target_count(declared_index, label_counts, &edge_label_counts); + if range.total_entries > target_count { return Err(format!( - "planner stats range index {} entries exceed type count", + "planner stats range index {} entries exceed target count", range.index_id )); } @@ -3220,6 +3634,35 @@ fn validate_declared_index_stats( Ok(()) } +fn edge_label_counts_from_adjacency_stats(stats: &SegmentPlannerStatsV1) -> BTreeMap { + let mut counts: BTreeMap = BTreeMap::new(); + for adjacency in &stats.adjacency_stats { + let Some(edge_label_id) = adjacency.edge_label_id else { + continue; + }; + counts + .entry(edge_label_id) + .and_modify(|count| *count = (*count).max(adjacency.total_edges)) + .or_insert(adjacency.total_edges); + } + counts +} + +fn declared_index_target_count( + declared_index: &DeclaredIndexStatsFingerprint, + node_label_counts: &BTreeMap, + edge_label_counts: &BTreeMap, +) -> u64 { + match declared_index.target { + PlannerStatsDeclaredIndexTarget::NodeProperty => *node_label_counts + .get(&declared_index.target_label_id) + .unwrap_or(&0), + PlannerStatsDeclaredIndexTarget::EdgeProperty => *edge_label_counts + .get(&declared_index.target_label_id) + .unwrap_or(&0), + } +} + fn declared_index_map( stats: &SegmentPlannerStatsV1, ) -> Result, String> { @@ -3238,7 +3681,7 @@ fn declared_index_map( fn validate_adjacency_stats(stats: &SegmentPlannerStatsV1) -> Result<(), String> { let mut seen = BTreeMap::new(); for adjacency in &stats.adjacency_stats { - let key = (adjacency.direction, adjacency.edge_type_id); + let key = (adjacency.direction, adjacency.edge_label_id); if seen.insert(key, ()).is_some() { return Err("planner stats adjacency section repeats a direction/type".to_string()); } @@ -3253,7 +3696,7 @@ fn validate_adjacency_stats(stats: &SegmentPlannerStatsV1) -> Result<(), String> if adjacency.total_edges > stats.edge_count { return Err("planner stats adjacency total exceeds segment edge count".to_string()); } - if adjacency.edge_type_id.is_none() && adjacency.total_edges != stats.edge_count { + if adjacency.edge_label_id.is_none() && adjacency.total_edges != stats.edge_count { return Err( "planner stats global adjacency total does not match edge count".to_string(), ); @@ -3409,13 +3852,13 @@ fn validate_ordered_option_pair( } } -fn observe_type( - type_accs: &mut BTreeMap, +fn observe_label( + label_accs: &mut BTreeMap, node_id: u64, - type_id: u32, + label_id: u32, updated_at_ms: i64, ) { - let acc = type_accs.entry(type_id).or_default(); + let acc = label_accs.entry(label_id).or_default(); acc.node_count += 1; acc.min_node_id = Some(acc.min_node_id.map_or(node_id, |value| value.min(node_id))); acc.max_node_id = Some(acc.max_node_id.map_or(node_id, |value| value.max(node_id))); @@ -3430,11 +3873,13 @@ fn observe_type( acc.updated_values.push(updated_at_ms); } -fn finalize_type_stats(type_accs: BTreeMap) -> Vec { - type_accs +fn finalize_node_label_stats( + label_accs: BTreeMap, +) -> Vec { + label_accs .into_iter() - .map(|(type_id, acc)| TypePlannerStats { - type_id, + .map(|(label_id, acc)| NodeLabelPlannerStats { + label_id, node_count: acc.node_count, min_node_id: acc.min_node_id, max_node_id: acc.max_node_id, @@ -3444,43 +3889,22 @@ fn finalize_type_stats(type_accs: BTreeMap) -> Vec Vec { - let mut by_type: BTreeMap> = BTreeMap::new(); - for node in nodes { - by_type - .entry(node.type_id) - .or_default() - .push(node.updated_at); - } - finalize_timestamp_groups(by_type) -} - -fn finalize_timestamp_stats_from_meta( - node_metas: &[CompactNodeMeta], +fn finalize_timestamp_stats_from_label_accs( + label_accs: &BTreeMap, ) -> Vec { - let mut by_type: BTreeMap> = BTreeMap::new(); - for meta in node_metas { - by_type - .entry(meta.type_id) - .or_default() - .push(meta.updated_at); - } - finalize_timestamp_groups(by_type) -} - -fn finalize_timestamp_groups(by_type: BTreeMap>) -> Vec { - by_type - .into_iter() - .filter_map(|(type_id, mut values)| { - if values.is_empty() { + label_accs + .iter() + .filter_map(|(&label_id, acc)| { + if acc.updated_values.is_empty() { return None; } + let mut values = acc.updated_values.clone(); values.sort_unstable(); let min_ms = *values.first().unwrap(); let max_ms = *values.last().unwrap(); let buckets = timestamp_buckets(&values, PLANNER_STATS_TIMESTAMP_BUCKETS); Some(TimestampPlannerStats { - type_id, + label_id, count: values.len() as u64, min_ms, max_ms, @@ -3495,13 +3919,15 @@ fn declared_property_reasons( ) -> BTreeMap<(u32, String), PropertyStatsTrackedReason> { let mut reasons: BTreeMap<(u32, String), PropertyStatsTrackedReason> = BTreeMap::new(); for entry in secondary_indexes { - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; + let SecondaryIndexTarget::NodeProperty { label_id, prop_key } = &entry.target else { + continue; + }; let new_reason = match entry.kind { SecondaryIndexKind::Equality => PropertyStatsTrackedReason::DeclaredEquality, SecondaryIndexKind::Range { .. } => PropertyStatsTrackedReason::DeclaredRange, }; reasons - .entry((*type_id, prop_key.clone())) + .entry((*label_id, prop_key.clone())) .and_modify(|reason| *reason = combine_property_reason(*reason, new_reason)) .or_insert(new_reason); } @@ -3511,23 +3937,23 @@ fn declared_property_reasons( fn seed_property_accumulators( declared_reasons: &BTreeMap<(u32, String), PropertyStatsTrackedReason>, property_candidates: BTreeMap, - type_accs: &BTreeMap, + label_accs: &BTreeMap, ) -> BTreeMap<(u32, String), PropertyAccumulator> { let mut accs = BTreeMap::new(); - for ((type_id, prop_key), reason) in declared_reasons { - if !type_accs.contains_key(type_id) { + for ((label_id, prop_key), reason) in declared_reasons { + if !label_accs.contains_key(label_id) { continue; } accs.insert( - (*type_id, prop_key.clone()), - PropertyAccumulator::new(*type_id, prop_key.clone(), *reason), + (*label_id, prop_key.clone()), + PropertyAccumulator::new(*label_id, prop_key.clone(), *reason), ); } - for (type_id, tracker) in property_candidates { + for (label_id, tracker) in property_candidates { for prop_key in tracker.into_keys() { - accs.entry((type_id, prop_key.clone())).or_insert_with(|| { + accs.entry((label_id, prop_key.clone())).or_insert_with(|| { PropertyAccumulator::new( - type_id, + label_id, prop_key, PropertyStatsTrackedReason::GeneralTopProperty, ) @@ -3561,14 +3987,14 @@ fn combine_property_reason( fn observe_general_property_candidates( candidates: &mut BTreeMap, declared_reasons: &BTreeMap<(u32, String), PropertyStatsTrackedReason>, - type_id: u32, + label_id: u32, props: &BTreeMap, ) { - let tracker = candidates.entry(type_id).or_insert_with(|| { - PropertyKeyCandidateTracker::new(PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_TYPE) + let tracker = candidates.entry(label_id).or_insert_with(|| { + PropertyKeyCandidateTracker::new(PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_LABEL) }); for key in props.keys() { - if declared_reasons.contains_key(&(type_id, key.clone())) { + if declared_reasons.contains_key(&(label_id, key.clone())) { continue; } tracker.observe(key); @@ -3577,11 +4003,11 @@ fn observe_general_property_candidates( fn observe_selected_node_properties( accs: &mut BTreeMap<(u32, String), PropertyAccumulator>, - type_id: u32, + label_id: u32, props: &BTreeMap, ) { for (key, value) in props { - if let Some(acc) = accs.get_mut(&(type_id, key.clone())) { + if let Some(acc) = accs.get_mut(&(label_id, key.clone())) { acc.observe(value); } } @@ -3590,13 +4016,13 @@ fn observe_selected_node_properties( fn finalize_property_stats( accs: BTreeMap<(u32, String), PropertyAccumulator>, ) -> Vec { - let mut by_type: BTreeMap> = BTreeMap::new(); + let mut by_label: BTreeMap> = BTreeMap::new(); for acc in accs.into_values() { - by_type.entry(acc.type_id).or_default().push(acc); + by_label.entry(acc.label_id).or_default().push(acc); } let mut stats = Vec::new(); - for (_type_id, mut props) in by_type { + for (_label_id, mut props) in by_label { let mut declared = Vec::new(); let mut general = Vec::new(); for acc in props.drain(..) { @@ -3620,13 +4046,13 @@ fn finalize_property_stats( stats.extend( general .into_iter() - .take(PLANNER_STATS_MAX_PROPERTY_KEYS_PER_TYPE) + .take(PLANNER_STATS_MAX_PROPERTY_KEYS_PER_LABEL) .map(PropertyAccumulator::into_stats), ); } stats.sort_by(|a, b| { - a.type_id - .cmp(&b.type_id) + a.label_id + .cmp(&b.label_id) .then_with(|| a.tracked_reason.cmp(&b.tracked_reason)) .then_with(|| a.prop_key.cmp(&b.prop_key)) }); @@ -3638,8 +4064,13 @@ fn declared_index_fingerprints( ) -> Vec { let mut declared: Vec<_> = secondary_indexes .iter() + .filter(|entry| entry.state == SecondaryIndexState::Ready) .map(|entry| { - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; + let target = planner_stats_declared_index_target(entry); + let (target_label_id, prop_key) = match &entry.target { + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => (*label_id, prop_key), + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => (*label_id, prop_key), + }; let (kind, range_domain) = match entry.kind { SecondaryIndexKind::Equality => (PlannerStatsDeclaredIndexKind::Equality, None), SecondaryIndexKind::Range { domain } => { @@ -3648,8 +4079,9 @@ fn declared_index_fingerprints( }; DeclaredIndexStatsFingerprint { index_id: entry.index_id, + target, kind, - type_id: *type_id, + target_label_id, prop_key: prop_key.clone(), range_domain, } @@ -3658,18 +4090,31 @@ fn declared_index_fingerprints( declared.sort_by(|a, b| { a.index_id .cmp(&b.index_id) + .then_with(|| { + declared_index_target_rank(a.target).cmp(&declared_index_target_rank(b.target)) + }) .then_with(|| declared_index_kind_rank(a.kind).cmp(&declared_index_kind_rank(b.kind))) - .then_with(|| a.type_id.cmp(&b.type_id)) + .then_with(|| a.target_label_id.cmp(&b.target_label_id)) .then_with(|| a.prop_key.cmp(&b.prop_key)) .then_with(|| range_domain_rank(a.range_domain).cmp(&range_domain_rank(b.range_domain))) }); declared } +pub(crate) fn planner_stats_declared_index_target( + entry: &SecondaryIndexManifestEntry, +) -> PlannerStatsDeclaredIndexTarget { + match &entry.target { + SecondaryIndexTarget::NodeProperty { .. } => PlannerStatsDeclaredIndexTarget::NodeProperty, + SecondaryIndexTarget::EdgeProperty { .. } => PlannerStatsDeclaredIndexTarget::EdgeProperty, + } +} + fn declaration_fingerprint(declared: &[DeclaredIndexStatsFingerprint]) -> u64 { let mut hash = FNV_OFFSET; for entry in declared { hash = fnv_update_u64(hash, entry.index_id); + hash = fnv_update_u8(hash, declared_index_target_rank(entry.target)); hash = fnv_update_u8( hash, match entry.kind { @@ -3677,7 +4122,7 @@ fn declaration_fingerprint(declared: &[DeclaredIndexStatsFingerprint]) -> u64 { PlannerStatsDeclaredIndexKind::Range => 2, }, ); - hash = fnv_update_u32(hash, entry.type_id); + hash = fnv_update_u32(hash, entry.target_label_id); hash = fnv_update_bytes(hash, entry.prop_key.as_bytes()); hash = fnv_update_u8( hash, @@ -3715,6 +4160,13 @@ fn fnv_update_u64(hash: u64, value: u64) -> u64 { fnv_update_bytes(hash, &value.to_le_bytes()) } +fn declared_index_target_rank(target: PlannerStatsDeclaredIndexTarget) -> u8 { + match target { + PlannerStatsDeclaredIndexTarget::NodeProperty => 0, + PlannerStatsDeclaredIndexTarget::EdgeProperty => 1, + } +} + fn declared_index_kind_rank(kind: PlannerStatsDeclaredIndexKind) -> u8 { match kind { PlannerStatsDeclaredIndexKind::Equality => 0, @@ -3820,14 +4272,14 @@ fn node_id_sample(ids: impl Iterator) -> Vec { } trait EdgeLike { - fn edge_type_id(&self) -> u32; + fn edge_label_id(&self) -> u32; fn source_node_id(&self) -> u64; fn target_node_id(&self) -> u64; } impl EdgeLike for EdgeRecord { - fn edge_type_id(&self) -> u32 { - self.type_id + fn edge_label_id(&self) -> u32 { + self.label_id } fn source_node_id(&self) -> u64 { self.from @@ -3838,8 +4290,8 @@ impl EdgeLike for EdgeRecord { } impl EdgeLike for &T { - fn edge_type_id(&self) -> u32 { - (*self).edge_type_id() + fn edge_label_id(&self) -> u32 { + (*self).edge_label_id() } fn source_node_id(&self) -> u64 { @@ -3853,7 +4305,7 @@ impl EdgeLike for &T { #[derive(Clone, Copy)] struct EdgeMetaRef { - type_id: u32, + label_id: u32, from: u64, to: u64, } @@ -3861,7 +4313,7 @@ struct EdgeMetaRef { impl From<&CompactEdgeMeta> for EdgeMetaRef { fn from(meta: &CompactEdgeMeta) -> Self { Self { - type_id: meta.type_id, + label_id: meta.label_id, from: meta.from, to: meta.to, } @@ -3869,8 +4321,8 @@ impl From<&CompactEdgeMeta> for EdgeMetaRef { } impl EdgeLike for EdgeMetaRef { - fn edge_type_id(&self) -> u32 { - self.type_id + fn edge_label_id(&self) -> u32 { + self.label_id } fn source_node_id(&self) -> u64 { self.from @@ -3898,15 +4350,15 @@ fn build_adjacency_stats( let mut groups: BTreeMap<(PlannerStatsDirection, Option), BTreeMap> = BTreeMap::new(); for edge in edges { - let type_id = edge.edge_type_id(); - for edge_type_id in [None, Some(type_id)] { + let label_id = edge.edge_label_id(); + for edge_label_id in [None, Some(label_id)] { *groups - .entry((PlannerStatsDirection::Outgoing, edge_type_id)) + .entry((PlannerStatsDirection::Outgoing, edge_label_id)) .or_default() .entry(edge.source_node_id()) .or_default() += 1; *groups - .entry((PlannerStatsDirection::Incoming, edge_type_id)) + .entry((PlannerStatsDirection::Incoming, edge_label_id)) .or_default() .entry(edge.target_node_id()) .or_default() += 1; @@ -3915,13 +4367,13 @@ fn build_adjacency_stats( groups .into_iter() - .filter_map(|((direction, edge_type_id), fanouts)| { + .filter_map(|((direction, edge_label_id), fanouts)| { if fanouts.is_empty() { return None; } Some(adjacency_stats_from_fanouts( direction, - edge_type_id, + edge_label_id, fanouts, )) }) @@ -3930,7 +4382,7 @@ fn build_adjacency_stats( fn adjacency_stats_from_fanouts( direction: PlannerStatsDirection, - edge_type_id: Option, + edge_label_id: Option, fanouts: BTreeMap, ) -> AdjacencyPlannerStats { let mut counts: Vec = fanouts.values().copied().collect(); @@ -3950,10 +4402,10 @@ fn adjacency_stats_from_fanouts( .cmp(&a.count) .then_with(|| a.node_id.cmp(&b.node_id)) }); - top_hubs.truncate(PLANNER_STATS_TOP_HUBS_PER_EDGE_TYPE); + top_hubs.truncate(PLANNER_STATS_TOP_HUBS_PER_EDGE_LABEL); AdjacencyPlannerStats { direction, - edge_type_id, + edge_label_id, source_node_count: counts.len() as u64, total_edges, min_fanout, @@ -3979,7 +4431,16 @@ fn decode_node_props_at( node_id: u64, ) -> Result, EngineError> { let start = data_offset as usize; - let key_len_start = start.checked_add(4).ok_or_else(|| { + let label_count = *data.get(start).ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} record too short for label count", node_id)) + })? as usize; + if label_count == 0 || label_count > crate::types::MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::CorruptRecord(format!( + "node {} record has invalid label count {}", + node_id, label_count + ))); + } + let key_len_start = start.checked_add(1 + label_count * 4).ok_or_else(|| { EngineError::CorruptRecord(format!("node {} props offset overflow", node_id)) })?; let key_len_end = key_len_start.checked_add(2).ok_or_else(|| { @@ -4017,11 +4478,11 @@ fn decode_node_props_at( }) } +#[cfg(test)] fn read_secondary_eq_group_counts(path: &Path) -> Result>, EngineError> { - let data = match fs::read(path) { - Ok(data) => data, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), + let data = match read_optional_component_payload(path)? { + Some(data) => data, + None => return Ok(None), }; if data.len() < 8 { return Err(EngineError::CorruptRecord(format!( @@ -4065,11 +4526,11 @@ fn read_secondary_eq_group_counts(path: &Path) -> Result Result>, EngineError> { - let data = match fs::read(path) { - Ok(data) => data, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), + let data = match read_optional_component_payload(path)? { + Some(data) => data, + None => return Ok(None), }; if data.len() < 8 { return Err(EngineError::CorruptRecord(format!( @@ -4097,10 +4558,48 @@ fn read_secondary_range_encoded_values(path: &Path) -> Result>, Ok(Some(encoded_values)) } +#[cfg(test)] +fn read_optional_component_payload(path: &Path) -> Result>, EngineError> { + let data = match fs::read(path) { + Ok(data) => data, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if data.len() >= COMPONENT_IDENTITY_HEADER_LEN + && data[0..COMPONENT_IDENTITY_HEADER_MAGIC.len()] == COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = decode_identity_header(&data)?; + let end = header + .payload_offset + .checked_add(header.payload_len) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "component payload range overflows for {}", + path.display() + )) + })?; + if end > data.len() as u64 { + return Err(EngineError::CorruptRecord(format!( + "component payload range [{}, {}) exceeds file length {} for {}", + header.payload_offset, + end, + data.len(), + path.display() + ))); + } + return Ok(Some( + data[header.payload_offset as usize..end as usize].to_vec(), + )); + } + Ok(Some(data)) +} + +#[cfg(test)] fn cleanup_stats_tmp(seg_dir: &Path) { let _ = fs::remove_file(seg_dir.join(PLANNER_STATS_TMP_FILENAME)); } +#[cfg(test)] fn fsync_dir(dir: &Path) -> Result<(), EngineError> { #[cfg(not(target_os = "windows"))] { @@ -4115,7 +4614,10 @@ fn fsync_dir(dir: &Path) -> Result<(), EngineError> { #[cfg(test)] mod tests { use super::*; - use crate::types::SecondaryIndexState; + use crate::types::{ + SecondaryIndexKind, SecondaryIndexManifestEntry, SecondaryIndexRangeDomain, + SecondaryIndexState, SecondaryIndexTarget, + }; fn minimal_stats(segment_id: u64) -> SegmentPlannerStatsV1 { SegmentPlannerStatsV1 { @@ -4132,8 +4634,8 @@ mod tests { general_property_sampled_node_count: 1, general_property_sampled_raw_bytes: 0, general_property_budget_exhausted: false, - type_stats: vec![TypePlannerStats { - type_id: 7, + node_label_stats: vec![NodeLabelPlannerStats { + label_id: 7, node_count: 1, min_node_id: Some(42), max_node_id: Some(42), @@ -4149,6 +4651,81 @@ mod tests { } } + fn equality_entry(index_id: u64) -> SecondaryIndexManifestEntry { + SecondaryIndexManifestEntry { + index_id, + target: SecondaryIndexTarget::NodeProperty { + label_id: 7, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + } + } + + fn range_entry(index_id: u64) -> SecondaryIndexManifestEntry { + SecondaryIndexManifestEntry { + index_id, + target: SecondaryIndexTarget::NodeProperty { + label_id: 7, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + } + } + + #[test] + fn equality_stats_from_written_groups_match_sidecar_read_stats() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(crate::segment_writer::secondary_indexes_dir(dir.path())).unwrap(); + let entry = equality_entry(41); + let mut groups = BTreeMap::new(); + groups.insert(100, vec![1, 3, 5]); + groups.insert(200, vec![2]); + groups.insert(300, Vec::new()); + + crate::segment_writer::write_node_prop_eq_sidecar_to_path( + &crate::segment_writer::node_prop_eq_sidecar_path(dir.path(), entry.index_id), + &groups, + ) + .unwrap(); + + let from_written = equality_index_stats_from_written_groups(&entry, &groups); + let from_sidecar = + build_equality_index_stats_from_sidecars(dir.path(), std::slice::from_ref(&entry)) + .unwrap() + .pop() + .unwrap(); + assert_eq!(from_written, from_sidecar); + } + + #[test] + fn range_stats_from_written_entries_match_sidecar_read_stats() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(crate::segment_writer::secondary_indexes_dir(dir.path())).unwrap(); + let entry = range_entry(42); + let entries = vec![(30, 3), (10, 1), (20, 2), (20, 4)]; + + crate::segment_writer::write_node_prop_range_sidecar_to_path( + &crate::segment_writer::node_prop_range_sidecar_path(dir.path(), entry.index_id), + &entries, + ) + .unwrap(); + + let from_written = range_index_stats_from_written_entries(&entry, &entries); + let from_sidecar = + build_range_index_stats_from_sidecars(dir.path(), std::slice::from_ref(&entry)) + .unwrap() + .pop() + .unwrap(); + assert_eq!(from_written, from_sidecar); + } + #[test] fn stale_risk_classification_uses_bounded_sample_signals() { assert_eq!( @@ -4183,7 +4760,7 @@ mod tests { stats.edge_count = 4; stats.adjacency_stats.push(AdjacencyPlannerStats { direction: PlannerStatsDirection::Outgoing, - edge_type_id: Some(10), + edge_label_id: Some(10), source_node_count: 2, total_edges: 4, min_fanout: 1, @@ -4229,11 +4806,95 @@ mod tests { ); } - fn ready_eq_entry(index_id: u64, type_id: u32, prop_key: &str) -> SecondaryIndexManifestEntry { + #[test] + fn validate_declared_index_stats_uses_edge_label_counts_for_edge_targets() { + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let mut stats = minimal_stats(1); + stats.node_count = 1; + stats.edge_count = 3; + stats.adjacency_stats.push(AdjacencyPlannerStats { + direction: PlannerStatsDirection::Outgoing, + edge_label_id: Some(7), + source_node_count: 1, + total_edges: 3, + min_fanout: 3, + max_fanout: 3, + p50_fanout: 3, + p90_fanout: 3, + p99_fanout: 3, + top_hubs: Vec::new(), + }); + stats.declared_indexes.push(DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::EdgeProperty, + index_id: 31, + kind: PlannerStatsDeclaredIndexKind::Equality, + target_label_id: 7, + prop_key: "color".to_string(), + range_domain: None, + }); + stats.equality_index_stats.push(EqualityIndexPlannerStats { + index_id: 31, + target_label_id: 7, + prop_key: "color".to_string(), + total_postings: 3, + value_group_count: 1, + max_group_postings: 3, + top_value_hashes: vec![ValueFrequency { + value_hash: red_hash, + count: 3, + }], + sidecar_present_at_build: true, + }); + stats.declared_indexes.push(DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::EdgeProperty, + index_id: 32, + kind: PlannerStatsDeclaredIndexKind::Range, + target_label_id: 7, + prop_key: "score".to_string(), + range_domain: Some(SecondaryIndexRangeDomain::Float), + }); + stats.range_index_stats.push(RangeIndexPlannerStats { + index_id: 32, + target_label_id: 7, + prop_key: "score".to_string(), + domain: SecondaryIndexRangeDomain::Float, + total_entries: 3, + min_encoded: Some(10), + max_encoded: Some(30), + buckets: vec![RangeBucket { + upper_encoded: 30, + count: 3, + }], + sidecar_present_at_build: true, + }); + let mut node_label_counts = BTreeMap::new(); + node_label_counts.insert(7, 1); + + assert!(validate_declared_index_stats(&stats, &node_label_counts).is_ok()); + } + + fn ready_eq_entry(index_id: u64, label_id: u32, prop_key: &str) -> SecondaryIndexManifestEntry { SecondaryIndexManifestEntry { index_id, target: SecondaryIndexTarget::NodeProperty { - type_id, + label_id, + prop_key: prop_key.to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + } + } + + fn ready_edge_eq_entry( + index_id: u64, + label_id: u32, + prop_key: &str, + ) -> SecondaryIndexManifestEntry { + SecondaryIndexManifestEntry { + index_id, + target: SecondaryIndexTarget::EdgeProperty { + label_id: label_id, prop_key: prop_key.to_string(), }, kind: SecondaryIndexKind::Equality, @@ -4244,14 +4905,14 @@ mod tests { fn ready_range_entry( index_id: u64, - type_id: u32, + label_id: u32, prop_key: &str, domain: SecondaryIndexRangeDomain, ) -> SecondaryIndexManifestEntry { SecondaryIndexManifestEntry { index_id, target: SecondaryIndexTarget::NodeProperty { - type_id, + label_id, prop_key: prop_key.to_string(), }, kind: SecondaryIndexKind::Range { domain }, @@ -4263,22 +4924,23 @@ mod tests { fn add_eq_stats( stats: &mut SegmentPlannerStatsV1, index_id: u64, - type_id: u32, + target_label_id: u32, prop_key: &str, total_postings: u64, value_group_count: u64, top_value_hashes: Vec, ) { stats.declared_indexes.push(DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id, kind: PlannerStatsDeclaredIndexKind::Equality, - type_id, + target_label_id, prop_key: prop_key.to_string(), range_domain: None, }); stats.equality_index_stats.push(EqualityIndexPlannerStats { index_id, - type_id, + target_label_id, prop_key: prop_key.to_string(), total_postings, value_group_count, @@ -4295,21 +4957,22 @@ mod tests { fn add_range_stats( stats: &mut SegmentPlannerStatsV1, index_id: u64, - type_id: u32, + target_label_id: u32, prop_key: &str, domain: SecondaryIndexRangeDomain, total_entries: u64, ) { stats.declared_indexes.push(DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id, kind: PlannerStatsDeclaredIndexKind::Range, - type_id, + target_label_id, prop_key: prop_key.to_string(), range_domain: Some(domain), }); stats.range_index_stats.push(RangeIndexPlannerStats { index_id, - type_id, + target_label_id, prop_key: prop_key.to_string(), domain, total_entries, @@ -4323,9 +4986,86 @@ mod tests { }); } + #[test] + fn targeted_stats_merge_replaces_only_target_block_and_keeps_siblings() { + let mut stats = minimal_stats(9); + add_eq_stats( + &mut stats, + 11, + 7, + "color", + 3, + 2, + vec![ValueFrequency { + value_hash: 111, + count: 2, + }], + ); + add_range_stats( + &mut stats, + 12, + 7, + "score", + SecondaryIndexRangeDomain::Int, + 1, + ); + let ready_indexes = vec![ + ready_range_entry(12, 7, "score", SecondaryIndexRangeDomain::Int), + ready_eq_entry(11, 7, "color"), + ]; + let replacement_range = RangeIndexPlannerStats { + index_id: 12, + target_label_id: 7, + prop_key: "score".to_string(), + domain: SecondaryIndexRangeDomain::Int, + total_entries: 3, + min_encoded: Some(10), + max_encoded: Some(30), + buckets: vec![RangeBucket { + upper_encoded: 30, + count: 3, + }], + sidecar_present_at_build: true, + }; + + let merged = merge_targeted_declared_index_stats( + stats, + &ready_indexes, + 12, + None, + Some(replacement_range), + ); + + assert_eq!( + merged.build_kind, + PlannerStatsBuildKind::SecondaryIndexRefresh + ); + assert_eq!(merged.equality_index_stats.len(), 1); + assert_eq!(merged.equality_index_stats[0].index_id, 11); + assert_eq!(merged.range_index_stats.len(), 1); + assert_eq!(merged.range_index_stats[0].index_id, 12); + assert_eq!(merged.range_index_stats[0].total_entries, 3); + assert_eq!( + merged + .range_index_stats + .iter() + .filter(|stats| stats.index_id == 12) + .count(), + 1 + ); + assert_eq!( + merged + .declared_indexes + .iter() + .map(|declared| declared.index_id) + .collect::>(), + vec![11, 12] + ); + } + fn stats_with_timestamp_histogram( segment_id: u64, - type_id: u32, + label_id: u32, count: u64, min_ms: i64, max_ms: i64, @@ -4335,8 +5075,8 @@ mod tests { stats.general_property_sampled_node_count = stats .general_property_sampled_node_count .min(stats.node_count); - stats.type_stats = vec![TypePlannerStats { - type_id, + stats.node_label_stats = vec![NodeLabelPlannerStats { + label_id, node_count: count, min_node_id: Some(segment_id.saturating_mul(1_000)), max_node_id: Some(segment_id.saturating_mul(1_000).saturating_add(count)), @@ -4344,7 +5084,7 @@ mod tests { max_updated_at_ms: Some(max_ms), }]; stats.timestamp_stats = vec![TimestampPlannerStats { - type_id, + label_id, count, min_ms, max_ms, @@ -4358,7 +5098,7 @@ mod tests { struct RangeHistogramInput { index_id: u64, - type_id: u32, + label_id: u32, prop_key: &'static str, domain: SecondaryIndexRangeDomain, count: u64, @@ -4368,15 +5108,16 @@ mod tests { fn add_range_histogram_stats(stats: &mut SegmentPlannerStatsV1, input: RangeHistogramInput) { stats.declared_indexes.push(DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id: input.index_id, kind: PlannerStatsDeclaredIndexKind::Range, - type_id: input.type_id, + target_label_id: input.label_id, prop_key: input.prop_key.to_string(), range_domain: Some(input.domain), }); stats.range_index_stats.push(RangeIndexPlannerStats { index_id: input.index_id, - type_id: input.type_id, + target_label_id: input.label_id, prop_key: input.prop_key.to_string(), domain: input.domain, total_entries: input.count, @@ -4493,17 +5234,159 @@ mod tests { )); } + #[test] + fn envelope_allows_label_memberships_above_physical_node_count() { + let mut stats = minimal_stats(29); + stats.node_count = 2; + stats.general_property_sampled_node_count = 2; + stats.node_id_sample = vec![41, 42]; + stats.node_label_stats = vec![ + NodeLabelPlannerStats { + label_id: 7, + node_count: 2, + min_node_id: Some(41), + max_node_id: Some(42), + min_updated_at_ms: Some(1000), + max_updated_at_ms: Some(1001), + }, + NodeLabelPlannerStats { + label_id: 9, + node_count: 1, + min_node_id: Some(42), + max_node_id: Some(42), + min_updated_at_ms: Some(1001), + max_updated_at_ms: Some(1001), + }, + ]; + stats.timestamp_stats = vec![ + TimestampPlannerStats { + label_id: 7, + count: 2, + min_ms: 1000, + max_ms: 1001, + buckets: vec![TimestampBucket { + upper_ms: 1001, + count: 2, + }], + }, + TimestampPlannerStats { + label_id: 9, + count: 1, + min_ms: 1001, + max_ms: 1001, + buckets: vec![TimestampBucket { + upper_ms: 1001, + count: 1, + }], + }, + ]; + + let data = encode_enveloped_stats(&stats).unwrap(); + let decoded = decode_planner_stats_envelope(&data, 29, 2, 0).unwrap(); + assert_eq!( + decoded + .node_label_stats + .iter() + .map(|stat| stat.node_count) + .sum::(), + 3 + ); + assert_eq!(decoded.node_count, 2); + } + + #[test] + fn envelope_rejects_label_memberships_beyond_max_label_bound() { + let mut stats = minimal_stats(30); + stats.node_count = 1; + stats.node_label_stats = (1..=11) + .map(|label_id| NodeLabelPlannerStats { + label_id, + node_count: 1, + min_node_id: Some(42), + max_node_id: Some(42), + min_updated_at_ms: Some(1000), + max_updated_at_ms: Some(1000), + }) + .collect(); + stats.timestamp_stats = (1..=11) + .map(|label_id| TimestampPlannerStats { + label_id, + count: 1, + min_ms: 1000, + max_ms: 1000, + buckets: vec![TimestampBucket { + upper_ms: 1000, + count: 1, + }], + }) + .collect(); + + assert_decode_err_contains(stats, 1, 0, "expected between 1 and 10"); + } + + #[test] + fn envelope_rejects_per_label_membership_count_above_node_count() { + let mut stats = minimal_stats(31); + stats.node_count = 2; + stats.general_property_sampled_node_count = 2; + stats.node_id_sample = vec![41, 42]; + stats.node_label_stats = vec![ + NodeLabelPlannerStats { + label_id: 7, + node_count: 3, + min_node_id: Some(41), + max_node_id: Some(42), + min_updated_at_ms: Some(1000), + max_updated_at_ms: Some(1001), + }, + NodeLabelPlannerStats { + label_id: 9, + node_count: 1, + min_node_id: Some(42), + max_node_id: Some(42), + min_updated_at_ms: Some(1001), + max_updated_at_ms: Some(1001), + }, + ]; + stats.timestamp_stats = vec![ + TimestampPlannerStats { + label_id: 7, + count: 3, + min_ms: 1000, + max_ms: 1001, + buckets: vec![TimestampBucket { + upper_ms: 1001, + count: 3, + }], + }, + TimestampPlannerStats { + label_id: 9, + count: 1, + min_ms: 1001, + max_ms: 1001, + buckets: vec![TimestampBucket { + upper_ms: 1001, + count: 1, + }], + }, + ]; + + assert_decode_err_contains(stats, 2, 0, "exceeds segment node count"); + } + #[test] fn envelope_rejects_internal_count_sanity_failures() { - let mut bad_type_count = minimal_stats(9); - bad_type_count.type_stats[0].node_count = 2; - assert_decode_err_contains(bad_type_count, 1, 0, "type counts sum"); + let mut bad_label_count = minimal_stats(9); + bad_label_count.node_count = 2; + bad_label_count.general_property_sampled_node_count = 2; + bad_label_count.node_id_sample = vec![42, 43]; + assert_decode_err_contains(bad_label_count, 2, 0, "label counts sum"); let mut bad_property_count = minimal_stats(9); bad_property_count .property_stats .push(PropertyPlannerStats { - type_id: 7, + label_id: 7, prop_key: "score".to_string(), tracked_reason: PropertyStatsTrackedReason::GeneralTopProperty, present_count: 2, @@ -4527,7 +5410,7 @@ mod tests { duplicate_property .property_stats .push(PropertyPlannerStats { - type_id: 7, + label_id: 7, prop_key: "score".to_string(), tracked_reason, present_count: 1, @@ -4548,7 +5431,7 @@ mod tests { bad_timestamp_bucket .timestamp_stats .push(TimestampPlannerStats { - type_id: 7, + label_id: 7, count: 1, min_ms: 1000, max_ms: 1000, @@ -4560,9 +5443,10 @@ mod tests { assert_decode_err_contains(bad_timestamp_bucket, 1, 0, "timestamp buckets sum"); let declared_eq = DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id: 11, kind: PlannerStatsDeclaredIndexKind::Equality, - type_id: 7, + target_label_id: 7, prop_key: "color".to_string(), range_domain: None, }; @@ -4572,7 +5456,7 @@ mod tests { .equality_index_stats .push(EqualityIndexPlannerStats { index_id: 11, - type_id: 7, + target_label_id: 7, prop_key: "color".to_string(), total_postings: 2, value_group_count: 1, @@ -4583,9 +5467,10 @@ mod tests { assert_decode_err_contains(bad_equality_count, 1, 0, "postings exceed"); let declared_range = DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id: 12, kind: PlannerStatsDeclaredIndexKind::Range, - type_id: 7, + target_label_id: 7, prop_key: "score".to_string(), range_domain: Some(SecondaryIndexRangeDomain::Int), }; @@ -4595,7 +5480,7 @@ mod tests { .range_index_stats .push(RangeIndexPlannerStats { index_id: 12, - type_id: 7, + target_label_id: 7, prop_key: "score".to_string(), domain: SecondaryIndexRangeDomain::Int, total_entries: 1, @@ -4613,7 +5498,7 @@ mod tests { bad_adjacency.edge_count = 1; bad_adjacency.adjacency_stats.push(AdjacencyPlannerStats { direction: PlannerStatsDirection::Outgoing, - edge_type_id: None, + edge_label_id: None, source_node_count: 1, total_edges: 2, min_fanout: 1, @@ -4647,8 +5532,8 @@ mod tests { #[test] fn bounded_property_candidate_tracker_keeps_late_frequent_key() { let mut tracker = - PropertyKeyCandidateTracker::new(PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_TYPE); - for idx in 0..PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_TYPE { + PropertyKeyCandidateTracker::new(PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_LABEL); + for idx in 0..PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_LABEL { tracker.observe(&format!("one_off_{:04}", idx)); } for _ in 0..32 { @@ -4658,7 +5543,7 @@ mod tests { let keys: Vec<_> = tracker.into_keys().collect(); assert_eq!( keys.len(), - PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_TYPE + PLANNER_STATS_PROPERTY_KEY_CANDIDATE_CAP_PER_LABEL ); assert!(keys.iter().any(|key| key == "zz_late_hot")); } @@ -4668,7 +5553,7 @@ mod tests { let red_hash = hash_prop_value(&PropValue::String("red".to_string())); let mut stats = minimal_stats(1); stats.timestamp_stats.push(TimestampPlannerStats { - type_id: 7, + label_id: 7, count: 1, min_ms: 1000, max_ms: 1000, @@ -4678,7 +5563,7 @@ mod tests { }], }); stats.property_stats.push(PropertyPlannerStats { - type_id: 7, + label_id: 7, prop_key: "color".to_string(), tracked_reason: PropertyStatsTrackedReason::DeclaredEquality, present_count: 1, @@ -4718,7 +5603,7 @@ mod tests { stats.edge_count = 1; stats.adjacency_stats.push(AdjacencyPlannerStats { direction: PlannerStatsDirection::Outgoing, - edge_type_id: Some(5), + edge_label_id: Some(5), source_node_count: 1, total_edges: 1, min_fanout: 1, @@ -4771,19 +5656,19 @@ mod tests { assert_eq!(view.full_rollup.node_count, 1); assert_eq!(view.full_rollup.coverage.covered_segment_ids, vec![1]); assert_eq!(view.full_rollup.coverage.uncovered_segment_ids, vec![2, 3]); - assert_eq!(view.type_node_count(7), 1); + assert_eq!(view.node_label_count(7), 1); assert_eq!(view.timestamp_coverage.covered_segment_ids, vec![1]); - assert_eq!(view.type_rollups.get(&7).unwrap().type_id, 7); - assert_eq!(view.timestamp_rollups.get(&7).unwrap().type_id, 7); + assert_eq!(view.node_label_rollups.get(&7).unwrap().label_id, 7); + assert_eq!(view.timestamp_rollups.get(&7).unwrap().label_id, 7); let property = view .property_rollups .get(&(7, "color".to_string())) .unwrap(); - assert_eq!(property.type_id, 7); + assert_eq!(property.label_id, 7); assert_eq!(property.prop_key, "color"); assert_eq!(property.present_count, 1); let equality = view.equality_index_rollups.get(&11).unwrap(); - assert_eq!(equality.type_id, 7); + assert_eq!(equality.target_label_id, 7); assert_eq!(equality.prop_key, "color"); assert_eq!(equality.coverage.covered_segment_ids, vec![1]); assert_eq!(equality.coverage.uncovered_segment_ids, vec![2, 3]); @@ -4795,7 +5680,7 @@ mod tests { }) ); let range = view.range_index_rollups.get(&12).unwrap(); - assert_eq!(range.type_id, 7); + assert_eq!(range.target_label_id, 7); assert_eq!(range.prop_key, "score"); assert_eq!(range.domain, SecondaryIndexRangeDomain::Int); assert_eq!(range.total_entries, 1); @@ -4804,7 +5689,7 @@ mod tests { .get(&(PlannerStatsDirection::Outgoing, Some(5))) .unwrap(); assert_eq!(adjacency.direction, PlannerStatsDirection::Outgoing); - assert_eq!(adjacency.edge_type_id, Some(5)); + assert_eq!(adjacency.edge_label_id, Some(5)); assert_eq!(adjacency.total_edges, 1); } @@ -4839,7 +5724,7 @@ mod tests { ); assert_eq!(view.full_rollup.coverage.covered_segment_ids, vec![1]); - assert_eq!(view.type_node_count(7), 1); + assert_eq!(view.node_label_count(7), 1); let equality = view.equality_index_rollups.get(&11).unwrap(); assert_eq!(equality.coverage.mismatched_segment_ids, vec![1]); assert_eq!(view.equality_segment_estimate(11, 1, &[red_hash]), None); @@ -4886,8 +5771,20 @@ mod tests { ready_range_entry(12, 7, "score", SecondaryIndexRangeDomain::Int), ]; let mut runtime_coverage = DeclaredIndexRuntimeCoverage::default(); - runtime_coverage.insert(1, 11, PlannerStatsDeclaredIndexKind::Equality, state); - runtime_coverage.insert(1, 12, PlannerStatsDeclaredIndexKind::Range, state); + runtime_coverage.insert( + 1, + 11, + PlannerStatsDeclaredIndexTarget::NodeProperty, + PlannerStatsDeclaredIndexKind::Equality, + state, + ); + runtime_coverage.insert( + 1, + 12, + PlannerStatsDeclaredIndexTarget::NodeProperty, + PlannerStatsDeclaredIndexKind::Range, + state, + ); let view = build_planner_stats_view_from_snapshots_with_runtime_coverage( 1, @@ -4955,7 +5852,87 @@ mod tests { } #[test] - fn rollup_declared_index_type_zero_is_valid_shape() { + fn rollup_declared_index_stats_require_matching_target_coverage() { + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let mut stats = minimal_stats(1); + stats.declared_indexes.push(DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::EdgeProperty, + index_id: 31, + kind: PlannerStatsDeclaredIndexKind::Equality, + target_label_id: 7, + prop_key: "color".to_string(), + range_domain: None, + }); + stats.equality_index_stats.push(EqualityIndexPlannerStats { + index_id: 31, + target_label_id: 7, + prop_key: "color".to_string(), + total_postings: 1, + value_group_count: 1, + max_group_postings: 1, + top_value_hashes: vec![ValueFrequency { + value_hash: red_hash, + count: 1, + }], + sidecar_present_at_build: true, + }); + let available = PlannerStatsAvailability::Available(Box::new(stats)); + let segments = vec![PlannerStatsSegmentSnapshot { + segment_id: 1, + node_count: 0, + edge_count: 1, + availability: &available, + }]; + let indexes = [ready_edge_eq_entry(31, 7, "color")]; + let mut runtime_coverage = DeclaredIndexRuntimeCoverage::default(); + runtime_coverage.insert( + 1, + 31, + PlannerStatsDeclaredIndexTarget::NodeProperty, + PlannerStatsDeclaredIndexKind::Equality, + DeclaredIndexRuntimeCoverageState::Available, + ); + + let view = build_planner_stats_view_from_snapshots_with_runtime_coverage( + 1, + &segments, + &indexes, + &runtime_coverage, + ); + assert_eq!( + view.equality_index_rollups + .get(&31) + .unwrap() + .coverage + .mismatched_segment_ids, + vec![1] + ); + + runtime_coverage.insert( + 1, + 31, + PlannerStatsDeclaredIndexTarget::EdgeProperty, + PlannerStatsDeclaredIndexKind::Equality, + DeclaredIndexRuntimeCoverageState::Available, + ); + let view = build_planner_stats_view_from_snapshots_with_runtime_coverage( + 1, + &segments, + &indexes, + &runtime_coverage, + ); + assert_eq!( + view.equality_index_rollups + .get(&31) + .unwrap() + .coverage + .covered_segment_ids, + vec![1] + ); + } + + #[test] + fn rollup_declared_index_label_zero_is_valid_shape() { let mut stats = minimal_stats(1); add_eq_stats(&mut stats, 11, 0, "color", 0, 0, Vec::new()); add_range_stats( @@ -4983,15 +5960,24 @@ mod tests { ], ); - assert_eq!(view.equality_index_rollups.get(&11).unwrap().type_id, 0); - assert_eq!(view.range_index_rollups.get(&12).unwrap().type_id, 0); + assert_eq!( + view.equality_index_rollups + .get(&11) + .unwrap() + .target_label_id, + 0 + ); + assert_eq!( + view.range_index_rollups.get(&12).unwrap().target_label_id, + 0 + ); } #[test] fn rollup_range_and_timestamp_histograms_use_conservative_upper_estimates() { let mut stats = minimal_stats(1); stats.timestamp_stats = vec![TimestampPlannerStats { - type_id: 7, + label_id: 7, count: 6, min_ms: 10, max_ms: 60, @@ -5011,15 +5997,16 @@ mod tests { ], }]; stats.declared_indexes.push(DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id: 12, kind: PlannerStatsDeclaredIndexKind::Range, - type_id: 7, + target_label_id: 7, prop_key: "score".to_string(), range_domain: Some(SecondaryIndexRangeDomain::Int), }); stats.range_index_stats.push(RangeIndexPlannerStats { index_id: 12, - type_id: 7, + target_label_id: 7, prop_key: "score".to_string(), domain: SecondaryIndexRangeDomain::Int, total_entries: 6, @@ -5097,7 +6084,7 @@ mod tests { &mut stats_a, RangeHistogramInput { index_id: 12, - type_id: 7, + label_id: 7, prop_key: "score", domain: SecondaryIndexRangeDomain::Int, count: 100, @@ -5110,7 +6097,7 @@ mod tests { &mut stats_b, RangeHistogramInput { index_id: 12, - type_id: 7, + label_id: 7, prop_key: "score", domain: SecondaryIndexRangeDomain::Int, count: 100, @@ -5194,7 +6181,7 @@ mod tests { &mut stats_a, RangeHistogramInput { index_id: 12, - type_id: 7, + label_id: 7, prop_key: "score", domain: SecondaryIndexRangeDomain::Int, count: 100, @@ -5207,7 +6194,7 @@ mod tests { &mut stats_b, RangeHistogramInput { index_id: 12, - type_id: 7, + label_id: 7, prop_key: "score", domain: SecondaryIndexRangeDomain::Int, count: 100, @@ -5291,7 +6278,7 @@ mod tests { &mut stats, RangeHistogramInput { index_id: 12, - type_id: 7, + label_id: 7, prop_key: "score", domain: SecondaryIndexRangeDomain::Int, count: 100, @@ -5376,7 +6363,7 @@ mod tests { } #[test] - fn timestamp_absent_type_is_exact_zero_only_when_type_stats_cover_segments() { + fn timestamp_absent_label_is_exact_zero_only_when_node_label_stats_cover_segments() { let stats = stats_with_timestamp_histogram(1, 7, 100, 0, 100); let available = PlannerStatsAvailability::Available(Box::new(stats)); let segments = vec![PlannerStatsSegmentSnapshot { @@ -5516,7 +6503,7 @@ mod tests { assert_eq!(view.segment_count, 256); assert_eq!(view.available_segment_stats, 256); assert_eq!(view.full_rollup.node_count, 256); - assert_eq!(view.type_node_count(7), 256); + assert_eq!(view.node_label_count(7), 256); let equality = view.equality_index_rollups.get(&11).unwrap(); assert_eq!(equality.coverage.covered_segment_ids.len(), 256); assert_eq!(equality.total_postings, 256); @@ -5594,7 +6581,7 @@ mod tests { fn size_reduction_preserves_core_before_skip() { let mut stats = minimal_stats(1); stats.property_stats.push(PropertyPlannerStats { - type_id: 1, + label_id: 1, prop_key: "large_general".to_string(), tracked_reason: PropertyStatsTrackedReason::GeneralTopProperty, present_count: 1, @@ -5616,7 +6603,7 @@ mod tests { let reduced = decode_planner_stats_envelope(&encoded, 1, 1, 0).unwrap(); assert!(reduced.truncated); assert!(reduced.property_stats.is_empty()); - assert_eq!(reduced.type_stats, stats.type_stats); + assert_eq!(reduced.node_label_stats, stats.node_label_stats); assert!(serialize_stats_with_limits(stats, 64, 64) .unwrap() @@ -5629,23 +6616,25 @@ mod tests { stats.edge_count = 1; stats.declared_indexes = vec![ DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id: 11, kind: PlannerStatsDeclaredIndexKind::Equality, - type_id: 7, + target_label_id: 7, prop_key: "color".to_string(), range_domain: None, }, DeclaredIndexStatsFingerprint { + target: PlannerStatsDeclaredIndexTarget::NodeProperty, index_id: 12, kind: PlannerStatsDeclaredIndexKind::Range, - type_id: 7, + target_label_id: 7, prop_key: "score".to_string(), range_domain: Some(SecondaryIndexRangeDomain::Int), }, ]; stats.equality_index_stats.push(EqualityIndexPlannerStats { index_id: 11, - type_id: 7, + target_label_id: 7, prop_key: "color".to_string(), total_postings: 1, value_group_count: 1, @@ -5658,7 +6647,7 @@ mod tests { }); stats.range_index_stats.push(RangeIndexPlannerStats { index_id: 12, - type_id: 7, + target_label_id: 7, prop_key: "score".to_string(), domain: SecondaryIndexRangeDomain::Int, total_entries: 1, @@ -5671,7 +6660,7 @@ mod tests { sidecar_present_at_build: true, }); stats.property_stats.push(PropertyPlannerStats { - type_id: 7, + label_id: 7, prop_key: "general".to_string(), tracked_reason: PropertyStatsTrackedReason::GeneralTopProperty, present_count: 1, @@ -5690,10 +6679,10 @@ mod tests { .collect(), numeric_summaries: Vec::new(), }); - for edge_type_id in 0..256 { + for edge_label_id in 0..256 { stats.adjacency_stats.push(AdjacencyPlannerStats { direction: PlannerStatsDirection::Outgoing, - edge_type_id: Some(edge_type_id), + edge_label_id: Some(edge_label_id), source_node_count: 1, total_edges: 1, min_fanout: 1, diff --git a/src/scrub.rs b/src/scrub.rs new file mode 100644 index 0000000..50e919f --- /dev/null +++ b/src/scrub.rs @@ -0,0 +1,2070 @@ +use crate::error::EngineError; +use crate::memtable::encode_range_prop_value; +use crate::parallel::engine_cpu_install; +use crate::segment_components::{ + component_id, decode_identity_header, decode_manifest_envelope, dependency_digest, + ComponentHandleV1, ComponentIdentityHeaderV1, SegmentComponentKind, SegmentComponentManifestV1, + SegmentComponentRecordV1, COMPONENT_IDENTITY_HEADER_LEN, PACKED_CORE_FILENAME, + SEGMENT_COMPONENT_MANIFEST_FILENAME, +}; +use crate::segment_reader::{validate_segment_manifest_identity, SegmentReader}; +use crate::segment_writer::segment_dir; +use crate::types::{ + hash_prop_value, ComponentScrubFinding, ManifestState, ScrubFindingType, ScrubReport, + SecondaryIndexKind, SecondaryIndexManifestEntry, SecondaryIndexState, SecondaryIndexTarget, + SegmentInfo, SegmentScrubResult, +}; +use rayon::prelude::*; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashSet}; +use std::fmt::Debug; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; +use std::time::Instant; + +const SCRUB_READ_BUFFER_SIZE: usize = 64 * 1024; + +pub(crate) fn scrub_database( + db_dir: &Path, + manifest: &ManifestState, +) -> Result { + let start = Instant::now(); + + let segment_results: Vec = engine_cpu_install(|| { + manifest + .segments + .par_iter() + .map(|seg_info| scrub_one_segment(db_dir, manifest, seg_info)) + .collect() + }); + + let mut total_components_checked: u64 = 0; + let mut total_components_ok: u64 = 0; + let mut total_components_failed: u64 = 0; + let mut total_bytes_digested: u64 = 0; + + for seg in &segment_results { + let failed = seg + .findings + .iter() + .map(|finding| finding.component_kind.as_str()) + .collect::>() + .len() as u64; + let ok = seg.components_ok; + total_components_checked += ok + failed; + total_components_ok += ok; + total_components_failed += failed; + total_bytes_digested += seg.bytes_digested; + } + + Ok(ScrubReport { + segments: segment_results, + total_components_checked, + total_components_ok, + total_components_failed, + total_bytes_digested, + duration_ms: start.elapsed().as_millis() as u64, + }) +} + +fn scrub_one_segment( + db_dir: &Path, + manifest_state: &ManifestState, + seg_info: &SegmentInfo, +) -> SegmentScrubResult { + let seg_dir = segment_dir(db_dir, seg_info.id); + + if !seg_dir.exists() { + return SegmentScrubResult { + segment_id: seg_info.id, + findings: vec![ComponentScrubFinding { + component_kind: "segment".into(), + finding_type: ScrubFindingType::FileMissing, + detail: format!("segment directory does not exist: {}", seg_dir.display()), + }], + components_ok: 0, + bytes_digested: 0, + }; + } + + let manifest_path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + let manifest_data = match std::fs::read(&manifest_path) { + Ok(d) => d, + Err(e) => { + return SegmentScrubResult { + segment_id: seg_info.id, + findings: vec![ComponentScrubFinding { + component_kind: "segment_manifest".into(), + finding_type: ScrubFindingType::FileMissing, + detail: format!("cannot read segment manifest: {e}"), + }], + components_ok: 0, + bytes_digested: 0, + }; + } + }; + + let manifest = match decode_manifest_envelope(&manifest_data) { + Ok(m) => m, + Err(e) => { + return SegmentScrubResult { + segment_id: seg_info.id, + findings: vec![ComponentScrubFinding { + component_kind: "segment_manifest".into(), + finding_type: ScrubFindingType::IoError, + detail: format!("cannot decode segment manifest: {e}"), + }], + components_ok: 0, + bytes_digested: 0, + }; + } + }; + + let mut findings: Vec = Vec::new(); + let mut components_ok: u64 = 0; + let mut bytes_digested: u64 = 0; + + if let Err(error) = validate_segment_manifest_identity(seg_info, &manifest) { + findings.push(ComponentScrubFinding { + component_kind: "segment".into(), + finding_type: ScrubFindingType::SegmentIdentityMismatch, + detail: error.to_string(), + }); + } + + let packed_outcome = scrub_packed_core(&seg_dir, &manifest, &mut bytes_digested); + components_ok += packed_outcome.components_ok; + findings.extend(packed_outcome.findings); + + for record in &manifest.components { + if record.kind == SegmentComponentKind::PackedSegmentContainer + || matches!(record.handle, ComponentHandleV1::PackedRange { .. }) + { + continue; + } + + let component_findings = match &record.handle { + ComponentHandleV1::ExternalFile { .. } => { + scrub_external_component(&seg_dir, record, &manifest, &mut bytes_digested) + } + ComponentHandleV1::PackedRange { .. } => unreachable!("packed ranges handled above"), + }; + + if component_findings.is_empty() { + components_ok += 1; + } else { + findings.extend(component_findings); + } + } + + let semantic_findings = scrub_segment_node_semantics(&seg_dir, manifest_state, seg_info); + if semantic_findings.is_empty() { + components_ok += 1; + } else { + findings.extend(semantic_findings); + } + + SegmentScrubResult { + segment_id: seg_info.id, + findings, + components_ok, + bytes_digested, + } +} + +fn scrub_segment_node_semantics( + seg_dir: &Path, + manifest_state: &ManifestState, + seg_info: &SegmentInfo, +) -> Vec { + let mut findings = Vec::new(); + let reader = match SegmentReader::open_with_info( + seg_dir, + seg_info, + manifest_state.dense_vector.as_ref(), + &manifest_state.secondary_indexes, + ) { + Ok(reader) => reader, + Err(error) => { + findings.push(semantic_finding(format!( + "cannot open segment for node semantic scrub: {error}" + ))); + return findings; + } + }; + + let known_label_ids: HashSet = + manifest_state.node_label_tokens.values().copied().collect(); + let node_record_index = match reader.node_record_index_entries_for_scrub(seg_info.node_count) { + Ok(entries) => entries, + Err(error) => { + findings.push(semantic_finding(format!( + "cannot read node record index for semantic scrub: {error}" + ))); + return findings; + } + }; + let meta_count = match reader.node_meta_count_for_scrub(seg_info.node_count) { + Ok(count) => count, + Err(error) => { + findings.push(semantic_finding(format!( + "cannot read node metadata count for semantic scrub: {error}" + ))); + return findings; + } + }; + if node_record_index.len() != meta_count { + findings.push(semantic_finding(format!( + "node record index count {} does not match node metadata row count {}", + node_record_index.len(), + meta_count + ))); + } + + let node_property_indexes: Vec<&SecondaryIndexManifestEntry> = manifest_state + .secondary_indexes + .iter() + .filter(|entry| { + entry.state == SecondaryIndexState::Ready + && matches!(entry.target, SecondaryIndexTarget::NodeProperty { .. }) + }) + .collect(); + let mut expected_secondary_eq_groups: BTreeMap>> = BTreeMap::new(); + let mut expected_secondary_range_entries: BTreeMap> = BTreeMap::new(); + for entry in &node_property_indexes { + match entry.kind { + SecondaryIndexKind::Equality => { + expected_secondary_eq_groups + .entry(entry.index_id) + .or_default(); + } + SecondaryIndexKind::Range { .. } => { + expected_secondary_range_entries + .entry(entry.index_id) + .or_default(); + } + } + } + + let mut expected_label_entries: Vec<(u32, u64)> = Vec::new(); + let mut expected_key_entries: Vec<(u32, String, u64)> = Vec::new(); + let mut expected_timestamp_entries: Vec<(u32, i64, u64)> = Vec::new(); + let mut previous_node_id = None; + + for index in 0..meta_count { + let meta = match reader.node_meta_at(index) { + Ok(meta) => meta, + Err(error) => { + findings.push(semantic_finding(format!( + "cannot read node metadata row {}: {error}", + index + ))); + continue; + } + }; + if previous_node_id.is_some_and(|previous| previous >= meta.node_id) { + findings.push(semantic_finding(format!( + "node metadata row {} is not sorted by unique node_id: previous {:?}, current {}", + index, previous_node_id, meta.node_id + ))); + } + previous_node_id = Some(meta.node_id); + let index_entry = node_record_index.get(index).copied(); + if index_entry.map(|(node_id, _)| node_id) != Some(meta.node_id) { + findings.push(semantic_finding(format!( + "node record index row {} id {:?} does not match metadata node_id {}", + index, + index_entry.map(|(node_id, _)| node_id), + meta.node_id + ))); + } + if index_entry.map(|(_, data_offset)| data_offset) != Some(meta.data_offset) { + findings.push(semantic_finding(format!( + "node record index row {} offset {:?} does not match metadata data_offset {}", + index, + index_entry.map(|(_, data_offset)| data_offset), + meta.data_offset + ))); + } + + let node = match reader.node_record_for_meta_scrub(&meta) { + Ok(node) => node, + Err(error) => { + findings.push(semantic_finding(format!( + "cannot decode node record {} from metadata span: {error}", + meta.node_id + ))); + continue; + } + }; + if node.label_ids != meta.label_ids { + findings.push(semantic_finding(format!( + "node {} record labels {:?} do not match metadata labels {:?}", + meta.node_id, node.label_ids, meta.label_ids + ))); + } + if node.key.len() != meta.key_len as usize { + findings.push(semantic_finding(format!( + "node {} record key length {} does not match metadata key_len {}", + meta.node_id, + node.key.len(), + meta.key_len + ))); + } + if node.updated_at != meta.updated_at { + findings.push(semantic_finding(format!( + "node {} record updated_at {} does not match metadata updated_at {}", + meta.node_id, node.updated_at, meta.updated_at + ))); + } + if node.weight.to_bits() != meta.weight.to_bits() { + findings.push(semantic_finding(format!( + "node {} record weight {} does not match metadata weight {}", + meta.node_id, node.weight, meta.weight + ))); + } + + for &label_id in meta.label_ids.as_slice() { + if !known_label_ids.contains(&label_id) { + findings.push(semantic_finding(format!( + "node {} references node label_id {} that is absent from the manifest catalog", + meta.node_id, label_id + ))); + } + expected_label_entries.push((label_id, meta.node_id)); + expected_key_entries.push((label_id, node.key.clone(), meta.node_id)); + expected_timestamp_entries.push((label_id, meta.updated_at, meta.node_id)); + } + + for entry in &node_property_indexes { + let SecondaryIndexTarget::NodeProperty { label_id, prop_key } = &entry.target else { + continue; + }; + if !meta.label_ids.contains(*label_id) { + continue; + } + let Some(value) = node.props.get(prop_key) else { + continue; + }; + match entry.kind { + SecondaryIndexKind::Equality => { + expected_secondary_eq_groups + .entry(entry.index_id) + .or_default() + .entry(hash_prop_value(value)) + .or_default() + .push(meta.node_id); + } + SecondaryIndexKind::Range { domain } => { + if let Some(encoded_value) = encode_range_prop_value(domain, value) { + expected_secondary_range_entries + .entry(entry.index_id) + .or_default() + .push((encoded_value, meta.node_id)); + } + } + } + } + } + + expected_label_entries.sort_unstable(); + expected_key_entries.sort(); + expected_timestamp_entries.sort_unstable(); + for groups in expected_secondary_eq_groups.values_mut() { + for ids in groups.values_mut() { + ids.sort_unstable(); + ids.dedup(); + } + } + for entries in expected_secondary_range_entries.values_mut() { + entries.sort_unstable(); + entries.dedup(); + } + + match reader.node_label_index_entries_for_scrub(seg_info.node_count) { + Ok(actual) => compare_semantic_entries( + "node label index", + &expected_label_entries, + &actual, + &mut findings, + ), + Err(error) => findings.push(semantic_finding(format!( + "cannot read node label index for semantic scrub: {error}" + ))), + } + match reader.node_key_index_entries_for_scrub() { + Ok(actual) => compare_semantic_entries( + "node key index", + &expected_key_entries, + &actual, + &mut findings, + ), + Err(error) => findings.push(semantic_finding(format!( + "cannot read node key index for semantic scrub: {error}" + ))), + } + match reader.node_timestamp_index_entries_for_scrub() { + Ok(actual) => compare_semantic_entries( + "node timestamp index", + &expected_timestamp_entries, + &actual, + &mut findings, + ), + Err(error) => findings.push(semantic_finding(format!( + "cannot read node timestamp index for semantic scrub: {error}" + ))), + } + scrub_declared_node_property_indexes( + &reader, + &node_property_indexes, + &expected_secondary_eq_groups, + &expected_secondary_range_entries, + &mut findings, + ); + + findings +} + +fn scrub_declared_node_property_indexes( + reader: &SegmentReader, + node_property_indexes: &[&SecondaryIndexManifestEntry], + expected_secondary_eq_groups: &BTreeMap>>, + expected_secondary_range_entries: &BTreeMap>, + findings: &mut Vec, +) { + for entry in node_property_indexes { + match entry.kind { + SecondaryIndexKind::Equality => { + let expected = expected_secondary_eq_groups + .get(&entry.index_id) + .cloned() + .unwrap_or_default(); + let mut actual = BTreeMap::new(); + match reader.for_each_secondary_eq_group(entry.index_id, |value_hash, ids| { + actual.insert(value_hash, ids.to_vec()); + Ok(()) + }) { + Ok(true) => { + let name = + format!("declared node-property equality index {}", entry.index_id); + let expected_entries: Vec<(u64, Vec)> = + expected.into_iter().collect(); + let actual_entries: Vec<(u64, Vec)> = actual.into_iter().collect(); + compare_semantic_entries( + &name, + &expected_entries, + &actual_entries, + findings, + ); + } + Ok(false) => { + if entry.state == SecondaryIndexState::Ready || !expected.is_empty() { + findings.push(semantic_finding(format!( + "declared node-property equality index {} sidecar is missing", + entry.index_id + ))); + } + } + Err(error) => findings.push(semantic_finding(format!( + "cannot read declared node-property equality index {} for semantic scrub: {error}", + entry.index_id + ))), + } + } + SecondaryIndexKind::Range { .. } => { + let expected = expected_secondary_range_entries + .get(&entry.index_id) + .cloned() + .unwrap_or_default(); + let mut actual = Vec::new(); + match reader.for_each_secondary_range_entry(entry.index_id, |encoded_value, node_id| { + actual.push((encoded_value, node_id)); + Ok(()) + }) { + Ok(true) => { + let name = + format!("declared node-property range index {}", entry.index_id); + compare_semantic_entries(&name, &expected, &actual, findings); + } + Ok(false) => { + if entry.state == SecondaryIndexState::Ready || !expected.is_empty() { + findings.push(semantic_finding(format!( + "declared node-property range index {} sidecar is missing", + entry.index_id + ))); + } + } + Err(error) => findings.push(semantic_finding(format!( + "cannot read declared node-property range index {} for semantic scrub: {error}", + entry.index_id + ))), + } + } + } + } +} + +fn semantic_finding(detail: String) -> ComponentScrubFinding { + ComponentScrubFinding { + component_kind: "NodeSemantic".into(), + finding_type: ScrubFindingType::SemanticMismatch, + detail, + } +} + +fn compare_semantic_entries( + name: &str, + expected: &[T], + actual: &[T], + findings: &mut Vec, +) { + if expected == actual { + return; + } + let mismatch_index = expected + .iter() + .zip(actual.iter()) + .position(|(left, right)| left != right) + .unwrap_or_else(|| expected.len().min(actual.len())); + findings.push(semantic_finding(format!( + "{} mismatch: expected {} entries, found {}; first mismatch at {} expected {:?}, found {:?}", + name, + expected.len(), + actual.len(), + mismatch_index, + expected.get(mismatch_index), + actual.get(mismatch_index) + ))); +} + +fn scrub_external_component( + seg_dir: &Path, + record: &crate::segment_components::SegmentComponentRecordV1, + manifest: &crate::segment_components::SegmentComponentManifestV1, + bytes_digested: &mut u64, +) -> Vec { + let mut findings = Vec::new(); + let kind_name = format!("{:?}", record.kind); + + let (relative_path, payload_offset, payload_len) = match &record.handle { + ComponentHandleV1::ExternalFile { + relative_path, + payload_offset, + payload_len, + } => (relative_path, *payload_offset, *payload_len), + _ => return findings, + }; + + let file_path = seg_dir.join(relative_path); + let mut file = match File::open(&file_path) { + Ok(file) => file, + Err(e) => { + findings.push(ComponentScrubFinding { + component_kind: kind_name, + finding_type: ScrubFindingType::FileMissing, + detail: format!("cannot open component file {relative_path}: {e}"), + }); + return findings; + } + }; + let file_len = match file.metadata() { + Ok(metadata) => metadata.len(), + Err(e) => { + findings.push(ComponentScrubFinding { + component_kind: kind_name, + finding_type: ScrubFindingType::IoError, + detail: format!("cannot stat component file {relative_path}: {e}"), + }); + return findings; + } + }; + + if payload_offset > 0 { + match read_identity_header_from_file(&mut file) { + Ok(header) => { + findings.extend(validate_header_vs_record(&header, record, manifest)); + } + Err(e) => { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!("cannot decode identity header: {e}"), + }); + } + } + } + + let payload_end = + match checked_component_payload_end(&kind_name, payload_offset, payload_len, file_len) { + Ok(end) => end, + Err(finding) => { + findings.push(finding); + payload_offset.saturating_add(payload_len).min(file_len) + } + }; + + if let Some(expected_digest) = &record.payload_digest { + if payload_offset + .checked_add(payload_len) + .is_some_and(|expected_end| payload_end == expected_end && expected_end <= file_len) + { + match streaming_sha256(&mut file, payload_offset, payload_len) { + Ok(actual_digest) => { + *bytes_digested += payload_len; + if actual_digest != *expected_digest { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::PayloadDigestMismatch, + detail: format!( + "payload SHA-256 mismatch: expected {}, got {}", + hex_short(expected_digest), + hex_short(&actual_digest), + ), + }); + } + } + Err(e) => { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IoError, + detail: format!("I/O error computing payload digest: {e}"), + }); + } + } + } + } + + findings.extend(validate_component_record_metadata( + record, manifest, &kind_name, + )); + + findings +} + +fn checked_component_payload_end( + kind_name: &str, + payload_offset: u64, + payload_len: u64, + file_len: u64, +) -> Result { + let Some(end) = payload_offset.checked_add(payload_len) else { + return Err(ComponentScrubFinding { + component_kind: kind_name.into(), + finding_type: ScrubFindingType::RangeOverflow, + detail: format!( + "payload range overflows: offset={}, len={}", + payload_offset, payload_len + ), + }); + }; + if end != file_len { + return Err(ComponentScrubFinding { + component_kind: kind_name.into(), + finding_type: ScrubFindingType::RangeOverflow, + detail: format!( + "payload range [{}, {}) does not match file length {}", + payload_offset, end, file_len + ), + }); + } + Ok(end) +} + +fn read_identity_header_from_file(file: &mut File) -> std::io::Result { + file.seek(SeekFrom::Start(0))?; + let mut hdr_buf = [0u8; COMPONENT_IDENTITY_HEADER_LEN]; + file.read_exact(&mut hdr_buf)?; + decode_identity_header(&hdr_buf) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string())) +} + +fn validate_component_record_metadata( + record: &SegmentComponentRecordV1, + manifest: &SegmentComponentManifestV1, + kind_name: &str, +) -> Vec { + let mut findings = Vec::new(); + + let recomputed_dep_digest = dependency_digest(&record.dependencies); + if recomputed_dep_digest != record.dependency_digest { + findings.push(ComponentScrubFinding { + component_kind: kind_name.into(), + finding_type: ScrubFindingType::DependencyDigestMismatch, + detail: "recomputed dependency digest does not match stored value".into(), + }); + } + + let recomputed_id = component_id( + manifest.segment_id, + &record.kind, + record.logical_format_version, + record.payload_len, + record.payload_digest.as_ref(), + &record.dependency_digest, + record.build_fingerprint, + ); + if recomputed_id != record.component_id { + findings.push(ComponentScrubFinding { + component_kind: kind_name.into(), + finding_type: ScrubFindingType::ComponentIdMismatch, + detail: "recomputed component_id does not match stored value".into(), + }); + } + + findings +} + +fn scrub_packed_core( + seg_dir: &Path, + manifest: &SegmentComponentManifestV1, + bytes_digested: &mut u64, +) -> PackedScrubOutcome { + let relevant_indices: Vec = manifest + .components + .iter() + .enumerate() + .filter_map(|(index, record)| { + if record.kind == SegmentComponentKind::PackedSegmentContainer + || matches!(record.handle, ComponentHandleV1::PackedRange { .. }) + { + Some(index) + } else { + None + } + }) + .collect(); + if relevant_indices.is_empty() { + return PackedScrubOutcome::default(); + } + + let mut per_record_findings: Vec> = + (0..manifest.components.len()).map(|_| Vec::new()).collect(); + let Some(container_index) = manifest + .components + .iter() + .position(|record| record.kind == SegmentComponentKind::PackedSegmentContainer) + else { + for &index in &relevant_indices { + let kind_name = format!("{:?}", manifest.components[index].kind); + per_record_findings[index].push(ComponentScrubFinding { + component_kind: kind_name, + finding_type: ScrubFindingType::FileMissing, + detail: "packed range has no segment.core container record".into(), + }); + } + return finish_packed_outcome(manifest, relevant_indices, per_record_findings); + }; + + let container_record = &manifest.components[container_index]; + let container_kind_name = format!("{:?}", container_record.kind); + per_record_findings[container_index].extend(validate_component_record_metadata( + container_record, + manifest, + &container_kind_name, + )); + + let (relative_path, payload_offset, payload_len) = match &container_record.handle { + ComponentHandleV1::ExternalFile { + relative_path, + payload_offset, + payload_len, + } => (relative_path, *payload_offset, *payload_len), + ComponentHandleV1::PackedRange { .. } => { + per_record_findings[container_index].push(ComponentScrubFinding { + component_kind: container_kind_name, + finding_type: ScrubFindingType::RangeOverflow, + detail: "packed container must use an external file handle".into(), + }); + mark_packed_ranges_unverified( + manifest, + &relevant_indices, + &mut per_record_findings, + ScrubFindingType::RangeOverflow, + "packed range could not be validated because segment.core container record is not an external file", + ); + return finish_packed_outcome(manifest, relevant_indices, per_record_findings); + } + }; + + if relative_path != PACKED_CORE_FILENAME { + per_record_findings[container_index].push(ComponentScrubFinding { + component_kind: container_kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!("packed container path must be {PACKED_CORE_FILENAME}"), + }); + } + + let core_path = seg_dir.join(relative_path); + let mut file = match File::open(&core_path) { + Ok(file) => file, + Err(e) => { + per_record_findings[container_index].push(ComponentScrubFinding { + component_kind: container_kind_name.clone(), + finding_type: ScrubFindingType::FileMissing, + detail: format!("cannot open segment.core: {e}"), + }); + mark_packed_ranges_unverified( + manifest, + &relevant_indices, + &mut per_record_findings, + ScrubFindingType::FileMissing, + format!("packed range could not be validated because segment.core could not be opened: {e}"), + ); + return finish_packed_outcome(manifest, relevant_indices, per_record_findings); + } + }; + let file_len = match file.metadata() { + Ok(metadata) => metadata.len(), + Err(e) => { + per_record_findings[container_index].push(ComponentScrubFinding { + component_kind: container_kind_name.clone(), + finding_type: ScrubFindingType::IoError, + detail: format!("cannot stat segment.core: {e}"), + }); + mark_packed_ranges_unverified( + manifest, + &relevant_indices, + &mut per_record_findings, + ScrubFindingType::IoError, + format!("packed range could not be validated because segment.core metadata could not be read: {e}"), + ); + return finish_packed_outcome(manifest, relevant_indices, per_record_findings); + } + }; + + match read_identity_header_from_file(&mut file) { + Ok(header) => { + per_record_findings[container_index].extend(validate_header_vs_record( + &header, + container_record, + manifest, + )); + } + Err(e) => { + per_record_findings[container_index].push(ComponentScrubFinding { + component_kind: container_kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!("cannot decode identity header: {e}"), + }); + } + } + + let payload_end = match checked_component_payload_end( + &container_kind_name, + payload_offset, + payload_len, + file_len, + ) { + Ok(end) => Some(end), + Err(finding) => { + per_record_findings[container_index].push(finding); + payload_offset + .checked_add(payload_len) + .filter(|end| *end <= file_len) + } + }; + + let mut range_targets = Vec::new(); + for (index, record) in manifest.components.iter().enumerate() { + let ComponentHandleV1::PackedRange { + container_component_id, + offset, + len, + } = &record.handle + else { + continue; + }; + let kind_name = format!("{:?}", record.kind); + per_record_findings[index].extend(validate_component_record_metadata( + record, manifest, &kind_name, + )); + if *container_component_id != container_record.component_id { + per_record_findings[index].push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::ContainerIdMismatch, + detail: format!( + "packed range container_component_id {} does not match container {}", + hex_short(container_component_id), + hex_short(&container_record.component_id), + ), + }); + } + if record.payload_len != *len { + per_record_findings[index].push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::RangeOverflow, + detail: format!( + "packed range length {} does not match record payload_len {}", + len, record.payload_len + ), + }); + } + let Some(end) = offset.checked_add(*len) else { + per_record_findings[index].push(ComponentScrubFinding { + component_kind: kind_name, + finding_type: ScrubFindingType::RangeOverflow, + detail: format!("packed range overflows: offset={}, len={}", offset, len), + }); + continue; + }; + if end > payload_len { + per_record_findings[index].push(ComponentScrubFinding { + component_kind: kind_name, + finding_type: ScrubFindingType::RangeOverflow, + detail: format!( + "packed range [{}, {}) exceeds segment.core payload length {}", + offset, end, payload_len + ), + }); + continue; + } + if per_record_findings[index].iter().any(|finding| { + matches!( + finding.finding_type, + ScrubFindingType::RangeOverflow | ScrubFindingType::ContainerIdMismatch + ) + }) { + continue; + } + range_targets.push(PackedRangeDigestTarget { + record_index: index, + kind_name, + start: *offset, + end, + hasher: Sha256::new(), + }); + } + + range_targets.sort_by(|left, right| { + left.start + .cmp(&right.start) + .then_with(|| left.end.cmp(&right.end)) + .then_with(|| { + manifest.components[left.record_index] + .kind + .kind_tag() + .cmp(&manifest.components[right.record_index].kind.kind_tag()) + }) + .then_with(|| { + manifest.components[left.record_index] + .kind + .index_id() + .cmp(&manifest.components[right.record_index].kind.index_id()) + }) + }); + + let mut overlapped = HashSet::new(); + let overlap_targets: Vec<&PackedRangeDigestTarget> = range_targets + .iter() + .filter(|target| target.start < target.end) + .collect(); + for pair in overlap_targets.windows(2) { + let previous = pair[0]; + let current = pair[1]; + if current.start < previous.end { + let detail = format!( + "packed component ranges overlap: previous=[{}, {}), current=[{}, {})", + previous.start, previous.end, current.start, current.end + ); + for target in [previous, current] { + if overlapped.insert(target.record_index) { + per_record_findings[target.record_index].push(ComponentScrubFinding { + component_kind: target.kind_name.clone(), + finding_type: ScrubFindingType::RangeOverlap, + detail: detail.clone(), + }); + } + } + } + } + range_targets.retain(|target| !overlapped.contains(&target.record_index)); + + let mut range_digests_verified = false; + if payload_end.is_some() { + match stream_packed_core_payload(&mut file, payload_offset, payload_len, &mut range_targets) + { + Ok(container_digest) => { + range_digests_verified = true; + *bytes_digested += payload_len; + if let Some(expected_digest) = &container_record.payload_digest { + if container_digest != *expected_digest { + per_record_findings[container_index].push(ComponentScrubFinding { + component_kind: container_kind_name.clone(), + finding_type: ScrubFindingType::PayloadDigestMismatch, + detail: format!( + "payload SHA-256 mismatch: expected {}, got {}", + hex_short(expected_digest), + hex_short(&container_digest), + ), + }); + } + } + } + Err(e) => { + per_record_findings[container_index].push(ComponentScrubFinding { + component_kind: container_kind_name.clone(), + finding_type: ScrubFindingType::IoError, + detail: format!("I/O error computing segment.core payload digest: {e}"), + }); + mark_packed_ranges_unverified( + manifest, + &relevant_indices, + &mut per_record_findings, + ScrubFindingType::IoError, + format!( + "packed range could not be validated because segment.core streaming failed: {e}" + ), + ); + } + } + } else { + mark_packed_ranges_unverified( + manifest, + &relevant_indices, + &mut per_record_findings, + ScrubFindingType::RangeOverflow, + "packed range could not be validated because segment.core payload range is invalid", + ); + } + + if range_digests_verified { + for target in range_targets { + let record = &manifest.components[target.record_index]; + if let Some(expected_digest) = &record.payload_digest { + let actual_digest: [u8; 32] = target.hasher.finalize().into(); + if actual_digest != *expected_digest { + per_record_findings[target.record_index].push(ComponentScrubFinding { + component_kind: target.kind_name, + finding_type: ScrubFindingType::PayloadDigestMismatch, + detail: format!( + "payload SHA-256 mismatch: expected {}, got {}", + hex_short(expected_digest), + hex_short(&actual_digest), + ), + }); + } + } + } + } + + finish_packed_outcome(manifest, relevant_indices, per_record_findings) +} + +#[derive(Default)] +struct PackedScrubOutcome { + findings: Vec, + components_ok: u64, +} + +struct PackedRangeDigestTarget { + record_index: usize, + kind_name: String, + start: u64, + end: u64, + hasher: Sha256, +} + +fn finish_packed_outcome( + manifest: &SegmentComponentManifestV1, + relevant_indices: Vec, + per_record_findings: Vec>, +) -> PackedScrubOutcome { + let mut outcome = PackedScrubOutcome::default(); + for index in relevant_indices { + if per_record_findings[index].is_empty() { + outcome.components_ok += 1; + } else { + outcome.findings.extend(per_record_findings[index].clone()); + } + } + debug_assert!( + outcome.components_ok as usize + outcome.findings.len() >= 1 + || manifest.components.is_empty() + ); + outcome +} + +fn mark_packed_ranges_unverified( + manifest: &SegmentComponentManifestV1, + relevant_indices: &[usize], + per_record_findings: &mut [Vec], + finding_type: ScrubFindingType, + detail: impl Into, +) { + let detail = detail.into(); + for &index in relevant_indices { + let record = &manifest.components[index]; + if matches!(record.handle, ComponentHandleV1::PackedRange { .. }) { + per_record_findings[index].push(ComponentScrubFinding { + component_kind: format!("{:?}", record.kind), + finding_type, + detail: detail.clone(), + }); + } + } +} + +fn stream_packed_core_payload( + file: &mut File, + payload_offset: u64, + payload_len: u64, + targets: &mut [PackedRangeDigestTarget], +) -> std::io::Result<[u8; 32]> { + file.seek(SeekFrom::Start(payload_offset))?; + let mut container_hasher = Sha256::new(); + let mut remaining = payload_len; + let mut relative_pos = 0u64; + let mut target_cursor = 0usize; + let mut buf = vec![0u8; SCRUB_READ_BUFFER_SIZE]; + while remaining > 0 { + let to_read = remaining.min(SCRUB_READ_BUFFER_SIZE as u64) as usize; + let n = file.read(&mut buf[..to_read])?; + if n == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "unexpected EOF during packed core scrub digest", + )); + } + let chunk = &buf[..n]; + container_hasher.update(chunk); + let chunk_start = relative_pos; + let chunk_end = relative_pos + n as u64; + + while target_cursor < targets.len() && targets[target_cursor].end <= chunk_start { + target_cursor += 1; + } + let mut index = target_cursor; + while index < targets.len() && targets[index].start < chunk_end { + let overlap_start = targets[index].start.max(chunk_start); + let overlap_end = targets[index].end.min(chunk_end); + if overlap_start < overlap_end { + let local_start = (overlap_start - chunk_start) as usize; + let local_end = (overlap_end - chunk_start) as usize; + targets[index].hasher.update(&chunk[local_start..local_end]); + } + index += 1; + } + + relative_pos = chunk_end; + remaining -= n as u64; + } + Ok(container_hasher.finalize().into()) +} + +fn streaming_sha256(file: &mut File, offset: u64, len: u64) -> std::io::Result<[u8; 32]> { + file.seek(SeekFrom::Start(offset))?; + let mut hasher = Sha256::new(); + let mut remaining = len; + let mut buf = vec![0u8; SCRUB_READ_BUFFER_SIZE]; + while remaining > 0 { + let to_read = (remaining as usize).min(SCRUB_READ_BUFFER_SIZE); + let n = file.read(&mut buf[..to_read])?; + if n == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "unexpected EOF during scrub digest", + )); + } + hasher.update(&buf[..n]); + remaining -= n as u64; + } + Ok(hasher.finalize().into()) +} + +fn validate_header_vs_record( + header: &ComponentIdentityHeaderV1, + record: &crate::segment_components::SegmentComponentRecordV1, + manifest: &crate::segment_components::SegmentComponentManifestV1, +) -> Vec { + let mut findings = Vec::new(); + let kind_name = format!("{:?}", record.kind); + + if header.segment_id != manifest.segment_id { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header segment_id {} != manifest segment_id {}", + header.segment_id, manifest.segment_id, + ), + }); + } + + if header.component_kind != record.kind { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header component_kind {:?} != record {:?}", + header.component_kind, record.kind + ), + }); + } + + if let ComponentHandleV1::ExternalFile { payload_offset, .. } = &record.handle { + if header.payload_offset != *payload_offset { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header payload_offset {} != record {}", + header.payload_offset, payload_offset + ), + }); + } + } + + if header.component_id != record.component_id { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: "header component_id does not match manifest record".into(), + }); + } + + if header.dependency_digest != record.dependency_digest { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: "header dependency_digest does not match manifest record".into(), + }); + } + + if header.build_fingerprint != record.build_fingerprint { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header build_fingerprint {} != record {}", + header.build_fingerprint, record.build_fingerprint, + ), + }); + } + + if header.payload_len != record.payload_len { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header payload_len {} != record {}", + header.payload_len, record.payload_len, + ), + }); + } + + if header.logical_format_version != record.logical_format_version { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header logical_format_version {} != record {}", + header.logical_format_version, record.logical_format_version, + ), + }); + } + + if header.segment_format_version != manifest.segment_format_version { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header segment_format_version {} != manifest {}", + header.segment_format_version, manifest.segment_format_version, + ), + }); + } + + if header.created_generation != record.created_generation { + findings.push(ComponentScrubFinding { + component_kind: kind_name.clone(), + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: format!( + "header created_generation {} != record {}", + header.created_generation, record.created_generation, + ), + }); + } + + if header.payload_digest != record.payload_digest { + findings.push(ComponentScrubFinding { + component_kind: kind_name, + finding_type: ScrubFindingType::IdentityHeaderMismatch, + detail: "header payload_digest does not match manifest record".into(), + }); + } + + findings +} + +fn hex_short(digest: &[u8; 32]) -> String { + format!("{}..{}", hex_byte(digest[0]), hex_byte(digest[31]),) +} + +fn hex_byte(b: u8) -> String { + format!("{b:02x}") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::segment_components::{ + encode_manifest_envelope, SegmentComponentManifestV1, SegmentComponentRecordV1, + }; + use crate::{ + DatabaseEngine, DbOptions, NodeInput, PropValue, ScrubFindingType, SecondaryIndexKind, + SecondaryIndexRangeDomain, UpsertEdgeOptions, UpsertNodeOptions, + }; + use std::collections::BTreeMap; + use std::fs::OpenOptions; + use std::io::{Seek, SeekFrom, Write}; + use std::path::{Path, PathBuf}; + use std::time::Duration; + use tempfile::TempDir; + + fn open_test_db(dir: &Path) -> DatabaseEngine { + let opts = DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }; + DatabaseEngine::open(dir, &opts).unwrap() + } + + fn populate_and_flush(db: &DatabaseEngine) { + let nodes: Vec = (0..10) + .map(|i| NodeInput { + labels: vec!["Person".to_string()], + key: format!("node_{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + let ids = db.batch_upsert_nodes(nodes.clone()).unwrap(); + + for i in 0..5 { + db.upsert_edge( + ids[i], + ids[i + 5], + "RELATES_TO", + UpsertEdgeOptions::default(), + ) + .unwrap(); + } + + db.flush().unwrap(); + } + + fn populated_db() -> (TempDir, PathBuf, DatabaseEngine) { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + populate_and_flush(&db); + (dir, db_path, db) + } + + fn first_seg_dir(db_path: &Path) -> PathBuf { + db_path.join("segments").join("seg_0001") + } + + fn read_segment_manifest(seg_dir: &Path) -> SegmentComponentManifestV1 { + let data = std::fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + decode_manifest_envelope(&data).unwrap() + } + + fn write_segment_manifest(seg_dir: &Path, manifest: &SegmentComponentManifestV1) { + let data = encode_manifest_envelope(manifest).unwrap(); + std::fs::write(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME), data).unwrap(); + } + + fn any_finding(report: &ScrubReport, finding_type: ScrubFindingType) -> bool { + report + .segments + .iter() + .flat_map(|segment| &segment.findings) + .any(|finding| finding.finding_type == finding_type) + } + + fn any_finding_detail( + report: &ScrubReport, + finding_type: ScrubFindingType, + detail: &str, + ) -> bool { + report + .segments + .iter() + .flat_map(|segment| &segment.findings) + .any(|finding| finding.finding_type == finding_type && finding.detail.contains(detail)) + } + + fn assert_scrub_reports_segment_identity_mismatch( + mutate: impl FnOnce(&mut SegmentInfo, &mut SegmentComponentManifestV1), + ) { + let (_dir, db_path, db) = populated_db(); + let root_manifest = db.manifest().unwrap(); + let mut root_segment = root_manifest.segments[0].clone(); + let seg_dir = first_seg_dir(&db_path); + let mut local_manifest = read_segment_manifest(&seg_dir); + mutate(&mut root_segment, &mut local_manifest); + write_segment_manifest(&seg_dir, &local_manifest); + + let result = scrub_one_segment(db.path(), &root_manifest, &root_segment); + assert!( + result + .findings + .iter() + .any(|finding| finding.finding_type == ScrubFindingType::SegmentIdentityMismatch), + "expected segment identity mismatch, got: {:?}", + result.findings + ); + } + + fn first_external_non_container_record_mut( + manifest: &mut SegmentComponentManifestV1, + ) -> &mut SegmentComponentRecordV1 { + manifest + .components + .iter_mut() + .find(|record| { + record.kind != SegmentComponentKind::PackedSegmentContainer + && matches!(record.handle, ComponentHandleV1::ExternalFile { .. }) + }) + .expect("test precondition: expected an external sidecar") + } + + fn read_test_u64(data: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap()) + } + + fn packed_component_payload_start( + manifest: &SegmentComponentManifestV1, + kind: SegmentComponentKind, + ) -> usize { + let container_payload_offset = manifest + .components + .iter() + .find_map(|record| { + if record.kind == SegmentComponentKind::PackedSegmentContainer { + if let ComponentHandleV1::ExternalFile { payload_offset, .. } = &record.handle { + return Some(*payload_offset); + } + } + None + }) + .expect("test precondition: packed container must be external"); + let component_offset = manifest + .components + .iter() + .find_map(|record| { + if record.kind == kind { + if let ComponentHandleV1::PackedRange { offset, .. } = &record.handle { + return Some(*offset); + } + } + None + }) + .expect("test precondition: component must be packed"); + (container_payload_offset + component_offset) as usize + } + + fn packed_node_records_payload_start(manifest: &SegmentComponentManifestV1) -> usize { + packed_component_payload_start(manifest, SegmentComponentKind::NodeRecords) + } + + fn external_component_payload_offset( + manifest: &SegmentComponentManifestV1, + kind: SegmentComponentKind, + ) -> usize { + manifest + .components + .iter() + .find_map(|record| { + if record.kind == kind { + if let ComponentHandleV1::ExternalFile { payload_offset, .. } = record.handle { + return Some(payload_offset as usize); + } + } + None + }) + .expect("test precondition: component must be external") + } + + fn create_declared_node_property_scrub_db() -> (TempDir, PathBuf, DatabaseEngine, u64, u64) { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + let eq = db + .ensure_node_property_index("Researcher", "status", SecondaryIndexKind::Equality) + .unwrap(); + let range = db + .ensure_node_property_index( + "Researcher", + "score", + SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + ) + .unwrap(); + db.shutdown_secondary_index_worker(); + + let nodes: Vec = (0..4) + .map(|i| { + let mut props = BTreeMap::new(); + props.insert( + "status".to_string(), + PropValue::String(if i % 2 == 0 { "active" } else { "idle" }.to_string()), + ); + props.insert("score".to_string(), PropValue::Int(i as i64 + 10)); + NodeInput { + labels: vec!["Person".to_string(), "Researcher".to_string()], + key: format!("node_{i}"), + props, + weight: 1.0, + dense_vector: None, + sparse_vector: None, + } + }) + .collect(); + db.batch_upsert_nodes(nodes).unwrap(); + db.flush().unwrap(); + db.with_runtime_manifest_write(|manifest| { + for entry in &mut manifest.secondary_indexes { + if entry.index_id == eq.index_id || entry.index_id == range.index_id { + entry.state = SecondaryIndexState::Ready; + entry.last_error = None; + } + } + Ok(()) + }) + .unwrap(); + (dir, db_path, db, eq.index_id, range.index_id) + } + + #[test] + fn scrub_semantic_comparator_reports_index_divergence_without_payload_corruption() { + let mut findings = Vec::new(); + compare_semantic_entries( + "node label index", + &[(1u32, 1u64), (2, 1)], + &[(1u32, 1u64)], + &mut findings, + ); + compare_semantic_entries( + "node key index", + &[(1u32, "alice".to_string(), 1u64)], + &[(1u32, "bob".to_string(), 1u64)], + &mut findings, + ); + compare_semantic_entries( + "node timestamp index", + &[(1u32, 100i64, 1u64)], + &[(1u32, 101i64, 1u64)], + &mut findings, + ); + + assert_eq!(findings.len(), 3); + for finding in &findings { + assert_eq!(finding.finding_type, ScrubFindingType::SemanticMismatch); + assert_eq!(finding.component_kind, "NodeSemantic"); + } + assert!(findings[0].detail.contains("node label index mismatch")); + assert!(findings[1].detail.contains("node key index mismatch")); + assert!(findings[2].detail.contains("node timestamp index mismatch")); + } + + #[test] + fn scrub_detects_multi_label_node_record_metadata_mismatch() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + let nodes: Vec = (0..4) + .map(|i| NodeInput { + labels: vec!["Person".to_string(), "Researcher".to_string()], + key: format!("node_{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + db.batch_upsert_nodes(nodes).unwrap(); + db.flush().unwrap(); + + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let node_records_payload_start = packed_node_records_payload_start(&manifest); + let core_path = seg_dir.join(PACKED_CORE_FILENAME); + let mut core = std::fs::read(&core_path).unwrap(); + let first_record_offset = read_test_u64(&core, node_records_payload_start + 8 + 8) as usize; + let first_label_id_offset = node_records_payload_start + first_record_offset + 1; + core[first_label_id_offset..first_label_id_offset + 4].copy_from_slice(&0u32.to_le_bytes()); + std::fs::write(&core_path, core).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding(&report, ScrubFindingType::SemanticMismatch), + "expected semantic mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_detects_node_record_index_offset_metadata_mismatch() { + let (_dir, db_path, db) = populated_db(); + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let node_records_payload_start = packed_node_records_payload_start(&manifest); + let core_path = seg_dir.join(PACKED_CORE_FILENAME); + let mut core = std::fs::read(&core_path).unwrap(); + let first_offset_pos = node_records_payload_start + 8 + 8; + let second_offset_pos = node_records_payload_start + 8 + 16 + 8; + let second_record_offset = read_test_u64(&core, second_offset_pos); + core[first_offset_pos..first_offset_pos + 8] + .copy_from_slice(&second_record_offset.to_le_bytes()); + std::fs::write(&core_path, core).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding_detail( + &report, + ScrubFindingType::SemanticMismatch, + "node record index row 0 offset" + ), + "expected node record index offset semantic mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_detects_node_label_index_overlapping_posting_ranges() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + let nodes: Vec = (0..4) + .map(|i| NodeInput { + labels: vec!["Person".to_string(), "Researcher".to_string()], + key: format!("node_{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + db.batch_upsert_nodes(nodes).unwrap(); + db.flush().unwrap(); + + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let node_label_index_start = + packed_component_payload_start(&manifest, SegmentComponentKind::NodeLabelIndex); + let core_path = seg_dir.join(PACKED_CORE_FILENAME); + let mut core = std::fs::read(&core_path).unwrap(); + let label_count = read_test_u64(&core, node_label_index_start); + assert!( + label_count >= 2, + "test precondition: expected at least two node-label rows" + ); + let first_posting_offset = read_test_u64(&core, node_label_index_start + 8 + 4); + let second_posting_offset_pos = node_label_index_start + 8 + 16 + 4; + core[second_posting_offset_pos..second_posting_offset_pos + 8] + .copy_from_slice(&first_posting_offset.to_le_bytes()); + std::fs::write(&core_path, core).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding_detail( + &report, + ScrubFindingType::SemanticMismatch, + "node label index posting range" + ), + "expected node label index posting range semantic mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_bounds_node_record_count_before_semantic_iteration() { + let (_dir, db_path, db) = populated_db(); + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let node_records_payload_start = + packed_component_payload_start(&manifest, SegmentComponentKind::NodeRecords); + let core_path = seg_dir.join(PACKED_CORE_FILENAME); + let mut core = std::fs::read(&core_path).unwrap(); + core[node_records_payload_start..node_records_payload_start + 8] + .copy_from_slice(&u64::MAX.to_le_bytes()); + std::fs::write(&core_path, core).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding_detail( + &report, + ScrubFindingType::SemanticMismatch, + "node records count" + ), + "expected bounded node record count semantic mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_bounds_node_metadata_count_before_semantic_iteration() { + let (_dir, db_path, db) = populated_db(); + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let node_metadata_payload_start = + packed_component_payload_start(&manifest, SegmentComponentKind::NodeMetadata); + let core_path = seg_dir.join(PACKED_CORE_FILENAME); + let mut core = std::fs::read(&core_path).unwrap(); + core[node_metadata_payload_start..node_metadata_payload_start + 8] + .copy_from_slice(&u64::MAX.to_le_bytes()); + std::fs::write(&core_path, core).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding_detail( + &report, + ScrubFindingType::SemanticMismatch, + "node metadata row count" + ), + "expected bounded node metadata count semantic mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_accepts_healthy_multi_label_declared_node_property_sidecars() { + let (_dir, _db_path, db, _eq_index_id, _range_index_id) = + create_declared_node_property_scrub_db(); + + let report = db.scrub().unwrap(); + assert!( + report.segments[0].findings.is_empty(), + "unexpected findings: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_does_not_require_missing_building_declared_node_property_sidecar() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + let mut props = BTreeMap::new(); + props.insert( + "status".to_string(), + PropValue::String("active".to_string()), + ); + db.upsert_node( + "Person", + "alice", + UpsertNodeOptions { + props, + ..Default::default() + }, + ) + .unwrap(); + db.flush().unwrap(); + + let (ready_rx, release_tx) = db.set_secondary_index_build_pause(); + let info = db + .ensure_node_property_index("Person", "status", SecondaryIndexKind::Equality) + .unwrap(); + assert_eq!(info.state, SecondaryIndexState::Building); + ready_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + report.segments[0].findings.is_empty(), + "unexpected findings for missing building sidecar: {:?}", + report.segments[0].findings + ); + + release_tx.send(()).unwrap(); + db.shutdown_secondary_index_worker(); + } + + #[test] + fn scrub_detects_declared_node_property_equality_sidecar_semantic_mismatch() { + let (_dir, db_path, db, eq_index_id, _range_index_id) = + create_declared_node_property_scrub_db(); + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let payload_offset = external_component_payload_offset( + &manifest, + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: eq_index_id, + }, + ); + let sidecar_path = seg_dir + .join("secondary_indexes") + .join(format!("node_prop_eq_{eq_index_id}.dat")); + let mut sidecar = std::fs::read(&sidecar_path).unwrap(); + let group_payload_offset = read_test_u64(&sidecar, payload_offset + 16) as usize; + sidecar[payload_offset + group_payload_offset..payload_offset + group_payload_offset + 8] + .copy_from_slice(&999_999u64.to_le_bytes()); + std::fs::write(&sidecar_path, sidecar).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding_detail( + &report, + ScrubFindingType::SemanticMismatch, + "declared node-property equality index" + ), + "expected declared equality sidecar semantic mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_detects_missing_declared_node_property_membership() { + let (_dir, db_path, db, eq_index_id, _range_index_id) = + create_declared_node_property_scrub_db(); + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let payload_offset = external_component_payload_offset( + &manifest, + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: eq_index_id, + }, + ); + let sidecar_path = seg_dir + .join("secondary_indexes") + .join(format!("node_prop_eq_{eq_index_id}.dat")); + let mut sidecar = std::fs::read(&sidecar_path).unwrap(); + let id_count_offset = payload_offset + 24; + let id_count = u32::from_le_bytes( + sidecar[id_count_offset..id_count_offset + 4] + .try_into() + .unwrap(), + ); + assert!( + id_count > 1, + "test precondition: expected at least two node IDs in the first equality group" + ); + sidecar[id_count_offset..id_count_offset + 4] + .copy_from_slice(&(id_count - 1).to_le_bytes()); + std::fs::write(&sidecar_path, sidecar).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding_detail( + &report, + ScrubFindingType::SemanticMismatch, + "declared node-property equality index" + ), + "expected declared equality sidecar missing-membership mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_detects_declared_node_property_range_sidecar_semantic_mismatch() { + let (_dir, db_path, db, _eq_index_id, range_index_id) = + create_declared_node_property_scrub_db(); + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let payload_offset = external_component_payload_offset( + &manifest, + SegmentComponentKind::NodePropertyRangeIndex { + index_id: range_index_id, + }, + ); + let sidecar_path = seg_dir + .join("secondary_indexes") + .join(format!("node_prop_range_{range_index_id}.dat")); + let mut sidecar = std::fs::read(&sidecar_path).unwrap(); + sidecar[payload_offset + 16..payload_offset + 24] + .copy_from_slice(&999_999u64.to_le_bytes()); + std::fs::write(&sidecar_path, sidecar).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding_detail( + &report, + ScrubFindingType::SemanticMismatch, + "declared node-property range index" + ), + "expected declared range sidecar semantic mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_detects_packed_container_identity_header_tamper() { + let (_dir, db_path, db) = populated_db(); + let core_path = first_seg_dir(&db_path).join(PACKED_CORE_FILENAME); + + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(&core_path) + .unwrap(); + file.seek(SeekFrom::Start(56)).unwrap(); + file.write_all(&[0xAA]).unwrap(); + file.sync_all().unwrap(); + + let report = db.scrub().unwrap(); + assert!( + any_finding(&report, ScrubFindingType::IdentityHeaderMismatch), + "expected container identity header mismatch, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_detects_packed_container_trailing_bytes() { + let (_dir, db_path, db) = populated_db(); + let core_path = first_seg_dir(&db_path).join(PACKED_CORE_FILENAME); + + let mut file = OpenOptions::new().append(true).open(&core_path).unwrap(); + file.write_all(&[0xAA]).unwrap(); + file.sync_all().unwrap(); + + let report = db.scrub().unwrap(); + assert!( + report.segments.iter().flat_map(|s| &s.findings).any(|f| { + f.component_kind == "PackedSegmentContainer" + && f.finding_type == ScrubFindingType::RangeOverflow + }), + "expected packed container range finding, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_marks_packed_ranges_failed_when_container_missing() { + let (_dir, db_path, db) = populated_db(); + let seg_dir = first_seg_dir(&db_path); + let manifest = read_segment_manifest(&seg_dir); + let packed_kinds: Vec = manifest + .components + .iter() + .filter(|record| matches!(record.handle, ComponentHandleV1::PackedRange { .. })) + .map(|record| format!("{:?}", record.kind)) + .collect(); + assert!( + !packed_kinds.is_empty(), + "test precondition: expected packed components" + ); + + std::fs::remove_file(seg_dir.join(PACKED_CORE_FILENAME)).unwrap(); + + let report = db.scrub().unwrap(); + assert!( + report.segments[0].findings.iter().any(|finding| { + finding.component_kind == "PackedSegmentContainer" + && finding.finding_type == ScrubFindingType::FileMissing + }), + "expected missing packed container finding, got: {:?}", + report.segments[0].findings + ); + for kind in packed_kinds { + assert!( + report + .segments + .iter() + .flat_map(|segment| &segment.findings) + .any(|finding| finding.component_kind == kind), + "expected packed range {kind} to be marked unverified, got: {:?}", + report.segments[0].findings + ); + } + } + + #[test] + fn scrub_does_not_treat_zero_length_packed_ranges_as_overlaps() { + let (_dir, db_path, db) = populated_db(); + let seg_dir = first_seg_dir(&db_path); + let mut manifest = read_segment_manifest(&seg_dir); + let (node_offset, _) = manifest + .components + .iter() + .find_map(|record| { + if record.kind == SegmentComponentKind::NodeRecords { + if let ComponentHandleV1::PackedRange { offset, len, .. } = &record.handle { + return Some((*offset, *len)); + } + } + None + }) + .expect("test precondition: NodeRecords should be packed"); + let segment_id = manifest.segment_id; + let empty_digest: [u8; 32] = Sha256::new().finalize().into(); + let record = manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::EdgeWeightIndex) + .expect("test precondition: expected packed edge weight index"); + let ComponentHandleV1::PackedRange { offset, len, .. } = &mut record.handle else { + panic!("EdgeWeightIndex should be packed"); + }; + *offset = node_offset; + *len = 0; + record.payload_len = 0; + record.payload_digest = Some(empty_digest); + record.component_id = component_id( + segment_id, + &record.kind, + record.logical_format_version, + record.payload_len, + record.payload_digest.as_ref(), + &record.dependency_digest, + record.build_fingerprint, + ); + write_segment_manifest(&seg_dir, &manifest); + + let report = db.scrub().unwrap(); + assert!( + !report.segments.iter().flat_map(|s| &s.findings).any(|f| { + f.component_kind == "EdgeWeightIndex" + && f.finding_type == ScrubFindingType::RangeOverlap + }), + "zero-length packed range should not overlap, got: {:?}", + report.segments[0].findings + ); + } + + #[test] + fn scrub_reports_root_local_segment_identity_mismatches() { + assert_scrub_reports_segment_identity_mismatch(|_, local| { + local.node_count += 1; + }); + assert_scrub_reports_segment_identity_mismatch(|_, local| { + local.segment_data_id = [9; 32]; + }); + assert_scrub_reports_segment_identity_mismatch(|root, _| { + root.segment_data_id = [7; 32]; + }); + assert_scrub_reports_segment_identity_mismatch(|_, local| { + local.segment_format_version += 1; + }); + assert_scrub_reports_segment_identity_mismatch(|_, local| { + local.segment_id += 100; + }); + } + + #[test] + fn scrub_summary_counts_failed_components_not_findings() { + let (_dir, db_path, db) = populated_db(); + let seg_dir = first_seg_dir(&db_path); + let mut manifest = read_segment_manifest(&seg_dir); + let record = first_external_non_container_record_mut(&mut manifest); + let digest = record + .payload_digest + .as_mut() + .expect("test precondition: sidecar should have a payload digest"); + digest[0] ^= 0x7F; + write_segment_manifest(&seg_dir, &manifest); + + let report = db.scrub().unwrap(); + assert!( + report.segments[0].findings.len() > 1, + "test precondition: expected multiple findings for one component, got: {:?}", + report.segments[0].findings + ); + assert_eq!(report.total_components_failed, 1); + } +} diff --git a/src/segment_components.rs b/src/segment_components.rs new file mode 100644 index 0000000..8c06753 --- /dev/null +++ b/src/segment_components.rs @@ -0,0 +1,3011 @@ +use crate::error::EngineError; +use crate::types::{ + SecondaryIndexKind, SecondaryIndexManifestEntry, SecondaryIndexRangeDomain, + SecondaryIndexTarget, +}; +use crc32fast::Hasher as Crc32Hasher; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::fs::File; +use std::io::{BufWriter, Seek, SeekFrom, Write}; +use std::path::{Component, Path}; + +pub(crate) type ComponentDigest32 = [u8; 32]; + +pub(crate) const SEGMENT_COMPONENT_MANIFEST_FILENAME: &str = "segment_manifest.dat"; +pub(crate) const SEGMENT_COMPONENT_MANIFEST_TMP_FILENAME: &str = "segment_manifest.tmp"; +pub(crate) const PACKED_CORE_FILENAME: &str = "segment.core"; +pub(crate) const PACKED_CORE_TMP_FILENAME: &str = "segment.core.tmp"; +pub(crate) const SEGMENT_COMPONENT_MANIFEST_MAGIC: [u8; 8] = *b"OGSID01\0"; +pub(crate) const SEGMENT_COMPONENT_MANIFEST_ENVELOPE_VERSION: u32 = 1; +pub(crate) const SEGMENT_COMPONENT_MANIFEST_PAYLOAD_VERSION: u32 = 1; +pub(crate) const SEGMENT_COMPONENT_MANIFEST_ENVELOPE_LEN: usize = 28; +pub(crate) const SEGMENT_COMPONENT_MANIFEST_MAX_PAYLOAD_LEN: usize = 16 * 1024 * 1024; + +pub(crate) const COMPONENT_IDENTITY_HEADER_MAGIC: [u8; 8] = *b"OGCID01\0"; +pub(crate) const COMPONENT_IDENTITY_HEADER_VERSION: u16 = 1; +pub(crate) const COMPONENT_IDENTITY_HEADER_LEN: usize = 192; +pub(crate) const ZERO_DIGEST: ComponentDigest32 = [0; 32]; + +const COMPONENT_IDENTITY_DOMAIN: &[u8] = b"overgraph.component.identity.v1"; +const DEPENDENCY_DIGEST_DOMAIN: &[u8] = b"overgraph.component.dependencies.v1"; +const SOURCE_GROUP_DIGEST_DOMAIN: &[u8] = b"overgraph.component.source_group.v1"; +const SEGMENT_DATA_DIGEST_DOMAIN: &[u8] = b"overgraph.segment_data.v1"; +const SEMANTIC_FINGERPRINT_DOMAIN: &[u8] = b"overgraph.semantic_fingerprint.v1"; +const SECONDARY_DECLARATION_FINGERPRINT_DOMAIN: &[u8] = b"overgraph.secondary_index_declaration.v1"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SegmentComponentBuildKind { + Flush, + Compaction, + OptionalRefresh, + TestFixture, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SegmentComponentManifestV1 { + pub format_version: u32, + pub segment_format_version: u32, + pub segment_id: u64, + pub generation: u64, + pub built_at_ms: i64, + pub build_kind: SegmentComponentBuildKind, + pub segment_data_id: ComponentDigest32, + pub node_count: u64, + pub edge_count: u64, + pub components: Vec, + pub unknown_optional_components: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SegmentComponentRecordV1 { + pub component_id: ComponentDigest32, + pub kind: SegmentComponentKind, + pub logical_format_version: u32, + pub created_generation: u64, + pub requirement: ComponentRequirement, + pub trust_class: ComponentTrustClass, + pub handle: ComponentHandleV1, + pub payload_len: u64, + pub payload_digest: Option, + pub dependency_digest: ComponentDigest32, + pub dependencies: Vec, + pub build_fingerprint: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct UnknownOptionalComponentRecordV1 { + pub wire: SegmentComponentRecordWireV1, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct SegmentComponentManifestWireV1 { + pub format_version: u32, + pub segment_format_version: u32, + pub segment_id: u64, + pub generation: u64, + pub built_at_ms: i64, + pub build_kind_tag: u8, + pub segment_data_id: ComponentDigest32, + pub node_count: u64, + pub edge_count: u64, + pub components: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct SegmentComponentRecordWireV1 { + pub component_id: ComponentDigest32, + pub kind_tag: u32, + pub index_id: Option, + pub logical_format_version: u32, + pub created_generation: u64, + pub requirement_tag: u8, + pub fallback_tag: u8, + pub trust_class_tag: u8, + pub handle: ComponentHandleWireV1, + pub payload_len: u64, + pub payload_digest: Option, + pub dependency_digest: ComponentDigest32, + pub dependencies: Vec, + pub build_fingerprint: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ComponentHandleWireV1 { + pub handle_tag: u8, + pub relative_path: Option, + pub payload_offset: u64, + pub payload_len: u64, + pub container_component_id: Option, + pub offset: u64, + pub len: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ComponentHandleV1 { + ExternalFile { + relative_path: String, + payload_offset: u64, + payload_len: u64, + }, + PackedRange { + container_component_id: ComponentDigest32, + offset: u64, + len: u64, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) enum SegmentComponentKind { + NodeRecords, + EdgeRecords, + NodeMetadata, + EdgeMetadata, + Tombstones, + KeyIndex, + NodeLabelIndex, + EdgeLabelIndex, + EdgeTripleIndex, + AdjOutIndex, + AdjOutPostings, + AdjInIndex, + AdjInPostings, + TimestampIndex, + LegacyNodePropertyIndex, + NodePropertyHashMetadata, + NodePropertyEqualityIndex { index_id: u64 }, + NodePropertyRangeIndex { index_id: u64 }, + EdgeWeightIndex, + EdgeUpdatedAtIndex, + EdgeValidFromIndex, + EdgeValidToIndex, + DegreeDelta, + PlannerStats, + NodeVectorMetadata, + NodeDenseVectorBlob, + NodeSparseVectorBlob, + DenseHnswMetadata, + DenseHnswGraph, + SparsePostingIndex, + SparsePostings, + EdgePropertyEqualityIndex { index_id: u64 }, + EdgePropertyRangeIndex { index_id: u64 }, + PackedSegmentContainer, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ComponentRequirement { + Required, + Optional { fallback: ComponentFallbackClass }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ComponentFallbackClass { + MetadataScan, + TypeScan, + AdjacencyWalk, + RecordScan, + ExactVectorScan, + PlannerStatsUnavailable, + FeatureUnavailable, + None, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ComponentTrustClass { + PrimaryData, + PrimaryMetadata, + CoreMaintainedIndex, + OptionalCandidateIndex, + OptionalExactAccelerator, + OptionalAdvisoryStats, + OptionalApproximateAccelerator, + AuxiliaryBlob, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ComponentAvailability { + Available, + Missing, + Incompatible { reason: String }, + CorruptIdentity { reason: String }, + Unsupported { reason: String }, +} + +impl ComponentAvailability { + pub(crate) fn is_available(&self) -> bool { + matches!(self, ComponentAvailability::Available) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SegmentComponentSourceGroups { + pub node_source: ComponentDigest32, + pub edge_source: ComponentDigest32, + pub node_property_content_source: ComponentDigest32, + pub node_property_hash_source: ComponentDigest32, + pub edge_metadata_source: ComponentDigest32, + pub degree_source: ComponentDigest32, + pub dense_vector_source: ComponentDigest32, + pub sparse_vector_source: ComponentDigest32, + pub segment_data_id: ComponentDigest32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ComponentDependencyV1 { + SourceComponent { + kind: SegmentComponentKind, + component_id: ComponentDigest32, + }, + SourceGroup { + group: SegmentSourceGroupKind, + group_id: ComponentDigest32, + }, + SecondaryIndexDeclaration { + index_id: u64, + target_kind: SecondaryIndexTargetKindForComponents, + kind: SecondaryIndexKindFingerprint, + fingerprint: u64, + }, + DenseVectorConfig { + fingerprint: u64, + }, + SparseVectorConfig { + fingerprint: u64, + }, + WriterBuildParams { + fingerprint: u64, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ComponentDependencyWireV1 { + pub dependency_tag: u8, + pub component_kind_tag: Option, + pub component_index_id: Option, + pub component_id: Option, + pub group_tag: Option, + pub group_id: Option, + pub index_id: Option, + pub target_kind_tag: Option, + pub secondary_index_kind_tag: Option, + pub fingerprint: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum SegmentSourceGroupKind { + NodeSource, + EdgeSource, + NodePropertyContentSource, + NodePropertyHashSource, + EdgeMetadataSource, + DegreeSource, + DenseVectorSource, + SparseVectorSource, + SegmentData, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum SecondaryIndexTargetKindForComponents { + Node, + Edge, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum SecondaryIndexKindFingerprint { + Equality, + Range, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ComponentIdentityHeaderV1 { + pub segment_format_version: u32, + pub segment_id: u64, + pub component_kind: SegmentComponentKind, + pub logical_format_version: u32, + pub created_generation: u64, + pub payload_offset: u64, + pub payload_len: u64, + pub component_id: ComponentDigest32, + pub dependency_digest: ComponentDigest32, + pub build_fingerprint: u64, + pub payload_digest: Option, +} + +pub(crate) struct ComponentIdentityWriter { + segment_format_version: u32, + segment_id: u64, + kind: SegmentComponentKind, + logical_format_version: u32, + created_generation: u64, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + relative_path: String, + build_fingerprint: u64, + writer: BufWriter, + payload_digest: Sha256, + payload_len: u64, + writes_identity_header: bool, +} + +impl ComponentIdentityWriter { + #[allow(clippy::too_many_arguments)] + pub(crate) fn create( + path: &Path, + relative_path: String, + segment_format_version: u32, + segment_id: u64, + kind: SegmentComponentKind, + logical_format_version: u32, + created_generation: u64, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + build_fingerprint: u64, + writes_identity_header: bool, + ) -> Result { + validate_relative_component_path(&relative_path)?; + let mut writer = BufWriter::new(File::create(path)?); + if writes_identity_header { + writer.write_all(&[0; COMPONENT_IDENTITY_HEADER_LEN])?; + } + Ok(Self { + segment_format_version, + segment_id, + kind, + logical_format_version, + created_generation, + requirement, + trust_class, + relative_path, + build_fingerprint, + writer, + payload_digest: Sha256::new(), + payload_len: 0, + writes_identity_header, + }) + } + + pub(crate) fn finish( + mut self, + dependencies: Vec, + ) -> Result { + self.writer.flush()?; + let payload_digest: ComponentDigest32 = self.payload_digest.finalize().into(); + let dependency_digest = dependency_digest(&dependencies); + let payload_offset = if self.writes_identity_header { + COMPONENT_IDENTITY_HEADER_LEN as u64 + } else { + 0 + }; + let computed_component_id = component_id( + self.segment_id, + &self.kind, + self.logical_format_version, + self.payload_len, + Some(&payload_digest), + &dependency_digest, + self.build_fingerprint, + ); + if self.writes_identity_header { + let header = ComponentIdentityHeaderV1 { + segment_format_version: self.segment_format_version, + segment_id: self.segment_id, + component_kind: self.kind.clone(), + logical_format_version: self.logical_format_version, + created_generation: self.created_generation, + payload_offset, + payload_len: self.payload_len, + component_id: computed_component_id, + dependency_digest, + build_fingerprint: self.build_fingerprint, + payload_digest: Some(payload_digest), + }; + self.writer.seek(SeekFrom::Start(0))?; + self.writer.write_all(&encode_identity_header(&header))?; + self.writer.flush()?; + } + self.writer.get_ref().sync_all()?; + Ok(SegmentComponentRecordV1 { + component_id: computed_component_id, + kind: self.kind, + logical_format_version: self.logical_format_version, + created_generation: self.created_generation, + requirement: self.requirement, + trust_class: self.trust_class, + handle: ComponentHandleV1::ExternalFile { + relative_path: self.relative_path, + payload_offset, + payload_len: self.payload_len, + }, + payload_len: self.payload_len, + payload_digest: Some(payload_digest), + dependency_digest, + dependencies, + build_fingerprint: self.build_fingerprint, + }) + } +} + +impl Write for ComponentIdentityWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.writer.write(buf)?; + self.payload_digest.update(&buf[..written]); + self.payload_len += written as u64; + Ok(written) + } + + fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> { + self.writer.write_all(buf)?; + self.payload_digest.update(buf); + self.payload_len += buf.len() as u64; + Ok(()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() + } +} + +pub(crate) fn encode_manifest_envelope( + manifest: &SegmentComponentManifestV1, +) -> Result, EngineError> { + validate_manifest(manifest)?; + let wire = manifest_to_wire(manifest); + let payload = rmp_serde::to_vec(&wire) + .map_err(|error| EngineError::SerializationError(error.to_string()))?; + if payload.len() > SEGMENT_COMPONENT_MANIFEST_MAX_PAYLOAD_LEN { + return Err(component_manifest_error( + "segment component manifest exceeds hard cap", + )); + } + let mut crc = Crc32Hasher::new(); + crc.update(&payload); + let checksum = crc.finalize(); + let mut data = Vec::with_capacity(SEGMENT_COMPONENT_MANIFEST_ENVELOPE_LEN + payload.len()); + data.extend_from_slice(&SEGMENT_COMPONENT_MANIFEST_MAGIC); + data.extend_from_slice(&SEGMENT_COMPONENT_MANIFEST_ENVELOPE_VERSION.to_le_bytes()); + data.extend_from_slice(&manifest.segment_format_version.to_le_bytes()); + data.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + data.extend_from_slice(&checksum.to_le_bytes()); + data.extend_from_slice(&payload); + Ok(data) +} + +pub(crate) fn decode_manifest_envelope( + data: &[u8], +) -> Result { + if data.len() < SEGMENT_COMPONENT_MANIFEST_ENVELOPE_LEN { + return Err(component_manifest_error( + "segment component manifest is shorter than envelope", + )); + } + if data[0..8] != SEGMENT_COMPONENT_MANIFEST_MAGIC { + return Err(component_manifest_error( + "segment component manifest has bad magic", + )); + } + let envelope_version = u32::from_le_bytes(data[8..12].try_into().unwrap()); + if envelope_version != SEGMENT_COMPONENT_MANIFEST_ENVELOPE_VERSION { + return Err(component_manifest_error(format!( + "unsupported segment component manifest envelope version {}", + envelope_version + ))); + } + let segment_format_version = u32::from_le_bytes(data[12..16].try_into().unwrap()); + let payload_len = u64::from_le_bytes(data[16..24].try_into().unwrap()) as usize; + if payload_len > SEGMENT_COMPONENT_MANIFEST_MAX_PAYLOAD_LEN { + return Err(component_manifest_error( + "segment component manifest exceeds hard cap", + )); + } + let expected_payload_len = data.len() - SEGMENT_COMPONENT_MANIFEST_ENVELOPE_LEN; + if payload_len != expected_payload_len { + return Err(component_manifest_error(format!( + "segment component manifest payload length mismatch: header={}, actual={}", + payload_len, expected_payload_len + ))); + } + let expected_crc = u32::from_le_bytes(data[24..28].try_into().unwrap()); + let payload = &data[SEGMENT_COMPONENT_MANIFEST_ENVELOPE_LEN..]; + let mut crc = Crc32Hasher::new(); + crc.update(payload); + if crc.finalize() != expected_crc { + return Err(component_manifest_error( + "segment component manifest payload crc mismatch", + )); + } + let wire: SegmentComponentManifestWireV1 = rmp_serde::from_slice(payload) + .map_err(|error| EngineError::SerializationError(error.to_string()))?; + let mut manifest = manifest_from_wire(wire)?; + manifest.segment_format_version = segment_format_version; + validate_manifest(&manifest)?; + Ok(manifest) +} + +pub(crate) fn encode_identity_header( + header: &ComponentIdentityHeaderV1, +) -> [u8; COMPONENT_IDENTITY_HEADER_LEN] { + let mut data = [0u8; COMPONENT_IDENTITY_HEADER_LEN]; + data[0..8].copy_from_slice(&COMPONENT_IDENTITY_HEADER_MAGIC); + data[8..10].copy_from_slice(&COMPONENT_IDENTITY_HEADER_VERSION.to_le_bytes()); + data[10..12].copy_from_slice(&0u16.to_le_bytes()); + data[12..16].copy_from_slice(&header.segment_format_version.to_le_bytes()); + data[16..24].copy_from_slice(&header.segment_id.to_le_bytes()); + data[24..28].copy_from_slice(&header.component_kind.kind_tag().to_le_bytes()); + data[28..32].copy_from_slice(&header.logical_format_version.to_le_bytes()); + data[32..40].copy_from_slice(&header.created_generation.to_le_bytes()); + data[40..48].copy_from_slice(&header.payload_offset.to_le_bytes()); + data[48..56].copy_from_slice(&header.payload_len.to_le_bytes()); + data[56..88].copy_from_slice(&header.component_id); + data[88..120].copy_from_slice(&header.dependency_digest); + data[120..128].copy_from_slice(&header.build_fingerprint.to_le_bytes()); + data[128] = u8::from(header.payload_digest.is_some()); + data[129] = u8::from(header.component_kind.index_id().is_some()); + if let Some(payload_digest) = header.payload_digest { + data[136..168].copy_from_slice(&payload_digest); + } + data[168..176].copy_from_slice(&header.component_kind.index_id().unwrap_or(0).to_le_bytes()); + data +} + +pub(crate) fn decode_identity_header( + data: &[u8], +) -> Result { + if data.len() < COMPONENT_IDENTITY_HEADER_LEN { + return Err(component_manifest_error( + "component identity header is too short", + )); + } + if data[0..8] != COMPONENT_IDENTITY_HEADER_MAGIC { + return Err(component_manifest_error( + "component identity header has bad magic", + )); + } + let header_version = u16::from_le_bytes(data[8..10].try_into().unwrap()); + if header_version != COMPONENT_IDENTITY_HEADER_VERSION { + return Err(component_manifest_error(format!( + "unsupported component identity header version {}", + header_version + ))); + } + if u16::from_le_bytes(data[10..12].try_into().unwrap()) != 0 { + return Err(component_manifest_error( + "component identity header reserved field is nonzero", + )); + } + if data[130..136] != [0; 6] + || data[176..COMPONENT_IDENTITY_HEADER_LEN] + .iter() + .any(|b| *b != 0) + { + return Err(component_manifest_error( + "component identity header reserved bytes are nonzero", + )); + } + let kind_tag = u32::from_le_bytes(data[24..28].try_into().unwrap()); + let payload_digest = match data[128] { + 0 => { + if data[136..168].iter().any(|b| *b != 0) { + return Err(component_manifest_error( + "component identity header has digest bytes without payload digest flag", + )); + } + None + } + 1 => Some(data[136..168].try_into().unwrap()), + _ => { + return Err(component_manifest_error( + "component identity header has invalid payload digest flag", + )); + } + }; + let index_id_value = u64::from_le_bytes(data[168..176].try_into().unwrap()); + let index_id = match data[129] { + 0 if index_id_value == 0 => None, + 0 => { + return Err(component_manifest_error( + "component identity header has index_id bytes without index flag", + )); + } + 1 => Some(index_id_value), + _ => { + return Err(component_manifest_error( + "component identity header has invalid index_id flag", + )); + } + }; + let component_kind = + SegmentComponentKind::from_tag_and_index(kind_tag, index_id)?.ok_or_else(|| { + component_manifest_error("component identity header has unknown kind tag") + })?; + Ok(ComponentIdentityHeaderV1 { + segment_format_version: u32::from_le_bytes(data[12..16].try_into().unwrap()), + segment_id: u64::from_le_bytes(data[16..24].try_into().unwrap()), + component_kind, + logical_format_version: u32::from_le_bytes(data[28..32].try_into().unwrap()), + created_generation: u64::from_le_bytes(data[32..40].try_into().unwrap()), + payload_offset: u64::from_le_bytes(data[40..48].try_into().unwrap()), + payload_len: u64::from_le_bytes(data[48..56].try_into().unwrap()), + component_id: data[56..88].try_into().unwrap(), + dependency_digest: data[88..120].try_into().unwrap(), + build_fingerprint: u64::from_le_bytes(data[120..128].try_into().unwrap()), + payload_digest, + }) +} + +pub(crate) fn validate_relative_component_path(relative_path: &str) -> Result<(), EngineError> { + if relative_path.is_empty() { + return Err(component_manifest_error("component relative path is empty")); + } + if relative_path.ends_with('/') || relative_path.ends_with('\\') { + return Err(component_manifest_error( + "component relative path must not end with a separator", + )); + } + if relative_path.contains('\\') { + return Err(component_manifest_error( + "component relative path must not contain backslashes", + )); + } + if relative_path.chars().any(char::is_control) { + return Err(component_manifest_error( + "component relative path must not contain control characters", + )); + } + let path = Path::new(relative_path); + if path.is_absolute() { + return Err(component_manifest_error( + "component relative path must not be absolute", + )); + } + let mut saw_normal = false; + for component in path.components() { + match component { + Component::Normal(part) if !part.is_empty() => saw_normal = true, + Component::CurDir | Component::ParentDir => { + return Err(component_manifest_error( + "component relative path must be normalized", + )); + } + _ => { + return Err(component_manifest_error( + "component relative path contains an invalid component", + )); + } + } + } + if !saw_normal { + return Err(component_manifest_error( + "component relative path must include a file name", + )); + } + Ok(()) +} + +pub(crate) fn component_id( + segment_id: u64, + kind: &SegmentComponentKind, + logical_format_version: u32, + payload_len: u64, + payload_digest: Option<&ComponentDigest32>, + dependency_digest: &ComponentDigest32, + build_fingerprint: u64, +) -> ComponentDigest32 { + let mut hasher = Sha256::new(); + hasher.update(COMPONENT_IDENTITY_DOMAIN); + put_u64(&mut hasher, segment_id); + put_u32(&mut hasher, kind.kind_tag()); + put_u64(&mut hasher, kind.index_id().unwrap_or(0)); + put_u32(&mut hasher, logical_format_version); + put_u64(&mut hasher, payload_len); + hasher.update(payload_digest.unwrap_or(&ZERO_DIGEST)); + hasher.update(dependency_digest); + put_u64(&mut hasher, build_fingerprint); + hasher.finalize().into() +} + +pub(crate) fn dependency_digest(dependencies: &[ComponentDependencyV1]) -> ComponentDigest32 { + let mut wires: Vec = + dependencies.iter().map(dependency_to_wire).collect(); + dependency_digest_from_wire(&mut wires).expect("dependency_to_wire produces valid dependencies") +} + +fn dependency_digest_from_wire( + dependencies: &mut [ComponentDependencyWireV1], +) -> Result { + for dependency in dependencies.iter() { + dependency_from_wire(dependency)?; + } + let mut canonical: Vec> = dependencies + .iter() + .map(dependency_canonical_bytes) + .collect(); + canonical.sort(); + let mut hasher = Sha256::new(); + hasher.update(DEPENDENCY_DIGEST_DOMAIN); + for dependency in canonical { + hasher.update(dependency); + } + Ok(hasher.finalize().into()) +} + +pub(crate) fn digest_source_group( + group: SegmentSourceGroupKind, + digests: &[ComponentDigest32], +) -> ComponentDigest32 { + let mut hasher = Sha256::new(); + hasher.update(SOURCE_GROUP_DIGEST_DOMAIN); + put_u8(&mut hasher, group.tag()); + for digest in digests { + hasher.update(digest); + } + hasher.finalize().into() +} + +pub(crate) fn digest_segment_data( + segment_id: u64, + node_count: u64, + edge_count: u64, + node_source: &ComponentDigest32, + edge_source: &ComponentDigest32, + dense_vector_source: &ComponentDigest32, + sparse_vector_source: &ComponentDigest32, +) -> ComponentDigest32 { + let mut hasher = Sha256::new(); + hasher.update(SEGMENT_DATA_DIGEST_DOMAIN); + put_u64(&mut hasher, segment_id); + put_u64(&mut hasher, node_count); + put_u64(&mut hasher, edge_count); + hasher.update(node_source); + hasher.update(edge_source); + hasher.update(dense_vector_source); + hasher.update(sparse_vector_source); + hasher.finalize().into() +} + +pub(crate) fn component_semantic_fingerprint(namespace: &str, fields: &[u64]) -> u64 { + let mut hasher = Sha256::new(); + hasher.update(SEMANTIC_FINGERPRINT_DOMAIN); + put_bytes_with_len(&mut hasher, namespace.as_bytes()); + for field in fields { + put_u64(&mut hasher, *field); + } + fingerprint_from_digest(hasher.finalize().into()) +} + +pub(crate) fn component_build_fingerprint( + segment_format_version: u32, + namespace: &str, + fields: &[u64], +) -> u64 { + let mut all_fields = Vec::with_capacity(fields.len() + 1); + all_fields.push(segment_format_version as u64); + all_fields.extend_from_slice(fields); + component_semantic_fingerprint(namespace, &all_fields) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn secondary_index_declaration_fingerprint( + index_id: u64, + target_kind: SecondaryIndexTargetKindForComponents, + target_label_id: u32, + property_key: &[u8], + index_kind: SecondaryIndexKindFingerprint, + range_domain: u64, + declaration_generation: u64, + value_encoding_version: u64, +) -> u64 { + let mut hasher = Sha256::new(); + hasher.update(SECONDARY_DECLARATION_FINGERPRINT_DOMAIN); + put_u64(&mut hasher, index_id); + put_u8(&mut hasher, target_kind.tag()); + put_u32(&mut hasher, target_label_id); + put_bytes_with_len(&mut hasher, property_key); + put_u8(&mut hasher, index_kind.tag()); + put_u64(&mut hasher, range_domain); + put_u64(&mut hasher, declaration_generation); + put_u64(&mut hasher, value_encoding_version); + fingerprint_from_digest(hasher.finalize().into()) +} + +pub(crate) fn secondary_declaration_dependency( + entry: &SecondaryIndexManifestEntry, +) -> ComponentDependencyV1 { + let (target_kind, target_label_id, prop_key) = match &entry.target { + SecondaryIndexTarget::NodeProperty { label_id, prop_key } => ( + SecondaryIndexTargetKindForComponents::Node, + *label_id, + prop_key, + ), + SecondaryIndexTarget::EdgeProperty { label_id, prop_key } => ( + SecondaryIndexTargetKindForComponents::Edge, + *label_id, + prop_key, + ), + }; + let (kind, range_domain) = match entry.kind { + SecondaryIndexKind::Equality => (SecondaryIndexKindFingerprint::Equality, 0), + SecondaryIndexKind::Range { domain } => { + let domain_tag = match domain { + SecondaryIndexRangeDomain::Int => 1, + SecondaryIndexRangeDomain::UInt => 2, + SecondaryIndexRangeDomain::Float => 3, + }; + (SecondaryIndexKindFingerprint::Range, domain_tag) + } + }; + let fingerprint = secondary_index_declaration_fingerprint( + entry.index_id, + target_kind, + target_label_id, + prop_key.as_bytes(), + kind, + range_domain, + 1, + 1, + ); + ComponentDependencyV1::SecondaryIndexDeclaration { + index_id: entry.index_id, + target_kind, + kind, + fingerprint, + } +} + +pub(crate) fn source_group_dependency( + group: SegmentSourceGroupKind, + group_id: ComponentDigest32, +) -> ComponentDependencyV1 { + ComponentDependencyV1::SourceGroup { group, group_id } +} + +pub(crate) fn source_component_dependency( + record: &SegmentComponentRecordV1, +) -> ComponentDependencyV1 { + ComponentDependencyV1::SourceComponent { + kind: record.kind.clone(), + component_id: record.component_id, + } +} + +pub(crate) fn segment_source_groups_from_records( + segment_id: u64, + node_count: u64, + edge_count: u64, + records: &[SegmentComponentRecordV1], +) -> Result { + let node_records = required_component_id(records, &SegmentComponentKind::NodeRecords)?; + let edge_records = required_component_id(records, &SegmentComponentKind::EdgeRecords)?; + let node_meta = required_component_id(records, &SegmentComponentKind::NodeMetadata)?; + let edge_meta = required_component_id(records, &SegmentComponentKind::EdgeMetadata)?; + let tombstones = required_component_id(records, &SegmentComponentKind::Tombstones)?; + let node_vector_meta = + optional_component_id(records, &SegmentComponentKind::NodeVectorMetadata); + let node_dense_vectors = + optional_component_id(records, &SegmentComponentKind::NodeDenseVectorBlob); + let node_sparse_vectors = + optional_component_id(records, &SegmentComponentKind::NodeSparseVectorBlob); + let node_prop_hashes = + optional_component_id(records, &SegmentComponentKind::NodePropertyHashMetadata); + + let node_source = digest_source_group( + SegmentSourceGroupKind::NodeSource, + &[node_records, node_meta, tombstones], + ); + let edge_source = digest_source_group( + SegmentSourceGroupKind::EdgeSource, + &[edge_records, edge_meta, tombstones], + ); + let node_property_content_source = digest_source_group( + SegmentSourceGroupKind::NodePropertyContentSource, + &[node_source], + ); + let node_property_hash_source = digest_source_group( + SegmentSourceGroupKind::NodePropertyHashSource, + &[node_source, node_prop_hashes], + ); + let edge_metadata_source = digest_source_group( + SegmentSourceGroupKind::EdgeMetadataSource, + &[edge_meta, tombstones], + ); + let degree_source = digest_source_group(SegmentSourceGroupKind::DegreeSource, &[edge_source]); + let dense_vector_source = digest_source_group( + SegmentSourceGroupKind::DenseVectorSource, + &[node_source, node_vector_meta, node_dense_vectors], + ); + let sparse_vector_source = digest_source_group( + SegmentSourceGroupKind::SparseVectorSource, + &[node_source, node_vector_meta, node_sparse_vectors], + ); + let segment_data_id = digest_segment_data( + segment_id, + node_count, + edge_count, + &node_source, + &edge_source, + &dense_vector_source, + &sparse_vector_source, + ); + + Ok(SegmentComponentSourceGroups { + node_source, + edge_source, + node_property_content_source, + node_property_hash_source, + edge_metadata_source, + degree_source, + dense_vector_source, + sparse_vector_source, + segment_data_id, + }) +} + +pub(crate) fn is_packed_core_component_kind(kind: &SegmentComponentKind) -> bool { + matches!( + kind, + SegmentComponentKind::NodeRecords + | SegmentComponentKind::EdgeRecords + | SegmentComponentKind::NodeMetadata + | SegmentComponentKind::EdgeMetadata + | SegmentComponentKind::Tombstones + | SegmentComponentKind::KeyIndex + | SegmentComponentKind::NodeLabelIndex + | SegmentComponentKind::EdgeLabelIndex + | SegmentComponentKind::EdgeTripleIndex + | SegmentComponentKind::AdjOutIndex + | SegmentComponentKind::AdjOutPostings + | SegmentComponentKind::AdjInIndex + | SegmentComponentKind::AdjInPostings + | SegmentComponentKind::TimestampIndex + | SegmentComponentKind::NodeVectorMetadata + | SegmentComponentKind::NodeDenseVectorBlob + | SegmentComponentKind::NodeSparseVectorBlob + | SegmentComponentKind::EdgeWeightIndex + | SegmentComponentKind::EdgeUpdatedAtIndex + | SegmentComponentKind::EdgeValidFromIndex + | SegmentComponentKind::EdgeValidToIndex + ) +} + +pub(crate) fn is_refreshable_external_component_kind(kind: &SegmentComponentKind) -> bool { + matches!( + kind, + SegmentComponentKind::LegacyNodePropertyIndex + | SegmentComponentKind::NodePropertyHashMetadata + | SegmentComponentKind::NodePropertyEqualityIndex { .. } + | SegmentComponentKind::NodePropertyRangeIndex { .. } + | SegmentComponentKind::DegreeDelta + | SegmentComponentKind::PlannerStats + | SegmentComponentKind::DenseHnswMetadata + | SegmentComponentKind::DenseHnswGraph + | SegmentComponentKind::SparsePostingIndex + | SegmentComponentKind::SparsePostings + | SegmentComponentKind::EdgePropertyEqualityIndex { .. } + | SegmentComponentKind::EdgePropertyRangeIndex { .. } + ) +} + +pub(crate) fn is_container_component_kind(kind: &SegmentComponentKind) -> bool { + matches!(kind, SegmentComponentKind::PackedSegmentContainer) +} + +pub(crate) fn packed_core_container_record( + manifest: &SegmentComponentManifestV1, +) -> Result<&SegmentComponentRecordV1, EngineError> { + manifest + .components + .iter() + .find(|record| is_container_component_kind(&record.kind)) + .ok_or_else(|| { + component_manifest_error("packed core manifest missing segment.core container") + }) +} + +pub(crate) fn patch_packed_range_container_id( + records: &mut [SegmentComponentRecordV1], + container_component_id: ComponentDigest32, +) { + for record in records { + if let ComponentHandleV1::PackedRange { + container_component_id: handle_container_id, + .. + } = &mut record.handle + { + *handle_container_id = container_component_id; + } + } +} + +#[cfg(test)] +pub(crate) fn validate_packed_core_manifest_contract( + manifest: &SegmentComponentManifestV1, +) -> Result<(), EngineError> { + validate_packed_core_records_contract_impl( + &manifest.components, + PackedRangeOverlapPolicy::AllValidRanges, + ) +} + +pub(crate) fn validate_packed_core_manifest_contract_for_open( + manifest: &SegmentComponentManifestV1, +) -> Result<(), EngineError> { + validate_packed_core_records_contract_impl( + &manifest.components, + PackedRangeOverlapPolicy::RequiredRangesOnly, + ) +} + +pub(crate) fn validate_packed_core_records_contract( + records: &[SegmentComponentRecordV1], +) -> Result<(), EngineError> { + validate_packed_core_records_contract_impl(records, PackedRangeOverlapPolicy::AllValidRanges) +} + +#[derive(Clone, Copy)] +enum PackedRangeOverlapPolicy { + AllValidRanges, + RequiredRangesOnly, +} + +fn validate_packed_core_records_contract_impl( + records: &[SegmentComponentRecordV1], + overlap_policy: PackedRangeOverlapPolicy, +) -> Result<(), EngineError> { + for record in records { + if is_packed_core_component_kind(&record.kind) + && matches!(record.handle, ComponentHandleV1::ExternalFile { .. }) + { + return Err(component_manifest_error(format!( + "packed core component {:?} must use a PackedRange handle", + record.kind + ))); + } + } + + let packed_records: Vec<&SegmentComponentRecordV1> = records + .iter() + .filter(|record| matches!(record.handle, ComponentHandleV1::PackedRange { .. })) + .collect(); + let Some(container_record) = records + .iter() + .find(|record| record.kind == SegmentComponentKind::PackedSegmentContainer) + else { + if !packed_records + .iter() + .any(|record| record.requirement == ComponentRequirement::Required) + { + return Ok(()); + } + return Err(component_manifest_error( + "packed core manifest has packed ranges without segment.core container", + )); + }; + + if container_record.requirement != ComponentRequirement::Required { + return Err(component_manifest_error( + "packed core container must be required", + )); + } + if container_record.trust_class != ComponentTrustClass::AuxiliaryBlob { + return Err(component_manifest_error( + "packed core container must use AuxiliaryBlob trust class", + )); + } + let ComponentHandleV1::ExternalFile { + relative_path, + payload_offset, + payload_len: container_payload_len, + } = &container_record.handle + else { + return Err(component_manifest_error( + "packed core container must use an external segment.core handle", + )); + }; + if relative_path != PACKED_CORE_FILENAME { + return Err(component_manifest_error(format!( + "packed core container path must be {PACKED_CORE_FILENAME}" + ))); + } + if *payload_offset != COMPONENT_IDENTITY_HEADER_LEN as u64 { + return Err(component_manifest_error( + "packed core container payload offset must follow identity header", + )); + } + + let mut ranges: Vec<(u64, u64, u32, Option)> = Vec::new(); + for record in packed_records { + if record.kind == SegmentComponentKind::PackedSegmentContainer { + return Err(component_manifest_error( + "packed core container cannot be a packed range", + )); + } + if !is_packed_core_component_kind(&record.kind) { + if record.requirement == ComponentRequirement::Required { + return Err(component_manifest_error(format!( + "component {:?} is not allowed in segment.core", + record.kind + ))); + } + continue; + } + let ComponentHandleV1::PackedRange { + container_component_id, + offset, + len, + } = &record.handle + else { + unreachable!("packed_records filtered by handle") + }; + if record.payload_len != *len { + if record.requirement == ComponentRequirement::Required { + return Err(component_manifest_error(format!( + "packed component {:?} payload length does not match packed range", + record.kind + ))); + } + continue; + } + if *container_component_id != container_record.component_id { + if record.requirement == ComponentRequirement::Required { + return Err(component_manifest_error(format!( + "required packed component {:?} points at the wrong container", + record.kind + ))); + } + continue; + } + let Some(end) = offset.checked_add(*len) else { + if record.requirement == ComponentRequirement::Required { + return Err(component_manifest_error(format!( + "packed component {:?} range overflows", + record.kind + ))); + } + continue; + }; + if end > *container_payload_len { + if record.requirement == ComponentRequirement::Required { + return Err(component_manifest_error(format!( + "packed component {:?} range [{}, {}) exceeds segment.core payload length {}", + record.kind, offset, end, container_payload_len + ))); + } + continue; + } + let include_in_overlap_check = match overlap_policy { + PackedRangeOverlapPolicy::AllValidRanges => true, + PackedRangeOverlapPolicy::RequiredRangesOnly => { + record.requirement == ComponentRequirement::Required + } + }; + if *len > 0 && include_in_overlap_check { + ranges.push((*offset, end, record.kind.kind_tag(), record.kind.index_id())); + } + } + + ranges.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| a.1.cmp(&b.1)) + .then_with(|| a.2.cmp(&b.2)) + .then_with(|| a.3.cmp(&b.3)) + }); + for pair in ranges.windows(2) { + let previous = pair[0]; + let current = pair[1]; + if current.0 < previous.1 { + return Err(component_manifest_error(format!( + "packed component ranges overlap: previous=[{}, {}), current=[{}, {})", + previous.0, previous.1, current.0, current.1 + ))); + } + } + + Ok(()) +} + +fn required_component_id( + records: &[SegmentComponentRecordV1], + kind: &SegmentComponentKind, +) -> Result { + records + .iter() + .find(|record| &record.kind == kind) + .map(|record| record.component_id) + .ok_or_else(|| component_manifest_error(format!("missing source component {:?}", kind))) +} + +fn optional_component_id( + records: &[SegmentComponentRecordV1], + kind: &SegmentComponentKind, +) -> ComponentDigest32 { + records + .iter() + .find(|record| &record.kind == kind) + .map(|record| record.component_id) + .unwrap_or(ZERO_DIGEST) +} + +fn fingerprint_from_digest(digest: ComponentDigest32) -> u64 { + let mut value = u64::from_le_bytes(digest[0..8].try_into().unwrap()); + if value == 0 { + value = 1; + } + value +} + +impl SegmentComponentKind { + pub(crate) fn kind_tag(&self) -> u32 { + match self { + SegmentComponentKind::NodeRecords => 1, + SegmentComponentKind::EdgeRecords => 2, + SegmentComponentKind::NodeMetadata => 3, + SegmentComponentKind::EdgeMetadata => 4, + SegmentComponentKind::Tombstones => 5, + SegmentComponentKind::KeyIndex => 6, + SegmentComponentKind::NodeLabelIndex => 7, + SegmentComponentKind::EdgeLabelIndex => 8, + SegmentComponentKind::EdgeTripleIndex => 9, + SegmentComponentKind::AdjOutIndex => 10, + SegmentComponentKind::AdjOutPostings => 11, + SegmentComponentKind::AdjInIndex => 12, + SegmentComponentKind::AdjInPostings => 13, + SegmentComponentKind::TimestampIndex => 14, + SegmentComponentKind::LegacyNodePropertyIndex => 15, + SegmentComponentKind::NodePropertyHashMetadata => 16, + SegmentComponentKind::NodePropertyEqualityIndex { .. } => 17, + SegmentComponentKind::NodePropertyRangeIndex { .. } => 18, + SegmentComponentKind::EdgeWeightIndex => 19, + SegmentComponentKind::EdgeUpdatedAtIndex => 20, + SegmentComponentKind::EdgeValidFromIndex => 21, + SegmentComponentKind::EdgeValidToIndex => 22, + SegmentComponentKind::DegreeDelta => 23, + SegmentComponentKind::PlannerStats => 24, + SegmentComponentKind::NodeVectorMetadata => 25, + SegmentComponentKind::NodeDenseVectorBlob => 26, + SegmentComponentKind::NodeSparseVectorBlob => 27, + SegmentComponentKind::DenseHnswMetadata => 28, + SegmentComponentKind::DenseHnswGraph => 29, + SegmentComponentKind::SparsePostingIndex => 30, + SegmentComponentKind::SparsePostings => 31, + SegmentComponentKind::EdgePropertyEqualityIndex { .. } => 32, + SegmentComponentKind::EdgePropertyRangeIndex { .. } => 33, + SegmentComponentKind::PackedSegmentContainer => 34, + } + } + + pub(crate) fn index_id(&self) -> Option { + match self { + SegmentComponentKind::NodePropertyEqualityIndex { index_id } + | SegmentComponentKind::NodePropertyRangeIndex { index_id } + | SegmentComponentKind::EdgePropertyEqualityIndex { index_id } + | SegmentComponentKind::EdgePropertyRangeIndex { index_id } => Some(*index_id), + _ => None, + } + } + + fn from_tag_and_index( + kind_tag: u32, + index_id: Option, + ) -> Result, EngineError> { + let known = match kind_tag { + 1 => require_no_index(kind_tag, index_id, SegmentComponentKind::NodeRecords)?, + 2 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeRecords)?, + 3 => require_no_index(kind_tag, index_id, SegmentComponentKind::NodeMetadata)?, + 4 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeMetadata)?, + 5 => require_no_index(kind_tag, index_id, SegmentComponentKind::Tombstones)?, + 6 => require_no_index(kind_tag, index_id, SegmentComponentKind::KeyIndex)?, + 7 => require_no_index(kind_tag, index_id, SegmentComponentKind::NodeLabelIndex)?, + 8 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeLabelIndex)?, + 9 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeTripleIndex)?, + 10 => require_no_index(kind_tag, index_id, SegmentComponentKind::AdjOutIndex)?, + 11 => require_no_index(kind_tag, index_id, SegmentComponentKind::AdjOutPostings)?, + 12 => require_no_index(kind_tag, index_id, SegmentComponentKind::AdjInIndex)?, + 13 => require_no_index(kind_tag, index_id, SegmentComponentKind::AdjInPostings)?, + 14 => require_no_index(kind_tag, index_id, SegmentComponentKind::TimestampIndex)?, + 15 => require_no_index( + kind_tag, + index_id, + SegmentComponentKind::LegacyNodePropertyIndex, + )?, + 16 => require_no_index( + kind_tag, + index_id, + SegmentComponentKind::NodePropertyHashMetadata, + )?, + 17 => SegmentComponentKind::NodePropertyEqualityIndex { + index_id: require_index(kind_tag, index_id)?, + }, + 18 => SegmentComponentKind::NodePropertyRangeIndex { + index_id: require_index(kind_tag, index_id)?, + }, + 19 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeWeightIndex)?, + 20 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeUpdatedAtIndex)?, + 21 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeValidFromIndex)?, + 22 => require_no_index(kind_tag, index_id, SegmentComponentKind::EdgeValidToIndex)?, + 23 => require_no_index(kind_tag, index_id, SegmentComponentKind::DegreeDelta)?, + 24 => require_no_index(kind_tag, index_id, SegmentComponentKind::PlannerStats)?, + 25 => require_no_index(kind_tag, index_id, SegmentComponentKind::NodeVectorMetadata)?, + 26 => require_no_index( + kind_tag, + index_id, + SegmentComponentKind::NodeDenseVectorBlob, + )?, + 27 => require_no_index( + kind_tag, + index_id, + SegmentComponentKind::NodeSparseVectorBlob, + )?, + 28 => require_no_index(kind_tag, index_id, SegmentComponentKind::DenseHnswMetadata)?, + 29 => require_no_index(kind_tag, index_id, SegmentComponentKind::DenseHnswGraph)?, + 30 => require_no_index(kind_tag, index_id, SegmentComponentKind::SparsePostingIndex)?, + 31 => require_no_index(kind_tag, index_id, SegmentComponentKind::SparsePostings)?, + 32 => SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: require_index(kind_tag, index_id)?, + }, + 33 => SegmentComponentKind::EdgePropertyRangeIndex { + index_id: require_index(kind_tag, index_id)?, + }, + 34 => require_no_index( + kind_tag, + index_id, + SegmentComponentKind::PackedSegmentContainer, + )?, + _ => return Ok(None), + }; + Ok(Some(known)) + } +} + +fn require_no_index( + kind_tag: u32, + index_id: Option, + kind: SegmentComponentKind, +) -> Result { + if index_id.is_some() { + return Err(component_manifest_error(format!( + "component kind tag {} does not accept index_id", + kind_tag + ))); + } + Ok(kind) +} + +fn require_index(kind_tag: u32, index_id: Option) -> Result { + index_id.ok_or_else(|| { + component_manifest_error(format!("component kind tag {} requires index_id", kind_tag)) + }) +} + +fn manifest_to_wire(manifest: &SegmentComponentManifestV1) -> SegmentComponentManifestWireV1 { + let mut components: Vec = + manifest.components.iter().map(record_to_wire).collect(); + components.extend( + manifest + .unknown_optional_components + .iter() + .map(|record| record.wire.clone()), + ); + SegmentComponentManifestWireV1 { + format_version: manifest.format_version, + segment_format_version: manifest.segment_format_version, + segment_id: manifest.segment_id, + generation: manifest.generation, + built_at_ms: manifest.built_at_ms, + build_kind_tag: manifest.build_kind.tag(), + segment_data_id: manifest.segment_data_id, + node_count: manifest.node_count, + edge_count: manifest.edge_count, + components, + } +} + +fn manifest_from_wire( + wire: SegmentComponentManifestWireV1, +) -> Result { + let mut components = Vec::new(); + let mut unknown_optional_components = Vec::new(); + for record in wire.components { + match SegmentComponentKind::from_tag_and_index(record.kind_tag, record.index_id)? { + Some(kind) => components.push(record_from_wire(record, kind)?), + None if requirement_from_wire(record.requirement_tag, record.fallback_tag)? + .is_optional() => + { + unknown_optional_components.push(UnknownOptionalComponentRecordV1 { wire: record }); + } + None => { + return Err(component_manifest_error(format!( + "unknown required component kind tag {}", + record.kind_tag + ))); + } + } + } + Ok(SegmentComponentManifestV1 { + format_version: wire.format_version, + segment_format_version: wire.segment_format_version, + segment_id: wire.segment_id, + generation: wire.generation, + built_at_ms: wire.built_at_ms, + build_kind: SegmentComponentBuildKind::from_tag(wire.build_kind_tag)?, + segment_data_id: wire.segment_data_id, + node_count: wire.node_count, + edge_count: wire.edge_count, + components, + unknown_optional_components, + }) +} + +fn validate_manifest(manifest: &SegmentComponentManifestV1) -> Result<(), EngineError> { + if manifest.format_version != SEGMENT_COMPONENT_MANIFEST_PAYLOAD_VERSION { + return Err(component_manifest_error(format!( + "unsupported segment component manifest payload version {}", + manifest.format_version + ))); + } + let mut component_ids = HashSet::new(); + let mut known_keys = HashSet::new(); + let mut unknown_keys = HashSet::new(); + for record in &manifest.components { + if !component_ids.insert(record.component_id) { + return Err(component_manifest_error("duplicate component id")); + } + let key = (record.kind.kind_tag(), record.kind.index_id()); + if !known_keys.insert(key) { + return Err(component_manifest_error("duplicate known component kind")); + } + if let ComponentHandleV1::ExternalFile { relative_path, .. } = &record.handle { + validate_relative_component_path(relative_path)?; + } + validate_record_payload_len(record.payload_len, &record.handle)?; + let expected_dependency_digest = dependency_digest(&record.dependencies); + if record.dependency_digest != expected_dependency_digest { + return Err(component_manifest_error( + "component dependency digest does not match dependency list", + )); + } + } + for record in &manifest.unknown_optional_components { + if !component_ids.insert(record.wire.component_id) { + return Err(component_manifest_error("duplicate component id")); + } + if SegmentComponentKind::from_tag_and_index(record.wire.kind_tag, record.wire.index_id)? + .is_some() + { + return Err(component_manifest_error( + "unknown optional component record uses a known kind tag", + )); + } + if !requirement_from_wire(record.wire.requirement_tag, record.wire.fallback_tag)? + .is_optional() + { + return Err(component_manifest_error( + "unknown component record must be optional", + )); + } + let key = (record.wire.kind_tag, record.wire.index_id); + if !unknown_keys.insert(key) { + return Err(component_manifest_error( + "duplicate unknown optional component kind", + )); + } + let handle = handle_from_wire(record.wire.handle.clone())?; + validate_record_payload_len(record.wire.payload_len, &handle)?; + let mut dependencies = record.wire.dependencies.clone(); + let expected_dependency_digest = dependency_digest_from_wire(&mut dependencies)?; + if record.wire.dependency_digest != expected_dependency_digest { + return Err(component_manifest_error( + "unknown component dependency digest does not match dependency list", + )); + } + } + Ok(()) +} + +fn validate_record_payload_len( + payload_len: u64, + handle: &ComponentHandleV1, +) -> Result<(), EngineError> { + let handle_payload_len = match handle { + ComponentHandleV1::ExternalFile { + payload_len: handle_payload_len, + .. + } => *handle_payload_len, + ComponentHandleV1::PackedRange { len, .. } => *len, + }; + if payload_len != handle_payload_len { + return Err(component_manifest_error(format!( + "component payload length mismatch: record={}, handle={}", + payload_len, handle_payload_len + ))); + } + Ok(()) +} + +fn record_to_wire(record: &SegmentComponentRecordV1) -> SegmentComponentRecordWireV1 { + let (requirement_tag, fallback_tag) = requirement_to_wire(&record.requirement); + SegmentComponentRecordWireV1 { + component_id: record.component_id, + kind_tag: record.kind.kind_tag(), + index_id: record.kind.index_id(), + logical_format_version: record.logical_format_version, + created_generation: record.created_generation, + requirement_tag, + fallback_tag, + trust_class_tag: record.trust_class.tag(), + handle: handle_to_wire(&record.handle), + payload_len: record.payload_len, + payload_digest: record.payload_digest, + dependency_digest: record.dependency_digest, + dependencies: record.dependencies.iter().map(dependency_to_wire).collect(), + build_fingerprint: record.build_fingerprint, + } +} + +fn record_from_wire( + wire: SegmentComponentRecordWireV1, + kind: SegmentComponentKind, +) -> Result { + Ok(SegmentComponentRecordV1 { + component_id: wire.component_id, + kind, + logical_format_version: wire.logical_format_version, + created_generation: wire.created_generation, + requirement: requirement_from_wire(wire.requirement_tag, wire.fallback_tag)?, + trust_class: ComponentTrustClass::from_tag(wire.trust_class_tag)?, + handle: handle_from_wire(wire.handle)?, + payload_len: wire.payload_len, + payload_digest: wire.payload_digest, + dependency_digest: wire.dependency_digest, + dependencies: wire + .dependencies + .iter() + .map(dependency_from_wire) + .collect::, _>>()?, + build_fingerprint: wire.build_fingerprint, + }) +} + +fn handle_to_wire(handle: &ComponentHandleV1) -> ComponentHandleWireV1 { + match handle { + ComponentHandleV1::ExternalFile { + relative_path, + payload_offset, + payload_len, + } => ComponentHandleWireV1 { + handle_tag: 1, + relative_path: Some(relative_path.clone()), + payload_offset: *payload_offset, + payload_len: *payload_len, + container_component_id: None, + offset: 0, + len: 0, + }, + ComponentHandleV1::PackedRange { + container_component_id, + offset, + len, + } => ComponentHandleWireV1 { + handle_tag: 2, + relative_path: None, + payload_offset: 0, + payload_len: 0, + container_component_id: Some(*container_component_id), + offset: *offset, + len: *len, + }, + } +} + +fn handle_from_wire(wire: ComponentHandleWireV1) -> Result { + match wire.handle_tag { + 1 => { + let relative_path = wire.relative_path.ok_or_else(|| { + component_manifest_error("external component handle missing relative_path") + })?; + validate_relative_component_path(&relative_path)?; + Ok(ComponentHandleV1::ExternalFile { + relative_path, + payload_offset: wire.payload_offset, + payload_len: wire.payload_len, + }) + } + 2 => Ok(ComponentHandleV1::PackedRange { + container_component_id: wire.container_component_id.ok_or_else(|| { + component_manifest_error("packed component handle missing container id") + })?, + offset: wire.offset, + len: wire.len, + }), + _ => Err(component_manifest_error(format!( + "unknown component handle tag {}", + wire.handle_tag + ))), + } +} + +fn requirement_to_wire(requirement: &ComponentRequirement) -> (u8, u8) { + match requirement { + ComponentRequirement::Required => (1, 0), + ComponentRequirement::Optional { fallback } => (2, fallback.tag()), + } +} + +fn requirement_from_wire( + requirement_tag: u8, + fallback_tag: u8, +) -> Result { + match requirement_tag { + 1 if fallback_tag == 0 => Ok(ComponentRequirement::Required), + 1 => Err(component_manifest_error( + "required component must use empty fallback tag", + )), + 2 => Ok(ComponentRequirement::Optional { + fallback: ComponentFallbackClass::from_tag(fallback_tag)?, + }), + _ => Err(component_manifest_error(format!( + "unknown component requirement tag {}", + requirement_tag + ))), + } +} + +impl ComponentRequirement { + fn is_optional(&self) -> bool { + matches!(self, ComponentRequirement::Optional { .. }) + } +} + +impl ComponentFallbackClass { + fn tag(self) -> u8 { + match self { + ComponentFallbackClass::None => 0, + ComponentFallbackClass::MetadataScan => 1, + ComponentFallbackClass::TypeScan => 2, + ComponentFallbackClass::AdjacencyWalk => 3, + ComponentFallbackClass::RecordScan => 4, + ComponentFallbackClass::ExactVectorScan => 5, + ComponentFallbackClass::PlannerStatsUnavailable => 6, + ComponentFallbackClass::FeatureUnavailable => 7, + } + } + + fn from_tag(tag: u8) -> Result { + match tag { + 0 => Ok(ComponentFallbackClass::None), + 1 => Ok(ComponentFallbackClass::MetadataScan), + 2 => Ok(ComponentFallbackClass::TypeScan), + 3 => Ok(ComponentFallbackClass::AdjacencyWalk), + 4 => Ok(ComponentFallbackClass::RecordScan), + 5 => Ok(ComponentFallbackClass::ExactVectorScan), + 6 => Ok(ComponentFallbackClass::PlannerStatsUnavailable), + 7 => Ok(ComponentFallbackClass::FeatureUnavailable), + _ => Err(component_manifest_error(format!( + "unknown component fallback tag {}", + tag + ))), + } + } +} + +impl ComponentTrustClass { + fn tag(self) -> u8 { + match self { + ComponentTrustClass::PrimaryData => 1, + ComponentTrustClass::PrimaryMetadata => 2, + ComponentTrustClass::CoreMaintainedIndex => 3, + ComponentTrustClass::OptionalCandidateIndex => 4, + ComponentTrustClass::OptionalExactAccelerator => 5, + ComponentTrustClass::OptionalAdvisoryStats => 6, + ComponentTrustClass::OptionalApproximateAccelerator => 7, + ComponentTrustClass::AuxiliaryBlob => 8, + } + } + + fn from_tag(tag: u8) -> Result { + match tag { + 1 => Ok(ComponentTrustClass::PrimaryData), + 2 => Ok(ComponentTrustClass::PrimaryMetadata), + 3 => Ok(ComponentTrustClass::CoreMaintainedIndex), + 4 => Ok(ComponentTrustClass::OptionalCandidateIndex), + 5 => Ok(ComponentTrustClass::OptionalExactAccelerator), + 6 => Ok(ComponentTrustClass::OptionalAdvisoryStats), + 7 => Ok(ComponentTrustClass::OptionalApproximateAccelerator), + 8 => Ok(ComponentTrustClass::AuxiliaryBlob), + _ => Err(component_manifest_error(format!( + "unknown component trust class tag {}", + tag + ))), + } + } +} + +impl SegmentComponentBuildKind { + fn tag(self) -> u8 { + match self { + SegmentComponentBuildKind::Flush => 1, + SegmentComponentBuildKind::Compaction => 2, + SegmentComponentBuildKind::OptionalRefresh => 3, + SegmentComponentBuildKind::TestFixture => 4, + } + } + + fn from_tag(tag: u8) -> Result { + match tag { + 1 => Ok(SegmentComponentBuildKind::Flush), + 2 => Ok(SegmentComponentBuildKind::Compaction), + 3 => Ok(SegmentComponentBuildKind::OptionalRefresh), + 4 => Ok(SegmentComponentBuildKind::TestFixture), + _ => Err(component_manifest_error(format!( + "unknown component manifest build kind tag {}", + tag + ))), + } + } +} + +impl SegmentSourceGroupKind { + fn tag(self) -> u8 { + match self { + SegmentSourceGroupKind::NodeSource => 1, + SegmentSourceGroupKind::EdgeSource => 2, + SegmentSourceGroupKind::NodePropertyContentSource => 3, + SegmentSourceGroupKind::NodePropertyHashSource => 4, + SegmentSourceGroupKind::EdgeMetadataSource => 5, + SegmentSourceGroupKind::DegreeSource => 6, + SegmentSourceGroupKind::DenseVectorSource => 7, + SegmentSourceGroupKind::SparseVectorSource => 8, + SegmentSourceGroupKind::SegmentData => 9, + } + } + + fn from_tag(tag: u8) -> Result { + match tag { + 1 => Ok(SegmentSourceGroupKind::NodeSource), + 2 => Ok(SegmentSourceGroupKind::EdgeSource), + 3 => Ok(SegmentSourceGroupKind::NodePropertyContentSource), + 4 => Ok(SegmentSourceGroupKind::NodePropertyHashSource), + 5 => Ok(SegmentSourceGroupKind::EdgeMetadataSource), + 6 => Ok(SegmentSourceGroupKind::DegreeSource), + 7 => Ok(SegmentSourceGroupKind::DenseVectorSource), + 8 => Ok(SegmentSourceGroupKind::SparseVectorSource), + 9 => Ok(SegmentSourceGroupKind::SegmentData), + _ => Err(component_manifest_error(format!( + "unknown source group tag {}", + tag + ))), + } + } +} + +impl SecondaryIndexTargetKindForComponents { + fn tag(self) -> u8 { + match self { + SecondaryIndexTargetKindForComponents::Node => 1, + SecondaryIndexTargetKindForComponents::Edge => 2, + } + } + + fn from_tag(tag: u8) -> Result { + match tag { + 1 => Ok(SecondaryIndexTargetKindForComponents::Node), + 2 => Ok(SecondaryIndexTargetKindForComponents::Edge), + _ => Err(component_manifest_error(format!( + "unknown secondary index target tag {}", + tag + ))), + } + } +} + +impl SecondaryIndexKindFingerprint { + fn tag(self) -> u8 { + match self { + SecondaryIndexKindFingerprint::Equality => 1, + SecondaryIndexKindFingerprint::Range => 2, + } + } + + fn from_tag(tag: u8) -> Result { + match tag { + 1 => Ok(SecondaryIndexKindFingerprint::Equality), + 2 => Ok(SecondaryIndexKindFingerprint::Range), + _ => Err(component_manifest_error(format!( + "unknown secondary index kind tag {}", + tag + ))), + } + } +} + +fn dependency_to_wire(dependency: &ComponentDependencyV1) -> ComponentDependencyWireV1 { + match dependency { + ComponentDependencyV1::SourceComponent { kind, component_id } => { + ComponentDependencyWireV1 { + dependency_tag: 1, + component_kind_tag: Some(kind.kind_tag()), + component_index_id: kind.index_id(), + component_id: Some(*component_id), + group_tag: None, + group_id: None, + index_id: None, + target_kind_tag: None, + secondary_index_kind_tag: None, + fingerprint: None, + } + } + ComponentDependencyV1::SourceGroup { group, group_id } => ComponentDependencyWireV1 { + dependency_tag: 2, + component_kind_tag: None, + component_index_id: None, + component_id: None, + group_tag: Some(group.tag()), + group_id: Some(*group_id), + index_id: None, + target_kind_tag: None, + secondary_index_kind_tag: None, + fingerprint: None, + }, + ComponentDependencyV1::SecondaryIndexDeclaration { + index_id, + target_kind, + kind, + fingerprint, + } => ComponentDependencyWireV1 { + dependency_tag: 3, + component_kind_tag: None, + component_index_id: None, + component_id: None, + group_tag: None, + group_id: None, + index_id: Some(*index_id), + target_kind_tag: Some(target_kind.tag()), + secondary_index_kind_tag: Some(kind.tag()), + fingerprint: Some(*fingerprint), + }, + ComponentDependencyV1::DenseVectorConfig { fingerprint } => ComponentDependencyWireV1 { + dependency_tag: 4, + component_kind_tag: None, + component_index_id: None, + component_id: None, + group_tag: None, + group_id: None, + index_id: None, + target_kind_tag: None, + secondary_index_kind_tag: None, + fingerprint: Some(*fingerprint), + }, + ComponentDependencyV1::SparseVectorConfig { fingerprint } => ComponentDependencyWireV1 { + dependency_tag: 5, + component_kind_tag: None, + component_index_id: None, + component_id: None, + group_tag: None, + group_id: None, + index_id: None, + target_kind_tag: None, + secondary_index_kind_tag: None, + fingerprint: Some(*fingerprint), + }, + ComponentDependencyV1::WriterBuildParams { fingerprint } => ComponentDependencyWireV1 { + dependency_tag: 6, + component_kind_tag: None, + component_index_id: None, + component_id: None, + group_tag: None, + group_id: None, + index_id: None, + target_kind_tag: None, + secondary_index_kind_tag: None, + fingerprint: Some(*fingerprint), + }, + } +} + +fn dependency_from_wire( + wire: &ComponentDependencyWireV1, +) -> Result { + match wire.dependency_tag { + 1 => Ok(ComponentDependencyV1::SourceComponent { + kind: SegmentComponentKind::from_tag_and_index( + required_field(wire.component_kind_tag, "dependency component kind tag")?, + wire.component_index_id, + )? + .ok_or_else(|| { + component_manifest_error("dependency references unknown component kind") + })?, + component_id: required_field(wire.component_id, "dependency component id")?, + }), + 2 => Ok(ComponentDependencyV1::SourceGroup { + group: SegmentSourceGroupKind::from_tag(required_field( + wire.group_tag, + "dependency source group tag", + )?)?, + group_id: required_field(wire.group_id, "dependency source group id")?, + }), + 3 => Ok(ComponentDependencyV1::SecondaryIndexDeclaration { + index_id: required_field(wire.index_id, "dependency index id")?, + target_kind: SecondaryIndexTargetKindForComponents::from_tag(required_field( + wire.target_kind_tag, + "dependency target kind tag", + )?)?, + kind: SecondaryIndexKindFingerprint::from_tag(required_field( + wire.secondary_index_kind_tag, + "dependency secondary index kind tag", + )?)?, + fingerprint: required_field(wire.fingerprint, "dependency fingerprint")?, + }), + 4 => Ok(ComponentDependencyV1::DenseVectorConfig { + fingerprint: required_field(wire.fingerprint, "dependency fingerprint")?, + }), + 5 => Ok(ComponentDependencyV1::SparseVectorConfig { + fingerprint: required_field(wire.fingerprint, "dependency fingerprint")?, + }), + 6 => Ok(ComponentDependencyV1::WriterBuildParams { + fingerprint: required_field(wire.fingerprint, "dependency fingerprint")?, + }), + _ => Err(component_manifest_error(format!( + "unknown dependency tag {}", + wire.dependency_tag + ))), + } +} + +fn dependency_canonical_bytes(dependency: &ComponentDependencyWireV1) -> Vec { + let mut bytes = Vec::with_capacity(96); + push_u8(&mut bytes, dependency.dependency_tag); + push_u32(&mut bytes, dependency.component_kind_tag.unwrap_or(0)); + push_u64(&mut bytes, dependency.component_index_id.unwrap_or(0)); + bytes.extend_from_slice(&dependency.component_id.unwrap_or(ZERO_DIGEST)); + push_u8(&mut bytes, dependency.group_tag.unwrap_or(0)); + bytes.extend_from_slice(&dependency.group_id.unwrap_or(ZERO_DIGEST)); + push_u64(&mut bytes, dependency.index_id.unwrap_or(0)); + push_u8(&mut bytes, dependency.target_kind_tag.unwrap_or(0)); + push_u8(&mut bytes, dependency.secondary_index_kind_tag.unwrap_or(0)); + push_u64(&mut bytes, dependency.fingerprint.unwrap_or(0)); + bytes +} + +fn required_field(value: Option, name: &str) -> Result { + value.ok_or_else(|| component_manifest_error(format!("missing {}", name))) +} + +fn put_u8(hasher: &mut Sha256, value: u8) { + hasher.update([value]); +} + +fn put_u32(hasher: &mut Sha256, value: u32) { + hasher.update(value.to_le_bytes()); +} + +fn put_u64(hasher: &mut Sha256, value: u64) { + hasher.update(value.to_le_bytes()); +} + +fn put_bytes_with_len(hasher: &mut Sha256, bytes: &[u8]) { + put_u64(hasher, bytes.len() as u64); + hasher.update(bytes); +} + +fn push_u8(bytes: &mut Vec, value: u8) { + bytes.push(value); +} + +fn push_u32(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_le_bytes()); +} + +fn push_u64(bytes: &mut Vec, value: u64) { + bytes.extend_from_slice(&value.to_le_bytes()); +} + +fn component_manifest_error(message: impl Into) -> EngineError { + EngineError::ManifestError(message.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::segment_writer::SEGMENT_FORMAT_VERSION; + use std::io::Read; + use tempfile::tempdir; + + fn digest(seed: u8) -> ComponentDigest32 { + [seed; 32] + } + + fn known_component( + kind: SegmentComponentKind, + component_id: ComponentDigest32, + ) -> SegmentComponentRecordV1 { + let dependencies = vec![ComponentDependencyV1::SourceGroup { + group: SegmentSourceGroupKind::NodeSource, + group_id: digest(3), + }]; + let dependency_digest = dependency_digest(&dependencies); + SegmentComponentRecordV1 { + component_id, + kind, + logical_format_version: 1, + created_generation: 1, + requirement: ComponentRequirement::Required, + trust_class: ComponentTrustClass::CoreMaintainedIndex, + handle: ComponentHandleV1::ExternalFile { + relative_path: "component_payload.dat".to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: 12, + }, + payload_len: 12, + payload_digest: Some(digest(9)), + dependency_digest, + dependencies, + build_fingerprint: 11, + } + } + + fn manifest_with_components( + components: Vec, + unknown_optional_components: Vec, + ) -> SegmentComponentManifestV1 { + SegmentComponentManifestV1 { + format_version: SEGMENT_COMPONENT_MANIFEST_PAYLOAD_VERSION, + segment_format_version: 10, + segment_id: 42, + generation: 1, + built_at_ms: 123, + build_kind: SegmentComponentBuildKind::TestFixture, + segment_data_id: digest(4), + node_count: 2, + edge_count: 3, + components, + unknown_optional_components, + } + } + + fn unknown_optional( + tag: u32, + index_id: Option, + component_id: ComponentDigest32, + ) -> UnknownOptionalComponentRecordV1 { + let dependency_digest = dependency_digest(&[]); + UnknownOptionalComponentRecordV1 { + wire: SegmentComponentRecordWireV1 { + component_id, + kind_tag: tag, + index_id, + logical_format_version: 1, + created_generation: 1, + requirement_tag: 2, + fallback_tag: ComponentFallbackClass::RecordScan.tag(), + trust_class_tag: ComponentTrustClass::OptionalCandidateIndex.tag(), + handle: ComponentHandleWireV1 { + handle_tag: 1, + relative_path: Some("future_sidecar.dat".to_string()), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: 8, + container_component_id: None, + offset: 0, + len: 0, + }, + payload_len: 8, + payload_digest: Some(digest(8)), + dependency_digest, + dependencies: vec![], + build_fingerprint: 5, + }, + } + } + + #[test] + fn component_kind_tags_match_snapshot_and_are_unique() { + let snapshot = vec![ + (SegmentComponentKind::NodeRecords, 1, None), + (SegmentComponentKind::EdgeRecords, 2, None), + (SegmentComponentKind::NodeMetadata, 3, None), + (SegmentComponentKind::EdgeMetadata, 4, None), + (SegmentComponentKind::Tombstones, 5, None), + (SegmentComponentKind::KeyIndex, 6, None), + (SegmentComponentKind::NodeLabelIndex, 7, None), + (SegmentComponentKind::EdgeLabelIndex, 8, None), + (SegmentComponentKind::EdgeTripleIndex, 9, None), + (SegmentComponentKind::AdjOutIndex, 10, None), + (SegmentComponentKind::AdjOutPostings, 11, None), + (SegmentComponentKind::AdjInIndex, 12, None), + (SegmentComponentKind::AdjInPostings, 13, None), + (SegmentComponentKind::TimestampIndex, 14, None), + (SegmentComponentKind::LegacyNodePropertyIndex, 15, None), + (SegmentComponentKind::NodePropertyHashMetadata, 16, None), + ( + SegmentComponentKind::NodePropertyEqualityIndex { index_id: 91 }, + 17, + Some(91), + ), + ( + SegmentComponentKind::NodePropertyRangeIndex { index_id: 92 }, + 18, + Some(92), + ), + (SegmentComponentKind::EdgeWeightIndex, 19, None), + (SegmentComponentKind::EdgeUpdatedAtIndex, 20, None), + (SegmentComponentKind::EdgeValidFromIndex, 21, None), + (SegmentComponentKind::EdgeValidToIndex, 22, None), + (SegmentComponentKind::DegreeDelta, 23, None), + (SegmentComponentKind::PlannerStats, 24, None), + (SegmentComponentKind::NodeVectorMetadata, 25, None), + (SegmentComponentKind::NodeDenseVectorBlob, 26, None), + (SegmentComponentKind::NodeSparseVectorBlob, 27, None), + (SegmentComponentKind::DenseHnswMetadata, 28, None), + (SegmentComponentKind::DenseHnswGraph, 29, None), + (SegmentComponentKind::SparsePostingIndex, 30, None), + (SegmentComponentKind::SparsePostings, 31, None), + ( + SegmentComponentKind::EdgePropertyEqualityIndex { index_id: 93 }, + 32, + Some(93), + ), + ( + SegmentComponentKind::EdgePropertyRangeIndex { index_id: 94 }, + 33, + Some(94), + ), + (SegmentComponentKind::PackedSegmentContainer, 34, None), + ]; + let mut tags = HashSet::new(); + for (kind, expected_tag, expected_index_id) in snapshot { + assert_eq!(kind.kind_tag(), expected_tag); + assert_eq!(kind.index_id(), expected_index_id); + assert!(tags.insert(expected_tag)); + assert_eq!( + SegmentComponentKind::from_tag_and_index(expected_tag, expected_index_id) + .unwrap() + .unwrap(), + kind + ); + } + } + + #[test] + fn indexed_kind_manifest_round_trip_preserves_index_id() { + let record = known_component( + SegmentComponentKind::NodePropertyEqualityIndex { index_id: 77 }, + digest(1), + ); + let manifest = manifest_with_components(vec![record], vec![]); + let decoded = + decode_manifest_envelope(&encode_manifest_envelope(&manifest).unwrap()).unwrap(); + assert_eq!( + decoded.components[0].kind, + SegmentComponentKind::NodePropertyEqualityIndex { index_id: 77 } + ); + } + + #[test] + fn manifest_envelope_round_trips_and_rejects_malformed_inputs() { + let manifest = manifest_with_components( + vec![known_component(SegmentComponentKind::KeyIndex, digest(1))], + vec![], + ); + let encoded = encode_manifest_envelope(&manifest).unwrap(); + assert_eq!(decode_manifest_envelope(&encoded).unwrap(), manifest); + + let mut bad_magic = encoded.clone(); + bad_magic[0] ^= 1; + assert!(decode_manifest_envelope(&bad_magic) + .unwrap_err() + .to_string() + .contains("magic")); + + let mut bad_version = encoded.clone(); + bad_version[8..12].copy_from_slice(&99u32.to_le_bytes()); + assert!(decode_manifest_envelope(&bad_version) + .unwrap_err() + .to_string() + .contains("version")); + + let mut bad_crc = encoded.clone(); + let last = bad_crc.len() - 1; + bad_crc[last] ^= 1; + assert!(decode_manifest_envelope(&bad_crc) + .unwrap_err() + .to_string() + .contains("crc")); + + let mut bad_len = encoded; + bad_len[16..24].copy_from_slice(&1u64.to_le_bytes()); + assert!(decode_manifest_envelope(&bad_len) + .unwrap_err() + .to_string() + .contains("length")); + } + + #[test] + fn manifest_envelope_future_segment_format_version_decoded_correctly() { + let mut manifest = manifest_with_components( + vec![known_component(SegmentComponentKind::KeyIndex, digest(1))], + vec![], + ); + manifest.segment_format_version = 99; + let encoded = encode_manifest_envelope(&manifest).unwrap(); + let version_in_envelope = u32::from_le_bytes(encoded[12..16].try_into().unwrap()); + assert_eq!(version_in_envelope, 99); + let decoded = decode_manifest_envelope(&encoded).unwrap(); + assert_eq!(decoded.segment_format_version, 99); + } + + #[test] + fn manifest_rejects_duplicate_component_ids_and_known_kinds() { + let duplicate_id = manifest_with_components( + vec![ + known_component(SegmentComponentKind::KeyIndex, digest(1)), + known_component(SegmentComponentKind::NodeLabelIndex, digest(1)), + ], + vec![], + ); + assert!(encode_manifest_envelope(&duplicate_id) + .unwrap_err() + .to_string() + .contains("duplicate component id")); + + let duplicate_kind = manifest_with_components( + vec![ + known_component(SegmentComponentKind::KeyIndex, digest(1)), + known_component(SegmentComponentKind::KeyIndex, digest(2)), + ], + vec![], + ); + assert!(encode_manifest_envelope(&duplicate_kind) + .unwrap_err() + .to_string() + .contains("duplicate known")); + } + + #[test] + fn manifest_rejects_loose_wire_contract_records() { + let mut bad_version = manifest_with_components( + vec![known_component(SegmentComponentKind::KeyIndex, digest(1))], + vec![], + ); + bad_version.format_version = SEGMENT_COMPONENT_MANIFEST_PAYLOAD_VERSION + 1; + assert!(encode_manifest_envelope(&bad_version) + .unwrap_err() + .to_string() + .contains("payload version")); + + let mut bad_payload_len = manifest_with_components( + vec![known_component(SegmentComponentKind::KeyIndex, digest(1))], + vec![], + ); + bad_payload_len.components[0].payload_len += 1; + assert!(encode_manifest_envelope(&bad_payload_len) + .unwrap_err() + .to_string() + .contains("payload length")); + + let mut bad_dependency_digest = manifest_with_components( + vec![known_component(SegmentComponentKind::KeyIndex, digest(1))], + vec![], + ); + bad_dependency_digest.components[0].dependency_digest = digest(99); + assert!(encode_manifest_envelope(&bad_dependency_digest) + .unwrap_err() + .to_string() + .contains("dependency digest")); + + let mut bad_unknown_path = unknown_optional(5000, Some(6), digest(2)); + bad_unknown_path.wire.handle.relative_path = Some("a/../b".to_string()); + let manifest = manifest_with_components(vec![], vec![bad_unknown_path]); + assert!(encode_manifest_envelope(&manifest) + .unwrap_err() + .to_string() + .contains("normalized")); + + let mut bad_unknown_dependency = unknown_optional(5001, None, digest(3)); + bad_unknown_dependency.wire.dependency_digest = digest(88); + let manifest = manifest_with_components(vec![], vec![bad_unknown_dependency]); + assert!(encode_manifest_envelope(&manifest) + .unwrap_err() + .to_string() + .contains("dependency digest")); + } + + #[test] + fn unknown_optional_records_are_preserved_without_known_duplicate_collision() { + let manifest = manifest_with_components( + vec![known_component(SegmentComponentKind::KeyIndex, digest(1))], + vec![unknown_optional(5000, Some(6), digest(2))], + ); + let decoded = + decode_manifest_envelope(&encode_manifest_envelope(&manifest).unwrap()).unwrap(); + assert_eq!(decoded.components.len(), 1); + assert_eq!(decoded.unknown_optional_components.len(), 1); + assert_eq!(decoded.unknown_optional_components[0].wire.kind_tag, 5000); + assert_eq!( + decoded.unknown_optional_components[0].wire.index_id, + Some(6) + ); + } + + #[test] + fn duplicate_unknown_optional_key_is_rejected() { + let manifest = manifest_with_components( + vec![], + vec![ + unknown_optional(5000, Some(6), digest(2)), + unknown_optional(5000, Some(6), digest(3)), + ], + ); + assert!(encode_manifest_envelope(&manifest) + .unwrap_err() + .to_string() + .contains("duplicate unknown")); + } + + #[test] + fn unknown_required_record_fails_decode_before_runtime_use() { + let wire = SegmentComponentManifestWireV1 { + format_version: 1, + segment_format_version: 10, + segment_id: 42, + generation: 1, + built_at_ms: 123, + build_kind_tag: SegmentComponentBuildKind::TestFixture.tag(), + segment_data_id: digest(4), + node_count: 2, + edge_count: 3, + components: vec![SegmentComponentRecordWireV1 { + requirement_tag: 1, + fallback_tag: 0, + ..unknown_optional(5000, None, digest(1)).wire + }], + }; + let payload = rmp_serde::to_vec(&wire).unwrap(); + let mut crc = Crc32Hasher::new(); + crc.update(&payload); + let mut data = Vec::new(); + data.extend_from_slice(&SEGMENT_COMPONENT_MANIFEST_MAGIC); + data.extend_from_slice(&SEGMENT_COMPONENT_MANIFEST_ENVELOPE_VERSION.to_le_bytes()); + data.extend_from_slice(&SEGMENT_FORMAT_VERSION.to_le_bytes()); + data.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + data.extend_from_slice(&crc.finalize().to_le_bytes()); + data.extend_from_slice(&payload); + assert!(decode_manifest_envelope(&data) + .unwrap_err() + .to_string() + .contains("unknown required")); + } + + #[test] + fn relative_path_validation_allows_nested_sidecar_and_rejects_unsafe_paths() { + validate_relative_component_path("secondary_indexes/node_prop_eq_1.dat").unwrap(); + for invalid in [ + "", + ".", + "..", + "a/../b", + "/absolute", + "secondary_indexes/", + "secondary_indexes\\node_prop_eq_1.dat", + "secondary_indexes/\u{1f}.dat", + ] { + assert!( + validate_relative_component_path(invalid).is_err(), + "{invalid:?} should be rejected" + ); + } + } + + #[test] + fn identity_header_round_trips_and_rejects_mismatch() { + let header = ComponentIdentityHeaderV1 { + segment_format_version: 10, + segment_id: 42, + component_kind: SegmentComponentKind::KeyIndex, + logical_format_version: 1, + created_generation: 2, + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: 12, + component_id: digest(1), + dependency_digest: digest(2), + build_fingerprint: 3, + payload_digest: Some(digest(4)), + }; + let encoded = encode_identity_header(&header); + assert_eq!(decode_identity_header(&encoded).unwrap(), header); + let mut bad_magic = encoded; + bad_magic[0] ^= 1; + assert!(decode_identity_header(&bad_magic) + .unwrap_err() + .to_string() + .contains("magic")); + } + + #[test] + fn indexed_identity_header_round_trips_and_rejects_stale_flag_bytes() { + let indexed_header = ComponentIdentityHeaderV1 { + segment_format_version: 10, + segment_id: 42, + component_kind: SegmentComponentKind::NodePropertyEqualityIndex { index_id: 77 }, + logical_format_version: 1, + created_generation: 2, + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: 12, + component_id: digest(1), + dependency_digest: digest(2), + build_fingerprint: 3, + payload_digest: Some(digest(4)), + }; + let encoded = encode_identity_header(&indexed_header); + assert_eq!(decode_identity_header(&encoded).unwrap(), indexed_header); + + let no_digest_header = ComponentIdentityHeaderV1 { + payload_digest: None, + ..indexed_header + }; + let mut stale_digest_bytes = encode_identity_header(&no_digest_header); + stale_digest_bytes[136] = 1; + assert!(decode_identity_header(&stale_digest_bytes) + .unwrap_err() + .to_string() + .contains("digest bytes")); + + let mut stale_index_bytes = encode_identity_header(&ComponentIdentityHeaderV1 { + component_kind: SegmentComponentKind::KeyIndex, + payload_digest: None, + ..no_digest_header + }); + stale_index_bytes[168..176].copy_from_slice(&77u64.to_le_bytes()); + assert!(decode_identity_header(&stale_index_bytes) + .unwrap_err() + .to_string() + .contains("index_id bytes")); + } + + #[test] + fn dependency_digest_sorts_deterministically() { + let a = ComponentDependencyV1::SourceGroup { + group: SegmentSourceGroupKind::NodeSource, + group_id: digest(1), + }; + let b = ComponentDependencyV1::WriterBuildParams { fingerprint: 9 }; + assert_eq!( + dependency_digest(&[a.clone(), b.clone()]), + dependency_digest(&[b, a]) + ); + + let same_shape_a = ComponentDependencyV1::SourceGroup { + group: SegmentSourceGroupKind::NodeSource, + group_id: digest(11), + }; + let same_shape_b = ComponentDependencyV1::SourceGroup { + group: SegmentSourceGroupKind::NodeSource, + group_id: digest(12), + }; + assert_eq!( + dependency_digest(&[same_shape_a.clone(), same_shape_b.clone()]), + dependency_digest(&[same_shape_b.clone(), same_shape_a.clone()]) + ); + assert_ne!( + dependency_digest(&[same_shape_a.clone(), same_shape_a]), + dependency_digest(&[same_shape_b.clone(), same_shape_b]) + ); + } + + #[test] + fn source_group_digests_are_deterministic() { + assert_eq!( + digest_source_group(SegmentSourceGroupKind::NodeSource, &[digest(1), digest(2)]), + digest_source_group(SegmentSourceGroupKind::NodeSource, &[digest(1), digest(2)]) + ); + assert_ne!( + digest_source_group(SegmentSourceGroupKind::NodeSource, &[digest(1), digest(2)]), + digest_source_group(SegmentSourceGroupKind::NodeSource, &[digest(2), digest(1)]) + ); + } + + #[test] + fn packed_container_is_ignored_by_segment_data_source_groups() { + let required = vec![ + known_component(SegmentComponentKind::NodeRecords, digest(1)), + known_component(SegmentComponentKind::EdgeRecords, digest(2)), + known_component(SegmentComponentKind::NodeMetadata, digest(3)), + known_component(SegmentComponentKind::EdgeMetadata, digest(4)), + known_component(SegmentComponentKind::Tombstones, digest(5)), + ]; + let without_container = segment_source_groups_from_records(42, 2, 3, &required).unwrap(); + let mut with_container_records = required; + with_container_records.push(known_component( + SegmentComponentKind::PackedSegmentContainer, + digest(99), + )); + let with_container = + segment_source_groups_from_records(42, 2, 3, &with_container_records).unwrap(); + + assert_eq!( + without_container.segment_data_id, + with_container.segment_data_id + ); + assert_eq!(without_container.node_source, with_container.node_source); + assert_eq!(without_container.edge_source, with_container.edge_source); + assert_eq!( + without_container.dense_vector_source, + with_container.dense_vector_source + ); + assert_eq!( + without_container.sparse_vector_source, + with_container.sparse_vector_source + ); + } + + #[test] + fn component_id_sensitivity_matches_policy() { + let base = component_id( + 42, + &SegmentComponentKind::KeyIndex, + 1, + 100, + Some(&digest(1)), + &digest(2), + 3, + ); + assert_ne!( + base, + component_id( + 42, + &SegmentComponentKind::KeyIndex, + 1, + 100, + Some(&digest(1)), + &digest(9), + 3, + ) + ); + assert_ne!( + base, + component_id( + 42, + &SegmentComponentKind::KeyIndex, + 1, + 100, + Some(&digest(1)), + &digest(2), + 9, + ) + ); + assert_ne!( + base, + component_id( + 42, + &SegmentComponentKind::KeyIndex, + 1, + 100, + Some(&digest(9)), + &digest(2), + 3, + ) + ); + assert_eq!( + base, + component_id( + 42, + &SegmentComponentKind::KeyIndex, + 1, + 100, + Some(&digest(1)), + &digest(2), + 3, + ) + ); + } + + #[test] + fn semantic_fingerprints_are_nonzero_stable_and_field_sensitive() { + let planner = component_semantic_fingerprint("planner_stats", &[1, 2, 3, 4]); + let sparse = component_semantic_fingerprint("sparse_postings", &[1, 2, 3, 4]); + let dense = component_semantic_fingerprint("dense_hnsw", &[64, 1, 16, 200]); + assert_ne!(planner, 0); + assert_eq!( + planner, + component_semantic_fingerprint("planner_stats", &[1, 2, 3, 4]) + ); + assert_ne!( + planner, + component_semantic_fingerprint("planner_stats", &[1, 2, 3, 5]) + ); + assert_ne!(planner, sparse); + assert_ne!(dense, 0); + } + + #[test] + fn declaration_fingerprint_changes_for_declared_index_fields() { + let base = secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email", + SecondaryIndexKindFingerprint::Equality, + 0, + 7, + 3, + ); + assert_ne!(base, 0); + assert_eq!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email", + SecondaryIndexKindFingerprint::Equality, + 0, + 7, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 2, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email", + SecondaryIndexKindFingerprint::Equality, + 0, + 7, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Edge, + 10, + b"email", + SecondaryIndexKindFingerprint::Equality, + 0, + 7, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 11, + b"email", + SecondaryIndexKindFingerprint::Equality, + 0, + 7, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email\0", + SecondaryIndexKindFingerprint::Equality, + 0, + 7, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email", + SecondaryIndexKindFingerprint::Range, + 0, + 7, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email", + SecondaryIndexKindFingerprint::Equality, + 1, + 7, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email", + SecondaryIndexKindFingerprint::Equality, + 0, + 8, + 3, + ) + ); + assert_ne!( + base, + secondary_index_declaration_fingerprint( + 1, + SecondaryIndexTargetKindForComponents::Node, + 10, + b"email", + SecondaryIndexKindFingerprint::Equality, + 0, + 7, + 4, + ) + ); + } + + #[test] + fn component_identity_writer_is_single_pass_and_unwrapped_from_production_paths() { + let dir = tempdir().unwrap(); + let path = dir.path().join("planner_stats.dat"); + let mut writer = ComponentIdentityWriter::create( + &path, + "planner_stats.dat".to_string(), + 10, + 42, + SegmentComponentKind::PlannerStats, + 1, + 1, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::PlannerStatsUnavailable, + }, + ComponentTrustClass::OptionalAdvisoryStats, + 17, + true, + ) + .unwrap(); + writer.write_all(b"hello").unwrap(); + writer.write_all(b" world").unwrap(); + let dependencies = vec![ComponentDependencyV1::SourceGroup { + group: SegmentSourceGroupKind::NodeSource, + group_id: digest(44), + }]; + let expected_dependency_digest = dependency_digest(&dependencies); + let record = writer.finish(dependencies).unwrap(); + assert_eq!(record.payload_len, 11); + assert_eq!(record.dependency_digest, expected_dependency_digest); + assert_eq!( + record.handle, + ComponentHandleV1::ExternalFile { + relative_path: "planner_stats.dat".to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: 11, + } + ); + let mut bytes = Vec::new(); + File::open(&path).unwrap().read_to_end(&mut bytes).unwrap(); + assert_eq!(&bytes[COMPONENT_IDENTITY_HEADER_LEN..], b"hello world"); + let header = decode_identity_header(&bytes[..COMPONENT_IDENTITY_HEADER_LEN]).unwrap(); + assert_eq!(header.component_id, record.component_id); + assert_eq!(header.payload_digest, record.payload_digest); + } + + fn packed_container_record(payload_len: u64) -> SegmentComponentRecordV1 { + let dependencies = Vec::new(); + let dependency_digest = dependency_digest(&dependencies); + let payload_digest = digest(88); + SegmentComponentRecordV1 { + component_id: digest(77), + kind: SegmentComponentKind::PackedSegmentContainer, + logical_format_version: 1, + created_generation: 1, + requirement: ComponentRequirement::Required, + trust_class: ComponentTrustClass::AuxiliaryBlob, + handle: ComponentHandleV1::ExternalFile { + relative_path: PACKED_CORE_FILENAME.to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len, + }, + payload_len, + payload_digest: Some(payload_digest), + dependency_digest, + dependencies, + build_fingerprint: 101, + } + } + + fn packed_record( + kind: SegmentComponentKind, + offset: u64, + len: u64, + ) -> SegmentComponentRecordV1 { + let dependencies = Vec::new(); + let dependency_digest = dependency_digest(&dependencies); + let payload_digest = digest(kind.kind_tag() as u8); + let computed_component_id = component_id( + 42, + &kind, + 1, + len, + Some(&payload_digest), + &dependency_digest, + 11, + ); + SegmentComponentRecordV1 { + component_id: computed_component_id, + kind, + logical_format_version: 1, + created_generation: 1, + requirement: ComponentRequirement::Required, + trust_class: ComponentTrustClass::CoreMaintainedIndex, + handle: ComponentHandleV1::PackedRange { + container_component_id: digest(77), + offset, + len, + }, + payload_len: len, + payload_digest: Some(payload_digest), + dependency_digest, + dependencies, + build_fingerprint: 11, + } + } + + #[test] + fn packed_range_wire_roundtrip_preserves_fields() { + let mut record = known_component(SegmentComponentKind::KeyIndex, digest(9)); + record.handle = ComponentHandleV1::PackedRange { + container_component_id: digest(7), + offset: 24, + len: 12, + }; + record.payload_len = 12; + let manifest = manifest_with_components(vec![record], vec![]); + let decoded = + decode_manifest_envelope(&encode_manifest_envelope(&manifest).unwrap()).unwrap(); + assert_eq!( + decoded.components[0].handle, + ComponentHandleV1::PackedRange { + container_component_id: digest(7), + offset: 24, + len: 12, + } + ); + } + + #[test] + fn packed_component_id_ignores_physical_handle() { + let kind = SegmentComponentKind::NodeRecords; + let dependencies = Vec::new(); + let dependency_digest = dependency_digest(&dependencies); + let payload_digest = digest(5); + let computed_component_id = component_id( + 42, + &kind, + 1, + 99, + Some(&payload_digest), + &dependency_digest, + 11, + ); + let external = SegmentComponentRecordV1 { + component_id: computed_component_id, + kind: kind.clone(), + logical_format_version: 1, + created_generation: 1, + requirement: ComponentRequirement::Required, + trust_class: ComponentTrustClass::PrimaryData, + handle: ComponentHandleV1::ExternalFile { + relative_path: "external_component_payload.dat".to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: 99, + }, + payload_len: 99, + payload_digest: Some(payload_digest), + dependency_digest, + dependencies: dependencies.clone(), + build_fingerprint: 11, + }; + let mut packed = external.clone(); + packed.handle = ComponentHandleV1::PackedRange { + container_component_id: digest(7), + offset: 128, + len: 99, + }; + assert_ne!(external.handle, packed.handle); + assert_eq!(external.component_id, packed.component_id); + assert_eq!( + packed.component_id, + component_id( + 42, + &packed.kind, + packed.logical_format_version, + packed.payload_len, + packed.payload_digest.as_ref(), + &packed.dependency_digest, + packed.build_fingerprint, + ) + ); + } + + #[test] + fn packed_manifest_validation_rejects_missing_container_length_and_overlap() { + let node = packed_record(SegmentComponentKind::NodeRecords, 0, 16); + let manifest = manifest_with_components(vec![node.clone()], vec![]); + assert!(validate_packed_core_manifest_contract(&manifest) + .unwrap_err() + .to_string() + .contains("without segment.core container")); + + let mut bad_len = manifest_with_components(vec![packed_container_record(64), node], vec![]); + bad_len.components[1].payload_len = 15; + assert!(validate_packed_core_manifest_contract(&bad_len) + .unwrap_err() + .to_string() + .contains("payload length")); + + let overflow = manifest_with_components( + vec![ + packed_container_record(64), + packed_record(SegmentComponentKind::NodeRecords, u64::MAX - 1, 8), + ], + vec![], + ); + assert!(validate_packed_core_manifest_contract(&overflow) + .unwrap_err() + .to_string() + .contains("overflows")); + + let overlap = manifest_with_components( + vec![ + packed_container_record(64), + packed_record(SegmentComponentKind::NodeRecords, 0, 16), + packed_record(SegmentComponentKind::EdgeRecords, 8, 16), + ], + vec![], + ); + assert!(validate_packed_core_manifest_contract(&overlap) + .unwrap_err() + .to_string() + .contains("overlap")); + } + + #[test] + fn packed_manifest_validation_allows_zero_length_shared_offsets() { + let manifest = manifest_with_components( + vec![ + packed_container_record(64), + packed_record(SegmentComponentKind::NodeRecords, 0, 0), + packed_record(SegmentComponentKind::EdgeRecords, 0, 0), + ], + vec![], + ); + validate_packed_core_manifest_contract(&manifest).unwrap(); + } + + #[test] + fn packed_manifest_validation_rejects_optional_range_overlap() { + let mut optional = packed_record(SegmentComponentKind::EdgeWeightIndex, 8, 16); + optional.requirement = ComponentRequirement::Optional { + fallback: ComponentFallbackClass::MetadataScan, + }; + optional.trust_class = ComponentTrustClass::OptionalCandidateIndex; + + let overlap = manifest_with_components( + vec![ + packed_container_record(64), + packed_record(SegmentComponentKind::NodeRecords, 0, 16), + optional, + ], + vec![], + ); + + assert!(validate_packed_core_manifest_contract(&overlap) + .unwrap_err() + .to_string() + .contains("overlap")); + } +} diff --git a/src/segment_reader.rs b/src/segment_reader.rs index ffc4649..15d85cd 100644 --- a/src/segment_reader.rs +++ b/src/segment_reader.rs @@ -1,23 +1,40 @@ -use crate::degree_cache::{DegreeDelta, DegreeSidecar, DEGREE_DELTA_FILENAME}; +use crate::degree_cache::{DegreeDelta, DegreeSidecar}; use crate::dense_hnsw::{ dense_score_from_bytes, load_dense_hnsw_query_points, search_dense_hnsw_scoped_with_points, - search_dense_hnsw_with_points, validate_dense_hnsw_files, DenseHnswHeader, DenseQueryPoint, - DENSE_HNSW_GRAPH_FILENAME, DENSE_HNSW_META_FILENAME, + search_dense_hnsw_with_points, validate_dense_hnsw_files_for_open, DenseHnswHeader, + DenseQueryPoint, +}; +use crate::edge_metadata::{ + encode_edge_weight_key, EdgeMetadataCandidate, RangeBoundFlags, + EDGE_I64_METADATA_INDEX_ENTRY_SIZE, EDGE_UPDATED_AT_INDEX_LOGICAL_NAME, + EDGE_VALID_FROM_INDEX_LOGICAL_NAME, EDGE_VALID_TO_INDEX_LOGICAL_NAME, + EDGE_WEIGHT_INDEX_ENTRY_SIZE, EDGE_WEIGHT_INDEX_LOGICAL_NAME, }; use crate::error::EngineError; #[cfg(test)] use crate::planner_stats::SegmentPlannerStatsV1; use crate::planner_stats::{ - read_planner_stats_sidecar, DeclaredIndexRuntimeCoverageState, PlannerStatsAvailability, - PlannerStatsDeclaredIndexKind, + planner_stats_declared_index_target, read_planner_stats_payload, + DeclaredIndexRuntimeCoverageState, PlannerStatsAvailability, PlannerStatsDeclaredIndexKind, + PlannerStatsDeclaredIndexTarget, +}; +use crate::segment_components::{ + component_id, decode_identity_header, decode_manifest_envelope, dependency_digest, + is_packed_core_component_kind, packed_core_container_record, secondary_declaration_dependency, + segment_source_groups_from_records, source_component_dependency, source_group_dependency, + validate_packed_core_manifest_contract_for_open, ComponentAvailability, ComponentDependencyV1, + ComponentFallbackClass, ComponentHandleV1, ComponentRequirement, ComponentTrustClass, + SegmentComponentKind, SegmentComponentManifestV1, SegmentComponentRecordV1, + SegmentComponentSourceGroups, SegmentSourceGroupKind, PACKED_CORE_FILENAME, + SEGMENT_COMPONENT_MANIFEST_FILENAME, }; use crate::segment_writer::{ - NODE_DENSE_VECTOR_BLOB_FILENAME, NODE_SPARSE_VECTOR_BLOB_FILENAME, NODE_VECTOR_META_ENTRY_SIZE, - NODE_VECTOR_META_FILENAME, SEGMENT_FORMAT_VERSION, SEGMENT_MAGIC, + component_fingerprint, dense_config_fingerprint, planner_stats_component_dependencies, + planner_stats_component_fingerprint, NODE_VECTOR_META_ENTRY_SIZE, SEGMENT_FORMAT_VERSION, }; use crate::sparse_postings::{ - read_sparse_posting_groups, validate_sparse_posting_files, SPARSE_POSTINGS_FILENAME, - SPARSE_POSTING_INDEX_FILENAME, + accumulate_sparse_posting_scores as accumulate_sparse_posting_scores_from_bytes, + validate_sparse_posting_files_for_open, validate_sparse_posting_index_shape_for_search, }; use crate::types::*; use memmap2::Mmap; @@ -28,16 +45,34 @@ use std::fs::File; use std::ops::ControlFlow; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::sync::{Mutex, OnceLock}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; /// Segment file data, either mmap'd or an empty placeholder. -/// Segment data files (e.g. adj_out.dat) can be 0 bytes when empty. +/// Segment component payloads can be 0 bytes when empty. /// memmap2 can't map empty files, so we handle that case explicitly. enum MappedData { - Mmap(Mmap), + Mmap { + mmap: Arc, + payload_offset: usize, + payload_len: usize, + }, Empty, } +#[derive(Clone, Copy)] +pub(crate) enum SegmentAdjacencyFile { + Out, + In, +} + +pub(crate) struct SegmentAdjPostingCursor { + file: SegmentAdjacencyFile, + cur_off: usize, + remaining: usize, + prev_edge_id: u64, +} + struct SecondaryEqSidecarCacheEntry { data: MappedData, validated: bool, @@ -50,16 +85,111 @@ struct SecondaryRangeSidecarCacheEntry { header_validated: bool, } +fn secondary_eq_component_kind( + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, +) -> SegmentComponentKind { + match target { + PlannerStatsDeclaredIndexTarget::NodeProperty => { + SegmentComponentKind::NodePropertyEqualityIndex { index_id } + } + PlannerStatsDeclaredIndexTarget::EdgeProperty => { + SegmentComponentKind::EdgePropertyEqualityIndex { index_id } + } + } +} + +fn secondary_range_component_kind( + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, +) -> SegmentComponentKind { + match target { + PlannerStatsDeclaredIndexTarget::NodeProperty => { + SegmentComponentKind::NodePropertyRangeIndex { index_id } + } + PlannerStatsDeclaredIndexTarget::EdgeProperty => { + SegmentComponentKind::EdgePropertyRangeIndex { index_id } + } + } +} + +struct SegmentComponentRegistry { + segment_id: u64, + records: HashMap, + availability: Mutex>, +} + +impl SegmentComponentRegistry { + fn new(manifest: &SegmentComponentManifestV1) -> Self { + Self { + segment_id: manifest.segment_id, + records: manifest + .components + .iter() + .map(|record| (record.kind.clone(), record.clone())) + .collect(), + availability: Mutex::new(HashMap::new()), + } + } + + fn record(&self, kind: &SegmentComponentKind) -> Option<&SegmentComponentRecordV1> { + self.records.get(kind) + } + + fn set_availability(&self, kind: SegmentComponentKind, state: ComponentAvailability) { + self.availability.lock().unwrap().insert(kind, state); + } + + fn availability(&self, kind: &SegmentComponentKind) -> ComponentAvailability { + self.availability + .lock() + .unwrap() + .get(kind) + .cloned() + .unwrap_or(ComponentAvailability::Missing) + } + + fn recorded_availability(&self, kind: &SegmentComponentKind) -> Option { + self.availability.lock().unwrap().get(kind).cloned() + } +} + +struct PackedCoreMapping { + component_id: crate::segment_components::ComponentDigest32, + mmap: Arc, + payload_offset: usize, + payload_len: usize, +} + +struct ComponentOpenContext { + packed_core: Option, + invalid_optional_packed_ranges: HashMap, +} + impl Deref for MappedData { type Target = [u8]; fn deref(&self) -> &[u8] { match self { - MappedData::Mmap(m) => m, + MappedData::Mmap { + mmap, + payload_offset, + payload_len, + } => &mmap[*payload_offset..*payload_offset + *payload_len], MappedData::Empty => &[], } } } +impl MappedData { + #[cfg(test)] + fn mapping_identity_for_test(&self) -> Option { + match self { + MappedData::Mmap { mmap, .. } => Some(Arc::as_ptr(mmap) as usize), + MappedData::Empty => None, + } + } +} + // --- Binary read helpers (little-endian, from byte slices) --- fn read_u16_at(data: &[u8], offset: usize) -> Result { @@ -161,6 +291,292 @@ fn read_f32_at(data: &[u8], offset: usize) -> Result { Ok(f32::from_le_bytes(slice.try_into().unwrap())) } +fn usize_from_u64(value: u64, context: &str) -> Result { + usize::try_from(value).map_err(|_| { + EngineError::CorruptRecord(format!("{context} does not fit in usize: {value}")) + }) +} + +fn checked_range_end(start: usize, len: usize, context: &str) -> Result { + start + .checked_add(len) + .ok_or_else(|| EngineError::CorruptRecord(format!("{context} range overflow"))) +} + +fn parse_node_meta_layout(data: &[u8]) -> Result, EngineError> { + if data.len() < 8 { + return Ok(None); + } + let node_count = usize_from_u64(read_u64_at(data, 0)?, "node metadata count")?; + if node_count == 0 { + return Ok(Some(NodeMetaLayout { + node_count, + fixed_entry_size: NODE_META_FIXED_ENTRY_SIZE, + label_offset_entry_size: NODE_META_LABEL_OFFSET_ENTRY_SIZE, + fixed_entries_offset: NODE_META_HEADER_SIZE, + label_offsets_offset: NODE_META_HEADER_SIZE, + label_ids_offset: NODE_META_HEADER_SIZE, + label_id_count: 0, + })); + } + if data.len() < NODE_META_HEADER_SIZE { + return Err(EngineError::CorruptRecord(format!( + "node metadata header length {} is shorter than {}", + data.len(), + NODE_META_HEADER_SIZE + ))); + } + + let fixed_entry_size = read_u16_at(data, 8)? as usize; + let label_offset_entry_size = read_u16_at(data, 10)? as usize; + if fixed_entry_size != NODE_META_FIXED_ENTRY_SIZE { + return Err(EngineError::CorruptRecord(format!( + "unsupported node metadata fixed entry size {}, expected {}", + fixed_entry_size, NODE_META_FIXED_ENTRY_SIZE + ))); + } + if label_offset_entry_size != NODE_META_LABEL_OFFSET_ENTRY_SIZE { + return Err(EngineError::CorruptRecord(format!( + "unsupported node metadata label offset entry size {}, expected {}", + label_offset_entry_size, NODE_META_LABEL_OFFSET_ENTRY_SIZE + ))); + } + + let fixed_entries_offset = + usize_from_u64(read_u64_at(data, 16)?, "node metadata fixed entries offset")?; + let label_offsets_offset = + usize_from_u64(read_u64_at(data, 24)?, "node metadata label offsets offset")?; + let label_ids_offset = + usize_from_u64(read_u64_at(data, 32)?, "node metadata label IDs offset")?; + let label_id_count = usize_from_u64(read_u64_at(data, 40)?, "node metadata label ID count")?; + + let fixed_bytes = node_count + .checked_mul(fixed_entry_size) + .ok_or_else(|| EngineError::CorruptRecord("node metadata fixed table overflow".into()))?; + let label_offset_entries = node_count.checked_add(1).ok_or_else(|| { + EngineError::CorruptRecord("node metadata label offset count overflow".into()) + })?; + let label_offset_bytes = label_offset_entries + .checked_mul(label_offset_entry_size) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata label offset table overflow".into()) + })?; + let label_id_bytes = label_id_count.checked_mul(4).ok_or_else(|| { + EngineError::CorruptRecord("node metadata label ID region overflow".into()) + })?; + + if checked_range_end( + fixed_entries_offset, + fixed_bytes, + "node metadata fixed table", + )? > data.len() + { + return Err(EngineError::CorruptRecord( + "node metadata fixed table exceeds payload length".into(), + )); + } + if checked_range_end( + label_offsets_offset, + label_offset_bytes, + "node metadata label offset table", + )? > data.len() + { + return Err(EngineError::CorruptRecord( + "node metadata label offset table exceeds payload length".into(), + )); + } + if checked_range_end( + label_ids_offset, + label_id_bytes, + "node metadata label ID region", + )? > data.len() + { + return Err(EngineError::CorruptRecord( + "node metadata label ID region exceeds payload length".into(), + )); + } + if label_id_count > node_count.saturating_mul(MAX_NODE_LABELS_PER_NODE) { + return Err(EngineError::CorruptRecord(format!( + "node metadata label ID count {} exceeds maximum {}", + label_id_count, + node_count.saturating_mul(MAX_NODE_LABELS_PER_NODE) + ))); + } + let first_label_offset = usize_from_u64( + read_u64_at(data, label_offsets_offset)?, + "node metadata first label offset", + )?; + if first_label_offset != 0 { + return Err(EngineError::CorruptRecord(format!( + "node metadata first label offset must be 0, got {}", + first_label_offset + ))); + } + let terminal_offset_pos = label_offsets_offset + .checked_add( + node_count + .checked_mul(label_offset_entry_size) + .ok_or_else(|| { + EngineError::CorruptRecord( + "node metadata terminal label offset overflow".into(), + ) + })?, + ) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata terminal label offset overflow".into()) + })?; + let terminal_label_offset = usize_from_u64( + read_u64_at(data, terminal_offset_pos)?, + "node metadata terminal label offset", + )?; + if terminal_label_offset != label_id_count { + return Err(EngineError::CorruptRecord(format!( + "node metadata terminal label offset {} does not match label ID count {}", + terminal_label_offset, label_id_count + ))); + } + + Ok(Some(NodeMetaLayout { + node_count, + fixed_entry_size, + label_offset_entry_size, + fixed_entries_offset, + label_offsets_offset, + label_ids_offset, + label_id_count, + })) +} + +fn read_node_meta_entry_at( + data: &[u8], + layout: NodeMetaLayout, + index: usize, +) -> Result { + if index >= layout.node_count { + return Err(EngineError::CorruptRecord(format!( + "node metadata index {} out of bounds for count {}", + index, layout.node_count + ))); + } + let off = layout + .fixed_entries_offset + .checked_add(index.checked_mul(layout.fixed_entry_size).ok_or_else(|| { + EngineError::CorruptRecord("node metadata fixed entry offset overflow".into()) + })?) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata fixed entry offset overflow".into()) + })?; + let node_id = read_u64_at(data, off)?; + let data_offset = read_u64_at(data, off + 8)?; + let data_len = read_u32_at(data, off + 16)?; + let updated_at = read_i64_at(data, off + 20)?; + let weight = read_f32_at(data, off + 28)?; + let key_len = read_u16_at(data, off + 32)?; + let last_write_seq = read_u64_at(data, off + 34)?; + + let label_offset_pos = layout + .label_offsets_offset + .checked_add( + index + .checked_mul(layout.label_offset_entry_size) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata label offset entry overflow".into()) + })?, + ) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata label offset entry overflow".into()) + })?; + let label_start = usize_from_u64( + read_u64_at(data, label_offset_pos)?, + "node metadata label start offset", + )?; + let label_end = usize_from_u64( + read_u64_at(data, label_offset_pos + layout.label_offset_entry_size)?, + "node metadata label end offset", + )?; + if label_start > label_end { + return Err(EngineError::CorruptRecord(format!( + "node metadata label offsets are not monotonic at row {}: {} > {}", + index, label_start, label_end + ))); + } + if label_end > layout.label_id_count { + return Err(EngineError::CorruptRecord(format!( + "node metadata label end {} exceeds label ID count {}", + label_end, layout.label_id_count + ))); + } + let label_count = label_end - label_start; + if label_count == 0 || label_count > MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::CorruptRecord(format!( + "node metadata row {} has invalid label count {}", + index, label_count + ))); + } + + let mut label_ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + for label_index in 0..label_count { + let label_pos = layout + .label_ids_offset + .checked_add((label_start + label_index).checked_mul(4).ok_or_else(|| { + EngineError::CorruptRecord("node metadata label ID offset overflow".into()) + })?) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata label ID offset overflow".into()) + })?; + label_ids[label_index] = read_u32_at(data, label_pos)?; + if label_index > 0 && label_ids[label_index - 1] >= label_ids[label_index] { + return Err(EngineError::CorruptRecord( + "node metadata label IDs must be sorted ascending and unique".into(), + )); + } + } + let label_ids = NodeLabelSet::from_canonical_ids(&label_ids[..label_count]).map_err(|err| { + EngineError::CorruptRecord(format!("invalid node metadata label set: {err}")) + })?; + + Ok(SegmentNodeMeta { + node_id, + data_offset, + data_len, + label_ids, + updated_at, + weight, + key_len, + last_write_seq, + }) +} + +fn binary_search_node_meta_index( + data: &[u8], + layout: NodeMetaLayout, + target_id: u64, +) -> Result, EngineError> { + let mut lo = 0usize; + let mut hi = layout.node_count; + + while lo < hi { + let mid = lo + (hi - lo) / 2; + let entry_off = layout + .fixed_entries_offset + .checked_add(mid.checked_mul(layout.fixed_entry_size).ok_or_else(|| { + EngineError::CorruptRecord("node metadata fixed entry offset overflow".into()) + })?) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata fixed entry offset overflow".into()) + })?; + let id = read_u64_at(data, entry_off)?; + if id < target_id { + lo = mid + 1; + } else if id > target_id { + hi = mid; + } else { + return Ok(Some(mid)); + } + } + Ok(None) +} + /// Read a LEB128 varint from data starting at `offset`. /// Returns (value, bytes_consumed). fn read_varint_at(data: &[u8], offset: usize) -> Result<(u64, usize), EngineError> { @@ -188,6 +604,13 @@ fn read_varint_at(data: &[u8], offset: usize) -> Result<(u64, usize), EngineErro } } +#[inline] +fn checked_adj_edge_id_delta(prev_edge_id: u64, delta: u64) -> Result { + prev_edge_id + .checked_add(delta) + .ok_or_else(|| EngineError::CorruptRecord("adjacency edge id delta overflow".into())) +} + /// Safe byte slice extraction with bounds checking. fn read_bytes_at(data: &[u8], offset: usize, len: usize) -> Result<&[u8], EngineError> { let end = offset @@ -213,12 +636,11 @@ const BATCH_RANDOM_ACCESS_PENALTY: usize = 4; const NODE_INDEX_ENTRY_SIZE: usize = 16; // node_id (8) + offset (8) const EDGE_INDEX_ENTRY_SIZE: usize = 16; // edge_id (8) + offset (8) -const ADJ_INDEX_ENTRY_SIZE: usize = 24; // node_id (8) + type_id (4) + offset (8) + count (4) +const ADJ_INDEX_ENTRY_SIZE: usize = 24; // node_id (8) + label_id (4) + offset (8) + count (4) // ADJ_POSTING_SIZE removed. Postings are now variable-length (delta + varint encoded) const TOMBSTONE_ENTRY_SIZE: usize = 25; // kind (1) + id (8) + deleted_at (8) + last_write_seq (8) -const TYPE_INDEX_ENTRY_SIZE: usize = 16; // type_id (4) + offset (8) + count (4) -const PROP_INDEX_ENTRY_SIZE: usize = 32; // type_id (4) + key_hash (8) + value_hash (8) + offset (8) + count (4) -const EDGE_TRIPLE_ENTRY_SIZE: usize = 28; // from (8) + to (8) + type_id (4) + edge_id (8) +const LABEL_POSTING_INDEX_ENTRY_SIZE: usize = 16; // label_id (4) + offset (8) + count (4) +const EDGE_TRIPLE_ENTRY_SIZE: usize = 28; // from (8) + to (8) + label_id (4) + edge_id (8) const SECONDARY_EQ_ENTRY_SIZE: usize = 20; // value_hash (8) + offset (8) + count (4) const SECONDARY_RANGE_ENTRY_SIZE: usize = 16; // encoded_value (8) + node_id (8) @@ -289,8 +711,10 @@ fn upper_bound_u64_index( /// Choose between per-key binary seek and merge-walk using a lightweight /// shared cost model reused across batch index readers. +#[allow(clippy::too_many_arguments)] fn choose_batch_read_strategy( index_data: &[u8], + idx_start: usize, index_count: usize, entry_size: usize, key_offset: usize, @@ -302,7 +726,6 @@ fn choose_batch_read_strategy( return Ok(BatchReadStrategy::SeekPerKey); } - let idx_start = 8; let span_start = lower_bound_u64_index( index_data, idx_start, @@ -333,13 +756,13 @@ fn choose_batch_read_strategy( } /// Lower bound for the key index (variable-length entries addressed via offset -/// table). Returns the first entry index where `(type_id, key) >= (target_type, +/// table). Returns the first entry index where `(label_id, key) >= (target_label, /// target_key)`, in [0, count]. fn lower_bound_key_index( data: &[u8], offset_table_start: usize, count: usize, - target_type: u32, + target_label: u32, target_key: &str, ) -> Result { let mut lo = 0usize; @@ -347,7 +770,7 @@ fn lower_bound_key_index( while lo < hi { let mid = lo + (hi - lo) / 2; let entry_offset = read_u64_at(data, offset_table_start + mid * 8)? as usize; - let entry_type = read_u32_at(data, entry_offset)?; + let entry_label_id = read_u32_at(data, entry_offset)?; let key_len = read_u16_at(data, entry_offset + 12)? as usize; let key_bytes = read_bytes_at(data, entry_offset + 14, key_len)?; let entry_key = std::str::from_utf8(key_bytes).map_err(|_| { @@ -356,7 +779,7 @@ fn lower_bound_key_index( entry_offset + 14 )) })?; - if (entry_type, entry_key) < (target_type, target_key) { + if (entry_label_id, entry_key) < (target_label, target_key) { lo = mid + 1; } else { hi = mid; @@ -366,12 +789,12 @@ fn lower_bound_key_index( } /// Upper bound for the key index. Returns the first entry index where -/// `(type_id, key) > (target_type, target_key)`, in [0, count]. +/// `(label_id, key) > (target_label, target_key)`, in [0, count]. fn upper_bound_key_index( data: &[u8], offset_table_start: usize, count: usize, - target_type: u32, + target_label: u32, target_key: &str, ) -> Result { let mut lo = 0usize; @@ -379,7 +802,7 @@ fn upper_bound_key_index( while lo < hi { let mid = lo + (hi - lo) / 2; let entry_offset = read_u64_at(data, offset_table_start + mid * 8)? as usize; - let entry_type = read_u32_at(data, entry_offset)?; + let entry_label_id = read_u32_at(data, entry_offset)?; let key_len = read_u16_at(data, entry_offset + 12)? as usize; let key_bytes = read_bytes_at(data, entry_offset + 14, key_len)?; let entry_key = std::str::from_utf8(key_bytes).map_err(|_| { @@ -388,7 +811,58 @@ fn upper_bound_key_index( entry_offset + 14 )) })?; - if (entry_type, entry_key) <= (target_type, target_key) { + if (entry_label_id, entry_key) <= (target_label, target_key) { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) +} + +fn read_edge_triple_key_at( + data: &[u8], + entries_start: usize, + index: usize, +) -> Result<(u64, u64, u32), EngineError> { + let off = entries_start + index * EDGE_TRIPLE_ENTRY_SIZE; + Ok(( + read_u64_at(data, off)?, + read_u64_at(data, off + 8)?, + read_u32_at(data, off + 16)?, + )) +} + +fn lower_bound_edge_triple_index( + data: &[u8], + entries_start: usize, + count: usize, + target: (u64, u64, u32), +) -> Result { + let mut lo = 0usize; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + if read_edge_triple_key_at(data, entries_start, mid)? < target { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) +} + +fn upper_bound_edge_triple_index( + data: &[u8], + entries_start: usize, + count: usize, + target: (u64, u64, u32), +) -> Result { + let mut lo = 0usize; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + if read_edge_triple_key_at(data, entries_start, mid)? <= target { lo = mid + 1; } else { hi = mid; @@ -397,81 +871,303 @@ fn upper_bound_key_index( Ok(lo) } +fn binary_search_edge_triple_index( + data: &[u8], + entries_start: usize, + count: usize, + target: (u64, u64, u32), +) -> Result, EngineError> { + let pos = lower_bound_edge_triple_index(data, entries_start, count, target)?; + if pos >= count || read_edge_triple_key_at(data, entries_start, pos)? != target { + return Ok(None); + } + Ok(Some(read_u64_at( + data, + entries_start + pos * EDGE_TRIPLE_ENTRY_SIZE + 20, + )?)) +} + /// An mmap-backed reader for an immutable segment directory. /// /// Provides O(log N) lookups by ID for nodes and edges, adjacency queries /// from pre-built indexes, key-based lookups, and tombstone checks. -/// Size of a node metadata entry in node_meta.dat (60 bytes, v9). -const NODE_META_ENTRY_SIZE: usize = 60; -/// Size of an edge metadata entry in edge_meta.dat (80 bytes, v9). +/// Size of a node metadata payload entry (60 bytes, v9). +const NODE_META_HEADER_SIZE: usize = 48; +const NODE_META_FIXED_ENTRY_SIZE: usize = 48; +const NODE_META_LABEL_OFFSET_ENTRY_SIZE: usize = 8; + +#[derive(Clone, Copy)] +struct NodeMetaLayout { + node_count: usize, + fixed_entry_size: usize, + label_offset_entry_size: usize, + fixed_entries_offset: usize, + label_offsets_offset: usize, + label_ids_offset: usize, + label_id_count: usize, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct SegmentNodeMeta { + pub(crate) node_id: u64, + pub(crate) data_offset: u64, + pub(crate) data_len: u32, + pub(crate) label_ids: NodeLabelSet, + pub(crate) updated_at: i64, + pub(crate) weight: f32, + pub(crate) key_len: u16, + pub(crate) last_write_seq: u64, +} +/// Size of an edge metadata payload entry (80 bytes, v9). const EDGE_META_ENTRY_SIZE: usize = 80; const NODE_VECTOR_FLAG_DENSE: u8 = 0b0000_0001; const NODE_VECTOR_FLAG_SPARSE: u8 = 0b0000_0010; const DENSE_VECTOR_VALUE_SIZE: usize = 4; const SPARSE_VECTOR_ENTRY_SIZE: usize = 8; -#[derive(Clone, Copy)] -struct DenseScoringMeta { - type_id: u32, - updated_at: i64, - weight: f32, - dense_offset: usize, - dense_len: usize, +fn edge_metadata_entry_count( + data: &[u8], + entry_size: usize, + logical_name: &str, +) -> Result { + if data.len() < 8 { + return Err(EngineError::CorruptRecord(format!( + "{logical_name} missing or truncated count header" + ))); + } + let count = usize::try_from(read_u64_at(data, 0)?).map_err(|_| { + EngineError::CorruptRecord(format!("{logical_name} count exceeds addressable memory")) + })?; + let expected_len = 8usize + .checked_add(count.checked_mul(entry_size).ok_or_else(|| { + EngineError::CorruptRecord(format!("{logical_name} entry count overflow")) + })?) + .ok_or_else(|| EngineError::CorruptRecord(format!("{logical_name} length overflow")))?; + if expected_len != data.len() { + return Err(EngineError::CorruptRecord(format!( + "{logical_name} length {} does not match count {} and entry size {}", + data.len(), + count, + entry_size + ))); + } + Ok(count) } -#[derive(Clone, Copy)] -struct SparseScoringMeta { - type_id: u32, - updated_at: i64, - weight: f32, - sparse_offset: usize, - sparse_len: usize, +#[cfg(test)] +fn validate_edge_weight_index_data(data: &[u8]) -> Result { + let count = edge_metadata_entry_count( + data, + EDGE_WEIGHT_INDEX_ENTRY_SIZE, + EDGE_WEIGHT_INDEX_LOGICAL_NAME, + )?; + let mut previous = None; + for index in 0..count { + let offset = 8 + index * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + let entry = ( + read_u32_at(data, offset)?, + read_u32_at(data, offset + 4)?, + read_u64_at(data, offset + 8)?, + ); + if previous.is_some_and(|prev| prev > entry) { + return Err(EngineError::CorruptRecord(format!( + "{EDGE_WEIGHT_INDEX_LOGICAL_NAME} is not sorted" + ))); + } + previous = Some(entry); + } + Ok(count) } -pub struct SegmentReader { - pub segment_id: u64, - seg_dir: PathBuf, - nodes_mmap: MappedData, - edges_mmap: MappedData, - adj_out_idx: MappedData, - adj_out_dat: MappedData, - adj_in_idx: MappedData, - adj_in_dat: MappedData, - key_index_mmap: MappedData, - node_type_index_mmap: MappedData, - edge_type_index_mmap: MappedData, - prop_node_index_mmap: MappedData, - edge_triple_index_mmap: MappedData, - // V5 metadata sidecars - node_meta_mmap: MappedData, - edge_meta_mmap: MappedData, - node_prop_hashes_mmap: MappedData, - node_vector_meta_mmap: MappedData, - node_dense_vectors_mmap: MappedData, - node_sparse_vectors_mmap: MappedData, - dense_hnsw_meta_mmap: MappedData, - dense_hnsw_graph_mmap: MappedData, - dense_hnsw_header: Option, - dense_hnsw_points: Vec, +#[cfg(test)] +fn validate_edge_i64_metadata_index_data( + data: &[u8], + logical_name: &str, +) -> Result { + let count = edge_metadata_entry_count(data, EDGE_I64_METADATA_INDEX_ENTRY_SIZE, logical_name)?; + let mut previous = None; + for index in 0..count { + let offset = 8 + index * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + let entry = ( + read_u32_at(data, offset)?, + read_i64_at(data, offset + 4)?, + read_u64_at(data, offset + 12)?, + ); + if previous.is_some_and(|prev| prev > entry) { + return Err(EngineError::CorruptRecord(format!( + "{logical_name} is not sorted" + ))); + } + previous = Some(entry); + } + Ok(count) +} + +fn mark_optional_component_corrupt( + registry: &SegmentComponentRegistry, + kind: SegmentComponentKind, + reason: String, +) { + if !matches!( + registry.availability(&kind), + ComponentAvailability::Missing + | ComponentAvailability::Incompatible { .. } + | ComponentAvailability::Unsupported { .. } + ) { + registry.set_availability(kind, ComponentAvailability::CorruptIdentity { reason }); + } +} + +fn edge_metadata_index_count_from_header( + registry: &SegmentComponentRegistry, + kind: SegmentComponentKind, + mmap: &MappedData, + entry_size: usize, + logical_name: &str, +) -> Option { + let data = &mmap[..]; + if data.is_empty() { + if registry.availability(&kind).is_available() { + mark_optional_component_corrupt( + registry, + kind, + format!("{logical_name} missing or truncated count header"), + ); + } + return None; + } + match edge_metadata_entry_count(data, entry_size, logical_name) { + Ok(count) => Some(count), + Err(error) => { + mark_optional_component_corrupt(registry, kind, error.to_string()); + None + } + } +} + +fn edge_i64_metadata_component_kind(logical_name: &str) -> Option { + match logical_name { + EDGE_UPDATED_AT_INDEX_LOGICAL_NAME => Some(SegmentComponentKind::EdgeUpdatedAtIndex), + EDGE_VALID_FROM_INDEX_LOGICAL_NAME => Some(SegmentComponentKind::EdgeValidFromIndex), + EDGE_VALID_TO_INDEX_LOGICAL_NAME => Some(SegmentComponentKind::EdgeValidToIndex), + _ => None, + } +} + +fn key_matches_bounds( + value: T, + lower: Option, + lower_inclusive: bool, + upper: Option, + upper_inclusive: bool, +) -> bool { + if let Some(lower) = lower { + if lower_inclusive { + if value < lower { + return false; + } + } else if value <= lower { + return false; + } + } + if let Some(upper) = upper { + if upper_inclusive { + if value > upper { + return false; + } + } else if value >= upper { + return false; + } + } + true +} + +#[derive(Clone, Copy)] +struct DenseScoringMeta { + label_ids: NodeLabelSet, + updated_at: i64, + weight: f32, + dense_offset: usize, + dense_len: usize, +} + +#[derive(Clone, Copy)] +struct SparseScoringMeta { + label_ids: NodeLabelSet, + updated_at: i64, + weight: f32, + sparse_offset: usize, + sparse_len: usize, +} + +pub struct SegmentReader { + pub segment_id: u64, + seg_dir: PathBuf, + segment_data_id: [u8; 32], + component_manifest_generation: u64, + component_registry: SegmentComponentRegistry, + nodes_mmap: MappedData, + edges_mmap: MappedData, + adj_out_idx: MappedData, + adj_out_dat: MappedData, + adj_in_idx: MappedData, + adj_in_dat: MappedData, + key_index_mmap: MappedData, + node_label_index_mmap: MappedData, + edge_label_index_mmap: MappedData, + edge_triple_index_mmap: MappedData, + edge_weight_index_mmap: MappedData, + edge_weight_index_count: Option, + edge_updated_at_index_mmap: MappedData, + edge_updated_at_index_count: Option, + edge_valid_from_index_mmap: MappedData, + edge_valid_from_index_count: Option, + edge_valid_to_index_mmap: MappedData, + edge_valid_to_index_count: Option, + // Metadata payloads + node_meta_mmap: MappedData, + edge_meta_mmap: MappedData, + node_vector_meta_mmap: MappedData, + node_dense_vectors_mmap: MappedData, + node_sparse_vectors_mmap: MappedData, + dense_hnsw_meta_mmap: MappedData, + dense_hnsw_graph_mmap: MappedData, + dense_hnsw_header: Option, + dense_hnsw_available: AtomicBool, + dense_hnsw_points: OnceLock, String>>, + dense_vector_count: usize, + sparse_vector_count: usize, sparse_posting_index_mmap: MappedData, sparse_postings_mmap: MappedData, + sparse_postings_available: AtomicBool, + sparse_posting_index_shape: OnceLock>, degree_delta: Option, planner_stats: PlannerStatsAvailability, // Timestamp range index timestamp_index_mmap: MappedData, deleted_nodes: NodeIdMap, deleted_edges: NodeIdMap, - secondary_eq_sidecars: Mutex>, - secondary_range_sidecars: Mutex>, - declared_index_runtime_coverage: - Mutex>, + secondary_eq_sidecars: + Mutex>, + secondary_range_sidecars: + Mutex>, + declared_index_runtime_coverage: Mutex< + HashMap< + ( + u64, + PlannerStatsDeclaredIndexTarget, + PlannerStatsDeclaredIndexKind, + ), + DeclaredIndexRuntimeCoverageState, + >, + >, node_ids: OnceLock>, node_count: u64, edge_count: u64, } #[derive(Clone, Copy)] -pub(crate) struct SegmentTypePosting { +pub(crate) struct SegmentLabelPosting { offset: usize, count: usize, } @@ -483,48 +1179,233 @@ pub(crate) struct SecondaryEqPostingChunk { } impl SegmentReader { - /// Open a segment directory and mmap all files. - /// Validates the format version file. - pub fn open( + /// Test-only unpinned open helper. + /// + /// Production callers must pass the root manifest's `SegmentInfo` through + /// `open_with_info` so the local segment manifest is pinned to the DB + /// manifest. + #[cfg(test)] + pub(crate) fn open_unpinned_for_test( seg_dir: &Path, segment_id: u64, dense_config: Option<&DenseVectorConfig>, ) -> Result { - let format_version = read_format_version(seg_dir)?; - let vector_meta_path = seg_dir.join(NODE_VECTOR_META_FILENAME); - let dense_blob_path = seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME); - let sparse_blob_path = seg_dir.join(NODE_SPARSE_VECTOR_BLOB_FILENAME); - let dense_hnsw_meta_path = seg_dir.join(DENSE_HNSW_META_FILENAME); - let dense_hnsw_graph_path = seg_dir.join(DENSE_HNSW_GRAPH_FILENAME); - let sparse_posting_index_path = seg_dir.join(SPARSE_POSTING_INDEX_FILENAME); - let sparse_postings_path = seg_dir.join(SPARSE_POSTINGS_FILENAME); - let degree_delta_path = seg_dir.join(DEGREE_DELTA_FILENAME); - let nodes_mmap = mmap_file(&seg_dir.join("nodes.dat"))?; - let edges_mmap = mmap_file(&seg_dir.join("edges.dat"))?; - let adj_out_idx = mmap_file(&seg_dir.join("adj_out.idx"))?; - let adj_out_dat = mmap_file(&seg_dir.join("adj_out.dat"))?; - let adj_in_idx = mmap_file(&seg_dir.join("adj_in.idx"))?; - let adj_in_dat = mmap_file(&seg_dir.join("adj_in.dat"))?; - let key_index_mmap = mmap_file(&seg_dir.join("key_index.dat"))?; - let node_type_index_mmap = mmap_file_optional(&seg_dir.join("node_type_index.dat"))?; - let edge_type_index_mmap = mmap_file_optional(&seg_dir.join("edge_type_index.dat"))?; - let prop_node_index_mmap = mmap_file_optional(&seg_dir.join("prop_index.dat"))?; - let edge_triple_index_mmap = mmap_file_optional(&seg_dir.join("edge_triple_index.dat"))?; - // V5 metadata sidecars - let node_meta_mmap = mmap_file(&seg_dir.join("node_meta.dat"))?; - let edge_meta_mmap = mmap_file(&seg_dir.join("edge_meta.dat"))?; - let node_prop_hashes_mmap = mmap_file_optional(&seg_dir.join("node_prop_hashes.dat"))?; - let node_vector_meta_mmap = mmap_file_optional(&vector_meta_path)?; - let node_dense_vectors_mmap = mmap_file_optional(&dense_blob_path)?; - let node_sparse_vectors_mmap = mmap_file_optional(&sparse_blob_path)?; - let dense_hnsw_meta_mmap = mmap_file_optional(&dense_hnsw_meta_path)?; - let dense_hnsw_graph_mmap = mmap_file_optional(&dense_hnsw_graph_path)?; - let sparse_posting_index_mmap = mmap_file_optional(&sparse_posting_index_path)?; - let sparse_postings_mmap = mmap_file_optional(&sparse_postings_path)?; - let degree_delta = DegreeSidecar::open_optional(°ree_delta_path); - let timestamp_index_mmap = mmap_file(&seg_dir.join("timestamp_index.dat"))?; - - let (deleted_nodes, deleted_edges) = load_tombstones(&seg_dir.join("tombstones.dat"))?; + let component_manifest = load_component_manifest(seg_dir, segment_id)?; + let segment_info = SegmentInfo { + id: segment_id, + node_count: component_manifest.node_count, + edge_count: component_manifest.edge_count, + segment_format_version: component_manifest.segment_format_version, + segment_data_id: component_manifest.segment_data_id, + }; + Self::open_with_info(seg_dir, &segment_info, dense_config, &[]) + } + + pub(crate) fn open_with_info( + seg_dir: &Path, + segment_info: &SegmentInfo, + dense_config: Option<&DenseVectorConfig>, + secondary_indexes: &[SecondaryIndexManifestEntry], + ) -> Result { + let component_manifest = read_component_manifest(seg_dir)?; + let source_groups = validate_segment_manifest_identity(segment_info, &component_manifest)?; + let component_registry = SegmentComponentRegistry::new(&component_manifest); + validate_manifest_component_contracts( + &component_registry, + &source_groups, + dense_config, + secondary_indexes, + )?; + let component_open_context = + ComponentOpenContext::open(seg_dir, segment_info.id, &component_manifest)?; + warm_edge_property_sidecar_availability( + &component_registry, + &component_open_context, + seg_dir, + secondary_indexes, + ); + let nodes_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::NodeRecords, + )?; + let edges_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::EdgeRecords, + )?; + let adj_out_idx = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::AdjOutIndex, + )?; + let adj_out_dat = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::AdjOutPostings, + )?; + let adj_in_idx = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::AdjInIndex, + )?; + let adj_in_dat = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::AdjInPostings, + )?; + let key_index_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::KeyIndex, + )?; + let node_label_index_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::NodeLabelIndex, + )?; + let edge_label_index_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::EdgeLabelIndex, + )?; + let edge_triple_index_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::EdgeTripleIndex, + )?; + // Metadata payloads. + let node_meta_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::NodeMetadata, + )?; + let edge_meta_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::EdgeMetadata, + )?; + let edge_weight_index_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::EdgeWeightIndex, + )?; + let edge_weight_index_count = edge_metadata_index_count_from_header( + &component_registry, + SegmentComponentKind::EdgeWeightIndex, + &edge_weight_index_mmap, + EDGE_WEIGHT_INDEX_ENTRY_SIZE, + EDGE_WEIGHT_INDEX_LOGICAL_NAME, + ); + let edge_updated_at_index_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::EdgeUpdatedAtIndex, + )?; + let edge_updated_at_index_count = edge_metadata_index_count_from_header( + &component_registry, + SegmentComponentKind::EdgeUpdatedAtIndex, + &edge_updated_at_index_mmap, + EDGE_I64_METADATA_INDEX_ENTRY_SIZE, + EDGE_UPDATED_AT_INDEX_LOGICAL_NAME, + ); + let edge_valid_from_index_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::EdgeValidFromIndex, + )?; + let edge_valid_from_index_count = edge_metadata_index_count_from_header( + &component_registry, + SegmentComponentKind::EdgeValidFromIndex, + &edge_valid_from_index_mmap, + EDGE_I64_METADATA_INDEX_ENTRY_SIZE, + EDGE_VALID_FROM_INDEX_LOGICAL_NAME, + ); + let edge_valid_to_index_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::EdgeValidToIndex, + )?; + let edge_valid_to_index_count = edge_metadata_index_count_from_header( + &component_registry, + SegmentComponentKind::EdgeValidToIndex, + &edge_valid_to_index_mmap, + EDGE_I64_METADATA_INDEX_ENTRY_SIZE, + EDGE_VALID_TO_INDEX_LOGICAL_NAME, + ); + let node_vector_meta_mmap = open_manifested_required_payload_or_empty( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::NodeVectorMetadata, + )?; + let node_dense_vectors_mmap = open_manifested_required_payload_or_empty( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::NodeDenseVectorBlob, + )?; + let node_sparse_vectors_mmap = open_manifested_required_payload_or_empty( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::NodeSparseVectorBlob, + )?; + let dense_hnsw_meta_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::DenseHnswMetadata, + )?; + let dense_hnsw_graph_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::DenseHnswGraph, + )?; + let mut sparse_posting_index_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::SparsePostingIndex, + )?; + let mut sparse_postings_mmap = open_optional_manifest_payload( + &component_registry, + Some(&component_open_context), + seg_dir, + SegmentComponentKind::SparsePostings, + )?; + let degree_delta = open_degree_delta_sidecar(&component_registry, seg_dir); + let timestamp_index_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::TimestampIndex, + )?; + + let tombstones_mmap = open_required_manifest_payload( + &component_registry, + &component_open_context, + seg_dir, + SegmentComponentKind::Tombstones, + )?; + let (deleted_nodes, deleted_edges) = load_tombstones_from_bytes(&tombstones_mmap)?; let node_count = if nodes_mmap.len() >= 8 { read_u64_at(&nodes_mmap, 0)? @@ -536,83 +1417,76 @@ impl SegmentReader { } else { 0 }; - let planner_stats = read_planner_stats_sidecar(seg_dir, segment_id, node_count, edge_count); + let planner_stats = open_planner_stats( + &component_registry, + seg_dir, + segment_info.id, + node_count, + edge_count, + ); let node_meta_count = if node_meta_mmap.len() >= 8 { read_u64_at(&node_meta_mmap, 0)? } else { 0 }; - if format_version < 6 - && (!node_vector_meta_mmap.is_empty() - || !node_dense_vectors_mmap.is_empty() - || !node_sparse_vectors_mmap.is_empty()) - { - return Err(EngineError::CorruptRecord(format!( - "segment {} has unexpected vector sidecars for format version {}", - segment_id, format_version - ))); - } - if format_version < 7 - && (!dense_hnsw_meta_mmap.is_empty() || !dense_hnsw_graph_mmap.is_empty()) - { - return Err(EngineError::CorruptRecord(format!( - "segment {} has unexpected dense HNSW files for format version {}", - segment_id, format_version - ))); - } - if format_version < 8 - && (!sparse_posting_index_mmap.is_empty() || !sparse_postings_mmap.is_empty()) - { - return Err(EngineError::CorruptRecord(format!( - "segment {} has unexpected sparse posting files for format version {}", - segment_id, format_version - ))); - } - let vector_summary = validate_node_vector_sidecars( - segment_id, + segment_info.id, &node_vector_meta_mmap, &node_dense_vectors_mmap, &node_sparse_vectors_mmap, node_meta_count, )?; - let dense_hnsw_header = validate_dense_hnsw_files( + let dense_hnsw_validation = validate_dense_hnsw_files_for_open( &dense_hnsw_meta_mmap, &dense_hnsw_graph_mmap, node_dense_vectors_mmap.len(), vector_summary.dense_count, dense_config, - )?; - let dense_hnsw_points = if let Some(header) = dense_hnsw_header { - load_dense_hnsw_query_points(&dense_hnsw_meta_mmap, header)? - } else { - Vec::new() + ); + let dense_hnsw_header = match dense_hnsw_validation { + Ok(header) => header, + Err(error) => { + mark_optional_components_corrupt( + &component_registry, + &[ + SegmentComponentKind::DenseHnswMetadata, + SegmentComponentKind::DenseHnswGraph, + ], + error.to_string(), + ); + None + } }; - if format_version < 8 && vector_summary.sparse_count > 0 { - return Err(EngineError::CorruptRecord(format!( - "segment {} format version {} predates sparse posting support; sparse-bearing segments must be rewritten as v8", - segment_id, format_version - ))); - } - validate_sparse_posting_files( + if validate_sparse_posting_files_for_open( &sparse_posting_index_mmap, &sparse_postings_mmap, vector_summary.sparse_count, - true, - )?; - validate_sparse_posting_parity( - segment_id, - &node_meta_mmap, - &node_vector_meta_mmap, - &node_sparse_vectors_mmap, - &sparse_posting_index_mmap, - &sparse_postings_mmap, - )?; + ) + .map_err(|error| { + mark_optional_components_corrupt( + &component_registry, + &[ + SegmentComponentKind::SparsePostingIndex, + SegmentComponentKind::SparsePostings, + ], + error.to_string(), + ); + }) + .is_err() + { + sparse_posting_index_mmap = MappedData::Empty; + sparse_postings_mmap = MappedData::Empty; + } + let sparse_postings_available = + !sparse_posting_index_mmap.is_empty() && !sparse_postings_mmap.is_empty(); Ok(SegmentReader { - segment_id, + segment_id: segment_info.id, seg_dir: seg_dir.to_path_buf(), + segment_data_id: segment_info.segment_data_id, + component_manifest_generation: component_manifest.generation, + component_registry, nodes_mmap, edges_mmap, adj_out_idx, @@ -620,22 +1494,33 @@ impl SegmentReader { adj_in_idx, adj_in_dat, key_index_mmap, - node_type_index_mmap, - edge_type_index_mmap, - prop_node_index_mmap, + node_label_index_mmap, + edge_label_index_mmap, edge_triple_index_mmap, + edge_weight_index_mmap, + edge_weight_index_count, + edge_updated_at_index_mmap, + edge_updated_at_index_count, + edge_valid_from_index_mmap, + edge_valid_from_index_count, + edge_valid_to_index_mmap, + edge_valid_to_index_count, node_meta_mmap, edge_meta_mmap, - node_prop_hashes_mmap, node_vector_meta_mmap, node_dense_vectors_mmap, node_sparse_vectors_mmap, dense_hnsw_meta_mmap, dense_hnsw_graph_mmap, dense_hnsw_header, - dense_hnsw_points, + dense_hnsw_available: AtomicBool::new(dense_hnsw_header.is_some()), + dense_hnsw_points: OnceLock::new(), + dense_vector_count: vector_summary.dense_count, + sparse_vector_count: vector_summary.sparse_count, sparse_posting_index_mmap, sparse_postings_mmap, + sparse_postings_available: AtomicBool::new(sparse_postings_available), + sparse_posting_index_shape: OnceLock::new(), degree_delta, planner_stats, timestamp_index_mmap, @@ -662,9 +1547,8 @@ impl SegmentReader { }; let mut node = decode_node_at(&self.nodes_mmap, offset, id)?; self.hydrate_node_vectors(index, &mut node)?; - // Hydrate last_write_seq from metadata sidecar - let (_, _, _, _, _, _, _, _, _, last_write_seq) = self.node_meta_at(index)?; - node.last_write_seq = last_write_seq; + // Hydrate last_write_seq from metadata. + node.last_write_seq = self.node_meta_at(index)?.last_write_seq; Ok(Some(node)) } @@ -679,7 +1563,7 @@ impl SegmentReader { None => return Ok(None), }; let mut edge = decode_edge_at(&self.edges_mmap, offset, id)?; - // Hydrate last_write_seq from metadata sidecar + // Hydrate last_write_seq from metadata. let (_, _, _, _, _, _, _, _, _, _, last_write_seq) = self.edge_meta_at(index)?; edge.last_write_seq = last_write_seq; Ok(Some(edge)) @@ -713,61 +1597,46 @@ impl SegmentReader { ))) } - /// Look up a node by (type_id, key). Returns None if not found or tombstoned. - pub fn node_by_key(&self, type_id: u32, key: &str) -> Result, EngineError> { - let node_id = match self.binary_search_key_index(type_id, key)? { + fn edge_metadata_at_index(&self, index: usize) -> Result { + let ( + edge_id, + _data_offset, + _data_len, + from, + to, + label_id, + updated_at, + weight, + valid_from, + valid_to, + _last_write_seq, + ) = self.edge_meta_at(index)?; + Ok(EdgeMetadataCandidate { + edge_id, + from, + to, + label_id, + updated_at, + weight, + valid_from, + valid_to, + }) + } + + /// Look up a node by (label_id, key). Returns None if not found or tombstoned. + #[cfg(test)] + pub fn node_by_key(&self, label_id: u32, key: &str) -> Result, EngineError> { + let node_id = match self.binary_search_key_index(label_id, key)? { Some(id) => id, None => return Ok(None), }; self.get_node(node_id) } - /// Batch resolve (type_id, key) pairs to node records using the key index. - /// - /// `lookups` must be sorted by `(type_id, key)`. Each entry is - /// `(orig_idx, type_id, key)`. Found (non-tombstoned) records are written - /// into `results[orig_idx]`. Returns the set of `orig_idx` values that - /// were found in this segment's key index (regardless of tombstone status), - /// so the caller can remove them from further searching. - /// - /// Two-phase design: - /// Phase 1: resolve keys -> node_ids via key_index (dual-strategy) - /// Phase 2: batch-fetch node records via get_nodes_batch (one sorted - /// merge-walk through nodes.dat, not N binary searches) - pub fn resolve_keys_batch( - &self, - lookups: &[(usize, u32, &str)], - results: &mut [Option], - ) -> Result, EngineError> { - if lookups.is_empty() { - return Ok(Vec::new()); - } - - // Phase 1: key_index → (orig_idx, node_id) pairs - let resolved = self.resolve_keys_to_ids(lookups)?; - if resolved.is_empty() { - return Ok(Vec::new()); - } - - let found_indices: Vec = resolved.iter().map(|&(orig_idx, _)| orig_idx).collect(); - - // Phase 2: batch-fetch node records from nodes.dat - // Build (orig_idx, node_id) sorted by node_id for get_nodes_batch. - let mut node_lookups: Vec<(usize, u64)> = resolved - .iter() - .filter(|&&(_, nid)| !self.deleted_nodes.contains_key(&nid)) - .copied() - .collect(); - node_lookups.sort_unstable_by_key(|&(_, nid)| nid); - self.get_nodes_batch(&node_lookups, results)?; - - Ok(found_indices) - } - /// Phase 1 of resolve_keys_batch: walk the key index to map each - /// (type_id, key) query to a node_id. Returns (orig_idx, node_id) pairs + /// (label_id, key) query to a node_id. Returns (orig_idx, node_id) pairs /// for keys found in this segment's key index. - fn resolve_keys_to_ids( + pub(crate) fn resolve_keys_to_ids( &self, lookups: &[(usize, u32, &str)], ) -> Result, EngineError> { @@ -822,12 +1691,12 @@ impl SegmentReader { if strategy == BatchReadStrategy::SeekPerKey { let mut prev_query: Option<(u32, &str)> = None; let mut prev_node_id: Option = None; - for &(orig_idx, type_id, key) in lookups { - let node_id = if prev_query == Some((type_id, key)) { + for &(orig_idx, label_id, key) in lookups { + let node_id = if prev_query == Some((label_id, key)) { prev_node_id } else { - let found = self.binary_search_key_index(type_id, key)?; - prev_query = Some((type_id, key)); + let found = self.binary_search_key_index(label_id, key)?; + prev_query = Some((label_id, key)); prev_node_id = found; found }; @@ -838,11 +1707,11 @@ impl SegmentReader { } else { // Merge-walk: single cursor through key index entries let mut idx_pos = 0usize; - for &(orig_idx, type_id, key) in lookups { + for &(orig_idx, label_id, key) in lookups { while idx_pos < count { let entry_offset = read_u64_at(data, offset_table_start + idx_pos * 8)? as usize; - let entry_type = read_u32_at(data, entry_offset)?; + let entry_label_id = read_u32_at(data, entry_offset)?; let key_len = read_u16_at(data, entry_offset + 12)? as usize; let key_bytes = read_bytes_at(data, entry_offset + 14, key_len)?; let entry_key = std::str::from_utf8(key_bytes).map_err(|_| { @@ -852,7 +1721,7 @@ impl SegmentReader { )) })?; - match (entry_type, entry_key).cmp(&(type_id, key)) { + match (entry_label_id, entry_key).cmp(&(label_id, key)) { std::cmp::Ordering::Less => { idx_pos += 1; } @@ -874,13 +1743,13 @@ impl SegmentReader { /// Query neighbors of a node. Checks both outgoing and incoming adjacency /// based on the direction parameter. - pub fn neighbors( + pub(crate) fn neighbors( &self, node_id: u64, direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, limit: usize, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let mut results = Vec::new(); match direction { @@ -889,7 +1758,7 @@ impl SegmentReader { &self.adj_out_idx, &self.adj_out_dat, node_id, - type_filter, + label_filter_ids, limit, None, None, @@ -902,7 +1771,7 @@ impl SegmentReader { &self.adj_in_idx, &self.adj_in_dat, node_id, - type_filter, + label_filter_ids, limit, None, None, @@ -917,7 +1786,7 @@ impl SegmentReader { &self.adj_out_idx, &self.adj_out_dat, node_id, - type_filter, + label_filter_ids, 0, Some(&mut self_loop_edge_ids), None, @@ -928,7 +1797,7 @@ impl SegmentReader { &self.adj_in_idx, &self.adj_in_dat, node_id, - type_filter, + label_filter_ids, 0, None, Some(&self_loop_edge_ids), @@ -941,7 +1810,7 @@ impl SegmentReader { &self.adj_out_idx, &self.adj_out_dat, node_id, - type_filter, + label_filter_ids, limit, Some(&mut self_loop_edge_ids), None, @@ -956,7 +1825,7 @@ impl SegmentReader { &self.adj_in_idx, &self.adj_in_dat, node_id, - type_filter, + label_filter_ids, 0, None, Some(&self_loop_edge_ids), @@ -1014,6 +1883,7 @@ impl SegmentReader { }; let strategy = choose_batch_read_strategy( data, + idx_start, count, NODE_INDEX_ENTRY_SIZE, 0, @@ -1041,8 +1911,7 @@ impl SegmentReader { if let Some((index, offset)) = offset { let mut node = decode_node_at(&self.nodes_mmap, offset, target_id)?; self.hydrate_node_vectors(index, &mut node)?; - let (_, _, _, _, _, _, _, _, _, lws) = self.node_meta_at(index)?; - node.last_write_seq = lws; + node.last_write_seq = self.node_meta_at(index)?.last_write_seq; results[orig_idx] = Some(node); } } @@ -1062,8 +1931,7 @@ impl SegmentReader { let offset = read_u64_at(data, entry_off + 8)? as usize; let mut node = decode_node_at(&self.nodes_mmap, offset, id)?; self.hydrate_node_vectors(idx_pos, &mut node)?; - let (_, _, _, _, _, _, _, _, _, lws) = self.node_meta_at(idx_pos)?; - node.last_write_seq = lws; + node.last_write_seq = self.node_meta_at(idx_pos)?.last_write_seq; results[orig_idx] = Some(node); break; } else { @@ -1078,25 +1946,25 @@ impl SegmentReader { /// Batch metadata lookup: resolve multiple node IDs without decoding full /// node records or hydrating vectors. `lookups` must be sorted by ID. /// Found metadata is written into `results[original_index]` as - /// `(type_id, updated_at, weight)`. + /// `(label_ids, updated_at, weight)`. pub(crate) fn get_node_meta_batch( &self, lookups: &[(usize, u64)], - results: &mut [Option<(u32, i64, f32)>], + results: &mut [Option<(NodeLabelSet, i64, f32)>], ) -> Result<(), EngineError> { if lookups.is_empty() { return Ok(()); } - let data = &self.nodes_mmap[..]; - if data.len() < 8 { + let data = &self.node_meta_mmap[..]; + let Some(layout) = parse_node_meta_layout(data)? else { return Ok(()); - } - let count = read_u64_at(data, 0)? as usize; + }; + let count = layout.node_count; if count == 0 { return Ok(()); } - let idx_start = 8; + let idx_start = layout.fixed_entries_offset; let min_key = lookups.first().map(|&(_, id)| id).unwrap_or(0); let max_key = lookups.last().map(|&(_, id)| id).unwrap_or(0); let unique_keys = { @@ -1112,8 +1980,9 @@ impl SegmentReader { }; let strategy = choose_batch_read_strategy( data, + idx_start, count, - NODE_INDEX_ENTRY_SIZE, + layout.fixed_entry_size, 0, unique_keys, min_key, @@ -1122,27 +1991,17 @@ impl SegmentReader { if strategy == BatchReadStrategy::SeekPerKey { let mut prev_id: Option = None; - let mut prev_meta: Option<(u32, i64, f32)> = None; + let mut prev_meta: Option<(NodeLabelSet, i64, f32)> = None; for &(orig_idx, target_id) in lookups { if self.deleted_nodes.contains_key(&target_id) { continue; } let meta = if prev_id == Some(target_id) { prev_meta - } else if let Some((index, _offset)) = self.binary_search_node_index(target_id)? { - let ( - _node_id, - _data_offset, - _data_len, - type_id, - updated_at, - weight, - _key_len, - _prop_hash_offset, - _prop_hash_count, - _last_write_seq, - ) = self.node_meta_at(index)?; - let found = Some((type_id, updated_at, weight)); + } else if let Some(index) = binary_search_node_meta_index(data, layout, target_id)? + { + let meta = read_node_meta_entry_at(data, layout, index)?; + let found = Some((meta.label_ids, meta.updated_at, meta.weight)); prev_id = Some(target_id); prev_meta = found; found @@ -1160,24 +2019,13 @@ impl SegmentReader { continue; } while idx_pos < count { - let entry_off = idx_start + idx_pos * NODE_INDEX_ENTRY_SIZE; + let entry_off = idx_start + idx_pos * layout.fixed_entry_size; let id = read_u64_at(data, entry_off)?; if id < target_id { idx_pos += 1; } else if id == target_id { - let ( - _node_id, - _data_offset, - _data_len, - type_id, - updated_at, - weight, - _key_len, - _prop_hash_offset, - _prop_hash_count, - _last_write_seq, - ) = self.node_meta_at(idx_pos)?; - results[orig_idx] = Some((type_id, updated_at, weight)); + let meta = read_node_meta_entry_at(data, layout, idx_pos)?; + results[orig_idx] = Some((meta.label_ids, meta.updated_at, meta.weight)); break; } else { break; @@ -1204,25 +2052,25 @@ impl SegmentReader { remaining_out: &mut Vec, ) -> Result<(), EngineError> where - F: FnMut(u32, i64, f32) -> bool, + F: FnMut(NodeLabelSet, i64, f32) -> bool, { if ids.is_empty() { return Ok(()); } - let data = &self.nodes_mmap[..]; let node_meta = &self.node_meta_mmap[..]; let vector_meta = &self.node_vector_meta_mmap[..]; - if data.len() < 8 { + let Some(node_meta_layout) = parse_node_meta_layout(node_meta)? else { remaining_out.extend_from_slice(ids); return Ok(()); - } - let count = read_u64_at(data, 0)? as usize; + }; + let data = node_meta; + let count = node_meta_layout.node_count; if count == 0 { remaining_out.extend_from_slice(ids); return Ok(()); } - let idx_start = 8; + let idx_start = node_meta_layout.fixed_entries_offset; let min_key = ids.first().copied().unwrap_or(0); let max_key = ids.last().copied().unwrap_or(0); let mut unique_keys = 0usize; @@ -1235,8 +2083,9 @@ impl SegmentReader { } let strategy = choose_batch_read_strategy( data, + idx_start, count, - NODE_INDEX_ENTRY_SIZE, + node_meta_layout.fixed_entry_size, 0, unique_keys, min_key, @@ -1253,8 +2102,15 @@ impl SegmentReader { let found = if prev_id == Some(target_id) { prev_found - } else if let Some((index, _offset)) = self.binary_search_node_index(target_id)? { - let found = Some(read_dense_scoring_meta(node_meta, vector_meta, index)?); + } else if let Some(index) = + binary_search_node_meta_index(node_meta, node_meta_layout, target_id)? + { + let found = Some(read_dense_scoring_meta( + node_meta, + node_meta_layout, + vector_meta, + index, + )?); prev_id = Some(target_id); prev_found = found; found @@ -1268,7 +2124,8 @@ impl SegmentReader { remaining_out.push(target_id); continue; }; - if found.dense_len == 0 || !include(found.type_id, found.updated_at, found.weight) { + if found.dense_len == 0 || !include(found.label_ids, found.updated_at, found.weight) + { continue; } hits_out.push(VectorHit { @@ -1292,12 +2149,17 @@ impl SegmentReader { let mut found = None; while idx_pos < count { - let entry_off = idx_start + idx_pos * NODE_INDEX_ENTRY_SIZE; + let entry_off = idx_start + idx_pos * node_meta_layout.fixed_entry_size; let id = read_u64_at(data, entry_off)?; if id < target_id { idx_pos += 1; } else if id == target_id { - found = Some(read_dense_scoring_meta(node_meta, vector_meta, idx_pos)?); + found = Some(read_dense_scoring_meta( + node_meta, + node_meta_layout, + vector_meta, + idx_pos, + )?); break; } else { break; @@ -1308,7 +2170,8 @@ impl SegmentReader { remaining_out.push(target_id); continue; }; - if found.dense_len == 0 || !include(found.type_id, found.updated_at, found.weight) { + if found.dense_len == 0 || !include(found.label_ids, found.updated_at, found.weight) + { continue; } hits_out.push(VectorHit { @@ -1342,26 +2205,26 @@ impl SegmentReader { remaining_out: &mut Vec, ) -> Result<(), EngineError> where - F: FnMut(u32, i64, f32) -> bool, + F: FnMut(NodeLabelSet, i64, f32) -> bool, { if ids.is_empty() || query.is_empty() { return Ok(()); } - let data = &self.nodes_mmap[..]; let node_meta = &self.node_meta_mmap[..]; let vector_meta = &self.node_vector_meta_mmap[..]; let sparse_blob = &self.node_sparse_vectors_mmap[..]; - if data.len() < 8 { + let Some(node_meta_layout) = parse_node_meta_layout(node_meta)? else { remaining_out.extend_from_slice(ids); return Ok(()); - } - let count = read_u64_at(data, 0)? as usize; + }; + let data = node_meta; + let count = node_meta_layout.node_count; if count == 0 { remaining_out.extend_from_slice(ids); return Ok(()); } - let idx_start = 8; + let idx_start = node_meta_layout.fixed_entries_offset; let min_key = ids.first().copied().unwrap_or(0); let max_key = ids.last().copied().unwrap_or(0); let mut unique_keys = 0usize; @@ -1374,8 +2237,9 @@ impl SegmentReader { } let strategy = choose_batch_read_strategy( data, + idx_start, count, - NODE_INDEX_ENTRY_SIZE, + node_meta_layout.fixed_entry_size, 0, unique_keys, min_key, @@ -1392,8 +2256,15 @@ impl SegmentReader { let found = if prev_id == Some(target_id) { prev_found - } else if let Some((index, _offset)) = self.binary_search_node_index(target_id)? { - let found = Some(read_sparse_scoring_meta(node_meta, vector_meta, index)?); + } else if let Some(index) = + binary_search_node_meta_index(node_meta, node_meta_layout, target_id)? + { + let found = Some(read_sparse_scoring_meta( + node_meta, + node_meta_layout, + vector_meta, + index, + )?); prev_id = Some(target_id); prev_found = found; found @@ -1407,7 +2278,8 @@ impl SegmentReader { remaining_out.push(target_id); continue; }; - if found.sparse_len == 0 || !include(found.type_id, found.updated_at, found.weight) + if found.sparse_len == 0 + || !include(found.label_ids, found.updated_at, found.weight) { continue; } @@ -1430,12 +2302,17 @@ impl SegmentReader { let mut found = None; while idx_pos < count { - let entry_off = idx_start + idx_pos * NODE_INDEX_ENTRY_SIZE; + let entry_off = idx_start + idx_pos * node_meta_layout.fixed_entry_size; let id = read_u64_at(data, entry_off)?; if id < target_id { idx_pos += 1; } else if id == target_id { - found = Some(read_sparse_scoring_meta(node_meta, vector_meta, idx_pos)?); + found = Some(read_sparse_scoring_meta( + node_meta, + node_meta_layout, + vector_meta, + idx_pos, + )?); break; } else { break; @@ -1446,7 +2323,8 @@ impl SegmentReader { remaining_out.push(target_id); continue; }; - if found.sparse_len == 0 || !include(found.type_id, found.updated_at, found.weight) + if found.sparse_len == 0 + || !include(found.label_ids, found.updated_at, found.weight) { continue; } @@ -1504,6 +2382,7 @@ impl SegmentReader { }; let strategy = choose_batch_read_strategy( data, + idx_start, count, EDGE_INDEX_ENTRY_SIZE, 0, @@ -1563,29 +2442,112 @@ impl SegmentReader { Ok(()) } - /// Check if an edge ID is tombstoned in this segment. - pub fn is_edge_deleted(&self, id: u64) -> bool { - self.deleted_edges.contains_key(&id) - } - - /// Check if a node ID exists (has a live record) in this segment's index. - /// Does NOT check tombstones; only checks whether the node index contains this ID. - pub fn has_node(&self, id: u64) -> bool { - self.binary_search_node_index(id).ok().flatten().is_some() - } - - /// Check if an edge ID exists (has a live record) in this segment's index. - /// Does NOT check tombstones; only checks whether the edge index contains this ID. - pub fn has_edge(&self, id: u64) -> bool { - self.binary_search_edge_index(id).ok().flatten().is_some() - } - - /// Return the deleted node tombstone map in this segment. - pub fn deleted_node_tombstones(&self) -> &NodeIdMap { - &self.deleted_nodes - } + /// Batch metadata lookup: resolve multiple edge IDs without decoding full + /// edge records. `lookups` must be sorted by ID. Found metadata is written + /// into `results[original_index]`. + pub(crate) fn get_edge_metadata_batch( + &self, + lookups: &[(usize, u64)], + results: &mut [Option], + ) -> Result<(), EngineError> { + if lookups.is_empty() { + return Ok(()); + } + let data = &self.edges_mmap[..]; + if data.len() < 8 { + return Ok(()); + } + let count = read_u64_at(data, 0)? as usize; + if count == 0 { + return Ok(()); + } - /// Return the deleted edge tombstone map in this segment. + let idx_start = 8; + let min_key = lookups.first().map(|&(_, id)| id).unwrap_or(0); + let max_key = lookups.last().map(|&(_, id)| id).unwrap_or(0); + let unique_keys = { + let mut n = 0usize; + let mut prev: Option = None; + for &(_, id) in lookups { + if prev != Some(id) { + n += 1; + prev = Some(id); + } + } + n + }; + let strategy = choose_batch_read_strategy( + data, + idx_start, + count, + EDGE_INDEX_ENTRY_SIZE, + 0, + unique_keys, + min_key, + max_key, + )?; + + if strategy == BatchReadStrategy::SeekPerKey { + let mut prev_id: Option = None; + let mut prev_meta: Option = None; + for &(orig_idx, target_id) in lookups { + if self.deleted_edges.contains_key(&target_id) { + continue; + } + let meta = if prev_id == Some(target_id) { + prev_meta + } else if let Some((index, _offset)) = self.binary_search_edge_index(target_id)? { + let found = Some(self.edge_metadata_at_index(index)?); + prev_id = Some(target_id); + prev_meta = found; + found + } else { + prev_id = Some(target_id); + prev_meta = None; + None + }; + results[orig_idx] = meta; + } + } else { + let mut idx_pos = 0usize; + for &(orig_idx, target_id) in lookups { + if self.deleted_edges.contains_key(&target_id) { + continue; + } + while idx_pos < count { + let entry_off = idx_start + idx_pos * EDGE_INDEX_ENTRY_SIZE; + let id = read_u64_at(data, entry_off)?; + if id < target_id { + idx_pos += 1; + } else if id == target_id { + results[orig_idx] = Some(self.edge_metadata_at_index(idx_pos)?); + break; + } else { + break; + } + } + } + } + Ok(()) + } + + /// Check if an edge ID is tombstoned in this segment. + pub fn is_edge_deleted(&self, id: u64) -> bool { + self.deleted_edges.contains_key(&id) + } + + /// Check if an edge ID exists (has a live record) in this segment's index. + /// Does NOT check tombstones; only checks whether the edge index contains this ID. + pub fn has_edge(&self, id: u64) -> bool { + self.binary_search_edge_index(id).ok().flatten().is_some() + } + + /// Return the deleted node tombstone map in this segment. + pub fn deleted_node_tombstones(&self) -> &NodeIdMap { + &self.deleted_nodes + } + + /// Return the deleted edge tombstone map in this segment. pub fn deleted_edge_tombstones(&self) -> &NodeIdMap { &self.deleted_edges } @@ -1630,42 +2592,66 @@ impl SegmentReader { .as_ref()) } - /// Enumerate unique node IDs that appear in this segment's adjacency indexes. - /// Scans both adj_out and adj_in index files and returns the union of node IDs. - /// Used by degree cache rebuild to find adjacency-bearing nodes without - /// enumerating all visible node records. - pub fn adj_node_ids(&self) -> Result { - let mut ids = NodeIdSet::default(); - Self::collect_adj_index_node_ids(&self.adj_out_idx, &mut ids)?; - Self::collect_adj_index_node_ids(&self.adj_in_idx, &mut ids)?; - Ok(ids) - } - - /// Extract unique node IDs from a single adjacency index file. - /// Index format: [count: u64] then count entries of ADJ_INDEX_ENTRY_SIZE, - /// each starting with [node_id: u64]. Entries are sorted by node_id so - /// consecutive duplicates (same node, different type_id) are common. - fn collect_adj_index_node_ids( - idx_mmap: &MappedData, - out: &mut NodeIdSet, - ) -> Result<(), EngineError> { - let idx_data = &idx_mmap[..]; - if idx_data.len() < 8 { - return Ok(()); + pub(crate) fn node_record_index_entries_for_scrub( + &self, + expected_count: u64, + ) -> Result, EngineError> { + let data = &self.nodes_mmap[..]; + if data.len() < 8 { + return Err(EngineError::CorruptRecord(format!( + "node records payload length {} is too short for count header", + data.len() + ))); } - let count = read_u64_at(idx_data, 0)? as usize; - let idx_start = 8; - let mut prev_node: u64 = u64::MAX; - for i in 0..count { - let entry_off = idx_start + i * ADJ_INDEX_ENTRY_SIZE; - let node_id = read_u64_at(idx_data, entry_off)?; - // Index is sorted by node_id; skip consecutive duplicates - if node_id != prev_node { - out.insert(node_id); - prev_node = node_id; + let actual_count = read_u64_at(data, 0)?; + if actual_count != expected_count { + return Err(EngineError::CorruptRecord(format!( + "node records count {} does not match segment manifest node_count {}", + actual_count, expected_count + ))); + } + let count = usize_from_u64(actual_count, "node records count")?; + let index_start = 8usize; + let index_end = index_start + .checked_add(count.checked_mul(NODE_INDEX_ENTRY_SIZE).ok_or_else(|| { + EngineError::CorruptRecord("node records index size overflow".into()) + })?) + .ok_or_else(|| EngineError::CorruptRecord("node records index end overflow".into()))?; + if index_end > data.len() { + return Err(EngineError::CorruptRecord(format!( + "node records index [{}, {}) exceeds payload length {}", + index_start, + index_end, + data.len() + ))); + } + + let mut entries = Vec::with_capacity(count); + let mut previous = None; + for index in 0..count { + let entry_offset = index_start + index * NODE_INDEX_ENTRY_SIZE; + let node_id = read_u64_at(data, entry_offset)?; + if previous.is_some_and(|prev| prev >= node_id) { + return Err(EngineError::CorruptRecord(format!( + "node records index row {} is not sorted by unique node_id", + index + ))); + } + previous = Some(node_id); + let data_offset = read_u64_at(data, entry_offset + 8)?; + let data_offset_usize = usize_from_u64(data_offset, "node records data offset")?; + if data_offset_usize < index_end || data_offset_usize >= data.len() { + return Err(EngineError::CorruptRecord(format!( + "node records index row {} offset {} is outside payload data region [{}, {})", + index, + data_offset_usize, + index_end, + data.len() + ))); } + entries.push((node_id, data_offset)); } - Ok(()) + Ok(entries) } pub fn node_count(&self) -> u64 { @@ -1757,16 +2743,71 @@ impl SegmentReader { Some((first_id, last_id)) } - /// Raw mmap bytes for nodes.dat (used by V3 compaction for raw binary copy). + /// Raw mmap bytes for the node records payload (used by V3 compaction). pub(crate) fn raw_nodes_mmap(&self) -> &[u8] { &self.nodes_mmap[..] } - /// Raw mmap bytes for edges.dat (used by V3 compaction for raw binary copy). + /// Raw mmap bytes for the edge records payload (used by V3 compaction). pub(crate) fn raw_edges_mmap(&self) -> &[u8] { &self.edges_mmap[..] } + pub(crate) fn segment_data_id(&self) -> [u8; 32] { + self.segment_data_id + } + + pub(crate) fn component_manifest_generation(&self) -> u64 { + self.component_manifest_generation + } + + #[cfg(test)] + pub(crate) fn optional_component_availability_for_test( + &self, + kind: SegmentComponentKind, + ) -> ComponentAvailability { + self.optional_component_availability(kind) + } + + #[cfg(test)] + pub(crate) fn component_mapping_identity_for_test( + &self, + kind: SegmentComponentKind, + ) -> Option { + let data = match kind { + SegmentComponentKind::NodeRecords => &self.nodes_mmap, + SegmentComponentKind::EdgeRecords => &self.edges_mmap, + SegmentComponentKind::NodeMetadata => &self.node_meta_mmap, + SegmentComponentKind::EdgeMetadata => &self.edge_meta_mmap, + SegmentComponentKind::Tombstones => return None, + SegmentComponentKind::KeyIndex => &self.key_index_mmap, + SegmentComponentKind::NodeLabelIndex => &self.node_label_index_mmap, + SegmentComponentKind::EdgeLabelIndex => &self.edge_label_index_mmap, + SegmentComponentKind::EdgeTripleIndex => &self.edge_triple_index_mmap, + SegmentComponentKind::AdjOutIndex => &self.adj_out_idx, + SegmentComponentKind::AdjOutPostings => &self.adj_out_dat, + SegmentComponentKind::AdjInIndex => &self.adj_in_idx, + SegmentComponentKind::AdjInPostings => &self.adj_in_dat, + SegmentComponentKind::TimestampIndex => &self.timestamp_index_mmap, + SegmentComponentKind::NodeVectorMetadata => &self.node_vector_meta_mmap, + SegmentComponentKind::NodeDenseVectorBlob => &self.node_dense_vectors_mmap, + SegmentComponentKind::NodeSparseVectorBlob => &self.node_sparse_vectors_mmap, + SegmentComponentKind::EdgeWeightIndex => &self.edge_weight_index_mmap, + SegmentComponentKind::EdgeUpdatedAtIndex => &self.edge_updated_at_index_mmap, + SegmentComponentKind::EdgeValidFromIndex => &self.edge_valid_from_index_mmap, + SegmentComponentKind::EdgeValidToIndex => &self.edge_valid_to_index_mmap, + _ => return None, + }; + data.mapping_identity_for_test() + } + + pub(crate) fn optional_component_availability( + &self, + kind: SegmentComponentKind, + ) -> ComponentAvailability { + self.component_registry.availability(&kind) + } + #[cfg(test)] pub(crate) fn planner_stats(&self) -> Option<&SegmentPlannerStatsV1> { self.planner_stats.stats() @@ -1786,7 +2827,7 @@ impl SegmentReader { self.planner_stats.clone() } - // --- V5 metadata sidecar accessors (for V3 compaction) --- + // --- Metadata payload accessors (for V3 compaction) --- /// Number of node metadata entries. pub(crate) fn node_meta_count(&self) -> u64 { @@ -1797,103 +2838,368 @@ impl SegmentReader { read_u64_at(data, 0).unwrap_or(0) } - /// Read a node metadata entry by index (0-based). - /// Returns (node_id, data_offset, data_len, type_id, updated_at, weight, key_len, - /// prop_hash_offset, prop_hash_count, last_write_seq). - #[allow(clippy::type_complexity)] - pub(crate) fn node_meta_at( + pub(crate) fn node_meta_count_for_scrub( &self, - index: usize, - ) -> Result<(u64, u64, u32, u32, i64, f32, u16, u64, u32, u64), EngineError> { + expected_count: u64, + ) -> Result { let data = &self.node_meta_mmap[..]; - let off = 8 + index * NODE_META_ENTRY_SIZE; - let node_id = read_u64_at(data, off)?; - let data_offset = read_u64_at(data, off + 8)?; - let data_len = read_u32_at(data, off + 16)?; - let type_id = read_u32_at(data, off + 20)?; - let updated_at = read_i64_at(data, off + 24)?; - let weight = read_f32_at(data, off + 32)?; - let key_len = read_u16_at(data, off + 36)?; - let prop_hash_offset = read_u64_at(data, off + 38)?; - let prop_hash_count = read_u32_at(data, off + 46)?; - let last_write_seq = read_u64_at(data, off + 50)?; - Ok(( - node_id, - data_offset, - data_len, - type_id, - updated_at, - weight, - key_len, - prop_hash_offset, - prop_hash_count, - last_write_seq, - )) + if data.len() < 8 { + return Err(EngineError::CorruptRecord(format!( + "node metadata payload length {} is too short for count header", + data.len() + ))); + } + let actual_count = read_u64_at(data, 0)?; + if actual_count != expected_count { + return Err(EngineError::CorruptRecord(format!( + "node metadata row count {} does not match segment manifest node_count {}", + actual_count, expected_count + ))); + } + let expected_count = usize_from_u64(expected_count, "segment manifest node_count")?; + let Some(layout) = parse_node_meta_layout(data)? else { + return Err(EngineError::CorruptRecord( + "node metadata payload is missing".into(), + )); + }; + if layout.node_count != expected_count { + return Err(EngineError::CorruptRecord(format!( + "node metadata parsed row count {} does not match segment manifest node_count {}", + layout.node_count, expected_count + ))); + } + Ok(layout.node_count) } - pub(crate) fn node_vector_meta_at( + /// Read a node metadata entry by index (0-based). + pub(crate) fn node_meta_at(&self, index: usize) -> Result { + let data = &self.node_meta_mmap[..]; + let Some(layout) = parse_node_meta_layout(data)? else { + return Err(EngineError::CorruptRecord( + "node metadata payload is missing".into(), + )); + }; + read_node_meta_entry_at(data, layout, index) + } + + pub(crate) fn node_record_for_meta_scrub( &self, - index: usize, - ) -> Result<(u64, u32, u64, u32), EngineError> { - let data = &self.node_vector_meta_mmap[..]; - if data.is_empty() { - return Ok((0, 0, 0, 0)); + meta: &SegmentNodeMeta, + ) -> Result { + let offset = usize_from_u64(meta.data_offset, "node record offset")?; + let expected_end = offset + .checked_add(meta.data_len as usize) + .ok_or_else(|| EngineError::CorruptRecord("node record span overflow".into()))?; + let (node, actual_end) = decode_node_at_with_end(&self.nodes_mmap, offset, meta.node_id)?; + if actual_end != expected_end { + return Err(EngineError::CorruptRecord(format!( + "node record {} decoded span [{}, {}) does not match metadata span [{}, {})", + meta.node_id, offset, actual_end, offset, expected_end + ))); } - let (flags, dense_offset, dense_len, sparse_offset, sparse_len) = - read_node_vector_meta_entry(data, index)?; - Ok(( - if flags & NODE_VECTOR_FLAG_DENSE != 0 { - dense_offset - } else { - 0 - }, - if flags & NODE_VECTOR_FLAG_DENSE != 0 { - dense_len - } else { - 0 - }, - if flags & NODE_VECTOR_FLAG_SPARSE != 0 { - sparse_offset - } else { - 0 - }, - if flags & NODE_VECTOR_FLAG_SPARSE != 0 { - sparse_len - } else { - 0 - }, - )) + Ok(node) } - /// Number of edge metadata entries. - pub(crate) fn edge_meta_count(&self) -> u64 { - let data = &self.edge_meta_mmap[..]; + pub(crate) fn node_label_index_entries_for_scrub( + &self, + expected_node_count: u64, + ) -> Result, EngineError> { + let data = &self.node_label_index_mmap[..]; if data.len() < 8 { - return 0; + return Err(EngineError::CorruptRecord( + "node label index missing or truncated (< 8 bytes)".into(), + )); } - read_u64_at(data, 0).unwrap_or(0) + let max_memberships = usize_from_u64( + expected_node_count + .checked_mul(MAX_NODE_LABELS_PER_NODE as u64) + .ok_or_else(|| { + EngineError::CorruptRecord( + "node label index expected membership cap overflow".into(), + ) + })?, + "node label index expected membership cap", + )?; + let count = usize_from_u64(read_u64_at(data, 0)?, "node label index row count")?; + if count > max_memberships { + return Err(EngineError::CorruptRecord(format!( + "node label index row count {} exceeds maximum label memberships {}", + count, max_memberships + ))); + } + let index_start = 8usize; + let index_end = index_start + .checked_add( + count + .checked_mul(LABEL_POSTING_INDEX_ENTRY_SIZE) + .ok_or_else(|| { + EngineError::CorruptRecord("node label index table size overflow".into()) + })?, + ) + .ok_or_else(|| { + EngineError::CorruptRecord("node label index table end overflow".into()) + })?; + if index_end > data.len() { + return Err(EngineError::CorruptRecord(format!( + "node label index table [{}, {}) exceeds payload length {}", + index_start, + index_end, + data.len() + ))); + } + + let mut entries = Vec::new(); + let mut prev_label_id = None; + let mut expected_posting_offset = index_end; + let mut total_postings = 0usize; + for index in 0..count { + let off = index_start + index * LABEL_POSTING_INDEX_ENTRY_SIZE; + let label_id = read_u32_at(data, off)?; + if prev_label_id.is_some_and(|prev| prev >= label_id) { + return Err(EngineError::CorruptRecord( + "node label index label IDs must be sorted ascending and unique".into(), + )); + } + prev_label_id = Some(label_id); + + let posting_offset = usize_from_u64( + read_u64_at(data, off + 4)?, + "node label index posting offset", + )?; + let posting_count = read_u32_at(data, off + 12)? as usize; + if posting_count == 0 { + return Err(EngineError::CorruptRecord(format!( + "node label index label_id {} has an empty posting list", + label_id + ))); + } + total_postings = total_postings.checked_add(posting_count).ok_or_else(|| { + EngineError::CorruptRecord("node label index posting count overflow".into()) + })?; + if total_postings > max_memberships { + return Err(EngineError::CorruptRecord(format!( + "node label index posting count {} exceeds maximum label memberships {}", + total_postings, max_memberships + ))); + } + let posting_end = posting_offset + .checked_add(posting_count.checked_mul(8).ok_or_else(|| { + EngineError::CorruptRecord("node label index posting size overflow".into()) + })?) + .ok_or_else(|| { + EngineError::CorruptRecord("node label index posting end overflow".into()) + })?; + if posting_offset != expected_posting_offset { + return Err(EngineError::CorruptRecord(format!( + "node label index posting range for label_id {} starts at {}, expected {}", + label_id, posting_offset, expected_posting_offset + ))); + } + if posting_offset < index_end || posting_end > data.len() { + return Err(EngineError::CorruptRecord(format!( + "node label index posting [{}, {}) is outside payload data region [{}, {})", + posting_offset, + posting_end, + index_end, + data.len() + ))); + } + + let mut prev_node_id = None; + for posting_index in 0..posting_count { + let node_id = read_u64_at(data, posting_offset + posting_index * 8)?; + if prev_node_id.is_some_and(|prev| prev >= node_id) { + return Err(EngineError::CorruptRecord(format!( + "node label index posting for label_id {} must be sorted ascending and unique", + label_id + ))); + } + prev_node_id = Some(node_id); + entries.push((label_id, node_id)); + } + expected_posting_offset = posting_end; + } + Ok(entries) } - /// Read an edge metadata entry by index (0-based). - /// Returns (edge_id, data_offset, data_len, from, to, type_id, updated_at, - /// weight, valid_from, valid_to, last_write_seq). - #[allow(clippy::type_complexity)] - pub(crate) fn edge_meta_at( + pub(crate) fn node_key_index_entries_for_scrub( &self, - index: usize, - ) -> Result<(u64, u64, u32, u64, u64, u32, i64, f32, i64, i64, u64), EngineError> { - let data = &self.edge_meta_mmap[..]; - let off = 8 + index * EDGE_META_ENTRY_SIZE; - let edge_id = read_u64_at(data, off)?; - let data_offset = read_u64_at(data, off + 8)?; - let data_len = read_u32_at(data, off + 16)?; - let from = read_u64_at(data, off + 20)?; - let to = read_u64_at(data, off + 28)?; - let type_id = read_u32_at(data, off + 36)?; - let updated_at = read_i64_at(data, off + 40)?; - let weight = read_f32_at(data, off + 48)?; - let valid_from = read_i64_at(data, off + 52)?; - let valid_to = read_i64_at(data, off + 60)?; + ) -> Result, EngineError> { + let data = &self.key_index_mmap[..]; + if data.len() < 8 { + return Err(EngineError::CorruptRecord( + "node key index missing or truncated (< 8 bytes)".into(), + )); + } + let count = read_u64_at(data, 0)? as usize; + let offset_table_start = 8usize; + let data_start = offset_table_start + .checked_add(count.checked_mul(8).ok_or_else(|| { + EngineError::CorruptRecord("node key index offset table size overflow".into()) + })?) + .ok_or_else(|| { + EngineError::CorruptRecord("node key index offset table end overflow".into()) + })?; + if data_start > data.len() { + return Err(EngineError::CorruptRecord(format!( + "node key index offset table end {} exceeds payload length {}", + data_start, + data.len() + ))); + } + + let mut entries = Vec::with_capacity(count); + let mut prev_entry: Option<(u32, String, u64)> = None; + let mut prev_offset = data_start; + for index in 0..count { + let entry_offset = usize_from_u64( + read_u64_at(data, offset_table_start + index * 8)?, + "node key index entry offset", + )?; + if entry_offset < data_start || entry_offset < prev_offset || entry_offset >= data.len() + { + return Err(EngineError::CorruptRecord(format!( + "node key index entry offset {} at row {} is outside or before the data region", + entry_offset, index + ))); + } + prev_offset = entry_offset; + + let label_id = read_u32_at(data, entry_offset)?; + let node_id = read_u64_at(data, entry_offset + 4)?; + let key_len = read_u16_at(data, entry_offset + 12)? as usize; + let key_bytes = read_bytes_at(data, entry_offset + 14, key_len)?; + let key = std::str::from_utf8(key_bytes) + .map_err(|_| { + EngineError::CorruptRecord(format!( + "invalid UTF-8 in node key index at offset {}", + entry_offset + 14 + )) + })? + .to_string(); + let entry = (label_id, key, node_id); + if prev_entry.as_ref().is_some_and(|prev| prev >= &entry) { + return Err(EngineError::CorruptRecord( + "node key index entries must be sorted ascending and unique".into(), + )); + } + prev_entry = Some(entry.clone()); + entries.push(entry); + } + Ok(entries) + } + + pub(crate) fn node_timestamp_index_entries_for_scrub( + &self, + ) -> Result, EngineError> { + let data = &self.timestamp_index_mmap[..]; + if data.len() < 8 { + return Err(EngineError::CorruptRecord( + "node timestamp index missing or truncated (< 8 bytes)".into(), + )); + } + let count = read_u64_at(data, 0)? as usize; + let entry_start = 8usize; + let entry_size = 20usize; + let entry_end = entry_start + .checked_add(count.checked_mul(entry_size).ok_or_else(|| { + EngineError::CorruptRecord("node timestamp index size overflow".into()) + })?) + .ok_or_else(|| { + EngineError::CorruptRecord("node timestamp index end overflow".into()) + })?; + if entry_end > data.len() { + return Err(EngineError::CorruptRecord(format!( + "node timestamp index entries [{}, {}) exceed payload length {}", + entry_start, + entry_end, + data.len() + ))); + } + + let mut entries = Vec::with_capacity(count); + let mut prev_entry: Option<(u32, i64, u64)> = None; + for index in 0..count { + let off = entry_start + index * entry_size; + let entry = ( + read_u32_at(data, off)?, + read_i64_at(data, off + 4)?, + read_u64_at(data, off + 12)?, + ); + if prev_entry.is_some_and(|prev| prev >= entry) { + return Err(EngineError::CorruptRecord( + "node timestamp index entries must be sorted ascending and unique".into(), + )); + } + prev_entry = Some(entry); + entries.push(entry); + } + Ok(entries) + } + + pub(crate) fn node_vector_meta_at( + &self, + index: usize, + ) -> Result<(u64, u32, u64, u32), EngineError> { + let data = &self.node_vector_meta_mmap[..]; + if data.is_empty() { + return Ok((0, 0, 0, 0)); + } + let (flags, dense_offset, dense_len, sparse_offset, sparse_len) = + read_node_vector_meta_entry(data, index)?; + Ok(( + if flags & NODE_VECTOR_FLAG_DENSE != 0 { + dense_offset + } else { + 0 + }, + if flags & NODE_VECTOR_FLAG_DENSE != 0 { + dense_len + } else { + 0 + }, + if flags & NODE_VECTOR_FLAG_SPARSE != 0 { + sparse_offset + } else { + 0 + }, + if flags & NODE_VECTOR_FLAG_SPARSE != 0 { + sparse_len + } else { + 0 + }, + )) + } + + /// Number of edge metadata entries. + pub(crate) fn edge_meta_count(&self) -> u64 { + let data = &self.edge_meta_mmap[..]; + if data.len() < 8 { + return 0; + } + read_u64_at(data, 0).unwrap_or(0) + } + + /// Read an edge metadata entry by index (0-based). + /// Returns (edge_id, data_offset, data_len, from, to, label_id, updated_at, + /// weight, valid_from, valid_to, last_write_seq). + #[allow(clippy::type_complexity)] + pub(crate) fn edge_meta_at( + &self, + index: usize, + ) -> Result<(u64, u64, u32, u64, u64, u32, i64, f32, i64, i64, u64), EngineError> { + let data = &self.edge_meta_mmap[..]; + let off = 8 + index * EDGE_META_ENTRY_SIZE; + let edge_id = read_u64_at(data, off)?; + let data_offset = read_u64_at(data, off + 8)?; + let data_len = read_u32_at(data, off + 16)?; + let from = read_u64_at(data, off + 20)?; + let to = read_u64_at(data, off + 28)?; + let label_id = read_u32_at(data, off + 36)?; + let updated_at = read_i64_at(data, off + 40)?; + let weight = read_f32_at(data, off + 48)?; + let valid_from = read_i64_at(data, off + 52)?; + let valid_to = read_i64_at(data, off + 60)?; let last_write_seq = read_u64_at(data, off + 68)?; Ok(( edge_id, @@ -1901,7 +3207,7 @@ impl SegmentReader { data_len, from, to, - type_id, + label_id, updated_at, weight, valid_from, @@ -1910,11 +3216,6 @@ impl SegmentReader { )) } - /// Raw mmap bytes for node_prop_hashes.dat (used by V3 compaction). - pub(crate) fn raw_node_prop_hashes_mmap(&self) -> &[u8] { - &self.node_prop_hashes_mmap[..] - } - pub(crate) fn raw_node_dense_vectors_mmap(&self) -> &[u8] { &self.node_dense_vectors_mmap[..] } @@ -1923,36 +3224,229 @@ impl SegmentReader { &self.node_sparse_vectors_mmap[..] } + #[cfg(test)] pub(crate) fn raw_sparse_posting_index_mmap(&self) -> &[u8] { &self.sparse_posting_index_mmap[..] } + #[cfg(test)] pub(crate) fn raw_sparse_postings_mmap(&self) -> &[u8] { &self.sparse_postings_mmap[..] } pub(crate) fn dense_hnsw_header(&self) -> Option { + if !self.dense_hnsw_available.load(Ordering::Acquire) { + return None; + } self.dense_hnsw_header } + pub(crate) fn dense_vector_count(&self) -> usize { + self.dense_vector_count + } + + pub(crate) fn sparse_vector_count(&self) -> usize { + self.sparse_vector_count + } + + pub(crate) fn sparse_postings_available(&self) -> bool { + self.sparse_postings_available.load(Ordering::Acquire) + && !self.sparse_posting_index_mmap.is_empty() + && !self.sparse_postings_mmap.is_empty() + } + + fn mark_dense_hnsw_unavailable(&self, error: impl ToString) { + self.dense_hnsw_available.store(false, Ordering::Release); + mark_optional_components_corrupt( + &self.component_registry, + &[ + SegmentComponentKind::DenseHnswMetadata, + SegmentComponentKind::DenseHnswGraph, + ], + error.to_string(), + ); + } + + fn mark_sparse_postings_unavailable(&self, error: impl ToString) { + self.sparse_postings_available + .store(false, Ordering::Release); + mark_optional_components_corrupt( + &self.component_registry, + &[ + SegmentComponentKind::SparsePostingIndex, + SegmentComponentKind::SparsePostings, + ], + error.to_string(), + ); + } + + fn dense_hnsw_points_for_search( + &self, + header: DenseHnswHeader, + ) -> Result<&[DenseQueryPoint], EngineError> { + let points = self.dense_hnsw_points.get_or_init(|| { + load_dense_hnsw_query_points(&self.dense_hnsw_meta_mmap, header) + .map(Vec::into_boxed_slice) + .map_err(|error| error.to_string()) + }); + match points { + Ok(points) => Ok(points.as_ref()), + Err(error) => { + self.mark_dense_hnsw_unavailable(error); + Err(EngineError::CorruptRecord(error.clone())) + } + } + } + + fn ensure_sparse_posting_index_shape_for_search(&self) -> Result<(), EngineError> { + let validation = self.sparse_posting_index_shape.get_or_init(|| { + validate_sparse_posting_index_shape_for_search( + &self.sparse_posting_index_mmap, + &self.sparse_postings_mmap, + ) + .map_err(|error| error.to_string()) + }); + match validation { + Ok(()) => Ok(()), + Err(error) => { + self.mark_sparse_postings_unavailable(error); + Err(EngineError::CorruptRecord(error.clone())) + } + } + } + + pub(crate) fn accumulate_sparse_posting_scores( + &self, + query: &[(u32, f32)], + scores: &mut NodeIdMap, + ) -> Result<(), EngineError> { + if !self.sparse_postings_available() { + return Ok(()); + } + self.ensure_sparse_posting_index_shape_for_search()?; + let result = accumulate_sparse_posting_scores_from_bytes( + &self.sparse_posting_index_mmap, + &self.sparse_postings_mmap, + query, + scores, + ); + if let Err(error) = &result { + self.mark_sparse_postings_unavailable(error.to_string()); + } + result + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn exact_dense_vector_search( + &self, + query: &[f32], + metric: DenseMetric, + query_norm: Option, + scope_ids: Option<&NodeIdSet>, + hidden_ids: &NodeIdSet, + mut include: F, + hits_out: &mut Vec, + ) -> Result<(), EngineError> + where + F: FnMut(NodeLabelSet, i64, f32) -> bool, + { + if self.dense_vector_count == 0 { + return Ok(()); + } + + for index in 0..self.node_meta_count() as usize { + let meta = self.node_meta_at(index)?; + if self.deleted_nodes.contains_key(&meta.node_id) + || hidden_ids.contains(&meta.node_id) + || scope_ids.is_some_and(|scope| !scope.contains(&meta.node_id)) + { + continue; + } + let (dense_offset, dense_len, _, _) = self.node_vector_meta_at(index)?; + if dense_len == 0 || !include(meta.label_ids, meta.updated_at, meta.weight) { + continue; + } + hits_out.push(VectorHit { + node_id: meta.node_id, + score: dense_score_from_bytes( + metric, + query, + query_norm, + &self.node_dense_vectors_mmap, + dense_offset as usize, + dense_len as usize, + )?, + }); + } + + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn exact_sparse_vector_scores( + &self, + query: &[(u32, f32)], + scope_ids: Option<&NodeIdSet>, + hidden_ids: &NodeIdSet, + mut include: F, + hits_out: &mut Vec<(u64, f32)>, + ) -> Result<(), EngineError> + where + F: FnMut(NodeLabelSet, i64, f32) -> bool, + { + if self.sparse_vector_count == 0 || query.is_empty() { + return Ok(()); + } + + for index in 0..self.node_meta_count() as usize { + let meta = self.node_meta_at(index)?; + if self.deleted_nodes.contains_key(&meta.node_id) + || hidden_ids.contains(&meta.node_id) + || scope_ids.is_some_and(|scope| !scope.contains(&meta.node_id)) + { + continue; + } + let (_, _, sparse_offset, sparse_len) = self.node_vector_meta_at(index)?; + if sparse_len == 0 || !include(meta.label_ids, meta.updated_at, meta.weight) { + continue; + } + let score = sparse_dot_score_from_blob( + query, + &self.node_sparse_vectors_mmap, + sparse_offset as usize, + sparse_len as usize, + )?; + if score > 0.0 { + hits_out.push((meta.node_id, score)); + } + } + + Ok(()) + } + pub(crate) fn search_dense_hnsw( &self, query: &[f32], ef_search: usize, limit: usize, ) -> Result, EngineError> { - let Some(header) = self.dense_hnsw_header else { + let Some(header) = self.dense_hnsw_header() else { return Ok(Vec::new()); }; - search_dense_hnsw_with_points( + let points = self.dense_hnsw_points_for_search(header)?; + let result = search_dense_hnsw_with_points( header, - &self.dense_hnsw_points, + points, &self.dense_hnsw_graph_mmap, &self.node_dense_vectors_mmap, query, ef_search, limit, - ) + ); + if let Err(error) = &result { + self.mark_dense_hnsw_unavailable(error.to_string()); + } + result } pub(crate) fn search_dense_hnsw_scoped( @@ -1962,19 +3456,24 @@ impl SegmentReader { limit: usize, scope_ids: &crate::types::NodeIdSet, ) -> Result, EngineError> { - let Some(header) = self.dense_hnsw_header else { + let Some(header) = self.dense_hnsw_header() else { return Ok(Vec::new()); }; - search_dense_hnsw_scoped_with_points( + let points = self.dense_hnsw_points_for_search(header)?; + let result = search_dense_hnsw_scoped_with_points( header, - &self.dense_hnsw_points, + points, &self.dense_hnsw_graph_mmap, &self.node_dense_vectors_mmap, query, ef_search, limit, scope_ids, - ) + ); + if let Err(error) = &result { + self.mark_dense_hnsw_unavailable(error.to_string()); + } + result } pub(crate) fn raw_dense_hnsw_meta_mmap(&self) -> &[u8] { @@ -2021,6 +3520,7 @@ impl SegmentReader { /// Collect all node records in this segment (including tombstoned ones). /// Returns records in index order (sorted by node_id). + #[cfg(test)] pub fn all_nodes(&self) -> Result, EngineError> { let data = &self.nodes_mmap[..]; if data.len() < 8 { @@ -2040,69 +3540,50 @@ impl SegmentReader { Ok(nodes) } - /// Collect all edge records in this segment (including tombstoned ones). - /// Returns records in index order (sorted by edge_id). - pub fn all_edges(&self) -> Result, EngineError> { - let data = &self.edges_mmap[..]; - if data.len() < 8 { - return Ok(Vec::new()); - } - let count = read_u64_at(data, 0)? as usize; - let idx_start = 8; - let mut edges = Vec::with_capacity(count); - for i in 0..count { - let entry_off = idx_start + i * EDGE_INDEX_ENTRY_SIZE; - let id = read_u64_at(data, entry_off)?; - let offset = read_u64_at(data, entry_off + 8)? as usize; - edges.push(decode_edge_at(data, offset, id)?); - } - Ok(edges) - } - - // --- Type index queries --- + // --- Label posting index queries --- - /// Return node IDs for a given type_id from this segment's type index. + /// Return node IDs for a given label_id from this segment's node-label index. /// Excludes tombstoned nodes. - pub fn nodes_by_type(&self, type_id: u32) -> Result, EngineError> { - self.query_type_index(&self.node_type_index_mmap, type_id, &self.deleted_nodes) + pub fn nodes_by_label_id(&self, label_id: u32) -> Result, EngineError> { + self.query_label_posting_index(&self.node_label_index_mmap, label_id, &self.deleted_nodes) } - /// Return the posting count for a given node type without decoding the posting list. - pub(crate) fn node_type_posting_count(&self, type_id: u32) -> Result { - self.type_index_posting_count(&self.node_type_index_mmap, type_id) + /// Return the posting count for a given node label without decoding the posting list. + pub(crate) fn node_label_posting_count(&self, label_id: u32) -> Result { + self.label_posting_index_count(&self.node_label_index_mmap, label_id) } - pub(crate) fn node_type_posting( + pub(crate) fn node_label_posting( &self, - type_id: u32, - ) -> Result, EngineError> { - self.type_index_posting(&self.node_type_index_mmap, type_id) + label_id: u32, + ) -> Result, EngineError> { + self.label_posting_index(&self.node_label_index_mmap, label_id) } - pub(crate) fn node_type_id_at_posting( + pub(crate) fn node_id_at_label_posting( &self, - posting: SegmentTypePosting, + posting: SegmentLabelPosting, index: usize, ) -> Result, EngineError> { if index >= posting.count { return Ok(None); } Ok(Some(read_u64_at( - &self.node_type_index_mmap, + &self.node_label_index_mmap, posting.offset + index * 8, )?)) } - pub(crate) fn node_type_id_lower_bound_posting( + pub(crate) fn node_label_posting_lower_bound( &self, - posting: SegmentTypePosting, + posting: SegmentLabelPosting, after: u64, ) -> Result { let mut lo = 0usize; let mut hi = posting.count; while lo < hi { let mid = lo + (hi - lo) / 2; - let node_id = read_u64_at(&self.node_type_index_mmap, posting.offset + mid * 8)?; + let node_id = read_u64_at(&self.node_label_index_mmap, posting.offset + mid * 8)?; if node_id <= after { lo = mid + 1; } else { @@ -2112,18 +3593,62 @@ impl SegmentReader { Ok(lo) } - /// Return edge IDs for a given type_id from this segment's type index. + /// Return edge IDs for a given label_id from this segment's label index. /// Excludes tombstoned edges. - pub fn edges_by_type(&self, type_id: u32) -> Result, EngineError> { - self.query_type_index(&self.edge_type_index_mmap, type_id, &self.deleted_edges) + pub fn edges_by_label_id(&self, label_id: u32) -> Result, EngineError> { + self.query_label_posting_index(&self.edge_label_index_mmap, label_id, &self.deleted_edges) + } + + pub(crate) fn edge_label_posting_count(&self, label_id: u32) -> Result { + self.label_posting_index_count(&self.edge_label_index_mmap, label_id) + } + + pub(crate) fn edge_label_posting( + &self, + label_id: u32, + ) -> Result, EngineError> { + self.label_posting_index(&self.edge_label_index_mmap, label_id) + } + + pub(crate) fn edge_label_id_at_posting( + &self, + posting: SegmentLabelPosting, + index: usize, + ) -> Result, EngineError> { + if index >= posting.count { + return Ok(None); + } + Ok(Some(read_u64_at( + &self.edge_label_index_mmap, + posting.offset + index * 8, + )?)) + } + + pub(crate) fn edge_label_id_lower_bound_posting( + &self, + posting: SegmentLabelPosting, + after: u64, + ) -> Result { + let mut lo = 0usize; + let mut hi = posting.count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let edge_id = read_u64_at(&self.edge_label_index_mmap, posting.offset + mid * 8)?; + if edge_id <= after { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) } - /// Binary search a type index file for a given type_id. + /// Binary search a label posting index file for a given label ID. /// Returns record IDs, excluding any in the deleted set. - fn query_type_index( + fn query_label_posting_index( &self, mmap: &MappedData, - target_type: u32, + target_label_id: u32, deleted: &NodeIdMap, ) -> Result, EngineError> { let data = &mmap[..]; @@ -2135,20 +3660,20 @@ impl SegmentReader { return Ok(Vec::new()); } - // Binary search the index section for target_type + // Binary search the index section for target_label_id. let idx_start = 8; - // Entry: type_id (4) + offset (8) + count (4) = 16 bytes - let entry_size = TYPE_INDEX_ENTRY_SIZE; + // Entry: label_id (4) + offset (8) + count (4) = 16 bytes. + let entry_size = LABEL_POSTING_INDEX_ENTRY_SIZE; let mut lo = 0usize; let mut hi = count; while lo < hi { let mid = lo + (hi - lo) / 2; let entry_off = idx_start + mid * entry_size; - let entry_type = read_u32_at(data, entry_off)?; - if entry_type < target_type { + let entry_label_id = read_u32_at(data, entry_off)?; + if entry_label_id < target_label_id { lo = mid + 1; - } else if entry_type > target_type { + } else if entry_label_id > target_label_id { hi = mid; } else { // Found, read the IDs @@ -2168,22 +3693,22 @@ impl SegmentReader { Ok(Vec::new()) } - fn type_index_posting_count( + fn label_posting_index_count( &self, mmap: &MappedData, - target_type: u32, + target_label_id: u32, ) -> Result { Ok(self - .type_index_posting(mmap, target_type)? + .label_posting_index(mmap, target_label_id)? .map(|posting| posting.count) .unwrap_or(0)) } - fn type_index_posting( + fn label_posting_index( &self, mmap: &MappedData, - target_type: u32, - ) -> Result, EngineError> { + target_label_id: u32, + ) -> Result, EngineError> { let data = &mmap[..]; if data.len() < 8 { return Ok(None); @@ -2194,15 +3719,15 @@ impl SegmentReader { } let idx_start = 8; - let entry_size = TYPE_INDEX_ENTRY_SIZE; + let entry_size = LABEL_POSTING_INDEX_ENTRY_SIZE; let mut lo = 0usize; let mut hi = count; while lo < hi { let mid = lo + (hi - lo) / 2; let entry_off = idx_start + mid * entry_size; - let entry_type = read_u32_at(data, entry_off)?; - match entry_type.cmp(&target_type) { + let entry_label_id = read_u32_at(data, entry_off)?; + match entry_label_id.cmp(&target_label_id) { std::cmp::Ordering::Less => lo = mid + 1, std::cmp::Ordering::Greater => hi = mid, std::cmp::Ordering::Equal => { @@ -2210,20 +3735,24 @@ impl SegmentReader { let id_count = read_u32_at(data, entry_off + 12)? as usize; let end = offset .checked_add(id_count.checked_mul(8).ok_or_else(|| { - EngineError::CorruptRecord("type index posting overflow".into()) + EngineError::CorruptRecord( + "label posting index payload overflow".into(), + ) })?) .ok_or_else(|| { - EngineError::CorruptRecord("type index posting end overflow".into()) + EngineError::CorruptRecord( + "label posting index payload end overflow".into(), + ) })?; if end > data.len() { return Err(EngineError::CorruptRecord(format!( - "type index posting [{}, {}) exceeds file length {}", + "label posting index payload [{}, {}) exceeds file length {}", offset, end, data.len() ))); } - return Ok(Some(SegmentTypePosting { + return Ok(Some(SegmentLabelPosting { offset, count: id_count, })); @@ -2234,18 +3763,13 @@ impl SegmentReader { Ok(None) } - /// Return all distinct node type IDs present in this segment's type index. - pub fn node_type_ids(&self) -> Result, EngineError> { - Self::type_ids_from_index(&self.node_type_index_mmap) + /// Return all distinct node label IDs present in this segment's label index. + pub fn node_label_ids(&self) -> Result, EngineError> { + Self::label_posting_index_ids(&self.node_label_index_mmap) } - /// Return all distinct edge type IDs present in this segment's type index. - pub fn edge_type_ids(&self) -> Result, EngineError> { - Self::type_ids_from_index(&self.edge_type_index_mmap) - } - - /// Extract all type IDs from a type index mmap header. - fn type_ids_from_index(mmap: &MappedData) -> Result, EngineError> { + /// Extract all label IDs from a label posting index mmap header. + fn label_posting_index_ids(mmap: &MappedData) -> Result, EngineError> { let data = &mmap[..]; if data.len() < 8 { return Ok(Vec::new()); @@ -2254,7 +3778,7 @@ impl SegmentReader { let mut result = Vec::with_capacity(count); let idx_start = 8; for i in 0..count { - let entry_off = idx_start + i * TYPE_INDEX_ENTRY_SIZE; + let entry_off = idx_start + i * LABEL_POSTING_INDEX_ENTRY_SIZE; result.push(read_u32_at(data, entry_off)?); } Ok(result) @@ -2262,19 +3786,19 @@ impl SegmentReader { // --- Timestamp index queries --- - /// Return node IDs within a time range for a given type_id. + /// Return node IDs within a time range for a given label_id. /// Binary search for range start, scan to range end. O(log N + results). /// Results are sorted by node_id for K-way merge compatibility. pub fn nodes_by_time_range( &self, - type_id: u32, + label_id: u32, from_ms: i64, to_ms: i64, ) -> Result, EngineError> { let data = &self.timestamp_index_mmap[..]; if data.len() < 8 { return Err(EngineError::CorruptRecord( - "timestamp_index.dat missing or truncated (< 8 bytes)".into(), + "timestamp index missing or truncated (< 8 bytes)".into(), )); } let count = read_u64_at(data, 0)? as usize; @@ -2283,30 +3807,30 @@ impl SegmentReader { } let entry_start = 8usize; - let entry_size = 20usize; // type_id(4) + updated_at(8) + node_id(8) + let entry_size = 20usize; // label_id(4) + updated_at(8) + node_id(8) - // Binary search for the first entry >= (type_id, from_ms, 0) + // Binary search for the first entry >= (label_id, from_ms, 0) let mut lo = 0usize; let mut hi = count; while lo < hi { let mid = lo + (hi - lo) / 2; let off = entry_start + mid * entry_size; - let e_type = read_u32_at(data, off)?; + let entry_label_id = read_u32_at(data, off)?; let e_time = read_i64_at(data, off + 4)?; - if (e_type, e_time) < (type_id, from_ms) { + if (entry_label_id, e_time) < (label_id, from_ms) { lo = mid + 1; } else { hi = mid; } } - // Scan from lo until type_id changes or updated_at > to_ms + // Scan from lo until label_id changes or updated_at > to_ms let mut result = Vec::new(); let mut pos = lo; while pos < count { let off = entry_start + pos * entry_size; - let e_type = read_u32_at(data, off)?; - if e_type != type_id { + let entry_label_id = read_u32_at(data, off)?; + if entry_label_id != label_id { break; } let e_time = read_i64_at(data, off + 4)?; @@ -2327,7 +3851,7 @@ impl SegmentReader { pub(crate) fn for_each_node_by_time_range( &self, - type_id: u32, + label_id: u32, from_ms: i64, to_ms: i64, mut callback: F, @@ -2338,7 +3862,7 @@ impl SegmentReader { let data = &self.timestamp_index_mmap[..]; if data.len() < 8 { return Err(EngineError::CorruptRecord( - "timestamp_index.dat missing or truncated (< 8 bytes)".into(), + "timestamp index missing or truncated (< 8 bytes)".into(), )); } let count = read_u64_at(data, 0)? as usize; @@ -2354,9 +3878,9 @@ impl SegmentReader { while lo < hi { let mid = lo + (hi - lo) / 2; let off = entry_start + mid * entry_size; - let e_type = read_u32_at(data, off)?; + let entry_label_id = read_u32_at(data, off)?; let e_time = read_i64_at(data, off + 4)?; - if (e_type, e_time) < (type_id, from_ms) { + if (entry_label_id, e_time) < (label_id, from_ms) { lo = mid + 1; } else { hi = mid; @@ -2366,8 +3890,8 @@ impl SegmentReader { let mut pos = lo; while pos < count { let off = entry_start + pos * entry_size; - let e_type = read_u32_at(data, off)?; - if e_type != type_id { + let entry_label_id = read_u32_at(data, off)?; + if entry_label_id != label_id { break; } let e_time = read_i64_at(data, off + 4)?; @@ -2386,13 +3910,13 @@ impl SegmentReader { // --- Edge triple index --- - /// Look up an edge by (from, to, type_id) triple. Returns the edge record + /// Look up an edge by (from, to, label_id) triple. Returns the edge record /// if found and not tombstoned, or None. pub fn edge_by_triple( &self, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result, EngineError> { let data = &self.edge_triple_index_mmap[..]; if data.len() < 8 { @@ -2412,7 +3936,7 @@ impl SegmentReader { let off = entries_start + mid * EDGE_TRIPLE_ENTRY_SIZE; let e_from = read_u64_at(data, off)?; let e_to = read_u64_at(data, off + 8)?; - let e_type = read_u32_at(data, off + 16)?; + let e_label_id = read_u32_at(data, off + 16)?; match e_from.cmp(&from) { std::cmp::Ordering::Less => lo = mid + 1, @@ -2420,7 +3944,7 @@ impl SegmentReader { std::cmp::Ordering::Equal => match e_to.cmp(&to) { std::cmp::Ordering::Less => lo = mid + 1, std::cmp::Ordering::Greater => hi = mid, - std::cmp::Ordering::Equal => match e_type.cmp(&type_id) { + std::cmp::Ordering::Equal => match e_label_id.cmp(&label_id) { std::cmp::Ordering::Less => lo = mid + 1, std::cmp::Ordering::Greater => hi = mid, std::cmp::Ordering::Equal => { @@ -2435,101 +3959,1290 @@ impl SegmentReader { Ok(None) } - // --- Property index queries --- - - /// Find candidate node IDs matching (type_id, key_hash, value_hash) from - /// this segment's property index. Excludes tombstoned nodes. - /// Returns candidate IDs. Caller must post-filter with actual property values. - pub fn find_nodes_by_prop_hash( + pub fn resolve_triples_batch( &self, - type_id: u32, - key_hash: u64, - value_hash: u64, - ) -> Result, EngineError> { - let data = &self.prop_node_index_mmap[..]; - if data.len() < 8 { + lookups: &[(usize, u64, u64, u32)], + results: &mut [Option], + ) -> Result, EngineError> { + if lookups.is_empty() { return Ok(Vec::new()); } - let count = read_u64_at(data, 0)? as usize; - if count == 0 { - return Ok(Vec::new()); + + let resolved = self.resolve_triples_to_ids(lookups)?; + if resolved.is_empty() { + return Ok(Vec::new()); + } + + let found_indices: Vec = resolved.iter().map(|&(orig_idx, _)| orig_idx).collect(); + let mut edge_lookups: Vec<(usize, u64)> = resolved + .iter() + .filter(|&&(_, eid)| !self.deleted_edges.contains_key(&eid)) + .copied() + .collect(); + edge_lookups.sort_unstable_by_key(|&(_, eid)| eid); + self.get_edges_batch(&edge_lookups, results)?; + + Ok(found_indices) + } + + fn resolve_triples_to_ids( + &self, + lookups: &[(usize, u64, u64, u32)], + ) -> Result, EngineError> { + let mut resolved = Vec::new(); + let data = &self.edge_triple_index_mmap[..]; + if data.len() < 8 { + return Ok(resolved); + } + let count = read_u64_at(data, 0)? as usize; + if count == 0 { + return Ok(resolved); + } + + let entries_start = 8; + let unique_triples = { + let mut n = 0usize; + let mut prev: Option<(u64, u64, u32)> = None; + for &(_, from, to, label_id) in lookups { + if prev != Some((from, to, label_id)) { + n += 1; + prev = Some((from, to, label_id)); + } + } + n + }; + + let strategy = if unique_triples <= 2 || count <= 1 { + BatchReadStrategy::SeekPerKey + } else { + let min_triple = (lookups[0].1, lookups[0].2, lookups[0].3); + let last = lookups[lookups.len() - 1]; + let max_triple = (last.1, last.2, last.3); + let span_start = lower_bound_edge_triple_index(data, entries_start, count, min_triple)?; + let span_end = upper_bound_edge_triple_index(data, entries_start, count, max_triple)?; + let span = span_end.saturating_sub(span_start).max(unique_triples); + let seek_cost = unique_triples + .saturating_mul(ceil_log2_usize(count)) + .saturating_mul(BATCH_RANDOM_ACCESS_PENALTY); + if seek_cost <= span { + BatchReadStrategy::SeekPerKey + } else { + BatchReadStrategy::MergeWalk + } + }; + + if strategy == BatchReadStrategy::SeekPerKey { + let mut prev_query: Option<(u64, u64, u32)> = None; + let mut prev_edge_id: Option = None; + for &(orig_idx, from, to, label_id) in lookups { + let edge_id = if prev_query == Some((from, to, label_id)) { + prev_edge_id + } else { + let found = binary_search_edge_triple_index( + data, + entries_start, + count, + (from, to, label_id), + )?; + prev_query = Some((from, to, label_id)); + prev_edge_id = found; + found + }; + if let Some(eid) = edge_id { + resolved.push((orig_idx, eid)); + } + } + } else { + let mut idx_pos = 0usize; + let mut prev_query: Option<(u64, u64, u32)> = None; + let mut prev_edge_id: Option = None; + for &(orig_idx, from, to, label_id) in lookups { + if prev_query == Some((from, to, label_id)) { + if let Some(eid) = prev_edge_id { + resolved.push((orig_idx, eid)); + } + continue; + } + prev_query = Some((from, to, label_id)); + prev_edge_id = None; + + while idx_pos < count { + let off = entries_start + idx_pos * EDGE_TRIPLE_ENTRY_SIZE; + let entry_from = read_u64_at(data, off)?; + let entry_to = read_u64_at(data, off + 8)?; + let entry_label_id = read_u32_at(data, off + 16)?; + match (entry_from, entry_to, entry_label_id).cmp(&(from, to, label_id)) { + std::cmp::Ordering::Less => { + idx_pos += 1; + } + std::cmp::Ordering::Equal => { + let edge_id = read_u64_at(data, off + 20)?; + prev_edge_id = Some(edge_id); + resolved.push((orig_idx, edge_id)); + break; + } + std::cmp::Ordering::Greater => { + break; + } + } + } + } + } + + Ok(resolved) + } + + pub(crate) fn edge_ids_by_triple( + &self, + from: u64, + to: u64, + label_id: u32, + ) -> Result, EngineError> { + let data = &self.edge_triple_index_mmap[..]; + if data.len() < 8 { + return Ok(Vec::new()); + } + let count = read_u64_at(data, 0)? as usize; + if count == 0 { + return Ok(Vec::new()); + } + + let entries_start = 8; + let mut lo = 0usize; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = entries_start + mid * EDGE_TRIPLE_ENTRY_SIZE; + let e_from = read_u64_at(data, off)?; + let e_to = read_u64_at(data, off + 8)?; + let e_label_id = read_u32_at(data, off + 16)?; + if (e_from, e_to, e_label_id) < (from, to, label_id) { + lo = mid + 1; + } else { + hi = mid; + } + } + + let mut ids = Vec::new(); + let mut pos = lo; + while pos < count { + let off = entries_start + pos * EDGE_TRIPLE_ENTRY_SIZE; + let e_from = read_u64_at(data, off)?; + let e_to = read_u64_at(data, off + 8)?; + let e_label_id = read_u32_at(data, off + 16)?; + if (e_from, e_to, e_label_id) != (from, to, label_id) { + break; + } + let edge_id = read_u64_at(data, off + 20)?; + if !self.deleted_edges.contains_key(&edge_id) { + ids.push(edge_id); + } + pos += 1; + } + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } + + pub(crate) fn edge_weight_index_available(&self) -> bool { + self.edge_weight_index_count.is_some() + && self + .component_registry + .availability(&SegmentComponentKind::EdgeWeightIndex) + .is_available() + } + + pub(crate) fn edge_updated_at_index_available(&self) -> bool { + self.edge_i64_metadata_index_available(EDGE_UPDATED_AT_INDEX_LOGICAL_NAME) + } + + pub(crate) fn edge_valid_from_index_available(&self) -> bool { + self.edge_i64_metadata_index_available(EDGE_VALID_FROM_INDEX_LOGICAL_NAME) + } + + pub(crate) fn edge_valid_to_index_available(&self) -> bool { + self.edge_i64_metadata_index_available(EDGE_VALID_TO_INDEX_LOGICAL_NAME) + } + + fn edge_i64_metadata_index_available(&self, logical_name: &str) -> bool { + let Some(kind) = edge_i64_metadata_component_kind(logical_name) else { + return false; + }; + self.edge_i64_metadata_index_count(logical_name).is_some() + && self.component_registry.availability(&kind).is_available() + } + + fn mark_edge_metadata_component_corrupt( + &self, + kind: SegmentComponentKind, + error: &EngineError, + ) { + mark_optional_component_corrupt(&self.component_registry, kind, error.to_string()); + } + + fn edge_i64_metadata_index_count(&self, logical_name: &str) -> Option { + match logical_name { + EDGE_UPDATED_AT_INDEX_LOGICAL_NAME => self.edge_updated_at_index_count, + EDGE_VALID_FROM_INDEX_LOGICAL_NAME => self.edge_valid_from_index_count, + EDGE_VALID_TO_INDEX_LOGICAL_NAME => self.edge_valid_to_index_count, + _ => None, + } + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_weight_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option> { + self.edge_ids_by_weight_range_limited(label_id, bounds, usize::MAX) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_weight_range_limited( + &self, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Option> { + let data = &self.edge_weight_index_mmap[..]; + if !self.edge_weight_index_available() { + return None; + } + let count = self.edge_weight_index_count?; + match self.edge_ids_by_weight_range_inner(data, count, label_id, bounds, limit) { + Ok(ids) => Some(ids), + Err(error) => { + self.mark_edge_metadata_component_corrupt( + SegmentComponentKind::EdgeWeightIndex, + &error, + ); + None + } + } + } + + pub(crate) fn for_each_edge_id_by_weight_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + callback: &mut F, + ) -> Result>, EngineError> + where + F: FnMut(u64) -> ControlFlow<()>, + { + let data = &self.edge_weight_index_mmap[..]; + if !self.edge_weight_index_available() { + return Ok(None); + } + let count = self.edge_weight_index_count.expect("availability checked"); + match self.for_each_edge_id_by_weight_range_inner(data, count, label_id, bounds, callback) { + Ok(flow) => Ok(Some(flow)), + Err(error) => { + self.mark_edge_metadata_component_corrupt( + SegmentComponentKind::EdgeWeightIndex, + &error, + ); + Err(error) + } + } + } + + pub(crate) fn edge_weight_range_count( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option { + let data = &self.edge_weight_index_mmap[..]; + if !self.edge_weight_index_available() { + return None; + } + let count = self.edge_weight_index_count?; + match self.edge_weight_range_count_inner(data, count, label_id, bounds) { + Ok(count) => Some(count), + Err(error) => { + self.mark_edge_metadata_component_corrupt( + SegmentComponentKind::EdgeWeightIndex, + &error, + ); + None + } + } + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_updated_at_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option> { + self.edge_ids_by_updated_at_range_limited(label_id, bounds, usize::MAX) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_updated_at_range_limited( + &self, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Option> { + self.edge_ids_by_i64_metadata_range( + &self.edge_updated_at_index_mmap, + EDGE_UPDATED_AT_INDEX_LOGICAL_NAME, + label_id, + bounds, + limit, + ) + } + + pub(crate) fn for_each_edge_id_by_updated_at_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + callback: &mut F, + ) -> Result>, EngineError> + where + F: FnMut(u64) -> ControlFlow<()>, + { + self.for_each_edge_id_by_i64_metadata_range( + &self.edge_updated_at_index_mmap, + EDGE_UPDATED_AT_INDEX_LOGICAL_NAME, + label_id, + bounds, + callback, + ) + } + + pub(crate) fn edge_updated_at_range_count( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option { + self.edge_i64_metadata_range_count( + &self.edge_updated_at_index_mmap, + EDGE_UPDATED_AT_INDEX_LOGICAL_NAME, + label_id, + bounds, + ) + } + + pub(crate) fn for_each_edge_id_by_valid_from_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + callback: &mut F, + ) -> Result>, EngineError> + where + F: FnMut(u64) -> ControlFlow<()>, + { + self.for_each_edge_id_by_i64_metadata_range( + &self.edge_valid_from_index_mmap, + EDGE_VALID_FROM_INDEX_LOGICAL_NAME, + label_id, + bounds, + callback, + ) + } + + pub(crate) fn edge_valid_from_range_count( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option { + self.edge_i64_metadata_range_count( + &self.edge_valid_from_index_mmap, + EDGE_VALID_FROM_INDEX_LOGICAL_NAME, + label_id, + bounds, + ) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_valid_to_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option> { + self.edge_ids_by_valid_to_range_limited(label_id, bounds, usize::MAX) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_valid_to_range_limited( + &self, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Option> { + self.edge_ids_by_i64_metadata_range( + &self.edge_valid_to_index_mmap, + EDGE_VALID_TO_INDEX_LOGICAL_NAME, + label_id, + bounds, + limit, + ) + } + + pub(crate) fn for_each_edge_id_by_valid_to_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + callback: &mut F, + ) -> Result>, EngineError> + where + F: FnMut(u64) -> ControlFlow<()>, + { + self.for_each_edge_id_by_i64_metadata_range( + &self.edge_valid_to_index_mmap, + EDGE_VALID_TO_INDEX_LOGICAL_NAME, + label_id, + bounds, + callback, + ) + } + + pub(crate) fn edge_valid_to_range_count( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option { + self.edge_i64_metadata_range_count( + &self.edge_valid_to_index_mmap, + EDGE_VALID_TO_INDEX_LOGICAL_NAME, + label_id, + bounds, + ) + } + + #[cfg(test)] + fn edge_ids_by_weight_range_inner( + &self, + data: &[u8], + count: usize, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Result, EngineError> { + let lower_key = match bounds.lower { + Some(lower) => match encode_edge_weight_key(lower) { + Some(key) => Some(key), + None => return Ok(Vec::new()), + }, + None => None, + }; + let upper_key = match bounds.upper { + Some(upper) => match encode_edge_weight_key(upper) { + Some(key) => Some(key), + None => return Ok(Vec::new()), + }, + None => None, + }; + let mut ids = Vec::new(); + let start = match label_id { + Some(target) => { + self.edge_weight_label_value_lower_bound(data, count, target, lower_key)? + } + None => 0, + }; + let mut pos = start; + while pos < count { + let off = 8 + pos * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + let entry_label_id = read_u32_at(data, off)?; + if label_id.is_some_and(|target| entry_label_id != target) { + break; + } + let value = read_u32_at(data, off + 4)?; + if label_id.is_some() + && upper_key.is_some_and(|upper| { + if bounds.upper_inclusive { + value > upper + } else { + value >= upper + } + }) + { + break; + } + if key_matches_bounds( + value, + lower_key, + bounds.lower_inclusive, + upper_key, + bounds.upper_inclusive, + ) { + let edge_id = read_u64_at(data, off + 8)?; + if !self.deleted_edges.contains_key(&edge_id) { + ids.push(edge_id); + if ids.len() > limit { + break; + } + } + } + pos += 1; + } + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } + + fn for_each_edge_id_by_weight_range_inner( + &self, + data: &[u8], + count: usize, + label_id: Option, + bounds: RangeBoundFlags, + callback: &mut F, + ) -> Result, EngineError> + where + F: FnMut(u64) -> ControlFlow<()>, + { + let lower_key = match bounds.lower { + Some(lower) => match encode_edge_weight_key(lower) { + Some(key) => Some(key), + None => return Ok(ControlFlow::Continue(())), + }, + None => None, + }; + let upper_key = match bounds.upper { + Some(upper) => match encode_edge_weight_key(upper) { + Some(key) => Some(key), + None => return Ok(ControlFlow::Continue(())), + }, + None => None, + }; + let start = match label_id { + Some(target) => { + self.edge_weight_label_value_lower_bound(data, count, target, lower_key)? + } + None => 0, + }; + let mut pos = start; + while pos < count { + let off = 8 + pos * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + let entry_label_id = read_u32_at(data, off)?; + if label_id.is_some_and(|target| entry_label_id != target) { + break; + } + let value = read_u32_at(data, off + 4)?; + if label_id.is_some() + && upper_key.is_some_and(|upper| { + if bounds.upper_inclusive { + value > upper + } else { + value >= upper + } + }) + { + break; + } + if key_matches_bounds( + value, + lower_key, + bounds.lower_inclusive, + upper_key, + bounds.upper_inclusive, + ) { + let edge_id = read_u64_at(data, off + 8)?; + if !self.deleted_edges.contains_key(&edge_id) && callback(edge_id).is_break() { + return Ok(ControlFlow::Break(())); + } + } + pos += 1; + } + Ok(ControlFlow::Continue(())) + } + + fn edge_weight_label_value_lower_bound( + &self, + data: &[u8], + count: usize, + target_label: u32, + lower_key: Option, + ) -> Result { + let mut lo = 0usize; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + let entry = ( + read_u32_at(data, off)?, + read_u32_at(data, off + 4)?, + read_u64_at(data, off + 8)?, + ); + let target = (target_label, lower_key.unwrap_or(0), 0); + if entry < target { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } + + fn edge_weight_label_lower_bound( + &self, + data: &[u8], + count: usize, + target_label: u32, + ) -> Result { + let mut lo = 0usize; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + if read_u32_at(data, off)? < target_label { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } + + fn edge_weight_label_upper_bound( + &self, + data: &[u8], + count: usize, + target_label: u32, + ) -> Result { + let mut lo = 0usize; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + if read_u32_at(data, off)? <= target_label { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } + + fn edge_weight_value_lower_bound_in_span( + &self, + data: &[u8], + start: usize, + end: usize, + target_value: u32, + ) -> Result { + let mut lo = start; + let mut hi = end; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + if read_u32_at(data, off + 4)? < target_value { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } + + fn edge_weight_value_upper_bound_in_span( + &self, + data: &[u8], + start: usize, + end: usize, + target_value: u32, + ) -> Result { + let mut lo = start; + let mut hi = end; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + if read_u32_at(data, off + 4)? <= target_value { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } + + fn edge_weight_range_count_in_label_span( + &self, + data: &[u8], + start: usize, + end: usize, + bounds: RangeBoundFlags, + ) -> Result { + if start >= end { + return Ok(0); + } + let range_start = match bounds.lower { + Some(lower) if bounds.lower_inclusive => { + self.edge_weight_value_lower_bound_in_span(data, start, end, lower)? + } + Some(lower) => self.edge_weight_value_upper_bound_in_span(data, start, end, lower)?, + None => start, + }; + let range_end = match bounds.upper { + Some(upper) if bounds.upper_inclusive => { + self.edge_weight_value_upper_bound_in_span(data, start, end, upper)? + } + Some(upper) => self.edge_weight_value_lower_bound_in_span(data, start, end, upper)?, + None => end, + }; + Ok(range_end.saturating_sub(range_start)) + } + + fn edge_weight_range_count_inner( + &self, + data: &[u8], + count: usize, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Result { + let lower_key = match bounds.lower { + Some(lower) => match encode_edge_weight_key(lower) { + Some(key) => Some(key), + None => return Ok(0), + }, + None => None, + }; + let upper_key = match bounds.upper { + Some(upper) => match encode_edge_weight_key(upper) { + Some(key) => Some(key), + None => return Ok(0), + }, + None => None, + }; + let key_bounds = RangeBoundFlags { + lower: lower_key, + lower_inclusive: bounds.lower_inclusive, + upper: upper_key, + upper_inclusive: bounds.upper_inclusive, + }; + if count == 0 { + return Ok(0); + } + if let Some(target) = label_id { + let start = self.edge_weight_label_lower_bound(data, count, target)?; + if start == count { + return Ok(0); + } + let off = 8 + start * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + if read_u32_at(data, off)? != target { + return Ok(0); + } + let end = self.edge_weight_label_upper_bound(data, count, target)?; + return self.edge_weight_range_count_in_label_span(data, start, end, key_bounds); + } + if lower_key.is_none() && upper_key.is_none() { + return Ok(count); + } + let mut matched = 0usize; + let mut pos = 0usize; + while pos < count { + let off = 8 + pos * EDGE_WEIGHT_INDEX_ENTRY_SIZE; + let entry_label_id = read_u32_at(data, off)?; + let end = self.edge_weight_label_upper_bound(data, count, entry_label_id)?; + matched = matched.saturating_add( + self.edge_weight_range_count_in_label_span(data, pos, end, key_bounds)?, + ); + pos = end; + } + Ok(matched) + } + + #[cfg(test)] + fn edge_ids_by_i64_metadata_range( + &self, + mmap: &MappedData, + logical_name: &str, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Option> { + let data = &mmap[..]; + if !self.edge_i64_metadata_index_available(logical_name) { + return None; + } + let count = self.edge_i64_metadata_index_count(logical_name)?; + match self.edge_ids_by_i64_metadata_range_inner(data, count, label_id, bounds, limit) { + Ok(ids) => Some(ids), + Err(error) => { + if let Some(kind) = edge_i64_metadata_component_kind(logical_name) { + self.mark_edge_metadata_component_corrupt(kind, &error); + } + None + } + } + } + + fn for_each_edge_id_by_i64_metadata_range( + &self, + mmap: &MappedData, + logical_name: &str, + label_id: Option, + bounds: RangeBoundFlags, + callback: &mut F, + ) -> Result>, EngineError> + where + F: FnMut(u64) -> ControlFlow<()>, + { + let data = &mmap[..]; + if !self.edge_i64_metadata_index_available(logical_name) { + return Ok(None); + } + let count = self + .edge_i64_metadata_index_count(logical_name) + .expect("availability checked"); + match self + .for_each_edge_id_by_i64_metadata_range_inner(data, count, label_id, bounds, callback) + { + Ok(flow) => Ok(Some(flow)), + Err(error) => { + if let Some(kind) = edge_i64_metadata_component_kind(logical_name) { + self.mark_edge_metadata_component_corrupt(kind, &error); + } + Err(error) + } + } + } + + fn edge_i64_metadata_range_count( + &self, + mmap: &MappedData, + logical_name: &str, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Option { + let data = &mmap[..]; + if !self.edge_i64_metadata_index_available(logical_name) { + return None; + } + let count = self.edge_i64_metadata_index_count(logical_name)?; + match self.edge_i64_metadata_range_count_inner(data, count, label_id, bounds) { + Ok(count) => Some(count), + Err(error) => { + if let Some(kind) = edge_i64_metadata_component_kind(logical_name) { + self.mark_edge_metadata_component_corrupt(kind, &error); + } + None + } + } + } + + #[cfg(test)] + fn edge_ids_by_i64_metadata_range_inner( + &self, + data: &[u8], + count: usize, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Result, EngineError> { + let mut ids = Vec::new(); + let start = match label_id { + Some(target) => { + self.edge_i64_metadata_label_value_lower_bound(data, count, target, bounds.lower)? + } + None => 0, + }; + let mut pos = start; + while pos < count { + let off = 8 + pos * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + let entry_label_id = read_u32_at(data, off)?; + if label_id.is_some_and(|target| entry_label_id != target) { + break; + } + let value = read_i64_at(data, off + 4)?; + if label_id.is_some() + && bounds.upper.is_some_and(|upper| { + if bounds.upper_inclusive { + value > upper + } else { + value >= upper + } + }) + { + break; + } + if crate::edge_metadata::i64_matches_bounds(value, bounds) { + let edge_id = read_u64_at(data, off + 12)?; + if !self.deleted_edges.contains_key(&edge_id) { + ids.push(edge_id); + if ids.len() > limit { + break; + } + } + } + pos += 1; + } + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } + + fn for_each_edge_id_by_i64_metadata_range_inner( + &self, + data: &[u8], + count: usize, + label_id: Option, + bounds: RangeBoundFlags, + callback: &mut F, + ) -> Result, EngineError> + where + F: FnMut(u64) -> ControlFlow<()>, + { + let start = match label_id { + Some(target) => { + self.edge_i64_metadata_label_value_lower_bound(data, count, target, bounds.lower)? + } + None => 0, + }; + let mut pos = start; + while pos < count { + let off = 8 + pos * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + let entry_label_id = read_u32_at(data, off)?; + if label_id.is_some_and(|target| entry_label_id != target) { + break; + } + let value = read_i64_at(data, off + 4)?; + if label_id.is_some() + && bounds.upper.is_some_and(|upper| { + if bounds.upper_inclusive { + value > upper + } else { + value >= upper + } + }) + { + break; + } + if crate::edge_metadata::i64_matches_bounds(value, bounds) { + let edge_id = read_u64_at(data, off + 12)?; + if !self.deleted_edges.contains_key(&edge_id) && callback(edge_id).is_break() { + return Ok(ControlFlow::Break(())); + } + } + pos += 1; } + Ok(ControlFlow::Continue(())) + } - // Binary search the index for (type_id, key_hash, value_hash) - let idx_start = 8; + fn edge_i64_metadata_label_value_lower_bound( + &self, + data: &[u8], + count: usize, + target_label: u32, + lower_value: Option, + ) -> Result { let mut lo = 0usize; let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + let entry = ( + read_u32_at(data, off)?, + read_i64_at(data, off + 4)?, + read_u64_at(data, off + 12)?, + ); + let target = (target_label, lower_value.unwrap_or(i64::MIN), 0); + if entry < target { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } + fn edge_i64_metadata_label_lower_bound( + &self, + data: &[u8], + count: usize, + target_label: u32, + ) -> Result { + let mut lo = 0usize; + let mut hi = count; while lo < hi { let mid = lo + (hi - lo) / 2; - let entry_off = idx_start + mid * PROP_INDEX_ENTRY_SIZE; - let e_type = read_u32_at(data, entry_off)?; - let e_key_hash = read_u64_at(data, entry_off + 4)?; - let e_val_hash = read_u64_at(data, entry_off + 12)?; + let off = 8 + mid * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + if read_u32_at(data, off)? < target_label { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } - match e_type.cmp(&type_id) { - std::cmp::Ordering::Less => lo = mid + 1, - std::cmp::Ordering::Greater => hi = mid, - std::cmp::Ordering::Equal => match e_key_hash.cmp(&key_hash) { - std::cmp::Ordering::Less => lo = mid + 1, - std::cmp::Ordering::Greater => hi = mid, - std::cmp::Ordering::Equal => match e_val_hash.cmp(&value_hash) { - std::cmp::Ordering::Less => lo = mid + 1, - std::cmp::Ordering::Greater => hi = mid, - std::cmp::Ordering::Equal => { - // Found, read candidate IDs - let offset = read_u64_at(data, entry_off + 20)? as usize; - let id_count = read_u32_at(data, entry_off + 28)? as usize; - let mut result = Vec::with_capacity(id_count); - for i in 0..id_count { - let id = read_u64_at(data, offset + i * 8)?; - if !self.deleted_nodes.contains_key(&id) { - result.push(id); - } - } - return Ok(result); - } - }, - }, + fn edge_i64_metadata_label_upper_bound( + &self, + data: &[u8], + count: usize, + target_label: u32, + ) -> Result { + let mut lo = 0usize; + let mut hi = count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + if read_u32_at(data, off)? <= target_label { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) + } + + fn edge_i64_metadata_value_lower_bound_in_span( + &self, + data: &[u8], + start: usize, + end: usize, + target_value: i64, + ) -> Result { + let mut lo = start; + let mut hi = end; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + if read_i64_at(data, off + 4)? < target_value { + lo = mid + 1; + } else { + hi = mid; } } + Ok(lo) + } - Ok(Vec::new()) + fn edge_i64_metadata_value_upper_bound_in_span( + &self, + data: &[u8], + start: usize, + end: usize, + target_value: i64, + ) -> Result { + let mut lo = start; + let mut hi = end; + while lo < hi { + let mid = lo + (hi - lo) / 2; + let off = 8 + mid * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + if read_i64_at(data, off + 4)? <= target_value { + lo = mid + 1; + } else { + hi = mid; + } + } + Ok(lo) } - fn secondary_eq_sidecar_path(&self, index_id: u64) -> PathBuf { - self.seg_dir - .join("secondary_indexes") - .join(format!("node_prop_eq_{}.dat", index_id)) + fn edge_i64_metadata_range_count_in_label_span( + &self, + data: &[u8], + start: usize, + end: usize, + bounds: RangeBoundFlags, + ) -> Result { + if start >= end { + return Ok(0); + } + let range_start = match bounds.lower { + Some(lower) if bounds.lower_inclusive => { + self.edge_i64_metadata_value_lower_bound_in_span(data, start, end, lower)? + } + Some(lower) => { + self.edge_i64_metadata_value_upper_bound_in_span(data, start, end, lower)? + } + None => start, + }; + let range_end = match bounds.upper { + Some(upper) if bounds.upper_inclusive => { + self.edge_i64_metadata_value_upper_bound_in_span(data, start, end, upper)? + } + Some(upper) => { + self.edge_i64_metadata_value_lower_bound_in_span(data, start, end, upper)? + } + None => end, + }; + Ok(range_end.saturating_sub(range_start)) } - fn secondary_range_sidecar_path(&self, index_id: u64) -> PathBuf { - self.seg_dir - .join("secondary_indexes") - .join(format!("node_prop_range_{}.dat", index_id)) + fn edge_i64_metadata_range_count_inner( + &self, + data: &[u8], + count: usize, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Result { + if count == 0 { + return Ok(0); + } + if let Some(target) = label_id { + let start = self.edge_i64_metadata_label_lower_bound(data, count, target)?; + if start == count { + return Ok(0); + } + let off = 8 + start * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + if read_u32_at(data, off)? != target { + return Ok(0); + } + let end = self.edge_i64_metadata_label_upper_bound(data, count, target)?; + return self.edge_i64_metadata_range_count_in_label_span(data, start, end, bounds); + } + if bounds.lower.is_none() && bounds.upper.is_none() { + return Ok(count); + } + let mut matched = 0usize; + let mut pos = 0usize; + while pos < count { + let off = 8 + pos * EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + let entry_label_id = read_u32_at(data, off)?; + let end = self.edge_i64_metadata_label_upper_bound(data, count, entry_label_id)?; + matched = matched.saturating_add( + self.edge_i64_metadata_range_count_in_label_span(data, pos, end, bounds)?, + ); + pos = end; + } + Ok(matched) + } + + #[cfg(test)] + pub(crate) fn edge_metadata_scan_ids( + &self, + mut predicate: F, + ) -> Result, EngineError> + where + F: FnMut(EdgeMetadataCandidate) -> bool, + { + let mut ids = Vec::new(); + for index in 0..self.edge_meta_count() as usize { + let ( + edge_id, + _data_offset, + _data_len, + from, + to, + label_id, + updated_at, + weight, + valid_from, + valid_to, + _last_write_seq, + ) = self.edge_meta_at(index)?; + if self.deleted_edges.contains_key(&edge_id) { + continue; + } + let meta = EdgeMetadataCandidate { + edge_id, + from, + to, + label_id: label_id, + updated_at, + weight, + valid_from, + valid_to, + }; + if predicate(meta) { + ids.push(edge_id); + } + } + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } + + pub(crate) fn for_each_edge_metadata( + &self, + mut callback: F, + ) -> Result, EngineError> + where + F: FnMut(EdgeMetadataCandidate) -> ControlFlow<()>, + { + for index in 0..self.edge_meta_count() as usize { + let meta = self.edge_metadata_at_index(index)?; + if self.deleted_edges.contains_key(&meta.edge_id) { + continue; + } + if callback(meta).is_break() { + return Ok(ControlFlow::Break(())); + } + } + Ok(ControlFlow::Continue(())) + } + + fn secondary_component_unavailable_state( + &self, + kind: SegmentComponentKind, + ) -> DeclaredIndexRuntimeCoverageState { + match self.component_registry.availability(&kind) { + ComponentAvailability::Missing => DeclaredIndexRuntimeCoverageState::Missing, + ComponentAvailability::Available => DeclaredIndexRuntimeCoverageState::Available, + ComponentAvailability::Incompatible { .. } + | ComponentAvailability::CorruptIdentity { .. } + | ComponentAvailability::Unsupported { .. } => { + DeclaredIndexRuntimeCoverageState::Corrupt + } + } + } + + fn open_secondary_eq_sidecar_payload( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) -> Result, EngineError> { + try_open_optional_manifest_payload( + &self.component_registry, + None, + &self.seg_dir, + secondary_eq_component_kind(index_id, target), + ) + } + + fn open_secondary_range_sidecar_payload( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) -> Result, EngineError> { + try_open_optional_manifest_payload( + &self.component_registry, + None, + &self.seg_dir, + secondary_range_component_kind(index_id, target), + ) } fn set_declared_index_runtime_coverage_state( &self, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, kind: PlannerStatsDeclaredIndexKind, state: DeclaredIndexRuntimeCoverageState, ) { self.declared_index_runtime_coverage .lock() .unwrap() - .insert((index_id, kind), state); + .insert((index_id, target, kind), state); } + #[cfg(test)] pub(crate) fn declared_index_runtime_coverage_state( &self, index_id: u64, kind: PlannerStatsDeclaredIndexKind, + ) -> DeclaredIndexRuntimeCoverageState { + self.declared_index_runtime_coverage_state_for_target( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + kind, + ) + } + + pub(crate) fn declared_index_runtime_coverage_state_for_target( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + kind: PlannerStatsDeclaredIndexKind, ) -> DeclaredIndexRuntimeCoverageState { self.declared_index_runtime_coverage .lock() .unwrap() - .get(&(index_id, kind)) + .get(&(index_id, target, kind)) .copied() .unwrap_or(DeclaredIndexRuntimeCoverageState::Unknown) } @@ -2538,35 +5251,55 @@ impl SegmentReader { if entry.state != SecondaryIndexState::Ready { return; } + let target = planner_stats_declared_index_target(entry); match entry.kind { - SecondaryIndexKind::Equality => self.warm_secondary_eq_runtime_coverage(entry.index_id), + SecondaryIndexKind::Equality => { + self.warm_secondary_eq_runtime_coverage(entry.index_id, target) + } SecondaryIndexKind::Range { .. } => { - self.warm_secondary_range_runtime_coverage(entry.index_id) + self.warm_secondary_range_runtime_coverage(entry.index_id, target) } } } - fn warm_secondary_eq_runtime_coverage(&self, index_id: u64) { - let state = match self.with_secondary_eq_sidecar_index_validated(index_id, |_| Ok(())) { - Ok(Some(())) => DeclaredIndexRuntimeCoverageState::Available, - Ok(None) => DeclaredIndexRuntimeCoverageState::Missing, - Err(_) => DeclaredIndexRuntimeCoverageState::Corrupt, - }; + fn warm_secondary_eq_runtime_coverage( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) { + let component_kind = secondary_eq_component_kind(index_id, target); + let state = + match self.with_secondary_eq_sidecar_index_validated(index_id, target, |_| Ok(())) { + Ok(Some(())) => DeclaredIndexRuntimeCoverageState::Available, + Ok(None) => self.secondary_component_unavailable_state(component_kind), + Err(_) => DeclaredIndexRuntimeCoverageState::Corrupt, + }; self.set_declared_index_runtime_coverage_state( index_id, + target, PlannerStatsDeclaredIndexKind::Equality, state, ); } - fn warm_secondary_range_runtime_coverage(&self, index_id: u64) { - let state = match self.with_secondary_range_sidecar_header_validated(index_id, |_| Ok(())) { + fn warm_secondary_range_runtime_coverage( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) { + let component_kind = secondary_range_component_kind(index_id, target); + let state = match self.with_secondary_range_sidecar_header_validated( + index_id, + target, + |_| Ok(()), + ) { Ok(Some(())) => DeclaredIndexRuntimeCoverageState::Available, - Ok(None) => DeclaredIndexRuntimeCoverageState::Missing, + Ok(None) => self.secondary_component_unavailable_state(component_kind), Err(_) => DeclaredIndexRuntimeCoverageState::Corrupt, }; self.set_declared_index_runtime_coverage_state( index_id, + target, PlannerStatsDeclaredIndexKind::Range, state, ); @@ -2575,34 +5308,20 @@ impl SegmentReader { fn with_secondary_eq_sidecar( &self, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, callback: impl FnOnce(&[u8]) -> Result, ) -> Result, EngineError> { - let path = self.secondary_eq_sidecar_path(index_id); - if !path.exists() { - self.secondary_eq_sidecars.lock().unwrap().remove(&index_id); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Equality, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - let mut cache = self.secondary_eq_sidecars.lock().unwrap(); - if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry(index_id) { - let data = match mmap_file(&path) { - Ok(data) => data, - Err(EngineError::IoError(error)) - if error.kind() == std::io::ErrorKind::NotFound => - { - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Equality, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - Err(error) => return Err(error), + if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry((index_id, target)) { + let component_kind = secondary_eq_component_kind(index_id, target); + let Some(data) = self.open_secondary_eq_sidecar_payload(index_id, target)? else { + self.set_declared_index_runtime_coverage_state( + index_id, + target, + PlannerStatsDeclaredIndexKind::Equality, + self.secondary_component_unavailable_state(component_kind), + ); + return Ok(None); }; entry.insert(SecondaryEqSidecarCacheEntry { data, @@ -2613,7 +5332,7 @@ impl SegmentReader { let validation_error = { let entry = cache - .get_mut(&index_id) + .get_mut(&(index_id, target)) .expect("secondary equality sidecar cache entry must exist"); if entry.validated { None @@ -2629,59 +5348,50 @@ impl SegmentReader { } }; if let Some(error) = validation_error { - cache.remove(&index_id); + cache.remove(&(index_id, target)); drop(cache); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Equality, - DeclaredIndexRuntimeCoverageState::Corrupt, - ); + self.mark_secondary_eq_sidecar_corrupt(index_id, target, &error); return Err(error); } self.set_declared_index_runtime_coverage_state( index_id, + target, PlannerStatsDeclaredIndexKind::Equality, DeclaredIndexRuntimeCoverageState::Available, ); let data = &cache - .get(&index_id) + .get(&(index_id, target)) .expect("secondary equality sidecar cache entry must exist") .data[..]; - Ok(Some(callback(data)?)) + match callback(data) { + Ok(value) => Ok(Some(value)), + Err(error) => { + cache.remove(&(index_id, target)); + drop(cache); + self.mark_secondary_eq_sidecar_corrupt(index_id, target, &error); + Err(error) + } + } } fn with_secondary_eq_sidecar_index_validated( &self, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, callback: impl FnOnce(&[u8]) -> Result, ) -> Result, EngineError> { - let path = self.secondary_eq_sidecar_path(index_id); - if !path.exists() { - self.secondary_eq_sidecars.lock().unwrap().remove(&index_id); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Equality, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - let mut cache = self.secondary_eq_sidecars.lock().unwrap(); - if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry(index_id) { - let data = match mmap_file(&path) { - Ok(data) => data, - Err(EngineError::IoError(error)) - if error.kind() == std::io::ErrorKind::NotFound => - { - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Equality, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - Err(error) => return Err(error), + if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry((index_id, target)) { + let component_kind = secondary_eq_component_kind(index_id, target); + let Some(data) = self.open_secondary_eq_sidecar_payload(index_id, target)? else { + self.set_declared_index_runtime_coverage_state( + index_id, + target, + PlannerStatsDeclaredIndexKind::Equality, + self.secondary_component_unavailable_state(component_kind), + ); + return Ok(None); }; entry.insert(SecondaryEqSidecarCacheEntry { data, @@ -2692,12 +5402,12 @@ impl SegmentReader { let validation_error = { let entry = cache - .get_mut(&index_id) + .get_mut(&(index_id, target)) .expect("secondary equality sidecar cache entry must exist"); if entry.validated || entry.index_validated { None } else { - match validate_secondary_eq_sidecar_index(&entry.data) { + match validate_secondary_eq_sidecar_index_header(&entry.data) { Ok(()) => { entry.index_validated = true; None @@ -2707,62 +5417,50 @@ impl SegmentReader { } }; if let Some(error) = validation_error { - cache.remove(&index_id); + cache.remove(&(index_id, target)); drop(cache); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Equality, - DeclaredIndexRuntimeCoverageState::Corrupt, - ); + self.mark_secondary_eq_sidecar_corrupt(index_id, target, &error); return Err(error); } self.set_declared_index_runtime_coverage_state( index_id, + target, PlannerStatsDeclaredIndexKind::Equality, DeclaredIndexRuntimeCoverageState::Available, ); let data = &cache - .get(&index_id) + .get(&(index_id, target)) .expect("secondary equality sidecar cache entry must exist") .data[..]; - Ok(Some(callback(data)?)) + match callback(data) { + Ok(value) => Ok(Some(value)), + Err(error) => { + cache.remove(&(index_id, target)); + drop(cache); + self.mark_secondary_eq_sidecar_corrupt(index_id, target, &error); + Err(error) + } + } } fn with_secondary_range_sidecar( &self, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, callback: impl FnOnce(&[u8]) -> Result, ) -> Result, EngineError> { - let path = self.secondary_range_sidecar_path(index_id); - if !path.exists() { - self.secondary_range_sidecars - .lock() - .unwrap() - .remove(&index_id); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Range, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - let mut cache = self.secondary_range_sidecars.lock().unwrap(); - if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry(index_id) { - let data = match mmap_file(&path) { - Ok(data) => data, - Err(EngineError::IoError(error)) - if error.kind() == std::io::ErrorKind::NotFound => - { - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Range, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - Err(error) => return Err(error), + if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry((index_id, target)) { + let component_kind = secondary_range_component_kind(index_id, target); + let Some(data) = self.open_secondary_range_sidecar_payload(index_id, target)? else { + self.set_declared_index_runtime_coverage_state( + index_id, + target, + PlannerStatsDeclaredIndexKind::Range, + self.secondary_component_unavailable_state(component_kind), + ); + return Ok(None); }; entry.insert(SecondaryRangeSidecarCacheEntry { data, @@ -2773,7 +5471,7 @@ impl SegmentReader { let validation_error = { let entry = cache - .get_mut(&index_id) + .get_mut(&(index_id, target)) .expect("secondary range sidecar cache entry must exist"); if entry.validated { None @@ -2789,62 +5487,50 @@ impl SegmentReader { } }; if let Some(error) = validation_error { - cache.remove(&index_id); + cache.remove(&(index_id, target)); drop(cache); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Range, - DeclaredIndexRuntimeCoverageState::Corrupt, - ); + self.mark_secondary_range_sidecar_corrupt(index_id, target, &error); return Err(error); } self.set_declared_index_runtime_coverage_state( index_id, + target, PlannerStatsDeclaredIndexKind::Range, DeclaredIndexRuntimeCoverageState::Available, ); let data = &cache - .get(&index_id) + .get(&(index_id, target)) .expect("secondary range sidecar cache entry must exist") .data[..]; - Ok(Some(callback(data)?)) + match callback(data) { + Ok(value) => Ok(Some(value)), + Err(error) => { + cache.remove(&(index_id, target)); + drop(cache); + self.mark_secondary_range_sidecar_corrupt(index_id, target, &error); + Err(error) + } + } } fn with_secondary_range_sidecar_header_validated( &self, index_id: u64, + target: PlannerStatsDeclaredIndexTarget, callback: impl FnOnce(&[u8]) -> Result, ) -> Result, EngineError> { - let path = self.secondary_range_sidecar_path(index_id); - if !path.exists() { - self.secondary_range_sidecars - .lock() - .unwrap() - .remove(&index_id); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Range, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - let mut cache = self.secondary_range_sidecars.lock().unwrap(); - if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry(index_id) { - let data = match mmap_file(&path) { - Ok(data) => data, - Err(EngineError::IoError(error)) - if error.kind() == std::io::ErrorKind::NotFound => - { - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Range, - DeclaredIndexRuntimeCoverageState::Missing, - ); - return Ok(None); - } - Err(error) => return Err(error), + if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry((index_id, target)) { + let component_kind = secondary_range_component_kind(index_id, target); + let Some(data) = self.open_secondary_range_sidecar_payload(index_id, target)? else { + self.set_declared_index_runtime_coverage_state( + index_id, + target, + PlannerStatsDeclaredIndexKind::Range, + self.secondary_component_unavailable_state(component_kind), + ); + return Ok(None); }; entry.insert(SecondaryRangeSidecarCacheEntry { data, @@ -2855,7 +5541,7 @@ impl SegmentReader { let validation_error = { let entry = cache - .get_mut(&index_id) + .get_mut(&(index_id, target)) .expect("secondary range sidecar cache entry must exist"); if entry.validated || entry.header_validated { None @@ -2870,35 +5556,117 @@ impl SegmentReader { } }; if let Some(error) = validation_error { - cache.remove(&index_id); + cache.remove(&(index_id, target)); drop(cache); - self.set_declared_index_runtime_coverage_state( - index_id, - PlannerStatsDeclaredIndexKind::Range, - DeclaredIndexRuntimeCoverageState::Corrupt, - ); + self.mark_secondary_range_sidecar_corrupt(index_id, target, &error); return Err(error); } self.set_declared_index_runtime_coverage_state( index_id, + target, PlannerStatsDeclaredIndexKind::Range, DeclaredIndexRuntimeCoverageState::Available, ); let data = &cache - .get(&index_id) + .get(&(index_id, target)) .expect("secondary range sidecar cache entry must exist") .data[..]; - Ok(Some(callback(data)?)) + match callback(data) { + Ok(value) => Ok(Some(value)), + Err(error) => { + cache.remove(&(index_id, target)); + drop(cache); + self.mark_secondary_range_sidecar_corrupt(index_id, target, &error); + Err(error) + } + } + } + + fn mark_secondary_eq_sidecar_corrupt( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + error: &EngineError, + ) { + let component_kind = secondary_eq_component_kind(index_id, target); + mark_optional_component_corrupt( + &self.component_registry, + component_kind, + error.to_string(), + ); + self.set_declared_index_runtime_coverage_state( + index_id, + target, + PlannerStatsDeclaredIndexKind::Equality, + DeclaredIndexRuntimeCoverageState::Corrupt, + ); + } + + fn mark_secondary_range_sidecar_corrupt( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + error: &EngineError, + ) { + let component_kind = secondary_range_component_kind(index_id, target); + mark_optional_component_corrupt( + &self.component_registry, + component_kind, + error.to_string(), + ); + self.set_declared_index_runtime_coverage_state( + index_id, + target, + PlannerStatsDeclaredIndexKind::Range, + DeclaredIndexRuntimeCoverageState::Corrupt, + ); } pub(crate) fn validate_secondary_eq_sidecar(&self, index_id: u64) -> Result { - match self.with_secondary_eq_sidecar(index_id, validate_secondary_eq_sidecar_data)? { + self.validate_secondary_eq_sidecar_for_target( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + ) + } + + pub(crate) fn validate_secondary_eq_sidecar_for_target( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) -> Result { + match self.with_secondary_eq_sidecar( + index_id, + target, + validate_secondary_eq_sidecar_data, + )? { + Some(()) => Ok(true), + None => Ok(false), + } + } + + pub(crate) fn secondary_eq_sidecar_lightweight_available_for_target( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) -> Result { + match self.with_secondary_eq_sidecar_index_validated(index_id, target, |_| Ok(()))? { Some(()) => Ok(true), None => Ok(false), } } + pub(crate) fn validate_secondary_eq_sidecar_uncached( + &self, + index_id: u64, + ) -> Result { + self.secondary_eq_sidecars + .lock() + .unwrap() + .remove(&(index_id, PlannerStatsDeclaredIndexTarget::NodeProperty)); + self.validate_secondary_eq_sidecar(index_id) + } + #[cfg(test)] pub(crate) fn find_nodes_by_secondary_eq_index( &self, @@ -2916,9 +5684,11 @@ impl SegmentReader { index_id: u64, value_hash: u64, ) -> Result>, EngineError> { - self.with_secondary_eq_sidecar(index_id, |data| { - find_nodes_in_secondary_eq_sidecar(data, &self.deleted_nodes, value_hash) - }) + self.with_secondary_eq_sidecar_index_validated( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + |data| find_nodes_in_secondary_eq_sidecar(data, &self.deleted_nodes, value_hash), + ) } pub(crate) fn secondary_eq_posting_chunk_if_present( @@ -2928,9 +5698,29 @@ impl SegmentReader { start: usize, raw_limit: usize, ) -> Result, EngineError> { - self.with_secondary_eq_sidecar_index_validated(index_id, |data| { - secondary_eq_posting_chunk(data, &self.deleted_nodes, value_hash, start, raw_limit) - }) + self.with_secondary_eq_sidecar_index_validated( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + |data| { + secondary_eq_posting_chunk(data, &self.deleted_nodes, value_hash, start, raw_limit) + }, + ) + } + + pub(crate) fn edge_secondary_eq_posting_chunk_if_present( + &self, + index_id: u64, + value_hash: u64, + start: usize, + raw_limit: usize, + ) -> Result, EngineError> { + self.with_secondary_eq_sidecar_index_validated( + index_id, + PlannerStatsDeclaredIndexTarget::EdgeProperty, + |data| { + secondary_eq_posting_chunk(data, &self.deleted_edges, value_hash, start, raw_limit) + }, + ) } pub(crate) fn secondary_eq_posting_count_if_present( @@ -2938,20 +5728,65 @@ impl SegmentReader { index_id: u64, value_hash: u64, ) -> Result, EngineError> { - self.with_secondary_eq_sidecar_index_validated(index_id, |data| { - secondary_eq_posting_count(data, value_hash) - }) + self.with_secondary_eq_sidecar_index_validated( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + |data| secondary_eq_posting_count(data, value_hash), + ) + } + + pub(crate) fn edge_secondary_eq_posting_count_if_present( + &self, + index_id: u64, + value_hash: u64, + ) -> Result, EngineError> { + self.with_secondary_eq_sidecar_index_validated( + index_id, + PlannerStatsDeclaredIndexTarget::EdgeProperty, + |data| secondary_eq_visible_posting_count(data, &self.deleted_edges, value_hash), + ) } pub(crate) fn for_each_secondary_eq_group( &self, index_id: u64, + callback: F, + ) -> Result + where + F: FnMut(u64, &[u64]) -> Result<(), EngineError>, + { + self.for_each_secondary_eq_group_for_target( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + callback, + ) + } + + pub(crate) fn for_each_declared_secondary_eq_group( + &self, + entry: &SecondaryIndexManifestEntry, + callback: F, + ) -> Result + where + F: FnMut(u64, &[u64]) -> Result<(), EngineError>, + { + self.for_each_secondary_eq_group_for_target( + entry.index_id, + planner_stats_declared_index_target(entry), + callback, + ) + } + + fn for_each_secondary_eq_group_for_target( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, mut callback: F, ) -> Result where F: FnMut(u64, &[u64]) -> Result<(), EngineError>, { - match self.with_secondary_eq_sidecar(index_id, |data| { + match self.with_secondary_eq_sidecar(index_id, target, |data| { let count = read_u64_at(data, 0)? as usize; let idx_start = 8; for index in 0..count { @@ -2976,12 +5811,49 @@ impl SegmentReader { &self, index_id: u64, ) -> Result { - match self.with_secondary_range_sidecar(index_id, validate_secondary_range_sidecar_data)? { + self.validate_secondary_range_sidecar_for_target( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + ) + } + + pub(crate) fn validate_secondary_range_sidecar_for_target( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) -> Result { + match self.with_secondary_range_sidecar( + index_id, + target, + validate_secondary_range_sidecar_data, + )? { + Some(()) => Ok(true), + None => Ok(false), + } + } + + pub(crate) fn secondary_range_sidecar_lightweight_available_for_target( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, + ) -> Result { + match self.with_secondary_range_sidecar_header_validated(index_id, target, |_| Ok(()))? { Some(()) => Ok(true), None => Ok(false), } } + pub(crate) fn validate_secondary_range_sidecar_uncached( + &self, + index_id: u64, + ) -> Result { + self.secondary_range_sidecars + .lock() + .unwrap() + .remove(&(index_id, PlannerStatsDeclaredIndexTarget::NodeProperty)); + self.validate_secondary_range_sidecar(index_id) + } + #[cfg(test)] pub(crate) fn find_nodes_by_secondary_range_index_if_present( &self, @@ -3003,27 +5875,86 @@ impl SegmentReader { after: Option<(u64, u64)>, limit: Option, ) -> Result>, EngineError> { - self.with_secondary_range_sidecar(index_id, |data| { - find_nodes_in_secondary_range_sidecar( - data, - &self.deleted_nodes, - lower, - upper, - after, - limit, - ) - }) + self.with_secondary_range_sidecar_header_validated( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + |data| { + find_nodes_in_secondary_range_sidecar( + data, + &self.deleted_nodes, + lower, + upper, + after, + limit, + ) + }, + ) + } + + pub(crate) fn find_edges_by_secondary_range_index_if_present_limited( + &self, + index_id: u64, + lower: Option<(u64, bool)>, + upper: Option<(u64, bool)>, + after: Option<(u64, u64)>, + limit: Option, + ) -> Result>, EngineError> { + self.with_secondary_range_sidecar_header_validated( + index_id, + PlannerStatsDeclaredIndexTarget::EdgeProperty, + |data| { + find_nodes_in_secondary_range_sidecar( + data, + &self.deleted_edges, + lower, + upper, + after, + limit, + ) + }, + ) } pub(crate) fn for_each_secondary_range_entry( &self, index_id: u64, + callback: F, + ) -> Result + where + F: FnMut(u64, u64) -> Result<(), EngineError>, + { + self.for_each_secondary_range_entry_for_target( + index_id, + PlannerStatsDeclaredIndexTarget::NodeProperty, + callback, + ) + } + + pub(crate) fn for_each_declared_secondary_range_entry( + &self, + entry: &SecondaryIndexManifestEntry, + callback: F, + ) -> Result + where + F: FnMut(u64, u64) -> Result<(), EngineError>, + { + self.for_each_secondary_range_entry_for_target( + entry.index_id, + planner_stats_declared_index_target(entry), + callback, + ) + } + + fn for_each_secondary_range_entry_for_target( + &self, + index_id: u64, + target: PlannerStatsDeclaredIndexTarget, mut callback: F, ) -> Result where F: FnMut(u64, u64) -> Result<(), EngineError>, { - match self.with_secondary_range_sidecar(index_id, |data| { + match self.with_secondary_range_sidecar(index_id, target, |data| { let count = read_u64_at(data, 0)? as usize; for index in 0..count { let entry_off = 8 + index * SECONDARY_RANGE_ENTRY_SIZE; @@ -3047,6 +5978,29 @@ impl SegmentReader { decode_node_property_at(&self.nodes_mmap, data_offset as usize, node_id, prop_key) } + pub(crate) fn edge_property_value_at_offset( + &self, + edge_id: u64, + data_offset: u64, + prop_key: &str, + ) -> Result, EngineError> { + decode_edge_property_at(&self.edges_mmap, data_offset as usize, edge_id, prop_key) + } + + pub(crate) fn edge_properties( + &self, + edge_id: u64, + prop_keys: &[String], + ) -> Result>, EngineError> { + if self.is_edge_deleted(edge_id) { + return Ok(None); + } + let Some((_index, offset)) = self.binary_search_edge_index(edge_id)? else { + return Ok(None); + }; + decode_edge_properties_at(&self.edges_mmap, offset, edge_id, prop_keys).map(Some) + } + // --- Internal binary search methods --- /// Binary search the node index for a given node_id. @@ -3118,11 +6072,11 @@ impl SegmentReader { Ok(None) } - /// Binary search the key index for a (type_id, key) pair. + /// Binary search the key index for a (label_id, key) pair. /// Returns the node_id if found, or None. fn binary_search_key_index( &self, - target_type: u32, + target_label: u32, target_key: &str, ) -> Result, EngineError> { let data = &self.key_index_mmap[..]; @@ -3144,8 +6098,8 @@ impl SegmentReader { let mid = lo + (hi - lo) / 2; let entry_offset = read_u64_at(data, offset_table_start + mid * 8)? as usize; - // Read entry: type_id (4) + node_id (8) + key_len (2) + key - let entry_type = read_u32_at(data, entry_offset)?; + // Read entry: label_id (4) + node_id (8) + key_len (2) + key + let entry_label_id = read_u32_at(data, entry_offset)?; let key_len = read_u16_at(data, entry_offset + 12)? as usize; let key_bytes = read_bytes_at(data, entry_offset + 14, key_len)?; let entry_key = std::str::from_utf8(key_bytes).map_err(|_| { @@ -3155,7 +6109,7 @@ impl SegmentReader { )) })?; - match entry_type.cmp(&target_type) { + match entry_label_id.cmp(&target_label) { std::cmp::Ordering::Less => lo = mid + 1, std::cmp::Ordering::Greater => hi = mid, std::cmp::Ordering::Equal => match entry_key.cmp(target_key) { @@ -3208,78 +6162,237 @@ impl SegmentReader { hi = mid; } } - - Ok(found) + + Ok(found) + } + + /// Collect neighbor entries from an adjacency index + data file pair. + /// Postings are delta-encoded with varints. + #[allow(clippy::too_many_arguments)] + fn collect_adj_neighbors( + &self, + idx_mmap: &MappedData, + dat_mmap: &MappedData, + node_id: u64, + label_filter_ids: Option<&[u32]>, + limit: usize, + mut record_self_loop_edge_ids: Option<&mut NodeIdSet>, + skip_self_loop_edge_ids: Option<&NodeIdSet>, + mut raw_budget: Option<&mut usize>, + results: &mut Vec, + ) -> Result<(), EngineError> { + let idx_data = &idx_mmap[..]; + let dat_data = &dat_mmap[..]; + + let first = match self.find_first_adj_entry(idx_data, node_id)? { + Some(i) => i, + None => return Ok(()), + }; + + let count = read_u64_at(idx_data, 0)? as usize; + let idx_start = 8; + + // Scan forward from first entry while node_id matches + for i in first..count { + if let Some(remaining) = raw_budget.as_ref() { + if **remaining == 0 { + break; + } + } else if limit > 0 && results.len() >= limit { + break; + } + + let entry_off = idx_start + i * ADJ_INDEX_ENTRY_SIZE; + let entry_node = read_u64_at(idx_data, entry_off)?; + if entry_node != node_id { + break; + } + + let entry_label_id = read_u32_at(idx_data, entry_off + 8)?; + let posting_offset = read_u64_at(idx_data, entry_off + 12)? as usize; + let posting_count = read_u32_at(idx_data, entry_off + 20)? as usize; + + if let Some(label_ids) = label_filter_ids { + if !label_ids.contains(&entry_label_id) { + continue; + } + } + + // Decode delta-encoded postings sequentially + let mut cur_off = posting_offset; + let mut prev_edge_id: u64 = 0; + + for _j in 0..posting_count { + if let Some(remaining) = raw_budget.as_ref() { + if **remaining == 0 { + break; + } + } else if limit > 0 && results.len() >= limit { + break; + } + + let (delta, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + let edge_id = checked_adj_edge_id_delta(prev_edge_id, delta)?; + prev_edge_id = edge_id; + + let (neighbor_id, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + + let weight = read_f32_at(dat_data, cur_off)?; + cur_off += 4; + + let (vf_enc, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + let valid_from = vf_enc as i64; + + let (vt_enc, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + let valid_to = if vt_enc == 0 { + i64::MAX + } else { + (vt_enc - 1) as i64 + }; + + if self.deleted_edges.contains_key(&edge_id) { + continue; + } + if self.deleted_nodes.contains_key(&neighbor_id) { + continue; + } + if let Some(remaining) = raw_budget.as_deref_mut() { + *remaining = remaining.saturating_sub(1); + } + if neighbor_id == node_id { + if let Some(skip) = skip_self_loop_edge_ids { + if skip.contains(&edge_id) { + continue; + } + } + if let Some(record) = record_self_loop_edge_ids.as_deref_mut() { + record.insert(edge_id); + } + } + + results.push(NeighborRecord { + node_id: neighbor_id, + edge_id, + edge_label_id: entry_label_id, + weight, + valid_from, + valid_to, + }); + } + } + + Ok(()) + } + + /// Iterate adjacency postings for a node, calling the callback for each valid + /// (non-tombstoned, type-matching) posting. Used by degree/weight aggregation + /// to avoid materializing `Vec`. + /// + /// Callback receives `(edge_id, neighbor_id, weight, valid_from, valid_to)`. + /// For `Direction::Both`, self-loops may invoke the callback twice. Caller + /// handles dedup. + pub fn for_each_adj_posting( + &self, + node_id: u64, + direction: Direction, + label_filter_ids: Option<&[u32]>, + callback: &mut F, + ) -> Result, EngineError> + where + F: FnMut(u64, u64, f32, i64, i64) -> ControlFlow<()>, + { + match direction { + Direction::Outgoing => self.decode_adj_postings_cb( + &self.adj_out_idx, + &self.adj_out_dat, + node_id, + label_filter_ids, + callback, + ), + Direction::Incoming => self.decode_adj_postings_cb( + &self.adj_in_idx, + &self.adj_in_dat, + node_id, + label_filter_ids, + callback, + ), + Direction::Both => { + if self + .decode_adj_postings_cb( + &self.adj_out_idx, + &self.adj_out_dat, + node_id, + label_filter_ids, + callback, + )? + .is_break() + { + return Ok(ControlFlow::Break(())); + } + self.decode_adj_postings_cb( + &self.adj_in_idx, + &self.adj_in_dat, + node_id, + label_filter_ids, + callback, + ) + } + } } - /// Collect neighbor entries from an adjacency index + data file pair. - /// Postings are delta-encoded with varints. - #[allow(clippy::too_many_arguments)] - fn collect_adj_neighbors( + /// Decode adjacency postings from one index+data file pair, invoking the + /// callback for each non-tombstoned posting. Passes valid_from/valid_to + /// through for caller-side temporal filtering. + fn decode_adj_postings_cb( &self, idx_mmap: &MappedData, dat_mmap: &MappedData, node_id: u64, - type_filter: Option<&[u32]>, - limit: usize, - mut record_self_loop_edge_ids: Option<&mut NodeIdSet>, - skip_self_loop_edge_ids: Option<&NodeIdSet>, - mut raw_budget: Option<&mut usize>, - results: &mut Vec, - ) -> Result<(), EngineError> { + label_filter_ids: Option<&[u32]>, + callback: &mut F, + ) -> Result, EngineError> + where + F: FnMut(u64, u64, f32, i64, i64) -> ControlFlow<()>, + { let idx_data = &idx_mmap[..]; let dat_data = &dat_mmap[..]; let first = match self.find_first_adj_entry(idx_data, node_id)? { Some(i) => i, - None => return Ok(()), + None => return Ok(ControlFlow::Continue(())), }; let count = read_u64_at(idx_data, 0)? as usize; let idx_start = 8; - // Scan forward from first entry while node_id matches for i in first..count { - if let Some(remaining) = raw_budget.as_ref() { - if **remaining == 0 { - break; - } - } else if limit > 0 && results.len() >= limit { - break; - } - let entry_off = idx_start + i * ADJ_INDEX_ENTRY_SIZE; let entry_node = read_u64_at(idx_data, entry_off)?; if entry_node != node_id { break; } - let entry_type = read_u32_at(idx_data, entry_off + 8)?; + let entry_label_id = read_u32_at(idx_data, entry_off + 8)?; let posting_offset = read_u64_at(idx_data, entry_off + 12)? as usize; let posting_count = read_u32_at(idx_data, entry_off + 20)? as usize; - if let Some(types) = type_filter { - if !types.contains(&entry_type) { + if let Some(label_ids) = label_filter_ids { + if !label_ids.contains(&entry_label_id) { continue; } } - // Decode delta-encoded postings sequentially let mut cur_off = posting_offset; let mut prev_edge_id: u64 = 0; - for _j in 0..posting_count { - if let Some(remaining) = raw_budget.as_ref() { - if **remaining == 0 { - break; - } - } else if limit > 0 && results.len() >= limit { - break; - } - + for _ in 0..posting_count { let (delta, n) = read_varint_at(dat_data, cur_off)?; cur_off += n; - let edge_id = prev_edge_id + delta; + let edge_id = checked_adj_edge_id_delta(prev_edge_id, delta)?; prev_edge_id = edge_id; let (neighbor_id, n) = read_varint_at(dat_data, cur_off)?; @@ -3288,10 +6401,8 @@ impl SegmentReader { let weight = read_f32_at(dat_data, cur_off)?; cur_off += 4; - let (vf_enc, n) = read_varint_at(dat_data, cur_off)?; + let (valid_from_raw, n) = read_varint_at(dat_data, cur_off)?; cur_off += n; - let valid_from = vf_enc as i64; - let (vt_enc, n) = read_varint_at(dat_data, cur_off)?; cur_off += n; let valid_to = if vt_enc == 0 { @@ -3300,295 +6411,499 @@ impl SegmentReader { (vt_enc - 1) as i64 }; - if self.deleted_edges.contains_key(&edge_id) { - continue; - } - if self.deleted_nodes.contains_key(&neighbor_id) { - continue; - } - if let Some(remaining) = raw_budget.as_deref_mut() { - *remaining = remaining.saturating_sub(1); - } - if neighbor_id == node_id { - if let Some(skip) = skip_self_loop_edge_ids { - if skip.contains(&edge_id) { - continue; - } + if self.deleted_edges.contains_key(&edge_id) { + continue; + } + if self.deleted_nodes.contains_key(&neighbor_id) { + continue; + } + + if callback( + edge_id, + neighbor_id, + weight, + valid_from_raw as i64, + valid_to, + ) + .is_break() + { + return Ok(ControlFlow::Break(())); + } + } + } + + Ok(ControlFlow::Continue(())) + } + + /// Batch neighbor query: collect neighbors for multiple node IDs in a single + /// cursor walk through the adjacency index. Input `node_ids` must be sorted + /// and deduplicated. O(N+M) per direction where N = index entry count, + /// M = number of queried nodes, vs O(M log N) for M individual binary searches. + pub(crate) fn neighbors_batch( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + ) -> Result>, EngineError> { + let mut results: NodeIdMap> = + NodeIdMap::with_capacity_and_hasher(node_ids.len(), Default::default()); + + match direction { + Direction::Outgoing => { + self.collect_adj_neighbors_batch( + &self.adj_out_idx, + &self.adj_out_dat, + node_ids, + label_filter_ids, + &mut results, + )?; + } + Direction::Incoming => { + self.collect_adj_neighbors_batch( + &self.adj_in_idx, + &self.adj_in_dat, + node_ids, + label_filter_ids, + &mut results, + )?; + } + Direction::Both => { + self.collect_adj_neighbors_batch( + &self.adj_out_idx, + &self.adj_out_dat, + node_ids, + label_filter_ids, + &mut results, + )?; + self.collect_adj_neighbors_batch( + &self.adj_in_idx, + &self.adj_in_dat, + node_ids, + label_filter_ids, + &mut results, + )?; + // Deduplicate by edge_id per node (self-loops appear in both) + for entries in results.values_mut() { + let mut seen = NodeIdSet::default(); + entries.retain(|e| seen.insert(e.edge_id)); + } + } + } + + Ok(results) + } + + /// Single-pass cursor walk through an adjacency index file, collecting + /// neighbors for all requested node IDs. `node_ids` must be sorted. + /// Appends results into the existing HashMap (for Direction::Both merging). + fn collect_adj_neighbors_batch( + &self, + idx_mmap: &MappedData, + dat_mmap: &MappedData, + node_ids: &[u64], + label_filter_ids: Option<&[u32]>, + results: &mut NodeIdMap>, + ) -> Result<(), EngineError> { + let idx_data = &idx_mmap[..]; + let dat_data = &dat_mmap[..]; + + if idx_data.len() < 8 { + return Ok(()); + } + let count = read_u64_at(idx_data, 0)? as usize; + if count == 0 { + return Ok(()); + } + + let idx_start = 8; + let min_key = node_ids.first().copied().unwrap_or(0); + let max_key = node_ids.last().copied().unwrap_or(0); + let unique_keys = { + let mut n = 0usize; + let mut prev: Option = None; + for &id in node_ids { + if prev != Some(id) { + n += 1; + prev = Some(id); + } + } + n + }; + let use_seek = choose_batch_read_strategy( + idx_data, + idx_start, + count, + ADJ_INDEX_ENTRY_SIZE, + 0, + unique_keys, + min_key, + max_key, + )? == BatchReadStrategy::SeekPerKey; + let mut idx_pos = 0usize; // cursor for merge-walk path + + for &target_id in node_ids { + // Find starting position via the strategy selected by the shared + // cost model: per-key seek or merge-walk cursor advance. + if use_seek { + idx_pos = match self.find_first_adj_entry(idx_data, target_id)? { + Some(pos) => pos, + None => continue, + }; + } else { + while idx_pos < count { + let entry_off = idx_start + idx_pos * ADJ_INDEX_ENTRY_SIZE; + let entry_node = read_u64_at(idx_data, entry_off)?; + if entry_node < target_id { + idx_pos += 1; + } else { + break; + } + } + } + + // Collect all entries with node_id == target_id + while idx_pos < count { + let entry_off = idx_start + idx_pos * ADJ_INDEX_ENTRY_SIZE; + let entry_node = read_u64_at(idx_data, entry_off)?; + if entry_node != target_id { + break; + } + + let entry_label_id = read_u32_at(idx_data, entry_off + 8)?; + let posting_offset = read_u64_at(idx_data, entry_off + 12)? as usize; + let posting_count = read_u32_at(idx_data, entry_off + 20)? as usize; + + idx_pos += 1; + + if let Some(label_ids) = label_filter_ids { + if !label_ids.contains(&entry_label_id) { + continue; + } + } + + // Decode delta-encoded postings + let entries = results.entry(target_id).or_default(); + let mut cur_off = posting_offset; + let mut prev_edge_id: u64 = 0; + + for _ in 0..posting_count { + let (delta, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + let edge_id = checked_adj_edge_id_delta(prev_edge_id, delta)?; + prev_edge_id = edge_id; + + let (neighbor_id, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + + let weight = read_f32_at(dat_data, cur_off)?; + cur_off += 4; + + let (vf_enc, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + let valid_from = vf_enc as i64; + + let (vt_enc, n) = read_varint_at(dat_data, cur_off)?; + cur_off += n; + let valid_to = if vt_enc == 0 { + i64::MAX + } else { + (vt_enc - 1) as i64 + }; + + if self.deleted_edges.contains_key(&edge_id) { + continue; } - if let Some(record) = record_self_loop_edge_ids.as_deref_mut() { - record.insert(edge_id); + if self.deleted_nodes.contains_key(&neighbor_id) { + continue; } - } - results.push(NeighborEntry { - node_id: neighbor_id, - edge_id, - edge_type_id: entry_type, - weight, - valid_from, - valid_to, - }); + entries.push(NeighborRecord { + node_id: neighbor_id, + edge_id, + edge_label_id: entry_label_id, + weight, + valid_from, + valid_to, + }); + } } } Ok(()) } - /// Iterate adjacency postings for a node, calling the callback for each valid - /// (non-tombstoned, type-matching) posting. Used by degree/weight aggregation - /// to avoid materializing `Vec`. + /// Batch callback-based adjacency posting iteration. Same adaptive cost model + /// as `collect_adj_neighbors_batch` (SeekPerKey vs MergeWalk) but invokes a + /// callback instead of building `Vec`. /// - /// Callback receives `(edge_id, neighbor_id, weight, valid_from, valid_to)`. - /// For `Direction::Both`, self-loops may invoke the callback twice. Caller - /// handles dedup. - pub fn for_each_adj_posting( + /// `node_ids` must be sorted and deduplicated. For `Direction::Both`, self-loops + /// may invoke the callback twice per edge. Caller handles dedup. + /// + /// Callback receives `(queried_node_id, edge_id, neighbor_id, weight, valid_from, valid_to)`. + pub fn for_each_adj_posting_batch( &self, - node_id: u64, + node_ids: &[u64], direction: Direction, - type_filter: Option<&[u32]>, + label_filter_ids: Option<&[u32]>, callback: &mut F, ) -> Result, EngineError> where - F: FnMut(u64, u64, f32, i64, i64) -> ControlFlow<()>, + F: FnMut(u64, u64, u64, f32, i64, i64) -> ControlFlow<()>, { match direction { - Direction::Outgoing => self.decode_adj_postings_cb( + Direction::Outgoing => self.decode_adj_postings_batch_cb( &self.adj_out_idx, &self.adj_out_dat, - node_id, - type_filter, + node_ids, + label_filter_ids, callback, ), - Direction::Incoming => self.decode_adj_postings_cb( + Direction::Incoming => self.decode_adj_postings_batch_cb( &self.adj_in_idx, &self.adj_in_dat, - node_id, - type_filter, + node_ids, + label_filter_ids, callback, ), Direction::Both => { if self - .decode_adj_postings_cb( + .decode_adj_postings_batch_cb( &self.adj_out_idx, &self.adj_out_dat, - node_id, - type_filter, + node_ids, + label_filter_ids, callback, )? .is_break() { return Ok(ControlFlow::Break(())); } - self.decode_adj_postings_cb( + self.decode_adj_postings_batch_cb( &self.adj_in_idx, &self.adj_in_dat, - node_id, - type_filter, + node_ids, + label_filter_ids, callback, ) } } } - /// Decode adjacency postings from one index+data file pair, invoking the - /// callback for each non-tombstoned posting. Passes valid_from/valid_to - /// through for caller-side temporal filtering. - fn decode_adj_postings_cb( - &self, - idx_mmap: &MappedData, - dat_mmap: &MappedData, - node_id: u64, - type_filter: Option<&[u32]>, - callback: &mut F, - ) -> Result, EngineError> - where - F: FnMut(u64, u64, f32, i64, i64) -> ControlFlow<()>, - { - let idx_data = &idx_mmap[..]; - let dat_data = &dat_mmap[..]; - - let first = match self.find_first_adj_entry(idx_data, node_id)? { - Some(i) => i, - None => return Ok(ControlFlow::Continue(())), - }; - - let count = read_u64_at(idx_data, 0)? as usize; - let idx_start = 8; - - for i in first..count { - let entry_off = idx_start + i * ADJ_INDEX_ENTRY_SIZE; - let entry_node = read_u64_at(idx_data, entry_off)?; - if entry_node != node_id { - break; - } - - let entry_type = read_u32_at(idx_data, entry_off + 8)?; - let posting_offset = read_u64_at(idx_data, entry_off + 12)? as usize; - let posting_count = read_u32_at(idx_data, entry_off + 20)? as usize; - - if let Some(types) = type_filter { - if !types.contains(&entry_type) { - continue; - } - } - - let mut cur_off = posting_offset; - let mut prev_edge_id: u64 = 0; - - for _ in 0..posting_count { - let (delta, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; - let edge_id = prev_edge_id + delta; - prev_edge_id = edge_id; - - let (neighbor_id, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; - - let weight = read_f32_at(dat_data, cur_off)?; - cur_off += 4; - - let (valid_from_raw, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; - let (vt_enc, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; - let valid_to = if vt_enc == 0 { - i64::MAX - } else { - (vt_enc - 1) as i64 - }; - - if self.deleted_edges.contains_key(&edge_id) { - continue; - } - if self.deleted_nodes.contains_key(&neighbor_id) { - continue; - } - - if callback( - edge_id, - neighbor_id, - weight, - valid_from_raw as i64, - valid_to, - ) - .is_break() - { - return Ok(ControlFlow::Break(())); - } - } - } - - Ok(ControlFlow::Continue(())) - } - - /// Batch neighbor query: collect neighbors for multiple node IDs in a single - /// cursor walk through the adjacency index. Input `node_ids` must be sorted - /// and deduplicated. O(N+M) per direction where N = index entry count, - /// M = number of queried nodes, vs O(M log N) for M individual binary searches. - pub fn neighbors_batch( + pub(crate) fn endpoint_adj_posting_cursors( &self, node_ids: &[u64], direction: Direction, - type_filter: Option<&[u32]>, - ) -> Result>, EngineError> { - let mut results: NodeIdMap> = - NodeIdMap::with_capacity_and_hasher(node_ids.len(), Default::default()); - + label_filter_ids: Option<&[u32]>, + ) -> Result, EngineError> { + let mut cursors = Vec::new(); match direction { - Direction::Outgoing => { - self.collect_adj_neighbors_batch( - &self.adj_out_idx, - &self.adj_out_dat, - node_ids, - type_filter, - &mut results, - )?; - } - Direction::Incoming => { - self.collect_adj_neighbors_batch( - &self.adj_in_idx, - &self.adj_in_dat, - node_ids, - type_filter, - &mut results, - )?; - } + Direction::Outgoing => self.collect_adj_posting_cursors( + &self.adj_out_idx, + SegmentAdjacencyFile::Out, + node_ids, + label_filter_ids, + &mut cursors, + )?, + Direction::Incoming => self.collect_adj_posting_cursors( + &self.adj_in_idx, + SegmentAdjacencyFile::In, + node_ids, + label_filter_ids, + &mut cursors, + )?, Direction::Both => { - self.collect_adj_neighbors_batch( + self.collect_adj_posting_cursors( &self.adj_out_idx, - &self.adj_out_dat, + SegmentAdjacencyFile::Out, node_ids, - type_filter, - &mut results, + label_filter_ids, + &mut cursors, )?; - self.collect_adj_neighbors_batch( + self.collect_adj_posting_cursors( &self.adj_in_idx, - &self.adj_in_dat, + SegmentAdjacencyFile::In, node_ids, - type_filter, - &mut results, + label_filter_ids, + &mut cursors, )?; - // Deduplicate by edge_id per node (self-loops appear in both) - for entries in results.values_mut() { - let mut seen = NodeIdSet::default(); - entries.retain(|e| seen.insert(e.edge_id)); + } + } + Ok(cursors) + } + + pub(crate) fn endpoint_adj_posting_count( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + ) -> Result { + let cursors = self.endpoint_adj_posting_cursors(node_ids, direction, label_filter_ids)?; + Ok(cursors.iter().fold(0usize, |total, cursor| { + total.saturating_add(cursor.remaining) + })) + } + + fn collect_adj_posting_cursors( + &self, + idx_mmap: &MappedData, + file: SegmentAdjacencyFile, + node_ids: &[u64], + label_filter_ids: Option<&[u32]>, + cursors: &mut Vec, + ) -> Result<(), EngineError> { + let idx_data = &idx_mmap[..]; + if idx_data.len() < 8 { + return Ok(()); + } + let count = read_u64_at(idx_data, 0)? as usize; + if count == 0 { + return Ok(()); + } + + let idx_start = 8; + let min_key = node_ids.first().copied().unwrap_or(0); + let max_key = node_ids.last().copied().unwrap_or(0); + let use_seek = choose_batch_read_strategy( + idx_data, + idx_start, + count, + ADJ_INDEX_ENTRY_SIZE, + 0, + node_ids.len(), + min_key, + max_key, + )? == BatchReadStrategy::SeekPerKey; + let mut idx_pos = 0usize; + + for &target_id in node_ids { + if use_seek { + idx_pos = match self.find_first_adj_entry(idx_data, target_id)? { + Some(pos) => pos, + None => continue, + }; + } else { + while idx_pos < count { + let entry_off = idx_start + idx_pos * ADJ_INDEX_ENTRY_SIZE; + let entry_node = read_u64_at(idx_data, entry_off)?; + if entry_node < target_id { + idx_pos += 1; + } else { + break; + } + } + } + + while idx_pos < count { + let entry_off = idx_start + idx_pos * ADJ_INDEX_ENTRY_SIZE; + let entry_node = read_u64_at(idx_data, entry_off)?; + if entry_node != target_id { + break; + } + + let entry_label_id = read_u32_at(idx_data, entry_off + 8)?; + let posting_offset = read_u64_at(idx_data, entry_off + 12)? as usize; + let posting_count = read_u32_at(idx_data, entry_off + 20)? as usize; + idx_pos += 1; + + if label_filter_ids.is_some_and(|label_ids| !label_ids.contains(&entry_label_id)) { + continue; + } + + if posting_count > 0 { + cursors.push(SegmentAdjPostingCursor { + file, + cur_off: posting_offset, + remaining: posting_count, + prev_edge_id: 0, + }); } } } - Ok(results) + Ok(()) } - /// Single-pass cursor walk through an adjacency index file, collecting - /// neighbors for all requested node IDs. `node_ids` must be sorted. - /// Appends results into the existing HashMap (for Direction::Both merging). - fn collect_adj_neighbors_batch( + pub(crate) fn next_adj_posting_edge_id( + &self, + cursor: &mut SegmentAdjPostingCursor, + ) -> Result, EngineError> { + let dat_data = match cursor.file { + SegmentAdjacencyFile::Out => &self.adj_out_dat[..], + SegmentAdjacencyFile::In => &self.adj_in_dat[..], + }; + + while cursor.remaining > 0 { + cursor.remaining -= 1; + + let (delta, n) = read_varint_at(dat_data, cursor.cur_off)?; + cursor.cur_off += n; + let edge_id = checked_adj_edge_id_delta(cursor.prev_edge_id, delta)?; + cursor.prev_edge_id = edge_id; + + let (neighbor_id, n) = read_varint_at(dat_data, cursor.cur_off)?; + cursor.cur_off += n; + + let _ = read_f32_at(dat_data, cursor.cur_off)?; + cursor.cur_off += 4; + + let (_, n) = read_varint_at(dat_data, cursor.cur_off)?; + cursor.cur_off += n; + let (_, n) = read_varint_at(dat_data, cursor.cur_off)?; + cursor.cur_off += n; + + if self.deleted_edges.contains_key(&edge_id) { + continue; + } + if self.deleted_nodes.contains_key(&neighbor_id) { + continue; + } + return Ok(Some(edge_id)); + } + + Ok(None) + } + + /// Batch decode adjacency postings from one index+data file pair using the + /// adaptive cost model. Invokes the callback for each non-tombstoned posting. + fn decode_adj_postings_batch_cb( &self, idx_mmap: &MappedData, dat_mmap: &MappedData, node_ids: &[u64], - type_filter: Option<&[u32]>, - results: &mut NodeIdMap>, - ) -> Result<(), EngineError> { + label_filter_ids: Option<&[u32]>, + callback: &mut F, + ) -> Result, EngineError> + where + F: FnMut(u64, u64, u64, f32, i64, i64) -> ControlFlow<()>, + { let idx_data = &idx_mmap[..]; let dat_data = &dat_mmap[..]; if idx_data.len() < 8 { - return Ok(()); + return Ok(ControlFlow::Continue(())); } let count = read_u64_at(idx_data, 0)? as usize; if count == 0 { - return Ok(()); + return Ok(ControlFlow::Continue(())); } let idx_start = 8; let min_key = node_ids.first().copied().unwrap_or(0); let max_key = node_ids.last().copied().unwrap_or(0); - let unique_keys = { - let mut n = 0usize; - let mut prev: Option = None; - for &id in node_ids { - if prev != Some(id) { - n += 1; - prev = Some(id); - } - } - n - }; + // node_ids is pre-sorted and deduped, so len() == unique count let use_seek = choose_batch_read_strategy( idx_data, + idx_start, count, ADJ_INDEX_ENTRY_SIZE, 0, - unique_keys, + node_ids.len(), min_key, max_key, )? == BatchReadStrategy::SeekPerKey; - let mut idx_pos = 0usize; // cursor for merge-walk path + let mut idx_pos = 0usize; for &target_id in node_ids { - // Find starting position via the strategy selected by the shared - // cost model: per-key seek or merge-walk cursor advance. if use_seek { idx_pos = match self.find_first_adj_entry(idx_data, target_id)? { Some(pos) => pos, @@ -3606,7 +6921,6 @@ impl SegmentReader { } } - // Collect all entries with node_id == target_id while idx_pos < count { let entry_off = idx_start + idx_pos * ADJ_INDEX_ENTRY_SIZE; let entry_node = read_u64_at(idx_data, entry_off)?; @@ -3614,27 +6928,25 @@ impl SegmentReader { break; } - let entry_type = read_u32_at(idx_data, entry_off + 8)?; + let entry_label_id = read_u32_at(idx_data, entry_off + 8)?; let posting_offset = read_u64_at(idx_data, entry_off + 12)? as usize; let posting_count = read_u32_at(idx_data, entry_off + 20)? as usize; idx_pos += 1; - if let Some(types) = type_filter { - if !types.contains(&entry_type) { + if let Some(label_ids) = label_filter_ids { + if !label_ids.contains(&entry_label_id) { continue; } } - // Decode delta-encoded postings - let entries = results.entry(target_id).or_default(); let mut cur_off = posting_offset; let mut prev_edge_id: u64 = 0; for _ in 0..posting_count { let (delta, n) = read_varint_at(dat_data, cur_off)?; cur_off += n; - let edge_id = prev_edge_id + delta; + let edge_id = checked_adj_edge_id_delta(prev_edge_id, delta)?; prev_edge_id = edge_id; let (neighbor_id, n) = read_varint_at(dat_data, cur_off)?; @@ -3643,10 +6955,8 @@ impl SegmentReader { let weight = read_f32_at(dat_data, cur_off)?; cur_off += 4; - let (vf_enc, n) = read_varint_at(dat_data, cur_off)?; + let (valid_from_raw, n) = read_varint_at(dat_data, cur_off)?; cur_off += n; - let valid_from = vf_enc as i64; - let (vt_enc, n) = read_varint_at(dat_data, cur_off)?; cur_off += n; let valid_to = if vt_enc == 0 { @@ -3662,268 +6972,1042 @@ impl SegmentReader { continue; } - entries.push(NeighborEntry { - node_id: neighbor_id, + if callback( + target_id, edge_id, - edge_type_id: entry_type, + neighbor_id, weight, - valid_from, + valid_from_raw as i64, valid_to, - }); + ) + .is_break() + { + return Ok(ControlFlow::Break(())); + } } } } - Ok(()) + Ok(ControlFlow::Continue(())) } +} - /// Batch callback-based adjacency posting iteration. Same adaptive cost model - /// as `collect_adj_neighbors_batch` (SeekPerKey vs MergeWalk) but invokes a - /// callback instead of building `Vec`. - /// - /// `node_ids` must be sorted and deduplicated. For `Direction::Both`, self-loops - /// may invoke the callback twice per edge. Caller handles dedup. - /// - /// Callback receives `(queried_node_id, edge_id, neighbor_id, weight, valid_from, valid_to)`. - pub fn for_each_adj_posting_batch( - &self, - node_ids: &[u64], - direction: Direction, - type_filter: Option<&[u32]>, - callback: &mut F, - ) -> Result, EngineError> - where - F: FnMut(u64, u64, u64, f32, i64, i64) -> ControlFlow<()>, - { - match direction { - Direction::Outgoing => self.decode_adj_postings_batch_cb( - &self.adj_out_idx, - &self.adj_out_dat, - node_ids, - type_filter, - callback, - ), - Direction::Incoming => self.decode_adj_postings_batch_cb( - &self.adj_in_idx, - &self.adj_in_dat, - node_ids, - type_filter, - callback, - ), - Direction::Both => { - if self - .decode_adj_postings_batch_cb( - &self.adj_out_idx, - &self.adj_out_dat, - node_ids, - type_filter, - callback, - )? - .is_break() - { - return Ok(ControlFlow::Break(())); - } - self.decode_adj_postings_batch_cb( - &self.adj_in_idx, - &self.adj_in_dat, - node_ids, - type_filter, - callback, - ) +// --- Helpers --- + +#[cfg(test)] +fn load_component_manifest( + seg_dir: &Path, + segment_id: u64, +) -> Result { + let manifest = read_component_manifest(seg_dir)?; + if manifest.segment_id != segment_id { + return Err(EngineError::CorruptRecord(format!( + "segment manifest id {} does not match directory segment {}", + manifest.segment_id, segment_id + ))); + } + Ok(manifest) +} + +pub(crate) fn validate_segment_manifest_identity( + segment_info: &SegmentInfo, + manifest: &SegmentComponentManifestV1, +) -> Result { + if manifest.segment_id != segment_info.id { + return Err(EngineError::CorruptRecord(format!( + "segment manifest id {} does not match root segment {}", + manifest.segment_id, segment_info.id + ))); + } + if manifest.node_count != segment_info.node_count { + return Err(EngineError::CorruptRecord(format!( + "segment manifest node_count {} does not match root node_count {} for segment {}", + manifest.node_count, segment_info.node_count, segment_info.id + ))); + } + if manifest.edge_count != segment_info.edge_count { + return Err(EngineError::CorruptRecord(format!( + "segment manifest edge_count {} does not match root edge_count {} for segment {}", + manifest.edge_count, segment_info.edge_count, segment_info.id + ))); + } + if manifest.segment_data_id != segment_info.segment_data_id { + return Err(EngineError::CorruptRecord(format!( + "segment manifest segment_data_id does not match root for segment {}", + segment_info.id + ))); + } + validate_root_segment_info(segment_info, manifest.segment_format_version)?; + validate_manifest_segment_data_id(manifest) +} + +fn read_component_manifest(seg_dir: &Path) -> Result { + let path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + if !path.exists() { + return Err(EngineError::CorruptRecord( + "unsupported segment format: missing segment_manifest.dat; rebuild the database".into(), + )); + } + let data = std::fs::read(&path)?; + let manifest = decode_manifest_envelope(&data)?; + if manifest.segment_format_version != SEGMENT_FORMAT_VERSION { + return Err(EngineError::CorruptRecord(format!( + "unsupported segment manifest version {}; rebuild the database", + manifest.segment_format_version + ))); + } + Ok(manifest) +} + +fn validate_root_segment_info( + segment_info: &SegmentInfo, + format_version: u32, +) -> Result<(), EngineError> { + if format_version != SEGMENT_FORMAT_VERSION { + return Err(EngineError::CorruptRecord(format!( + "unsupported segment manifest version {}; rebuild the database", + format_version + ))); + } + if segment_info.segment_format_version != SEGMENT_FORMAT_VERSION { + return Err(EngineError::CorruptRecord(format!( + "root manifest segment {} has unsupported segment manifest version {}; rebuild the database", + segment_info.id, segment_info.segment_format_version + ))); + } + if segment_info.segment_data_id == [0; 32] { + return Err(EngineError::CorruptRecord(format!( + "root manifest segment {} is missing segment_data_id; rebuild the database", + segment_info.id + ))); + } + Ok(()) +} + +fn validate_manifest_segment_data_id( + manifest: &SegmentComponentManifestV1, +) -> Result { + let source_groups = segment_source_groups_from_records( + manifest.segment_id, + manifest.node_count, + manifest.edge_count, + &manifest.components, + )?; + if source_groups.segment_data_id != manifest.segment_data_id { + return Err(EngineError::CorruptRecord(format!( + "segment manifest segment_data_id does not match component source groups for segment {}", + manifest.segment_id + ))); + } + Ok(source_groups) +} + +fn validate_manifest_component_contracts( + registry: &SegmentComponentRegistry, + source_groups: &SegmentComponentSourceGroups, + dense_config: Option<&DenseVectorConfig>, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result<(), EngineError> { + for record in registry.records.values() { + let result = validate_manifest_component_contract( + record, + registry, + source_groups, + dense_config, + secondary_indexes, + ); + if let Err(error) = result { + match record.requirement { + ComponentRequirement::Required => return Err(error), + ComponentRequirement::Optional { .. } => registry.set_availability( + record.kind.clone(), + ComponentAvailability::Incompatible { + reason: error.to_string(), + }, + ), } } } + Ok(()) +} - /// Batch decode adjacency postings from one index+data file pair using the - /// adaptive cost model. Invokes the callback for each non-tombstoned posting. - fn decode_adj_postings_batch_cb( - &self, - idx_mmap: &MappedData, - dat_mmap: &MappedData, - node_ids: &[u64], - type_filter: Option<&[u32]>, - callback: &mut F, - ) -> Result, EngineError> - where - F: FnMut(u64, u64, u64, f32, i64, i64) -> ControlFlow<()>, - { - let idx_data = &idx_mmap[..]; - let dat_data = &dat_mmap[..]; +fn warm_edge_property_sidecar_availability( + registry: &SegmentComponentRegistry, + context: &ComponentOpenContext, + seg_dir: &Path, + secondary_indexes: &[SecondaryIndexManifestEntry], +) { + for entry in secondary_indexes { + let kind = match (&entry.target, &entry.kind) { + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Equality) => { + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + } + } + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Range { .. }) => { + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: entry.index_id, + } + } + _ => continue, + }; + let _ = try_open_optional_manifest_payload(registry, Some(context), seg_dir, kind); + } +} - if idx_data.len() < 8 { - return Ok(ControlFlow::Continue(())); - } - let count = read_u64_at(idx_data, 0)? as usize; - if count == 0 { - return Ok(ControlFlow::Continue(())); - } +fn validate_manifest_component_contract( + record: &SegmentComponentRecordV1, + registry: &SegmentComponentRegistry, + source_groups: &SegmentComponentSourceGroups, + dense_config: Option<&DenseVectorConfig>, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result<(), EngineError> { + let (expected_requirement, expected_trust_class) = expected_component_contract(&record.kind) + .ok_or_else(|| { + EngineError::CorruptRecord(format!("unsupported component kind {:?}", record.kind)) + })?; + if record.requirement != expected_requirement { + return Err(EngineError::CorruptRecord(format!( + "component {:?} requirement does not match manifest contract", + record.kind + ))); + } + if record.trust_class != expected_trust_class { + return Err(EngineError::CorruptRecord(format!( + "component {:?} trust class does not match manifest contract", + record.kind + ))); + } + if record.logical_format_version != 1 { + return Err(EngineError::CorruptRecord(format!( + "component {:?} has unsupported logical format version {}", + record.kind, record.logical_format_version + ))); + } + let dependencies = expected_component_dependencies( + &record.kind, + registry, + source_groups, + dense_config, + secondary_indexes, + )?; + if record.dependency_digest != dependency_digest(&dependencies) { + return Err(EngineError::CorruptRecord(format!( + "component {:?} dependency digest does not match current source identity", + record.kind + ))); + } + if !component_build_fingerprint_matches( + &record.kind, + record.build_fingerprint, + secondary_indexes, + ) { + return Err(EngineError::CorruptRecord(format!( + "component {:?} build fingerprint does not match current semantics", + record.kind + ))); + } + let expected_component_id = component_id( + registry.segment_id, + &record.kind, + record.logical_format_version, + record.payload_len, + record.payload_digest.as_ref(), + &record.dependency_digest, + record.build_fingerprint, + ); + if record.component_id != expected_component_id { + return Err(EngineError::CorruptRecord(format!( + "component {:?} id does not match expected identity fields", + record.kind + ))); + } + Ok(()) +} + +fn expected_component_contract( + kind: &SegmentComponentKind, +) -> Option<(ComponentRequirement, ComponentTrustClass)> { + use SegmentComponentKind::*; + let contract = match kind { + NodeRecords | EdgeRecords => ( + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + ), + NodeMetadata | EdgeMetadata | Tombstones => ( + ComponentRequirement::Required, + ComponentTrustClass::PrimaryMetadata, + ), + KeyIndex | NodeLabelIndex | EdgeLabelIndex | EdgeTripleIndex | AdjOutIndex + | AdjOutPostings | AdjInIndex | AdjInPostings | TimestampIndex => ( + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + ), + NodeVectorMetadata | NodeDenseVectorBlob | NodeSparseVectorBlob => ( + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + ), + LegacyNodePropertyIndex + | NodePropertyHashMetadata + | NodePropertyEqualityIndex { .. } + | NodePropertyRangeIndex { .. } => ( + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + ), + EdgeWeightIndex | EdgeUpdatedAtIndex | EdgeValidFromIndex | EdgeValidToIndex => ( + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::MetadataScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + ), + DegreeDelta => ( + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::AdjacencyWalk, + }, + ComponentTrustClass::OptionalExactAccelerator, + ), + PlannerStats => ( + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::PlannerStatsUnavailable, + }, + ComponentTrustClass::OptionalAdvisoryStats, + ), + DenseHnswMetadata | DenseHnswGraph | SparsePostingIndex | SparsePostings => ( + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::ExactVectorScan, + }, + ComponentTrustClass::OptionalApproximateAccelerator, + ), + EdgePropertyEqualityIndex { .. } | EdgePropertyRangeIndex { .. } => ( + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + ), + PackedSegmentContainer => ( + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + ), + }; + Some(contract) +} - let idx_start = 8; - let min_key = node_ids.first().copied().unwrap_or(0); - let max_key = node_ids.last().copied().unwrap_or(0); - // node_ids is pre-sorted and deduped, so len() == unique count - let use_seek = choose_batch_read_strategy( - idx_data, - count, - ADJ_INDEX_ENTRY_SIZE, - 0, - node_ids.len(), - min_key, - max_key, - )? == BatchReadStrategy::SeekPerKey; - let mut idx_pos = 0usize; +fn expected_component_dependencies( + kind: &SegmentComponentKind, + registry: &SegmentComponentRegistry, + source_groups: &SegmentComponentSourceGroups, + dense_config: Option<&DenseVectorConfig>, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result, EngineError> { + use SegmentComponentKind::*; + Ok(match kind { + NodeRecords | EdgeRecords | NodeMetadata | EdgeMetadata | Tombstones => Vec::new(), + KeyIndex | NodeLabelIndex | TimestampIndex => vec![source_group_dependency( + SegmentSourceGroupKind::NodeSource, + source_groups.node_source, + )], + EdgeLabelIndex | EdgeTripleIndex | AdjOutIndex | AdjOutPostings | AdjInIndex + | AdjInPostings => vec![source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + )], + LegacyNodePropertyIndex => vec![source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + )], + NodePropertyHashMetadata => vec![source_group_dependency( + SegmentSourceGroupKind::NodeSource, + source_groups.node_source, + )], + NodeVectorMetadata => vec![source_group_dependency( + SegmentSourceGroupKind::NodeSource, + source_groups.node_source, + )], + NodeDenseVectorBlob | NodeSparseVectorBlob => { + let vector_meta = registry.record(&NodeVectorMetadata).ok_or_else(|| { + EngineError::CorruptRecord("vector blob missing metadata component".into()) + })?; + vec![ + source_group_dependency( + SegmentSourceGroupKind::NodeSource, + source_groups.node_source, + ), + source_component_dependency(vector_meta), + ] + } + NodePropertyEqualityIndex { index_id } | NodePropertyRangeIndex { index_id } => { + let entry = secondary_indexes + .iter() + .find(|entry| { + entry.index_id == *index_id + && matches!(entry.target, SecondaryIndexTarget::NodeProperty { .. }) + }) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "component {:?} has no matching secondary index declaration", + kind + )) + })?; + vec![ + source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + ), + secondary_declaration_dependency(entry), + ] + } + EdgePropertyEqualityIndex { index_id } | EdgePropertyRangeIndex { index_id } => { + let entry = secondary_indexes + .iter() + .find(|entry| { + entry.index_id == *index_id + && matches!(entry.target, SecondaryIndexTarget::EdgeProperty { .. }) + }) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "component {:?} has no matching secondary index declaration", + kind + )) + })?; + vec![ + source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + ), + secondary_declaration_dependency(entry), + ] + } + EdgeWeightIndex | EdgeUpdatedAtIndex | EdgeValidFromIndex | EdgeValidToIndex => { + vec![source_group_dependency( + SegmentSourceGroupKind::EdgeMetadataSource, + source_groups.edge_metadata_source, + )] + } + DegreeDelta => vec![source_group_dependency( + SegmentSourceGroupKind::DegreeSource, + source_groups.degree_source, + )], + PlannerStats => { + planner_stats_component_dependencies(source_groups.segment_data_id, secondary_indexes) + } + DenseHnswMetadata | DenseHnswGraph => vec![ + source_group_dependency( + SegmentSourceGroupKind::DenseVectorSource, + source_groups.dense_vector_source, + ), + ComponentDependencyV1::DenseVectorConfig { + fingerprint: dense_config_fingerprint(dense_config), + }, + ], + SparsePostingIndex | SparsePostings => vec![ + source_group_dependency( + SegmentSourceGroupKind::SparseVectorSource, + source_groups.sparse_vector_source, + ), + ComponentDependencyV1::SparseVectorConfig { + fingerprint: component_fingerprint("sparse_vector_config", &[]), + }, + ], + PackedSegmentContainer => Vec::new(), + }) +} - for &target_id in node_ids { - if use_seek { - idx_pos = match self.find_first_adj_entry(idx_data, target_id)? { - Some(pos) => pos, - None => continue, - }; - } else { - while idx_pos < count { - let entry_off = idx_start + idx_pos * ADJ_INDEX_ENTRY_SIZE; - let entry_node = read_u64_at(idx_data, entry_off)?; - if entry_node < target_id { - idx_pos += 1; - } else { - break; - } - } - } +fn component_build_fingerprint_matches( + kind: &SegmentComponentKind, + actual: u64, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> bool { + use SegmentComponentKind::*; + match kind { + EdgePropertyEqualityIndex { index_id } => { + actual == component_fingerprint("flush.edge_prop_eq", &[*index_id]) + || actual == component_fingerprint("compaction.edge_prop_eq", &[*index_id]) + || actual == component_fingerprint("build.edge_prop_eq", &[*index_id]) + } + EdgePropertyRangeIndex { index_id } => { + actual == component_fingerprint("flush.edge_prop_range", &[*index_id]) + || actual == component_fingerprint("compaction.edge_prop_range", &[*index_id]) + || actual == component_fingerprint("build.edge_prop_range", &[*index_id]) + } + _ => actual == expected_component_build_fingerprint(kind, secondary_indexes), + } +} - while idx_pos < count { - let entry_off = idx_start + idx_pos * ADJ_INDEX_ENTRY_SIZE; - let entry_node = read_u64_at(idx_data, entry_off)?; - if entry_node != target_id { - break; - } +fn expected_component_build_fingerprint( + kind: &SegmentComponentKind, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> u64 { + use SegmentComponentKind::*; + match kind { + NodeRecords => component_fingerprint("flush.nodes", &[]), + EdgeRecords => component_fingerprint("flush.edges", &[]), + NodeMetadata => component_fingerprint("flush.node_meta", &[]), + EdgeMetadata => component_fingerprint("flush.edge_meta", &[]), + Tombstones => component_fingerprint("flush.tombstones", &[]), + KeyIndex => component_fingerprint("flush.key_index", &[]), + NodeLabelIndex => component_fingerprint("flush.node_label_index", &[]), + EdgeLabelIndex => component_fingerprint("flush.edge_label_index", &[]), + EdgeTripleIndex => component_fingerprint("flush.edge_triple_index", &[]), + AdjOutIndex => component_fingerprint("flush.adj_out_idx", &[]), + AdjOutPostings => component_fingerprint("flush.adj_out_dat", &[]), + AdjInIndex => component_fingerprint("flush.adj_in_idx", &[]), + AdjInPostings => component_fingerprint("flush.adj_in_dat", &[]), + TimestampIndex => component_fingerprint("flush.timestamp_index", &[]), + LegacyNodePropertyIndex => component_fingerprint("flush.prop_index", &[]), + NodePropertyHashMetadata => component_fingerprint("flush.node_prop_hashes", &[]), + NodePropertyEqualityIndex { index_id } => { + component_fingerprint("flush.node_prop_eq", &[*index_id]) + } + NodePropertyRangeIndex { index_id } => { + component_fingerprint("flush.node_prop_range", &[*index_id]) + } + EdgeWeightIndex => component_fingerprint("flush.edge_weight_index", &[]), + EdgeUpdatedAtIndex => component_fingerprint("flush.edge_updated_at_index", &[]), + EdgeValidFromIndex => component_fingerprint("flush.edge_valid_from_index", &[]), + EdgeValidToIndex => component_fingerprint("flush.edge_valid_to_index", &[]), + DegreeDelta => component_fingerprint("flush.degree_delta", &[]), + PlannerStats => planner_stats_component_fingerprint(secondary_indexes), + NodeVectorMetadata => component_fingerprint("flush.node_vector_meta", &[]), + NodeDenseVectorBlob => component_fingerprint("flush.node_dense_vectors", &[]), + NodeSparseVectorBlob => component_fingerprint("flush.node_sparse_vectors", &[]), + DenseHnswMetadata => component_fingerprint("flush.dense_hnsw_meta", &[]), + DenseHnswGraph => component_fingerprint("flush.dense_hnsw_graph", &[]), + SparsePostingIndex => component_fingerprint("flush.sparse_posting_index", &[]), + SparsePostings => component_fingerprint("flush.sparse_postings", &[]), + EdgePropertyEqualityIndex { index_id } => { + component_fingerprint("flush.edge_prop_eq", &[*index_id]) + } + EdgePropertyRangeIndex { index_id } => { + component_fingerprint("flush.edge_prop_range", &[*index_id]) + } + PackedSegmentContainer => component_fingerprint("flush.packed_segment_container", &[]), + } +} - let entry_type = read_u32_at(idx_data, entry_off + 8)?; - let posting_offset = read_u64_at(idx_data, entry_off + 12)? as usize; - let posting_count = read_u32_at(idx_data, entry_off + 20)? as usize; +fn open_required_manifest_payload( + registry: &SegmentComponentRegistry, + context: &ComponentOpenContext, + seg_dir: &Path, + kind: SegmentComponentKind, +) -> Result { + let record = registry.record(&kind).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment manifest missing required component {:?}", + kind + )) + })?; + match open_manifest_component_record(seg_dir, registry.segment_id, record, Some(context)) { + Ok(data) => { + registry.set_availability(kind, ComponentAvailability::Available); + Ok(data) + } + Err(error) => { + registry.set_availability( + kind.clone(), + ComponentAvailability::CorruptIdentity { + reason: error.to_string(), + }, + ); + Err(error) + } + } +} - idx_pos += 1; +fn open_manifested_required_payload_or_empty( + registry: &SegmentComponentRegistry, + context: &ComponentOpenContext, + seg_dir: &Path, + kind: SegmentComponentKind, +) -> Result { + if registry.record(&kind).is_none() { + registry.set_availability(kind, ComponentAvailability::Missing); + return Ok(MappedData::Empty); + } + open_required_manifest_payload(registry, context, seg_dir, kind) +} - if let Some(types) = type_filter { - if !types.contains(&entry_type) { - continue; - } - } +fn open_optional_manifest_payload( + registry: &SegmentComponentRegistry, + context: Option<&ComponentOpenContext>, + seg_dir: &Path, + kind: SegmentComponentKind, +) -> Result { + try_open_optional_manifest_payload(registry, context, seg_dir, kind) + .map(|data| data.unwrap_or(MappedData::Empty)) +} - let mut cur_off = posting_offset; - let mut prev_edge_id: u64 = 0; +fn try_open_optional_manifest_payload( + registry: &SegmentComponentRegistry, + context: Option<&ComponentOpenContext>, + seg_dir: &Path, + kind: SegmentComponentKind, +) -> Result, EngineError> { + if let Some(state) = registry.recorded_availability(&kind) { + if !state.is_available() { + return Ok(None); + } + } + let Some(record) = registry.record(&kind) else { + registry.set_availability(kind, ComponentAvailability::Missing); + return Ok(None); + }; + match open_manifest_component_record(seg_dir, registry.segment_id, record, context) { + Ok(data) => { + registry.set_availability(kind, ComponentAvailability::Available); + Ok(Some(data)) + } + Err(EngineError::IoError(error)) if error.kind() == std::io::ErrorKind::NotFound => { + registry.set_availability(kind, ComponentAvailability::Missing); + Ok(None) + } + Err(error) => { + registry.set_availability( + kind, + ComponentAvailability::CorruptIdentity { + reason: error.to_string(), + }, + ); + Ok(None) + } + } +} - for _ in 0..posting_count { - let (delta, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; - let edge_id = prev_edge_id + delta; - prev_edge_id = edge_id; +fn open_planner_stats( + registry: &SegmentComponentRegistry, + seg_dir: &Path, + segment_id: u64, + node_count: u64, + edge_count: u64, +) -> PlannerStatsAvailability { + match try_open_optional_manifest_payload( + registry, + None, + seg_dir, + SegmentComponentKind::PlannerStats, + ) { + Ok(Some(data)) => { + let availability = + read_planner_stats_payload(&data, segment_id, node_count, edge_count); + if let PlannerStatsAvailability::Unavailable { reason } = &availability { + registry.set_availability( + SegmentComponentKind::PlannerStats, + ComponentAvailability::CorruptIdentity { + reason: reason.clone(), + }, + ); + } + availability + } + Ok(None) => match registry.availability(&SegmentComponentKind::PlannerStats) { + ComponentAvailability::Missing => PlannerStatsAvailability::Missing, + state => PlannerStatsAvailability::Unavailable { + reason: format!("{:?}", state), + }, + }, + Err(error) => PlannerStatsAvailability::Unavailable { + reason: error.to_string(), + }, + } +} - let (neighbor_id, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; +fn open_degree_delta_sidecar( + registry: &SegmentComponentRegistry, + seg_dir: &Path, +) -> Option { + let record = registry.record(&SegmentComponentKind::DegreeDelta)?; + let ComponentHandleV1::ExternalFile { relative_path, .. } = &record.handle else { + registry.set_availability( + SegmentComponentKind::DegreeDelta, + ComponentAvailability::Unsupported { + reason: "packed degree sidecar handles are unsupported".into(), + }, + ); + return None; + }; + try_open_optional_manifest_payload(registry, None, seg_dir, SegmentComponentKind::DegreeDelta) + .ok() + .flatten()?; + match DegreeSidecar::open(&seg_dir.join(relative_path)) { + Ok(sidecar) => Some(sidecar), + Err(error) => { + registry.set_availability( + SegmentComponentKind::DegreeDelta, + ComponentAvailability::CorruptIdentity { + reason: error.to_string(), + }, + ); + None + } + } +} - let weight = read_f32_at(dat_data, cur_off)?; - cur_off += 4; +fn mark_optional_components_corrupt( + registry: &SegmentComponentRegistry, + kinds: &[SegmentComponentKind], + reason: String, +) { + for kind in kinds { + if registry.record(kind).is_some() + && !matches!(registry.availability(kind), ComponentAvailability::Missing) + { + registry.set_availability( + kind.clone(), + ComponentAvailability::CorruptIdentity { + reason: reason.clone(), + }, + ); + } + } +} - let (valid_from_raw, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; - let (vt_enc, n) = read_varint_at(dat_data, cur_off)?; - cur_off += n; - let valid_to = if vt_enc == 0 { - i64::MAX - } else { - (vt_enc - 1) as i64 - }; +impl ComponentOpenContext { + fn open( + seg_dir: &Path, + segment_id: u64, + manifest: &SegmentComponentManifestV1, + ) -> Result { + validate_packed_core_manifest_contract_for_open(manifest)?; + let has_packed_ranges = manifest + .components + .iter() + .any(|record| matches!(record.handle, ComponentHandleV1::PackedRange { .. })); + let has_container = manifest + .components + .iter() + .any(|record| record.kind == SegmentComponentKind::PackedSegmentContainer); + if !has_packed_ranges && !has_container { + return Ok(Self { + packed_core: None, + invalid_optional_packed_ranges: HashMap::new(), + }); + } + if !has_container { + return Ok(Self { + packed_core: None, + invalid_optional_packed_ranges: HashMap::new(), + }); + } - if self.deleted_edges.contains_key(&edge_id) { - continue; - } - if self.deleted_nodes.contains_key(&neighbor_id) { - continue; - } + let container = packed_core_container_record(manifest)?; + let mapped = open_external_manifest_component_record(seg_dir, segment_id, container)?; + let MappedData::Mmap { + mmap, + payload_offset, + payload_len, + } = mapped + else { + return Err(EngineError::CorruptRecord(format!( + "{PACKED_CORE_FILENAME} must contain an identity header" + ))); + }; + let packed_core = PackedCoreMapping { + component_id: container.component_id, + mmap, + payload_offset, + payload_len, + }; + let invalid_optional_packed_ranges = + collect_invalid_optional_packed_ranges(manifest, &packed_core); + Ok(Self { + packed_core: Some(packed_core), + invalid_optional_packed_ranges, + }) + } +} - if callback( - target_id, - edge_id, - neighbor_id, - weight, - valid_from_raw as i64, - valid_to, - ) - .is_break() - { - return Ok(ControlFlow::Break(())); - } - } +#[derive(Clone)] +struct PackedRangeForOpen { + kind: SegmentComponentKind, + optional: bool, + start: u64, + end: u64, +} + +fn collect_invalid_optional_packed_ranges( + manifest: &SegmentComponentManifestV1, + packed_core: &PackedCoreMapping, +) -> HashMap { + let mut invalid = HashMap::new(); + let mut ranges = Vec::new(); + for record in &manifest.components { + if !matches!(record.handle, ComponentHandleV1::PackedRange { .. }) { + continue; + } + match validate_packed_range_for_open(record, packed_core) { + Ok(Some(range)) => ranges.push(range), + Ok(None) => {} + Err(error) if record.requirement != ComponentRequirement::Required => { + invalid.insert(record.kind.clone(), error); + } + Err(_) => {} + } + } + + ranges.sort_by(|left, right| { + left.start + .cmp(&right.start) + .then_with(|| left.end.cmp(&right.end)) + .then_with(|| left.kind.kind_tag().cmp(&right.kind.kind_tag())) + .then_with(|| left.kind.index_id().cmp(&right.kind.index_id())) + }); + for pair in ranges.windows(2) { + let previous = &pair[0]; + let current = &pair[1]; + if current.start < previous.end { + let reason = format!( + "packed component range overlaps another range: previous={:?} [{}, {}), current={:?} [{}, {})", + previous.kind, previous.start, previous.end, current.kind, current.start, current.end + ); + if previous.optional { + invalid.insert(previous.kind.clone(), reason.clone()); + } + if current.optional { + invalid.insert(current.kind.clone(), reason); } } + } - Ok(ControlFlow::Continue(())) + invalid +} + +fn validate_packed_range_for_open( + record: &SegmentComponentRecordV1, + packed_core: &PackedCoreMapping, +) -> Result, String> { + if !is_packed_core_component_kind(&record.kind) { + return Err(format!( + "component {:?} is not allowed in {PACKED_CORE_FILENAME}", + record.kind + )); + } + let ComponentHandleV1::PackedRange { + container_component_id, + offset, + len, + } = &record.handle + else { + return Ok(None); + }; + if *container_component_id != packed_core.component_id { + return Err(format!( + "packed component {:?} points at the wrong {PACKED_CORE_FILENAME} container", + record.kind + )); + } + if record.payload_len != *len { + return Err(format!( + "packed component {:?} payload length does not match packed range", + record.kind + )); + } + let end = offset + .checked_add(*len) + .ok_or_else(|| format!("packed component {:?} range overflows", record.kind))?; + if end > packed_core.payload_len as u64 { + return Err(format!( + "packed component {:?} range [{}, {}) exceeds {PACKED_CORE_FILENAME} payload length {}", + record.kind, offset, end, packed_core.payload_len + )); + } + if *len == 0 { + return Ok(None); } + Ok(Some(PackedRangeForOpen { + kind: record.kind.clone(), + optional: record.requirement != ComponentRequirement::Required, + start: *offset, + end, + })) } -// --- Helpers --- +fn open_manifest_component_record( + seg_dir: &Path, + segment_id: u64, + record: &SegmentComponentRecordV1, + context: Option<&ComponentOpenContext>, +) -> Result { + match &record.handle { + ComponentHandleV1::ExternalFile { .. } => { + open_external_manifest_component_record(seg_dir, segment_id, record) + } + ComponentHandleV1::PackedRange { .. } => { + open_packed_manifest_component_record(record, context) + } + } +} -/// Validate the segment format version file. If the file is absent (pre-version -/// segment), this is allowed. If present, the magic and version must match. -/// Validate and return the segment format version. -/// Returns 0 for pre-version segments (no format.ver file). -fn read_format_version(seg_dir: &Path) -> Result { - let path = seg_dir.join("format.ver"); - if !path.exists() { +fn open_external_manifest_component_record( + seg_dir: &Path, + segment_id: u64, + record: &SegmentComponentRecordV1, +) -> Result { + let ComponentHandleV1::ExternalFile { + relative_path, + payload_offset, + payload_len, + } = &record.handle + else { return Err(EngineError::CorruptRecord( - "segment format version 0 is too old (minimum supported: 5)".into(), + "external component opener received a packed range handle".into(), )); + }; + let path = seg_dir.join(relative_path); + let data = mmap_file_payload(&path, *payload_offset, *payload_len)?; + let MappedData::Mmap { mmap, .. } = &data else { + return Ok(data); + }; + let header = decode_identity_header(mmap)?; + if header.segment_format_version != SEGMENT_FORMAT_VERSION + || header.segment_id != segment_id + || header.component_kind != record.kind + || header.logical_format_version != record.logical_format_version + || header.created_generation != record.created_generation + || header.payload_offset != *payload_offset + || header.payload_len != *payload_len + || header.component_id != record.component_id + || header.dependency_digest != record.dependency_digest + || header.build_fingerprint != record.build_fingerprint + || header.payload_digest != record.payload_digest + { + return Err(EngineError::CorruptRecord(format!( + "component identity header does not match manifest for {:?}", + record.kind + ))); } - let data = std::fs::read(&path)?; - if data.len() != 8 { + Ok(data) +} + +fn open_packed_manifest_component_record( + record: &SegmentComponentRecordV1, + context: Option<&ComponentOpenContext>, +) -> Result { + let Some(context) = context else { return Err(EngineError::CorruptRecord(format!( - "format.ver has invalid size {} (expected 8)", - data.len() + "packed component {:?} cannot be opened without {PACKED_CORE_FILENAME}", + record.kind + ))); + }; + let Some(packed_core) = &context.packed_core else { + return Err(EngineError::CorruptRecord(format!( + "packed component {:?} has no {PACKED_CORE_FILENAME} mapping", + record.kind ))); + }; + if let Some(reason) = context.invalid_optional_packed_ranges.get(&record.kind) { + return Err(EngineError::CorruptRecord(reason.clone())); } - if data[..4] != SEGMENT_MAGIC { + if !is_packed_core_component_kind(&record.kind) { return Err(EngineError::CorruptRecord(format!( - "format.ver has invalid magic {:?} (expected {:?})", - &data[..4], - SEGMENT_MAGIC + "component {:?} is not allowed in {PACKED_CORE_FILENAME}", + record.kind ))); } - let version = u32::from_le_bytes(data[4..8].try_into().unwrap()); - if version < 5 { + let ComponentHandleV1::PackedRange { + container_component_id, + offset, + len, + } = &record.handle + else { + unreachable!("packed component opener received external handle") + }; + if *container_component_id != packed_core.component_id { return Err(EngineError::CorruptRecord(format!( - "segment format version {} is too old (minimum supported: 5)", - version + "packed component {:?} points at the wrong {PACKED_CORE_FILENAME} container", + record.kind ))); } - if version > SEGMENT_FORMAT_VERSION { + if record.payload_len != *len { return Err(EngineError::CorruptRecord(format!( - "segment format version {} is newer than supported version {}", - version, SEGMENT_FORMAT_VERSION + "packed component {:?} payload length does not match packed range", + record.kind ))); } - Ok(version) -} - -/// Memory-map a file, returning Empty if the file doesn't exist. -/// Used for index files that may not be present in older segments. -fn mmap_file_optional(path: &Path) -> Result { - if !path.exists() { - return Ok(MappedData::Empty); + let end = offset.checked_add(*len).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "packed component {:?} range overflows", + record.kind + )) + })?; + if end > packed_core.payload_len as u64 { + return Err(EngineError::CorruptRecord(format!( + "packed component {:?} range [{}, {}) exceeds {PACKED_CORE_FILENAME} payload length {}", + record.kind, offset, end, packed_core.payload_len + ))); } - mmap_file(path) + let range_offset = usize::try_from(*offset).map_err(|_| { + EngineError::CorruptRecord(format!( + "packed component {:?} offset does not fit in usize", + record.kind + )) + })?; + let range_len = usize::try_from(*len).map_err(|_| { + EngineError::CorruptRecord(format!( + "packed component {:?} length does not fit in usize", + record.kind + )) + })?; + let payload_offset = packed_core + .payload_offset + .checked_add(range_offset) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "packed component {:?} physical offset overflows", + record.kind + )) + })?; + Ok(MappedData::Mmap { + mmap: Arc::clone(&packed_core.mmap), + payload_offset, + payload_len: range_len, + }) } -/// Memory-map a file. Returns MappedData::Empty for zero-byte files. -fn mmap_file(path: &Path) -> Result { +fn mmap_file_payload( + path: &Path, + payload_offset: u64, + payload_len: u64, +) -> Result { let file = File::open(path)?; let meta = file.metadata()?; - if meta.len() == 0 { - return Ok(MappedData::Empty); + let file_len = meta.len(); + let (payload_offset, payload_len) = if payload_len == u64::MAX { + if file_len == 0 { + return Ok(MappedData::Empty); + } + (0usize, file_len as usize) + } else { + let end = payload_offset.checked_add(payload_len).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "component payload range overflows for {}", + path.display() + )) + })?; + if end != file_len { + return Err(EngineError::CorruptRecord(format!( + "component payload range [{}, {}) does not match file length {} for {}", + payload_offset, + end, + file_len, + path.display() + ))); + } + (payload_offset as usize, payload_len as usize) + }; + if file_len == 0 { + return Err(EngineError::CorruptRecord(format!( + "manifest component file is empty for {}", + path.display() + ))); } // SAFETY: Segment files are immutable after write. No concurrent modification. let mmap = unsafe { Mmap::map(&file).map_err(EngineError::IoError)? }; - Ok(MappedData::Mmap(mmap)) + Ok(MappedData::Mmap { + mmap: Arc::new(mmap), + payload_offset, + payload_len, + }) } fn validate_secondary_eq_sidecar_data(data: &[u8]) -> Result<(), EngineError> { @@ -3995,23 +8079,8 @@ fn secondary_eq_sidecar_index_bounds(data: &[u8]) -> Result<(usize, usize), Engi Ok((count, idx_bytes)) } -fn validate_secondary_eq_sidecar_index(data: &[u8]) -> Result<(), EngineError> { - let (count, _) = secondary_eq_sidecar_index_bounds(data)?; - let mut previous = None; - for index in 0..count { - let entry_off = 8 + index * SECONDARY_EQ_ENTRY_SIZE; - let value_hash = read_u64_at(data, entry_off)?; - if let Some(previous) = previous { - if value_hash <= previous { - return Err(EngineError::CorruptRecord(format!( - "secondary equality sidecar value hashes are not strictly increasing at group {}", - index - ))); - } - } - previous = Some(value_hash); - } - Ok(()) +fn validate_secondary_eq_sidecar_index_header(data: &[u8]) -> Result<(), EngineError> { + secondary_eq_sidecar_index_bounds(data).map(|_| ()) } fn secondary_eq_group_range_from_entry( @@ -4180,6 +8249,24 @@ fn secondary_eq_posting_count(data: &[u8], value_hash: u64) -> Result, + value_hash: u64, +) -> Result { + let Some((offset, id_count)) = secondary_eq_group_range(data, value_hash)? else { + return Ok(0); + }; + let mut count = 0usize; + for id_index in 0..id_count { + let id = read_u64_at(data, offset + id_index * 8)?; + if !deleted_ids.contains_key(&id) { + count = count.saturating_add(1); + } + } + Ok(count) +} + fn secondary_range_sidecar_lower_bound( data: &[u8], target: (u64, u64), @@ -4309,17 +8396,71 @@ impl<'de> Visitor<'de> for PropLookupVisitor<'_> { } } +struct PropProjectionSeed<'a> { + targets: &'a [String], +} + +impl<'de> DeserializeSeed<'de> for PropProjectionSeed<'_> { + type Value = BTreeMap; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(PropProjectionVisitor { + targets: self.targets, + }) + } +} + +struct PropProjectionVisitor<'a> { + targets: &'a [String], +} + +impl<'de> Visitor<'de> for PropProjectionVisitor<'_> { + type Value = BTreeMap; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a property map") + } + + fn visit_map(self, mut map: M) -> Result + where + M: MapAccess<'de>, + { + let mut found = BTreeMap::new(); + while let Some(key) = map.next_key::()? { + if self.targets.iter().any(|target| target == &key) { + found.insert(key, map.next_value()?); + } else { + let _: IgnoredAny = map.next_value()?; + } + } + Ok(found) + } +} + fn decode_node_property_at( data: &[u8], offset: usize, id: u64, prop_key: &str, ) -> Result, EngineError> { - let _type_id = read_u32_at(data, offset)?; - let key_len = read_u16_at(data, offset + 4)? as usize; - let _key_bytes = read_bytes_at(data, offset + 6, key_len)?; + let label_count = read_u8_at(data, offset)? as usize; + if label_count == 0 || label_count > MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::CorruptRecord(format!( + "node {} targeted props decode has invalid label count {}", + id, label_count + ))); + } + let key_len_offset = offset + .checked_add(1 + label_count * 4) + .ok_or_else(|| EngineError::CorruptRecord("node property key offset overflow".into()))?; + let key_len = read_u16_at(data, key_len_offset)? as usize; + let key_start = key_len_offset + 2; + let _key_bytes = read_bytes_at(data, key_start, key_len)?; - let pos = offset + 6 + key_len; + let pos = key_start + key_len; let props_len = read_u32_at(data, pos + 20)? as usize; let props_bytes = read_bytes_at(data, pos + 24, props_len)?; let mut deserializer = rmp_serde::Deserializer::from_read_ref(props_bytes); @@ -4335,6 +8476,51 @@ fn decode_node_property_at( }) } +fn decode_edge_property_at( + data: &[u8], + offset: usize, + id: u64, + prop_key: &str, +) -> Result, EngineError> { + let props_len = read_u32_at(data, offset + 56)? as usize; + let props_bytes = read_bytes_at(data, offset + 60, props_len)?; + let mut deserializer = rmp_serde::Deserializer::from_read_ref(props_bytes); + PropLookupSeed { target: prop_key } + .deserialize(&mut deserializer) + .map_err(|error| { + EngineError::CorruptRecord(format!( + "edge {} targeted props decode at offset {}: {}", + id, + offset + 60, + error + )) + }) +} + +fn decode_edge_properties_at( + data: &[u8], + offset: usize, + id: u64, + prop_keys: &[String], +) -> Result, EngineError> { + if prop_keys.is_empty() { + return Ok(BTreeMap::new()); + } + let props_len = read_u32_at(data, offset + 56)? as usize; + let props_bytes = read_bytes_at(data, offset + 60, props_len)?; + let mut deserializer = rmp_serde::Deserializer::from_read_ref(props_bytes); + PropProjectionSeed { targets: prop_keys } + .deserialize(&mut deserializer) + .map_err(|error| { + EngineError::CorruptRecord(format!( + "edge {} projected props decode at offset {}: {}", + id, + offset + 60, + error + )) + }) +} + fn validate_node_vector_sidecars( segment_id: u64, vector_meta: &[u8], @@ -4371,8 +8557,17 @@ fn validate_node_vector_sidecars( ))); } + let count = usize::try_from(count).map_err(|_| { + EngineError::CorruptRecord(format!( + "segment {} node vector metadata count exceeds addressable memory", + segment_id + )) + })?; + let index_bytes = count + .checked_mul(NODE_VECTOR_META_ENTRY_SIZE) + .ok_or_else(|| EngineError::CorruptRecord("node vector metadata size overflow".into()))?; let expected_len = 8usize - .checked_add(count as usize * NODE_VECTOR_META_ENTRY_SIZE) + .checked_add(index_bytes) .ok_or_else(|| EngineError::CorruptRecord("node vector metadata size overflow".into()))?; if vector_meta.len() != expected_len { return Err(EngineError::CorruptRecord(format!( @@ -4390,7 +8585,7 @@ fn validate_node_vector_sidecars( let mut dense_count = 0usize; let mut sparse_count = 0usize; - for index in 0..count as usize { + for index in 0..count { let (flags, dense_offset, dense_len, sparse_offset, sparse_len) = read_node_vector_meta_entry(vector_meta, index)?; if flags & !(NODE_VECTOR_FLAG_DENSE | NODE_VECTOR_FLAG_SPARSE) != 0 { @@ -4410,7 +8605,21 @@ fn validate_node_vector_sidecars( } else { has_dense = true; dense_count += 1; - let dense_offset = dense_offset as usize; + let dense_offset = usize::try_from(dense_offset).map_err(|_| { + EngineError::CorruptRecord(format!( + "segment {} node vector entry {} dense offset exceeds addressable memory", + segment_id, index + )) + })?; + let dense_len = usize::try_from(dense_len).map_err(|_| { + EngineError::CorruptRecord(format!( + "segment {} node vector entry {} dense length exceeds addressable memory", + segment_id, index + )) + })?; + let dense_bytes = dense_len + .checked_mul(DENSE_VECTOR_VALUE_SIZE) + .ok_or_else(|| EngineError::CorruptRecord("dense blob size overflow".into()))?; if dense_offset != next_dense_offset { return Err(EngineError::CorruptRecord(format!( "segment {} node vector entry {} dense offset {} does not match expected {}", @@ -4420,13 +8629,13 @@ fn validate_node_vector_sidecars( validate_blob_range( dense_blob, dense_offset as u64, - dense_len as usize * DENSE_VECTOR_VALUE_SIZE, + dense_bytes, "dense", segment_id, index, )?; next_dense_offset = next_dense_offset - .checked_add(dense_len as usize * DENSE_VECTOR_VALUE_SIZE) + .checked_add(dense_bytes) .ok_or_else(|| EngineError::CorruptRecord("dense blob size overflow".into()))?; } @@ -4440,7 +8649,21 @@ fn validate_node_vector_sidecars( } else { has_sparse = true; sparse_count += 1; - let sparse_offset = sparse_offset as usize; + let sparse_offset = usize::try_from(sparse_offset).map_err(|_| { + EngineError::CorruptRecord(format!( + "segment {} node vector entry {} sparse offset exceeds addressable memory", + segment_id, index + )) + })?; + let sparse_len = usize::try_from(sparse_len).map_err(|_| { + EngineError::CorruptRecord(format!( + "segment {} node vector entry {} sparse length exceeds addressable memory", + segment_id, index + )) + })?; + let sparse_bytes = sparse_len + .checked_mul(SPARSE_VECTOR_ENTRY_SIZE) + .ok_or_else(|| EngineError::CorruptRecord("sparse blob size overflow".into()))?; if sparse_offset != next_sparse_offset { return Err(EngineError::CorruptRecord(format!( "segment {} node vector entry {} sparse offset {} does not match expected {}", @@ -4450,13 +8673,13 @@ fn validate_node_vector_sidecars( validate_blob_range( sparse_blob, sparse_offset as u64, - sparse_len as usize * SPARSE_VECTOR_ENTRY_SIZE, + sparse_bytes, "sparse", segment_id, index, )?; next_sparse_offset = next_sparse_offset - .checked_add(sparse_len as usize * SPARSE_VECTOR_ENTRY_SIZE) + .checked_add(sparse_bytes) .ok_or_else(|| EngineError::CorruptRecord("sparse blob size overflow".into()))?; } } @@ -4513,104 +8736,6 @@ struct NodeVectorSidecarSummary { sparse_count: usize, } -fn validate_sparse_posting_parity( - segment_id: u64, - node_meta: &[u8], - vector_meta: &[u8], - sparse_blob: &[u8], - sparse_posting_index: &[u8], - sparse_postings: &[u8], -) -> Result<(), EngineError> { - if vector_meta.is_empty() { - if !sparse_posting_index.is_empty() || !sparse_postings.is_empty() { - return Err(EngineError::CorruptRecord(format!( - "segment {} has sparse posting files without node vector metadata", - segment_id - ))); - } - return Ok(()); - } - - let mut expected = BTreeMap::>::new(); - let count = read_u64_at(node_meta, 0)? as usize; - for index in 0..count { - let node_id = read_u64_at(node_meta, 8 + index * NODE_META_ENTRY_SIZE)?; - let (flags, _dense_offset, _dense_len, sparse_offset, sparse_len) = - read_node_vector_meta_entry(vector_meta, index)?; - if flags & NODE_VECTOR_FLAG_SPARSE == 0 { - continue; - } - - let base = sparse_offset as usize; - for entry_index in 0..sparse_len as usize { - let entry_off = base + entry_index * SPARSE_VECTOR_ENTRY_SIZE; - let dimension_id = read_u32_at(sparse_blob, entry_off)?; - let weight = read_f32_at(sparse_blob, entry_off + 4)?; - if weight < 0.0 { - return Err(EngineError::CorruptRecord(format!( - "segment {} sparse vector payload for node {} dimension {} has negative weight", - segment_id, node_id, dimension_id - ))); - } - expected - .entry(dimension_id) - .or_default() - .push((node_id, weight)); - } - } - - let actual = read_sparse_posting_groups(sparse_posting_index, sparse_postings)?; - if expected.len() != actual.len() { - return Err(EngineError::CorruptRecord(format!( - "segment {} sparse posting dimension count {} does not match sparse vector payload count {}", - segment_id, - actual.len(), - expected.len() - ))); - } - - for (dimension_id, expected_postings) in &expected { - let Some(actual_postings) = actual.get(dimension_id) else { - return Err(EngineError::CorruptRecord(format!( - "segment {} sparse posting files are missing dimension {} from sparse vectors", - segment_id, dimension_id - ))); - }; - if expected_postings.len() != actual_postings.len() { - return Err(EngineError::CorruptRecord(format!( - "segment {} sparse posting dimension {} count {} does not match sparse vector payload count {}", - segment_id, - dimension_id, - actual_postings.len(), - expected_postings.len() - ))); - } - for (expected_posting, actual_posting) in - expected_postings.iter().zip(actual_postings.iter()) - { - if expected_posting.0 != actual_posting.0 - || expected_posting.1.to_bits() != actual_posting.1.to_bits() - { - return Err(EngineError::CorruptRecord(format!( - "segment {} sparse posting dimension {} does not match sparse vector payloads", - segment_id, dimension_id - ))); - } - } - } - - for dimension_id in actual.keys() { - if !expected.contains_key(dimension_id) { - return Err(EngineError::CorruptRecord(format!( - "segment {} sparse posting dimension {} is not present in sparse vector payloads", - segment_id, dimension_id - ))); - } - } - - Ok(()) -} - fn validate_blob_range( blob: &[u8], offset: u64, @@ -4652,20 +8777,11 @@ fn read_node_vector_meta_entry( fn read_dense_scoring_meta( node_meta: &[u8], + layout: NodeMetaLayout, vector_meta: &[u8], index: usize, ) -> Result { - let node_off = 8 + index * NODE_META_ENTRY_SIZE; - let node_end = node_off - .checked_add(NODE_META_ENTRY_SIZE) - .ok_or_else(|| EngineError::CorruptRecord("node meta offset overflow".into()))?; - let node_entry = node_meta.get(node_off..node_end).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "node meta read at index {} exceeds data length {}", - index, - node_meta.len() - )) - })?; + let node_entry = read_node_meta_entry_at(node_meta, layout, index)?; let vector_off = 8 + index * NODE_VECTOR_META_ENTRY_SIZE; let vector_end = vector_off @@ -4680,9 +8796,9 @@ fn read_dense_scoring_meta( })?; Ok(DenseScoringMeta { - type_id: u32::from_le_bytes(node_entry[20..24].try_into().unwrap()), - updated_at: i64::from_le_bytes(node_entry[24..32].try_into().unwrap()), - weight: f32::from_le_bytes(node_entry[32..36].try_into().unwrap()), + label_ids: node_entry.label_ids, + updated_at: node_entry.updated_at, + weight: node_entry.weight, dense_offset: u64::from_le_bytes(vector_entry[4..12].try_into().unwrap()) as usize, dense_len: u32::from_le_bytes(vector_entry[12..16].try_into().unwrap()) as usize, }) @@ -4690,20 +8806,11 @@ fn read_dense_scoring_meta( fn read_sparse_scoring_meta( node_meta: &[u8], + layout: NodeMetaLayout, vector_meta: &[u8], index: usize, ) -> Result { - let node_off = 8 + index * NODE_META_ENTRY_SIZE; - let node_end = node_off - .checked_add(NODE_META_ENTRY_SIZE) - .ok_or_else(|| EngineError::CorruptRecord("node meta offset overflow".into()))?; - let node_entry = node_meta.get(node_off..node_end).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "node meta read at index {} exceeds data length {}", - index, - node_meta.len() - )) - })?; + let node_entry = read_node_meta_entry_at(node_meta, layout, index)?; let vector_off = 8 + index * NODE_VECTOR_META_ENTRY_SIZE; let vector_end = vector_off @@ -4718,9 +8825,9 @@ fn read_sparse_scoring_meta( })?; Ok(SparseScoringMeta { - type_id: u32::from_le_bytes(node_entry[20..24].try_into().unwrap()), - updated_at: i64::from_le_bytes(node_entry[24..32].try_into().unwrap()), - weight: f32::from_le_bytes(node_entry[32..36].try_into().unwrap()), + label_ids: node_entry.label_ids, + updated_at: node_entry.updated_at, + weight: node_entry.weight, sparse_offset: u64::from_le_bytes(vector_entry[16..24].try_into().unwrap()) as usize, sparse_len: u32::from_le_bytes(vector_entry[24..28].try_into().unwrap()) as usize, }) @@ -4728,21 +8835,48 @@ fn read_sparse_scoring_meta( /// Decode a NodeRecord from mmap data at a given byte offset. /// The ID is passed separately; it comes from the index, not the data section. -/// Layout: type_id(4) key_len(2) key(N) created_at(8) updated_at(8) weight(4) props_len(4) props(M) +/// Layout: label_count(1) label_id(4)*count key_len(2) key(N) created_at(8) updated_at(8) weight(4) props_len(4) props(M) fn decode_node_at(data: &[u8], offset: usize, id: u64) -> Result { - let type_id = read_u32_at(data, offset)?; - let key_len = read_u16_at(data, offset + 4)? as usize; - let key_bytes = read_bytes_at(data, offset + 6, key_len)?; + decode_node_at_with_end(data, offset, id).map(|(node, _)| node) +} + +fn decode_node_at_with_end( + data: &[u8], + offset: usize, + id: u64, +) -> Result<(NodeRecord, usize), EngineError> { + let label_count = read_u8_at(data, offset)? as usize; + if label_count == 0 || label_count > MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::CorruptRecord(format!( + "node record {} has invalid label count {}", + id, label_count + ))); + } + let mut label_ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + for label_index in 0..label_count { + label_ids[label_index] = read_u32_at(data, offset + 1 + label_index * 4)?; + if label_index > 0 && label_ids[label_index - 1] >= label_ids[label_index] { + return Err(EngineError::CorruptRecord(format!( + "node record {} label IDs must be sorted ascending and unique", + id + ))); + } + } + let label_ids = NodeLabelSet::from_canonical_ids(&label_ids[..label_count]).map_err(|err| { + EngineError::CorruptRecord(format!("invalid label set on node record {}: {err}", id)) + })?; + + let key_len_offset = offset + 1 + label_count * 4; + let key_len = read_u16_at(data, key_len_offset)? as usize; + let key_start = key_len_offset + 2; + let key_bytes = read_bytes_at(data, key_start, key_len)?; let key = std::str::from_utf8(key_bytes) .map_err(|_| { - EngineError::CorruptRecord(format!( - "invalid UTF-8 in node key at offset {}", - offset + 6 - )) + EngineError::CorruptRecord(format!("invalid UTF-8 in node key at offset {}", key_start)) })? .to_string(); - let pos = offset + 6 + key_len; + let pos = key_start + key_len; let created_at = read_i64_at(data, pos)?; let updated_at = read_i64_at(data, pos + 8)?; let weight = read_f32_at(data, pos + 16)?; @@ -4751,28 +8885,35 @@ fn decode_node_at(data: &[u8], offset: usize, id: u64) -> Result = rmp_serde::from_slice(props_bytes).map_err(|e| { EngineError::CorruptRecord(format!("node props decode at offset {}: {}", pos + 24, e)) })?; + let end = pos + .checked_add(24) + .and_then(|base| base.checked_add(props_len)) + .ok_or_else(|| EngineError::CorruptRecord("node record end offset overflow".into()))?; - Ok(NodeRecord { - id, - type_id, - key, - props, - created_at, - updated_at, - weight, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }) + Ok(( + NodeRecord { + id, + label_ids, + key, + props, + created_at, + updated_at, + weight, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }, + end, + )) } /// Decode an EdgeRecord from mmap data at a given byte offset. /// The ID is passed separately; it comes from the index, not the data section. -/// Layout: from(8) to(8) type_id(4) created_at(8) updated_at(8) weight(4) valid_from(8) valid_to(8) props_len(4) props(N) +/// Layout: from(8) to(8) label_id(4) created_at(8) updated_at(8) weight(4) valid_from(8) valid_to(8) props_len(4) props(N) fn decode_edge_at(data: &[u8], offset: usize, id: u64) -> Result { let from = read_u64_at(data, offset)?; let to = read_u64_at(data, offset + 8)?; - let type_id = read_u32_at(data, offset + 16)?; + let label_id = read_u32_at(data, offset + 16)?; let created_at = read_i64_at(data, offset + 20)?; let updated_at = read_i64_at(data, offset + 28)?; let weight = read_f32_at(data, offset + 36)?; @@ -4793,7 +8934,7 @@ fn decode_edge_at(data: &[u8], offset: usize, id: u64) -> Result Result Result<(NodeIdMap, NodeIdMap), EngineError> { - let data = std::fs::read(path)?; if data.len() < 8 { return Ok((NodeIdMap::default(), NodeIdMap::default())); } - let count = read_u64_at(&data, 0)? as usize; + let count = read_u64_at(data, 0)? as usize; let mut deleted_nodes = NodeIdMap::default(); let mut deleted_edges = NodeIdMap::default(); @@ -4828,9 +8967,9 @@ fn load_tombstones( ))); } let kind = data[off]; - let id = read_u64_at(&data, off + 1)?; - let deleted_at = read_i64_at(&data, off + 9)?; - let last_write_seq = read_u64_at(&data, off + 17)?; + let id = read_u64_at(data, off + 1)?; + let deleted_at = read_i64_at(data, off + 9)?; + let last_write_seq = read_u64_at(data, off + 17)?; let entry = TombstoneEntry { deleted_at, last_write_seq, @@ -4881,31 +9020,241 @@ fn sparse_dot_score_from_blob( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::memtable::Memtable; - use crate::segment_writer::write_segment_without_degree_sidecar_for_test as write_segment; + use crate::memtable::{encode_range_prop_value, Memtable}; + use crate::segment_writer::{ + write_segment_without_degree_sidecar_for_test as write_segment, + write_segment_without_degree_sidecar_with_secondary_indexes_for_test as write_segment_with_secondary_indexes, + }; /// Test-only wrapper to expose read_varint_at for cross-module varint tests. pub fn read_varint_at_pub(data: &[u8], offset: usize) -> (u64, usize) { read_varint_at(data, offset).unwrap() } - /// Write a valid format.ver file for tests that build manual segment dirs. - fn write_format_ver(seg_dir: &std::path::Path) { - use crate::segment_writer::{SEGMENT_FORMAT_VERSION, SEGMENT_MAGIC}; - let mut data = Vec::new(); - data.extend_from_slice(&SEGMENT_MAGIC); - data.extend_from_slice(&SEGMENT_FORMAT_VERSION.to_le_bytes()); - std::fs::write(seg_dir.join("format.ver"), &data).unwrap(); + fn write_varint_for_test(buf: &mut Vec, mut value: u64) { + loop { + let mut byte = (value & 0x7F) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + buf.push(byte); + if value == 0 { + break; + } + } + } + + fn read_payload_file(path: &std::path::Path) -> Vec { + let data = std::fs::read(path).unwrap(); + if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] + == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + return data[start..end].to_vec(); + } + data + } + + fn rewrite_payload_file(path: &std::path::Path, rewrite: impl FnOnce(&mut [u8])) { + let mut data = std::fs::read(path).unwrap(); + let range = if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] + == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + start..end + } else { + 0..data.len() + }; + rewrite(&mut data[range]); + std::fs::write(path, data).unwrap(); + } + + fn tamper_envelope_format_version(seg_dir: &std::path::Path, version: u32) { + let path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + let mut data = std::fs::read(&path).unwrap(); + data[12..16].copy_from_slice(&version.to_le_bytes()); + std::fs::write(&path, data).unwrap(); + } + + fn read_segment_manifest_for_test(seg_dir: &std::path::Path) -> SegmentComponentManifestV1 { + let data = std::fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + crate::segment_components::decode_manifest_envelope(&data).unwrap() + } + + fn write_segment_manifest_for_test( + seg_dir: &std::path::Path, + manifest: &SegmentComponentManifestV1, + ) { + let data = crate::segment_components::encode_manifest_envelope(manifest).unwrap(); + std::fs::write(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME), data).unwrap(); + } + + fn manifest_component_path_for_test( + seg_dir: &std::path::Path, + kind: SegmentComponentKind, + ) -> std::path::PathBuf { + let manifest = read_segment_manifest_for_test(seg_dir); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .expect("component must be present in test manifest"); + match &record.handle { + ComponentHandleV1::ExternalFile { relative_path, .. } => seg_dir.join(relative_path), + ComponentHandleV1::PackedRange { .. } => { + panic!("test component unexpectedly used a packed handle") + } + } + } + + fn rewrite_component_payload_for_test( + seg_dir: &std::path::Path, + kind: SegmentComponentKind, + rewrite: impl FnOnce(&mut [u8]), + ) { + let manifest = read_segment_manifest_for_test(seg_dir); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .expect("component must be present in test manifest"); + match &record.handle { + ComponentHandleV1::ExternalFile { relative_path, .. } => { + let path = seg_dir.join(relative_path); + let mut data = std::fs::read(&path).unwrap(); + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + rewrite(&mut data[start..end]); + std::fs::write(path, data).unwrap(); + } + ComponentHandleV1::PackedRange { offset, len, .. } => { + let path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); + let mut data = std::fs::read(&path).unwrap(); + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize + *offset as usize; + let end = start + *len as usize; + rewrite(&mut data[start..end]); + std::fs::write(path, data).unwrap(); + } + } + } + + fn component_payload_bytes_for_test( + seg_dir: &std::path::Path, + kind: SegmentComponentKind, + ) -> Vec { + let manifest = read_segment_manifest_for_test(seg_dir); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .expect("component must be present in test manifest"); + match &record.handle { + ComponentHandleV1::ExternalFile { relative_path, .. } => { + let data = std::fs::read(seg_dir.join(relative_path)).unwrap(); + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + data[start..end].to_vec() + } + ComponentHandleV1::PackedRange { offset, len, .. } => { + let data = + std::fs::read(seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME)) + .unwrap(); + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize + *offset as usize; + let end = start + *len as usize; + data[start..end].to_vec() + } + } + } + + fn packed_range_for_test( + manifest: &SegmentComponentManifestV1, + kind: SegmentComponentKind, + ) -> (u64, u64) { + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .expect("component must be present in test manifest"); + let ComponentHandleV1::PackedRange { offset, len, .. } = &record.handle else { + panic!("component must be packed in test manifest"); + }; + (*offset, *len) + } + + fn write_u64_at_for_test(data: &mut [u8], offset: usize, value: u64) { + data[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + + fn reopen_test_segment_with_index( + seg_dir: &std::path::Path, + entry: &SecondaryIndexManifestEntry, + ) -> SegmentReader { + let manifest = read_segment_manifest_for_test(seg_dir); + let info = segment_info_from_manifest(&manifest); + SegmentReader::open_with_info(seg_dir, &info, None, std::slice::from_ref(entry)).unwrap() + } + + fn segment_info_from_manifest(manifest: &SegmentComponentManifestV1) -> SegmentInfo { + SegmentInfo { + id: manifest.segment_id, + node_count: manifest.node_count, + edge_count: manifest.edge_count, + segment_format_version: manifest.segment_format_version, + segment_data_id: manifest.segment_data_id, + } + } + + fn expect_engine_error(result: Result) -> String { + match result { + Ok(_) => panic!("expected EngineError"), + Err(error) => error.to_string(), + } + } + + fn write_segment_with_info( + mt: &Memtable, + dense_config: Option<&DenseVectorConfig>, + ) -> (tempfile::TempDir, std::path::PathBuf, SegmentInfo) { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let info = write_segment(&seg_dir, 1, mt, dense_config).unwrap(); + (dir, seg_dir, info) + } + + fn make_node(id: u64, label_id: u32, key: &str) -> NodeRecord { + NodeRecord { + id, + label_ids: NodeLabelSet::single(label_id).unwrap(), + key: key.to_string(), + props: BTreeMap::new(), + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + } } - fn make_node(id: u64, type_id: u32, key: &str) -> NodeRecord { + fn make_node_with_labels(id: u64, label_ids: &[u32], key: &str, updated_at: i64) -> NodeRecord { NodeRecord { id, - type_id, + label_ids: NodeLabelSet::from_canonical_ids(label_ids).unwrap(), key: key.to_string(), props: BTreeMap::new(), created_at: 1000, - updated_at: 1001, + updated_at, weight: 0.5, dense_vector: None, sparse_vector: None, @@ -4913,13 +9262,26 @@ pub(crate) mod tests { } } - fn make_node_with_props(id: u64, type_id: u32, key: &str) -> NodeRecord { + fn make_node_with_labels_and_props( + id: u64, + label_ids: &[u32], + key: &str, + props: BTreeMap, + updated_at: i64, + ) -> NodeRecord { + NodeRecord { + props, + ..make_node_with_labels(id, label_ids, key, updated_at) + } + } + + fn make_node_with_props(id: u64, label_id: u32, key: &str) -> NodeRecord { let mut props = BTreeMap::new(); props.insert("name".to_string(), PropValue::String(key.to_string())); props.insert("score".to_string(), PropValue::Float(0.95)); NodeRecord { id, - type_id, + label_ids: NodeLabelSet::single(label_id).unwrap(), key: key.to_string(), props, created_at: 1000, @@ -4931,12 +9293,12 @@ pub(crate) mod tests { } } - fn make_edge(id: u64, from: u64, to: u64, type_id: u32) -> EdgeRecord { + fn make_edge(id: u64, from: u64, to: u64, label_id: u32) -> EdgeRecord { EdgeRecord { id, from, to, - type_id, + label_id: label_id, props: BTreeMap::new(), created_at: 2000, updated_at: 2001, @@ -4959,7 +9321,7 @@ pub(crate) mod tests { let dir = tempfile::tempdir().unwrap(); let seg_dir = dir.path().join("seg_0001"); write_segment(&seg_dir, 1, mt, dense_config).unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, dense_config).unwrap(); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, dense_config).unwrap(); (dir, reader) } @@ -4990,77 +9352,143 @@ pub(crate) mod tests { let seg_dir = dir.path().join("seg_0001"); write_segment(&seg_dir, 1, &mt, None).unwrap(); std::fs::remove_file(seg_dir.join(crate::planner_stats::PLANNER_STATS_FILENAME)).unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); assert!(matches!( reader.planner_stats_debug_snapshot_for_test(), PlannerStatsAvailability::Missing )); + assert_eq!( + reader.optional_component_availability_for_test(SegmentComponentKind::PlannerStats), + ComponentAvailability::Missing + ); assert!(reader.get_node(1).unwrap().is_some()); - std::fs::write( - seg_dir.join(crate::planner_stats::PLANNER_STATS_FILENAME), + crate::segment_writer::publish_planner_stats_component_payload( + &seg_dir, + &[], b"not planner stats", ) .unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); assert!(matches!( reader.planner_stats_debug_snapshot_for_test(), PlannerStatsAvailability::Unavailable { .. } )); + assert!(matches!( + reader.optional_component_availability_for_test(SegmentComponentKind::PlannerStats), + ComponentAvailability::CorruptIdentity { .. } + )); assert!(reader.get_node(1).unwrap().is_some()); } - fn write_legacy_prop_index(seg_dir: &Path, groups: &[(u32, u64, u64, Vec)]) { - use std::fs::File; - use std::io::{BufWriter, Write}; - - let mut groups = groups.to_vec(); - groups.sort_unstable_by(|left, right| { - left.0 - .cmp(&right.0) - .then_with(|| left.1.cmp(&right.1)) - .then_with(|| left.2.cmp(&right.2)) - }); - - let path = seg_dir.join("prop_index.dat"); - let file = File::create(path).unwrap(); - let mut writer = BufWriter::new(file); - writer - .write_all(&(groups.len() as u64).to_le_bytes()) - .unwrap(); + #[test] + fn test_open_rejects_zero_byte_required_manifest_component() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + let (_dir, seg_dir, _info) = write_segment_with_info(&mt, None); + std::fs::write( + seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME), + [], + ) + .unwrap(); - let data_start = 8 + groups.len() * PROP_INDEX_ENTRY_SIZE; - let mut data_offset = data_start as u64; - for (type_id, key_hash, value_hash, ids) in &groups { - writer.write_all(&type_id.to_le_bytes()).unwrap(); - writer.write_all(&key_hash.to_le_bytes()).unwrap(); - writer.write_all(&value_hash.to_le_bytes()).unwrap(); - writer.write_all(&data_offset.to_le_bytes()).unwrap(); - writer.write_all(&(ids.len() as u32).to_le_bytes()).unwrap(); - data_offset += (ids.len() * 8) as u64; - } + let err = expect_engine_error(SegmentReader::open_unpinned_for_test(&seg_dir, 1, None)); + assert!( + err.contains("identity header") + || err.contains("does not match file length") + || err.contains("component file is empty"), + "got: {err}" + ); + } - for (_, _, _, ids) in &groups { - for id in ids { - writer.write_all(&id.to_le_bytes()).unwrap(); - } - } + #[test] + fn test_zero_byte_optional_manifest_component_is_unavailable() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + let (_dir, seg_dir, _info) = write_segment_with_info(&mt, None); + let stats_path = + manifest_component_path_for_test(&seg_dir, SegmentComponentKind::PlannerStats); + std::fs::write(stats_path, []).unwrap(); - writer.flush().unwrap(); - writer.get_ref().sync_all().unwrap(); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(matches!( + reader.planner_stats_debug_snapshot_for_test(), + PlannerStatsAvailability::Unavailable { .. } + )); + assert!(matches!( + reader.optional_component_availability_for_test(SegmentComponentKind::PlannerStats), + ComponentAvailability::CorruptIdentity { .. } + )); + assert!(reader.get_node(1).unwrap().is_some()); } - fn write_and_open_with_legacy_prop_index( - mt: &Memtable, - groups: &[(u32, u64, u64, Vec)], - ) -> (tempfile::TempDir, SegmentReader) { + #[test] + fn test_planner_stats_identity_ignores_unrepresented_building_indexes() { + let mt = Memtable::new(); + let mut props = BTreeMap::new(); + props.insert( + "status".to_string(), + PropValue::String("active".to_string()), + ); + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), + key: "alice".to_string(), + props, + created_at: 1000, + updated_at: 1001, + weight: 1.0, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 1, + ); + let building = SecondaryIndexManifestEntry { + index_id: 101, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "status".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, + }; + let ready = SecondaryIndexManifestEntry { + index_id: 102, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "queued".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + mt.register_secondary_index(&building); + mt.register_secondary_index(&ready); + let dir = tempfile::tempdir().unwrap(); let seg_dir = dir.path().join("seg_0001"); - crate::segment_writer::write_segment_without_degree_sidecar_for_test(&seg_dir, 1, mt, None) - .unwrap(); - write_legacy_prop_index(&seg_dir, groups); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); - (dir, reader) + let indexes = vec![ready, building]; + let info = crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + &seg_dir, + 1, + &mt, + None, + &indexes, + ) + .unwrap(); + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &indexes).unwrap(); + assert!( + reader.planner_stats_available(), + "{:?}", + reader.planner_stats_debug_snapshot_for_test() + ); + assert_eq!( + reader.optional_component_availability_for_test(SegmentComponentKind::PlannerStats), + ComponentAvailability::Available + ); } fn write_and_open_with_secondary_eq_sidecar( @@ -5071,7 +9499,7 @@ pub(crate) mod tests { let seg_dir = dir.path().join("seg_0001"); let mt = mt.clone(); mt.register_secondary_index(entry); - crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + let info = crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( &seg_dir, 1, &mt, @@ -5079,7 +9507,9 @@ pub(crate) mod tests { std::slice::from_ref(entry), ) .unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(entry)) + .unwrap(); (dir, reader) } @@ -5091,7 +9521,7 @@ pub(crate) mod tests { let seg_dir = dir.path().join("seg_0001"); let mt = mt.clone(); mt.register_secondary_index(entry); - crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + let info = crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( &seg_dir, 1, &mt, @@ -5099,7 +9529,9 @@ pub(crate) mod tests { std::slice::from_ref(entry), ) .unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(entry)) + .unwrap(); (dir, reader) } @@ -5111,7 +9543,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "apple".to_string(), props, created_at: 1000, @@ -5126,14 +9558,14 @@ pub(crate) mod tests { let entry = SecondaryIndexManifestEntry { index_id: 91, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, state: SecondaryIndexState::Ready, last_error: None, }; - let (dir, reader) = write_and_open_with_secondary_eq_sidecar(&mt, &entry); + let (_dir, reader) = write_and_open_with_secondary_eq_sidecar(&mt, &entry); assert_eq!( reader.declared_index_runtime_coverage_state( entry.index_id, @@ -5151,12 +9583,28 @@ pub(crate) mod tests { DeclaredIndexRuntimeCoverageState::Available ); - let sidecar_path = dir - .path() - .join("seg_0001") - .join("secondary_indexes") - .join(format!("node_prop_eq_{}.dat", entry.index_id)); + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let mt = mt.clone(); + mt.register_secondary_index(&entry); + let info = crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + &seg_dir, + 1, + &mt, + None, + std::slice::from_ref(&entry), + ) + .unwrap(); + let sidecar_path = manifest_component_path_for_test( + &seg_dir, + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }, + ); std::fs::remove_file(&sidecar_path).unwrap(); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(&entry)) + .unwrap(); reader.warm_declared_index_runtime_coverage(&entry); assert_eq!( reader.declared_index_runtime_coverage_state( @@ -5166,7 +9614,24 @@ pub(crate) mod tests { DeclaredIndexRuntimeCoverageState::Missing ); + crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + &seg_dir, + 1, + &mt, + None, + std::slice::from_ref(&entry), + ) + .unwrap(); + let sidecar_path = manifest_component_path_for_test( + &seg_dir, + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }, + ); std::fs::write(&sidecar_path, [1u8, 2, 3]).unwrap(); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(&entry)) + .unwrap(); reader.warm_declared_index_runtime_coverage(&entry); assert_eq!( reader.declared_index_runtime_coverage_state( @@ -5185,7 +9650,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "apple".to_string(), props, created_at: 1000, @@ -5200,7 +9665,7 @@ pub(crate) mod tests { let entry = SecondaryIndexManifestEntry { index_id: 92, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Range { @@ -5209,7 +9674,7 @@ pub(crate) mod tests { state: SecondaryIndexState::Ready, last_error: None, }; - let (dir, reader) = write_and_open_with_secondary_range_sidecar(&mt, &entry); + let (_dir, reader) = write_and_open_with_secondary_range_sidecar(&mt, &entry); reader.warm_declared_index_runtime_coverage(&entry); assert_eq!( @@ -5220,12 +9685,28 @@ pub(crate) mod tests { DeclaredIndexRuntimeCoverageState::Available ); - let sidecar_path = dir - .path() - .join("seg_0001") - .join("secondary_indexes") - .join(format!("node_prop_range_{}.dat", entry.index_id)); + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let mt = mt.clone(); + mt.register_secondary_index(&entry); + let info = crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + &seg_dir, + 1, + &mt, + None, + std::slice::from_ref(&entry), + ) + .unwrap(); + let sidecar_path = manifest_component_path_for_test( + &seg_dir, + SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }, + ); std::fs::remove_file(&sidecar_path).unwrap(); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(&entry)) + .unwrap(); reader.warm_declared_index_runtime_coverage(&entry); assert_eq!( reader.declared_index_runtime_coverage_state( @@ -5235,7 +9716,24 @@ pub(crate) mod tests { DeclaredIndexRuntimeCoverageState::Missing ); + crate::segment_writer::write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + &seg_dir, + 1, + &mt, + None, + std::slice::from_ref(&entry), + ) + .unwrap(); + let sidecar_path = manifest_component_path_for_test( + &seg_dir, + SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }, + ); std::fs::write(&sidecar_path, [1u8, 2, 3]).unwrap(); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(&entry)) + .unwrap(); reader.warm_declared_index_runtime_coverage(&entry); assert_eq!( reader.declared_index_runtime_coverage_state( @@ -5265,22 +9763,31 @@ pub(crate) mod tests { } } - fn build_u64_key_index(keys: &[u64], entry_size: usize, key_offset: usize) -> Vec { - let mut data = vec![0u8; 8 + keys.len() * entry_size]; + fn build_u64_key_index_with_start( + keys: &[u64], + idx_start: usize, + entry_size: usize, + key_offset: usize, + ) -> Vec { + let mut data = vec![0u8; idx_start + keys.len() * entry_size]; data[0..8].copy_from_slice(&(keys.len() as u64).to_le_bytes()); for (i, key) in keys.iter().enumerate() { - let off = 8 + i * entry_size + key_offset; + let off = idx_start + i * entry_size + key_offset; data[off..off + 8].copy_from_slice(&key.to_le_bytes()); } data } + fn build_u64_key_index(keys: &[u64], entry_size: usize, key_offset: usize) -> Vec { + build_u64_key_index_with_start(keys, 8, entry_size, key_offset) + } + #[test] fn test_batch_strategy_prefers_seek_for_tiny_key_count() { let keys: Vec = (1..=10_000).collect(); let idx = build_u64_key_index(&keys, NODE_INDEX_ENTRY_SIZE, 0); let strategy = - choose_batch_read_strategy(&idx, keys.len(), NODE_INDEX_ENTRY_SIZE, 0, 2, 500, 501) + choose_batch_read_strategy(&idx, 8, keys.len(), NODE_INDEX_ENTRY_SIZE, 0, 2, 500, 501) .unwrap(); assert_eq!(strategy, BatchReadStrategy::SeekPerKey); } @@ -5291,6 +9798,7 @@ pub(crate) mod tests { let idx = build_u64_key_index(&keys, NODE_INDEX_ENTRY_SIZE, 0); let strategy = choose_batch_read_strategy( &idx, + 8, keys.len(), NODE_INDEX_ENTRY_SIZE, 0, @@ -5306,9 +9814,35 @@ pub(crate) mod tests { fn test_batch_strategy_prefers_seek_for_sparse_range() { let keys: Vec = (1..=10_000).collect(); let idx = build_u64_key_index(&keys, NODE_INDEX_ENTRY_SIZE, 0); - let strategy = - choose_batch_read_strategy(&idx, keys.len(), NODE_INDEX_ENTRY_SIZE, 0, 64, 100, 9_900) - .unwrap(); + let strategy = choose_batch_read_strategy( + &idx, + 8, + keys.len(), + NODE_INDEX_ENTRY_SIZE, + 0, + 64, + 100, + 9_900, + ) + .unwrap(); + assert_eq!(strategy, BatchReadStrategy::SeekPerKey); + } + + #[test] + fn test_batch_strategy_uses_explicit_index_start() { + let keys: Vec = (1..=10_000).collect(); + let idx = build_u64_key_index_with_start(&keys, 48, NODE_INDEX_ENTRY_SIZE, 0); + let strategy = choose_batch_read_strategy( + &idx, + 48, + keys.len(), + NODE_INDEX_ENTRY_SIZE, + 0, + 64, + 100, + 9_900, + ) + .unwrap(); assert_eq!(strategy, BatchReadStrategy::SeekPerKey); } @@ -5322,7 +9856,7 @@ pub(crate) mod tests { let (_dir, reader) = write_and_open(&mt); let node = reader.get_node(42).unwrap().unwrap(); assert_eq!(node.id, 42); - assert_eq!(node.type_id, 1); + assert_eq!(node.label_ids.as_slice(), [1]); assert_eq!(node.key, "alice"); assert_eq!(node.created_at, 1000); assert!((node.weight - 0.5).abs() < f32::EPSILON); @@ -5337,37 +9871,219 @@ pub(crate) mod tests { assert!(reader.get_node(999).unwrap().is_none()); } - #[test] - fn test_get_node_with_properties() { - let mt = Memtable::new(); - mt.apply_op(&WalOp::UpsertNode(make_node_with_props(1, 1, "alice")), 0); + #[test] + fn test_get_node_with_properties() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node_with_props(1, 1, "alice")), 0); + + let (_dir, reader) = write_and_open(&mt); + let node = reader.get_node(1).unwrap().unwrap(); + assert_eq!( + node.props.get("name"), + Some(&PropValue::String("alice".to_string())) + ); + if let Some(PropValue::Float(f)) = node.props.get("score") { + assert!((f - 0.95).abs() < f64::EPSILON); + } else { + panic!("expected Float property"); + } + } + + #[test] + fn test_get_node_with_vectors() { + let mt = Memtable::new(); + let dense_config = dense_config(3); + let mut node = make_node(7, 1, "vector"); + node.dense_vector = Some(vec![0.1, 0.2, 0.3]); + node.sparse_vector = Some(vec![(2, 1.5), (9, 0.25)]); + mt.apply_op(&WalOp::UpsertNode(node), 0); + + let (_dir, reader) = write_and_open_with_dense_config(&mt, Some(&dense_config)); + let node = reader.get_node(7).unwrap().unwrap(); + assert_eq!(node.dense_vector, Some(vec![0.1, 0.2, 0.3])); + assert_eq!(node.sparse_vector, Some(vec![(2, 1.5), (9, 0.25)])); + } + + #[test] + fn test_multi_label_flush_reopen_indexes_every_member_label() { + let mt = Memtable::new(); + let cases: &[(u64, &[u32], &str, i64)] = &[ + (1, &[1], "n1", 100), + (2, &[10, 11], "n2", 200), + (3, &[20, 21, 22, 23, 24], "n5", 500), + ( + 4, + &[100, 101, 102, 103, 104, 105, 106, 107, 108, 109], + "n10", + 1000, + ), + ]; + for &(id, labels, key, updated_at) in cases { + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(id, labels, key, updated_at)), + id, + ); + } + + let (_dir, reader) = write_and_open(&mt); + assert_eq!(reader.node_meta_count(), cases.len() as u64); + for (meta_index, &(id, labels, key, updated_at)) in cases.iter().enumerate() { + let node = reader.get_node(id).unwrap().unwrap(); + assert_eq!(node.label_ids.as_slice(), labels); + let meta = reader.node_meta_at(meta_index).unwrap(); + assert_eq!(meta.node_id, id); + assert_eq!(meta.label_ids.as_slice(), labels); + + for &label_id in labels { + let by_key = reader.node_by_key(label_id, key).unwrap().unwrap(); + assert_eq!(by_key.id, id); + assert_eq!(by_key.label_ids.as_slice(), labels); + assert_eq!(reader.nodes_by_label_id(label_id).unwrap(), vec![id]); + assert_eq!(reader.node_label_posting_count(label_id).unwrap(), 1); + assert_eq!( + reader + .nodes_by_time_range(label_id, updated_at, updated_at) + .unwrap(), + vec![id] + ); + } + } + assert!(reader.node_by_key(999, "n10").unwrap().is_none()); + assert!(reader.nodes_by_label_id(999).unwrap().is_empty()); + } + + #[test] + fn test_multi_label_flush_declared_property_sidecars_by_member_label() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let mt = Memtable::new(); + + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + props.insert("score".to_string(), PropValue::Int(42)); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels_and_props( + 1, + &[1, 2, 3], + "item", + props, + 100, + )), + 1, + ); + + let eq_label_1 = SecondaryIndexManifestEntry { + index_id: 10, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let eq_label_2 = SecondaryIndexManifestEntry { + index_id: 11, + target: SecondaryIndexTarget::NodeProperty { + label_id: 2, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let eq_absent = SecondaryIndexManifestEntry { + index_id: 12, + target: SecondaryIndexTarget::NodeProperty { + label_id: 9, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_label_2 = SecondaryIndexManifestEntry { + index_id: 13, + target: SecondaryIndexTarget::NodeProperty { + label_id: 2, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_label_3 = SecondaryIndexManifestEntry { + index_id: 14, + target: SecondaryIndexTarget::NodeProperty { + label_id: 3, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let indexes = vec![ + eq_label_1.clone(), + eq_label_2.clone(), + eq_absent.clone(), + range_label_2.clone(), + range_label_3.clone(), + ]; + for entry in &indexes { + mt.register_secondary_index(entry); + } + + let info = write_segment_with_secondary_indexes(&seg_dir, 1, &mt, None, &indexes).unwrap(); + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &indexes).unwrap(); - let (_dir, reader) = write_and_open(&mt); - let node = reader.get_node(1).unwrap().unwrap(); + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); assert_eq!( - node.props.get("name"), - Some(&PropValue::String("alice".to_string())) + reader + .find_nodes_by_secondary_eq_index(eq_label_1.index_id, red_hash) + .unwrap(), + vec![1] ); - if let Some(PropValue::Float(f)) = node.props.get("score") { - assert!((f - 0.95).abs() < f64::EPSILON); - } else { - panic!("expected Float property"); - } - } - - #[test] - fn test_get_node_with_vectors() { - let mt = Memtable::new(); - let dense_config = dense_config(3); - let mut node = make_node(7, 1, "vector"); - node.dense_vector = Some(vec![0.1, 0.2, 0.3]); - node.sparse_vector = Some(vec![(2, 1.5), (9, 0.25)]); - mt.apply_op(&WalOp::UpsertNode(node), 0); + assert_eq!( + reader + .find_nodes_by_secondary_eq_index(eq_label_2.index_id, red_hash) + .unwrap(), + vec![1] + ); + assert!(reader + .find_nodes_by_secondary_eq_index(eq_absent.index_id, red_hash) + .unwrap() + .is_empty()); - let (_dir, reader) = write_and_open_with_dense_config(&mt, Some(&dense_config)); - let node = reader.get_node(7).unwrap().unwrap(); - assert_eq!(node.dense_vector, Some(vec![0.1, 0.2, 0.3])); - assert_eq!(node.sparse_vector, Some(vec![(2, 1.5), (9, 0.25)])); + let encoded_score = + encode_range_prop_value(SecondaryIndexRangeDomain::Int, &PropValue::Int(42)).unwrap(); + assert_eq!( + reader + .find_nodes_by_secondary_range_index_if_present( + range_label_2.index_id, + Some((encoded_score, true)), + Some((encoded_score, true)), + None, + ) + .unwrap() + .unwrap(), + vec![(encoded_score, 1)] + ); + assert_eq!( + reader + .find_nodes_by_secondary_range_index_if_present( + range_label_3.index_id, + Some((encoded_score, true)), + Some((encoded_score, true)), + None, + ) + .unwrap() + .unwrap(), + vec![(encoded_score, 1)] + ); } #[test] @@ -5417,7 +10133,7 @@ pub(crate) mod tests { assert_eq!(edge.id, 100); assert_eq!(edge.from, 1); assert_eq!(edge.to, 2); - assert_eq!(edge.type_id, 10); + assert_eq!(edge.label_id, 10); } #[test] @@ -5491,7 +10207,7 @@ pub(crate) mod tests { } #[test] - fn test_neighbors_with_type_filter() { + fn test_neighbors_with_label_filter() { let mt = Memtable::new(); mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 0); mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 0); @@ -5501,14 +10217,14 @@ pub(crate) mod tests { let (_dir, reader) = write_and_open(&mt); - // Filter type 10 only + // Filter label 10 only let nbrs = reader .neighbors(1, Direction::Outgoing, Some(&[10]), 0) .unwrap(); assert_eq!(nbrs.len(), 1); assert_eq!(nbrs[0].node_id, 2); - // Filter type 20 only + // Filter label 20 only let nbrs = reader .neighbors(1, Direction::Outgoing, Some(&[20]), 0) .unwrap(); @@ -5623,6 +10339,73 @@ pub(crate) mod tests { assert_eq!(seen, 1); } + #[test] + fn test_adjacency_posting_delta_overflow_returns_corruption() { + let mt = Memtable::new(); + for id in 1..=3 { + mt.apply_op(&WalOp::UpsertNode(make_node(id, 1, &format!("n{}", id))), 0); + } + mt.apply_op(&WalOp::UpsertEdge(make_edge(u64::MAX - 1, 1, 2, 10)), 0); + mt.apply_op(&WalOp::UpsertEdge(make_edge(u64::MAX, 1, 3, 10)), 0); + + let (dir, reader) = write_and_open(&mt); + assert_eq!( + reader + .neighbors(1, Direction::Outgoing, Some(&[10]), 0) + .unwrap() + .len(), + 2 + ); + + let seg_dir = dir.path().join("seg_0001"); + rewrite_component_payload_for_test( + &seg_dir, + SegmentComponentKind::AdjOutPostings, + |payload| { + let (first_delta, mut offset) = read_varint_at(payload, 0).unwrap(); + assert_eq!(first_delta, u64::MAX - 1); + let (_, len) = read_varint_at(payload, offset).unwrap(); + offset += len; + offset += 4; + let (_, len) = read_varint_at(payload, offset).unwrap(); + offset += len; + let (_, len) = read_varint_at(payload, offset).unwrap(); + offset += len; + + let (second_delta, second_len) = read_varint_at(payload, offset).unwrap(); + assert_eq!(second_delta, 1); + let mut replacement = Vec::new(); + write_varint_for_test(&mut replacement, 2); + assert_eq!(replacement.len(), second_len); + payload[offset..offset + second_len].copy_from_slice(&replacement); + }, + ); + + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + let err = reader + .neighbors(1, Direction::Outgoing, Some(&[10]), 0) + .unwrap_err(); + assert!( + matches!(&err, EngineError::CorruptRecord(message) if message.contains("delta overflow")), + "expected adjacency delta overflow corruption, got {err}" + ); + + let mut cursors = reader + .endpoint_adj_posting_cursors(&[1], Direction::Outgoing, Some(&[10])) + .unwrap(); + assert_eq!( + reader.next_adj_posting_edge_id(&mut cursors[0]).unwrap(), + Some(u64::MAX - 1) + ); + let err = reader + .next_adj_posting_edge_id(&mut cursors[0]) + .unwrap_err(); + assert!( + matches!(&err, EngineError::CorruptRecord(message) if message.contains("delta overflow")), + "expected cursor delta overflow corruption, got {err}" + ); + } + // --- Empty segment --- #[test] @@ -5676,9 +10459,9 @@ pub(crate) mod tests { // Every node should be findable by key for i in 1..=50 { - let type_id = (i % 3) as u32 + 1; + let label_id = (i % 3) as u32 + 1; let key = format!("key_{:04}", i); - let node = reader.node_by_key(type_id, &key).unwrap().unwrap(); + let node = reader.node_by_key(label_id, &key).unwrap().unwrap(); assert_eq!(node.id, i); } } @@ -5694,229 +10477,69 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(make_node_with_props(i, 1, &format!("node_{}", i))), 0, - ); - } - mt.apply_op(&WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), 0); - mt.apply_op(&WalOp::UpsertEdge(make_edge(2, 2, 3, 10)), 0); - mt.apply_op(&WalOp::UpsertEdge(make_edge(3, 1, 3, 20)), 0); - mt.apply_op(&WalOp::UpsertEdge(make_edge(4, 4, 5, 10)), 0); - - // Delete one node and one edge - mt.apply_op( - &WalOp::DeleteNode { - id: 99, - deleted_at: 9999, - }, - 0, - ); - mt.apply_op( - &WalOp::DeleteEdge { - id: 99, - deleted_at: 9999, - }, - 0, - ); - - let (_dir, reader) = write_and_open(&mt); - - // Verify nodes - assert_eq!(reader.node_count(), 5); - for i in 1..=5 { - let node = reader.get_node(i).unwrap().unwrap(); - assert_eq!(node.key, format!("node_{}", i)); - assert_eq!( - node.props.get("name"), - Some(&PropValue::String(format!("node_{}", i))) - ); - } - - // Verify edges - assert_eq!(reader.edge_count(), 4); - let e1 = reader.get_edge(1).unwrap().unwrap(); - assert_eq!(e1.from, 1); - assert_eq!(e1.to, 2); - - // Verify key lookup - let n = reader.node_by_key(1, "node_3").unwrap().unwrap(); - assert_eq!(n.id, 3); - - // Verify neighbors - let out1 = reader.neighbors(1, Direction::Outgoing, None, 0).unwrap(); - assert_eq!(out1.len(), 2); // edges to 2 and 3 - let ids: NodeIdSet = out1.iter().map(|n| n.node_id).collect(); - assert!(ids.contains(&2)); - assert!(ids.contains(&3)); - - // Verify type-filtered neighbors - let out1_t10 = reader - .neighbors(1, Direction::Outgoing, Some(&[10]), 0) - .unwrap(); - assert_eq!(out1_t10.len(), 1); - assert_eq!(out1_t10[0].node_id, 2); - - // Verify tombstones - assert!(reader.is_node_deleted(99)); - assert!(reader.is_edge_deleted(99)); - } - - // --- Property index roundtrip --- - - #[test] - fn test_legacy_prop_index_roundtrip() { - use crate::types::{hash_prop_key, hash_prop_value}; - - let mt = Memtable::new(); - - let mut props1 = BTreeMap::new(); - props1.insert("color".to_string(), PropValue::String("red".to_string())); - mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 1, - type_id: 1, - key: "apple".to_string(), - props: props1, - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), - 0, - ); - - let mut props2 = BTreeMap::new(); - props2.insert("color".to_string(), PropValue::String("red".to_string())); - mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 2, - type_id: 1, - key: "cherry".to_string(), - props: props2, - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), - 0, - ); - - let mut props3 = BTreeMap::new(); - props3.insert("color".to_string(), PropValue::String("green".to_string())); - mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 3, - type_id: 1, - key: "lime".to_string(), - props: props3, - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), - 0, - ); - - let key_hash = hash_prop_key("color"); - let val_hash = hash_prop_value(&PropValue::String("red".to_string())); - let val_hash_green = hash_prop_value(&PropValue::String("green".to_string())); - let (_dir, reader) = write_and_open_with_legacy_prop_index( - &mt, - &[ - (1, key_hash, val_hash, vec![1, 2]), - (1, key_hash, val_hash_green, vec![3]), - ], - ); - - // Query for red nodes - let mut reds = reader - .find_nodes_by_prop_hash(1, key_hash, val_hash) - .unwrap(); - reds.sort(); - assert_eq!(reds, vec![1, 2]); - - // Query for green nodes - let greens = reader - .find_nodes_by_prop_hash(1, key_hash, val_hash_green) - .unwrap(); - assert_eq!(greens, vec![3]); - - // Non-existent value - let val_hash_blue = hash_prop_value(&PropValue::String("blue".to_string())); - assert!(reader - .find_nodes_by_prop_hash(1, key_hash, val_hash_blue) - .unwrap() - .is_empty()); - - // Wrong type_id - assert!(reader - .find_nodes_by_prop_hash(99, key_hash, val_hash) - .unwrap() - .is_empty()); - } - - #[test] - fn test_legacy_prop_index_excludes_tombstoned() { - use crate::types::{hash_prop_key, hash_prop_value}; - - let mt = Memtable::new(); - - let mut props = BTreeMap::new(); - props.insert("tag".to_string(), PropValue::String("x".to_string())); - mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 1, - type_id: 1, - key: "a".to_string(), - props: props.clone(), - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), - 0, - ); + ); + } + mt.apply_op(&WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), 0); + mt.apply_op(&WalOp::UpsertEdge(make_edge(2, 2, 3, 10)), 0); + mt.apply_op(&WalOp::UpsertEdge(make_edge(3, 1, 3, 20)), 0); + mt.apply_op(&WalOp::UpsertEdge(make_edge(4, 4, 5, 10)), 0); + + // Delete one node and one edge mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 2, - type_id: 1, - key: "b".to_string(), - props, - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), + &WalOp::DeleteNode { + id: 99, + deleted_at: 9999, + }, 0, ); - - // Delete node 2 (tombstone) mt.apply_op( - &WalOp::DeleteNode { - id: 2, + &WalOp::DeleteEdge { + id: 99, deleted_at: 9999, }, 0, ); - let key_hash = hash_prop_key("tag"); - let val_hash = hash_prop_value(&PropValue::String("x".to_string())); - let (_dir, reader) = - write_and_open_with_legacy_prop_index(&mt, &[(1, key_hash, val_hash, vec![1, 2])]); + let (_dir, reader) = write_and_open(&mt); + + // Verify nodes + assert_eq!(reader.node_count(), 5); + for i in 1..=5 { + let node = reader.get_node(i).unwrap().unwrap(); + assert_eq!(node.key, format!("node_{}", i)); + assert_eq!( + node.props.get("name"), + Some(&PropValue::String(format!("node_{}", i))) + ); + } + + // Verify edges + assert_eq!(reader.edge_count(), 4); + let e1 = reader.get_edge(1).unwrap().unwrap(); + assert_eq!(e1.from, 1); + assert_eq!(e1.to, 2); + + // Verify key lookup + let n = reader.node_by_key(1, "node_3").unwrap().unwrap(); + assert_eq!(n.id, 3); + + // Verify neighbors + let out1 = reader.neighbors(1, Direction::Outgoing, None, 0).unwrap(); + assert_eq!(out1.len(), 2); // edges to 2 and 3 + let ids: NodeIdSet = out1.iter().map(|n| n.node_id).collect(); + assert!(ids.contains(&2)); + assert!(ids.contains(&3)); - // Node 2 is tombstoned, so only node 1 should be returned - let results = reader - .find_nodes_by_prop_hash(1, key_hash, val_hash) + // Verify label-filtered neighbors + let out1_t10 = reader + .neighbors(1, Direction::Outgoing, Some(&[10]), 0) .unwrap(); - assert_eq!(results, vec![1]); + assert_eq!(out1_t10.len(), 1); + assert_eq!(out1_t10[0].node_id, 2); + + // Verify tombstones + assert!(reader.is_node_deleted(99)); + assert!(reader.is_edge_deleted(99)); } #[test] @@ -5930,7 +10553,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "apple".to_string(), props: props1, created_at: 1000, @@ -5948,7 +10571,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 2, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "cherry".to_string(), props: props2, created_at: 1000, @@ -5966,7 +10589,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 3, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "lime".to_string(), props: props3, created_at: 1000, @@ -5982,11 +10605,11 @@ pub(crate) mod tests { let entry = SecondaryIndexManifestEntry { index_id: 41, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, - state: SecondaryIndexState::Building, + state: SecondaryIndexState::Ready, last_error: None, }; let (_dir, reader) = write_and_open_with_secondary_eq_sidecar(&mt, &entry); @@ -6016,7 +10639,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "apple".to_string(), props, created_at: 1000, @@ -6032,11 +10655,11 @@ pub(crate) mod tests { let entry = SecondaryIndexManifestEntry { index_id: 51, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, - state: SecondaryIndexState::Building, + state: SecondaryIndexState::Ready, last_error: None, }; let (dir, reader) = write_and_open_with_secondary_eq_sidecar(&mt, &entry); @@ -6050,9 +10673,9 @@ pub(crate) mod tests { std::fs::write(&corrupt_path, [1u8, 2, 3]).unwrap(); std::fs::rename(&corrupt_path, &sidecar_path).unwrap(); - assert!(reader + assert!(!reader .validate_secondary_eq_sidecar(entry.index_id) - .is_err()); + .unwrap()); let repaired_path = seg_dir.join("secondary_indexes").join(".repaired_eq.dat"); let mut repaired_groups = BTreeMap::new(); @@ -6061,12 +10684,10 @@ pub(crate) mod tests { .unwrap(); std::fs::rename(&repaired_path, &sidecar_path).unwrap(); - assert_eq!( - reader - .find_nodes_by_secondary_eq_index(entry.index_id, red_hash) - .unwrap(), - vec![1] - ); + assert!(reader + .find_nodes_by_secondary_eq_index(entry.index_id, red_hash) + .unwrap() + .is_empty()); } #[test] @@ -6079,7 +10700,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "apple".to_string(), props, created_at: 1000, @@ -6095,11 +10716,11 @@ pub(crate) mod tests { let entry = SecondaryIndexManifestEntry { index_id: 52, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, - state: SecondaryIndexState::Building, + state: SecondaryIndexState::Ready, last_error: None, }; let (dir, reader) = write_and_open_with_secondary_eq_sidecar(&mt, &entry); @@ -6128,6 +10749,154 @@ pub(crate) mod tests { ); } + #[test] + fn test_secondary_eq_lookup_does_not_full_validate_unqueried_group() { + use crate::types::hash_prop_value; + + let mt = Memtable::new(); + for (id, color) in [(1, "red"), (2, "red"), (3, "green"), (4, "green")] { + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String(color.to_string())); + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id, + label_ids: NodeLabelSet::single(1).unwrap(), + key: format!("node-{id}"), + props, + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + id, + ); + } + + let entry = SecondaryIndexManifestEntry { + index_id: 53, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let (dir, _) = write_and_open_with_secondary_eq_sidecar(&mt, &entry); + let seg_dir = dir.path().join("seg_0001"); + let kind = SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }; + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let green_hash = hash_prop_value(&PropValue::String("green".to_string())); + + rewrite_component_payload_for_test(&seg_dir, kind.clone(), |payload| { + let (offset, id_count) = secondary_eq_group_range(payload, green_hash) + .unwrap() + .expect("green group must exist"); + assert!(id_count >= 2); + let first = read_u64_at(payload, offset).unwrap(); + let second = read_u64_at(payload, offset + 8).unwrap(); + write_u64_at_for_test(payload, offset, second); + write_u64_at_for_test(payload, offset + 8, first); + }); + + let payload = component_payload_bytes_for_test(&seg_dir, kind); + assert!(validate_secondary_eq_sidecar_data(&payload).is_err()); + + let reader = reopen_test_segment_with_index(&seg_dir, &entry); + let mut reds = reader + .find_nodes_by_secondary_eq_index(entry.index_id, red_hash) + .unwrap(); + reds.sort_unstable(); + assert_eq!(reds, vec![1, 2]); + assert_eq!( + reader.declared_index_runtime_coverage_state( + entry.index_id, + PlannerStatsDeclaredIndexKind::Equality + ), + DeclaredIndexRuntimeCoverageState::Available + ); + } + + #[test] + fn test_secondary_eq_lookup_latches_selected_group_malformed() { + use crate::types::hash_prop_value; + + let mt = Memtable::new(); + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), + key: "apple".to_string(), + props, + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 0, + ); + + let entry = SecondaryIndexManifestEntry { + index_id: 54, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let (dir, _) = write_and_open_with_secondary_eq_sidecar(&mt, &entry); + let seg_dir = dir.path().join("seg_0001"); + let kind = SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }; + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + + rewrite_component_payload_for_test(&seg_dir, kind.clone(), |payload| { + let (count, _) = secondary_eq_sidecar_index_bounds(payload).unwrap(); + for index in 0..count { + let entry_off = 8 + index * SECONDARY_EQ_ENTRY_SIZE; + if read_u64_at(payload, entry_off).unwrap() == red_hash { + write_u64_at_for_test(payload, entry_off + 8, payload.len() as u64 + 8); + return; + } + } + panic!("red group must exist"); + }); + + let reader = reopen_test_segment_with_index(&seg_dir, &entry); + let err = reader + .find_nodes_by_secondary_eq_index_if_present(entry.index_id, red_hash) + .unwrap_err(); + assert!(err.to_string().contains("exceeds file length")); + assert_eq!( + reader.declared_index_runtime_coverage_state( + entry.index_id, + PlannerStatsDeclaredIndexKind::Equality + ), + DeclaredIndexRuntimeCoverageState::Corrupt + ); + assert!(matches!( + reader.component_registry.availability(&kind), + ComponentAvailability::CorruptIdentity { .. } + )); + assert_eq!( + reader + .find_nodes_by_secondary_eq_index_if_present(entry.index_id, red_hash) + .unwrap(), + None + ); + } + #[test] fn test_validate_secondary_eq_sidecar_rejects_unsorted_node_ids() { let mut data = Vec::new(); @@ -6264,7 +11033,7 @@ pub(crate) mod tests { mt.apply_op( &WalOp::UpsertNode(NodeRecord { id: 1, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: "apple".to_string(), props, created_at: 1000, @@ -6280,13 +11049,13 @@ pub(crate) mod tests { let entry = SecondaryIndexManifestEntry { index_id: 61, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Range { domain: SecondaryIndexRangeDomain::Int, }, - state: SecondaryIndexState::Building, + state: SecondaryIndexState::Ready, last_error: None, }; let (dir, reader) = write_and_open_with_secondary_range_sidecar(&mt, &entry); @@ -6299,9 +11068,9 @@ pub(crate) mod tests { std::fs::write(&corrupt_path, [1u8, 2, 3]).unwrap(); std::fs::rename(&corrupt_path, &sidecar_path).unwrap(); - assert!(reader + assert!(!reader .validate_secondary_range_sidecar(entry.index_id) - .is_err()); + .unwrap()); let repaired_path = seg_dir .join("secondary_indexes") @@ -6322,7 +11091,7 @@ pub(crate) mod tests { None, ) .unwrap(), - Some(vec![(10u64 ^ (1u64 << 63), 1)]) + None ); } @@ -6361,12 +11130,164 @@ pub(crate) mod tests { data.extend_from_slice(&(10u64 ^ (1u64 << 63)).to_le_bytes()); data.extend_from_slice(&1u64.to_le_bytes()); - match validate_secondary_range_sidecar_data(&data) { - Err(EngineError::CorruptRecord(message)) => { - assert!(message.contains("not strictly increasing")); - } - other => panic!("expected corrupt secondary range sidecar, got {:?}", other), - } + match validate_secondary_range_sidecar_data(&data) { + Err(EngineError::CorruptRecord(message)) => { + assert!(message.contains("not strictly increasing")); + } + other => panic!("expected corrupt secondary range sidecar, got {:?}", other), + } + } + + #[test] + fn test_secondary_range_lookup_does_not_full_validate_unqueried_entry() { + let mt = Memtable::new(); + for (id, score) in [(1, 10), (2, 20), (3, 30)] { + let mut props = BTreeMap::new(); + props.insert("score".to_string(), PropValue::Int(score)); + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id, + label_ids: NodeLabelSet::single(1).unwrap(), + key: format!("node-{id}"), + props, + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + id, + ); + } + + let entry = SecondaryIndexManifestEntry { + index_id: 62, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let (dir, _) = write_and_open_with_secondary_range_sidecar(&mt, &entry); + let seg_dir = dir.path().join("seg_0001"); + let kind = SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }; + let encoded_5 = 5u64 ^ (1u64 << 63); + let encoded_10 = 10u64 ^ (1u64 << 63); + + rewrite_component_payload_for_test(&seg_dir, kind.clone(), |payload| { + let entry_off = 8 + 2 * SECONDARY_RANGE_ENTRY_SIZE; + write_u64_at_for_test(payload, entry_off, encoded_5); + }); + + let payload = component_payload_bytes_for_test(&seg_dir, kind); + assert!(validate_secondary_range_sidecar_data(&payload).is_err()); + + let reader = reopen_test_segment_with_index(&seg_dir, &entry); + assert_eq!( + reader + .find_nodes_by_secondary_range_index_if_present( + entry.index_id, + Some((encoded_10, true)), + Some((encoded_10, true)), + None, + ) + .unwrap(), + Some(vec![(encoded_10, 1)]) + ); + assert_eq!( + reader.declared_index_runtime_coverage_state( + entry.index_id, + PlannerStatsDeclaredIndexKind::Range + ), + DeclaredIndexRuntimeCoverageState::Available + ); + } + + #[test] + fn test_secondary_range_lookup_latches_header_malformed() { + let mt = Memtable::new(); + let mut props = BTreeMap::new(); + props.insert("score".to_string(), PropValue::Int(10)); + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), + key: "apple".to_string(), + props, + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 0, + ); + + let entry = SecondaryIndexManifestEntry { + index_id: 63, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let (dir, _) = write_and_open_with_secondary_range_sidecar(&mt, &entry); + let seg_dir = dir.path().join("seg_0001"); + let kind = SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }; + let encoded_10 = 10u64 ^ (1u64 << 63); + + rewrite_component_payload_for_test(&seg_dir, kind.clone(), |payload| { + write_u64_at_for_test(payload, 0, 2); + }); + + let reader = reopen_test_segment_with_index(&seg_dir, &entry); + let err = reader + .find_nodes_by_secondary_range_index_if_present( + entry.index_id, + Some((encoded_10, true)), + Some((encoded_10, true)), + None, + ) + .unwrap_err(); + assert!(err + .to_string() + .contains("does not match expected fixed-width length")); + assert_eq!( + reader.declared_index_runtime_coverage_state( + entry.index_id, + PlannerStatsDeclaredIndexKind::Range + ), + DeclaredIndexRuntimeCoverageState::Corrupt + ); + assert!(matches!( + reader.component_registry.availability(&kind), + ComponentAvailability::CorruptIdentity { .. } + )); + assert_eq!( + reader + .find_nodes_by_secondary_range_index_if_present( + entry.index_id, + Some((encoded_10, true)), + Some((encoded_10, true)), + None, + ) + .unwrap(), + None + ); } // --- Weight preservation in adjacency postings --- @@ -6382,7 +11303,7 @@ pub(crate) mod tests { id: 10, from: 1, to: 2, - type_id: 5, + label_id: 5, props: BTreeMap::new(), created_at: 100, updated_at: 100, @@ -6398,7 +11319,7 @@ pub(crate) mod tests { id: 11, from: 1, to: 3, - type_id: 5, + label_id: 5, props: BTreeMap::new(), created_at: 100, updated_at: 100, @@ -6461,7 +11382,7 @@ pub(crate) mod tests { assert_eq!(e.id, 102); // Non-existent triples - assert!(reader.edge_by_triple(1, 2, 20).unwrap().is_none()); // wrong type + assert!(reader.edge_by_triple(1, 2, 20).unwrap().is_none()); // wrong label assert!(reader.edge_by_triple(2, 1, 10).unwrap().is_none()); // reversed direction assert!(reader.edge_by_triple(3, 1, 10).unwrap().is_none()); // no such edge } @@ -6486,51 +11407,479 @@ pub(crate) mod tests { assert!(reader.edge_by_triple(1, 2, 10).unwrap().is_none()); } - // --- Bounds checking regression tests --- + #[test] + fn test_edge_triple_index_returns_parallel_edges() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertEdge(make_edge(100, 1, 2, 10)), 1); + mt.apply_op(&WalOp::UpsertEdge(make_edge(101, 1, 2, 10)), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(102, 1, 3, 10)), 3); + + let (_dir, reader) = write_and_open(&mt); + + assert_eq!(reader.edge_ids_by_triple(1, 2, 10).unwrap(), vec![100, 101]); + assert_eq!(reader.edge_ids_by_triple(1, 3, 10).unwrap(), vec![102]); + } #[test] - fn test_truncated_nodes_dat_returns_error() { - // A nodes.dat with only 4 bytes (truncated count header) should - // still open fine (< 8 bytes → count = 0), but a file that claims - // many records while being truncated should return CorruptRecord. - let dir = tempfile::tempdir().unwrap(); + fn test_edge_metadata_index_ranges_and_weight_zero_nan() { + let mt = Memtable::new(); + let mut neg_zero = make_edge(10, 1, 2, 5); + neg_zero.weight = -0.0; + neg_zero.updated_at = 100; + neg_zero.valid_from = 0; + neg_zero.valid_to = 100; + let mut pos_zero = make_edge(11, 1, 3, 5); + pos_zero.weight = 0.0; + pos_zero.updated_at = 150; + pos_zero.valid_from = 10; + pos_zero.valid_to = 200; + let mut positive = make_edge(12, 1, 4, 5); + positive.weight = 2.0; + positive.updated_at = 250; + positive.valid_from = 20; + positive.valid_to = 300; + let mut other_label = make_edge(13, 1, 5, 6); + other_label.weight = 0.0; + other_label.updated_at = 175; + let mut nan = make_edge(14, 1, 6, 5); + nan.weight = f32::NAN; + + for edge in [neg_zero, pos_zero, positive, other_label, nan] { + mt.apply_op(&WalOp::UpsertEdge(edge), 1); + } + + let (_dir, reader) = write_and_open(&mt); + assert!(reader.edge_weight_index_available()); + assert!(reader.edge_updated_at_index_available()); + assert!(reader.edge_valid_from_index_available()); + assert!(reader.edge_valid_to_index_available()); + + let zero_bounds = RangeBoundFlags::inclusive(Some(0.0), Some(0.0)); + assert_eq!( + reader.edge_ids_by_weight_range(Some(5), zero_bounds), + Some(vec![10, 11]) + ); + assert_eq!( + reader.edge_weight_range_count(Some(5), zero_bounds), + Some(2) + ); + assert_eq!( + reader.edge_ids_by_weight_range(None, zero_bounds), + Some(vec![10, 11, 13]) + ); + assert_eq!(reader.edge_weight_range_count(None, zero_bounds), Some(3)); + assert!(!reader + .edge_ids_by_weight_range(None, RangeBoundFlags::inclusive(None, Some(f32::INFINITY))) + .unwrap() + .contains(&14)); + assert_eq!( + reader.edge_weight_range_count( + None, + RangeBoundFlags::inclusive(None, Some(f32::INFINITY)) + ), + Some(4) + ); + + assert_eq!( + reader.edge_ids_by_updated_at_range( + Some(5), + RangeBoundFlags::inclusive(Some(100), Some(200)) + ), + Some(vec![10, 11]) + ); + assert_eq!( + reader.edge_updated_at_range_count( + Some(5), + RangeBoundFlags::inclusive(Some(100), Some(200)) + ), + Some(2) + ); + assert_eq!( + reader.edge_ids_by_valid_to_range( + Some(5), + RangeBoundFlags { + lower: Some(100), + lower_inclusive: false, + upper: None, + upper_inclusive: true, + } + ), + Some(vec![11, 12, 14]) + ); + assert_eq!( + reader.edge_valid_to_range_count( + Some(5), + RangeBoundFlags { + lower: Some(100), + lower_inclusive: false, + upper: None, + upper_inclusive: true, + } + ), + Some(3) + ); + assert_eq!( + reader + .edge_metadata_scan_ids(|meta| meta.valid_from <= 100) + .unwrap(), + vec![10, 11, 12, 13, 14] + ); + + let mut metadata = vec![None; 3]; + reader + .get_edge_metadata_batch(&[(1, 10), (0, 12), (2, 999)], &mut metadata) + .unwrap(); + assert_eq!(metadata[0].unwrap().edge_id, 12); + assert_eq!(metadata[0].unwrap().weight, 2.0); + assert_eq!(metadata[1].unwrap().edge_id, 10); + assert_eq!(metadata[2], None); + } + + #[test] + fn test_packed_core_components_share_one_mapping_and_logical_slices() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 2, "b")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 3); + let (dir, reader) = write_and_open(&mt); let seg_dir = dir.path().join("seg_0001"); - std::fs::create_dir_all(&seg_dir).unwrap(); - write_format_ver(&seg_dir); - // Write a nodes.dat that says "1 node" but has no index/data - let mut data = Vec::new(); - data.extend_from_slice(&1u64.to_le_bytes()); // count = 1 - // No index entry follows (truncated) - std::fs::write(seg_dir.join("nodes.dat"), &data).unwrap(); - - // Write empty files for the other required segment files - for name in &[ - "edges.dat", - "adj_out.idx", - "adj_out.dat", - "adj_in.idx", - "adj_in.dat", - "key_index.dat", - "tombstones.dat", - "node_meta.dat", - "edge_meta.dat", - "timestamp_index.dat", + let node_mapping = reader + .component_mapping_identity_for_test(SegmentComponentKind::NodeRecords) + .unwrap(); + for kind in [ + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::NodeMetadata, + SegmentComponentKind::EdgeMetadata, + SegmentComponentKind::KeyIndex, + SegmentComponentKind::NodeLabelIndex, + SegmentComponentKind::AdjOutPostings, + SegmentComponentKind::AdjOutIndex, + SegmentComponentKind::EdgeWeightIndex, ] { - std::fs::write(seg_dir.join(name), []).unwrap(); + assert_eq!( + reader.component_mapping_identity_for_test(kind).unwrap(), + node_mapping + ); } - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); - // get_node triggers binary search which reads an index entry past EOF - let result = reader.get_node(42); + assert_eq!( + reader.raw_nodes_mmap(), + component_payload_bytes_for_test(&seg_dir, SegmentComponentKind::NodeRecords) + ); + assert_eq!( + &reader.adj_out_dat[..], + component_payload_bytes_for_test(&seg_dir, SegmentComponentKind::AdjOutPostings) + ); + assert_eq!( + &reader.adj_out_idx[..], + component_payload_bytes_for_test(&seg_dir, SegmentComponentKind::AdjOutIndex) + ); + } + + #[test] + fn test_open_rejects_missing_packed_core_container_file() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + let (_dir, seg_dir, _info) = write_segment_with_info(&mt, None); + std::fs::remove_file(seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME)) + .unwrap(); + + let err = expect_engine_error(SegmentReader::open_unpinned_for_test(&seg_dir, 1, None)); assert!( - result.is_err(), - "truncated segment should return error, not panic" + err.contains("No such file") || err.contains("segment.core"), + "got: {err}" + ); + } + + #[test] + fn test_open_rejects_external_packed_core_component_handles() { + fn rewrite_handle_to_external( + seg_dir: &std::path::Path, + kind: SegmentComponentKind, + ) -> SegmentInfo { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 3); + let info = write_segment(seg_dir, 1, &mt, None).unwrap(); + let mut manifest = read_segment_manifest_for_test(seg_dir); + let record = manifest + .components + .iter_mut() + .find(|record| record.kind == kind) + .expect("test component must exist"); + record.handle = ComponentHandleV1::ExternalFile { + relative_path: "external-packed-core-component.dat".to_string(), + payload_offset: crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: record.payload_len, + }; + write_segment_manifest_for_test(seg_dir, &manifest); + info + } + + for kind in [ + SegmentComponentKind::NodeRecords, + SegmentComponentKind::EdgeWeightIndex, + ] { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join(format!("seg_{:04}", kind.kind_tag())); + let info = rewrite_handle_to_external(&seg_dir, kind.clone()); + + let err = + expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("must use a PackedRange handle"), "got: {err}"); + assert!(err.contains(&format!("{:?}", kind)), "got: {err}"); + } + } + + #[test] + fn test_open_rejects_required_packed_range_wrong_container_id() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + let record = manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::NodeRecords) + .unwrap(); + let ComponentHandleV1::PackedRange { + container_component_id, + .. + } = &mut record.handle + else { + panic!("NodeRecords should be packed"); + }; + *container_component_id = [42; 32]; + write_segment_manifest_for_test(&seg_dir, &manifest); + + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("wrong container"), "got: {err}"); + } + + #[test] + fn test_open_rejects_required_packed_range_overflow_and_overlap() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 1, 5)), 2); + + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + let record = manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::KeyIndex) + .unwrap(); + let ComponentHandleV1::PackedRange { offset, .. } = &mut record.handle else { + panic!("KeyIndex should be packed"); + }; + *offset = u64::MAX - 1; + write_segment_manifest_for_test(&seg_dir, &manifest); + + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("range overflows"), "got: {err}"); + + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + let (node_offset, _) = packed_range_for_test(&manifest, SegmentComponentKind::NodeRecords); + let record = manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::EdgeRecords) + .unwrap(); + let ComponentHandleV1::PackedRange { offset, .. } = &mut record.handle else { + panic!("EdgeRecords should be packed"); + }; + *offset = node_offset; + write_segment_manifest_for_test(&seg_dir, &manifest); + + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("overlap"), "got: {err}"); + } + + #[test] + fn test_optional_packed_edge_metadata_bad_range_falls_back() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 3); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + let (node_offset, _) = packed_range_for_test(&manifest, SegmentComponentKind::NodeRecords); + let record = manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::EdgeWeightIndex) + .unwrap(); + let ComponentHandleV1::PackedRange { offset, .. } = &mut record.handle else { + panic!("EdgeWeightIndex should be packed"); + }; + *offset = node_offset; + write_segment_manifest_for_test(&seg_dir, &manifest); + + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &[]).unwrap(); + assert!(reader.get_edge(10).unwrap().is_some()); + assert!(!reader.edge_weight_index_available()); + assert!(matches!( + reader.optional_component_availability_for_test(SegmentComponentKind::EdgeWeightIndex), + ComponentAvailability::CorruptIdentity { .. } + )); + assert_eq!( + reader.edge_ids_by_weight_range( + Some(5), + RangeBoundFlags::inclusive(Some(0.0), Some(2.0)) + ), + None + ); + } + + #[test] + fn test_missing_and_corrupt_edge_metadata_indexes_are_unavailable() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 1); + let (dir, reader) = write_and_open(&mt); + assert!(reader.edge_weight_index_available()); + let seg_dir = dir.path().join("seg_0001"); + + let mut manifest = read_segment_manifest_for_test(&seg_dir); + manifest.components.retain(|record| { + !matches!( + record.kind, + SegmentComponentKind::EdgeWeightIndex + | SegmentComponentKind::EdgeUpdatedAtIndex + | SegmentComponentKind::EdgeValidFromIndex + | SegmentComponentKind::EdgeValidToIndex + ) + }); + write_segment_manifest_for_test(&seg_dir, &manifest); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.edge_weight_index_available()); + assert!(reader.get_edge(10).unwrap().is_some()); + assert_eq!( + reader.edge_ids_by_weight_range( + Some(5), + RangeBoundFlags::inclusive(Some(0.0), Some(2.0)) + ), + None + ); + + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + write_segment(&seg_dir, 1, &mt, None).unwrap(); + rewrite_component_payload_for_test( + &seg_dir, + SegmentComponentKind::EdgeWeightIndex, + |payload| { + payload[0..8].copy_from_slice(&2u64.to_le_bytes()); + }, + ); + rewrite_component_payload_for_test( + &seg_dir, + SegmentComponentKind::EdgeUpdatedAtIndex, + |payload| { + payload[0..8].copy_from_slice(&2u64.to_le_bytes()); + }, + ); + + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.edge_weight_index_available()); + assert!(!reader.edge_updated_at_index_available()); + assert!(reader.get_edge(10).unwrap().is_some()); + } + + #[test] + fn test_open_does_not_full_scan_edge_metadata_index_sortedness() { + let mt = Memtable::new(); + let mut first = make_edge(10, 1, 2, 5); + first.weight = 1.0; + let mut second = make_edge(11, 1, 3, 5); + second.weight = 2.0; + mt.apply_op(&WalOp::UpsertEdge(first), 1); + mt.apply_op(&WalOp::UpsertEdge(second), 2); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + + rewrite_component_payload_for_test( + &seg_dir, + SegmentComponentKind::EdgeWeightIndex, + |payload| { + assert_eq!(payload.len(), 8 + 2 * EDGE_WEIGHT_INDEX_ENTRY_SIZE); + let first_start = 8; + let second_start = first_start + EDGE_WEIGHT_INDEX_ENTRY_SIZE; + for offset in 0..EDGE_WEIGHT_INDEX_ENTRY_SIZE { + payload.swap(first_start + offset, second_start + offset); + } + assert!(validate_edge_weight_index_data(payload).is_err()); + }, + ); + rewrite_component_payload_for_test( + &seg_dir, + SegmentComponentKind::EdgeUpdatedAtIndex, + |payload| { + assert_eq!(payload.len(), 8 + 2 * EDGE_I64_METADATA_INDEX_ENTRY_SIZE); + let first_start = 8; + let second_start = first_start + EDGE_I64_METADATA_INDEX_ENTRY_SIZE; + for offset in 0..EDGE_I64_METADATA_INDEX_ENTRY_SIZE { + payload.swap(first_start + offset, second_start + offset); + } + assert!(validate_edge_i64_metadata_index_data( + payload, + EDGE_UPDATED_AT_INDEX_LOGICAL_NAME + ) + .is_err()); + }, + ); + + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &[]).unwrap(); + assert!(reader.edge_weight_index_available()); + assert!(reader.edge_updated_at_index_available()); + assert_eq!( + reader.optional_component_availability_for_test(SegmentComponentKind::EdgeWeightIndex), + ComponentAvailability::Available + ); + assert_eq!( + reader + .optional_component_availability_for_test(SegmentComponentKind::EdgeUpdatedAtIndex), + ComponentAvailability::Available ); - let err_msg = result.unwrap_err().to_string(); + } + + #[test] + fn test_validate_node_vector_sidecars_rejects_metadata_length_overflow() { + let mut vector_meta = Vec::new(); + vector_meta.extend_from_slice(&u64::MAX.to_le_bytes()); + + match validate_node_vector_sidecars(1, &vector_meta, &[], &[], u64::MAX) { + Err(EngineError::CorruptRecord(message)) => { + assert!(message.contains("overflow") || message.contains("addressable")); + } + Err(other) => panic!( + "expected node vector metadata overflow corruption, got {}", + other + ), + Ok(_) => panic!("expected node vector metadata overflow error"), + } + } + + // --- Bounds checking regression tests --- + + #[test] + fn test_truncated_packed_core_returns_error() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 0); + write_segment(&seg_dir, 1, &mt, None).unwrap(); + + let core_path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); + let mut data = std::fs::read(&core_path).unwrap(); + data.pop(); + std::fs::write(&core_path, data).unwrap(); + + let err_msg = expect_engine_error(SegmentReader::open_unpinned_for_test(&seg_dir, 1, None)); assert!( - err_msg.contains("exceeds data length"), - "error should describe bounds issue: {}", + err_msg.contains("does not match file length"), + "error should describe identity length issue: {}", err_msg ); } @@ -6539,44 +11888,33 @@ pub(crate) mod tests { fn test_truncated_tombstones_returns_error() { let dir = tempfile::tempdir().unwrap(); let seg_dir = dir.path().join("seg_0001"); - std::fs::create_dir_all(&seg_dir).unwrap(); - write_format_ver(&seg_dir); - - // Write empty required files - for name in &[ - "nodes.dat", - "edges.dat", - "adj_out.idx", - "adj_out.dat", - "adj_in.idx", - "adj_in.dat", - "key_index.dat", - "node_meta.dat", - "edge_meta.dat", - "timestamp_index.dat", - ] { - std::fs::write(seg_dir.join(name), []).unwrap(); - } + let mt = Memtable::new(); + mt.apply_op( + &WalOp::DeleteNode { + id: 99, + deleted_at: 1234, + }, + 1, + ); + write_segment(&seg_dir, 1, &mt, None).unwrap(); - // Write a tombstones.dat that claims 5 entries but only has the header - let mut data = Vec::new(); - data.extend_from_slice(&5u64.to_le_bytes()); // count = 5 - // No actual tombstone entries (truncated) - std::fs::write(seg_dir.join("tombstones.dat"), &data).unwrap(); + rewrite_component_payload_for_test(&seg_dir, SegmentComponentKind::Tombstones, |payload| { + payload[0..8].copy_from_slice(&5u64.to_le_bytes()); + }); - let result = SegmentReader::open(&seg_dir, 1, None); + let result = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None); assert!( result.is_err(), - "truncated tombstones should return error, not panic" + "truncated packed tombstones should return error, not panic" ); } #[test] fn test_decode_node_at_truncated_returns_error() { // Minimal data that starts a valid node but is truncated mid-record - // Format v4: no id in data, starts with type_id + // Format v4: no id in data, starts with label_id let mut data = Vec::new(); - data.extend_from_slice(&1u32.to_le_bytes()); // type_id + data.extend_from_slice(&1u32.to_le_bytes()); // label_id // Missing key_len and beyond (truncated) let result = decode_node_at(&data, 0, 42); assert!(result.is_err()); @@ -6587,101 +11925,250 @@ pub(crate) mod tests { // Partial edge record (format v4: no id in data, starts with from) let mut data = Vec::new(); data.extend_from_slice(&1u64.to_le_bytes()); // from - // Missing to, type_id, timestamps, etc. + // Missing to, label_id, timestamps, etc. let result = decode_edge_at(&data, 0, 100); assert!(result.is_err()); } #[test] - fn test_format_version_bad_magic_rejected() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_test"); - std::fs::create_dir_all(&seg_dir).unwrap(); - - let mut bad = Vec::new(); - bad.extend_from_slice(b"BAAD"); - bad.extend_from_slice(&1u32.to_le_bytes()); - std::fs::write(seg_dir.join("format.ver"), &bad).unwrap(); + fn test_decode_edge_property_at_projects_requested_property() { + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + props.insert("score".to_string(), PropValue::Int(10)); + let props_bytes = rmp_serde::to_vec(&props).unwrap(); + + let mut data = Vec::new(); + data.extend_from_slice(&1u64.to_le_bytes()); + data.extend_from_slice(&2u64.to_le_bytes()); + data.extend_from_slice(&7u32.to_le_bytes()); + data.extend_from_slice(&1000i64.to_le_bytes()); + data.extend_from_slice(&1001i64.to_le_bytes()); + data.extend_from_slice(&1.0f32.to_le_bytes()); + data.extend_from_slice(&0i64.to_le_bytes()); + data.extend_from_slice(&i64::MAX.to_le_bytes()); + data.extend_from_slice(&(props_bytes.len() as u32).to_le_bytes()); + data.extend_from_slice(&props_bytes); + + assert_eq!( + decode_edge_property_at(&data, 0, 100, "color").unwrap(), + Some(PropValue::String("red".to_string())) + ); + assert_eq!( + decode_edge_property_at(&data, 0, 100, "score").unwrap(), + Some(PropValue::Int(10)) + ); + assert_eq!( + decode_edge_property_at(&data, 0, 100, "missing").unwrap(), + None + ); + + let mut truncated = data; + truncated.pop(); + assert!(decode_edge_property_at(&truncated, 0, 100, "color").is_err()); + } + + #[test] + fn test_open_with_info_rejects_root_local_segment_id_mismatch() { + let mt = Memtable::new(); + let (_dir, seg_dir, mut info) = write_segment_with_info(&mt, None); + info.id = 2; + + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("does not match root segment"), "got: {err}"); + } + + #[test] + fn test_open_with_info_rejects_root_local_count_mismatch() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 0); + let (_dir, seg_dir, mut info) = write_segment_with_info(&mt, None); + info.node_count += 1; + + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("does not match root node_count"), "got: {err}"); + } + + #[test] + fn test_open_with_info_rejects_root_local_segment_data_id_mismatch() { + let mt = Memtable::new(); + let (_dir, seg_dir, mut info) = write_segment_with_info(&mt, None); + info.segment_data_id = [7; 32]; + + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!( + err.contains("segment_data_id does not match root"), + "got: {err}" + ); + } + + #[test] + fn test_open_with_info_rejects_segment_data_id_recompute_mismatch() { + let mt = Memtable::new(); + let (_dir, seg_dir, _info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + manifest.segment_data_id = [9; 32]; + write_segment_manifest_for_test(&seg_dir, &manifest); + let info = segment_info_from_manifest(&manifest); + + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!( + err.contains("segment_data_id does not match component source groups"), + "got: {err}" + ); + } + + #[test] + fn test_open_rejects_missing_v10_segment_manifest() { + let mt = Memtable::new(); + let (_dir, seg_dir, _info) = write_segment_with_info(&mt, None); + std::fs::remove_file(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + + let err = expect_engine_error(SegmentReader::open_unpinned_for_test(&seg_dir, 1, None)); + assert!(err.contains("missing segment_manifest.dat"), "got: {err}"); + } + + #[test] + fn test_open_rejects_missing_required_manifest_record() { + let mt = Memtable::new(); + let (_dir, seg_dir, _info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + manifest + .components + .retain(|record| record.kind != SegmentComponentKind::NodeRecords); + let info = segment_info_from_manifest(&manifest); + write_segment_manifest_for_test(&seg_dir, &manifest); - let err = read_format_version(&seg_dir).unwrap_err(); - let msg = err.to_string(); - assert!(msg.contains("invalid magic"), "got: {}", msg); + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("NodeRecords"), "got: {err}"); } #[test] - fn test_format_version_bad_size_rejected() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_test"); - std::fs::create_dir_all(&seg_dir).unwrap(); - - std::fs::write(seg_dir.join("format.ver"), b"short").unwrap(); + fn test_open_rejects_required_component_dependency_mismatch() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 0); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + let record = manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::KeyIndex) + .unwrap(); + record.dependencies.clear(); + record.dependency_digest = crate::segment_components::dependency_digest(&[]); + write_segment_manifest_for_test(&seg_dir, &manifest); - let err = read_format_version(&seg_dir).unwrap_err(); - let msg = err.to_string(); - assert!(msg.contains("invalid size"), "got: {}", msg); + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("dependency digest"), "got: {err}"); } #[test] - fn test_format_version_future_version_rejected() { - use crate::segment_writer::{SEGMENT_FORMAT_VERSION, SEGMENT_MAGIC}; + fn test_open_rejects_required_component_build_mismatch() { + let mt = Memtable::new(); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::NodeRecords) + .unwrap() + .build_fingerprint ^= 1; + write_segment_manifest_for_test(&seg_dir, &manifest); - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_test"); - std::fs::create_dir_all(&seg_dir).unwrap(); + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("build fingerprint"), "got: {err}"); + } - let mut data = Vec::new(); - data.extend_from_slice(&SEGMENT_MAGIC); - data.extend_from_slice(&(SEGMENT_FORMAT_VERSION + 1).to_le_bytes()); - std::fs::write(seg_dir.join("format.ver"), &data).unwrap(); + #[test] + fn test_open_rejects_required_component_id_recompute_mismatch() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + let bogus_id = [17u8; 32]; + manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::KeyIndex) + .unwrap() + .component_id = bogus_id; + write_segment_manifest_for_test(&seg_dir, &manifest); - let err = read_format_version(&seg_dir).unwrap_err(); - let msg = err.to_string(); - assert!(msg.contains("newer than supported"), "got: {}", msg); + let err = expect_engine_error(SegmentReader::open_with_info(&seg_dir, &info, None, &[])); + assert!(err.contains("component KeyIndex id"), "got: {err}"); } #[test] - fn test_format_version_absent_is_rejected() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_test"); - std::fs::create_dir_all(&seg_dir).unwrap(); - // No format.ver file. Pre-version segment, too old (v0) - let err = read_format_version(&seg_dir).unwrap_err(); - assert!(err.to_string().contains("too old"), "got: {}", err); + fn test_open_disables_only_optional_component_on_build_fingerprint_mismatch() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 3); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::EdgeWeightIndex) + .unwrap() + .build_fingerprint ^= 1; + write_segment_manifest_for_test(&seg_dir, &manifest); + + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &[]).unwrap(); + assert!(reader.get_node(1).unwrap().is_some()); + assert!(reader.get_edge(10).unwrap().is_some()); + assert!(matches!( + reader.optional_component_availability_for_test(SegmentComponentKind::EdgeWeightIndex), + ComponentAvailability::Incompatible { .. } + )); + assert_eq!( + reader + .optional_component_availability_for_test(SegmentComponentKind::EdgeUpdatedAtIndex), + ComponentAvailability::Available + ); } #[test] - fn test_format_version_v4_rejected() { - use crate::segment_writer::SEGMENT_MAGIC; - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_test"); - std::fs::create_dir_all(&seg_dir).unwrap(); - - let mut data = Vec::new(); - data.extend_from_slice(&SEGMENT_MAGIC); - data.extend_from_slice(&4u32.to_le_bytes()); - std::fs::write(seg_dir.join("format.ver"), &data).unwrap(); + fn test_open_disables_only_optional_component_on_component_id_mismatch() { + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 3); + let (_dir, seg_dir, info) = write_segment_with_info(&mt, None); + let mut manifest = read_segment_manifest_for_test(&seg_dir); + let bogus_id = [23u8; 32]; + manifest + .components + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::EdgeWeightIndex) + .unwrap() + .component_id = bogus_id; + write_segment_manifest_for_test(&seg_dir, &manifest); - let err = read_format_version(&seg_dir).unwrap_err(); - assert!(err.to_string().contains("too old"), "got: {}", err); + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &[]).unwrap(); + assert!(reader.get_node(1).unwrap().is_some()); + assert!(reader.get_edge(10).unwrap().is_some()); + assert!(matches!( + reader.optional_component_availability_for_test(SegmentComponentKind::EdgeWeightIndex), + ComponentAvailability::Incompatible { .. } + )); + assert_eq!( + reader + .optional_component_availability_for_test(SegmentComponentKind::EdgeUpdatedAtIndex), + ComponentAvailability::Available + ); } #[test] - fn test_open_rejects_orphan_vector_blob() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_0001"); + fn test_open_rejects_required_component_identity_header_mismatch() { let mt = Memtable::new(); - mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "plain")), 0); - write_segment(&seg_dir, 1, &mt, None).unwrap(); - - std::fs::write(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME), [0u8; 4]).unwrap(); + let (_dir, seg_dir, _info) = write_segment_with_info(&mt, None); + let core_path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); + let mut data = std::fs::read(&core_path).unwrap(); + data[120] ^= 1; + std::fs::write(&core_path, data).unwrap(); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); - assert!( - err.to_string() - .contains("vector blobs without node vector metadata"), - "got: {}", - err - ); + let err = expect_engine_error(SegmentReader::open_unpinned_for_test(&seg_dir, 1, None)); + assert!(err.contains("identity header"), "got: {err}"); } #[test] @@ -6699,11 +12186,15 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let mut meta = std::fs::read(seg_dir.join(NODE_VECTOR_META_FILENAME)).unwrap(); - meta[0..8].copy_from_slice(&2u64.to_le_bytes()); - std::fs::write(seg_dir.join(NODE_VECTOR_META_FILENAME), meta).unwrap(); + rewrite_component_payload_for_test( + &seg_dir, + SegmentComponentKind::NodeVectorMetadata, + |meta| { + meta[0..8].copy_from_slice(&2u64.to_le_bytes()); + }, + ); - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) + let err = SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)) .err() .unwrap(); assert!(err @@ -6722,14 +12213,20 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let mut dense_blob = std::fs::read(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME)).unwrap(); - dense_blob.extend_from_slice(&0.9f32.to_le_bytes()); - std::fs::write(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME), dense_blob).unwrap(); + let core_path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); + let mut core = std::fs::read(&core_path).unwrap(); + core.extend_from_slice(&0.9f32.to_le_bytes()); + std::fs::write(&core_path, core).unwrap(); - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) + let err = SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)) .err() .unwrap(); - assert!(err.to_string().contains("trailing or unreferenced bytes")); + let message = err.to_string(); + assert!( + message.contains("does not match file length") + || message.contains("exceeds blob length"), + "{message}" + ); } #[test] @@ -6749,7 +12246,8 @@ pub(crate) mod tests { write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, Some(&dense_config)).unwrap(); + let reader = + SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)).unwrap(); let header = reader.dense_hnsw_header().unwrap(); assert_eq!(header.point_count, 2); @@ -6758,11 +12256,11 @@ pub(crate) mod tests { assert_eq!(header.m, dense_config.hnsw.m); assert_eq!( reader.raw_dense_hnsw_meta_mmap(), - &std::fs::read(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap() + &read_payload_file(&seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)) ); assert_eq!( reader.raw_dense_hnsw_graph_mmap(), - &std::fs::read(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME)).unwrap() + &read_payload_file(&seg_dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME)) ); } @@ -6782,15 +12280,11 @@ pub(crate) mod tests { std::fs::remove_file(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap(); std::fs::remove_file(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME)).unwrap(); - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) - .err() - .unwrap(); - assert!( - err.to_string() - .contains("dense HNSW files are missing for 1 dense vectors"), - "got: {}", - err - ); + let reader = + SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)).unwrap(); + assert!(reader.dense_hnsw_header().is_none()); + assert!(reader.raw_dense_hnsw_meta_mmap().is_empty()); + assert!(reader.raw_dense_hnsw_graph_mmap().is_empty()); } #[test] @@ -6816,16 +12310,14 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let mut format_ver = std::fs::read(seg_dir.join("format.ver")).unwrap(); - format_ver[4..8].copy_from_slice(&6u32.to_le_bytes()); - std::fs::write(seg_dir.join("format.ver"), format_ver).unwrap(); + tamper_envelope_format_version(&seg_dir, 6); - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) + let err = SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)) .err() .unwrap(); assert!( err.to_string() - .contains("unexpected dense HNSW files for format version 6"), + .contains("unsupported segment manifest version"), "got: {}", err ); @@ -6842,23 +12334,25 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let mut meta = - std::fs::read(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap(); - meta[26] = 1; // Euclidean - std::fs::write( - seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), - meta, - ) - .unwrap(); - - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) - .err() - .unwrap(); - assert!( - err.to_string().contains("does not match configured metric"), - "got: {}", - err + rewrite_payload_file( + &seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), + |meta| { + meta[26] = 1; // Euclidean + }, ); + + let reader = + SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)).unwrap(); + assert!(reader.dense_hnsw_header().is_none()); + assert!(matches!( + reader + .optional_component_availability_for_test(SegmentComponentKind::DenseHnswMetadata), + ComponentAvailability::CorruptIdentity { .. } + )); + assert!(matches!( + reader.optional_component_availability_for_test(SegmentComponentKind::DenseHnswGraph), + ComponentAvailability::CorruptIdentity { .. } + )); } #[test] @@ -6877,13 +12371,9 @@ pub(crate) mod tests { std::fs::remove_file(seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME)) .unwrap(); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); - assert!( - err.to_string() - .contains("segment has sparse vectors but sparse posting files are missing"), - "got: {}", - err - ); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(reader.raw_sparse_posting_index_mmap().is_empty()); + assert!(reader.raw_sparse_postings_mmap().is_empty()); } #[test] @@ -6897,14 +12387,14 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, None).unwrap(); - let mut format_ver = std::fs::read(seg_dir.join("format.ver")).unwrap(); - format_ver[4..8].copy_from_slice(&7u32.to_le_bytes()); - std::fs::write(seg_dir.join("format.ver"), format_ver).unwrap(); + tamper_envelope_format_version(&seg_dir, 7); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); + let err = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None) + .err() + .unwrap(); assert!( err.to_string() - .contains("unexpected sparse posting files for format version 7"), + .contains("unsupported segment manifest version"), "got: {}", err ); @@ -6926,20 +12416,21 @@ pub(crate) mod tests { std::fs::remove_file(seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME)) .unwrap(); - let mut format_ver = std::fs::read(seg_dir.join("format.ver")).unwrap(); - format_ver[4..8].copy_from_slice(&7u32.to_le_bytes()); - std::fs::write(seg_dir.join("format.ver"), format_ver).unwrap(); + tamper_envelope_format_version(&seg_dir, 7); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); + let err = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None) + .err() + .unwrap(); assert!( - err.to_string().contains("predates sparse posting support"), + err.to_string() + .contains("unsupported segment manifest version"), "got: {}", err ); } #[test] - fn test_open_rejects_sparse_posting_parity_mismatch() { + fn test_sparse_posting_payload_semantic_mismatch_is_not_open_time_scrubbed() { let dir = tempfile::tempdir().unwrap(); let seg_dir = dir.path().join("seg_0001"); let mt = Memtable::new(); @@ -6949,66 +12440,72 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, None).unwrap(); - let mut postings = - std::fs::read(seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME)).unwrap(); - postings[8..12].copy_from_slice(&9.0f32.to_le_bytes()); - std::fs::write( - seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME), - postings, - ) - .unwrap(); + rewrite_payload_file( + &seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME), + |postings| { + postings[8..12].copy_from_slice(&9.0f32.to_le_bytes()); + }, + ); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); - assert!( - err.to_string() - .contains("does not match sparse vector payloads"), - "got: {}", - err + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.raw_sparse_posting_index_mmap().is_empty()); + assert!(!reader.raw_sparse_postings_mmap().is_empty()); + assert_eq!( + reader + .optional_component_availability_for_test(SegmentComponentKind::SparsePostingIndex), + ComponentAvailability::Available + ); + assert_eq!( + reader.optional_component_availability_for_test(SegmentComponentKind::SparsePostings), + ComponentAvailability::Available ); } #[test] - fn test_open_rejects_sparse_vector_blob_with_negative_weight() { + fn test_sparse_vector_source_payload_semantics_are_not_open_time_scrubbed() { let mut node = make_node(1, 1, "sparse-negative"); node.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); let (_dir, seg_dir) = write_sparse_segment(vec![node]); - let sparse_blob_path = - seg_dir.join(crate::segment_writer::NODE_SPARSE_VECTOR_BLOB_FILENAME); - let mut sparse_blob = std::fs::read(&sparse_blob_path).unwrap(); - sparse_blob[4..8].copy_from_slice(&(-1.5f32).to_le_bytes()); - std::fs::write(&sparse_blob_path, sparse_blob).unwrap(); - - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); - assert!( - err.to_string().contains("has negative weight"), - "got: {}", - err + rewrite_component_payload_for_test( + &seg_dir, + SegmentComponentKind::NodeSparseVectorBlob, + |sparse_blob| { + sparse_blob[4..8].copy_from_slice(&(-1.5f32).to_le_bytes()); + }, ); + + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.raw_node_sparse_vectors_mmap().is_empty()); } #[test] - fn test_open_rejects_sparse_postings_missing_expected_dimension() { + fn test_sparse_posting_dimension_mismatch_is_not_open_time_scrubbed() { let mut node = make_node(1, 1, "sparse-missing-dim"); node.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); let (_dir, seg_dir) = write_sparse_segment(vec![node]); let index_path = seg_dir.join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME); - let mut index = std::fs::read(&index_path).unwrap(); - index[24..28].copy_from_slice(&9u32.to_le_bytes()); - std::fs::write(&index_path, index).unwrap(); + rewrite_payload_file(&index_path, |index| { + index[24..28].copy_from_slice(&9u32.to_le_bytes()); + }); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); - assert!( - err.to_string() - .contains("sparse posting files are missing dimension 7 from sparse vectors"), - "got: {}", - err + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.raw_sparse_posting_index_mmap().is_empty()); + assert!(!reader.raw_sparse_postings_mmap().is_empty()); + assert_eq!( + reader + .optional_component_availability_for_test(SegmentComponentKind::SparsePostingIndex), + ComponentAvailability::Available + ); + assert_eq!( + reader.optional_component_availability_for_test(SegmentComponentKind::SparsePostings), + ComponentAvailability::Available ); } #[test] - fn test_open_rejects_sparse_posting_count_mismatch() { + fn test_sparse_posting_index_shape_mismatch_is_latched_on_first_use() { let mut first = make_node(1, 1, "sparse-count-a"); first.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); let mut second = make_node(2, 1, "sparse-count-b"); @@ -7016,27 +12513,34 @@ pub(crate) mod tests { let (_dir, seg_dir) = write_sparse_segment(vec![first, second]); let index_path = seg_dir.join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME); - let postings_path = seg_dir.join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME); - let mut index = std::fs::read(&index_path).unwrap(); - index[20..24].copy_from_slice(&1u32.to_le_bytes()); - index[28..36].copy_from_slice(&12u64.to_le_bytes()); - std::fs::write(&index_path, index).unwrap(); - - let postings = std::fs::read(&postings_path).unwrap(); - let mut rebuilt_postings = Vec::with_capacity(24); - rebuilt_postings.extend_from_slice(&postings[0..12]); - rebuilt_postings.extend_from_slice(&postings[24..36]); - std::fs::write(&postings_path, rebuilt_postings).unwrap(); + rewrite_payload_file(&index_path, |index| { + index[20..24].copy_from_slice(&1u32.to_le_bytes()); + index[28..36].copy_from_slice(&12u64.to_le_bytes()); + }); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); - assert!( - err.to_string().contains( - "sparse posting dimension 2 count 1 does not match sparse vector payload count 2" - ), - "got: {}", - err + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.raw_sparse_posting_index_mmap().is_empty()); + assert!(!reader.raw_sparse_postings_mmap().is_empty()); + assert!(reader.sparse_postings_available()); + assert_eq!( + reader + .optional_component_availability_for_test(SegmentComponentKind::SparsePostingIndex), + ComponentAvailability::Available ); + assert!(reader + .accumulate_sparse_posting_scores(&[(2, 1.0)], &mut NodeIdMap::default()) + .is_err()); + assert!(!reader.sparse_postings_available()); + assert!(matches!( + reader + .optional_component_availability_for_test(SegmentComponentKind::SparsePostingIndex), + ComponentAvailability::CorruptIdentity { .. } + )); + assert!(matches!( + reader.optional_component_availability_for_test(SegmentComponentKind::SparsePostings), + ComponentAvailability::CorruptIdentity { .. } + )); } #[test] @@ -7050,23 +12554,16 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let mut meta = - std::fs::read(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap(); - meta[22..24].copy_from_slice(&(dense_config.hnsw.m + 1).to_le_bytes()); - std::fs::write( - seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), - meta, - ) - .unwrap(); - - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) - .err() - .unwrap(); - assert!( - err.to_string().contains("does not match configured m"), - "got: {}", - err + rewrite_payload_file( + &seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), + |meta| { + meta[22..24].copy_from_slice(&(dense_config.hnsw.m + 1).to_le_bytes()); + }, ); + + let reader = + SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)).unwrap(); + assert!(reader.dense_hnsw_header().is_none()); } #[test] @@ -7080,24 +12577,16 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let mut meta = - std::fs::read(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap(); - meta[28..32].copy_from_slice(&3u32.to_le_bytes()); - std::fs::write( - seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), - meta, - ) - .unwrap(); - - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) - .err() - .unwrap(); - assert!( - err.to_string() - .contains("does not match configured dimension"), - "got: {}", - err + rewrite_payload_file( + &seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), + |meta| { + meta[28..32].copy_from_slice(&3u32.to_le_bytes()); + }, ); + + let reader = + SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)).unwrap(); + assert!(reader.dense_hnsw_header().is_none()); } #[test] @@ -7111,24 +12600,17 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let mut meta = - std::fs::read(seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME)).unwrap(); - meta[24..26].copy_from_slice(&(dense_config.hnsw.ef_construction + 1).to_le_bytes()); - std::fs::write( - seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), - meta, - ) - .unwrap(); - - let err = SegmentReader::open(&seg_dir, 1, Some(&dense_config)) - .err() - .unwrap(); - assert!( - err.to_string() - .contains("does not match configured ef_construction"), - "got: {}", - err + rewrite_payload_file( + &seg_dir.join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME), + |meta| { + meta[24..26] + .copy_from_slice(&(dense_config.hnsw.ef_construction + 1).to_le_bytes()); + }, ); + + let reader = + SegmentReader::open_unpinned_for_test(&seg_dir, 1, Some(&dense_config)).unwrap(); + assert!(reader.dense_hnsw_header().is_none()); } #[test] @@ -7142,19 +12624,60 @@ pub(crate) mod tests { mt.apply_op(&WalOp::UpsertNode(node), 0); write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); - let err = SegmentReader::open(&seg_dir, 1, None).err().unwrap(); - assert!( - err.to_string() - .contains("require DbOptions::dense_vector to be configured"), - "got: {}", - err - ); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(reader.dense_hnsw_header().is_none()); + } + + // --- Packed metadata reader tests --- + + fn build_node_meta_payload_for_layout_test( + label_offsets: &[u64], + label_ids: &[u32], + ) -> Vec { + let node_count = label_offsets.len() - 1; + let fixed_entries_offset = NODE_META_HEADER_SIZE; + let fixed_entries_len = node_count * NODE_META_FIXED_ENTRY_SIZE; + let label_offsets_offset = fixed_entries_offset + fixed_entries_len; + let label_ids_offset = + label_offsets_offset + label_offsets.len() * NODE_META_LABEL_OFFSET_ENTRY_SIZE; + let mut data = vec![0u8; label_ids_offset + label_ids.len() * 4]; + + data[0..8].copy_from_slice(&(node_count as u64).to_le_bytes()); + data[8..10].copy_from_slice(&(NODE_META_FIXED_ENTRY_SIZE as u16).to_le_bytes()); + data[10..12].copy_from_slice(&(NODE_META_LABEL_OFFSET_ENTRY_SIZE as u16).to_le_bytes()); + data[16..24].copy_from_slice(&(fixed_entries_offset as u64).to_le_bytes()); + data[24..32].copy_from_slice(&(label_offsets_offset as u64).to_le_bytes()); + data[32..40].copy_from_slice(&(label_ids_offset as u64).to_le_bytes()); + data[40..48].copy_from_slice(&(label_ids.len() as u64).to_le_bytes()); + + for (index, offset) in label_offsets.iter().enumerate() { + let pos = label_offsets_offset + index * NODE_META_LABEL_OFFSET_ENTRY_SIZE; + data[pos..pos + 8].copy_from_slice(&offset.to_le_bytes()); + } + for (index, label_id) in label_ids.iter().enumerate() { + let pos = label_ids_offset + index * 4; + data[pos..pos + 4].copy_from_slice(&label_id.to_le_bytes()); + } + + data } - // --- V5 sidecar reader tests --- + #[test] + fn test_node_meta_layout_rejects_invalid_label_offset_sentinels() { + let mut nonzero_first = build_node_meta_payload_for_layout_test(&[1, 1, 2], &[1, 2]); + assert!(parse_node_meta_layout(&nonzero_first).is_err()); + + let terminal_mismatch = build_node_meta_payload_for_layout_test(&[0, 1, 1], &[1, 2]); + assert!(parse_node_meta_layout(&terminal_mismatch).is_err()); + + let label_offsets_offset = NODE_META_HEADER_SIZE + 2 * NODE_META_FIXED_ENTRY_SIZE; + nonzero_first[label_offsets_offset..label_offsets_offset + 8] + .copy_from_slice(&0u64.to_le_bytes()); + assert!(parse_node_meta_layout(&nonzero_first).is_ok()); + } #[test] - fn test_sidecar_node_meta_roundtrip() { + fn test_packed_node_metadata_roundtrip() { let mt = Memtable::new(); mt.apply_op(&WalOp::UpsertNode(make_node_with_props(1, 1, "alice")), 0); mt.apply_op(&WalOp::UpsertNode(make_node(2, 2, "bob")), 0); @@ -7164,25 +12687,83 @@ pub(crate) mod tests { assert_eq!(reader.node_meta_count(), 2); // First entry: node_id=1 - let (nid, _off, _len, tid, updated_at, weight, key_len, _pho, phc, _lws) = - reader.node_meta_at(0).unwrap(); - assert_eq!(nid, 1); - assert_eq!(tid, 1); - assert_eq!(updated_at, 2000); - assert!((weight - 0.75).abs() < f32::EPSILON); - assert_eq!(key_len, 5); // "alice" - assert_eq!(phc, 0); // CP2 stops emitting legacy property hash counts for new segments. + let meta = reader.node_meta_at(0).unwrap(); + assert_eq!(meta.node_id, 1); + assert_eq!(meta.label_ids.as_slice(), [1]); + assert_eq!(meta.updated_at, 2000); + assert!((meta.weight - 0.75).abs() < f32::EPSILON); + assert_eq!(meta.key_len, 5); // "alice" // Second entry: node_id=2 - let (nid2, _, _, tid2, _, _, key_len2, _, phc2, _) = reader.node_meta_at(1).unwrap(); - assert_eq!(nid2, 2); - assert_eq!(tid2, 2); - assert_eq!(key_len2, 3); // "bob" - assert_eq!(phc2, 0); // no props + let meta2 = reader.node_meta_at(1).unwrap(); + assert_eq!(meta2.node_id, 2); + assert_eq!(meta2.label_ids.as_slice(), [2]); + assert_eq!(meta2.key_len, 3); // "bob" } #[test] - fn test_sidecar_edge_meta_roundtrip() { + fn test_node_meta_reader_rejects_malformed_label_sets() { + let layout_data = build_node_meta_payload_for_layout_test(&[0, 2], &[2, 1]); + let layout = parse_node_meta_layout(&layout_data).unwrap().unwrap(); + let error = read_node_meta_entry_at(&layout_data, layout, 0).unwrap_err(); + assert!( + error.to_string().contains("sorted ascending and unique"), + "got: {error}" + ); + + let layout_data = build_node_meta_payload_for_layout_test(&[0, 2], &[1, 1]); + let layout = parse_node_meta_layout(&layout_data).unwrap().unwrap(); + let error = read_node_meta_entry_at(&layout_data, layout, 0).unwrap_err(); + assert!( + error.to_string().contains("sorted ascending and unique"), + "got: {error}" + ); + } + + #[test] + fn test_node_meta_batch_reads_node_meta_without_node_records_index() { + let mt = Memtable::new(); + for id in 1..=8u64 { + mt.apply_op( + &WalOp::UpsertNode(make_node(id, id as u32, &format!("n{id}"))), + id, + ); + } + + let (_dir, mut reader) = write_and_open(&mt); + reader.nodes_mmap = MappedData::Empty; + + let seek_lookups = vec![(0usize, 1u64), (1, 8)]; + let mut seek_results = vec![None; seek_lookups.len()]; + reader + .get_node_meta_batch(&seek_lookups, &mut seek_results) + .unwrap(); + assert_eq!( + seek_results, + vec![ + Some((NodeLabelSet::single(1).unwrap(), 1001, 0.5)), + Some((NodeLabelSet::single(8).unwrap(), 1001, 0.5)), + ] + ); + + let merge_lookups = vec![(0usize, 2u64), (1, 3), (2, 4), (3, 5)]; + let mut merge_results = vec![None; merge_lookups.len()]; + reader + .get_node_meta_batch(&merge_lookups, &mut merge_results) + .unwrap(); + assert_eq!( + merge_results, + vec![ + Some((NodeLabelSet::single(2).unwrap(), 1001, 0.5)), + Some((NodeLabelSet::single(3).unwrap(), 1001, 0.5)), + Some((NodeLabelSet::single(4).unwrap(), 1001, 0.5)), + Some((NodeLabelSet::single(5).unwrap(), 1001, 0.5)), + ] + ); + } + + #[test] + fn test_packed_edge_metadata_roundtrip() { let mt = Memtable::new(); mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 0); mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 0); @@ -7205,22 +12786,192 @@ pub(crate) mod tests { } #[test] - fn test_sidecar_data_offset_matches_nodes_dat() { + fn test_packed_node_metadata_offsets_decode_node_records() { let mt = Memtable::new(); mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 0); mt.apply_op(&WalOp::UpsertNode(make_node(2, 2, "bb")), 0); let (_dir, reader) = write_and_open(&mt); - // Read data_offset/data_len from sidecar and verify node can be decoded there + // Read data_offset/data_len from metadata and verify node can be decoded there. for i in 0..reader.node_meta_count() as usize { - let (nid, data_offset, data_len, tid, _, _, _, _, _, _) = - reader.node_meta_at(i).unwrap(); - // Verify the offset points to valid data in nodes.dat - let node = decode_node_at(&reader.nodes_mmap, data_offset as usize, nid).unwrap(); - assert_eq!(node.id, nid); - assert_eq!(node.type_id, tid); - assert!(data_len > 0); + let meta = reader.node_meta_at(i).unwrap(); + // Verify the offset points to valid data in the node records payload. + let record_start = meta.data_offset as usize; + assert_eq!(reader.nodes_mmap[record_start], 1); + assert_eq!( + u32::from_le_bytes( + reader.nodes_mmap[record_start + 1..record_start + 5] + .try_into() + .unwrap() + ), + meta.label_ids.single_label_id() + ); + let node = decode_node_at(&reader.nodes_mmap, record_start, meta.node_id).unwrap(); + assert_eq!(node.id, meta.node_id); + assert_eq!(node.label_ids, meta.label_ids); + assert!(meta.data_len > 0); + } + } + + #[test] + fn test_legacy_single_type_node_record_payload_is_rejected() { + let mut legacy = Vec::new(); + legacy.extend_from_slice(&1u32.to_le_bytes()); + legacy.extend_from_slice(&1u16.to_le_bytes()); + legacy.extend_from_slice(b"a"); + legacy.extend_from_slice(&1000i64.to_le_bytes()); + legacy.extend_from_slice(&1001i64.to_le_bytes()); + legacy.extend_from_slice(&0.5f32.to_le_bytes()); + let props = rmp_serde::to_vec(&BTreeMap::::new()).unwrap(); + legacy.extend_from_slice(&(props.len() as u32).to_le_bytes()); + legacy.extend_from_slice(&props); + legacy.push(0); + + assert!(decode_node_at(&legacy, 0, 1).is_err()); + } + + #[test] + fn test_node_record_reader_rejects_malformed_label_sets() { + fn raw_node_record(label_ids: &[u32]) -> Vec { + let mut data = Vec::new(); + data.push(label_ids.len() as u8); + for &label_id in label_ids { + data.extend_from_slice(&label_id.to_le_bytes()); + } + data.extend_from_slice(&1u16.to_le_bytes()); + data.extend_from_slice(b"a"); + data.extend_from_slice(&1000i64.to_le_bytes()); + data.extend_from_slice(&1001i64.to_le_bytes()); + data.extend_from_slice(&0.5f32.to_le_bytes()); + let props = rmp_serde::to_vec(&BTreeMap::::new()).unwrap(); + data.extend_from_slice(&(props.len() as u32).to_le_bytes()); + data.extend_from_slice(&props); + data + } + + let empty = raw_node_record(&[]); + assert!(decode_node_at(&empty, 0, 1).is_err()); + + let unsorted = raw_node_record(&[2, 1]); + assert!(decode_node_at(&unsorted, 0, 1).is_err()); + + let duplicate = raw_node_record(&[1, 1]); + assert!(decode_node_at(&duplicate, 0, 1).is_err()); + } + + #[test] + fn test_cross_segment_copy_external_sidecar_rejected() { + // Create segment A (id=1) with edges to produce external sidecars + let mt_a = Memtable::new(); + mt_a.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a1")), 1); + mt_a.apply_op(&WalOp::UpsertNode(make_node(2, 1, "a2")), 2); + mt_a.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 1)), 3); + let dir_a = tempfile::tempdir().unwrap(); + let seg_dir_a = dir_a.path().join("seg_0001"); + write_segment(&seg_dir_a, 1, &mt_a, None).unwrap(); + + // Create segment B (id=2) with different data + let mt_b = Memtable::new(); + mt_b.apply_op(&WalOp::UpsertNode(make_node(3, 2, "b1")), 4); + mt_b.apply_op(&WalOp::UpsertNode(make_node(4, 2, "b2")), 5); + mt_b.apply_op(&WalOp::UpsertEdge(make_edge(20, 3, 4, 2)), 6); + let dir_b = tempfile::tempdir().unwrap(); + let seg_dir_b = dir_b.path().join("seg_0002"); + write_segment(&seg_dir_b, 2, &mt_b, None).unwrap(); + + // Find an external optional sidecar file in segment A + let manifest_a = read_segment_manifest_for_test(&seg_dir_a); + let external_record = manifest_a.components.iter().find(|r| { + matches!(r.handle, ComponentHandleV1::ExternalFile { .. }) + && matches!(r.requirement, ComponentRequirement::Optional { .. }) + }); + + let record = external_record + .expect("test precondition: expected optional external sidecar in segment A"); + let relative_path = match &record.handle { + ComponentHandleV1::ExternalFile { relative_path, .. } => relative_path, + _ => panic!("test precondition: expected ExternalFile handle"), + }; + let src_path = seg_dir_a.join(relative_path); + let dst_path = seg_dir_b.join(relative_path); + assert!( + src_path.exists(), + "test precondition: source sidecar file must exist" + ); + std::fs::copy(&src_path, &dst_path).unwrap(); + + // Patch segment B's manifest to reference the copied file's record + let mut manifest_b = read_segment_manifest_for_test(&seg_dir_b); + let matching = manifest_b + .components + .iter_mut() + .find(|r| r.kind == record.kind); + if let Some(target) = matching { + *target = record.clone(); } + write_segment_manifest_for_test(&seg_dir_b, &manifest_b); + + // Opening segment B succeeds but the optional component + // is unavailable due to identity header mismatch + let info_b = segment_info_from_manifest(&manifest_b); + let reader = SegmentReader::open_with_info(&seg_dir_b, &info_b, None, &[]).unwrap(); + + let avail = reader.optional_component_availability_for_test(record.kind.clone()); + assert!( + matches!( + avail, + ComponentAvailability::Incompatible { .. } + | ComponentAvailability::CorruptIdentity { .. } + ), + "cross-segment optional sidecar should be unavailable, got: {:?}", + avail, + ); + } + + #[test] + fn test_cross_segment_copy_packed_manifest_record_rejected() { + // Create segment A (id=1) + let mt_a = Memtable::new(); + mt_a.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a1")), 1); + mt_a.apply_op(&WalOp::UpsertNode(make_node(2, 1, "a2")), 2); + let dir_a = tempfile::tempdir().unwrap(); + let seg_dir_a = dir_a.path().join("seg_0001"); + write_segment(&seg_dir_a, 1, &mt_a, None).unwrap(); + + // Create segment B (id=2) with different data + let mt_b = Memtable::new(); + mt_b.apply_op(&WalOp::UpsertNode(make_node(3, 2, "b1")), 3); + mt_b.apply_op(&WalOp::UpsertNode(make_node(4, 2, "b2")), 4); + let dir_b = tempfile::tempdir().unwrap(); + let seg_dir_b = dir_b.path().join("seg_0002"); + write_segment(&seg_dir_b, 2, &mt_b, None).unwrap(); + + // Inject segment A's NodeRecords packed range record into B's manifest + let manifest_a = read_segment_manifest_for_test(&seg_dir_a); + let node_record_a = manifest_a + .components + .iter() + .find(|r| r.kind == SegmentComponentKind::NodeRecords) + .unwrap() + .clone(); + + let mut manifest_b = read_segment_manifest_for_test(&seg_dir_b); + let node_record_b = manifest_b + .components + .iter_mut() + .find(|r| r.kind == SegmentComponentKind::NodeRecords) + .unwrap(); + *node_record_b = node_record_a; + write_segment_manifest_for_test(&seg_dir_b, &manifest_b); + + // Opening should fail: the injected record has a different component_id + // (computed from segment A's segment_id) and a different container_component_id + let info_b = segment_info_from_manifest(&manifest_b); + let result = SegmentReader::open_with_info(&seg_dir_b, &info_b, None, &[]); + assert!( + result.is_err(), + "injecting another segment's packed record should be rejected, got Ok" + ); } } diff --git a/src/segment_writer.rs b/src/segment_writer.rs index da611f2..07f7c97 100644 --- a/src/segment_writer.rs +++ b/src/segment_writer.rs @@ -1,23 +1,50 @@ use crate::degree_cache::{ - write_folded_degree_delta_sidecar_from_sidecars, write_sorted_degree_delta_sidecar, - DegreeOverlaySnapshot, DEGREE_DELTA_FILENAME, + write_folded_degree_delta_sidecar_payload_from_sidecars, + write_sorted_degree_delta_sidecar_payload, DegreeDelta, DegreeOverlaySnapshot, + DEGREE_DELTA_FILENAME, }; -use crate::dense_hnsw::{write_dense_hnsw_index_from_points, DensePointInput}; +use crate::dense_hnsw::{ + build_dense_hnsw_from_points, write_prebuilt_hnsw_to_writers, BuiltHnsw, DensePointInput, +}; +use crate::dense_hnsw::{DENSE_HNSW_GRAPH_FILENAME, DENSE_HNSW_META_FILENAME}; +use crate::edge_metadata::EdgeMetadataIndexEntries; use crate::error::EngineError; use crate::memtable::{encode_range_prop_value, AdjEntry, Memtable}; -use crate::parallel::engine_cpu_try_join; +use crate::parallel::{engine_cpu_join, engine_cpu_try_join}; use crate::planner_stats::{ - write_compaction_planner_stats_sidecar_best_effort, - write_flush_planner_stats_sidecar_best_effort, + assemble_compaction_stats_from_partials, assemble_flush_stats_from_partials, + build_compaction_stats_core_partial, build_flush_stats_core_partial, + equality_index_stats_from_written_groups, planner_stats_sidecar_payload, + range_index_stats_from_written_entries, DeclaredIndexStatsEvidence, PLANNER_STATS_FILENAME, +}; +use crate::segment_components::{ + component_build_fingerprint, component_id, decode_identity_header, decode_manifest_envelope, + dependency_digest, encode_identity_header, encode_manifest_envelope, + is_packed_core_component_kind, is_refreshable_external_component_kind, + patch_packed_range_container_id, secondary_declaration_dependency, + segment_source_groups_from_records, source_component_dependency, source_group_dependency, + validate_packed_core_records_contract, ComponentAvailability, ComponentDependencyV1, + ComponentFallbackClass, ComponentHandleV1, ComponentIdentityHeaderV1, ComponentIdentityWriter, + ComponentRequirement, ComponentTrustClass, SegmentComponentBuildKind, SegmentComponentKind, + SegmentComponentManifestV1, SegmentComponentRecordV1, SegmentComponentSourceGroups, + SegmentSourceGroupKind, COMPONENT_IDENTITY_HEADER_LEN, PACKED_CORE_FILENAME, + PACKED_CORE_TMP_FILENAME, SEGMENT_COMPONENT_MANIFEST_FILENAME, + SEGMENT_COMPONENT_MANIFEST_PAYLOAD_VERSION, SEGMENT_COMPONENT_MANIFEST_TMP_FILENAME, + ZERO_DIGEST, }; use crate::segment_reader::SegmentReader; -use crate::sparse_postings::write_sparse_posting_files; +use crate::sparse_postings::{ + write_sparse_posting_files_to_writers, SPARSE_POSTINGS_FILENAME, SPARSE_POSTING_INDEX_FILENAME, +}; use crate::types::*; -use std::collections::{BTreeMap, HashMap}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs::{self, File}; -use std::io::{BufWriter, Write}; +use std::io::{BufWriter, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; // --- Binary write helpers (little-endian) --- @@ -43,27 +70,28 @@ fn write_u64(w: &mut impl Write, v: u64) -> Result<(), EngineError> { // --- Segment format version --- -/// Magic bytes identifying an OverGraph segment directory. -pub const SEGMENT_MAGIC: [u8; 4] = *b"EGRM"; /// Current segment format version. /// v1: original format /// v2: added valid_from/valid_to to edge records /// v3: added valid_from/valid_to to adjacency postings /// v4: BTreeMap props, removed redundant ID from records, delta-encoded adjacency -/// v5: metadata sidecars (node_meta.dat, edge_meta.dat, node_prop_hashes.dat) +/// v5: metadata payloads (node metadata, edge metadata, node property hash metadata) /// v6: optional node vector sidecars/blobs /// v7: optional dense HNSW sidecars (dense_hnsw_meta.dat, dense_hnsw_graph.dat) /// v8: optional sparse posting-list sidecars (sparse_posting_index.dat, sparse_postings.dat) /// v9: last_write_seq in node_meta (60B), edge_meta (80B), tombstones (25B) -pub const SEGMENT_FORMAT_VERSION: u32 = 9; +/// v10: component identity manifest, dependency-gated sidecars, and packed core segment file. +/// No checked mmap, no component CRC block tables, no read-time byte verification. +pub const SEGMENT_FORMAT_VERSION: u32 = 10; -pub(crate) const NODE_VECTOR_META_FILENAME: &str = "node_vector_meta.dat"; -pub(crate) const NODE_DENSE_VECTOR_BLOB_FILENAME: &str = "node_dense_vectors.dat"; -pub(crate) const NODE_SPARSE_VECTOR_BLOB_FILENAME: &str = "node_sparse_vectors.dat"; pub(crate) const NODE_VECTOR_META_ENTRY_SIZE: usize = 28; pub(crate) const SECONDARY_INDEX_DIRNAME: &str = "secondary_indexes"; +static OPTIONAL_REFRESH_TMP_NONCE: AtomicU64 = AtomicU64::new(1); const NODE_VECTOR_FLAG_DENSE: u8 = 0b0000_0001; const NODE_VECTOR_FLAG_SPARSE: u8 = 0b0000_0010; +const NODE_META_HEADER_SIZE: u64 = 48; +const NODE_META_FIXED_ENTRY_SIZE: u16 = 48; +const NODE_META_LABEL_OFFSET_ENTRY_SIZE: u16 = 8; // --- Segment file format constants --- @@ -71,3457 +99,8543 @@ const NODE_VECTOR_FLAG_SPARSE: u8 = 0b0000_0010; const NODE_INDEX_ENTRY_SIZE: u64 = 16; /// Size of an edge index entry: edge_id (8) + offset (8) = 16 bytes const EDGE_INDEX_ENTRY_SIZE: u64 = 16; -/// Size of a type index entry: type_id (4) + offset (8) + count (4) = 16 bytes -const TYPE_INDEX_ENTRY_SIZE: u64 = 16; +/// Size of a label posting index entry: label_id (4) + offset (8) + count (4) = 16 bytes +const LABEL_POSTING_INDEX_ENTRY_SIZE: u64 = 16; const SECONDARY_EQ_ENTRY_SIZE: u64 = 20; const DENSE_VECTOR_VALUE_SIZE: u64 = 4; const SPARSE_VECTOR_ENTRY_SIZE: u64 = 8; -#[derive(Debug, Default, Clone)] -pub(crate) struct SecondaryIndexMaintenanceReport { - pub failed_equality_indexes: Vec<(u64, String)>, - pub failed_range_indexes: Vec<(u64, String)>, +pub(crate) type RecordDataSpan = (u64, u64, u32); +type RecordDataSpans = Vec; +type CompactionDatOutput = (SegmentComponentRecordV1, RecordDataSpans); +type AdjacencyGroupKey = (u64, u32); +type AdjacencyPosting = (u64, u64, f32, i64, i64); +type AdjacencyGroups = BTreeMap>; + +struct KeyIndexPayloadPlan<'a> { + entries: Vec>, } -pub(crate) fn secondary_indexes_dir(seg_dir: &Path) -> PathBuf { - seg_dir.join(SECONDARY_INDEX_DIRNAME) +struct KeyIndexEntryPlan<'a> { + label_id: u32, + key: &'a [u8], + node_id: u64, + encoded_len: u64, } -pub(crate) fn node_prop_eq_sidecar_path(seg_dir: &Path, index_id: u64) -> PathBuf { - secondary_indexes_dir(seg_dir).join(format!("node_prop_eq_{}.dat", index_id)) +struct LabelPostingIndexPayloadPlan { + groups: Vec<(u32, Vec)>, } -pub(crate) fn node_prop_range_sidecar_path(seg_dir: &Path, index_id: u64) -> PathBuf { - secondary_indexes_dir(seg_dir).join(format!("node_prop_range_{}.dat", index_id)) +struct TimestampIndexPayloadPlan { + entries: Vec<(u32, i64, u64)>, } -/// Write all segment files for a frozen memtable into the given directory. -/// -/// Creates: nodes.dat, edges.dat, adj_out.idx, adj_out.dat, adj_in.idx, -/// adj_in.dat, key_index.dat, node_type_index.dat, edge_type_index.dat, -/// edge_triple_index.dat, tombstones.dat, and any declared secondary sidecars -/// -/// IMPORTANT: Two index-writing paths exist and must stay in sync: -/// 1. This function (flush path, builds indexes from Memtable) -/// 2. `write_indexes_from_metadata_with_secondary_indexes()` (compaction path, builds from sidecars) -/// -/// If you add a new index type, you MUST add it to BOTH paths. -pub(crate) fn write_segment_with_degree_overlay_and_secondary_indexes( - seg_dir: &Path, - segment_id: u64, - memtable: &Memtable, - dense_config: Option<&DenseVectorConfig>, - degree_overlay: &DegreeOverlaySnapshot, - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result { - write_segment_inner( - seg_dir, - segment_id, - memtable, - dense_config, - Some(degree_overlay), - secondary_indexes, - ) +struct EdgeTripleIndexPayloadPlan { + entries: Vec<(u64, u64, u32, u64)>, } -#[cfg(test)] -pub(crate) fn write_segment_without_degree_sidecar_for_test( - seg_dir: &Path, - segment_id: u64, - memtable: &Memtable, - dense_config: Option<&DenseVectorConfig>, -) -> Result { - write_segment_inner(seg_dir, segment_id, memtable, dense_config, None, &[]) +#[derive(Clone)] +struct AdjacencyPayloadPlan { + groups: Vec, } -#[cfg(test)] -pub(crate) fn write_segment_without_degree_sidecar_with_secondary_indexes_for_test( - seg_dir: &Path, - segment_id: u64, - memtable: &Memtable, - dense_config: Option<&DenseVectorConfig>, - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result { - write_segment_inner( - seg_dir, - segment_id, - memtable, - dense_config, - None, - secondary_indexes, - ) +#[derive(Clone)] +struct AdjacencyGroupPlan { + node_id: u64, + label_id: u32, + offset: u64, + postings: Vec, } -fn write_segment_inner( - seg_dir: &Path, - segment_id: u64, - memtable: &Memtable, - dense_config: Option<&DenseVectorConfig>, - degree_overlay: Option<&DegreeOverlaySnapshot>, - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result { - fs::create_dir_all(seg_dir)?; +struct NodeVectorSourcePlan { + rows: Vec, + has_dense: bool, + has_sparse: bool, + dense_points: Vec, +} - let nodes = memtable.nodes(); - let edges = memtable.edges(); - let degree_entries = degree_overlay.map(DegreeOverlaySnapshot::sorted_entries); +#[derive(Clone, Copy)] +struct NodeVectorSourceRow { + node_id: u64, + flags: u8, + dense_offset: u64, + dense_len: u32, + sparse_offset: u64, + sparse_len: u32, +} - let node_data = write_nodes_dat(seg_dir, &nodes)?; - let edge_data = write_edges_dat(seg_dir, &edges)?; - run_index_fanout( - || { - write_key_index(seg_dir, &nodes)?; - write_type_index(seg_dir, "node_type_index", &memtable.type_node_index())?; - write_declared_equality_sidecars(seg_dir, memtable, secondary_indexes)?; - write_declared_range_sidecars(seg_dir, memtable, secondary_indexes)?; - write_timestamp_index(seg_dir, &memtable.time_node_index())?; - Ok(()) - }, - || { - write_adjacency_index(seg_dir, "adj_out", &memtable.adj_out())?; - write_adjacency_index(seg_dir, "adj_in", &memtable.adj_in())?; - write_type_index(seg_dir, "edge_type_index", &memtable.type_edge_index())?; - write_edge_triple_index(seg_dir, &edges)?; - write_tombstones( - seg_dir, - &memtable.deleted_nodes(), - &memtable.deleted_edges(), - )?; - if let Some(entries) = degree_entries.as_ref() { - write_sorted_degree_delta_sidecar(&seg_dir.join(DEGREE_DELTA_FILENAME), entries)?; - } - Ok(()) - }, - || { - let dense_points = write_sidecars(seg_dir, &node_data, &edge_data, &nodes, &edges)?; - write_dense_hnsw_index_from_points(seg_dir, dense_config, dense_points)?; - Ok(()) - }, - || write_sparse_posting_index(seg_dir, &nodes), - )?; - write_flush_planner_stats_sidecar_best_effort( - seg_dir, - segment_id, - &nodes, - &edges, - secondary_indexes, - ); - write_format_version(seg_dir)?; +#[derive(Debug, Default, Clone)] +pub(crate) struct SecondaryIndexMaintenanceReport { + pub failed_equality_indexes: Vec<(u64, String)>, + pub failed_range_indexes: Vec<(u64, String)>, +} - // fsync all files and the directory - fsync_dir(seg_dir)?; +#[derive(Debug, Default, Clone)] +pub(crate) struct MaintainedSecondaryIndexIds { + pub equality_index_ids: NodeIdSet, + pub range_index_ids: NodeIdSet, +} - Ok(SegmentInfo { - id: segment_id, - node_count: nodes.len() as u64, - edge_count: edges.len() as u64, - }) +#[derive(Debug, Default, Clone)] +pub(crate) struct CompactionComponentBuildOutput { + pub records: Vec, + pub report: SecondaryIndexMaintenanceReport, } -fn run_index_fanout( - node_branch: NodeBranch, - edge_branch: EdgeBranch, - vector_branch: VectorBranch, - sparse_branch: SparseBranch, -) -> Result<(), EngineError> -where - NodeBranch: FnOnce() -> Result<(), EngineError> + Send, - EdgeBranch: FnOnce() -> Result<(), EngineError> + Send, - VectorBranch: FnOnce() -> Result<(), EngineError> + Send, - SparseBranch: FnOnce() -> Result<(), EngineError> + Send, -{ - let _ = engine_cpu_try_join(node_branch, edge_branch)?; - let _ = engine_cpu_try_join(vector_branch, sparse_branch)?; - Ok(()) +#[derive(Debug, Default, Clone)] +struct DeclaredSidecarWriteOutcome { + records: Vec, + report: SecondaryIndexMaintenanceReport, + stats_evidence: DeclaredIndexStatsEvidence, } -pub(crate) fn write_node_prop_eq_sidecar_to_path( - path: &Path, - groups: &BTreeMap>, -) -> Result<(), EngineError> { - let file = File::create(path)?; - let mut writer = BufWriter::new(file); +struct FlushNodeIndexOutput<'a> { + key_index: KeyIndexPayloadPlan<'a>, + node_label_index: LabelPostingIndexPayloadPlan, + timestamp_index: TimestampIndexPayloadPlan, + external_records: Vec, + declared_evidence: DeclaredIndexStatsEvidence, +} - let entry_count = groups.len() as u64; - write_u64(&mut writer, entry_count)?; +struct FlushEdgeIndexOutput { + adj_out: AdjacencyPayloadPlan, + adj_in: AdjacencyPayloadPlan, + edge_label_index: LabelPostingIndexPayloadPlan, + edge_triple_index: EdgeTripleIndexPayloadPlan, + edge_metadata_indexes: EdgeMetadataIndexEntries, + external_records: Vec, + declared_evidence: DeclaredIndexStatsEvidence, +} - let data_start = 8 + entry_count * SECONDARY_EQ_ENTRY_SIZE; - let mut data_offset = data_start; - for (&value_hash, ids) in groups { - write_u64(&mut writer, value_hash)?; - write_u64(&mut writer, data_offset)?; - write_u32(&mut writer, ids.len() as u32)?; - data_offset += ids.len() as u64 * 8; - } +#[derive(Default)] +struct SecondaryIndexPartitions<'a> { + node_eq: Vec<&'a SecondaryIndexManifestEntry>, + node_range: Vec<&'a SecondaryIndexManifestEntry>, + edge_eq: Vec<&'a SecondaryIndexManifestEntry>, + edge_range: Vec<&'a SecondaryIndexManifestEntry>, +} - for ids in groups.values() { - for &node_id in ids { - write_u64(&mut writer, node_id)?; +fn partition_secondary_indexes( + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> SecondaryIndexPartitions<'_> { + let mut partitions = SecondaryIndexPartitions { + node_eq: Vec::with_capacity(secondary_indexes.len()), + node_range: Vec::with_capacity(secondary_indexes.len()), + edge_eq: Vec::with_capacity(secondary_indexes.len()), + edge_range: Vec::with_capacity(secondary_indexes.len()), + }; + for entry in secondary_indexes { + match (&entry.target, &entry.kind) { + (SecondaryIndexTarget::NodeProperty { .. }, SecondaryIndexKind::Equality) => { + partitions.node_eq.push(entry); + } + (SecondaryIndexTarget::NodeProperty { .. }, SecondaryIndexKind::Range { .. }) => { + partitions.node_range.push(entry); + } + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Equality) => { + partitions.edge_eq.push(entry); + } + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Range { .. }) => { + partitions.edge_range.push(entry); + } } } - - writer.flush()?; - writer.get_ref().sync_all()?; - Ok(()) + partitions } -pub(crate) fn write_node_prop_range_sidecar_to_path( - path: &Path, - entries: &[(u64, u64)], -) -> Result<(), EngineError> { - let file = File::create(path)?; - let mut writer = BufWriter::new(file); - - write_u64(&mut writer, entries.len() as u64)?; - for &(encoded_value, node_id) in entries { - write_u64(&mut writer, encoded_value)?; - write_u64(&mut writer, node_id)?; - } +const FLUSH_COMPONENT_GENERATION: u64 = 1; +const FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION: u32 = 1; - writer.flush()?; - writer.get_ref().sync_all()?; - Ok(()) +struct FlushComponentBuildSession { + records: Vec, } -fn write_declared_equality_sidecars( - seg_dir: &Path, - memtable: &Memtable, - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result<(), EngineError> { - let eq_entries: Vec<&SecondaryIndexManifestEntry> = secondary_indexes - .iter() - .filter(|entry| matches!(entry.kind, SecondaryIndexKind::Equality)) - .collect(); - if eq_entries.is_empty() { - return Ok(()); +impl FlushComponentBuildSession { + fn create() -> Self { + Self { + records: Vec::new(), + } } - let index_dir = secondary_indexes_dir(seg_dir); - fs::create_dir_all(&index_dir)?; + fn push(&mut self, record: SegmentComponentRecordV1) -> SegmentComponentRecordV1 { + self.records.push(record.clone()); + record + } - for entry in eq_entries { - let mut groups = BTreeMap::new(); - if let Some(values) = memtable.secondary_eq_state().get(&entry.index_id) { - for (&value_hash, ids) in values { - let mut sorted_ids: Vec = ids.iter().copied().collect(); - sorted_ids.sort_unstable(); - groups.insert(value_hash, sorted_ids); + fn extend(&mut self, records: Vec) { + self.records.extend(records); + } + + fn fsync_component_parent_dirs(&self, seg_dir: &Path) -> Result<(), EngineError> { + let mut parent_dirs = BTreeSet::new(); + for record in &self.records { + if let ComponentHandleV1::ExternalFile { relative_path, .. } = &record.handle { + if let Some(parent) = seg_dir.join(relative_path).parent() { + parent_dirs.insert(parent.to_path_buf()); + } } } - write_node_prop_eq_sidecar_to_path( - &node_prop_eq_sidecar_path(seg_dir, entry.index_id), - &groups, - )?; + for parent in parent_dirs { + fsync_dir(&parent)?; + } + Ok(()) } - - fsync_dir(&index_dir)?; - Ok(()) } -fn write_declared_range_sidecars( - seg_dir: &Path, - memtable: &Memtable, - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result<(), EngineError> { - let range_entries: Vec<&SecondaryIndexManifestEntry> = secondary_indexes - .iter() - .filter(|entry| matches!(entry.kind, SecondaryIndexKind::Range { .. })) - .collect(); - if range_entries.is_empty() { - return Ok(()); - } +pub(crate) struct PackedCoreWriter { + seg_dir: PathBuf, + segment_id: u64, + generation: u64, + tmp_path: PathBuf, + final_path: PathBuf, + writer: BufWriter, + container_digest: Sha256, + container_payload_len: u64, + records: Vec, +} - let index_dir = secondary_indexes_dir(seg_dir); - fs::create_dir_all(&index_dir)?; +struct PackedCoreComponentSink<'a> { + writer: &'a mut BufWriter, + container_digest: &'a mut Sha256, + container_payload_len: &'a mut u64, + payload_digest: Sha256, + payload_len: u64, +} - for entry in range_entries { - let sidecar_entries: Vec<(u64, u64)> = memtable - .secondary_range_state() - .get(&entry.index_id) - .map(|entries| entries.iter().copied().collect()) - .unwrap_or_default(); - write_node_prop_range_sidecar_to_path( - &node_prop_range_sidecar_path(seg_dir, entry.index_id), - &sidecar_entries, - )?; +impl PackedCoreWriter { + fn create(seg_dir: &Path, segment_id: u64, generation: u64) -> Result { + fs::create_dir_all(seg_dir)?; + let tmp_path = seg_dir.join(PACKED_CORE_TMP_FILENAME); + let final_path = seg_dir.join(PACKED_CORE_FILENAME); + let mut writer = BufWriter::new(File::create(&tmp_path)?); + writer.write_all(&[0; COMPONENT_IDENTITY_HEADER_LEN])?; + Ok(Self { + seg_dir: seg_dir.to_path_buf(), + segment_id, + generation, + tmp_path, + final_path, + writer, + container_digest: Sha256::new(), + container_payload_len: 0, + records: Vec::new(), + }) } - fsync_dir(&index_dir)?; - Ok(()) + #[allow(clippy::too_many_arguments)] + fn write_component( + &mut self, + kind: SegmentComponentKind, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + dependencies: Vec, + build_fingerprint: u64, + encode: impl FnOnce(&mut PackedCoreComponentSink<'_>) -> Result, + ) -> Result<(SegmentComponentRecordV1, T), EngineError> { + if !is_packed_core_component_kind(&kind) { + return Err(EngineError::CorruptRecord(format!( + "component {:?} is not eligible for {PACKED_CORE_FILENAME}", + kind + ))); + } + self.align_next_component()?; + let start_offset = self.container_payload_len; + let mut sink = PackedCoreComponentSink { + writer: &mut self.writer, + container_digest: &mut self.container_digest, + container_payload_len: &mut self.container_payload_len, + payload_digest: Sha256::new(), + payload_len: 0, + }; + let output = encode(&mut sink)?; + let payload_len = sink.payload_len; + let payload_digest: [u8; 32] = sink.payload_digest.finalize().into(); + let dependency_digest = dependency_digest(&dependencies); + let component_id = component_id( + self.segment_id, + &kind, + FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + payload_len, + Some(&payload_digest), + &dependency_digest, + build_fingerprint, + ); + let record = SegmentComponentRecordV1 { + component_id, + kind, + logical_format_version: FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + created_generation: self.generation, + requirement, + trust_class, + handle: ComponentHandleV1::PackedRange { + container_component_id: ZERO_DIGEST, + offset: start_offset, + len: payload_len, + }, + payload_len, + payload_digest: Some(payload_digest), + dependency_digest, + dependencies, + build_fingerprint, + }; + self.records.push(record.clone()); + Ok((record, output)) + } + + fn finish(mut self) -> Result, EngineError> { + self.writer.flush()?; + let container_payload_digest: [u8; 32] = self.container_digest.finalize().into(); + let dependencies = Vec::new(); + let dependency_digest = dependency_digest(&dependencies); + let build_fingerprint = component_fingerprint("flush.packed_segment_container", &[]); + let kind = SegmentComponentKind::PackedSegmentContainer; + let container_component_id = component_id( + self.segment_id, + &kind, + FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + self.container_payload_len, + Some(&container_payload_digest), + &dependency_digest, + build_fingerprint, + ); + let container_record = SegmentComponentRecordV1 { + component_id: container_component_id, + kind: kind.clone(), + logical_format_version: FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + created_generation: self.generation, + requirement: ComponentRequirement::Required, + trust_class: ComponentTrustClass::AuxiliaryBlob, + handle: ComponentHandleV1::ExternalFile { + relative_path: PACKED_CORE_FILENAME.to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: self.container_payload_len, + }, + payload_len: self.container_payload_len, + payload_digest: Some(container_payload_digest), + dependency_digest, + dependencies, + build_fingerprint, + }; + let header = ComponentIdentityHeaderV1 { + segment_format_version: SEGMENT_FORMAT_VERSION, + segment_id: self.segment_id, + component_kind: kind, + logical_format_version: FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + created_generation: self.generation, + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: self.container_payload_len, + component_id: container_component_id, + dependency_digest, + build_fingerprint, + payload_digest: Some(container_payload_digest), + }; + self.writer.seek(SeekFrom::Start(0))?; + self.writer.write_all(&encode_identity_header(&header))?; + self.writer.flush()?; + self.writer.get_ref().sync_all()?; + drop(self.writer); + fs::rename(&self.tmp_path, &self.final_path)?; + fsync_dir(&self.seg_dir)?; + + let mut records = self.records; + patch_packed_range_container_id(&mut records, container_component_id); + records.push(container_record); + Ok(records) + } + + fn align_next_component(&mut self) -> Result<(), EngineError> { + let padding = (8 - (self.container_payload_len % 8)) % 8; + if padding == 0 { + return Ok(()); + } + const ZEROS: [u8; 8] = [0; 8]; + let padding = padding as usize; + self.writer.write_all(&ZEROS[..padding])?; + self.container_digest.update(&ZEROS[..padding]); + self.container_payload_len += padding as u64; + Ok(()) + } } -/// nodes.dat format: -/// [count: u64] -/// [index: (node_id: u64, offset: u64) × count, sorted by node_id] -/// [data: node records sequentially] -/// -/// Returns Vec of (node_id, data_offset, data_len) sorted by node_id, -/// used by sidecar writers to record raw byte spans. -fn write_nodes_dat( +pub(crate) fn create_compaction_core_writer( seg_dir: &Path, - nodes: &NodeIdMap, -) -> Result, EngineError> { - let path = seg_dir.join("nodes.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - - // Sort nodes by ID for binary search in the index - let mut sorted: Vec<&NodeRecord> = nodes.values().collect(); - sorted.sort_by_key(|n| n.id); + segment_id: u64, +) -> Result { + PackedCoreWriter::create(seg_dir, segment_id, FLUSH_COMPONENT_GENERATION) +} - let count = sorted.len() as u64; - write_u64(&mut w, count)?; +pub(crate) fn finish_compaction_core_writer( + core_writer: PackedCoreWriter, +) -> Result, EngineError> { + core_writer.finish() +} - // First pass: encode into reused buffer to collect sizes for offset table. - let mut buf = Vec::new(); - let mut sizes: Vec = Vec::with_capacity(sorted.len()); - for node in &sorted { - encode_node_record_into(&mut buf, node)?; - sizes.push(buf.len() as u64); +impl Write for PackedCoreComponentSink<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.writer.write(buf)?; + let bytes = &buf[..written]; + self.container_digest.update(bytes); + self.payload_digest.update(bytes); + *self.container_payload_len += written as u64; + self.payload_len += written as u64; + Ok(written) } - // Write index entries and collect data info for sidecars - let data_start = 8 + count * NODE_INDEX_ENTRY_SIZE; - let mut data_offset = data_start; - let mut node_data = Vec::with_capacity(sorted.len()); - for (i, node) in sorted.iter().enumerate() { - write_u64(&mut w, node.id)?; - write_u64(&mut w, data_offset)?; - node_data.push((node.id, data_offset, sizes[i] as u32)); - data_offset += sizes[i]; + fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> { + self.writer.write_all(buf)?; + self.container_digest.update(buf); + self.payload_digest.update(buf); + *self.container_payload_len += buf.len() as u64; + self.payload_len += buf.len() as u64; + Ok(()) } - // Second pass: re-encode into reused buffer and write directly. - for node in &sorted { - encode_node_record_into(&mut buf, node)?; - w.write_all(&buf)?; + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() } - - w.flush()?; - w.get_ref().sync_all()?; - Ok(node_data) } -/// edges.dat format: -/// [count: u64] -/// [index: (edge_id: u64, offset: u64) × count, sorted by edge_id] -/// [data: edge records sequentially] -/// -/// Returns Vec of (edge_id, data_offset, data_len) sorted by edge_id, -/// used by sidecar writers to record raw byte spans. -fn write_edges_dat( +#[allow(clippy::too_many_arguments)] +fn create_segment_component_writer( seg_dir: &Path, - edges: &NodeIdMap, -) -> Result, EngineError> { - let path = seg_dir.join("edges.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); + segment_id: u64, + relative_path: &str, + kind: SegmentComponentKind, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + build_fingerprint: u64, +) -> Result { + let path = seg_dir.join(relative_path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + ComponentIdentityWriter::create( + &path, + relative_path.to_string(), + SEGMENT_FORMAT_VERSION, + segment_id, + kind, + FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + FLUSH_COMPONENT_GENERATION, + requirement, + trust_class, + build_fingerprint, + true, + ) +} - let mut sorted: Vec<&EdgeRecord> = edges.values().collect(); - sorted.sort_by_key(|e| e.id); +#[allow(clippy::too_many_arguments)] +fn create_flush_component_writer( + seg_dir: &Path, + segment_id: u64, + relative_path: &str, + kind: SegmentComponentKind, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + build_fingerprint: u64, +) -> Result { + create_segment_component_writer( + seg_dir, + segment_id, + relative_path, + kind, + requirement, + trust_class, + build_fingerprint, + ) +} - let count = sorted.len() as u64; - write_u64(&mut w, count)?; +#[allow(clippy::too_many_arguments)] +fn write_flush_component( + seg_dir: &Path, + segment_id: u64, + relative_path: &str, + kind: SegmentComponentKind, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + dependencies: Vec, + build_fingerprint: u64, + encode: impl FnOnce(&mut ComponentIdentityWriter) -> Result, +) -> Result<(SegmentComponentRecordV1, T), EngineError> { + let mut writer = create_flush_component_writer( + seg_dir, + segment_id, + relative_path, + kind, + requirement, + trust_class, + build_fingerprint, + )?; + let output = encode(&mut writer)?; + let record = writer.finish(dependencies)?; + Ok((record, output)) +} - // First pass: encode into reused buffer to collect sizes for offset table. - let mut buf = Vec::new(); - let mut sizes: Vec = Vec::with_capacity(sorted.len()); - for edge in &sorted { - encode_edge_record_into(&mut buf, edge)?; - sizes.push(buf.len() as u64); - } +#[allow(clippy::too_many_arguments)] +fn write_flush_component_pair( + seg_dir: &Path, + segment_id: u64, + first_relative_path: &str, + first_kind: SegmentComponentKind, + first_build_fingerprint: u64, + second_relative_path: &str, + second_kind: SegmentComponentKind, + second_build_fingerprint: u64, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + dependencies: Vec, + encode: impl FnOnce( + &mut ComponentIdentityWriter, + &mut ComponentIdentityWriter, + ) -> Result, +) -> Result<(SegmentComponentRecordV1, SegmentComponentRecordV1, T), EngineError> { + let mut first = create_flush_component_writer( + seg_dir, + segment_id, + first_relative_path, + first_kind, + requirement.clone(), + trust_class, + first_build_fingerprint, + )?; + let mut second = create_flush_component_writer( + seg_dir, + segment_id, + second_relative_path, + second_kind, + requirement, + trust_class, + second_build_fingerprint, + )?; + let output = encode(&mut first, &mut second)?; + let first_record = first.finish(dependencies.clone())?; + let second_record = second.finish(dependencies)?; + Ok((first_record, second_record, output)) +} - let data_start = 8 + count * EDGE_INDEX_ENTRY_SIZE; - let mut data_offset = data_start; - let mut edge_data = Vec::with_capacity(sorted.len()); - for (i, edge) in sorted.iter().enumerate() { - write_u64(&mut w, edge.id)?; - write_u64(&mut w, data_offset)?; - edge_data.push((edge.id, data_offset, sizes[i] as u32)); - data_offset += sizes[i]; +#[allow(clippy::too_many_arguments)] +fn write_compaction_component( + seg_dir: &Path, + segment_id: u64, + relative_path: &str, + kind: SegmentComponentKind, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + dependencies: Vec, + build_fingerprint: u64, + encode: impl FnOnce(&mut ComponentIdentityWriter) -> Result, +) -> Result<(SegmentComponentRecordV1, T), EngineError> { + let mut writer = create_segment_component_writer( + seg_dir, + segment_id, + relative_path, + kind, + requirement, + trust_class, + build_fingerprint, + )?; + let output = encode(&mut writer)?; + let record = writer.finish(dependencies)?; + Ok((record, output)) +} + +#[allow(clippy::too_many_arguments)] +fn write_compaction_component_pair( + seg_dir: &Path, + segment_id: u64, + first_relative_path: &str, + first_kind: SegmentComponentKind, + first_build_fingerprint: u64, + second_relative_path: &str, + second_kind: SegmentComponentKind, + second_build_fingerprint: u64, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + dependencies: Vec, + encode: impl FnOnce( + &mut ComponentIdentityWriter, + &mut ComponentIdentityWriter, + ) -> Result, +) -> Result<(SegmentComponentRecordV1, SegmentComponentRecordV1, T), EngineError> { + let mut first = create_segment_component_writer( + seg_dir, + segment_id, + first_relative_path, + first_kind, + requirement.clone(), + trust_class, + first_build_fingerprint, + )?; + let mut second = create_segment_component_writer( + seg_dir, + segment_id, + second_relative_path, + second_kind, + requirement, + trust_class, + second_build_fingerprint, + )?; + let output = encode(&mut first, &mut second)?; + let first_record = first.finish(dependencies.clone())?; + let second_record = second.finish(dependencies)?; + Ok((first_record, second_record, output)) +} + +pub(crate) fn secondary_indexes_dir(seg_dir: &Path) -> PathBuf { + seg_dir.join(SECONDARY_INDEX_DIRNAME) +} + +#[cfg(test)] +pub(crate) fn node_prop_eq_sidecar_path(seg_dir: &Path, index_id: u64) -> PathBuf { + if let Some(path) = manifested_component_path( + seg_dir, + &SegmentComponentKind::NodePropertyEqualityIndex { index_id }, + ) { + return path; } + secondary_indexes_dir(seg_dir).join(format!("node_prop_eq_{}.dat", index_id)) +} - // Second pass: re-encode into reused buffer and write directly. - for edge in &sorted { - encode_edge_record_into(&mut buf, edge)?; - w.write_all(&buf)?; +#[cfg(test)] +pub(crate) fn node_prop_range_sidecar_path(seg_dir: &Path, index_id: u64) -> PathBuf { + if let Some(path) = manifested_component_path( + seg_dir, + &SegmentComponentKind::NodePropertyRangeIndex { index_id }, + ) { + return path; } + secondary_indexes_dir(seg_dir).join(format!("node_prop_range_{}.dat", index_id)) +} - w.flush()?; - w.get_ref().sync_all()?; - Ok(edge_data) +#[cfg(test)] +pub(crate) fn edge_prop_eq_sidecar_path(seg_dir: &Path, index_id: u64) -> PathBuf { + if let Some(path) = manifested_component_path( + seg_dir, + &SegmentComponentKind::EdgePropertyEqualityIndex { index_id }, + ) { + return path; + } + secondary_indexes_dir(seg_dir).join(format!("edge_prop_eq_{}.dat", index_id)) } -// --- Varint helpers for adjacency delta encoding --- +#[cfg(test)] +pub(crate) fn edge_prop_range_sidecar_path(seg_dir: &Path, index_id: u64) -> PathBuf { + if let Some(path) = manifested_component_path( + seg_dir, + &SegmentComponentKind::EdgePropertyRangeIndex { index_id }, + ) { + return path; + } + secondary_indexes_dir(seg_dir).join(format!("edge_prop_range_{}.dat", index_id)) +} -/// Write a u64 varint into a `Vec`. -fn write_varint_to_vec(buf: &mut Vec, mut val: u64) { - loop { - let mut byte = (val & 0x7F) as u8; - val >>= 7; - if val != 0 { - byte |= 0x80; +fn secondary_index_component_kind_for_entry( + entry: &SecondaryIndexManifestEntry, +) -> SegmentComponentKind { + match (&entry.target, &entry.kind) { + (SecondaryIndexTarget::NodeProperty { .. }, SecondaryIndexKind::Equality) => { + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + } } - buf.push(byte); - if val == 0 { - break; + (SecondaryIndexTarget::NodeProperty { .. }, SecondaryIndexKind::Range { .. }) => { + SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + } + } + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Equality) => { + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + } + } + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Range { .. }) => { + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: entry.index_id, + } } } } -/// Adjacency index + delta-encoded postings. -/// -/// Index file (adj_out.idx / adj_in.idx): -/// [count: u64] -/// [(node_id: u64, type_id: u32, offset: u64, count: u32) × count, sorted by (node_id, type_id)] -/// -/// Data file (adj_out.dat / adj_in.dat): -/// Per group: delta-encoded postings, variable length. -/// First posting: varint(edge_id) + varint(neighbor_id) + f32(weight) + varint(valid_from_enc) + varint(valid_to_enc) -/// Subsequent: varint(edge_id_delta) + varint(neighbor_id) + f32(weight) + varint(valid_from_enc) + varint(valid_to_enc) -/// valid_from_enc = valid_from as u64 (valid_from is always >= 0) -/// valid_to_enc = 0 if valid_to == i64::MAX, else (valid_to as u64) + 1 -fn write_adjacency_index( - seg_dir: &Path, - prefix: &str, - adj: &NodeIdMap>, -) -> Result<(), EngineError> { - let idx_path = seg_dir.join(format!("{}.idx", prefix)); - let dat_path = seg_dir.join(format!("{}.dat", prefix)); - - let idx_file = File::create(&idx_path)?; - let dat_file = File::create(&dat_path)?; - let mut idx_w = BufWriter::new(idx_file); - let mut dat_w = BufWriter::new(dat_file); - - // Group entries by (node_id, type_id) - let mut groups: Vec<(u64, u32, Vec<&AdjEntry>)> = Vec::new(); - for (&node_id, edge_map) in adj { - let mut by_type: HashMap> = HashMap::new(); - for entry in edge_map.values() { - by_type.entry(entry.type_id).or_default().push(entry); +fn secondary_index_base_relative_path_for_entry(entry: &SecondaryIndexManifestEntry) -> String { + match (&entry.target, &entry.kind) { + (SecondaryIndexTarget::NodeProperty { .. }, SecondaryIndexKind::Equality) => { + format!( + "{}/node_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ) + } + (SecondaryIndexTarget::NodeProperty { .. }, SecondaryIndexKind::Range { .. }) => { + format!( + "{}/node_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ) + } + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Equality) => { + format!( + "{}/edge_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ) } - for (type_id, mut postings) in by_type { - postings.sort_unstable_by_key(|e| e.edge_id); - groups.push((node_id, type_id, postings)); + (SecondaryIndexTarget::EdgeProperty { .. }, SecondaryIndexKind::Range { .. }) => { + format!( + "{}/edge_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ) } } +} - groups.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); +fn secondary_index_component_kind_matches_entry( + kind: &SegmentComponentKind, + entry: &SecondaryIndexManifestEntry, +) -> bool { + kind == &secondary_index_component_kind_for_entry(entry) +} - let count = groups.len() as u64; - write_u64(&mut idx_w, count)?; - - // Encode postings into a buffer first to measure byte sizes - let mut posting_buf = Vec::new(); - let mut dat_offset: u64 = 0; - let mut index_entries: Vec<(u64, u32, u64, u32)> = Vec::with_capacity(groups.len()); - - for (node_id, type_id, postings) in &groups { - let posting_count = postings.len() as u32; - index_entries.push((*node_id, *type_id, dat_offset, posting_count)); - - posting_buf.clear(); - let mut prev_edge_id: u64 = 0; - for entry in postings { - let delta = entry.edge_id - prev_edge_id; - prev_edge_id = entry.edge_id; - - write_varint_to_vec(&mut posting_buf, delta); - write_varint_to_vec(&mut posting_buf, entry.neighbor_id); - posting_buf.extend_from_slice(&entry.weight.to_le_bytes()); - debug_assert!( - entry.valid_from >= 0, - "valid_from must be non-negative for varint encoding" - ); - debug_assert!( - entry.valid_to >= 0, - "valid_to must be non-negative for sentinel encoding" - ); - write_varint_to_vec(&mut posting_buf, entry.valid_from as u64); - // Sentinel: 0 means i64::MAX, otherwise value + 1 - let vt_enc = if entry.valid_to == i64::MAX { - 0u64 - } else { - entry.valid_to as u64 + 1 +pub(crate) fn secondary_index_sidecar_paths_for_entry( + seg_dir: &Path, + entry: &SecondaryIndexManifestEntry, +) -> Vec { + let base_relative = secondary_index_base_relative_path_for_entry(entry); + let base_path = seg_dir.join(&base_relative); + let mut paths = vec![base_path.clone()]; + let Some(parent) = base_path.parent() else { + return paths; + }; + let Some(base_name) = base_path.file_name().and_then(|name| name.to_str()) else { + return paths; + }; + let stem = Path::new(base_name) + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or(base_name); + let generated_prefix = format!("{stem}.g"); + let refresh_prefix = format!(".{stem}.refresh_tmp."); + if let Ok(entries) = fs::read_dir(parent) { + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_file() { + continue; + } + let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else { + continue; }; - write_varint_to_vec(&mut posting_buf, vt_enc); + if file_name == base_name + || file_name.starts_with(&generated_prefix) + || file_name.starts_with(&refresh_prefix) + { + paths.push(path); + } + } + } + paths.sort(); + paths.dedup(); + paths +} + +pub(crate) fn remove_secondary_index_component_records( + seg_dir: &Path, + entry: &SecondaryIndexManifestEntry, +) -> Result, EngineError> { + let mut manifest = read_segment_component_manifest(seg_dir)?; + let mut removed_paths = Vec::new(); + let before_len = manifest.components.len(); + manifest.components.retain(|record| { + let matches_index = secondary_index_component_kind_matches_entry(&record.kind, entry); + if matches_index { + if let ComponentHandleV1::ExternalFile { relative_path, .. } = &record.handle { + removed_paths.push(seg_dir.join(relative_path)); + } + false + } else { + true } + }); - dat_w.write_all(&posting_buf)?; - dat_offset += posting_buf.len() as u64; + if manifest.components.len() == before_len { + return Ok(removed_paths); } - // Write index entries - for (node_id, type_id, offset, posting_count) in &index_entries { - write_u64(&mut idx_w, *node_id)?; - write_u32(&mut idx_w, *type_id)?; - write_u64(&mut idx_w, *offset)?; - write_u32(&mut idx_w, *posting_count)?; + manifest.generation = manifest.generation.saturating_add(1); + manifest.built_at_ms = current_time_millis(); + manifest.build_kind = SegmentComponentBuildKind::OptionalRefresh; + write_segment_component_manifest(seg_dir, &manifest)?; + Ok(removed_paths) +} + +pub(crate) fn maintained_secondary_index_ids_from_component_records( + records: &[SegmentComponentRecordV1], + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> MaintainedSecondaryIndexIds { + let mut maintained = MaintainedSecondaryIndexIds::default(); + for entry in secondary_indexes { + if records + .iter() + .any(|record| secondary_index_component_kind_matches_entry(&record.kind, entry)) + { + match entry.kind { + SecondaryIndexKind::Equality => { + maintained.equality_index_ids.insert(entry.index_id); + } + SecondaryIndexKind::Range { .. } => { + maintained.range_index_ids.insert(entry.index_id); + } + } + } } + maintained +} - idx_w.flush()?; - idx_w.get_ref().sync_all()?; - dat_w.flush()?; - dat_w.get_ref().sync_all()?; - Ok(()) +pub(crate) fn maintained_secondary_index_ids_from_segment_manifest( + seg_dir: &Path, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result { + let manifest = read_segment_component_manifest(seg_dir)?; + Ok(maintained_secondary_index_ids_from_component_records( + &manifest.components, + secondary_indexes, + )) } -/// key_index.dat format: -/// [entry_count: u64] -/// [offset_table: u64 × entry_count] (byte offset to each entry in data section) -/// [data section: entries sorted by (type_id, key)] -/// -/// Each entry: [type_id: u32][node_id: u64][key_len: u16][key: bytes] -fn write_key_index(seg_dir: &Path, nodes: &NodeIdMap) -> Result<(), EngineError> { - let path = seg_dir.join("key_index.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - - // Collect and sort entries by (type_id, key) - let mut entries: Vec<(u32, &str, u64)> = nodes - .values() - .map(|n| (n.type_id, n.key.as_str(), n.id)) - .collect(); - entries.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1))); +#[cfg(test)] +fn manifested_component_path(seg_dir: &Path, kind: &SegmentComponentKind) -> Option { + let manifest = read_segment_component_manifest(seg_dir).ok()?; + manifest + .components + .iter() + .find(|record| &record.kind == kind) + .and_then(|record| match &record.handle { + ComponentHandleV1::ExternalFile { relative_path, .. } => { + Some(seg_dir.join(relative_path)) + } + ComponentHandleV1::PackedRange { .. } => None, + }) +} - let count = entries.len() as u64; - write_u64(&mut w, count)?; +pub(crate) fn component_fingerprint(namespace: &str, fields: &[u64]) -> u64 { + component_build_fingerprint(SEGMENT_FORMAT_VERSION, namespace, fields) +} - // Pre-compute entry sizes to build offset table - // Each entry: type_id (4) + node_id (8) + key_len (2) + key_bytes - let entry_sizes: Vec = entries +pub(crate) fn planner_stats_component_dependencies( + segment_data_id: [u8; 32], + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Vec { + let mut entries: Vec<&SecondaryIndexManifestEntry> = secondary_indexes .iter() - .map(|(_, key, _)| 4 + 8 + 2 + key.len() as u64) + .filter(|entry| entry.state == SecondaryIndexState::Ready) .collect(); + entries.sort_by_key(|entry| entry.index_id); + let mut dependencies = Vec::with_capacity(entries.len() + 1); + dependencies.push(source_group_dependency( + SegmentSourceGroupKind::SegmentData, + segment_data_id, + )); + dependencies.extend(entries.into_iter().map(secondary_declaration_dependency)); + dependencies +} - // Data section starts after: count (8) + offset_table (count * 8) - let data_start = 8 + count * 8; - let mut offset = data_start; - for &size in &entry_sizes { - write_u64(&mut w, offset)?; - offset += size; - } - - // Write data entries - for (type_id, key, node_id) in &entries { - write_u32(&mut w, *type_id)?; - write_u64(&mut w, *node_id)?; - let key_bytes = key.as_bytes(); - if key_bytes.len() > u16::MAX as usize { - return Err(EngineError::SerializationError(format!( - "node key exceeds maximum length of {} bytes", - u16::MAX - ))); +pub(crate) fn planner_stats_component_fingerprint( + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> u64 { + let mut entries: Vec<&SecondaryIndexManifestEntry> = secondary_indexes + .iter() + .filter(|entry| entry.state == SecondaryIndexState::Ready) + .collect(); + entries.sort_by_key(|entry| entry.index_id); + let mut fields = Vec::with_capacity(entries.len() * 2 + 1); + fields.push(entries.len() as u64); + for entry in entries { + fields.push(entry.index_id); + if let ComponentDependencyV1::SecondaryIndexDeclaration { fingerprint, .. } = + secondary_declaration_dependency(entry) + { + fields.push(fingerprint); } - write_u16(&mut w, key_bytes.len() as u16)?; - w.write_all(key_bytes)?; } - - w.flush()?; - w.get_ref().sync_all()?; - Ok(()) + component_fingerprint("flush.planner_stats", &fields) } -/// type index format (node_type_index.dat / edge_type_index.dat): -/// [entry_count: u64] -/// [index: entry_count × (type_id: u32, offset: u64, count: u32), sorted by type_id] -/// [data: packed u64 record IDs per type, grouped contiguously] -fn write_type_index( +#[cfg(test)] +pub(crate) fn publish_planner_stats_component_payload( seg_dir: &Path, - filename: &str, - type_index: &HashMap, + ready_secondary_indexes: &[SecondaryIndexManifestEntry], + payload: &[u8], ) -> Result<(), EngineError> { - let path = seg_dir.join(format!("{}.dat", filename)); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); + let manifest = read_segment_component_manifest(seg_dir)?; + let dependencies = + planner_stats_component_dependencies(manifest.segment_data_id, ready_secondary_indexes); + refresh_optional_component_with_writer( + seg_dir, + SegmentComponentKind::PlannerStats, + PLANNER_STATS_FILENAME, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::PlannerStatsUnavailable, + }, + ComponentTrustClass::OptionalAdvisoryStats, + dependencies, + planner_stats_component_fingerprint(ready_secondary_indexes), + |writer| { + writer.write_all(payload)?; + Ok(()) + }, + ) +} - // Collect non-empty type groups, sorted by type_id - let mut groups: Vec<(u32, Vec)> = type_index - .iter() - .filter(|(_, ids)| !ids.is_empty()) - .map(|(&type_id, ids)| { - let mut sorted_ids: Vec = ids.iter().copied().collect(); - sorted_ids.sort_unstable(); - (type_id, sorted_ids) - }) - .collect(); - groups.sort_by_key(|(type_id, _)| *type_id); +pub(crate) fn publish_planner_stats_component_payload_from_latest( + seg_dir: &Path, + ready_secondary_indexes: &[SecondaryIndexManifestEntry], + build_payload: impl FnOnce(Option<&[u8]>, u64, u64, u64) -> Result>, EngineError>, +) -> Result { + let captured_manifest = read_segment_component_manifest(seg_dir)?; + let source_groups = segment_source_groups_from_records( + captured_manifest.segment_id, + captured_manifest.node_count, + captured_manifest.edge_count, + &captured_manifest.components, + )?; + if source_groups.segment_data_id != captured_manifest.segment_data_id { + return Err(EngineError::CorruptRecord(format!( + "segment {} source identity changed before optional publication", + captured_manifest.segment_id + ))); + } + let current_payload = read_external_component_payload_from_manifest( + seg_dir, + &captured_manifest, + SegmentComponentKind::PlannerStats, + )?; + let Some(payload) = build_payload( + current_payload.as_deref(), + captured_manifest.segment_id, + captured_manifest.node_count, + captured_manifest.edge_count, + )? + else { + return Ok(false); + }; - let entry_count = groups.len() as u64; - write_u64(&mut w, entry_count)?; + let generation = captured_manifest.generation.saturating_add(1); + let final_relative_path = optional_generation_relative_path(PLANNER_STATS_FILENAME, generation); + let tmp_relative_path = optional_refresh_tmp_relative_path(PLANNER_STATS_FILENAME, generation); + let tmp_path = seg_dir.join(&tmp_relative_path); + if let Some(parent) = tmp_path.parent() { + fs::create_dir_all(parent)?; + } - // Data section starts after header + index - let data_start = 8 + entry_count * TYPE_INDEX_ENTRY_SIZE; - let mut data_offset = data_start; + let dependencies = planner_stats_component_dependencies( + captured_manifest.segment_data_id, + ready_secondary_indexes, + ); + let mut writer = ComponentIdentityWriter::create( + &tmp_path, + final_relative_path.clone(), + SEGMENT_FORMAT_VERSION, + captured_manifest.segment_id, + SegmentComponentKind::PlannerStats, + FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + generation, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::PlannerStatsUnavailable, + }, + ComponentTrustClass::OptionalAdvisoryStats, + planner_stats_component_fingerprint(ready_secondary_indexes), + true, + )?; + writer.write_all(&payload)?; + let record = writer.finish(dependencies)?; - // Write index entries - for (type_id, ids) in &groups { - write_u32(&mut w, *type_id)?; - write_u64(&mut w, data_offset)?; - let count = ids.len() as u32; - write_u32(&mut w, count)?; - data_offset += count as u64 * 8; // each ID is u64 = 8 bytes + let current_manifest = read_segment_component_manifest(seg_dir)?; + if current_manifest.segment_id != captured_manifest.segment_id + || current_manifest.segment_data_id != captured_manifest.segment_data_id + || current_manifest.generation != captured_manifest.generation + { + return Err(EngineError::CorruptRecord(format!( + "segment {} {}", + captured_manifest.segment_id, OPTIONAL_COMPONENT_PUBLICATION_CONFLICT_MESSAGE + ))); } - // Write data section (packed u64 IDs) - for (_, ids) in &groups { - for &id in ids { - write_u64(&mut w, id)?; - } + let final_path = seg_dir.join(&final_relative_path); + fs::rename(&tmp_path, &final_path)?; + if let Some(parent) = final_path.parent() { + fsync_dir(parent)?; } - w.flush()?; - w.get_ref().sync_all()?; - Ok(()) + let mut replacement = current_manifest; + replacement.generation = generation; + replacement.built_at_ms = current_time_millis(); + replacement.build_kind = SegmentComponentBuildKind::OptionalRefresh; + replacement + .components + .retain(|existing| existing.kind != SegmentComponentKind::PlannerStats); + replacement.components.push(record); + write_segment_component_manifest(seg_dir, &replacement)?; + Ok(true) } -/// edge_triple_index.dat format: -/// [count: u64] -/// [entries: count × (from: u64, to: u64, type_id: u32, edge_id: u64), sorted by (from, to, type_id)] -fn write_edge_triple_index( +fn read_external_component_payload_from_manifest( seg_dir: &Path, - edges: &NodeIdMap, -) -> Result<(), EngineError> { - let path = seg_dir.join("edge_triple_index.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); + manifest: &SegmentComponentManifestV1, + kind: SegmentComponentKind, +) -> Result>, EngineError> { + let Some(record) = manifest + .components + .iter() + .find(|record| record.kind == kind) + else { + return Ok(None); + }; + let ComponentHandleV1::ExternalFile { + relative_path, + payload_offset, + payload_len, + } = &record.handle + else { + return Ok(None); + }; + let path = seg_dir.join(relative_path); + let data = match fs::read(&path) { + Ok(data) => data, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(EngineError::IoError(error)), + }; + let end = payload_offset.checked_add(*payload_len).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "component payload range overflows for {}", + path.display() + )) + })?; + if end != data.len() as u64 { + return Ok(None); + } + let Ok(header) = decode_identity_header(&data) else { + return Ok(None); + }; + if header.segment_format_version != SEGMENT_FORMAT_VERSION + || header.segment_id != manifest.segment_id + || header.component_kind != record.kind + || header.logical_format_version != record.logical_format_version + || header.created_generation != record.created_generation + || header.payload_offset != *payload_offset + || header.payload_len != *payload_len + || header.component_id != record.component_id + || header.dependency_digest != record.dependency_digest + || header.build_fingerprint != record.build_fingerprint + || header.payload_digest != record.payload_digest + { + return Ok(None); + } + Ok(Some(data[*payload_offset as usize..end as usize].to_vec())) +} - // Collect and sort by (from, to, type_id) - let mut entries: Vec<(u64, u64, u32, u64)> = edges - .values() - .map(|e| (e.from, e.to, e.type_id, e.id)) - .collect(); - entries.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)).then(a.2.cmp(&b.2))); +const OPTIONAL_COMPONENT_PUBLICATION_CONFLICT_MESSAGE: &str = + "changed before optional component publication"; - let count = entries.len() as u64; - write_u64(&mut w, count)?; +pub(crate) fn is_optional_component_publication_conflict(error: &EngineError) -> bool { + matches!( + error, + EngineError::CorruptRecord(message) + if message.contains(OPTIONAL_COMPONENT_PUBLICATION_CONFLICT_MESSAGE) + ) +} - for (from, to, type_id, edge_id) in &entries { - write_u64(&mut w, *from)?; - write_u64(&mut w, *to)?; - write_u32(&mut w, *type_id)?; - write_u64(&mut w, *edge_id)?; - } +pub(crate) fn dense_config_fingerprint(config: Option<&DenseVectorConfig>) -> u64 { + let Some(config) = config else { + return component_fingerprint("dense_vector_config.none", &[]); + }; + let metric = match config.metric { + DenseMetric::Cosine => 1, + DenseMetric::Euclidean => 2, + DenseMetric::DotProduct => 3, + }; + component_fingerprint( + "dense_vector_config", + &[ + config.dimension as u64, + metric, + config.hnsw.m as u64, + config.hnsw.ef_construction as u64, + ], + ) +} - w.flush()?; - w.get_ref().sync_all()?; - Ok(()) +fn current_time_millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis().min(i64::MAX as u128) as i64) + .unwrap_or(0) } -/// timestamp_index.dat format: -/// [entry_count: u64] -/// [entries: entry_count × (type_id: u32, updated_at: i64, node_id: u64), -/// sorted by (type_id, updated_at, node_id)] +/// Write all segment components for a frozen memtable into the given directory. +/// +/// Required core components are packed into `segment.core`; refreshable and +/// accelerator sidecars remain externally materialized. +/// +/// IMPORTANT: Two index-writing paths exist and must stay in sync: +/// 1. This function (flush path, builds indexes from Memtable) +/// 2. `write_indexes_from_metadata_with_secondary_indexes()` (compaction path, builds from sidecars) /// -/// Each entry is 20 bytes. Binary search for range start (type_id, from_ms), -/// scan to range end (type_id, to_ms). O(log N) seek + O(results) scan. -fn write_timestamp_index( +/// If you add a new index type, you MUST add it to BOTH paths. +pub(crate) fn write_segment_with_degree_overlay_and_secondary_indexes( seg_dir: &Path, - time_index: &std::collections::BTreeSet<(u32, i64, u64)>, -) -> Result<(), EngineError> { - let path = seg_dir.join("timestamp_index.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - - let count = time_index.len() as u64; - write_u64(&mut w, count)?; + segment_id: u64, + memtable: &Memtable, + dense_config: Option<&DenseVectorConfig>, + degree_overlay: &DegreeOverlaySnapshot, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result { + write_segment_inner( + seg_dir, + segment_id, + memtable, + dense_config, + Some(degree_overlay), + secondary_indexes, + ) +} - // BTreeSet is already sorted by (type_id, updated_at, node_id) - for &(type_id, updated_at, node_id) in time_index { - write_u32(&mut w, type_id)?; - w.write_all(&updated_at.to_le_bytes())?; - write_u64(&mut w, node_id)?; - } +#[cfg(test)] +pub(crate) fn write_segment_without_degree_sidecar_for_test( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + dense_config: Option<&DenseVectorConfig>, +) -> Result { + write_segment_inner(seg_dir, segment_id, memtable, dense_config, None, &[]) +} - w.flush()?; - w.get_ref().sync_all()?; - Ok(()) +#[cfg(test)] +pub(crate) fn write_segment_without_degree_sidecar_with_secondary_indexes_for_test( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + dense_config: Option<&DenseVectorConfig>, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result { + write_segment_inner( + seg_dir, + segment_id, + memtable, + dense_config, + None, + secondary_indexes, + ) } -/// tombstones.dat format (v9): -/// [count: u64] -/// [(kind: u8, id: u64, deleted_at: i64, last_write_seq: u64) × count] -/// kind: 0 = node, 1 = edge. Entry size: 25 bytes. -fn write_tombstones( +fn write_segment_inner( seg_dir: &Path, - deleted_nodes: &NodeIdMap, - deleted_edges: &NodeIdMap, -) -> Result<(), EngineError> { - let path = seg_dir.join("tombstones.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); + segment_id: u64, + memtable: &Memtable, + dense_config: Option<&DenseVectorConfig>, + degree_overlay: Option<&DegreeOverlaySnapshot>, + secondary_indexes: &[SecondaryIndexManifestEntry], +) -> Result { + fs::create_dir_all(seg_dir)?; - let count = (deleted_nodes.len() + deleted_edges.len()) as u64; - write_u64(&mut w, count)?; + let nodes = memtable.nodes(); + let edges = memtable.edges(); + let degree_entries = degree_overlay.map(DegreeOverlaySnapshot::sorted_entries); - // Write node tombstones (sorted by ID for determinism) - let mut node_entries: Vec<(u64, &TombstoneEntry)> = - deleted_nodes.iter().map(|(&id, ts)| (id, ts)).collect(); - node_entries.sort_unstable_by_key(|&(id, _)| id); - for (id, ts) in node_entries { - write_u8(&mut w, 0)?; // kind = node - write_u64(&mut w, id)?; - w.write_all(&ts.deleted_at.to_le_bytes())?; - write_u64(&mut w, ts.last_write_seq)?; - } + let (flush_result, stats_core_result) = engine_cpu_join( + || -> Result< + ( + FlushComponentBuildSession, + SegmentComponentSourceGroups, + DeclaredIndexStatsEvidence, + ), + EngineError, + > { + let mut component_session = FlushComponentBuildSession::create(); + let mut core_writer = + PackedCoreWriter::create(seg_dir, segment_id, FLUSH_COMPONENT_GENERATION)?; + + let (node_records, node_data) = core_writer.write_component( + SegmentComponentKind::NodeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.nodes", &[]), + |writer| write_nodes_payload(writer, &nodes), + )?; - // Write edge tombstones (sorted by ID for determinism) - let mut edge_entries: Vec<(u64, &TombstoneEntry)> = - deleted_edges.iter().map(|(&id, ts)| (id, ts)).collect(); - edge_entries.sort_unstable_by_key(|&(id, _)| id); - for (id, ts) in edge_entries { - write_u8(&mut w, 1)?; // kind = edge - write_u64(&mut w, id)?; - w.write_all(&ts.deleted_at.to_le_bytes())?; - write_u64(&mut w, ts.last_write_seq)?; - } + let (edge_records, edge_data) = core_writer.write_component( + SegmentComponentKind::EdgeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.edges", &[]), + |writer| write_edges_payload(writer, &edges), + )?; - w.flush()?; - w.get_ref().sync_all()?; - Ok(()) -} + let (node_meta, _) = core_writer.write_component( + SegmentComponentKind::NodeMetadata, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryMetadata, + Vec::new(), + component_fingerprint("flush.node_meta", &[]), + |writer| write_node_meta_payload(writer, &node_data, &nodes), + )?; -// --- Record encoding helpers --- + let (edge_meta, _) = core_writer.write_component( + SegmentComponentKind::EdgeMetadata, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryMetadata, + Vec::new(), + component_fingerprint("flush.edge_meta", &[]), + |writer| write_edge_meta_payload(writer, &edge_data, &edges), + )?; -fn encode_node_record_into(buf: &mut Vec, node: &NodeRecord) -> Result<(), EngineError> { - buf.clear(); - // Note: node.id is NOT written here. It's already in the index. - buf.extend_from_slice(&node.type_id.to_le_bytes()); - let key_bytes = node.key.as_bytes(); - if key_bytes.len() > u16::MAX as usize { - return Err(EngineError::SerializationError(format!( - "node key exceeds maximum length of {} bytes", - u16::MAX - ))); + let (tombstones, _) = core_writer.write_component( + SegmentComponentKind::Tombstones, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryMetadata, + Vec::new(), + component_fingerprint("flush.tombstones", &[]), + |writer| { + write_tombstones_payload( + writer, + &memtable.deleted_nodes(), + &memtable.deleted_edges(), + ) + }, + )?; + + let source_records = vec![ + node_records.clone(), + edge_records.clone(), + node_meta.clone(), + edge_meta.clone(), + tombstones.clone(), + ]; + let source_groups = segment_source_groups_from_records( + segment_id, + nodes.len() as u64, + edges.len() as u64, + &source_records, + )?; + let (vector_records, dense_points) = + write_node_vector_source_components( + &mut core_writer, + &node_data, + &nodes, + source_groups.node_source, + )?; + + let source_groups = compute_flush_source_groups( + segment_id, + nodes.len() as u64, + edges.len() as u64, + &node_records, + &edge_records, + &node_meta, + &edge_meta, + &tombstones, + &vector_records, + )?; + + let (dependent_records, declared_evidence) = write_flush_dependent_components( + seg_dir, + segment_id, + &mut core_writer, + memtable, + &nodes, + &edges, + degree_entries.as_deref(), + dense_config, + dense_points, + secondary_indexes, + source_groups, + )?; + component_session.extend(dependent_records); + component_session.extend(core_writer.finish()?); + Ok((component_session, source_groups, declared_evidence)) + }, + || build_flush_stats_core_partial(&nodes, &edges, secondary_indexes), + ); + let (mut component_session, source_groups, declared_evidence) = flush_result?; + + if let Ok(core_partial) = stats_core_result { + let stats = assemble_flush_stats_from_partials( + segment_id, + secondary_indexes, + core_partial, + declared_evidence, + ); + if let Ok(Some(payload)) = planner_stats_sidecar_payload(stats) { + if let Ok((record, _)) = write_flush_component( + seg_dir, + segment_id, + PLANNER_STATS_FILENAME, + SegmentComponentKind::PlannerStats, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::PlannerStatsUnavailable, + }, + ComponentTrustClass::OptionalAdvisoryStats, + planner_stats_component_dependencies( + source_groups.segment_data_id, + secondary_indexes, + ), + planner_stats_component_fingerprint(secondary_indexes), + |writer| { + writer.write_all(&payload)?; + Ok(()) + }, + ) { + component_session.push(record); + } + } } - buf.extend_from_slice(&(key_bytes.len() as u16).to_le_bytes()); - buf.extend_from_slice(key_bytes); - buf.extend_from_slice(&node.created_at.to_le_bytes()); - buf.extend_from_slice(&node.updated_at.to_le_bytes()); - buf.extend_from_slice(&node.weight.to_le_bytes()); - let props_bytes = rmp_serde::to_vec(&node.props) - .map_err(|e| EngineError::SerializationError(e.to_string()))?; - buf.extend_from_slice(&(props_bytes.len() as u32).to_le_bytes()); - buf.extend_from_slice(&props_bytes); - Ok(()) -} -fn encode_edge_record_into(buf: &mut Vec, edge: &EdgeRecord) -> Result<(), EngineError> { - buf.clear(); - // Note: edge.id is NOT written here. It's already in the index. - buf.extend_from_slice(&edge.from.to_le_bytes()); - buf.extend_from_slice(&edge.to.to_le_bytes()); - buf.extend_from_slice(&edge.type_id.to_le_bytes()); - buf.extend_from_slice(&edge.created_at.to_le_bytes()); - buf.extend_from_slice(&edge.updated_at.to_le_bytes()); - buf.extend_from_slice(&edge.weight.to_le_bytes()); - buf.extend_from_slice(&edge.valid_from.to_le_bytes()); - buf.extend_from_slice(&edge.valid_to.to_le_bytes()); - let props_bytes = rmp_serde::to_vec(&edge.props) - .map_err(|e| EngineError::SerializationError(e.to_string()))?; - buf.extend_from_slice(&(props_bytes.len() as u32).to_le_bytes()); - buf.extend_from_slice(&props_bytes); - Ok(()) -} + validate_required_components_before_manifest(segment_id, &component_session.records)?; + component_session.fsync_component_parent_dirs(seg_dir)?; + fsync_dir(seg_dir)?; + let mut records = component_session.records.clone(); + sort_component_records_for_manifest(&mut records); + let manifest = SegmentComponentManifestV1 { + format_version: SEGMENT_COMPONENT_MANIFEST_PAYLOAD_VERSION, + segment_format_version: SEGMENT_FORMAT_VERSION, + segment_id, + generation: FLUSH_COMPONENT_GENERATION, + built_at_ms: current_time_millis(), + build_kind: SegmentComponentBuildKind::Flush, + segment_data_id: source_groups.segment_data_id, + node_count: nodes.len() as u64, + edge_count: edges.len() as u64, + components: records, + unknown_optional_components: Vec::new(), + }; + write_segment_component_manifest(seg_dir, &manifest)?; + fsync_dir(seg_dir)?; -// --- V5 metadata sidecar writers --- + Ok(SegmentInfo { + id: segment_id, + node_count: nodes.len() as u64, + edge_count: edges.len() as u64, + segment_format_version: SEGMENT_FORMAT_VERSION, + segment_data_id: source_groups.segment_data_id, + }) +} -/// Write metadata sidecars for node and edge records. -/// -/// `node_data` and `edge_data` are (id, data_offset, data_len) tuples sorted by id, -/// matching the actual byte positions in nodes.dat/edges.dat. -pub(crate) fn write_sidecars( +pub(crate) fn finalize_compaction_segment( seg_dir: &Path, - node_data: &[(u64, u64, u32)], - edge_data: &[(u64, u64, u32)], - nodes: &NodeIdMap, - edges: &NodeIdMap, -) -> Result, EngineError> { - write_node_meta(seg_dir, node_data, nodes)?; - let dense_points = write_node_vector_sidecars(seg_dir, node_data, nodes)?; - write_edge_meta(seg_dir, edge_data, edges)?; - Ok(dense_points) + segment_id: u64, + node_count: u64, + edge_count: u64, + records: Vec, +) -> Result { + let source_groups = + segment_source_groups_from_records(segment_id, node_count, edge_count, &records)?; + let component_session = FlushComponentBuildSession { records }; + validate_required_components_before_manifest(segment_id, &component_session.records)?; + component_session.fsync_component_parent_dirs(seg_dir)?; + fsync_dir(seg_dir)?; + let mut records = component_session.records.clone(); + sort_component_records_for_manifest(&mut records); + let manifest = SegmentComponentManifestV1 { + format_version: SEGMENT_COMPONENT_MANIFEST_PAYLOAD_VERSION, + segment_format_version: SEGMENT_FORMAT_VERSION, + segment_id, + generation: FLUSH_COMPONENT_GENERATION, + built_at_ms: current_time_millis(), + build_kind: SegmentComponentBuildKind::Compaction, + segment_data_id: source_groups.segment_data_id, + node_count, + edge_count, + components: records, + unknown_optional_components: Vec::new(), + }; + write_segment_component_manifest(seg_dir, &manifest)?; + fsync_dir(seg_dir)?; + + Ok(SegmentInfo { + id: segment_id, + node_count, + edge_count, + segment_format_version: SEGMENT_FORMAT_VERSION, + segment_data_id: source_groups.segment_data_id, + }) } -/// node_meta.dat format: -/// [count: u64] -/// [entries: count × NodeMetaEntry, sorted by node_id] -/// -/// NodeMetaEntry (60 bytes): -/// node_id: u64, data_offset: u64, data_len: u32, type_id: u32, -/// updated_at: i64, weight: f32, key_len: u16, -/// prop_hash_offset: u64, prop_hash_count: u32, -/// last_write_seq: u64, reserved: u16 -/// -fn write_node_meta( +#[allow(clippy::too_many_arguments)] +fn compute_flush_source_groups( + segment_id: u64, + node_count: u64, + edge_count: u64, + node_records: &SegmentComponentRecordV1, + edge_records: &SegmentComponentRecordV1, + node_meta: &SegmentComponentRecordV1, + edge_meta: &SegmentComponentRecordV1, + tombstones: &SegmentComponentRecordV1, + vector_records: &[SegmentComponentRecordV1], +) -> Result { + let mut records = Vec::with_capacity(5 + vector_records.len()); + records.extend([ + node_records.clone(), + edge_records.clone(), + node_meta.clone(), + edge_meta.clone(), + tombstones.clone(), + ]); + records.extend_from_slice(vector_records); + segment_source_groups_from_records(segment_id, node_count, edge_count, &records) +} + +#[allow(clippy::too_many_arguments)] +fn write_flush_dependent_components( seg_dir: &Path, - node_data: &[(u64, u64, u32)], + segment_id: u64, + core_writer: &mut PackedCoreWriter, + memtable: &Memtable, nodes: &NodeIdMap, -) -> Result<(), EngineError> { - let meta_path = seg_dir.join("node_meta.dat"); - - let meta_file = File::create(&meta_path)?; - let mut meta_w = BufWriter::new(meta_file); + edges: &NodeIdMap, + degree_entries: Option<&[(u64, DegreeDelta)]>, + dense_config: Option<&DenseVectorConfig>, + dense_points: Vec, + secondary_indexes: &[SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result<(Vec, DeclaredIndexStatsEvidence), EngineError> { + let partitions = partition_secondary_indexes(secondary_indexes); + let (((node_output, edge_output), sparse_posting_records), built_hnsw) = engine_cpu_try_join( + || { + engine_cpu_try_join( + || { + engine_cpu_try_join( + || { + write_flush_node_index_components( + seg_dir, + segment_id, + memtable, + nodes, + &partitions.node_eq, + &partitions.node_range, + source_groups, + ) + }, + || { + write_flush_edge_index_components( + seg_dir, + segment_id, + memtable, + edges, + degree_entries, + &partitions.edge_eq, + &partitions.edge_range, + source_groups, + ) + }, + ) + }, + || write_flush_sparse_posting_components(seg_dir, segment_id, nodes, source_groups), + ) + }, + || maybe_build_dense_hnsw(dense_points, dense_config), + )?; - let count = node_data.len() as u64; - write_u64(&mut meta_w, count)?; + let mut dense_hnsw_records = write_flush_prebuilt_dense_hnsw_components( + seg_dir, + segment_id, + dense_config, + built_hnsw, + source_groups, + )?; - for &(node_id, data_offset, data_len) in node_data { - let node = nodes.get(&node_id).ok_or_else(|| { - EngineError::CorruptRecord(format!("node {} not found for sidecar", node_id)) - })?; + emit_flush_node_index_components(core_writer, source_groups, &node_output)?; + emit_flush_edge_index_components(core_writer, source_groups, &edge_output)?; + + let mut records = Vec::new(); + let mut declared_evidence = node_output.declared_evidence; + declared_evidence.extend(edge_output.declared_evidence); + declared_evidence.sort(); + records.extend(node_output.external_records); + records.extend(edge_output.external_records); + records.append(&mut dense_hnsw_records); + records.extend(sparse_posting_records); + Ok((records, declared_evidence)) +} - // Write node_meta entry (60 bytes) - write_u64(&mut meta_w, node_id)?; - write_u64(&mut meta_w, data_offset)?; - write_u32(&mut meta_w, data_len)?; - write_u32(&mut meta_w, node.type_id)?; - meta_w.write_all(&node.updated_at.to_le_bytes())?; - meta_w.write_all(&node.weight.to_le_bytes())?; - write_u16(&mut meta_w, node.key.len() as u16)?; - write_u64(&mut meta_w, 0)?; - write_u32(&mut meta_w, 0)?; - write_u64(&mut meta_w, node.last_write_seq)?; - write_u16(&mut meta_w, 0)?; // reserved - } +fn write_flush_node_index_components<'a>( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + nodes: &'a NodeIdMap, + node_eq_indexes: &[&SecondaryIndexManifestEntry], + node_range_indexes: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result, EngineError> { + let mut records = Vec::with_capacity(node_eq_indexes.len() + node_range_indexes.len()); + let key_index = prepare_key_index_payload(nodes)?; + let label_node_index = memtable.label_node_index(); + let node_label_index = prepare_label_posting_index_payload(&label_node_index); + + let mut evidence = write_flush_declared_equality_components( + seg_dir, + segment_id, + memtable, + node_eq_indexes, + source_groups, + &mut records, + )?; + evidence.extend(write_flush_declared_range_components( + seg_dir, + segment_id, + memtable, + node_range_indexes, + source_groups, + &mut records, + )?); + + let time_node_index = memtable.time_node_index(); + let timestamp_index = prepare_timestamp_index_payload(&time_node_index); + + evidence.sort(); + Ok(FlushNodeIndexOutput { + key_index, + node_label_index, + timestamp_index, + external_records: records, + declared_evidence: evidence, + }) +} - meta_w.flush()?; - meta_w.get_ref().sync_all()?; +fn emit_flush_node_index_components( + core_writer: &mut PackedCoreWriter, + source_groups: SegmentComponentSourceGroups, + output: &FlushNodeIndexOutput<'_>, +) -> Result<(), EngineError> { + let node_source_dep = vec![source_group_dependency( + SegmentSourceGroupKind::NodeSource, + source_groups.node_source, + )]; + core_writer.write_component( + SegmentComponentKind::KeyIndex, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + node_source_dep.clone(), + component_fingerprint("flush.key_index", &[]), + |writer| write_key_index_plan_payload(writer, &output.key_index), + )?; + core_writer.write_component( + SegmentComponentKind::NodeLabelIndex, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + node_source_dep.clone(), + component_fingerprint("flush.node_label_index", &[]), + |writer| write_label_posting_index_plan_payload(writer, &output.node_label_index), + )?; + core_writer.write_component( + SegmentComponentKind::TimestampIndex, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + node_source_dep, + component_fingerprint("flush.timestamp_index", &[]), + |writer| write_timestamp_index_plan_payload(writer, &output.timestamp_index), + )?; Ok(()) } -fn write_node_vector_sidecars( +fn write_flush_declared_equality_components( seg_dir: &Path, - node_data: &[(u64, u64, u32)], - nodes: &NodeIdMap, -) -> Result, EngineError> { - let mut has_dense = false; - let mut has_sparse = false; - for &(node_id, _, _) in node_data { - let node = nodes.get(&node_id).ok_or_else(|| { - EngineError::CorruptRecord(format!("node {} not found for vector sidecar", node_id)) - })?; - has_dense |= node.dense_vector.is_some(); - has_sparse |= node.sparse_vector.is_some(); + segment_id: u64, + memtable: &Memtable, + eq_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, + records: &mut Vec, +) -> Result { + if eq_entries.is_empty() { + return Ok(DeclaredIndexStatsEvidence::default()); } - if !has_dense && !has_sparse { - return Ok(Vec::new()); + let secondary_eq_state = memtable.secondary_eq_state(); + let mut evidence = DeclaredIndexStatsEvidence::default(); + for entry in eq_entries { + let mut groups = BTreeMap::new(); + if let Some(values) = secondary_eq_state.get(&entry.index_id) { + for (&value_hash, ids) in values { + let mut sorted_ids: Vec = ids.iter().copied().collect(); + sorted_ids.sort_unstable(); + groups.insert(value_hash, sorted_ids); + } + } + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_flush_component( + seg_dir, + segment_id, + &format!( + "{}/node_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.node_prop_eq", &[entry.index_id]), + |writer| write_node_prop_eq_sidecar_payload(writer, &groups), + )?; + records.push(record); + if entry.state == SecondaryIndexState::Ready { + evidence + .equality_index_stats + .push(equality_index_stats_from_written_groups(entry, &groups)); + } } + evidence.sort(); + Ok(evidence) +} - let meta_file = File::create(seg_dir.join(NODE_VECTOR_META_FILENAME))?; - let mut meta_w = BufWriter::new(meta_file); - write_u64(&mut meta_w, node_data.len() as u64)?; - - let mut dense_w = if has_dense { - Some(BufWriter::new(File::create( - seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME), - )?)) - } else { - None - }; - let mut sparse_w = if has_sparse { - Some(BufWriter::new(File::create( - seg_dir.join(NODE_SPARSE_VECTOR_BLOB_FILENAME), - )?)) - } else { - None - }; +fn write_flush_declared_range_components( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + range_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, + records: &mut Vec, +) -> Result { + if range_entries.is_empty() { + return Ok(DeclaredIndexStatsEvidence::default()); + } - let mut dense_offset = 0u64; - let mut sparse_offset = 0u64; - let mut dense_points = Vec::new(); + let secondary_range_state = memtable.secondary_range_state(); + let mut evidence = DeclaredIndexStatsEvidence::default(); + for entry in range_entries { + let sidecar_entries: Vec<(u64, u64)> = secondary_range_state + .get(&entry.index_id) + .map(|entries| entries.iter().copied().collect()) + .unwrap_or_default(); + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_flush_component( + seg_dir, + segment_id, + &format!( + "{}/node_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.node_prop_range", &[entry.index_id]), + |writer| write_node_prop_range_sidecar_payload(writer, &sidecar_entries), + )?; + records.push(record); + if entry.state == SecondaryIndexState::Ready { + evidence + .range_index_stats + .push(range_index_stats_from_written_entries( + entry, + &sidecar_entries, + )); + } + } + evidence.sort(); + Ok(evidence) +} - for &(node_id, _, _) in node_data { - let node = nodes.get(&node_id).ok_or_else(|| { - EngineError::CorruptRecord(format!("node {} not found for vector sidecar", node_id)) - })?; +#[allow(clippy::too_many_arguments)] +fn write_flush_edge_index_components( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + edges: &NodeIdMap, + degree_entries: Option<&[(u64, DegreeDelta)]>, + edge_eq_indexes: &[&SecondaryIndexManifestEntry], + edge_range_indexes: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result { + let mut records = Vec::with_capacity( + (if degree_entries.is_some() { 1 } else { 0 }) + + edge_eq_indexes.len() + + edge_range_indexes.len(), + ); + let adj_out = prepare_adjacency_payloads(memtable.adj_out()); + let adj_in = prepare_adjacency_payloads(memtable.adj_in()); + let label_edge_index = memtable.label_edge_index(); + let edge_label_index = prepare_label_posting_index_payload(&label_edge_index); + let edge_triple_index = prepare_edge_triple_index_payload(edges); + let edge_metadata_indexes = prepare_flush_edge_metadata_index_components(edges); + + if let Some(entries) = degree_entries { + let dependencies = vec![source_group_dependency( + SegmentSourceGroupKind::DegreeSource, + source_groups.degree_source, + )]; + let (record, _) = write_flush_component( + seg_dir, + segment_id, + DEGREE_DELTA_FILENAME, + SegmentComponentKind::DegreeDelta, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::AdjacencyWalk, + }, + ComponentTrustClass::OptionalExactAccelerator, + dependencies, + component_fingerprint("flush.degree_delta", &[]), + |writer| write_sorted_degree_delta_sidecar_payload(writer, entries), + )?; + records.push(record); + } - let mut flags = 0u8; - let mut dense_len = 0u32; - let mut sparse_len = 0u32; - let mut entry_dense_offset = 0u64; - let mut entry_sparse_offset = 0u64; + let mut evidence = write_flush_declared_edge_equality_components( + seg_dir, + segment_id, + memtable, + edge_eq_indexes, + source_groups, + &mut records, + )?; + evidence.extend(write_flush_declared_edge_range_components( + seg_dir, + segment_id, + memtable, + edge_range_indexes, + source_groups, + &mut records, + )?); + evidence.sort(); + + Ok(FlushEdgeIndexOutput { + adj_out, + adj_in, + edge_label_index, + edge_triple_index, + edge_metadata_indexes, + external_records: records, + declared_evidence: evidence, + }) +} - if let Some(values) = node.dense_vector.as_ref() { - flags |= NODE_VECTOR_FLAG_DENSE; - dense_len = values.len() as u32; - entry_dense_offset = dense_offset; - dense_points.push(DensePointInput { - node_id, - dense_vector_offset: entry_dense_offset, - values: values.clone(), - }); - let w = dense_w.as_mut().expect("dense blob writer must exist"); - for &value in values { - w.write_all(&value.to_le_bytes())?; - } - dense_offset = dense_offset - .checked_add(values.len() as u64 * DENSE_VECTOR_VALUE_SIZE) - .ok_or_else(|| { - EngineError::CorruptRecord("dense vector blob offset overflow".into()) - })?; - } +fn write_flush_declared_edge_equality_components( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + eq_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, + records: &mut Vec, +) -> Result { + if eq_entries.is_empty() { + return Ok(DeclaredIndexStatsEvidence::default()); + } - if let Some(values) = node.sparse_vector.as_ref() { - flags |= NODE_VECTOR_FLAG_SPARSE; - sparse_len = values.len() as u32; - entry_sparse_offset = sparse_offset; - let w = sparse_w.as_mut().expect("sparse blob writer must exist"); - for &(dimension_id, weight) in values { - write_u32(w, dimension_id)?; - w.write_all(&weight.to_le_bytes())?; + let secondary_eq_state = memtable.secondary_eq_state(); + let mut evidence = DeclaredIndexStatsEvidence::default(); + for entry in eq_entries { + let mut groups = BTreeMap::new(); + if let Some(values) = secondary_eq_state.get(&entry.index_id) { + for (&value_hash, ids) in values { + let mut sorted_ids: Vec = ids.iter().copied().collect(); + sorted_ids.sort_unstable(); + groups.insert(value_hash, sorted_ids); } - sparse_offset = sparse_offset - .checked_add(values.len() as u64 * SPARSE_VECTOR_ENTRY_SIZE) - .ok_or_else(|| { - EngineError::CorruptRecord("sparse vector blob offset overflow".into()) - })?; } - - write_u8(&mut meta_w, flags)?; - meta_w.write_all(&[0u8; 3])?; - write_u64(&mut meta_w, entry_dense_offset)?; - write_u32(&mut meta_w, dense_len)?; - write_u64(&mut meta_w, entry_sparse_offset)?; - write_u32(&mut meta_w, sparse_len)?; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_flush_component( + seg_dir, + segment_id, + &format!( + "{}/edge_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.edge_prop_eq", &[entry.index_id]), + |writer| write_node_prop_eq_sidecar_payload(writer, &groups), + )?; + records.push(record); + if entry.state == SecondaryIndexState::Ready { + evidence + .equality_index_stats + .push(equality_index_stats_from_written_groups(entry, &groups)); + } } + evidence.sort(); + Ok(evidence) +} - meta_w.flush()?; - meta_w.get_ref().sync_all()?; - - if let Some(mut w) = dense_w { - w.flush()?; - w.get_ref().sync_all()?; - } - if let Some(mut w) = sparse_w { - w.flush()?; - w.get_ref().sync_all()?; +fn write_flush_declared_edge_range_components( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + range_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, + records: &mut Vec, +) -> Result { + if range_entries.is_empty() { + return Ok(DeclaredIndexStatsEvidence::default()); } - Ok(dense_points) + let secondary_range_state = memtable.secondary_range_state(); + let mut evidence = DeclaredIndexStatsEvidence::default(); + for entry in range_entries { + let sidecar_entries: Vec<(u64, u64)> = secondary_range_state + .get(&entry.index_id) + .map(|entries| entries.iter().copied().collect()) + .unwrap_or_default(); + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_flush_component( + seg_dir, + segment_id, + &format!( + "{}/edge_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.edge_prop_range", &[entry.index_id]), + |writer| write_node_prop_range_sidecar_payload(writer, &sidecar_entries), + )?; + records.push(record); + if entry.state == SecondaryIndexState::Ready { + evidence + .range_index_stats + .push(range_index_stats_from_written_entries( + entry, + &sidecar_entries, + )); + } + } + evidence.sort(); + Ok(evidence) } -/// edge_meta.dat format: -/// [count: u64] -/// [entries: count × EdgeMetaEntry, sorted by edge_id] -/// -/// EdgeMetaEntry (80 bytes): -/// edge_id: u64, data_offset: u64, data_len: u32, -/// from: u64, to: u64, type_id: u32, -/// updated_at: i64, weight: f32, -/// valid_from: i64, valid_to: i64, -/// last_write_seq: u64, reserved: u32 -fn write_edge_meta( - seg_dir: &Path, - edge_data: &[(u64, u64, u32)], +fn prepare_flush_edge_metadata_index_components( edges: &NodeIdMap, -) -> Result<(), EngineError> { - let path = seg_dir.join("edge_meta.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - - let count = edge_data.len() as u64; - write_u64(&mut w, count)?; +) -> EdgeMetadataIndexEntries { + let mut entries = EdgeMetadataIndexEntries::with_capacity(edges.len()); + for edge in edges.values() { + entries.push( + edge.label_id, + edge.updated_at, + edge.weight, + edge.valid_from, + edge.valid_to, + edge.id, + ); + } + entries.sort_all(); + entries +} - for &(edge_id, data_offset, data_len) in edge_data { - let edge = edges.get(&edge_id).ok_or_else(|| { - EngineError::CorruptRecord(format!("edge {} not found for sidecar", edge_id)) - })?; +fn emit_flush_edge_index_components( + core_writer: &mut PackedCoreWriter, + source_groups: SegmentComponentSourceGroups, + output: &FlushEdgeIndexOutput, +) -> Result<(), EngineError> { + let edge_source_dep = vec![source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + )]; + core_writer.write_component( + SegmentComponentKind::EdgeLabelIndex, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + edge_source_dep.clone(), + component_fingerprint("flush.edge_label_index", &[]), + |writer| write_label_posting_index_plan_payload(writer, &output.edge_label_index), + )?; + core_writer.write_component( + SegmentComponentKind::EdgeTripleIndex, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + edge_source_dep.clone(), + component_fingerprint("flush.edge_triple_index", &[]), + |writer| write_edge_triple_index_plan_payload(writer, &output.edge_triple_index), + )?; + core_writer.write_component( + SegmentComponentKind::AdjOutPostings, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + edge_source_dep.clone(), + component_fingerprint("flush.adj_out_dat", &[]), + |writer| write_adjacency_postings_payload(writer, &output.adj_out), + )?; + core_writer.write_component( + SegmentComponentKind::AdjOutIndex, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + edge_source_dep.clone(), + component_fingerprint("flush.adj_out_idx", &[]), + |writer| write_adjacency_index_payload(writer, &output.adj_out), + )?; + core_writer.write_component( + SegmentComponentKind::AdjInPostings, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + edge_source_dep.clone(), + component_fingerprint("flush.adj_in_dat", &[]), + |writer| write_adjacency_postings_payload(writer, &output.adj_in), + )?; + core_writer.write_component( + SegmentComponentKind::AdjInIndex, + ComponentRequirement::Required, + ComponentTrustClass::CoreMaintainedIndex, + edge_source_dep, + component_fingerprint("flush.adj_in_idx", &[]), + |writer| write_adjacency_index_payload(writer, &output.adj_in), + )?; - write_u64(&mut w, edge_id)?; - write_u64(&mut w, data_offset)?; - write_u32(&mut w, data_len)?; - write_u64(&mut w, edge.from)?; - write_u64(&mut w, edge.to)?; - write_u32(&mut w, edge.type_id)?; - w.write_all(&edge.updated_at.to_le_bytes())?; - w.write_all(&edge.weight.to_le_bytes())?; - w.write_all(&edge.valid_from.to_le_bytes())?; - w.write_all(&edge.valid_to.to_le_bytes())?; - write_u64(&mut w, edge.last_write_seq)?; - write_u32(&mut w, 0)?; // reserved - } + let dependencies = vec![source_group_dependency( + SegmentSourceGroupKind::EdgeMetadataSource, + source_groups.edge_metadata_source, + )]; - w.flush()?; - w.get_ref().sync_all()?; - Ok(()) -} + core_writer.write_component( + SegmentComponentKind::EdgeWeightIndex, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::MetadataScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies.clone(), + component_fingerprint("flush.edge_weight_index", &[]), + |writer| { + write_edge_weight_metadata_index_payload(writer, &output.edge_metadata_indexes.weight) + }, + )?; + core_writer.write_component( + SegmentComponentKind::EdgeUpdatedAtIndex, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::MetadataScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies.clone(), + component_fingerprint("flush.edge_updated_at_index", &[]), + |writer| { + write_edge_i64_metadata_index_payload(writer, &output.edge_metadata_indexes.updated_at) + }, + )?; + core_writer.write_component( + SegmentComponentKind::EdgeValidFromIndex, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::MetadataScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies.clone(), + component_fingerprint("flush.edge_valid_from_index", &[]), + |writer| { + write_edge_i64_metadata_index_payload(writer, &output.edge_metadata_indexes.valid_from) + }, + )?; + core_writer.write_component( + SegmentComponentKind::EdgeValidToIndex, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::MetadataScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.edge_valid_to_index", &[]), + |writer| { + write_edge_i64_metadata_index_payload(writer, &output.edge_metadata_indexes.valid_to) + }, + )?; -/// Write segment format version file: 4-byte magic + 4-byte version (little-endian). -fn write_format_version(seg_dir: &Path) -> Result<(), EngineError> { - let path = seg_dir.join("format.ver"); - let mut data = Vec::with_capacity(8); - data.extend_from_slice(&SEGMENT_MAGIC); - data.extend_from_slice(&SEGMENT_FORMAT_VERSION.to_le_bytes()); - fs::write(path, &data)?; Ok(()) } -/// Fsync the directory to ensure metadata (file creation) is durable. -/// No-op on Windows. NTFS doesn't support directory fsync via File::open(). -fn fsync_dir(dir: &Path) -> Result<(), EngineError> { - #[cfg(not(target_os = "windows"))] - { - let d = File::open(dir)?; - d.sync_all()?; +fn maybe_build_dense_hnsw( + dense_points: Vec, + dense_config: Option<&DenseVectorConfig>, +) -> Result, EngineError> { + let Some(config) = dense_config else { + return Ok(None); + }; + if dense_points.is_empty() { + return Ok(None); } - #[cfg(target_os = "windows")] - let _ = dir; - Ok(()) + build_dense_hnsw_from_points(dense_points, config) } -/// Return the segment directory path for a given segment ID within a db directory. -pub fn segment_dir(db_dir: &Path, segment_id: u64) -> PathBuf { - db_dir - .join("segments") - .join(format!("seg_{:04}", segment_id)) +fn write_flush_prebuilt_dense_hnsw_components( + seg_dir: &Path, + segment_id: u64, + dense_config: Option<&DenseVectorConfig>, + built_hnsw: Option, + source_groups: SegmentComponentSourceGroups, +) -> Result, EngineError> { + let Some(config) = dense_config else { + return Ok(Vec::new()); + }; + let Some(built) = built_hnsw else { + return Ok(Vec::new()); + }; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::DenseVectorSource, + source_groups.dense_vector_source, + ), + ComponentDependencyV1::DenseVectorConfig { + fingerprint: dense_config_fingerprint(Some(config)), + }, + ]; + let (meta_record, graph_record, _) = write_flush_component_pair( + seg_dir, + segment_id, + DENSE_HNSW_META_FILENAME, + SegmentComponentKind::DenseHnswMetadata, + component_fingerprint("flush.dense_hnsw_meta", &[]), + DENSE_HNSW_GRAPH_FILENAME, + SegmentComponentKind::DenseHnswGraph, + component_fingerprint("flush.dense_hnsw_graph", &[]), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::ExactVectorScan, + }, + ComponentTrustClass::OptionalApproximateAccelerator, + dependencies, + |meta_writer, graph_writer| { + write_prebuilt_hnsw_to_writers(meta_writer, graph_writer, config, &built) + }, + )?; + Ok(vec![meta_record, graph_record]) } -/// Return the temporary segment directory path (used during flush before atomic rename). -pub fn segment_tmp_dir(db_dir: &Path, segment_id: u64) -> PathBuf { - db_dir - .join("segments") - .join(format!("seg_{:04}.tmp", segment_id)) +fn write_flush_sparse_posting_components( + seg_dir: &Path, + segment_id: u64, + nodes: &NodeIdMap, + source_groups: SegmentComponentSourceGroups, +) -> Result, EngineError> { + let groups = sparse_posting_groups_from_nodes(nodes)?; + if groups.is_empty() { + return Ok(Vec::new()); + } + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::SparseVectorSource, + source_groups.sparse_vector_source, + ), + ComponentDependencyV1::SparseVectorConfig { + fingerprint: component_fingerprint("sparse_vector_config", &[]), + }, + ]; + let (index_record, postings_record, _) = write_flush_component_pair( + seg_dir, + segment_id, + SPARSE_POSTING_INDEX_FILENAME, + SegmentComponentKind::SparsePostingIndex, + component_fingerprint("flush.sparse_posting_index", &[]), + SPARSE_POSTINGS_FILENAME, + SegmentComponentKind::SparsePostings, + component_fingerprint("flush.sparse_postings", &[]), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::ExactVectorScan, + }, + ComponentTrustClass::OptionalApproximateAccelerator, + dependencies, + |index_writer, postings_writer| { + write_sparse_posting_files_to_writers(index_writer, postings_writer, &groups) + }, + )?; + Ok(vec![index_record, postings_record]) } -// --- Fast-merge compaction support --- - -pub(crate) struct FastMergeCopyInfo { - pub orig_data_start: u64, - pub new_data_base: u64, +#[cfg(test)] +pub(crate) fn write_node_prop_eq_sidecar_to_path( + path: &Path, + groups: &BTreeMap>, +) -> Result<(), EngineError> { + let file = File::create(path)?; + let mut writer = BufWriter::new(file); + write_node_prop_eq_sidecar_payload(&mut writer, groups)?; + writer.flush()?; + writer.get_ref().sync_all()?; + Ok(()) } -/// Write merged nodes.dat by binary copy from multiple non-overlapping segments. -/// -/// Instead of deserializing and re-serializing every record, this copies raw -/// record bytes directly from mmap'd input segments and rebuilds the merged -/// index with adjusted offsets. Record lengths are derived from the source -/// nodes.dat index/data layout, which lets the fast path cross-check sidecar -/// metadata later instead of trusting it blindly. -/// -/// Returns per-segment offset rebasing info so compaction metadata can compute -/// merged `data_offset` values directly from sidecars without a second data scan. -pub(crate) fn write_merged_nodes_dat( +pub(crate) fn publish_node_prop_eq_sidecar_component( seg_dir: &Path, - segments: &[Arc], -) -> Result, EngineError> { - let path = seg_dir.join("nodes.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); + entry: &SecondaryIndexManifestEntry, + groups: &BTreeMap>, +) -> Result<(), EngineError> { + let manifest = read_segment_component_manifest(seg_dir)?; + let source_groups = segment_source_groups_from_records( + manifest.segment_id, + manifest.node_count, + manifest.edge_count, + &manifest.components, + )?; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + ), + secondary_declaration_dependency(entry), + ]; + refresh_optional_component_with_writer( + seg_dir, + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }, + &format!( + "{}/node_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.node_prop_eq", &[entry.index_id]), + |writer| write_node_prop_eq_sidecar_payload(writer, groups), + ) +} - let mut seg_info: Vec<(u64, usize, usize)> = Vec::with_capacity(segments.len()); - let mut total_count: u64 = 0; +fn write_node_prop_eq_sidecar_payload( + mut writer: &mut impl Write, + groups: &BTreeMap>, +) -> Result<(), EngineError> { + let entry_count = groups.len() as u64; + write_u64(&mut writer, entry_count)?; - for (seg_idx, seg) in segments.iter().enumerate() { - let mmap = seg.raw_nodes_mmap(); - if mmap.len() < 8 { - return Err(EngineError::CorruptRecord(format!( - "segment {} nodes.dat too short for count header: {} bytes", - seg.segment_id, - mmap.len() - ))); - } - let count = u64::from_le_bytes(mmap[0..8].try_into().unwrap()); - let index_bytes = (count as usize) - .checked_mul(NODE_INDEX_ENTRY_SIZE as usize) - .ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} node index size overflow for {} entries", - seg.segment_id, count - )) - })?; - let data_start = 8usize.checked_add(index_bytes).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} node data start overflow", - seg.segment_id - )) - })?; - if data_start > mmap.len() { - return Err(EngineError::CorruptRecord(format!( - "segment {} nodes.dat index exceeds file length: start={}, len={}", - seg.segment_id, - data_start, - mmap.len() - ))); - } - seg_info.push((count, data_start, mmap.len() - data_start)); - total_count = total_count.checked_add(count).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "total node count overflow while merging segment {} (index {})", - seg.segment_id, seg_idx - )) - })?; + let data_start = 8 + entry_count * SECONDARY_EQ_ENTRY_SIZE; + let mut data_offset = data_start; + for (&value_hash, ids) in groups { + write_u64(&mut writer, value_hash)?; + write_u64(&mut writer, data_offset)?; + write_u32(&mut writer, ids.len() as u32)?; + data_offset += ids.len() as u64 * 8; } - write_u64(&mut w, total_count)?; - - let merged_data_start = 8u64 - .checked_add( - total_count - .checked_mul(NODE_INDEX_ENTRY_SIZE) - .ok_or_else(|| { - EngineError::CorruptRecord("merged node index size overflow".into()) - })?, - ) - .ok_or_else(|| EngineError::CorruptRecord("merged node data start overflow".into()))?; - let mut cumulative_data_offset = merged_data_start; - let mut data_offsets: Vec = Vec::with_capacity(segments.len()); - for &(_, _, data_size) in &seg_info { - data_offsets.push(cumulative_data_offset); - cumulative_data_offset = cumulative_data_offset - .checked_add(data_size as u64) - .ok_or_else(|| EngineError::CorruptRecord("merged nodes.dat size overflow".into()))?; - } - let mut all_entries: Vec<(u64, u64)> = Vec::with_capacity(total_count as usize); - for (seg_idx, seg) in segments.iter().enumerate() { - let mmap = seg.raw_nodes_mmap(); - let (count, orig_data_start, _) = seg_info[seg_idx]; - if count == 0 { - continue; - } - - let offset_adj = data_offsets[seg_idx] - .checked_sub(orig_data_start as u64) - .ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} node offset adjustment underflow", - seg.segment_id - )) - })?; - - for i in 0..count as usize { - let entry_off = 8 + i * NODE_INDEX_ENTRY_SIZE as usize; - let node_id = u64::from_le_bytes(mmap[entry_off..entry_off + 8].try_into().unwrap()); - let old_offset = - u64::from_le_bytes(mmap[entry_off + 8..entry_off + 16].try_into().unwrap()); - let new_offset = old_offset.checked_add(offset_adj).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} node {} merged offset overflow", - seg.segment_id, node_id - )) - })?; - all_entries.push((node_id, new_offset)); + for ids in groups.values() { + for &node_id in ids { + write_u64(&mut writer, node_id)?; } } - all_entries.sort_unstable_by_key(|(id, _)| *id); - - for &(node_id, offset) in &all_entries { - write_u64(&mut w, node_id)?; - write_u64(&mut w, offset)?; - } - for (seg_idx, seg) in segments.iter().enumerate() { - let mmap = seg.raw_nodes_mmap(); - let (_, data_start, data_size) = seg_info[seg_idx]; - if data_size > 0 { - w.write_all(&mmap[data_start..data_start + data_size])?; - } - } + Ok(()) +} - w.flush()?; - w.get_ref().sync_all()?; - Ok(seg_info - .into_iter() - .zip(data_offsets) - .map(|((_, data_start, _), new_data_base)| FastMergeCopyInfo { - orig_data_start: data_start as u64, - new_data_base, - }) - .collect()) +#[cfg(test)] +pub(crate) fn write_node_prop_range_sidecar_to_path( + path: &Path, + entries: &[(u64, u64)], +) -> Result<(), EngineError> { + let file = File::create(path)?; + let mut writer = BufWriter::new(file); + write_node_prop_range_sidecar_payload(&mut writer, entries)?; + writer.flush()?; + writer.get_ref().sync_all()?; + Ok(()) } -/// Write merged edges.dat by binary copy from multiple non-overlapping segments. -/// Same approach as `write_merged_nodes_dat`. -/// -/// Returns per-segment offset rebasing info so compaction metadata can compute -/// merged `data_offset` values directly from sidecars without a second data scan. -pub(crate) fn write_merged_edges_dat( +pub(crate) fn publish_node_prop_range_sidecar_component( seg_dir: &Path, - segments: &[Arc], -) -> Result, EngineError> { - let path = seg_dir.join("edges.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - - let mut seg_info: Vec<(u64, usize, usize)> = Vec::with_capacity(segments.len()); - let mut total_count: u64 = 0; - - for (seg_idx, seg) in segments.iter().enumerate() { - let mmap = seg.raw_edges_mmap(); - if mmap.len() < 8 { - return Err(EngineError::CorruptRecord(format!( - "segment {} edges.dat too short for count header: {} bytes", - seg.segment_id, - mmap.len() - ))); - } - let count = u64::from_le_bytes(mmap[0..8].try_into().unwrap()); - let index_bytes = (count as usize) - .checked_mul(EDGE_INDEX_ENTRY_SIZE as usize) - .ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} edge index size overflow for {} entries", - seg.segment_id, count - )) - })?; - let data_start = 8usize.checked_add(index_bytes).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} edge data start overflow", - seg.segment_id - )) - })?; - if data_start > mmap.len() { - return Err(EngineError::CorruptRecord(format!( - "segment {} edges.dat index exceeds file length: start={}, len={}", - seg.segment_id, - data_start, - mmap.len() - ))); - } - seg_info.push((count, data_start, mmap.len() - data_start)); - total_count = total_count.checked_add(count).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "total edge count overflow while merging segment {} (index {})", - seg.segment_id, seg_idx - )) - })?; - } - - write_u64(&mut w, total_count)?; - - let merged_data_start = 8u64 - .checked_add( - total_count - .checked_mul(EDGE_INDEX_ENTRY_SIZE) - .ok_or_else(|| { - EngineError::CorruptRecord("merged edge index size overflow".into()) - })?, - ) - .ok_or_else(|| EngineError::CorruptRecord("merged edge data start overflow".into()))?; - let mut cumulative_data_offset = merged_data_start; - let mut data_offsets: Vec = Vec::with_capacity(segments.len()); - for &(_, _, data_size) in &seg_info { - data_offsets.push(cumulative_data_offset); - cumulative_data_offset = cumulative_data_offset - .checked_add(data_size as u64) - .ok_or_else(|| EngineError::CorruptRecord("merged edges.dat size overflow".into()))?; - } - - let mut all_entries: Vec<(u64, u64)> = Vec::with_capacity(total_count as usize); - for (seg_idx, seg) in segments.iter().enumerate() { - let mmap = seg.raw_edges_mmap(); - let (count, orig_data_start, _) = seg_info[seg_idx]; - if count == 0 { - continue; - } - - let offset_adj = data_offsets[seg_idx] - .checked_sub(orig_data_start as u64) - .ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} edge offset adjustment underflow", - seg.segment_id - )) - })?; + entry: &SecondaryIndexManifestEntry, + entries: &[(u64, u64)], +) -> Result<(), EngineError> { + let manifest = read_segment_component_manifest(seg_dir)?; + let source_groups = segment_source_groups_from_records( + manifest.segment_id, + manifest.node_count, + manifest.edge_count, + &manifest.components, + )?; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + ), + secondary_declaration_dependency(entry), + ]; + refresh_optional_component_with_writer( + seg_dir, + SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }, + &format!( + "{}/node_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.node_prop_range", &[entry.index_id]), + |writer| write_node_prop_range_sidecar_payload(writer, entries), + ) +} - for i in 0..count as usize { - let entry_off = 8 + i * EDGE_INDEX_ENTRY_SIZE as usize; - let edge_id = u64::from_le_bytes(mmap[entry_off..entry_off + 8].try_into().unwrap()); - let old_offset = - u64::from_le_bytes(mmap[entry_off + 8..entry_off + 16].try_into().unwrap()); - let new_offset = old_offset.checked_add(offset_adj).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "segment {} edge {} merged offset overflow", - seg.segment_id, edge_id - )) - })?; - all_entries.push((edge_id, new_offset)); - } - } - all_entries.sort_unstable_by_key(|(id, _)| *id); +pub(crate) fn publish_edge_prop_eq_sidecar_component( + seg_dir: &Path, + entry: &SecondaryIndexManifestEntry, + groups: &BTreeMap>, +) -> Result<(), EngineError> { + let manifest = read_segment_component_manifest(seg_dir)?; + let source_groups = segment_source_groups_from_records( + manifest.segment_id, + manifest.node_count, + manifest.edge_count, + &manifest.components, + )?; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + ), + secondary_declaration_dependency(entry), + ]; + refresh_optional_component_with_writer( + seg_dir, + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + }, + &format!( + "{}/edge_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("build.edge_prop_eq", &[entry.index_id]), + |writer| write_node_prop_eq_sidecar_payload(writer, groups), + ) +} - for &(edge_id, offset) in &all_entries { - write_u64(&mut w, edge_id)?; - write_u64(&mut w, offset)?; - } +pub(crate) fn publish_edge_prop_range_sidecar_component( + seg_dir: &Path, + entry: &SecondaryIndexManifestEntry, + entries: &[(u64, u64)], +) -> Result<(), EngineError> { + let manifest = read_segment_component_manifest(seg_dir)?; + let source_groups = segment_source_groups_from_records( + manifest.segment_id, + manifest.node_count, + manifest.edge_count, + &manifest.components, + )?; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + ), + secondary_declaration_dependency(entry), + ]; + refresh_optional_component_with_writer( + seg_dir, + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: entry.index_id, + }, + &format!( + "{}/edge_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("build.edge_prop_range", &[entry.index_id]), + |writer| write_node_prop_range_sidecar_payload(writer, entries), + ) +} - for (seg_idx, seg) in segments.iter().enumerate() { - let mmap = seg.raw_edges_mmap(); - let (_, data_start, data_size) = seg_info[seg_idx]; - if data_size > 0 { - w.write_all(&mmap[data_start..data_start + data_size])?; - } +fn write_node_prop_range_sidecar_payload( + mut writer: &mut impl Write, + entries: &[(u64, u64)], +) -> Result<(), EngineError> { + write_u64(&mut writer, entries.len() as u64)?; + for &(encoded_value, node_id) in entries { + write_u64(&mut writer, encoded_value)?; + write_u64(&mut writer, node_id)?; } - - w.flush()?; - w.get_ref().sync_all()?; - Ok(seg_info - .into_iter() - .zip(data_offsets) - .map(|((_, data_start, _), new_data_base)| FastMergeCopyInfo { - orig_data_start: data_start as u64, - new_data_base, - }) - .collect()) + Ok(()) } -/// Write nodes.dat by raw-copying only winning record byte spans from source segments. -/// -/// Used by V3 compaction: the planner has already decided which records win, -/// so we skip all dropped records entirely (never decode them). -/// -/// `winners` is sorted by node_id: `(node_id, seg_idx, data_offset, data_len)`. +/// Node records payload format: +/// [count: u64] +/// [index: (node_id: u64, offset: u64) × count, sorted by node_id] +/// [data: node records sequentially] /// -/// Returns Vec of `(node_id, new_data_offset, data_len)` matching the output file, -/// for sidecar writing. -pub(crate) fn write_v3_nodes_dat( - seg_dir: &Path, - segments: &[Arc], - winners: &[(u64, usize, u64, u32)], -) -> Result, EngineError> { - let path = seg_dir.join("nodes.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); +/// Returns Vec of (node_id, data_offset, data_len) sorted by node_id, +/// used by metadata writers to record raw byte spans. +fn write_nodes_payload( + mut w: &mut impl Write, + nodes: &NodeIdMap, +) -> Result { + // Sort nodes by ID for binary search in the index + let mut sorted: Vec<&NodeRecord> = nodes.values().collect(); + sorted.sort_by_key(|n| n.id); - let count = winners.len() as u64; + let count = sorted.len() as u64; write_u64(&mut w, count)?; - // Calculate data section start - let data_start = 8 + count * NODE_INDEX_ENTRY_SIZE; + // First pass: encode into reused buffer to collect sizes for offset table. + let mut buf = Vec::new(); + let mut sizes: Vec = Vec::with_capacity(sorted.len()); + for node in &sorted { + encode_node_record_into(&mut buf, node)?; + sizes.push(buf.len() as u64); + } - // Build index entries and output info - let mut node_data = Vec::with_capacity(winners.len()); + // Write index entries and collect data info for sidecars + let data_start = 8 + count * NODE_INDEX_ENTRY_SIZE; let mut data_offset = data_start; - for &(node_id, _, _, data_len) in winners { - // Write index entry: (node_id, offset) - write_u64(&mut w, node_id)?; + let mut node_data = Vec::with_capacity(sorted.len()); + for (i, node) in sorted.iter().enumerate() { + write_u64(&mut w, node.id)?; write_u64(&mut w, data_offset)?; - node_data.push((node_id, data_offset, data_len)); - data_offset += data_len as u64; + node_data.push((node.id, data_offset, sizes[i] as u32)); + data_offset += sizes[i]; } - // Write data section by copying raw bytes from source segments - for &(node_id, seg_idx, src_offset, data_len) in winners { - let mmap = segments[seg_idx].raw_nodes_mmap(); - let start = src_offset as usize; - let end = start.checked_add(data_len as usize).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "node {} data span offset overflow: start={}, len={}", - node_id, start, data_len - )) - })?; - if end > mmap.len() { - return Err(EngineError::CorruptRecord(format!( - "node {} data span [{}, {}) exceeds mmap length {}", - node_id, - start, - end, - mmap.len() - ))); - } - w.write_all(&mmap[start..end])?; + // Second pass: re-encode into reused buffer and write directly. + for node in &sorted { + encode_node_record_into(&mut buf, node)?; + w.write_all(&buf)?; } - w.flush()?; - w.get_ref().sync_all()?; Ok(node_data) } -/// Write edges.dat by raw-copying only winning record byte spans from source segments. -/// -/// Same approach as `write_v3_nodes_dat` but for edge records. -/// -/// `winners` is sorted by edge_id: `(edge_id, seg_idx, data_offset, data_len)`. +/// Edge records payload format: +/// [count: u64] +/// [index: (edge_id: u64, offset: u64) × count, sorted by edge_id] +/// [data: edge records sequentially] /// -/// Returns Vec of `(edge_id, new_data_offset, data_len)` matching the output file. -pub(crate) fn write_v3_edges_dat( - seg_dir: &Path, - segments: &[Arc], - winners: &[(u64, usize, u64, u32)], -) -> Result, EngineError> { - let path = seg_dir.join("edges.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); +/// Returns Vec of (edge_id, data_offset, data_len) sorted by edge_id, +/// used by metadata writers to record raw byte spans. +fn write_edges_payload( + mut w: &mut impl Write, + edges: &NodeIdMap, +) -> Result { + let mut sorted: Vec<&EdgeRecord> = edges.values().collect(); + sorted.sort_by_key(|e| e.id); - let count = winners.len() as u64; + let count = sorted.len() as u64; write_u64(&mut w, count)?; - let data_start = 8 + count * EDGE_INDEX_ENTRY_SIZE; - - let mut edge_data = Vec::with_capacity(winners.len()); + // First pass: encode into reused buffer to collect sizes for offset table. + let mut buf = Vec::new(); + let mut sizes: Vec = Vec::with_capacity(sorted.len()); + for edge in &sorted { + encode_edge_record_into(&mut buf, edge)?; + sizes.push(buf.len() as u64); + } + + let data_start = 8 + count * EDGE_INDEX_ENTRY_SIZE; let mut data_offset = data_start; - for &(edge_id, _, _, data_len) in winners { - write_u64(&mut w, edge_id)?; + let mut edge_data = Vec::with_capacity(sorted.len()); + for (i, edge) in sorted.iter().enumerate() { + write_u64(&mut w, edge.id)?; write_u64(&mut w, data_offset)?; - edge_data.push((edge_id, data_offset, data_len)); - data_offset += data_len as u64; + edge_data.push((edge.id, data_offset, sizes[i] as u32)); + data_offset += sizes[i]; } - for &(edge_id, seg_idx, src_offset, data_len) in winners { - let mmap = segments[seg_idx].raw_edges_mmap(); - let start = src_offset as usize; - let end = start.checked_add(data_len as usize).ok_or_else(|| { - EngineError::CorruptRecord(format!( - "edge {} data span offset overflow: start={}, len={}", - edge_id, start, data_len - )) - })?; - if end > mmap.len() { - return Err(EngineError::CorruptRecord(format!( - "edge {} data span [{}, {}) exceeds mmap length {}", - edge_id, - start, - end, - mmap.len() - ))); - } - w.write_all(&mmap[start..end])?; + // Second pass: re-encode into reused buffer and write directly. + for edge in &sorted { + encode_edge_record_into(&mut buf, edge)?; + w.write_all(&buf)?; } - w.flush()?; - w.get_ref().sync_all()?; Ok(edge_data) } -// ========================================================================== -// Metadata-driven compaction index writers (V3) -// ========================================================================== +// --- Varint helpers for adjacency delta encoding --- -/// Node metadata collected from source sidecars for metadata-driven index building. -pub(crate) struct CompactNodeMeta { - pub node_id: u64, - pub new_data_offset: u64, - pub data_len: u32, - pub type_id: u32, - pub updated_at: i64, - pub weight: f32, - pub key_len: u16, - pub dense_vector_offset: u64, - pub dense_vector_len: u32, - pub sparse_vector_offset: u64, - pub sparse_vector_len: u32, - pub src_seg_idx: usize, - pub src_data_offset: u64, - pub last_write_seq: u64, +/// Write a u64 varint into a `Vec`. +fn write_varint_to_vec(buf: &mut Vec, mut val: u64) { + loop { + let mut byte = (val & 0x7F) as u8; + val >>= 7; + if val != 0 { + byte |= 0x80; + } + buf.push(byte); + if val == 0 { + break; + } + } } -/// Edge metadata collected from source sidecars for metadata-driven index building. -pub(crate) struct CompactEdgeMeta { - pub edge_id: u64, - pub new_data_offset: u64, - pub data_len: u32, - pub from: u64, - pub to: u64, - pub type_id: u32, - pub updated_at: i64, - pub weight: f32, - pub valid_from: i64, - pub valid_to: i64, - pub last_write_seq: u64, +fn varint_len(mut val: u64) -> u64 { + let mut len = 1; + while val >= 0x80 { + val >>= 7; + len += 1; + } + len } -/// Build all secondary indexes and sidecars from metadata without Memtable decode. -/// Used by V3 compaction path. -/// -/// IMPORTANT: Two index-writing paths exist and must stay in sync: -/// 1. `write_segment()` (flush path, builds indexes from Memtable) -/// 2. `write_indexes_from_metadata_with_secondary_indexes()` [this fn] (compaction path) -/// -/// If you add a new index type, you MUST add it to BOTH paths. -/// -/// `node_metas` and `edge_metas` must be sorted by ID. -#[allow(clippy::too_many_arguments)] -pub(crate) fn write_indexes_from_metadata_with_secondary_indexes( - segment_id: u64, - seg_dir: &Path, - segments: &[Arc], - node_metas: &[CompactNodeMeta], - edge_metas: &[CompactEdgeMeta], - dense_config: Option<&DenseVectorConfig>, - write_degree_sidecar: bool, - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result { - let report = Arc::new(Mutex::new(SecondaryIndexMaintenanceReport::default())); - let node_report = Arc::clone(&report); - run_index_fanout( - || { - write_key_index_from_meta(seg_dir, segments, node_metas)?; - write_node_type_index_from_meta(seg_dir, node_metas)?; - let branch_report = write_declared_equality_sidecars_from_metadata( - seg_dir, - segments, - node_metas, - secondary_indexes, - )?; - node_report - .lock() - .unwrap() - .failed_equality_indexes - .extend(branch_report.failed_equality_indexes); - let branch_report = write_declared_range_sidecars_from_metadata( - seg_dir, - segments, - node_metas, - secondary_indexes, - )?; - let mut report = node_report.lock().unwrap(); - report - .failed_range_indexes - .extend(branch_report.failed_range_indexes); - write_timestamp_index_from_meta(seg_dir, node_metas)?; - Ok(()) - }, - || { - write_adjacency_from_meta(seg_dir, "adj_out", edge_metas, true)?; - write_adjacency_from_meta(seg_dir, "adj_in", edge_metas, false)?; - write_edge_type_index_from_meta(seg_dir, edge_metas)?; - write_edge_triple_index_from_meta(seg_dir, edge_metas)?; - write_empty_tombstones(seg_dir)?; - if write_degree_sidecar { - if let Some(sidecars) = degree_sidecars_for_segments(segments) { - write_folded_degree_delta_sidecar_from_sidecars( - &seg_dir.join(DEGREE_DELTA_FILENAME), - &sidecars, - )?; - } - } - Ok(()) - }, - || { - let dense_points = write_sidecars_from_meta(seg_dir, segments, node_metas, edge_metas)?; - write_dense_hnsw_index_from_points(seg_dir, dense_config, dense_points)?; - Ok(()) - }, - || write_sparse_posting_index_from_meta(seg_dir, segments, node_metas), - )?; - write_compaction_planner_stats_sidecar_best_effort( - seg_dir, - segment_id, - segments, - node_metas, - edge_metas, - secondary_indexes, +fn write_adjacency_posting_bytes( + w: &mut impl Write, + posting: AdjacencyPosting, + prev_edge_id: &mut u64, + scratch: &mut Vec, +) -> Result<(), EngineError> { + let (edge_id, neighbor_id, weight, valid_from, valid_to) = posting; + let delta = edge_id - *prev_edge_id; + *prev_edge_id = edge_id; + scratch.clear(); + write_varint_to_vec(scratch, delta); + write_varint_to_vec(scratch, neighbor_id); + scratch.extend_from_slice(&weight.to_le_bytes()); + debug_assert!( + valid_from >= 0, + "valid_from must be non-negative for varint encoding" ); - write_format_version(seg_dir)?; - fsync_dir(seg_dir)?; - let final_report = report.lock().unwrap().clone(); - Ok(final_report) + debug_assert!( + valid_to >= 0, + "valid_to must be non-negative for sentinel encoding" + ); + let vt_enc = if valid_to == i64::MAX { + 0u64 + } else { + valid_to as u64 + 1 + }; + write_varint_to_vec(scratch, valid_from as u64); + write_varint_to_vec(scratch, vt_enc); + w.write_all(scratch)?; + Ok(()) } -fn degree_sidecars_for_segments( - segments: &[Arc], -) -> Option> { - segments - .iter() - .map(|segment| segment.degree_delta_sidecar()) - .collect() +fn adjacency_postings_len(postings: &[AdjacencyPosting]) -> u64 { + let mut len = 0u64; + let mut prev_edge_id = 0u64; + for &(edge_id, neighbor_id, _, valid_from, valid_to) in postings { + let delta = edge_id - prev_edge_id; + prev_edge_id = edge_id; + let vt_enc = if valid_to == i64::MAX { + 0u64 + } else { + valid_to as u64 + 1 + }; + len += varint_len(delta); + len += varint_len(neighbor_id); + len += 4; + len += varint_len(valid_from as u64); + len += varint_len(vt_enc); + } + len } -/// key_index.dat from metadata: read key bytes via partial header parse from source segments. -fn write_key_index_from_meta( - seg_dir: &Path, - segments: &[Arc], - node_metas: &[CompactNodeMeta], -) -> Result<(), EngineError> { - let path = seg_dir.join("key_index.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - - // Collect (type_id, key_bytes, node_id) by reading key from source segment raw data - let mut entries: Vec<(u32, Vec, u64)> = Vec::with_capacity(node_metas.len()); - for nm in node_metas { - let src_mmap = segments[nm.src_seg_idx].raw_nodes_mmap(); - // Raw node record layout: type_id(4) + key_len(2) + key_bytes(key_len) + ... - let key_start = nm.src_data_offset as usize + 6; - let key_end = key_start + nm.key_len as usize; - if key_end > src_mmap.len() { - return Err(EngineError::CorruptRecord(format!( - "node {} key bytes [{}, {}) exceed source mmap length {}", - nm.node_id, - key_start, - key_end, - src_mmap.len() - ))); +fn prepare_adjacency_payloads(adj: NodeIdMap>) -> AdjacencyPayloadPlan { + let mut groups: Vec<(u64, u32, Vec)> = Vec::new(); + for (node_id, edge_map) in adj { + let mut by_label: HashMap> = HashMap::new(); + for entry in edge_map.into_values() { + by_label.entry(entry.label_id).or_default().push(( + entry.edge_id, + entry.neighbor_id, + entry.weight, + entry.valid_from, + entry.valid_to, + )); + } + for (label_id, mut postings) in by_label { + postings.sort_unstable_by_key(|&(edge_id, ..)| edge_id); + groups.push((node_id, label_id, postings)); } - entries.push(( - nm.type_id, - src_mmap[key_start..key_end].to_vec(), - nm.node_id, - )); } - // Sort by (type_id, key) - entries.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); - - let count = entries.len() as u64; - write_u64(&mut w, count)?; - - // Pre-compute entry sizes for offset table - // Each entry: type_id (4) + node_id (8) + key_len (2) + key_bytes - let entry_sizes: Vec = entries - .iter() - .map(|(_, key, _)| 4 + 8 + 2 + key.len() as u64) + groups.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + let mut offset = 0u64; + let groups = groups + .into_iter() + .map(|(node_id, label_id, postings)| { + let group = AdjacencyGroupPlan { + node_id, + label_id, + offset, + postings, + }; + offset += adjacency_postings_len(&group.postings); + group + }) .collect(); + AdjacencyPayloadPlan { groups } +} - let data_start = 8 + count * 8; - let mut offset = data_start; - for &size in &entry_sizes { - write_u64(&mut w, offset)?; - offset += size; +fn write_adjacency_postings_payload( + w: &mut impl Write, + plan: &AdjacencyPayloadPlan, +) -> Result<(), EngineError> { + let mut scratch = Vec::with_capacity(32); + for group in &plan.groups { + let mut prev_edge_id = 0u64; + for &posting in &group.postings { + write_adjacency_posting_bytes(w, posting, &mut prev_edge_id, &mut scratch)?; + } } + Ok(()) +} - // Write data entries - for (type_id, key_bytes, node_id) in &entries { - write_u32(&mut w, *type_id)?; - write_u64(&mut w, *node_id)?; - write_u16(&mut w, key_bytes.len() as u16)?; - w.write_all(key_bytes)?; +fn write_adjacency_index_payload( + mut w: &mut impl Write, + plan: &AdjacencyPayloadPlan, +) -> Result<(), EngineError> { + write_u64(&mut w, plan.groups.len() as u64)?; + for group in &plan.groups { + write_u64(&mut w, group.node_id)?; + write_u32(&mut w, group.label_id)?; + write_u64(&mut w, group.offset)?; + write_u32(&mut w, group.postings.len() as u32)?; } - - w.flush()?; - w.get_ref().sync_all()?; Ok(()) } -/// node_type_index.dat from metadata. -fn write_node_type_index_from_meta( - seg_dir: &Path, - node_metas: &[CompactNodeMeta], -) -> Result<(), EngineError> { - let mut groups: BTreeMap> = BTreeMap::new(); - for nm in node_metas { - groups.entry(nm.type_id).or_default().push(nm.node_id); +/// Adjacency index + delta-encoded postings. +/// +/// Index payload: +/// [count: u64] +/// [(node_id: u64, label_id: u32, offset: u64, count: u32) × count, sorted by (node_id, label_id)] +/// +/// Postings payload: +/// Per group: delta-encoded postings, variable length. +/// First posting: varint(edge_id) + varint(neighbor_id) + f32(weight) + varint(valid_from_enc) + varint(valid_to_enc) +/// Subsequent: varint(edge_id_delta) + varint(neighbor_id) + f32(weight) + varint(valid_from_enc) + varint(valid_to_enc) +/// valid_from_enc = valid_from as u64 (valid_from is always >= 0) +/// valid_to_enc = 0 if valid_to == i64::MAX, else (valid_to as u64) + 1 +/// Key index payload format: +/// [entry_count: u64] +/// [offset_table: u64 × entry_count] (byte offset to each entry in data section) +/// [data section: entries sorted by (label_id, key, node_id)] +/// +/// Each entry: [label_id: u32][node_id: u64][key_len: u16][key: bytes] +fn prepare_key_index_payload( + nodes: &NodeIdMap, +) -> Result, EngineError> { + let mut entries: Vec> = Vec::new(); + for node in nodes.values() { + if node.key.len() > u16::MAX as usize { + return Err(EngineError::SerializationError(format!( + "node key exceeds maximum length of {} bytes", + u16::MAX + ))); + } + for &label_id in node.label_ids.as_slice() { + entries.push(KeyIndexEntryPlan { + label_id, + key: node.key.as_bytes(), + node_id: node.id, + encoded_len: 4 + 8 + 2 + node.key.len() as u64, + }); + } } - for ids in groups.values_mut() { - ids.sort_unstable(); + entries.sort_by(|a, b| { + a.label_id + .cmp(&b.label_id) + .then_with(|| a.key.cmp(b.key)) + .then_with(|| a.node_id.cmp(&b.node_id)) + }); + for pair in entries.windows(2) { + let left = &pair[0]; + let right = &pair[1]; + if left.label_id == right.label_id && left.key == right.key && left.node_id != right.node_id + { + let key = std::str::from_utf8(left.key).unwrap_or(""); + return Err(EngineError::InvalidOperation(format!( + "duplicate live node key membership for label {} and key '{}'", + left.label_id, key + ))); + } } - write_type_index_groups(seg_dir, "node_type_index", &groups) + Ok(KeyIndexPayloadPlan { entries }) } -/// edge_type_index.dat from metadata. -fn write_edge_type_index_from_meta( - seg_dir: &Path, - edge_metas: &[CompactEdgeMeta], +fn write_key_index_plan_payload( + mut w: &mut impl Write, + plan: &KeyIndexPayloadPlan<'_>, ) -> Result<(), EngineError> { - let mut groups: BTreeMap> = BTreeMap::new(); - for em in edge_metas { - groups.entry(em.type_id).or_default().push(em.edge_id); + let count = plan.entries.len() as u64; + write_u64(&mut w, count)?; + + let data_start = 8 + count * 8; + let mut offset = data_start; + for entry in &plan.entries { + write_u64(&mut w, offset)?; + offset += entry.encoded_len; } - for ids in groups.values_mut() { - ids.sort_unstable(); + + for entry in &plan.entries { + write_u32(&mut w, entry.label_id)?; + write_u64(&mut w, entry.node_id)?; + write_u16(&mut w, entry.key.len() as u16)?; + w.write_all(entry.key)?; } - write_type_index_groups(seg_dir, "edge_type_index", &groups) + + Ok(()) } -/// Shared writer for type index files from pre-grouped data. -fn write_type_index_groups( - seg_dir: &Path, - filename: &str, - groups: &BTreeMap>, -) -> Result<(), EngineError> { - let path = seg_dir.join(format!("{}.dat", filename)); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); +/// Label posting index payload format: +/// [entry_count: u64] +/// [index: entry_count x (target_label_id: u32, offset: u64, count: u32), sorted by target_label_id] +/// [data: packed u64 record IDs per target, grouped contiguously] +fn prepare_label_posting_index_payload( + label_posting_index: &HashMap, +) -> LabelPostingIndexPayloadPlan { + let mut groups: Vec<(u32, Vec)> = label_posting_index + .iter() + .filter(|(_, ids)| !ids.is_empty()) + .map(|(&target_label_id, ids)| { + let mut sorted_ids: Vec = ids.iter().copied().collect(); + sorted_ids.sort_unstable(); + (target_label_id, sorted_ids) + }) + .collect(); + groups.sort_by_key(|(target_label_id, _)| *target_label_id); + LabelPostingIndexPayloadPlan { groups } +} - let entry_count = groups.len() as u64; +fn write_label_posting_index_plan_payload( + mut w: &mut impl Write, + plan: &LabelPostingIndexPayloadPlan, +) -> Result<(), EngineError> { + let entry_count = plan.groups.len() as u64; write_u64(&mut w, entry_count)?; - let data_start = 8 + entry_count * TYPE_INDEX_ENTRY_SIZE; + let data_start = 8 + entry_count * LABEL_POSTING_INDEX_ENTRY_SIZE; let mut data_offset = data_start; - for (&type_id, ids) in groups { - let count = ids.len() as u32; - write_u32(&mut w, type_id)?; + for (target_label_id, ids) in &plan.groups { + write_u32(&mut w, *target_label_id)?; write_u64(&mut w, data_offset)?; + let count = ids.len() as u32; write_u32(&mut w, count)?; - data_offset += count as u64 * 8; + data_offset += count as u64 * 8; // each ID is u64 = 8 bytes } - for ids in groups.values() { + for (_, ids) in &plan.groups { for &id in ids { write_u64(&mut w, id)?; } } - w.flush()?; - w.get_ref().sync_all()?; Ok(()) } -/// edge_triple_index.dat from metadata. -fn write_edge_triple_index_from_meta( - seg_dir: &Path, - edge_metas: &[CompactEdgeMeta], -) -> Result<(), EngineError> { - let path = seg_dir.join("edge_triple_index.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - - let mut entries: Vec<(u64, u64, u32, u64)> = edge_metas - .iter() - .map(|em| (em.from, em.to, em.type_id, em.edge_id)) +/// Edge triple index payload format: +/// [count: u64] +/// [entries: count × (from: u64, to: u64, label_id: u32, edge_id: u64), +/// sorted by (from, to, label_id, edge_id)] +fn prepare_edge_triple_index_payload(edges: &NodeIdMap) -> EdgeTripleIndexPayloadPlan { + let mut entries: Vec<(u64, u64, u32, u64)> = edges + .values() + .map(|e| (e.from, e.to, e.label_id, e.id)) .collect(); - entries.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)).then(a.2.cmp(&b.2))); + entries.sort_by(|a, b| { + a.0.cmp(&b.0) + .then(a.1.cmp(&b.1)) + .then(a.2.cmp(&b.2)) + .then(a.3.cmp(&b.3)) + }); + EdgeTripleIndexPayloadPlan { entries } +} - let count = entries.len() as u64; +fn write_edge_triple_index_plan_payload( + mut w: &mut impl Write, + plan: &EdgeTripleIndexPayloadPlan, +) -> Result<(), EngineError> { + let count = plan.entries.len() as u64; write_u64(&mut w, count)?; - for &(from, to, type_id, edge_id) in &entries { - write_u64(&mut w, from)?; - write_u64(&mut w, to)?; - write_u32(&mut w, type_id)?; + for (from, to, label_id, edge_id) in &plan.entries { + write_u64(&mut w, *from)?; + write_u64(&mut w, *to)?; + write_u32(&mut w, *label_id)?; + write_u64(&mut w, *edge_id)?; + } + + Ok(()) +} + +fn write_edge_weight_metadata_index_payload( + mut w: &mut impl Write, + entries: &[(u32, u32, u64)], +) -> Result<(), EngineError> { + write_u64(&mut w, entries.len() as u64)?; + for &(label_id, weight_key, edge_id) in entries { + write_u32(&mut w, label_id)?; + write_u32(&mut w, weight_key)?; write_u64(&mut w, edge_id)?; } - w.flush()?; - w.get_ref().sync_all()?; Ok(()) } -/// Adjacency index from edge metadata. Builds adj_out (is_outgoing=true) or adj_in (is_outgoing=false). -#[allow(clippy::type_complexity)] -fn write_adjacency_from_meta( - seg_dir: &Path, - prefix: &str, - edge_metas: &[CompactEdgeMeta], - is_outgoing: bool, +fn write_edge_i64_metadata_index_payload( + mut w: &mut impl Write, + entries: &[(u32, i64, u64)], ) -> Result<(), EngineError> { - let idx_path = seg_dir.join(format!("{}.idx", prefix)); - let dat_path = seg_dir.join(format!("{}.dat", prefix)); - - let idx_file = File::create(&idx_path)?; - let dat_file = File::create(&dat_path)?; - let mut idx_w = BufWriter::new(idx_file); - let mut dat_w = BufWriter::new(dat_file); - - // Group entries by (node_id, type_id) - // For adj_out: node_id = from, neighbor = to - // For adj_in: node_id = to, neighbor = from - let mut groups: BTreeMap<(u64, u32), Vec<(u64, u64, f32, i64, i64)>> = BTreeMap::new(); - for em in edge_metas { - let (node_id, neighbor_id) = if is_outgoing { - (em.from, em.to) - } else { - (em.to, em.from) - }; - groups.entry((node_id, em.type_id)).or_default().push(( - em.edge_id, - neighbor_id, - em.weight, - em.valid_from, - em.valid_to, - )); - } - - // Sort postings within each group by edge_id - for postings in groups.values_mut() { - postings.sort_unstable_by_key(|&(edge_id, ..)| edge_id); + write_u64(&mut w, entries.len() as u64)?; + for &(label_id, value, edge_id) in entries { + write_u32(&mut w, label_id)?; + w.write_all(&value.to_le_bytes())?; + write_u64(&mut w, edge_id)?; } - let count = groups.len() as u64; - write_u64(&mut idx_w, count)?; - - let mut posting_buf = Vec::new(); - let mut dat_offset: u64 = 0; - let mut index_entries: Vec<(u64, u32, u64, u32)> = Vec::with_capacity(groups.len()); - - for (&(node_id, type_id), postings) in &groups { - let posting_count = postings.len() as u32; - index_entries.push((node_id, type_id, dat_offset, posting_count)); - - posting_buf.clear(); - let mut prev_edge_id: u64 = 0; - for &(edge_id, neighbor_id, weight, valid_from, valid_to) in postings { - let delta = edge_id - prev_edge_id; - prev_edge_id = edge_id; - - write_varint_to_vec(&mut posting_buf, delta); - write_varint_to_vec(&mut posting_buf, neighbor_id); - posting_buf.extend_from_slice(&weight.to_le_bytes()); - write_varint_to_vec(&mut posting_buf, valid_from as u64); - let vt_enc = if valid_to == i64::MAX { - 0u64 - } else { - valid_to as u64 + 1 - }; - write_varint_to_vec(&mut posting_buf, vt_enc); - } + Ok(()) +} - dat_w.write_all(&posting_buf)?; - dat_offset += posting_buf.len() as u64; +/// Timestamp index payload format: +/// [entry_count: u64] +/// [entries: entry_count × (label_id: u32, updated_at: i64, node_id: u64), +/// sorted by (label_id, updated_at, node_id)] +/// +/// Each entry is 20 bytes. Binary search for range start (label_id, from_ms), +/// scan to range end (label_id, to_ms). O(log N) seek + O(results) scan. +fn prepare_timestamp_index_payload( + time_index: &std::collections::BTreeSet<(u32, i64, u64)>, +) -> TimestampIndexPayloadPlan { + TimestampIndexPayloadPlan { + entries: time_index.iter().copied().collect(), } +} + +fn write_timestamp_index_plan_payload( + mut w: &mut impl Write, + plan: &TimestampIndexPayloadPlan, +) -> Result<(), EngineError> { + let count = plan.entries.len() as u64; + write_u64(&mut w, count)?; - // Write index entries - for &(node_id, type_id, offset, posting_count) in &index_entries { - write_u64(&mut idx_w, node_id)?; - write_u32(&mut idx_w, type_id)?; - write_u64(&mut idx_w, offset)?; - write_u32(&mut idx_w, posting_count)?; + for &(label_id, updated_at, node_id) in &plan.entries { + write_u32(&mut w, label_id)?; + w.write_all(&updated_at.to_le_bytes())?; + write_u64(&mut w, node_id)?; } - idx_w.flush()?; - idx_w.get_ref().sync_all()?; - dat_w.flush()?; - dat_w.get_ref().sync_all()?; Ok(()) } -fn build_secondary_eq_groups_from_source_sidecars( - segments: &[Arc], - node_metas: &[CompactNodeMeta], - index_id: u64, - type_id: u32, -) -> Result>, EngineError> { - let winner_sources: HashMap = node_metas - .iter() - .filter(|meta| meta.type_id == type_id) - .map(|meta| (meta.node_id, meta.src_seg_idx)) - .collect(); - let mut groups: BTreeMap> = BTreeMap::new(); +/// Tombstones payload format: +/// [count: u64] +/// [(kind: u8, id: u64, deleted_at: i64, last_write_seq: u64) × count] +/// kind: 0 = node, 1 = edge. Entry size: 25 bytes. +fn write_tombstones_payload( + mut w: &mut impl Write, + deleted_nodes: &NodeIdMap, + deleted_edges: &NodeIdMap, +) -> Result<(), EngineError> { + let count = (deleted_nodes.len() + deleted_edges.len()) as u64; + write_u64(&mut w, count)?; - for (seg_idx, seg) in segments.iter().enumerate() { - seg.for_each_secondary_eq_group(index_id, |value_hash, ids| { - let group = groups.entry(value_hash).or_default(); - for &node_id in ids { - if winner_sources.get(&node_id) == Some(&seg_idx) { - group.push(node_id); - } - } - Ok(()) - })?; + // Write node tombstones (sorted by ID for determinism) + let mut node_entries: Vec<(u64, &TombstoneEntry)> = + deleted_nodes.iter().map(|(&id, ts)| (id, ts)).collect(); + node_entries.sort_unstable_by_key(|&(id, _)| id); + for (id, ts) in node_entries { + write_u8(&mut w, 0)?; // kind = node + write_u64(&mut w, id)?; + w.write_all(&ts.deleted_at.to_le_bytes())?; + write_u64(&mut w, ts.last_write_seq)?; } - for ids in groups.values_mut() { - ids.sort_unstable(); - ids.dedup(); + // Write edge tombstones (sorted by ID for determinism) + let mut edge_entries: Vec<(u64, &TombstoneEntry)> = + deleted_edges.iter().map(|(&id, ts)| (id, ts)).collect(); + edge_entries.sort_unstable_by_key(|&(id, _)| id); + for (id, ts) in edge_entries { + write_u8(&mut w, 1)?; // kind = edge + write_u64(&mut w, id)?; + w.write_all(&ts.deleted_at.to_le_bytes())?; + write_u64(&mut w, ts.last_write_seq)?; } - Ok(groups) + Ok(()) } -fn build_secondary_eq_groups_from_targeted_decode( - segments: &[Arc], - node_metas: &[CompactNodeMeta], - type_id: u32, - prop_key: &str, -) -> Result>, EngineError> { - let mut groups: BTreeMap> = BTreeMap::new(); +// --- Record encoding helpers --- - for meta in node_metas.iter().filter(|meta| meta.type_id == type_id) { - if let Some(value) = segments[meta.src_seg_idx].node_property_value_at_offset( - meta.node_id, - meta.src_data_offset, - prop_key, - )? { - groups - .entry(hash_prop_value(&value)) - .or_default() - .push(meta.node_id); - } +fn encode_node_record_into(buf: &mut Vec, node: &NodeRecord) -> Result<(), EngineError> { + buf.clear(); + // Note: node.id is NOT written here. It's already in the index. + buf.push(node.label_ids.len() as u8); + for &label_id in node.label_ids.as_slice() { + buf.extend_from_slice(&label_id.to_le_bytes()); } - - for ids in groups.values_mut() { - ids.sort_unstable(); - ids.dedup(); + let key_bytes = node.key.as_bytes(); + if key_bytes.len() > u16::MAX as usize { + return Err(EngineError::SerializationError(format!( + "node key exceeds maximum length of {} bytes", + u16::MAX + ))); } + buf.extend_from_slice(&(key_bytes.len() as u16).to_le_bytes()); + buf.extend_from_slice(key_bytes); + buf.extend_from_slice(&node.created_at.to_le_bytes()); + buf.extend_from_slice(&node.updated_at.to_le_bytes()); + buf.extend_from_slice(&node.weight.to_le_bytes()); + let props_bytes = rmp_serde::to_vec(&node.props) + .map_err(|e| EngineError::SerializationError(e.to_string()))?; + buf.extend_from_slice(&(props_bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(&props_bytes); + Ok(()) +} - Ok(groups) +fn encode_edge_record_into(buf: &mut Vec, edge: &EdgeRecord) -> Result<(), EngineError> { + buf.clear(); + // Note: edge.id is NOT written here. It's already in the index. + buf.extend_from_slice(&edge.from.to_le_bytes()); + buf.extend_from_slice(&edge.to.to_le_bytes()); + buf.extend_from_slice(&edge.label_id.to_le_bytes()); + buf.extend_from_slice(&edge.created_at.to_le_bytes()); + buf.extend_from_slice(&edge.updated_at.to_le_bytes()); + buf.extend_from_slice(&edge.weight.to_le_bytes()); + buf.extend_from_slice(&edge.valid_from.to_le_bytes()); + buf.extend_from_slice(&edge.valid_to.to_le_bytes()); + let props_bytes = rmp_serde::to_vec(&edge.props) + .map_err(|e| EngineError::SerializationError(e.to_string()))?; + buf.extend_from_slice(&(props_bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(&props_bytes); + Ok(()) } -fn build_secondary_range_entries_from_source_sidecars( - segments: &[Arc], - node_metas: &[CompactNodeMeta], - index_id: u64, - type_id: u32, -) -> Result, EngineError> { - let winner_sources: HashMap = node_metas - .iter() - .filter(|meta| meta.type_id == type_id) - .map(|meta| (meta.node_id, meta.src_seg_idx)) - .collect(); - let mut entries = Vec::new(); +// --- Metadata payload writers --- - for (seg_idx, seg) in segments.iter().enumerate() { - seg.for_each_secondary_range_entry(index_id, |encoded_value, node_id| { - if winner_sources.get(&node_id) == Some(&seg_idx) { - entries.push((encoded_value, node_id)); - } - Ok(()) +/// Node metadata payload format: +/// Header + fixed rows + label-offset table + compact label-ID region. +fn write_node_meta_payload( + mut meta_w: &mut impl Write, + node_data: &[(u64, u64, u32)], + nodes: &NodeIdMap, +) -> Result<(), EngineError> { + let count = node_data.len() as u64; + write_u64(&mut meta_w, count)?; + write_u16(&mut meta_w, NODE_META_FIXED_ENTRY_SIZE)?; + write_u16(&mut meta_w, NODE_META_LABEL_OFFSET_ENTRY_SIZE)?; + write_u32(&mut meta_w, 0)?; + let fixed_entries_offset = NODE_META_HEADER_SIZE; + let fixed_entries_len = count + .checked_mul(NODE_META_FIXED_ENTRY_SIZE as u64) + .ok_or_else(|| EngineError::CorruptRecord("node metadata fixed table overflow".into()))?; + let label_offsets_offset = fixed_entries_offset + .checked_add(fixed_entries_len) + .ok_or_else(|| EngineError::CorruptRecord("node metadata offset overflow".into()))?; + let label_offset_entries = count.checked_add(1).ok_or_else(|| { + EngineError::CorruptRecord("node metadata label offset count overflow".into()) + })?; + let label_ids_offset = label_offsets_offset + .checked_add(label_offset_entries * NODE_META_LABEL_OFFSET_ENTRY_SIZE as u64) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata label ID offset overflow".into()) + })?; + let mut label_offsets = Vec::with_capacity(label_offset_entries as usize); + let mut label_ids = Vec::new(); + label_offsets.push(0u64); + for &(node_id, _, _) in node_data { + let node = nodes.get(&node_id).ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} not found for metadata", node_id)) })?; + label_ids.extend_from_slice(node.label_ids.as_slice()); + label_offsets.push(label_ids.len() as u64); } + write_u64(&mut meta_w, fixed_entries_offset)?; + write_u64(&mut meta_w, label_offsets_offset)?; + write_u64(&mut meta_w, label_ids_offset)?; + write_u64(&mut meta_w, label_ids.len() as u64)?; - entries.sort_unstable(); - entries.dedup(); - Ok(entries) -} + for &(node_id, data_offset, data_len) in node_data { + let node = nodes.get(&node_id).ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} not found for metadata", node_id)) + })?; -fn build_secondary_range_entries_from_targeted_decode( - segments: &[Arc], - node_metas: &[CompactNodeMeta], - type_id: u32, - prop_key: &str, - domain: SecondaryIndexRangeDomain, -) -> Result, EngineError> { - let mut entries = Vec::new(); + write_u64(&mut meta_w, node_id)?; + write_u64(&mut meta_w, data_offset)?; + write_u32(&mut meta_w, data_len)?; + meta_w.write_all(&node.updated_at.to_le_bytes())?; + meta_w.write_all(&node.weight.to_le_bytes())?; + write_u16(&mut meta_w, node.key.len() as u16)?; + write_u64(&mut meta_w, node.last_write_seq)?; + meta_w.write_all(&[0u8; 6])?; + } - for meta in node_metas.iter().filter(|meta| meta.type_id == type_id) { - let Some(value) = segments[meta.src_seg_idx].node_property_value_at_offset( - meta.node_id, - meta.src_data_offset, - prop_key, - )? - else { - continue; - }; - let Some(encoded_value) = encode_range_prop_value(domain, &value) else { - continue; - }; - entries.push((encoded_value, meta.node_id)); + for offset in label_offsets { + write_u64(&mut meta_w, offset)?; + } + for label_id in label_ids { + write_u32(&mut meta_w, label_id)?; } - entries.sort_unstable(); - entries.dedup(); - Ok(entries) + Ok(()) } -fn write_declared_equality_sidecars_from_metadata( - seg_dir: &Path, - segments: &[Arc], - node_metas: &[CompactNodeMeta], - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result { - let eq_entries: Vec<&SecondaryIndexManifestEntry> = secondary_indexes - .iter() - .filter(|entry| matches!(entry.kind, SecondaryIndexKind::Equality)) - .collect(); - if eq_entries.is_empty() { - return Ok(SecondaryIndexMaintenanceReport::default()); +fn write_node_vector_source_components( + core_writer: &mut PackedCoreWriter, + node_data: &[(u64, u64, u32)], + nodes: &NodeIdMap, + node_source: [u8; 32], +) -> Result<(Vec, Vec), EngineError> { + let plan = prepare_node_vector_source_plan(node_data, nodes)?; + if !plan.has_dense && !plan.has_sparse { + return Ok((Vec::new(), Vec::new())); + } + let node_source_dep = source_group_dependency(SegmentSourceGroupKind::NodeSource, node_source); + let (meta_record, _) = core_writer.write_component( + SegmentComponentKind::NodeVectorMetadata, + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + vec![node_source_dep.clone()], + component_fingerprint("flush.node_vector_meta", &[]), + |writer| write_node_vector_meta_payload(writer, &plan), + )?; + let vector_blob_deps = vec![node_source_dep, source_component_dependency(&meta_record)]; + let mut records = Vec::with_capacity(3); + records.push(meta_record); + if plan.has_dense { + let (record, _) = core_writer.write_component( + SegmentComponentKind::NodeDenseVectorBlob, + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + vector_blob_deps.clone(), + component_fingerprint("flush.node_dense_vectors", &[]), + |writer| write_node_dense_vector_blob_payload(writer, &plan, nodes), + )?; + records.push(record); + } + if plan.has_sparse { + let (record, _) = core_writer.write_component( + SegmentComponentKind::NodeSparseVectorBlob, + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + vector_blob_deps, + component_fingerprint("flush.node_sparse_vectors", &[]), + |writer| write_node_sparse_vector_blob_payload(writer, &plan, nodes), + )?; + records.push(record); } + Ok((records, plan.dense_points)) +} - let index_dir = secondary_indexes_dir(seg_dir); - fs::create_dir_all(&index_dir)?; - let mut report = SecondaryIndexMaintenanceReport::default(); +fn prepare_node_vector_source_plan( + node_data: &[(u64, u64, u32)], + nodes: &NodeIdMap, +) -> Result { + let mut rows = Vec::with_capacity(node_data.len()); + let mut has_dense = false; + let mut has_sparse = false; + let mut dense_offset = 0u64; + let mut sparse_offset = 0u64; + let mut dense_points = Vec::new(); - for entry in eq_entries { - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; - let mut failure_message = None; - let use_source_sidecars = if entry.state == SecondaryIndexState::Failed { - false - } else { - let mut all_present = true; - for seg in segments { - match seg.validate_secondary_eq_sidecar(entry.index_id) { - Ok(true) => {} - Ok(false) => { - all_present = false; - break; - } - Err(error) => { - all_present = false; - if entry.state == SecondaryIndexState::Ready { - failure_message = Some(error.to_string()); - } - break; - } - } - } - all_present - }; + for &(node_id, _, _) in node_data { + let node = nodes.get(&node_id).ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} not found for vector source", node_id)) + })?; - let groups = if use_source_sidecars { - build_secondary_eq_groups_from_source_sidecars( - segments, - node_metas, - entry.index_id, - *type_id, - )? - } else { - build_secondary_eq_groups_from_targeted_decode( - segments, node_metas, *type_id, prop_key, - )? - }; + let mut flags = 0u8; + let mut dense_len = 0u32; + let mut sparse_len = 0u32; + let mut entry_dense_offset = 0u64; + let mut entry_sparse_offset = 0u64; - if let Some(message) = failure_message { - report - .failed_equality_indexes - .push((entry.index_id, message)); + if let Some(values) = node.dense_vector.as_ref() { + flags |= NODE_VECTOR_FLAG_DENSE; + dense_len = values.len() as u32; + entry_dense_offset = dense_offset; + has_dense = true; + dense_points.push(DensePointInput { + node_id, + dense_vector_offset: entry_dense_offset, + values: values.clone(), + }); + dense_offset = dense_offset + .checked_add(values.len() as u64 * DENSE_VECTOR_VALUE_SIZE) + .ok_or_else(|| { + EngineError::CorruptRecord("dense vector blob offset overflow".into()) + })?; } - write_node_prop_eq_sidecar_to_path( - &node_prop_eq_sidecar_path(seg_dir, entry.index_id), - &groups, - )?; - } + if let Some(values) = node.sparse_vector.as_ref() { + flags |= NODE_VECTOR_FLAG_SPARSE; + sparse_len = values.len() as u32; + entry_sparse_offset = sparse_offset; + has_sparse = true; + sparse_offset = sparse_offset + .checked_add(values.len() as u64 * SPARSE_VECTOR_ENTRY_SIZE) + .ok_or_else(|| { + EngineError::CorruptRecord("sparse vector blob offset overflow".into()) + })?; + } - fsync_dir(&index_dir)?; - Ok(report) + rows.push(NodeVectorSourceRow { + node_id, + flags, + dense_offset: entry_dense_offset, + dense_len, + sparse_offset: entry_sparse_offset, + sparse_len, + }); + } + + Ok(NodeVectorSourcePlan { + rows, + has_dense, + has_sparse, + dense_points, + }) } -fn write_declared_range_sidecars_from_metadata( - seg_dir: &Path, - segments: &[Arc], - node_metas: &[CompactNodeMeta], - secondary_indexes: &[SecondaryIndexManifestEntry], -) -> Result { - let range_entries: Vec<&SecondaryIndexManifestEntry> = secondary_indexes - .iter() - .filter(|entry| matches!(entry.kind, SecondaryIndexKind::Range { .. })) - .collect(); - if range_entries.is_empty() { - return Ok(SecondaryIndexMaintenanceReport::default()); +fn write_node_vector_meta_payload( + mut w: &mut impl Write, + plan: &NodeVectorSourcePlan, +) -> Result<(), EngineError> { + write_u64(&mut w, plan.rows.len() as u64)?; + for row in &plan.rows { + write_u8(&mut w, row.flags)?; + w.write_all(&[0u8; 3])?; + write_u64(&mut w, row.dense_offset)?; + write_u32(&mut w, row.dense_len)?; + write_u64(&mut w, row.sparse_offset)?; + write_u32(&mut w, row.sparse_len)?; } + Ok(()) +} - let index_dir = secondary_indexes_dir(seg_dir); - fs::create_dir_all(&index_dir)?; - let mut report = SecondaryIndexMaintenanceReport::default(); - - for entry in range_entries { - let SecondaryIndexTarget::NodeProperty { type_id, prop_key } = &entry.target; - let SecondaryIndexKind::Range { domain } = entry.kind else { +fn write_node_dense_vector_blob_payload( + w: &mut impl Write, + plan: &NodeVectorSourcePlan, + nodes: &NodeIdMap, +) -> Result<(), EngineError> { + for row in &plan.rows { + if row.dense_len == 0 { continue; + } + let node = nodes.get(&row.node_id).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "node {} not found for dense vector blob", + row.node_id + )) + })?; + let Some(values) = node.dense_vector.as_ref() else { + return Err(EngineError::CorruptRecord(format!( + "node {} missing dense vector for blob", + row.node_id + ))); }; - let mut failure_message = None; - let use_source_sidecars = if entry.state == SecondaryIndexState::Failed { - false - } else { - let mut all_present = true; - for seg in segments { - match seg.validate_secondary_range_sidecar(entry.index_id) { - Ok(true) => {} - Ok(false) => { - all_present = false; - break; - } - Err(error) => { - all_present = false; - if entry.state == SecondaryIndexState::Ready { - failure_message = Some(error.to_string()); - } - break; - } - } - } - all_present - }; + for &value in values { + w.write_all(&value.to_le_bytes())?; + } + } + Ok(()) +} - let sidecar_entries = if use_source_sidecars { - build_secondary_range_entries_from_source_sidecars( - segments, - node_metas, - entry.index_id, - *type_id, - )? - } else { - build_secondary_range_entries_from_targeted_decode( - segments, node_metas, *type_id, prop_key, domain, - )? +fn write_node_sparse_vector_blob_payload( + w: &mut impl Write, + plan: &NodeVectorSourcePlan, + nodes: &NodeIdMap, +) -> Result<(), EngineError> { + for row in &plan.rows { + if row.sparse_len == 0 { + continue; + } + let node = nodes.get(&row.node_id).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "node {} not found for sparse vector blob", + row.node_id + )) + })?; + let Some(values) = node.sparse_vector.as_ref() else { + return Err(EngineError::CorruptRecord(format!( + "node {} missing sparse vector for blob", + row.node_id + ))); }; - - if let Some(message) = failure_message { - report.failed_range_indexes.push((entry.index_id, message)); + for &(dimension_id, weight) in values { + write_u32(w, dimension_id)?; + w.write_all(&weight.to_le_bytes())?; } + } + Ok(()) +} - write_node_prop_range_sidecar_to_path( - &node_prop_range_sidecar_path(seg_dir, entry.index_id), - &sidecar_entries, - )?; +/// Edge metadata payload format: +/// [count: u64] +/// [entries: count × EdgeMetaEntry, sorted by edge_id] +/// +/// EdgeMetaEntry (80 bytes): +/// edge_id: u64, data_offset: u64, data_len: u32, +/// from: u64, to: u64, label_id: u32, +/// updated_at: i64, weight: f32, +/// valid_from: i64, valid_to: i64, +/// last_write_seq: u64, reserved: u32 +fn write_edge_meta_payload( + mut w: &mut impl Write, + edge_data: &[(u64, u64, u32)], + edges: &NodeIdMap, +) -> Result<(), EngineError> { + let count = edge_data.len() as u64; + write_u64(&mut w, count)?; + + for &(edge_id, data_offset, data_len) in edge_data { + let edge = edges.get(&edge_id).ok_or_else(|| { + EngineError::CorruptRecord(format!("edge {} not found for metadata", edge_id)) + })?; + + write_u64(&mut w, edge_id)?; + write_u64(&mut w, data_offset)?; + write_u32(&mut w, data_len)?; + write_u64(&mut w, edge.from)?; + write_u64(&mut w, edge.to)?; + write_u32(&mut w, edge.label_id)?; + w.write_all(&edge.updated_at.to_le_bytes())?; + w.write_all(&edge.weight.to_le_bytes())?; + w.write_all(&edge.valid_from.to_le_bytes())?; + w.write_all(&edge.valid_to.to_le_bytes())?; + write_u64(&mut w, edge.last_write_seq)?; + write_u32(&mut w, 0)?; // reserved } - fsync_dir(&index_dir)?; - Ok(report) + Ok(()) } -/// timestamp_index.dat from metadata. -fn write_timestamp_index_from_meta( - seg_dir: &Path, - node_metas: &[CompactNodeMeta], +fn validate_required_components_before_manifest( + segment_id: u64, + records: &[SegmentComponentRecordV1], ) -> Result<(), EngineError> { - let path = seg_dir.join("timestamp_index.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); + let has_component = + |kind: &SegmentComponentKind| records.iter().any(|record| record.kind == *kind); + if !has_component(&SegmentComponentKind::PackedSegmentContainer) { + return Err(EngineError::CorruptRecord(format!( + "refusing to publish {SEGMENT_COMPONENT_MANIFEST_FILENAME} for segment {segment_id}: missing packed core container" + ))); + } - // Build sorted entries from metadata - let mut entries: Vec<(u32, i64, u64)> = node_metas - .iter() - .map(|nm| (nm.type_id, nm.updated_at, nm.node_id)) - .collect(); - entries.sort_unstable(); + for kind in [ + SegmentComponentKind::NodeRecords, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::NodeMetadata, + SegmentComponentKind::EdgeMetadata, + SegmentComponentKind::Tombstones, + SegmentComponentKind::KeyIndex, + SegmentComponentKind::NodeLabelIndex, + SegmentComponentKind::EdgeLabelIndex, + SegmentComponentKind::EdgeTripleIndex, + SegmentComponentKind::AdjOutIndex, + SegmentComponentKind::AdjOutPostings, + SegmentComponentKind::AdjInIndex, + SegmentComponentKind::AdjInPostings, + SegmentComponentKind::TimestampIndex, + ] { + if !has_component(&kind) { + return Err(EngineError::CorruptRecord(format!( + "refusing to publish {SEGMENT_COMPONENT_MANIFEST_FILENAME} for segment {segment_id}: missing required component {kind:?}" + ))); + } + let record = records + .iter() + .find(|record| record.kind == kind) + .expect("required component was checked above"); + if !matches!(record.handle, ComponentHandleV1::PackedRange { .. }) { + return Err(EngineError::CorruptRecord(format!( + "refusing to publish {SEGMENT_COMPONENT_MANIFEST_FILENAME} for segment {segment_id}: required component {kind:?} is not packed" + ))); + } + } - let count = entries.len() as u64; - write_u64(&mut w, count)?; + let has_vector_blob = has_component(&SegmentComponentKind::NodeDenseVectorBlob) + || has_component(&SegmentComponentKind::NodeSparseVectorBlob); + if has_vector_blob && !has_component(&SegmentComponentKind::NodeVectorMetadata) { + return Err(EngineError::CorruptRecord(format!( + "refusing to publish {SEGMENT_COMPONENT_MANIFEST_FILENAME} for segment {segment_id}: vector blob component is missing NodeVectorMetadata" + ))); + } + for kind in [ + SegmentComponentKind::NodeVectorMetadata, + SegmentComponentKind::NodeDenseVectorBlob, + SegmentComponentKind::NodeSparseVectorBlob, + ] { + if let Some(record) = records.iter().find(|record| record.kind == kind) { + if !matches!(record.handle, ComponentHandleV1::PackedRange { .. }) { + return Err(EngineError::CorruptRecord(format!( + "refusing to publish {SEGMENT_COMPONENT_MANIFEST_FILENAME} for segment {segment_id}: vector source truth component {kind:?} is not packed" + ))); + } + } + } - for &(type_id, updated_at, node_id) in &entries { - write_u32(&mut w, type_id)?; - w.write_all(&updated_at.to_le_bytes())?; - write_u64(&mut w, node_id)?; + for record in records { + if matches!(record.handle, ComponentHandleV1::ExternalFile { .. }) + && record.kind != SegmentComponentKind::PackedSegmentContainer + && !is_refreshable_external_component_kind(&record.kind) + { + return Err(EngineError::CorruptRecord(format!( + "refusing to publish {SEGMENT_COMPONENT_MANIFEST_FILENAME} for segment {segment_id}: component {:?} is not allowed as an external file", + record.kind + ))); + } } - w.flush()?; - w.get_ref().sync_all()?; - Ok(()) -} + validate_packed_core_records_contract(records)?; -/// Write empty tombstones.dat (count=0). After compaction, tombstones are consumed. -fn write_empty_tombstones(seg_dir: &Path) -> Result<(), EngineError> { - let path = seg_dir.join("tombstones.dat"); - let file = File::create(&path)?; - let mut w = BufWriter::new(file); - write_u64(&mut w, 0)?; - w.flush()?; - w.get_ref().sync_all()?; Ok(()) } -/// Write output metadata sidecars from compaction metadata. -fn write_sidecars_from_meta( - seg_dir: &Path, - segments: &[Arc], - node_metas: &[CompactNodeMeta], - edge_metas: &[CompactEdgeMeta], -) -> Result, EngineError> { - // node_meta.dat - let meta_path = seg_dir.join("node_meta.dat"); - - let meta_file = File::create(&meta_path)?; - let mut meta_w = BufWriter::new(meta_file); +fn sort_component_records_for_manifest(records: &mut [SegmentComponentRecordV1]) { + records.sort_by(|left, right| { + let left_key = ( + left.kind.kind_tag(), + left.kind.index_id().unwrap_or(0), + left.created_generation, + ); + let right_key = ( + right.kind.kind_tag(), + right.kind.index_id().unwrap_or(0), + right.created_generation, + ); + left_key + .cmp(&right_key) + .then_with(|| compare_component_handles(&left.handle, &right.handle)) + }); +} - let count = node_metas.len() as u64; - write_u64(&mut meta_w, count)?; +fn compare_component_handles( + left: &ComponentHandleV1, + right: &ComponentHandleV1, +) -> std::cmp::Ordering { + use std::cmp::Ordering; + match (left, right) { + ( + ComponentHandleV1::ExternalFile { + relative_path: left_path, + payload_offset: left_offset, + payload_len: left_len, + }, + ComponentHandleV1::ExternalFile { + relative_path: right_path, + payload_offset: right_offset, + payload_len: right_len, + }, + ) => (0u8, left_path.as_str(), *left_offset, *left_len).cmp(&( + 0u8, + right_path.as_str(), + *right_offset, + *right_len, + )), + ( + ComponentHandleV1::PackedRange { + offset: left_offset, + len: left_len, + .. + }, + ComponentHandleV1::PackedRange { + offset: right_offset, + len: right_len, + .. + }, + ) => (1u8, *left_offset, *left_len).cmp(&(1u8, *right_offset, *right_len)), + (ComponentHandleV1::ExternalFile { .. }, ComponentHandleV1::PackedRange { .. }) => { + Ordering::Less + } + (ComponentHandleV1::PackedRange { .. }, ComponentHandleV1::ExternalFile { .. }) => { + Ordering::Greater + } + } +} - for nm in node_metas { - // Write node_meta entry with updated data_offset and prop_hash_offset (60 bytes) - write_u64(&mut meta_w, nm.node_id)?; - write_u64(&mut meta_w, nm.new_data_offset)?; - write_u32(&mut meta_w, nm.data_len)?; - write_u32(&mut meta_w, nm.type_id)?; - meta_w.write_all(&nm.updated_at.to_le_bytes())?; - meta_w.write_all(&nm.weight.to_le_bytes())?; - write_u16(&mut meta_w, nm.key_len)?; - write_u64(&mut meta_w, 0)?; - write_u32(&mut meta_w, 0)?; - write_u64(&mut meta_w, nm.last_write_seq)?; - write_u16(&mut meta_w, 0)?; // reserved +fn write_segment_component_manifest( + seg_dir: &Path, + manifest: &SegmentComponentManifestV1, +) -> Result<(), EngineError> { + let tmp_path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_TMP_FILENAME); + let final_path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + let data = encode_manifest_envelope(manifest)?; + { + let mut file = File::create(&tmp_path)?; + file.write_all(&data)?; + file.sync_all()?; } + fs::rename(&tmp_path, &final_path)?; + fsync_dir(seg_dir)?; + Ok(()) +} - meta_w.flush()?; - meta_w.get_ref().sync_all()?; +fn read_segment_component_manifest( + seg_dir: &Path, +) -> Result { + let path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + let data = fs::read(&path)?; + decode_manifest_envelope(&data) +} - let dense_points = write_node_vector_sidecars_from_meta(seg_dir, segments, node_metas)?; +pub(crate) fn cleanup_orphan_optional_component_files(seg_dir: &Path) { + let Ok(manifest) = read_segment_component_manifest(seg_dir) else { + return; + }; - // edge_meta.dat - let edge_meta_path = seg_dir.join("edge_meta.dat"); - let edge_meta_file = File::create(&edge_meta_path)?; - let mut em_w = BufWriter::new(edge_meta_file); + let mut referenced = HashSet::new(); + for record in &manifest.components { + if let ComponentHandleV1::ExternalFile { relative_path, .. } = &record.handle { + referenced.insert(PathBuf::from(relative_path)); + } + } + for record in &manifest.unknown_optional_components { + if record.wire.handle.handle_tag == 1 { + if let Some(relative_path) = &record.wire.handle.relative_path { + referenced.insert(PathBuf::from(relative_path)); + } + } + } - let edge_count = edge_metas.len() as u64; - write_u64(&mut em_w, edge_count)?; + let mut dirty_dirs = HashSet::new(); + cleanup_orphan_optional_component_files_in_dir(seg_dir, seg_dir, &referenced, &mut dirty_dirs); + for dir in dirty_dirs { + let _ = fsync_dir(&dir); + } +} - for em in edge_metas { - write_u64(&mut em_w, em.edge_id)?; - write_u64(&mut em_w, em.new_data_offset)?; - write_u32(&mut em_w, em.data_len)?; - write_u64(&mut em_w, em.from)?; - write_u64(&mut em_w, em.to)?; - write_u32(&mut em_w, em.type_id)?; - em_w.write_all(&em.updated_at.to_le_bytes())?; - em_w.write_all(&em.weight.to_le_bytes())?; - em_w.write_all(&em.valid_from.to_le_bytes())?; - em_w.write_all(&em.valid_to.to_le_bytes())?; - write_u64(&mut em_w, em.last_write_seq)?; - write_u32(&mut em_w, 0)?; // reserved - } - - em_w.flush()?; - em_w.get_ref().sync_all()?; - Ok(dense_points) -} - -fn write_node_vector_sidecars_from_meta( +fn cleanup_orphan_optional_component_files_in_dir( seg_dir: &Path, - segments: &[Arc], - node_metas: &[CompactNodeMeta], -) -> Result, EngineError> { - let has_dense = node_metas.iter().any(|nm| nm.dense_vector_len > 0); - let has_sparse = node_metas.iter().any(|nm| nm.sparse_vector_len > 0); + dir: &Path, + referenced: &HashSet, + dirty_dirs: &mut HashSet, +) { + let entries = match fs::read_dir(dir) { + Ok(entries) => entries, + Err(_) => return, + }; + for entry in entries.flatten() { + let path = entry.path(); + let file_type = match entry.file_type() { + Ok(file_type) => file_type, + Err(_) => continue, + }; + if file_type.is_dir() { + cleanup_orphan_optional_component_files_in_dir(seg_dir, &path, referenced, dirty_dirs); + continue; + } + if !file_type.is_file() { + continue; + } - if !has_dense && !has_sparse { - return Ok(Vec::new()); + let Ok(relative_path) = path.strip_prefix(seg_dir) else { + continue; + }; + if referenced.contains(relative_path) { + continue; + } + let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if (is_optional_refresh_tmp_file_name(file_name) + || is_optional_generation_file_name(file_name)) + && fs::remove_file(&path).is_ok() + { + if let Some(parent) = path.parent() { + dirty_dirs.insert(parent.to_path_buf()); + } + } } +} - let meta_file = File::create(seg_dir.join(NODE_VECTOR_META_FILENAME))?; - let mut meta_w = BufWriter::new(meta_file); - write_u64(&mut meta_w, node_metas.len() as u64)?; +fn is_optional_refresh_tmp_file_name(file_name: &str) -> bool { + file_name.contains(".refresh_tmp.") +} - let mut dense_w = if has_dense { - Some(BufWriter::new(File::create( - seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME), - )?)) - } else { - None +fn is_optional_generation_file_name(file_name: &str) -> bool { + let Some(g_pos) = file_name.rfind(".g") else { + return false; }; - let mut sparse_w = if has_sparse { - Some(BufWriter::new(File::create( - seg_dir.join(NODE_SPARSE_VECTOR_BLOB_FILENAME), - )?)) - } else { - None + let generation = &file_name[g_pos + 2..]; + let generation = generation + .split_once('.') + .map_or(generation, |(generation, _)| generation); + generation.len() == 16 && generation.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn optional_generation_relative_path(base_relative_path: &str, generation: u64) -> String { + let path = Path::new(base_relative_path); + let stem = path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or(base_relative_path); + let extension = path.extension().and_then(|value| value.to_str()); + let file_name = match extension { + Some(extension) => format!("{stem}.g{generation:016}.{extension}"), + None => format!("{stem}.g{generation:016}"), }; + match path.parent().and_then(|parent| parent.to_str()) { + Some(parent) if !parent.is_empty() => format!("{parent}/{file_name}"), + _ => file_name, + } +} - let mut new_dense_offset = 0u64; - let mut new_sparse_offset = 0u64; - let mut dense_points = Vec::new(); +fn optional_refresh_tmp_relative_path(base_relative_path: &str, generation: u64) -> String { + let path = Path::new(base_relative_path); + let stem = path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or(base_relative_path); + let nonce = OPTIONAL_REFRESH_TMP_NONCE.fetch_add(1, Ordering::Relaxed); + let file_name = format!( + ".{stem}.refresh_tmp.{}.{}.g{generation:016}.dat", + std::process::id(), + nonce + ); + match path.parent().and_then(|parent| parent.to_str()) { + Some(parent) if !parent.is_empty() => format!("{parent}/{file_name}"), + _ => file_name, + } +} - for nm in node_metas { - let mut flags = 0u8; - let mut entry_dense_offset = 0u64; - let mut entry_sparse_offset = 0u64; +#[allow(clippy::too_many_arguments)] +fn refresh_optional_component_with_writer( + seg_dir: &Path, + kind: SegmentComponentKind, + base_relative_path: &str, + requirement: ComponentRequirement, + trust_class: ComponentTrustClass, + dependencies: Vec, + build_fingerprint: u64, + write_payload: impl FnOnce(&mut ComponentIdentityWriter) -> Result<(), EngineError>, +) -> Result<(), EngineError> { + if !is_refreshable_external_component_kind(&kind) { + return Err(EngineError::CorruptRecord(format!( + "component {kind:?} is not eligible for optional external refresh" + ))); + } + if matches!(&requirement, ComponentRequirement::Required) { + return Err(EngineError::CorruptRecord(format!( + "component {kind:?} optional refresh cannot publish a required component" + ))); + } - if nm.dense_vector_len > 0 { - flags |= NODE_VECTOR_FLAG_DENSE; - entry_dense_offset = new_dense_offset; - } - if nm.sparse_vector_len > 0 { - flags |= NODE_VECTOR_FLAG_SPARSE; - entry_sparse_offset = new_sparse_offset; - } - - write_u8(&mut meta_w, flags)?; - meta_w.write_all(&[0u8; 3])?; - write_u64(&mut meta_w, entry_dense_offset)?; - write_u32(&mut meta_w, nm.dense_vector_len)?; - write_u64(&mut meta_w, entry_sparse_offset)?; - write_u32(&mut meta_w, nm.sparse_vector_len)?; + let captured_manifest = read_segment_component_manifest(seg_dir)?; + let source_groups = segment_source_groups_from_records( + captured_manifest.segment_id, + captured_manifest.node_count, + captured_manifest.edge_count, + &captured_manifest.components, + )?; + if source_groups.segment_data_id != captured_manifest.segment_data_id { + return Err(EngineError::CorruptRecord(format!( + "segment {} source identity changed before optional publication", + captured_manifest.segment_id + ))); + } + let generation = captured_manifest.generation.saturating_add(1); + let final_relative_path = optional_generation_relative_path(base_relative_path, generation); + let tmp_relative_path = optional_refresh_tmp_relative_path(base_relative_path, generation); + let tmp_path = seg_dir.join(&tmp_relative_path); + if let Some(parent) = tmp_path.parent() { + fs::create_dir_all(parent)?; + } + + let mut writer = ComponentIdentityWriter::create( + &tmp_path, + final_relative_path.clone(), + SEGMENT_FORMAT_VERSION, + captured_manifest.segment_id, + kind.clone(), + FLUSH_COMPONENT_LOGICAL_FORMAT_VERSION, + generation, + requirement, + trust_class, + build_fingerprint, + true, + )?; + write_payload(&mut writer)?; + let record = writer.finish(dependencies)?; - if nm.dense_vector_len > 0 { - let src = segments[nm.src_seg_idx].raw_node_dense_vectors_mmap(); - let base = nm.dense_vector_offset as usize; - let len = nm.dense_vector_len as usize * DENSE_VECTOR_VALUE_SIZE as usize; - let end = base + len; - if end > src.len() { - return Err(EngineError::CorruptRecord(format!( - "node {} dense vector range [{}, {}) exceeds source length {}", - nm.node_id, - base, - end, - src.len() - ))); - } - let mut values = Vec::with_capacity(nm.dense_vector_len as usize); - for index in 0..nm.dense_vector_len as usize { - let value_offset = base + index * DENSE_VECTOR_VALUE_SIZE as usize; - values.push(f32::from_le_bytes( - src[value_offset..value_offset + DENSE_VECTOR_VALUE_SIZE as usize] - .try_into() - .unwrap(), - )); - } - dense_points.push(DensePointInput { - node_id: nm.node_id, - dense_vector_offset: entry_dense_offset, - values, - }); - dense_w - .as_mut() - .expect("dense blob writer must exist") - .write_all(&src[base..end])?; - new_dense_offset = new_dense_offset.checked_add(len as u64).ok_or_else(|| { - EngineError::CorruptRecord("dense vector output offset overflow".into()) - })?; - } + let current_manifest = read_segment_component_manifest(seg_dir)?; + if current_manifest.segment_id != captured_manifest.segment_id + || current_manifest.segment_data_id != captured_manifest.segment_data_id + || current_manifest.generation != captured_manifest.generation + { + return Err(EngineError::CorruptRecord(format!( + "segment {} {}", + captured_manifest.segment_id, OPTIONAL_COMPONENT_PUBLICATION_CONFLICT_MESSAGE + ))); + } - if nm.sparse_vector_len > 0 { - let src = segments[nm.src_seg_idx].raw_node_sparse_vectors_mmap(); - let base = nm.sparse_vector_offset as usize; - let len = nm.sparse_vector_len as usize * SPARSE_VECTOR_ENTRY_SIZE as usize; - let end = base + len; - if end > src.len() { - return Err(EngineError::CorruptRecord(format!( - "node {} sparse vector range [{}, {}) exceeds source length {}", - nm.node_id, - base, - end, - src.len() - ))); - } - sparse_w - .as_mut() - .expect("sparse blob writer must exist") - .write_all(&src[base..end])?; - new_sparse_offset = new_sparse_offset.checked_add(len as u64).ok_or_else(|| { - EngineError::CorruptRecord("sparse vector output offset overflow".into()) - })?; - } + let final_path = seg_dir.join(&final_relative_path); + fs::rename(&tmp_path, &final_path)?; + if let Some(parent) = final_path.parent() { + fsync_dir(parent)?; } - meta_w.flush()?; - meta_w.get_ref().sync_all()?; + let mut replacement = current_manifest; + replacement.generation = generation; + replacement.built_at_ms = current_time_millis(); + replacement.build_kind = SegmentComponentBuildKind::OptionalRefresh; + replacement + .components + .retain(|existing| existing.kind != kind); + replacement.components.push(record); + write_segment_component_manifest(seg_dir, &replacement) +} - if let Some(mut w) = dense_w { - w.flush()?; - w.get_ref().sync_all()?; - } - if let Some(mut w) = sparse_w { - w.flush()?; - w.get_ref().sync_all()?; +/// Fsync the directory to ensure metadata (file creation) is durable. +/// No-op on Windows. NTFS doesn't support directory fsync via File::open(). +fn fsync_dir(dir: &Path) -> Result<(), EngineError> { + #[cfg(not(target_os = "windows"))] + { + let d = File::open(dir)?; + d.sync_all()?; } + #[cfg(target_os = "windows")] + let _ = dir; + Ok(()) +} - Ok(dense_points) +/// Return the segment directory path for a given segment ID within a db directory. +pub fn segment_dir(db_dir: &Path, segment_id: u64) -> PathBuf { + db_dir + .join("segments") + .join(format!("seg_{:04}", segment_id)) } -fn write_sparse_posting_index( - seg_dir: &Path, - nodes: &NodeIdMap, -) -> Result<(), EngineError> { - let mut groups: BTreeMap> = BTreeMap::new(); - for node in nodes.values() { - let Some(values) = node.sparse_vector.as_ref() else { - continue; - }; - for &(dimension_id, weight) in values { - groups - .entry(dimension_id) - .or_default() - .push((node.id, weight)); - } - } - sort_sparse_posting_groups(&mut groups)?; - write_sparse_posting_files(seg_dir, &groups) +/// Return the temporary segment directory path (used during flush before atomic rename). +pub fn segment_tmp_dir(db_dir: &Path, segment_id: u64) -> PathBuf { + db_dir + .join("segments") + .join(format!("seg_{:04}.tmp", segment_id)) } -fn write_sparse_posting_index_from_meta( - seg_dir: &Path, +// --- Fast-merge compaction support --- + +pub(crate) struct FastMergeCopyInfo { + pub orig_data_start: u64, + pub new_data_base: u64, +} + +/// Write merged node records payload by binary copy from multiple non-overlapping segments. +/// +/// Instead of deserializing and re-serializing every record, this copies raw +/// record bytes directly from mmap'd input segments and rebuilds the merged +/// index with adjusted offsets. Record lengths are derived from the source +/// node records index/data layout, which lets the fast path cross-check metadata +/// metadata later instead of trusting it blindly. +/// +/// Returns per-segment offset rebasing info so compaction metadata can compute +/// merged `data_offset` values directly from sidecars without a second data scan. +pub(crate) fn write_merged_nodes_dat( + core_writer: &mut PackedCoreWriter, segments: &[Arc], - node_metas: &[CompactNodeMeta], -) -> Result<(), EngineError> { - let mut groups: BTreeMap> = BTreeMap::new(); - for nm in node_metas { - if nm.sparse_vector_len == 0 { - continue; - } - let src = segments[nm.src_seg_idx].raw_node_sparse_vectors_mmap(); - let base = nm.sparse_vector_offset as usize; - let len = nm.sparse_vector_len as usize * SPARSE_VECTOR_ENTRY_SIZE as usize; - let end = base + len; - if end > src.len() { +) -> Result<(SegmentComponentRecordV1, Vec), EngineError> { + let mut seg_info: Vec<(u64, usize, usize)> = Vec::with_capacity(segments.len()); + let mut total_count: u64 = 0; + + for (seg_idx, seg) in segments.iter().enumerate() { + let mmap = seg.raw_nodes_mmap(); + if mmap.len() < 8 { return Err(EngineError::CorruptRecord(format!( - "node {} sparse posting source range [{}, {}) exceeds source length {}", - nm.node_id, - base, - end, - src.len() + "segment {} node records payload too short for count header: {} bytes", + seg.segment_id, + mmap.len() ))); } - for index in 0..nm.sparse_vector_len as usize { - let entry_offset = base + index * SPARSE_VECTOR_ENTRY_SIZE as usize; - let dimension_id = - u32::from_le_bytes(src[entry_offset..entry_offset + 4].try_into().unwrap()); - let weight = - f32::from_le_bytes(src[entry_offset + 4..entry_offset + 8].try_into().unwrap()); - groups - .entry(dimension_id) - .or_default() - .push((nm.node_id, weight)); + let count = u64::from_le_bytes(mmap[0..8].try_into().unwrap()); + let index_bytes = (count as usize) + .checked_mul(NODE_INDEX_ENTRY_SIZE as usize) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} node index size overflow for {} entries", + seg.segment_id, count + )) + })?; + let data_start = 8usize.checked_add(index_bytes).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} node data start overflow", + seg.segment_id + )) + })?; + if data_start > mmap.len() { + return Err(EngineError::CorruptRecord(format!( + "segment {} node records index exceeds payload length: start={}, len={}", + seg.segment_id, + data_start, + mmap.len() + ))); } + seg_info.push((count, data_start, mmap.len() - data_start)); + total_count = total_count.checked_add(count).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "total node count overflow while merging segment {} (index {})", + seg.segment_id, seg_idx + )) + })?; } - sort_sparse_posting_groups(&mut groups)?; - write_sparse_posting_files(seg_dir, &groups) -} -fn sort_sparse_posting_groups( - groups: &mut BTreeMap>, -) -> Result<(), EngineError> { - for (&dimension_id, postings) in groups.iter_mut() { - postings.sort_unstable_by_key(|&(node_id, _)| node_id); - for window in postings.windows(2) { - if window[0].0 == window[1].0 { - return Err(EngineError::CorruptRecord(format!( - "sparse posting dimension {} has duplicate node {}", - dimension_id, window[0].0 - ))); + let (record, copy_info) = core_writer.write_component( + SegmentComponentKind::NodeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.nodes", &[]), + |w| { + write_u64(w, total_count)?; + + let merged_data_start = + 8u64.checked_add(total_count.checked_mul(NODE_INDEX_ENTRY_SIZE).ok_or_else( + || EngineError::CorruptRecord("merged node index size overflow".into()), + )?) + .ok_or_else(|| { + EngineError::CorruptRecord("merged node data start overflow".into()) + })?; + let mut cumulative_data_offset = merged_data_start; + let mut data_offsets: Vec = Vec::with_capacity(segments.len()); + for &(_, _, data_size) in &seg_info { + data_offsets.push(cumulative_data_offset); + cumulative_data_offset = cumulative_data_offset + .checked_add(data_size as u64) + .ok_or_else(|| { + EngineError::CorruptRecord( + "merged node records payload size overflow".into(), + ) + })?; } - } - } - Ok(()) -} + let mut all_entries: Vec<(u64, u64)> = Vec::with_capacity(total_count as usize); + for (seg_idx, seg) in segments.iter().enumerate() { + let mmap = seg.raw_nodes_mmap(); + let (count, orig_data_start, _) = seg_info[seg_idx]; + if count == 0 { + continue; + } -#[cfg(test)] -mod tests { - use super::*; - use crate::degree_cache::DegreeDelta; - use std::sync::Arc; + let offset_adj = data_offsets[seg_idx] + .checked_sub(orig_data_start as u64) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} node offset adjustment underflow", + seg.segment_id + )) + })?; + + for i in 0..count as usize { + let entry_off = 8 + i * NODE_INDEX_ENTRY_SIZE as usize; + let node_id = + u64::from_le_bytes(mmap[entry_off..entry_off + 8].try_into().unwrap()); + let old_offset = + u64::from_le_bytes(mmap[entry_off + 8..entry_off + 16].try_into().unwrap()); + let new_offset = old_offset.checked_add(offset_adj).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} node {} merged offset overflow", + seg.segment_id, node_id + )) + })?; + all_entries.push((node_id, new_offset)); + } + } + all_entries.sort_unstable_by_key(|(id, _)| *id); - fn write_segment( - seg_dir: &Path, - segment_id: u64, - memtable: &Memtable, - dense_config: Option<&DenseVectorConfig>, - ) -> Result { - let degree_overlay = DegreeOverlaySnapshot::empty(); - super::write_segment_with_degree_overlay_and_secondary_indexes( - seg_dir, - segment_id, - memtable, - dense_config, - degree_overlay.as_ref(), - &[], - ) - } + for &(node_id, offset) in &all_entries { + write_u64(w, node_id)?; + write_u64(w, offset)?; + } - fn write_segment_with_secondary_indexes( - seg_dir: &Path, - segment_id: u64, - memtable: &Memtable, - dense_config: Option<&DenseVectorConfig>, - secondary_indexes: &[SecondaryIndexManifestEntry], - ) -> Result { - let degree_overlay = DegreeOverlaySnapshot::empty(); - super::write_segment_with_degree_overlay_and_secondary_indexes( - seg_dir, - segment_id, - memtable, - dense_config, - degree_overlay.as_ref(), - secondary_indexes, - ) - } + for (seg_idx, seg) in segments.iter().enumerate() { + let mmap = seg.raw_nodes_mmap(); + let (_, data_start, data_size) = seg_info[seg_idx]; + if data_size > 0 { + w.write_all(&mmap[data_start..data_start + data_size])?; + } + } - fn make_node(id: u64, type_id: u32, key: &str) -> NodeRecord { - NodeRecord { - id, - type_id, - key: key.to_string(), - props: BTreeMap::new(), - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - } - } + Ok(seg_info + .iter() + .zip(data_offsets) + .map(|((_, data_start, _), new_data_base)| FastMergeCopyInfo { + orig_data_start: *data_start as u64, + new_data_base, + }) + .collect()) + }, + )?; + Ok((record, copy_info)) +} - fn make_node_with_props(id: u64, type_id: u32, key: &str) -> NodeRecord { - let mut props = BTreeMap::new(); - props.insert("name".to_string(), PropValue::String(key.to_string())); - props.insert("score".to_string(), PropValue::Float(0.95)); - NodeRecord { - id, - type_id, - key: key.to_string(), - props, - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - } - } +/// Write merged edge records payload by binary copy from multiple non-overlapping segments. +/// Same approach as `write_merged_nodes_dat`. +/// +/// Returns per-segment offset rebasing info so compaction metadata can compute +/// merged `data_offset` values directly from sidecars without a second data scan. +pub(crate) fn write_merged_edges_dat( + core_writer: &mut PackedCoreWriter, + segments: &[Arc], +) -> Result<(SegmentComponentRecordV1, Vec), EngineError> { + let mut seg_info: Vec<(u64, usize, usize)> = Vec::with_capacity(segments.len()); + let mut total_count: u64 = 0; - fn make_edge(id: u64, from: u64, to: u64, type_id: u32) -> EdgeRecord { + for (seg_idx, seg) in segments.iter().enumerate() { + let mmap = seg.raw_edges_mmap(); + if mmap.len() < 8 { + return Err(EngineError::CorruptRecord(format!( + "segment {} edge records payload too short for count header: {} bytes", + seg.segment_id, + mmap.len() + ))); + } + let count = u64::from_le_bytes(mmap[0..8].try_into().unwrap()); + let index_bytes = (count as usize) + .checked_mul(EDGE_INDEX_ENTRY_SIZE as usize) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} edge index size overflow for {} entries", + seg.segment_id, count + )) + })?; + let data_start = 8usize.checked_add(index_bytes).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} edge data start overflow", + seg.segment_id + )) + })?; + if data_start > mmap.len() { + return Err(EngineError::CorruptRecord(format!( + "segment {} edge records index exceeds payload length: start={}, len={}", + seg.segment_id, + data_start, + mmap.len() + ))); + } + seg_info.push((count, data_start, mmap.len() - data_start)); + total_count = total_count.checked_add(count).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "total edge count overflow while merging segment {} (index {})", + seg.segment_id, seg_idx + )) + })?; + } + + let (record, copy_info) = core_writer.write_component( + SegmentComponentKind::EdgeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.edges", &[]), + |w| { + write_u64(w, total_count)?; + + let merged_data_start = + 8u64.checked_add(total_count.checked_mul(EDGE_INDEX_ENTRY_SIZE).ok_or_else( + || EngineError::CorruptRecord("merged edge index size overflow".into()), + )?) + .ok_or_else(|| { + EngineError::CorruptRecord("merged edge data start overflow".into()) + })?; + let mut cumulative_data_offset = merged_data_start; + let mut data_offsets: Vec = Vec::with_capacity(segments.len()); + for &(_, _, data_size) in &seg_info { + data_offsets.push(cumulative_data_offset); + cumulative_data_offset = cumulative_data_offset + .checked_add(data_size as u64) + .ok_or_else(|| { + EngineError::CorruptRecord( + "merged edge records payload size overflow".into(), + ) + })?; + } + + let mut all_entries: Vec<(u64, u64)> = Vec::with_capacity(total_count as usize); + for (seg_idx, seg) in segments.iter().enumerate() { + let mmap = seg.raw_edges_mmap(); + let (count, orig_data_start, _) = seg_info[seg_idx]; + if count == 0 { + continue; + } + + let offset_adj = data_offsets[seg_idx] + .checked_sub(orig_data_start as u64) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} edge offset adjustment underflow", + seg.segment_id + )) + })?; + + for i in 0..count as usize { + let entry_off = 8 + i * EDGE_INDEX_ENTRY_SIZE as usize; + let edge_id = + u64::from_le_bytes(mmap[entry_off..entry_off + 8].try_into().unwrap()); + let old_offset = + u64::from_le_bytes(mmap[entry_off + 8..entry_off + 16].try_into().unwrap()); + let new_offset = old_offset.checked_add(offset_adj).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "segment {} edge {} merged offset overflow", + seg.segment_id, edge_id + )) + })?; + all_entries.push((edge_id, new_offset)); + } + } + all_entries.sort_unstable_by_key(|(id, _)| *id); + + for &(edge_id, offset) in &all_entries { + write_u64(w, edge_id)?; + write_u64(w, offset)?; + } + + for (seg_idx, seg) in segments.iter().enumerate() { + let mmap = seg.raw_edges_mmap(); + let (_, data_start, data_size) = seg_info[seg_idx]; + if data_size > 0 { + w.write_all(&mmap[data_start..data_start + data_size])?; + } + } + + Ok(seg_info + .iter() + .zip(data_offsets) + .map(|((_, data_start, _), new_data_base)| FastMergeCopyInfo { + orig_data_start: *data_start as u64, + new_data_base, + }) + .collect()) + }, + )?; + Ok((record, copy_info)) +} + +/// Write node records payload by raw-copying only winning record byte spans from source segments. +/// +/// Used by V3 compaction: the planner has already decided which records win, +/// so we skip all dropped records entirely (never decode them). +/// +/// `winners` is sorted by node_id: `(node_id, seg_idx, data_offset, data_len)`. +/// +/// Returns Vec of `(node_id, new_data_offset, data_len)` matching the output payload, +/// for metadata writing. +pub(crate) fn write_v3_nodes_dat( + core_writer: &mut PackedCoreWriter, + segments: &[Arc], + winners: &[(u64, usize, u64, u32)], +) -> Result { + core_writer.write_component( + SegmentComponentKind::NodeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.nodes", &[]), + |w| { + let count = winners.len() as u64; + write_u64(w, count)?; + + // Calculate data section start + let data_start = 8 + count * NODE_INDEX_ENTRY_SIZE; + + // Build index entries and output info + let mut node_data = Vec::with_capacity(winners.len()); + let mut data_offset = data_start; + for &(node_id, _, _, data_len) in winners { + // Write index entry: (node_id, offset) + write_u64(w, node_id)?; + write_u64(w, data_offset)?; + node_data.push((node_id, data_offset, data_len)); + data_offset += data_len as u64; + } + + // Write data section by copying raw bytes from source segments + for &(node_id, seg_idx, src_offset, data_len) in winners { + let mmap = segments[seg_idx].raw_nodes_mmap(); + let start = src_offset as usize; + let end = start.checked_add(data_len as usize).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "node {} data span offset overflow: start={}, len={}", + node_id, start, data_len + )) + })?; + if end > mmap.len() { + return Err(EngineError::CorruptRecord(format!( + "node {} data span [{}, {}) exceeds mmap length {}", + node_id, + start, + end, + mmap.len() + ))); + } + w.write_all(&mmap[start..end])?; + } + + Ok(node_data) + }, + ) +} + +/// Write edge records payload by raw-copying only winning record byte spans from source segments. +/// +/// Same approach as `write_v3_nodes_dat` but for edge records. +/// +/// `winners` is sorted by edge_id: `(edge_id, seg_idx, data_offset, data_len)`. +/// +/// Returns Vec of `(edge_id, new_data_offset, data_len)` matching the output payload. +pub(crate) fn write_v3_edges_dat( + core_writer: &mut PackedCoreWriter, + segments: &[Arc], + winners: &[(u64, usize, u64, u32)], +) -> Result { + core_writer.write_component( + SegmentComponentKind::EdgeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.edges", &[]), + |w| { + let count = winners.len() as u64; + write_u64(w, count)?; + + let data_start = 8 + count * EDGE_INDEX_ENTRY_SIZE; + + let mut edge_data = Vec::with_capacity(winners.len()); + let mut data_offset = data_start; + for &(edge_id, _, _, data_len) in winners { + write_u64(w, edge_id)?; + write_u64(w, data_offset)?; + edge_data.push((edge_id, data_offset, data_len)); + data_offset += data_len as u64; + } + + for &(edge_id, seg_idx, src_offset, data_len) in winners { + let mmap = segments[seg_idx].raw_edges_mmap(); + let start = src_offset as usize; + let end = start.checked_add(data_len as usize).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "edge {} data span offset overflow: start={}, len={}", + edge_id, start, data_len + )) + })?; + if end > mmap.len() { + return Err(EngineError::CorruptRecord(format!( + "edge {} data span [{}, {}) exceeds mmap length {}", + edge_id, + start, + end, + mmap.len() + ))); + } + w.write_all(&mmap[start..end])?; + } + + Ok(edge_data) + }, + ) +} + +// ========================================================================== +// Metadata-driven compaction index writers (V3) +// ========================================================================== + +/// Node metadata collected from source sidecars for metadata-driven index building. +pub(crate) struct CompactNodeMeta { + pub node_id: u64, + pub new_data_offset: u64, + pub data_len: u32, + pub label_ids: NodeLabelSet, + pub updated_at: i64, + pub weight: f32, + pub key_len: u16, + pub dense_vector_offset: u64, + pub dense_vector_len: u32, + pub sparse_vector_offset: u64, + pub sparse_vector_len: u32, + pub src_seg_idx: usize, + pub src_data_offset: u64, + pub last_write_seq: u64, +} + +/// Edge metadata collected from source sidecars for metadata-driven index building. +pub(crate) struct CompactEdgeMeta { + pub edge_id: u64, + pub new_data_offset: u64, + pub data_len: u32, + pub from: u64, + pub to: u64, + pub label_id: u32, + pub updated_at: i64, + pub weight: f32, + pub valid_from: i64, + pub valid_to: i64, + pub src_seg_idx: usize, + pub src_data_offset: u64, + pub last_write_seq: u64, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn write_compaction_source_components( + segment_id: u64, + core_writer: &mut PackedCoreWriter, + segments: &[Arc], + node_record: SegmentComponentRecordV1, + edge_record: SegmentComponentRecordV1, + node_metas: &[CompactNodeMeta], + edge_metas: &[CompactEdgeMeta], +) -> Result<(SegmentComponentSourceGroups, Vec), EngineError> { + let mut records = vec![node_record, edge_record]; + + let (node_meta, _) = core_writer.write_component( + SegmentComponentKind::NodeMetadata, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryMetadata, + Vec::new(), + component_fingerprint("flush.node_meta", &[]), + |writer| write_compact_node_meta_payload(writer, node_metas), + )?; + records.push(node_meta); + + let (edge_meta, _) = core_writer.write_component( + SegmentComponentKind::EdgeMetadata, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryMetadata, + Vec::new(), + component_fingerprint("flush.edge_meta", &[]), + |writer| write_compact_edge_meta_payload(writer, edge_metas), + )?; + records.push(edge_meta); + + let (tombstones, _) = core_writer.write_component( + SegmentComponentKind::Tombstones, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryMetadata, + Vec::new(), + component_fingerprint("flush.tombstones", &[]), + |writer| write_u64(writer, 0), + )?; + records.push(tombstones); + + let preliminary_source_groups = segment_source_groups_from_records( + segment_id, + node_metas.len() as u64, + edge_metas.len() as u64, + &records, + )?; + let (vector_records, dense_points) = write_node_vector_source_components_from_meta( + core_writer, + segments, + node_metas, + preliminary_source_groups.node_source, + )?; + records.extend(vector_records); + + let source_groups = segment_source_groups_from_records( + segment_id, + node_metas.len() as u64, + edge_metas.len() as u64, + &records, + )?; + Ok((source_groups, dense_points)) +} + +fn write_compact_node_meta_payload( + mut w: &mut impl Write, + node_metas: &[CompactNodeMeta], +) -> Result<(), EngineError> { + write_u64(&mut w, node_metas.len() as u64)?; + write_u16(&mut w, NODE_META_FIXED_ENTRY_SIZE)?; + write_u16(&mut w, NODE_META_LABEL_OFFSET_ENTRY_SIZE)?; + write_u32(&mut w, 0)?; + let count = node_metas.len() as u64; + let fixed_entries_offset = NODE_META_HEADER_SIZE; + let fixed_entries_len = count + .checked_mul(NODE_META_FIXED_ENTRY_SIZE as u64) + .ok_or_else(|| EngineError::CorruptRecord("node metadata fixed table overflow".into()))?; + let label_offsets_offset = fixed_entries_offset + .checked_add(fixed_entries_len) + .ok_or_else(|| EngineError::CorruptRecord("node metadata offset overflow".into()))?; + let label_offset_entries = count.checked_add(1).ok_or_else(|| { + EngineError::CorruptRecord("node metadata label offset count overflow".into()) + })?; + let label_ids_offset = label_offsets_offset + .checked_add(label_offset_entries * NODE_META_LABEL_OFFSET_ENTRY_SIZE as u64) + .ok_or_else(|| { + EngineError::CorruptRecord("node metadata label ID offset overflow".into()) + })?; + let mut label_offsets = Vec::with_capacity(label_offset_entries as usize); + let mut label_ids = Vec::new(); + label_offsets.push(0u64); + for nm in node_metas { + label_ids.extend_from_slice(nm.label_ids.as_slice()); + label_offsets.push(label_ids.len() as u64); + } + write_u64(&mut w, fixed_entries_offset)?; + write_u64(&mut w, label_offsets_offset)?; + write_u64(&mut w, label_ids_offset)?; + write_u64(&mut w, label_ids.len() as u64)?; + for nm in node_metas { + write_u64(&mut w, nm.node_id)?; + write_u64(&mut w, nm.new_data_offset)?; + write_u32(&mut w, nm.data_len)?; + w.write_all(&nm.updated_at.to_le_bytes())?; + w.write_all(&nm.weight.to_le_bytes())?; + write_u16(&mut w, nm.key_len)?; + write_u64(&mut w, nm.last_write_seq)?; + w.write_all(&[0u8; 6])?; + } + for offset in label_offsets { + write_u64(&mut w, offset)?; + } + for label_id in label_ids { + write_u32(&mut w, label_id)?; + } + Ok(()) +} + +fn write_compact_edge_meta_payload( + mut w: &mut impl Write, + edge_metas: &[CompactEdgeMeta], +) -> Result<(), EngineError> { + write_u64(&mut w, edge_metas.len() as u64)?; + for em in edge_metas { + write_u64(&mut w, em.edge_id)?; + write_u64(&mut w, em.new_data_offset)?; + write_u32(&mut w, em.data_len)?; + write_u64(&mut w, em.from)?; + write_u64(&mut w, em.to)?; + write_u32(&mut w, em.label_id)?; + w.write_all(&em.updated_at.to_le_bytes())?; + w.write_all(&em.weight.to_le_bytes())?; + w.write_all(&em.valid_from.to_le_bytes())?; + w.write_all(&em.valid_to.to_le_bytes())?; + write_u64(&mut w, em.last_write_seq)?; + write_u32(&mut w, 0)?; + } + Ok(()) +} + +fn prepare_key_index_payload_from_meta<'a>( + segments: &'a [Arc], + node_metas: &[CompactNodeMeta], +) -> Result, EngineError> { + let entry_count = node_metas + .iter() + .map(|meta| meta.label_ids.len()) + .sum::(); + let mut entries: Vec> = Vec::with_capacity(entry_count); + for nm in node_metas { + let key = raw_node_key_bytes_from_meta(segments, nm)?; + for &label_id in nm.label_ids.as_slice() { + entries.push(KeyIndexEntryPlan { + label_id, + key, + node_id: nm.node_id, + encoded_len: 4 + 8 + 2 + nm.key_len as u64, + }); + } + } + entries.sort_by(|a, b| { + a.label_id + .cmp(&b.label_id) + .then_with(|| a.key.cmp(b.key)) + .then_with(|| a.node_id.cmp(&b.node_id)) + }); + for pair in entries.windows(2) { + let left = &pair[0]; + let right = &pair[1]; + if left.label_id == right.label_id && left.key == right.key && left.node_id != right.node_id + { + let key = std::str::from_utf8(left.key).unwrap_or(""); + return Err(EngineError::InvalidOperation(format!( + "duplicate live node key membership for label {} and key '{}'", + left.label_id, key + ))); + } + } + Ok(KeyIndexPayloadPlan { entries }) +} + +fn raw_node_key_bytes_from_meta<'a>( + segments: &'a [Arc], + nm: &CompactNodeMeta, +) -> Result<&'a [u8], EngineError> { + let src_mmap = segments[nm.src_seg_idx].raw_nodes_mmap(); + let record_start = nm.src_data_offset as usize; + let record_end = record_start + .checked_add(nm.data_len as usize) + .ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} source record span overflow", nm.node_id)) + })?; + if record_end > src_mmap.len() { + return Err(EngineError::CorruptRecord(format!( + "node {} source record span [{}, {}) exceeds source mmap length {}", + nm.node_id, + record_start, + record_end, + src_mmap.len() + ))); + } + if record_start >= record_end { + return Err(EngineError::CorruptRecord(format!( + "node {} source record is empty", + nm.node_id + ))); + } + + let raw_label_count = src_mmap[record_start] as usize; + if raw_label_count != nm.label_ids.len() { + return Err(EngineError::CorruptRecord(format!( + "node {} raw label count {} does not match metadata label count {}", + nm.node_id, + raw_label_count, + nm.label_ids.len() + ))); + } + let key_len_offset = record_start + .checked_add(1) + .and_then(|offset| offset.checked_add(raw_label_count.checked_mul(4)?)) + .ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} key length offset overflow", nm.node_id)) + })?; + let key_start = key_len_offset.checked_add(2).ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} key start offset overflow", nm.node_id)) + })?; + if key_start > record_end { + return Err(EngineError::CorruptRecord(format!( + "node {} key length field exceeds source record span", + nm.node_id + ))); + } + let raw_key_len = u16::from_le_bytes( + src_mmap[key_len_offset..key_len_offset + 2] + .try_into() + .unwrap(), + ); + if raw_key_len != nm.key_len { + return Err(EngineError::CorruptRecord(format!( + "node {} raw key length {} does not match metadata key length {}", + nm.node_id, raw_key_len, nm.key_len + ))); + } + let key_end = key_start.checked_add(nm.key_len as usize).ok_or_else(|| { + EngineError::CorruptRecord(format!("node {} key end offset overflow", nm.node_id)) + })?; + if key_end > record_end { + return Err(EngineError::CorruptRecord(format!( + "node {} key bytes [{}, {}) exceed source record span ending at {}", + nm.node_id, key_start, key_end, record_end + ))); + } + Ok(&src_mmap[key_start..key_end]) +} + +fn prepare_node_label_index_payload_from_meta( + node_metas: &[CompactNodeMeta], +) -> LabelPostingIndexPayloadPlan { + let mut groups: BTreeMap> = BTreeMap::new(); + for nm in node_metas { + for &label_id in nm.label_ids.as_slice() { + groups.entry(label_id).or_default().push(nm.node_id); + } + } + for ids in groups.values_mut() { + ids.sort_unstable(); + ids.dedup(); + } + LabelPostingIndexPayloadPlan { + groups: groups.into_iter().collect(), + } +} + +fn prepare_edge_label_index_payload_from_meta( + edge_metas: &[CompactEdgeMeta], +) -> LabelPostingIndexPayloadPlan { + let mut groups: BTreeMap> = BTreeMap::new(); + for em in edge_metas { + groups.entry(em.label_id).or_default().push(em.edge_id); + } + for ids in groups.values_mut() { + ids.sort_unstable(); + } + LabelPostingIndexPayloadPlan { + groups: groups.into_iter().collect(), + } +} + +fn prepare_timestamp_index_payload_from_meta( + node_metas: &[CompactNodeMeta], +) -> TimestampIndexPayloadPlan { + let mut entries: Vec<(u32, i64, u64)> = node_metas + .iter() + .flat_map(|nm| { + nm.label_ids + .as_slice() + .iter() + .map(move |&label_id| (label_id, nm.updated_at, nm.node_id)) + }) + .collect(); + entries.sort_unstable(); + TimestampIndexPayloadPlan { entries } +} + +fn prepare_edge_triple_index_payload_from_meta( + edge_metas: &[CompactEdgeMeta], +) -> EdgeTripleIndexPayloadPlan { + let mut entries: Vec<(u64, u64, u32, u64)> = edge_metas + .iter() + .map(|em| (em.from, em.to, em.label_id, em.edge_id)) + .collect(); + entries.sort_by(|a, b| { + a.0.cmp(&b.0) + .then(a.1.cmp(&b.1)) + .then(a.2.cmp(&b.2)) + .then(a.3.cmp(&b.3)) + }); + EdgeTripleIndexPayloadPlan { entries } +} + +fn prepare_edge_metadata_index_components_from_meta( + edge_metas: &[CompactEdgeMeta], +) -> EdgeMetadataIndexEntries { + let mut entries = EdgeMetadataIndexEntries::with_capacity(edge_metas.len()); + for edge in edge_metas { + entries.push( + edge.label_id, + edge.updated_at, + edge.weight, + edge.valid_from, + edge.valid_to, + edge.edge_id, + ); + } + entries.sort_all(); + entries +} + +fn prepare_adjacency_payloads_from_meta( + edge_metas: &[CompactEdgeMeta], + is_outgoing: bool, +) -> AdjacencyPayloadPlan { + let mut groups: AdjacencyGroups = BTreeMap::new(); + for em in edge_metas { + let (node_id, neighbor_id) = if is_outgoing { + (em.from, em.to) + } else { + (em.to, em.from) + }; + groups.entry((node_id, em.label_id)).or_default().push(( + em.edge_id, + neighbor_id, + em.weight, + em.valid_from, + em.valid_to, + )); + } + for postings in groups.values_mut() { + postings.sort_unstable_by_key(|&(edge_id, ..)| edge_id); + } + + let mut offset = 0u64; + let groups = groups + .into_iter() + .map(|((node_id, label_id), postings)| { + let group = AdjacencyGroupPlan { + node_id, + label_id, + offset, + postings, + }; + offset += adjacency_postings_len(&group.postings); + group + }) + .collect(); + AdjacencyPayloadPlan { groups } +} + +/// Build all secondary indexes and sidecars from metadata without Memtable decode. +/// Used by V3 compaction path. +/// +/// IMPORTANT: Two index-writing paths exist and must stay in sync: +/// 1. `write_segment()` (flush path, builds indexes from Memtable) +/// 2. `write_indexes_from_metadata_with_secondary_indexes()` [this fn] (compaction path) +/// +/// If you add a new index type, you MUST add it to BOTH paths. +/// +/// `node_metas` and `edge_metas` must be sorted by ID. +#[allow(clippy::too_many_arguments)] +pub(crate) fn write_indexes_from_metadata_with_secondary_indexes( + segment_id: u64, + seg_dir: &Path, + core_writer: &mut PackedCoreWriter, + segments: &[Arc], + node_metas: &[CompactNodeMeta], + edge_metas: &[CompactEdgeMeta], + dense_config: Option<&DenseVectorConfig>, + dense_points: Vec, + write_degree_sidecar: bool, + secondary_indexes: &[SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result { + let partitions = partition_secondary_indexes(secondary_indexes); + let (index_result, stats_core_result) = engine_cpu_join( + || { + engine_cpu_try_join( + || { + engine_cpu_try_join( + || { + engine_cpu_try_join( + || { + let key_index = + prepare_key_index_payload_from_meta(segments, node_metas)?; + let node_label_index = + prepare_node_label_index_payload_from_meta(node_metas); + let mut outcome = DeclaredSidecarWriteOutcome::default(); + let branch_outcome = + write_declared_equality_sidecars_from_metadata( + seg_dir, + segment_id, + segments, + node_metas, + &partitions.node_eq, + source_groups, + )?; + outcome + .report + .failed_equality_indexes + .extend(branch_outcome.report.failed_equality_indexes); + outcome.stats_evidence.extend(branch_outcome.stats_evidence); + outcome.records.extend(branch_outcome.records); + let branch_outcome = + write_declared_range_sidecars_from_metadata( + seg_dir, + segment_id, + segments, + node_metas, + &partitions.node_range, + source_groups, + )?; + outcome + .report + .failed_range_indexes + .extend(branch_outcome.report.failed_range_indexes); + outcome.stats_evidence.extend(branch_outcome.stats_evidence); + outcome.records.extend(branch_outcome.records); + let timestamp_index = + prepare_timestamp_index_payload_from_meta(node_metas); + outcome.stats_evidence.sort(); + Ok(( + FlushNodeIndexOutput { + key_index, + node_label_index, + timestamp_index, + external_records: outcome.records, + declared_evidence: outcome.stats_evidence, + }, + outcome.report, + )) + }, + || { + let adj_out = + prepare_adjacency_payloads_from_meta(edge_metas, true); + let adj_in = + prepare_adjacency_payloads_from_meta(edge_metas, false); + let edge_label_index = + prepare_edge_label_index_payload_from_meta(edge_metas); + let edge_triple_index = + prepare_edge_triple_index_payload_from_meta(edge_metas); + let edge_metadata_indexes = + prepare_edge_metadata_index_components_from_meta( + edge_metas, + ); + let mut external_records = Vec::new(); + if write_degree_sidecar { + if let Some(sidecars) = + degree_sidecars_for_segments(segments) + { + match write_degree_delta_from_sidecars( + seg_dir, + segment_id, + &sidecars, + source_groups, + ) { + Ok(record) => external_records.push(record), + Err(error) + if is_optional_degree_delta_invalidity( + &error, + ) => + { + let _ = fs::remove_file( + seg_dir.join(DEGREE_DELTA_FILENAME), + ); + } + Err(error) => return Err(error), + } + } + } + let edge_eq_outcome = + write_declared_edge_equality_sidecars_from_metadata( + seg_dir, + segment_id, + segments, + edge_metas, + &partitions.edge_eq, + source_groups, + )?; + let mut declared_evidence = edge_eq_outcome.stats_evidence; + external_records.extend(edge_eq_outcome.records); + let edge_range_outcome = + write_declared_edge_range_sidecars_from_metadata( + seg_dir, + segment_id, + segments, + edge_metas, + &partitions.edge_range, + source_groups, + )?; + declared_evidence.extend(edge_range_outcome.stats_evidence); + declared_evidence.sort(); + external_records.extend(edge_range_outcome.records); + Ok(FlushEdgeIndexOutput { + adj_out, + adj_in, + edge_label_index, + edge_triple_index, + edge_metadata_indexes, + external_records, + declared_evidence, + }) + }, + ) + }, + || { + write_sparse_posting_index_from_meta( + seg_dir, + segment_id, + segments, + node_metas, + source_groups, + ) + }, + ) + }, + || maybe_build_dense_hnsw(dense_points, dense_config), + ) + }, + || build_compaction_stats_core_partial(segments, node_metas, edge_metas, secondary_indexes), + ); + let (indexes_result, built_hnsw) = index_result?; + let (((node_output, report), edge_output), sparse_records) = indexes_result; + let dense_records = write_compaction_prebuilt_dense_hnsw_components( + seg_dir, + segment_id, + dense_config, + built_hnsw, + source_groups, + )?; + emit_flush_node_index_components(core_writer, source_groups, &node_output)?; + emit_flush_edge_index_components(core_writer, source_groups, &edge_output)?; + + let FlushNodeIndexOutput { + external_records: node_external_records, + declared_evidence: stats_evidence, + .. + } = node_output; + let FlushEdgeIndexOutput { + external_records: edge_external_records, + declared_evidence: edge_stats_evidence, + .. + } = edge_output; + let mut stats_evidence = stats_evidence; + stats_evidence.extend(edge_stats_evidence); + stats_evidence.sort(); + let mut records = Vec::new(); + records.extend(node_external_records); + records.extend(edge_external_records); + records.extend(dense_records); + records.extend(sparse_records); + if let Ok(core_partial) = stats_core_result { + let stats = assemble_compaction_stats_from_partials( + segment_id, + secondary_indexes, + core_partial, + stats_evidence, + ); + if let Ok(Some(payload)) = planner_stats_sidecar_payload(stats) { + if let Ok((record, _)) = write_compaction_component( + seg_dir, + segment_id, + PLANNER_STATS_FILENAME, + SegmentComponentKind::PlannerStats, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::PlannerStatsUnavailable, + }, + ComponentTrustClass::OptionalAdvisoryStats, + planner_stats_component_dependencies( + source_groups.segment_data_id, + secondary_indexes, + ), + planner_stats_component_fingerprint(secondary_indexes), + |writer| { + writer.write_all(&payload)?; + Ok(()) + }, + ) { + records.push(record); + } + } + } + Ok(CompactionComponentBuildOutput { records, report }) +} + +fn degree_sidecars_for_segments( + segments: &[Arc], +) -> Option> { + segments + .iter() + .map(|segment| segment.degree_delta_sidecar()) + .collect() +} + +fn write_degree_delta_from_sidecars( + seg_dir: &Path, + segment_id: u64, + sidecars: &[&crate::degree_cache::DegreeSidecar], + source_groups: SegmentComponentSourceGroups, +) -> Result { + let (record, _) = write_compaction_component( + seg_dir, + segment_id, + DEGREE_DELTA_FILENAME, + SegmentComponentKind::DegreeDelta, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::AdjacencyWalk, + }, + ComponentTrustClass::OptionalExactAccelerator, + vec![source_group_dependency( + SegmentSourceGroupKind::DegreeSource, + source_groups.degree_source, + )], + component_fingerprint("flush.degree_delta", &[]), + |writer| write_folded_degree_delta_sidecar_payload_from_sidecars(writer, sidecars), + )?; + Ok(record) +} + +fn is_optional_degree_delta_invalidity(error: &EngineError) -> bool { + matches!( + error, + EngineError::CorruptRecord(_) | EngineError::InvalidOperation(_) + ) +} + +fn write_compaction_prebuilt_dense_hnsw_components( + seg_dir: &Path, + segment_id: u64, + dense_config: Option<&DenseVectorConfig>, + built_hnsw: Option, + source_groups: SegmentComponentSourceGroups, +) -> Result, EngineError> { + let Some(config) = dense_config else { + return Ok(Vec::new()); + }; + let Some(built) = built_hnsw else { + return Ok(Vec::new()); + }; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::DenseVectorSource, + source_groups.dense_vector_source, + ), + ComponentDependencyV1::DenseVectorConfig { + fingerprint: dense_config_fingerprint(Some(config)), + }, + ]; + let (meta_record, graph_record, _) = write_compaction_component_pair( + seg_dir, + segment_id, + DENSE_HNSW_META_FILENAME, + SegmentComponentKind::DenseHnswMetadata, + component_fingerprint("flush.dense_hnsw_meta", &[]), + DENSE_HNSW_GRAPH_FILENAME, + SegmentComponentKind::DenseHnswGraph, + component_fingerprint("flush.dense_hnsw_graph", &[]), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::ExactVectorScan, + }, + ComponentTrustClass::OptionalApproximateAccelerator, + dependencies, + |meta_writer, graph_writer| { + write_prebuilt_hnsw_to_writers(meta_writer, graph_writer, config, &built) + }, + )?; + Ok(vec![meta_record, graph_record]) +} + +fn build_secondary_eq_groups_from_source_sidecars( + segments: &[Arc], + node_metas: &[CompactNodeMeta], + index_id: u64, + target_label_id: u32, +) -> Result>, EngineError> { + let winner_sources: HashMap = node_metas + .iter() + .filter(|meta| meta.label_ids.contains(target_label_id)) + .map(|meta| (meta.node_id, meta.src_seg_idx)) + .collect(); + let mut groups: BTreeMap> = BTreeMap::new(); + + for (seg_idx, seg) in segments.iter().enumerate() { + seg.for_each_secondary_eq_group(index_id, |value_hash, ids| { + let group = groups.entry(value_hash).or_default(); + for &node_id in ids { + if winner_sources.get(&node_id) == Some(&seg_idx) { + group.push(node_id); + } + } + Ok(()) + })?; + } + + for ids in groups.values_mut() { + ids.sort_unstable(); + ids.dedup(); + } + + Ok(groups) +} + +fn build_secondary_eq_groups_from_targeted_decode( + segments: &[Arc], + node_metas: &[CompactNodeMeta], + target_label_id: u32, + prop_key: &str, +) -> Result>, EngineError> { + let mut groups: BTreeMap> = BTreeMap::new(); + + for meta in node_metas + .iter() + .filter(|meta| meta.label_ids.contains(target_label_id)) + { + if let Some(value) = segments[meta.src_seg_idx].node_property_value_at_offset( + meta.node_id, + meta.src_data_offset, + prop_key, + )? { + groups + .entry(hash_prop_value(&value)) + .or_default() + .push(meta.node_id); + } + } + + for ids in groups.values_mut() { + ids.sort_unstable(); + ids.dedup(); + } + + Ok(groups) +} + +fn sidecar_unavailable_failure_reason( + segment: &SegmentReader, + kind: SegmentComponentKind, +) -> Option { + match segment.optional_component_availability(kind) { + ComponentAvailability::CorruptIdentity { reason } + | ComponentAvailability::Incompatible { reason } + | ComponentAvailability::Unsupported { reason } => Some(reason), + ComponentAvailability::Available | ComponentAvailability::Missing => None, + } +} + +fn build_secondary_range_entries_from_source_sidecars( + segments: &[Arc], + node_metas: &[CompactNodeMeta], + index_id: u64, + target_label_id: u32, +) -> Result, EngineError> { + let winner_sources: HashMap = node_metas + .iter() + .filter(|meta| meta.label_ids.contains(target_label_id)) + .map(|meta| (meta.node_id, meta.src_seg_idx)) + .collect(); + let mut entries = Vec::new(); + + for (seg_idx, seg) in segments.iter().enumerate() { + seg.for_each_secondary_range_entry(index_id, |encoded_value, node_id| { + if winner_sources.get(&node_id) == Some(&seg_idx) { + entries.push((encoded_value, node_id)); + } + Ok(()) + })?; + } + + entries.sort_unstable(); + entries.dedup(); + Ok(entries) +} + +fn build_secondary_range_entries_from_targeted_decode( + segments: &[Arc], + node_metas: &[CompactNodeMeta], + target_label_id: u32, + prop_key: &str, + domain: SecondaryIndexRangeDomain, +) -> Result, EngineError> { + let mut entries = Vec::new(); + + for meta in node_metas + .iter() + .filter(|meta| meta.label_ids.contains(target_label_id)) + { + let Some(value) = segments[meta.src_seg_idx].node_property_value_at_offset( + meta.node_id, + meta.src_data_offset, + prop_key, + )? + else { + continue; + }; + let Some(encoded_value) = encode_range_prop_value(domain, &value) else { + continue; + }; + entries.push((encoded_value, meta.node_id)); + } + + entries.sort_unstable(); + entries.dedup(); + Ok(entries) +} + +fn build_edge_secondary_eq_groups_from_targeted_decode( + segments: &[Arc], + edge_metas: &[CompactEdgeMeta], + label_id: u32, + prop_key: &str, +) -> Result>, EngineError> { + let mut groups: BTreeMap> = BTreeMap::new(); + + for meta in edge_metas.iter().filter(|meta| meta.label_id == label_id) { + if let Some(value) = segments[meta.src_seg_idx].edge_property_value_at_offset( + meta.edge_id, + meta.src_data_offset, + prop_key, + )? { + groups + .entry(hash_prop_value(&value)) + .or_default() + .push(meta.edge_id); + } + } + + for ids in groups.values_mut() { + ids.sort_unstable(); + ids.dedup(); + } + + Ok(groups) +} + +fn build_edge_secondary_range_entries_from_targeted_decode( + segments: &[Arc], + edge_metas: &[CompactEdgeMeta], + label_id: u32, + prop_key: &str, + domain: SecondaryIndexRangeDomain, +) -> Result, EngineError> { + let mut entries = Vec::new(); + + for meta in edge_metas.iter().filter(|meta| meta.label_id == label_id) { + let Some(value) = segments[meta.src_seg_idx].edge_property_value_at_offset( + meta.edge_id, + meta.src_data_offset, + prop_key, + )? + else { + continue; + }; + let Some(encoded_value) = encode_range_prop_value(domain, &value) else { + continue; + }; + entries.push((encoded_value, meta.edge_id)); + } + + entries.sort_unstable(); + entries.dedup(); + Ok(entries) +} + +fn write_declared_equality_sidecars_from_metadata( + seg_dir: &Path, + segment_id: u64, + segments: &[Arc], + node_metas: &[CompactNodeMeta], + eq_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result { + if eq_entries.is_empty() { + return Ok(DeclaredSidecarWriteOutcome::default()); + } + + let index_dir = secondary_indexes_dir(seg_dir); + fs::create_dir_all(&index_dir)?; + let mut outcome = DeclaredSidecarWriteOutcome::default(); + outcome.records.reserve(eq_entries.len()); + + for entry in eq_entries { + let SecondaryIndexTarget::NodeProperty { label_id, prop_key } = &entry.target else { + continue; + }; + let mut failure_message = None; + let use_source_sidecars = if entry.state == SecondaryIndexState::Failed { + false + } else { + let mut all_present = true; + for seg in segments { + match seg.validate_secondary_eq_sidecar_uncached(entry.index_id) { + Ok(true) => {} + Ok(false) => { + if entry.state == SecondaryIndexState::Ready { + let kind = SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }; + if let Some(reason) = sidecar_unavailable_failure_reason(seg, kind) { + failure_message = Some(reason); + } + } + all_present = false; + break; + } + Err(error) => { + all_present = false; + if entry.state == SecondaryIndexState::Ready { + failure_message = Some(error.to_string()); + } + break; + } + } + } + all_present + }; + + let groups = if use_source_sidecars { + build_secondary_eq_groups_from_source_sidecars( + segments, + node_metas, + entry.index_id, + *label_id, + )? + } else { + build_secondary_eq_groups_from_targeted_decode( + segments, node_metas, *label_id, prop_key, + )? + }; + + if let Some(message) = failure_message { + outcome + .report + .failed_equality_indexes + .push((entry.index_id, message)); + } + + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_compaction_component( + seg_dir, + segment_id, + &format!( + "{}/node_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.node_prop_eq", &[entry.index_id]), + |writer| write_node_prop_eq_sidecar_payload(writer, &groups), + )?; + outcome.records.push(record); + if entry.state == SecondaryIndexState::Ready { + outcome + .stats_evidence + .equality_index_stats + .push(equality_index_stats_from_written_groups(entry, &groups)); + } + } + + outcome.stats_evidence.sort(); + Ok(outcome) +} + +fn write_declared_range_sidecars_from_metadata( + seg_dir: &Path, + segment_id: u64, + segments: &[Arc], + node_metas: &[CompactNodeMeta], + range_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result { + if range_entries.is_empty() { + return Ok(DeclaredSidecarWriteOutcome::default()); + } + + let index_dir = secondary_indexes_dir(seg_dir); + fs::create_dir_all(&index_dir)?; + let mut outcome = DeclaredSidecarWriteOutcome::default(); + outcome.records.reserve(range_entries.len()); + + for entry in range_entries { + let SecondaryIndexTarget::NodeProperty { label_id, prop_key } = &entry.target else { + continue; + }; + let SecondaryIndexKind::Range { domain } = entry.kind else { + continue; + }; + let mut failure_message = None; + let use_source_sidecars = if entry.state == SecondaryIndexState::Failed { + false + } else { + let mut all_present = true; + for seg in segments { + match seg.validate_secondary_range_sidecar_uncached(entry.index_id) { + Ok(true) => {} + Ok(false) => { + if entry.state == SecondaryIndexState::Ready { + let kind = SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }; + if let Some(reason) = sidecar_unavailable_failure_reason(seg, kind) { + failure_message = Some(reason); + } + } + all_present = false; + break; + } + Err(error) => { + all_present = false; + if entry.state == SecondaryIndexState::Ready { + failure_message = Some(error.to_string()); + } + break; + } + } + } + all_present + }; + + let sidecar_entries = if use_source_sidecars { + build_secondary_range_entries_from_source_sidecars( + segments, + node_metas, + entry.index_id, + *label_id, + )? + } else { + build_secondary_range_entries_from_targeted_decode( + segments, node_metas, *label_id, prop_key, domain, + )? + }; + + if let Some(message) = failure_message { + outcome + .report + .failed_range_indexes + .push((entry.index_id, message)); + } + + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::NodePropertyContentSource, + source_groups.node_property_content_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_compaction_component( + seg_dir, + segment_id, + &format!( + "{}/node_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::NodePropertyRangeIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("flush.node_prop_range", &[entry.index_id]), + |writer| write_node_prop_range_sidecar_payload(writer, &sidecar_entries), + )?; + outcome.records.push(record); + if entry.state == SecondaryIndexState::Ready { + outcome + .stats_evidence + .range_index_stats + .push(range_index_stats_from_written_entries( + entry, + &sidecar_entries, + )); + } + } + + outcome.stats_evidence.sort(); + Ok(outcome) +} + +fn write_declared_edge_equality_sidecars_from_metadata( + seg_dir: &Path, + segment_id: u64, + segments: &[Arc], + edge_metas: &[CompactEdgeMeta], + eq_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result { + if eq_entries.is_empty() { + return Ok(DeclaredSidecarWriteOutcome::default()); + } + + let index_dir = secondary_indexes_dir(seg_dir); + fs::create_dir_all(&index_dir)?; + let mut outcome = DeclaredSidecarWriteOutcome::default(); + outcome.records.reserve(eq_entries.len()); + + for entry in eq_entries { + let SecondaryIndexTarget::EdgeProperty { label_id, prop_key } = &entry.target else { + continue; + }; + let groups = build_edge_secondary_eq_groups_from_targeted_decode( + segments, edge_metas, *label_id, prop_key, + )?; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_compaction_component( + seg_dir, + segment_id, + &format!( + "{}/edge_prop_eq_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("compaction.edge_prop_eq", &[entry.index_id]), + |writer| write_node_prop_eq_sidecar_payload(writer, &groups), + )?; + outcome.records.push(record); + if entry.state == SecondaryIndexState::Ready { + outcome + .stats_evidence + .equality_index_stats + .push(equality_index_stats_from_written_groups(entry, &groups)); + } + } + + outcome.stats_evidence.sort(); + Ok(outcome) +} + +fn write_declared_edge_range_sidecars_from_metadata( + seg_dir: &Path, + segment_id: u64, + segments: &[Arc], + edge_metas: &[CompactEdgeMeta], + range_entries: &[&SecondaryIndexManifestEntry], + source_groups: SegmentComponentSourceGroups, +) -> Result { + if range_entries.is_empty() { + return Ok(DeclaredSidecarWriteOutcome::default()); + } + + let index_dir = secondary_indexes_dir(seg_dir); + fs::create_dir_all(&index_dir)?; + let mut outcome = DeclaredSidecarWriteOutcome::default(); + outcome.records.reserve(range_entries.len()); + + for entry in range_entries { + let SecondaryIndexTarget::EdgeProperty { label_id, prop_key } = &entry.target else { + continue; + }; + let SecondaryIndexKind::Range { domain } = entry.kind else { + continue; + }; + let sidecar_entries = build_edge_secondary_range_entries_from_targeted_decode( + segments, edge_metas, *label_id, prop_key, domain, + )?; + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::EdgeSource, + source_groups.edge_source, + ), + secondary_declaration_dependency(entry), + ]; + let (record, _) = write_compaction_component( + seg_dir, + segment_id, + &format!( + "{}/edge_prop_range_{}.dat", + SECONDARY_INDEX_DIRNAME, entry.index_id + ), + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: entry.index_id, + }, + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + ComponentTrustClass::OptionalCandidateIndex, + dependencies, + component_fingerprint("compaction.edge_prop_range", &[entry.index_id]), + |writer| write_node_prop_range_sidecar_payload(writer, &sidecar_entries), + )?; + outcome.records.push(record); + if entry.state == SecondaryIndexState::Ready { + outcome + .stats_evidence + .range_index_stats + .push(range_index_stats_from_written_entries( + entry, + &sidecar_entries, + )); + } + } + + outcome.stats_evidence.sort(); + Ok(outcome) +} + +fn write_node_vector_source_components_from_meta( + core_writer: &mut PackedCoreWriter, + segments: &[Arc], + node_metas: &[CompactNodeMeta], + node_source: [u8; 32], +) -> Result<(Vec, Vec), EngineError> { + let plan = prepare_node_vector_source_plan_from_meta(segments, node_metas)?; + if !plan.has_dense && !plan.has_sparse { + return Ok((Vec::new(), Vec::new())); + } + + let node_source_dep = source_group_dependency(SegmentSourceGroupKind::NodeSource, node_source); + let (meta_record, _) = core_writer.write_component( + SegmentComponentKind::NodeVectorMetadata, + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + vec![node_source_dep.clone()], + component_fingerprint("flush.node_vector_meta", &[]), + |writer| write_node_vector_meta_payload(writer, &plan), + )?; + let vector_blob_deps = vec![node_source_dep, source_component_dependency(&meta_record)]; + let mut records = Vec::with_capacity(3); + records.push(meta_record); + if plan.has_dense { + let (record, _) = core_writer.write_component( + SegmentComponentKind::NodeDenseVectorBlob, + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + vector_blob_deps.clone(), + component_fingerprint("flush.node_dense_vectors", &[]), + |writer| write_node_dense_vector_blob_payload_from_meta(writer, segments, node_metas), + )?; + records.push(record); + } + if plan.has_sparse { + let (record, _) = core_writer.write_component( + SegmentComponentKind::NodeSparseVectorBlob, + ComponentRequirement::Required, + ComponentTrustClass::AuxiliaryBlob, + vector_blob_deps, + component_fingerprint("flush.node_sparse_vectors", &[]), + |writer| write_node_sparse_vector_blob_payload_from_meta(writer, segments, node_metas), + )?; + records.push(record); + } + + Ok((records, plan.dense_points)) +} + +fn prepare_node_vector_source_plan_from_meta( + segments: &[Arc], + node_metas: &[CompactNodeMeta], +) -> Result { + let mut rows = Vec::with_capacity(node_metas.len()); + let mut has_dense = false; + let mut has_sparse = false; + let mut dense_offset = 0u64; + let mut sparse_offset = 0u64; + let mut dense_points = Vec::new(); + + for nm in node_metas { + let mut flags = 0u8; + let mut entry_dense_offset = 0u64; + let mut entry_sparse_offset = 0u64; + + if nm.dense_vector_len > 0 { + flags |= NODE_VECTOR_FLAG_DENSE; + entry_dense_offset = dense_offset; + has_dense = true; + let src = segments[nm.src_seg_idx].raw_node_dense_vectors_mmap(); + let (base, len, end) = checked_compaction_vector_range( + nm.node_id, + "dense", + nm.dense_vector_offset, + nm.dense_vector_len, + DENSE_VECTOR_VALUE_SIZE, + src.len(), + )?; + let mut values = Vec::with_capacity(nm.dense_vector_len as usize); + for index in 0..nm.dense_vector_len as usize { + let value_offset = base + index * DENSE_VECTOR_VALUE_SIZE as usize; + values.push(f32::from_le_bytes( + src[value_offset..value_offset + DENSE_VECTOR_VALUE_SIZE as usize] + .try_into() + .unwrap(), + )); + } + dense_points.push(DensePointInput { + node_id: nm.node_id, + dense_vector_offset: entry_dense_offset, + values, + }); + dense_offset = dense_offset.checked_add(len as u64).ok_or_else(|| { + EngineError::CorruptRecord("dense vector output offset overflow".into()) + })?; + debug_assert!(end <= src.len()); + } + + if nm.sparse_vector_len > 0 { + flags |= NODE_VECTOR_FLAG_SPARSE; + entry_sparse_offset = sparse_offset; + has_sparse = true; + let src = segments[nm.src_seg_idx].raw_node_sparse_vectors_mmap(); + let (_, len, _) = checked_compaction_vector_range( + nm.node_id, + "sparse", + nm.sparse_vector_offset, + nm.sparse_vector_len, + SPARSE_VECTOR_ENTRY_SIZE, + src.len(), + )?; + sparse_offset = sparse_offset.checked_add(len as u64).ok_or_else(|| { + EngineError::CorruptRecord("sparse vector output offset overflow".into()) + })?; + } + + rows.push(NodeVectorSourceRow { + node_id: nm.node_id, + flags, + dense_offset: entry_dense_offset, + dense_len: nm.dense_vector_len, + sparse_offset: entry_sparse_offset, + sparse_len: nm.sparse_vector_len, + }); + } + + Ok(NodeVectorSourcePlan { + rows, + has_dense, + has_sparse, + dense_points, + }) +} + +fn checked_compaction_vector_range( + node_id: u64, + label: &str, + source_offset: u64, + element_count: u32, + element_size: u64, + source_len: usize, +) -> Result<(usize, usize, usize), EngineError> { + let base = usize::try_from(source_offset).map_err(|_| { + EngineError::CorruptRecord(format!("node {node_id} {label} vector offset too large")) + })?; + let len_u64 = (element_count as u64) + .checked_mul(element_size) + .ok_or_else(|| { + EngineError::CorruptRecord(format!( + "node {node_id} {label} vector byte length overflow" + )) + })?; + let len = usize::try_from(len_u64).map_err(|_| { + EngineError::CorruptRecord(format!( + "node {node_id} {label} vector byte length too large" + )) + })?; + let end = base.checked_add(len).ok_or_else(|| { + EngineError::CorruptRecord(format!( + "node {node_id} {label} vector range offset overflow: start={base}, len={len}" + )) + })?; + if end > source_len { + return Err(EngineError::CorruptRecord(format!( + "node {node_id} {label} vector range [{base}, {end}) exceeds source length {source_len}" + ))); + } + Ok((base, len, end)) +} + +fn write_node_dense_vector_blob_payload_from_meta( + w: &mut impl Write, + segments: &[Arc], + node_metas: &[CompactNodeMeta], +) -> Result<(), EngineError> { + for nm in node_metas { + if nm.dense_vector_len == 0 { + continue; + } + let src = segments[nm.src_seg_idx].raw_node_dense_vectors_mmap(); + let (base, _, end) = checked_compaction_vector_range( + nm.node_id, + "dense", + nm.dense_vector_offset, + nm.dense_vector_len, + DENSE_VECTOR_VALUE_SIZE, + src.len(), + )?; + w.write_all(&src[base..end])?; + } + Ok(()) +} + +fn write_node_sparse_vector_blob_payload_from_meta( + w: &mut impl Write, + segments: &[Arc], + node_metas: &[CompactNodeMeta], +) -> Result<(), EngineError> { + for nm in node_metas { + if nm.sparse_vector_len == 0 { + continue; + } + let src = segments[nm.src_seg_idx].raw_node_sparse_vectors_mmap(); + let (base, _, end) = checked_compaction_vector_range( + nm.node_id, + "sparse", + nm.sparse_vector_offset, + nm.sparse_vector_len, + SPARSE_VECTOR_ENTRY_SIZE, + src.len(), + )?; + w.write_all(&src[base..end])?; + } + Ok(()) +} + +fn sparse_posting_groups_from_nodes( + nodes: &NodeIdMap, +) -> Result>, EngineError> { + let mut groups: BTreeMap> = BTreeMap::new(); + for node in nodes.values() { + let Some(values) = node.sparse_vector.as_ref() else { + continue; + }; + for &(dimension_id, weight) in values { + groups + .entry(dimension_id) + .or_default() + .push((node.id, weight)); + } + } + sort_sparse_posting_groups(&mut groups)?; + Ok(groups) +} + +fn write_sparse_posting_index_from_meta( + seg_dir: &Path, + segment_id: u64, + segments: &[Arc], + node_metas: &[CompactNodeMeta], + source_groups: SegmentComponentSourceGroups, +) -> Result, EngineError> { + let mut groups: BTreeMap> = BTreeMap::new(); + for nm in node_metas { + if nm.sparse_vector_len == 0 { + continue; + } + let src = segments[nm.src_seg_idx].raw_node_sparse_vectors_mmap(); + let (base, _, end) = checked_compaction_vector_range( + nm.node_id, + "sparse", + nm.sparse_vector_offset, + nm.sparse_vector_len, + SPARSE_VECTOR_ENTRY_SIZE, + src.len(), + )?; + for entry_offset in (base..end).step_by(SPARSE_VECTOR_ENTRY_SIZE as usize) { + let dimension_id = + u32::from_le_bytes(src[entry_offset..entry_offset + 4].try_into().unwrap()); + let weight = + f32::from_le_bytes(src[entry_offset + 4..entry_offset + 8].try_into().unwrap()); + groups + .entry(dimension_id) + .or_default() + .push((nm.node_id, weight)); + } + } + sort_sparse_posting_groups(&mut groups)?; + if groups.is_empty() { + return Ok(Vec::new()); + } + let dependencies = vec![ + source_group_dependency( + SegmentSourceGroupKind::SparseVectorSource, + source_groups.sparse_vector_source, + ), + ComponentDependencyV1::SparseVectorConfig { + fingerprint: component_fingerprint("sparse_vector_config", &[]), + }, + ]; + let (index_record, postings_record, _) = write_compaction_component_pair( + seg_dir, + segment_id, + SPARSE_POSTING_INDEX_FILENAME, + SegmentComponentKind::SparsePostingIndex, + component_fingerprint("flush.sparse_posting_index", &[]), + SPARSE_POSTINGS_FILENAME, + SegmentComponentKind::SparsePostings, + component_fingerprint("flush.sparse_postings", &[]), + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::ExactVectorScan, + }, + ComponentTrustClass::OptionalApproximateAccelerator, + dependencies, + |index_writer, postings_writer| { + write_sparse_posting_files_to_writers(index_writer, postings_writer, &groups) + }, + )?; + Ok(vec![index_record, postings_record]) +} + +fn sort_sparse_posting_groups( + groups: &mut BTreeMap>, +) -> Result<(), EngineError> { + for (&dimension_id, postings) in groups.iter_mut() { + postings.sort_unstable_by_key(|&(node_id, _)| node_id); + for window in postings.windows(2) { + if window[0].0 == window[1].0 { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimension {} has duplicate node {}", + dimension_id, window[0].0 + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::degree_cache::DegreeDelta; + use std::sync::Arc; + + fn write_segment( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + dense_config: Option<&DenseVectorConfig>, + ) -> Result { + let degree_overlay = DegreeOverlaySnapshot::empty(); + super::write_segment_with_degree_overlay_and_secondary_indexes( + seg_dir, + segment_id, + memtable, + dense_config, + degree_overlay.as_ref(), + &[], + ) + } + + fn write_segment_with_secondary_indexes( + seg_dir: &Path, + segment_id: u64, + memtable: &Memtable, + dense_config: Option<&DenseVectorConfig>, + secondary_indexes: &[SecondaryIndexManifestEntry], + ) -> Result { + let degree_overlay = DegreeOverlaySnapshot::empty(); + super::write_segment_with_degree_overlay_and_secondary_indexes( + seg_dir, + segment_id, + memtable, + dense_config, + degree_overlay.as_ref(), + secondary_indexes, + ) + } + + fn make_node(id: u64, label_id: u32, key: &str) -> NodeRecord { + NodeRecord { + id, + label_ids: NodeLabelSet::single(label_id).unwrap(), + key: key.to_string(), + props: BTreeMap::new(), + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + } + } + + fn make_node_with_props(id: u64, label_id: u32, key: &str) -> NodeRecord { + let mut props = BTreeMap::new(); + props.insert("name".to_string(), PropValue::String(key.to_string())); + props.insert("score".to_string(), PropValue::Float(0.95)); + NodeRecord { + id, + label_ids: NodeLabelSet::single(label_id).unwrap(), + key: key.to_string(), + props, + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + } + } + + fn make_edge(id: u64, from: u64, to: u64, label_id: u32) -> EdgeRecord { EdgeRecord { id, from, to, - type_id, + label_id: label_id, props: BTreeMap::new(), created_at: 2000, updated_at: 2001, weight: 1.0, valid_from: 0, valid_to: i64::MAX, - last_write_seq: 0, + last_write_seq: 0, + } + } + + fn make_node_with_custom_props( + id: u64, + label_id: u32, + key: &str, + props: BTreeMap, + updated_at: i64, + ) -> NodeRecord { + NodeRecord { + id, + label_ids: NodeLabelSet::single(label_id).unwrap(), + key: key.to_string(), + props, + created_at: 1000, + updated_at, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + } + } + + fn make_node_with_labels( + id: u64, + label_ids: &[u32], + key: &str, + props: BTreeMap, + updated_at: i64, + ) -> NodeRecord { + NodeRecord { + id, + label_ids: NodeLabelSet::from_canonical_ids(label_ids).unwrap(), + key: key.to_string(), + props, + created_at: 1000, + updated_at, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + } + } + + fn read_payload_file(path: &Path) -> Vec { + let data = fs::read(path).unwrap(); + if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] + == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC + { + let header = crate::segment_components::decode_identity_header(&data).unwrap(); + let start = header.payload_offset as usize; + let end = start + header.payload_len as usize; + return data[start..end].to_vec(); + } + data + } + + fn read_manifest_component_payload(seg_dir: &Path, kind: SegmentComponentKind) -> Vec { + let manifest_bytes = fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap_or_else(|| panic!("missing component {:?}", kind)); + match &record.handle { + ComponentHandleV1::ExternalFile { relative_path, .. } => { + read_payload_file(&seg_dir.join(relative_path)) + } + ComponentHandleV1::PackedRange { offset, len, .. } => { + let core = read_payload_file( + &seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME), + ); + let start = *offset as usize; + let end = start + *len as usize; + core[start..end].to_vec() + } + } + } + + fn write_packed_segment_from_ops( + ops: Vec, + dense_config: Option<&DenseVectorConfig>, + ) -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let mt = Memtable::new(); + for (index, op) in ops.iter().enumerate() { + mt.apply_op(op, (index + 1) as u64); + } + write_segment(&seg_dir, 1, &mt, dense_config).unwrap(); + (dir, seg_dir) + } + + fn read_record_spans_from_payload(data: &[u8]) -> RecordDataSpans { + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()) as usize; + let mut entries = Vec::with_capacity(count); + for index in 0..count { + let entry_offset = 8 + index * 16; + let id = u64::from_le_bytes(data[entry_offset..entry_offset + 8].try_into().unwrap()); + let data_offset = u64::from_le_bytes( + data[entry_offset + 8..entry_offset + 16] + .try_into() + .unwrap(), + ); + let next_offset = if index + 1 == count { + data.len() as u64 + } else { + let next_entry_offset = 8 + (index + 1) * 16; + u64::from_le_bytes( + data[next_entry_offset + 8..next_entry_offset + 16] + .try_into() + .unwrap(), + ) + }; + entries.push((id, data_offset, (next_offset - data_offset) as u32)); + } + entries + } + + fn assert_component_handle_is_packed(seg_dir: &Path, kind: SegmentComponentKind) { + let manifest_bytes = fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap_or_else(|| panic!("missing component {:?}", kind)); + assert!( + matches!(record.handle, ComponentHandleV1::PackedRange { .. }), + "component {:?} should be packed", + record.kind + ); + } + + fn assert_component_handle_is_external(seg_dir: &Path, kind: SegmentComponentKind) { + let manifest_bytes = fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + let record = manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap_or_else(|| panic!("missing component {:?}", kind)); + assert!( + matches!(record.handle, ComponentHandleV1::ExternalFile { .. }), + "component {:?} should be external", + record.kind + ); + } + + fn assert_no_legacy_property_components(seg_dir: &Path) { + let manifest_bytes = fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + assert!(manifest.components.iter().all(|record| { + !matches!( + record.kind, + SegmentComponentKind::LegacyNodePropertyIndex + | SegmentComponentKind::NodePropertyHashMetadata + ) + })); + } + + fn assert_only_manifested_component_files(seg_dir: &Path) { + let manifest_bytes = fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + let mut expected = BTreeSet::from([PathBuf::from(SEGMENT_COMPONENT_MANIFEST_FILENAME)]); + for record in &manifest.components { + if let ComponentHandleV1::ExternalFile { relative_path, .. } = &record.handle { + expected.insert(PathBuf::from(relative_path)); + } + } + for record in &manifest.unknown_optional_components { + if record.wire.handle.handle_tag == 1 { + if let Some(relative_path) = &record.wire.handle.relative_path { + expected.insert(PathBuf::from(relative_path)); + } + } + } + + let mut actual = Vec::new(); + collect_regular_files_relative_to(seg_dir, seg_dir, &mut actual); + for relative_path in actual { + assert!( + expected.contains(&relative_path), + "unexpected unmanifested segment file {}", + relative_path.display() + ); + } + } + + fn collect_regular_files_relative_to(root: &Path, dir: &Path, files: &mut Vec) { + for entry in fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let file_type = entry.file_type().unwrap(); + if file_type.is_dir() { + collect_regular_files_relative_to(root, &path, files); + } else if file_type.is_file() { + files.push(path.strip_prefix(root).unwrap().to_path_buf()); + } + } + } + + fn read_weight_index_entries(seg_dir: &Path) -> Vec<(u32, u32, u64)> { + let data = read_manifest_component_payload(seg_dir, SegmentComponentKind::EdgeWeightIndex); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()) as usize; + let mut entries = Vec::with_capacity(count); + for index in 0..count { + let off = 8 + index * 16; + entries.push(( + u32::from_le_bytes(data[off..off + 4].try_into().unwrap()), + u32::from_le_bytes(data[off + 4..off + 8].try_into().unwrap()), + u64::from_le_bytes(data[off + 8..off + 16].try_into().unwrap()), + )); + } + entries + } + + fn read_i64_index_entries(seg_dir: &Path, kind: SegmentComponentKind) -> Vec<(u32, i64, u64)> { + let data = read_manifest_component_payload(seg_dir, kind); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()) as usize; + let mut entries = Vec::with_capacity(count); + for index in 0..count { + let off = 8 + index * 20; + entries.push(( + u32::from_le_bytes(data[off..off + 4].try_into().unwrap()), + i64::from_le_bytes(data[off + 4..off + 12].try_into().unwrap()), + u64::from_le_bytes(data[off + 12..off + 20].try_into().unwrap()), + )); + } + entries + } + + fn read_secondary_eq_groups( + seg_dir: &Path, + kind: SegmentComponentKind, + ) -> BTreeMap> { + let payload = read_manifest_component_payload(seg_dir, kind); + let count = u64::from_le_bytes(payload[0..8].try_into().unwrap()) as usize; + let mut groups = BTreeMap::new(); + for index in 0..count { + let entry_off = 8 + index * SECONDARY_EQ_ENTRY_SIZE as usize; + let value_hash = + u64::from_le_bytes(payload[entry_off..entry_off + 8].try_into().unwrap()); + let ids_offset = + u64::from_le_bytes(payload[entry_off + 8..entry_off + 16].try_into().unwrap()) + as usize; + let id_count = + u32::from_le_bytes(payload[entry_off + 16..entry_off + 20].try_into().unwrap()) + as usize; + let ids = (0..id_count) + .map(|id_index| { + let off = ids_offset + id_index * 8; + u64::from_le_bytes(payload[off..off + 8].try_into().unwrap()) + }) + .collect::>(); + groups.insert(value_hash, ids); + } + groups + } + + fn read_secondary_range_entries(seg_dir: &Path, kind: SegmentComponentKind) -> Vec<(u64, u64)> { + let payload = read_manifest_component_payload(seg_dir, kind); + let count = u64::from_le_bytes(payload[0..8].try_into().unwrap()) as usize; + (0..count) + .map(|index| { + let off = 8 + index * 16; + ( + u64::from_le_bytes(payload[off..off + 8].try_into().unwrap()), + u64::from_le_bytes(payload[off + 8..off + 16].try_into().unwrap()), + ) + }) + .collect() + } + + fn test_component_record(kind: SegmentComponentKind) -> SegmentComponentRecordV1 { + SegmentComponentRecordV1 { + component_id: [0; 32], + kind, + logical_format_version: 1, + created_generation: 0, + requirement: ComponentRequirement::Optional { + fallback: ComponentFallbackClass::RecordScan, + }, + trust_class: ComponentTrustClass::OptionalCandidateIndex, + handle: ComponentHandleV1::ExternalFile { + relative_path: "secondary_indexes/test.dat".to_string(), + payload_offset: 0, + payload_len: 0, + }, + payload_len: 0, + payload_digest: Some([0; 32]), + dependency_digest: [0; 32], + dependencies: Vec::new(), + build_fingerprint: 0, + } + } + + fn compact_copy_segment_for_test( + source: Arc, + out_dir: &Path, + out_segment_id: u64, + secondary_indexes: &[SecondaryIndexManifestEntry], + ) -> SegmentReader { + std::fs::create_dir_all(out_dir).unwrap(); + let segments = vec![source.clone()]; + let mut core_writer = create_compaction_core_writer(out_dir, out_segment_id).unwrap(); + let (node_record, node_copy_info) = + write_merged_nodes_dat(&mut core_writer, &segments).unwrap(); + let (edge_record, edge_copy_info) = + write_merged_edges_dat(&mut core_writer, &segments).unwrap(); + let node_copy = &node_copy_info[0]; + let edge_copy = &edge_copy_info[0]; + + let mut node_metas = Vec::new(); + for index in 0..source.node_meta_count() as usize { + let meta = source.node_meta_at(index).unwrap(); + let (dense_vector_offset, dense_vector_len, sparse_vector_offset, sparse_vector_len) = + source.node_vector_meta_at(index).unwrap(); + node_metas.push(CompactNodeMeta { + node_id: meta.node_id, + new_data_offset: meta.data_offset - node_copy.orig_data_start + + node_copy.new_data_base, + data_len: meta.data_len, + label_ids: meta.label_ids, + updated_at: meta.updated_at, + weight: meta.weight, + key_len: meta.key_len, + dense_vector_offset, + dense_vector_len, + sparse_vector_offset, + sparse_vector_len, + src_seg_idx: 0, + src_data_offset: meta.data_offset, + last_write_seq: meta.last_write_seq, + }); + } + + let mut edge_metas = Vec::new(); + for index in 0..source.edge_meta_count() as usize { + let ( + edge_id, + data_offset, + data_len, + from, + to, + label_id, + updated_at, + weight, + valid_from, + valid_to, + last_write_seq, + ) = source.edge_meta_at(index).unwrap(); + edge_metas.push(CompactEdgeMeta { + edge_id, + new_data_offset: data_offset - edge_copy.orig_data_start + edge_copy.new_data_base, + data_len, + from, + to, + label_id: label_id, + updated_at, + weight, + valid_from, + valid_to, + src_seg_idx: 0, + src_data_offset: data_offset, + last_write_seq, + }); + } + + let (source_groups, dense_points) = write_compaction_source_components( + out_segment_id, + &mut core_writer, + &segments, + node_record, + edge_record, + &node_metas, + &edge_metas, + ) + .unwrap(); + let component_output = write_indexes_from_metadata_with_secondary_indexes( + out_segment_id, + out_dir, + &mut core_writer, + &segments, + &node_metas, + &edge_metas, + None, + dense_points, + true, + secondary_indexes, + source_groups, + ) + .unwrap(); + let mut records = component_output.records; + records.extend(finish_compaction_core_writer(core_writer).unwrap()); + let info = finalize_compaction_segment( + out_dir, + out_segment_id, + node_metas.len() as u64, + edge_metas.len() as u64, + records, + ) + .unwrap(); + let reference_stats = crate::planner_stats::build_compaction_stats( + out_segment_id, + out_dir, + &segments, + &node_metas, + &edge_metas, + secondary_indexes, + ) + .unwrap(); + let reader = + SegmentReader::open_with_info(out_dir, &info, None, secondary_indexes).unwrap(); + assert_eq!( + reader.planner_stats().expect("planner stats should load"), + &reference_stats + ); + reader + } + + // --- encode_node_record / encode_edge_record --- + + #[test] + fn test_encode_node_record_roundtrip() { + let node = make_node_with_props(42, 1, "alice"); + let mut buf = Vec::new(); + encode_node_record_into(&mut buf, &node).unwrap(); + + // Verify structure (no id): label_count(1) + label_id(4) + key_len(2) + key(5) + created(8) + updated(8) + weight(4) + props_len(4) + props(N) + assert!(buf.len() > 30 + 5); // minimum size with key "alice" + + assert_eq!(buf[0], 1); + let label_id = u32::from_le_bytes(buf[1..5].try_into().unwrap()); + assert_eq!(label_id, 1); + + let key_len = u16::from_le_bytes(buf[5..7].try_into().unwrap()) as usize; + assert_eq!(key_len, 5); + + let key = std::str::from_utf8(&buf[7..7 + key_len]).unwrap(); + assert_eq!(key, "alice"); + } + + #[test] + fn test_encode_edge_record_roundtrip() { + let edge = make_edge(100, 1, 2, 10); + let mut buf = Vec::new(); + encode_edge_record_into(&mut buf, &edge).unwrap(); + + // No id in data section. Starts with from + let from = u64::from_le_bytes(buf[0..8].try_into().unwrap()); + assert_eq!(from, 1); + + let to = u64::from_le_bytes(buf[8..16].try_into().unwrap()); + assert_eq!(to, 2); + + let label_id = u32::from_le_bytes(buf[16..20].try_into().unwrap()); + assert_eq!(label_id, 10); + } + + // --- packed node records --- + + #[test] + fn test_write_node_records_empty() { + let (_dir, seg_dir) = write_packed_segment_from_ops(Vec::new(), None); + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeRecords); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 0); + assert_eq!(data.len(), 8); // just the count + } + + #[test] + fn test_write_node_records_multiple() { + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![ + WalOp::UpsertNode(make_node(3, 1, "charlie")), + WalOp::UpsertNode(make_node(1, 1, "alice")), + WalOp::UpsertNode(make_node(2, 1, "bob")), + ], + None, + ); + + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeRecords); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 3); + + // Index entries should be sorted by node_id + let idx_start = 8; + let id0 = u64::from_le_bytes(data[idx_start..idx_start + 8].try_into().unwrap()); + let id1 = u64::from_le_bytes(data[idx_start + 16..idx_start + 24].try_into().unwrap()); + let id2 = u64::from_le_bytes(data[idx_start + 32..idx_start + 40].try_into().unwrap()); + assert_eq!(id0, 1); + assert_eq!(id1, 2); + assert_eq!(id2, 3); + + // Verify the offset of the first record leads to valid data. + // The id is NOT in the record; first fields are label_count + labels. + let offset0 = + u64::from_le_bytes(data[idx_start + 8..idx_start + 16].try_into().unwrap()) as usize; + assert_eq!(data[offset0], 1); + let label_id = u32::from_le_bytes(data[offset0 + 1..offset0 + 5].try_into().unwrap()); + assert_eq!(label_id, 1); + } + + // --- packed edge records --- + + #[test] + fn test_write_edge_records_empty() { + let (_dir, seg_dir) = write_packed_segment_from_ops(Vec::new(), None); + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::EdgeRecords); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 0); + } + + #[test] + fn test_write_edge_records_multiple() { + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![ + WalOp::UpsertEdge(make_edge(2, 1, 3, 10)), + WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), + ], + None, + ); + + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::EdgeRecords); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 2); + + // Index should be sorted: edge 1 then edge 2 + let idx_start = 8; + let eid0 = u64::from_le_bytes(data[idx_start..idx_start + 8].try_into().unwrap()); + let eid1 = u64::from_le_bytes(data[idx_start + 16..idx_start + 24].try_into().unwrap()); + assert_eq!(eid0, 1); + assert_eq!(eid1, 2); + } + + // --- write_adjacency_index --- + + fn make_adj(edge_id: u64, label_id: u32, neighbor_id: u64, weight: f32) -> AdjEntry { + AdjEntry { + edge_id, + label_id: label_id, + neighbor_id, + weight, + valid_from: 1000, + valid_to: i64::MAX, + } + } + + #[test] + fn test_write_adjacency_empty() { + let (_dir, seg_dir) = write_packed_segment_from_ops(Vec::new(), None); + let idx_data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::AdjOutIndex); + let count = u64::from_le_bytes(idx_data[0..8].try_into().unwrap()); + assert_eq!(count, 0); + } + + #[test] + fn test_write_adjacency_single_node() { + let mut edge_10 = make_edge(10, 1, 2, 1); + edge_10.weight = 0.5; + let mut edge_11 = make_edge(11, 1, 3, 1); + edge_11.weight = 0.7; + let edge_12 = make_edge(12, 1, 4, 2); + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![ + WalOp::UpsertEdge(edge_10), + WalOp::UpsertEdge(edge_11), + WalOp::UpsertEdge(edge_12), + ], + None, + ); + + let idx_data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::AdjOutIndex); + let count = u64::from_le_bytes(idx_data[0..8].try_into().unwrap()); + // Node 1 has 2 label groups: label_id=1 (2 entries) and label_id=2 (1 entry) + assert_eq!(count, 2); + + let dat_data = + read_manifest_component_payload(&seg_dir, SegmentComponentKind::AdjOutPostings); + // Delta-encoded variable-length postings, much smaller than fixed-size. + // 3 postings with small ids/deltas → expect < 108 bytes (old fixed-size). + assert!(!dat_data.is_empty()); + assert!( + dat_data.len() < 108, + "delta encoding should be smaller than fixed 36-byte postings" + ); + } + + #[test] + fn test_write_adjacency_sorted_index() { + let mut edge_10 = make_edge(10, 5, 6, 1); + edge_10.weight = 0.5; + let mut edge_11 = make_edge(11, 1, 2, 1); + edge_11.weight = 0.7; + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![WalOp::UpsertEdge(edge_10), WalOp::UpsertEdge(edge_11)], + None, + ); + + let idx_data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::AdjOutIndex); + let count = u64::from_le_bytes(idx_data[0..8].try_into().unwrap()); + assert_eq!(count, 2); + + // First index entry should be node_id=1 (sorted) + let node_id_0 = u64::from_le_bytes(idx_data[8..16].try_into().unwrap()); + let node_id_1 = u64::from_le_bytes(idx_data[8 + 24..16 + 24].try_into().unwrap()); + assert_eq!(node_id_0, 1); + assert_eq!(node_id_1, 5); + } + + #[test] + fn test_adjacency_prepare_emit_matches_packed_payloads() { + let mut adj: NodeIdMap> = NodeIdMap::default(); + + let mut node_7 = NodeIdMap::default(); + let mut edge_30 = make_adj(30, 2, 9, 1.25); + edge_30.valid_from = 10; + edge_30.valid_to = 20; + node_7.insert(edge_30.edge_id, edge_30); + let mut edge_31 = make_adj(31, 2, 10, 2.5); + edge_31.valid_from = 11; + edge_31.valid_to = i64::MAX; + node_7.insert(edge_31.edge_id, edge_31); + adj.insert(7, node_7); + + let mut node_3 = NodeIdMap::default(); + node_3.insert(4, make_adj(4, 1, 2, 0.5)); + node_3.insert(6, make_adj(6, 3, 8, 0.75)); + adj.insert(3, node_3); + + let mut ops = Vec::new(); + for (&from, edges) in &adj { + for entry in edges.values() { + let mut edge = make_edge(entry.edge_id, from, entry.neighbor_id, entry.label_id); + edge.weight = entry.weight; + edge.valid_from = entry.valid_from; + edge.valid_to = entry.valid_to; + ops.push(WalOp::UpsertEdge(edge)); + } } + let (_dir, seg_dir) = write_packed_segment_from_ops(ops, None); + + let plan = prepare_adjacency_payloads(adj.clone()); + let mut direct_idx = Vec::new(); + let mut direct_dat = Vec::new(); + write_adjacency_postings_payload(&mut direct_dat, &plan).unwrap(); + write_adjacency_index_payload(&mut direct_idx, &plan).unwrap(); + + assert_eq!( + read_manifest_component_payload(&seg_dir, SegmentComponentKind::AdjOutIndex), + direct_idx + ); + assert_eq!( + read_manifest_component_payload(&seg_dir, SegmentComponentKind::AdjOutPostings), + direct_dat + ); + } + + // --- Packed key index payload --- + + #[test] + fn test_packed_key_index_payload_empty() { + let (_dir, seg_dir) = write_packed_segment_from_ops(Vec::new(), None); + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::KeyIndex); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 0); + assert_eq!(data.len(), 8); + } + + #[test] + fn test_packed_key_index_payload_sorted_by_label_and_key() { + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![ + WalOp::UpsertNode(make_node(1, 2, "zebra")), + WalOp::UpsertNode(make_node(2, 1, "bob")), + WalOp::UpsertNode(make_node(3, 1, "alice")), + ], + None, + ); + + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::KeyIndex); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 3); + + // Read offset table + let offsets: Vec = (0..3) + .map(|i| { + let start = 8 + i * 8; + u64::from_le_bytes(data[start..start + 8].try_into().unwrap()) + }) + .collect(); + + // First entry should be label_id=1, key="alice" + let off0 = offsets[0] as usize; + let label0 = u32::from_le_bytes(data[off0..off0 + 4].try_into().unwrap()); + let node0 = u64::from_le_bytes(data[off0 + 4..off0 + 12].try_into().unwrap()); + let klen0 = u16::from_le_bytes(data[off0 + 12..off0 + 14].try_into().unwrap()) as usize; + let key0 = std::str::from_utf8(&data[off0 + 14..off0 + 14 + klen0]).unwrap(); + assert_eq!(label0, 1); + assert_eq!(key0, "alice"); + assert_eq!(node0, 3); + + // Second entry should be label_id=1, key="bob" + let off1 = offsets[1] as usize; + let label1 = u32::from_le_bytes(data[off1..off1 + 4].try_into().unwrap()); + let klen1 = u16::from_le_bytes(data[off1 + 12..off1 + 14].try_into().unwrap()) as usize; + let key1 = std::str::from_utf8(&data[off1 + 14..off1 + 14 + klen1]).unwrap(); + assert_eq!(label1, 1); + assert_eq!(key1, "bob"); + + // Third entry should be label_id=2, key="zebra" + let off2 = offsets[2] as usize; + let label2 = u32::from_le_bytes(data[off2..off2 + 4].try_into().unwrap()); + assert_eq!(label2, 2); + } + + // --- Packed tombstones payload --- + + #[test] + fn test_packed_tombstones_payload_empty() { + let (_dir, seg_dir) = write_packed_segment_from_ops(Vec::new(), None); + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::Tombstones); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 0); + } + + #[test] + fn test_packed_tombstones_payload_mixed() { + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![ + WalOp::DeleteNode { + id: 5, + deleted_at: 1000, + }, + WalOp::DeleteNode { + id: 3, + deleted_at: 1001, + }, + WalOp::DeleteEdge { + id: 10, + deleted_at: 2000, + }, + ], + None, + ); + + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::Tombstones); + let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); + assert_eq!(count, 3); + + // Each tombstone: 1 byte kind + 8 bytes id + 8 bytes deleted_at + 8 bytes last_write_seq = 25 bytes + // Node tombstones first (sorted: 3, 5), then edge tombstones (sorted: 10) + let entry_size = 25; + let off = 8; + + assert_eq!(data[off], 0); // kind = node + let id0 = u64::from_le_bytes(data[off + 1..off + 9].try_into().unwrap()); + let ts0 = i64::from_le_bytes(data[off + 9..off + 17].try_into().unwrap()); + assert_eq!(id0, 3); + assert_eq!(ts0, 1001); + + assert_eq!(data[off + entry_size], 0); // kind = node + let id1 = u64::from_le_bytes( + data[off + entry_size + 1..off + entry_size + 9] + .try_into() + .unwrap(), + ); + let ts1 = i64::from_le_bytes( + data[off + entry_size + 9..off + entry_size + 17] + .try_into() + .unwrap(), + ); + assert_eq!(id1, 5); + assert_eq!(ts1, 1000); + + assert_eq!(data[off + 2 * entry_size], 1); // kind = edge + let id2 = u64::from_le_bytes( + data[off + 2 * entry_size + 1..off + 2 * entry_size + 9] + .try_into() + .unwrap(), + ); + let ts2 = i64::from_le_bytes( + data[off + 2 * entry_size + 9..off + 2 * entry_size + 17] + .try_into() + .unwrap(), + ); + assert_eq!(id2, 10); + assert_eq!(ts2, 2000); } - fn make_node_with_custom_props( - id: u64, - type_id: u32, - key: &str, - props: BTreeMap, - updated_at: i64, - ) -> NodeRecord { - NodeRecord { - id, - type_id, - key: key.to_string(), - props, - created_at: 1000, - updated_at, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, + // --- write_segment (full pipeline) --- + + #[test] + fn test_write_segment_full() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 0); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 0); + mt.apply_op(&WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), 0); + mt.apply_op( + &WalOp::DeleteNode { + id: 99, + deleted_at: 9999, + }, + 0, + ); + + let info = write_segment(&seg_dir, 1, &mt, None).unwrap(); + assert_eq!(info.id, 1); + assert_eq!(info.node_count, 2); + assert_eq!(info.edge_count, 1); + + assert!(seg_dir.join(PACKED_CORE_FILENAME).exists()); + assert!(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME).exists()); + assert_only_manifested_component_files(&seg_dir); + for kind in [ + SegmentComponentKind::NodeRecords, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::NodeMetadata, + SegmentComponentKind::EdgeMetadata, + SegmentComponentKind::Tombstones, + SegmentComponentKind::KeyIndex, + SegmentComponentKind::NodeLabelIndex, + SegmentComponentKind::EdgeLabelIndex, + SegmentComponentKind::EdgeTripleIndex, + SegmentComponentKind::AdjOutIndex, + SegmentComponentKind::AdjOutPostings, + SegmentComponentKind::AdjInIndex, + SegmentComponentKind::AdjInPostings, + SegmentComponentKind::TimestampIndex, + SegmentComponentKind::EdgeWeightIndex, + SegmentComponentKind::EdgeUpdatedAtIndex, + SegmentComponentKind::EdgeValidFromIndex, + SegmentComponentKind::EdgeValidToIndex, + ] { + assert_component_handle_is_packed(&seg_dir, kind); } + assert_component_handle_is_external(&seg_dir, SegmentComponentKind::PackedSegmentContainer); + assert_no_legacy_property_components(&seg_dir); + assert!(!seg_dir.join(SECONDARY_INDEX_DIRNAME).exists()); + assert!(!seg_dir + .join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME) + .exists()); + assert!(!seg_dir + .join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME) + .exists()); } - fn compact_copy_segment_for_test( - source: Arc, - out_dir: &Path, - out_segment_id: u64, - secondary_indexes: &[SecondaryIndexManifestEntry], - ) -> SegmentReader { - std::fs::create_dir_all(out_dir).unwrap(); - let segments = vec![source.clone()]; - let node_copy_info = write_merged_nodes_dat(out_dir, &segments).unwrap(); - let edge_copy_info = write_merged_edges_dat(out_dir, &segments).unwrap(); - let node_copy = &node_copy_info[0]; - let edge_copy = &edge_copy_info[0]; + #[test] + fn test_write_segment_degree_sidecar_overlay_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let mut node_metas = Vec::new(); - for index in 0..source.node_meta_count() as usize { - let ( - node_id, - data_offset, - data_len, - type_id, - updated_at, - weight, - key_len, - _prop_hash_offset, - _prop_hash_count, - last_write_seq, - ) = source.node_meta_at(index).unwrap(); - let (dense_vector_offset, dense_vector_len, sparse_vector_offset, sparse_vector_len) = - source.node_vector_meta_at(index).unwrap(); - node_metas.push(CompactNodeMeta { - node_id, - new_data_offset: data_offset - node_copy.orig_data_start + node_copy.new_data_base, - data_len, - type_id, - updated_at, - weight, - key_len, - dense_vector_offset, - dense_vector_len, - sparse_vector_offset, - sparse_vector_len, - src_seg_idx: 0, - src_data_offset: data_offset, - last_write_seq, - }); + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), 3); + + let mut deltas = NodeIdMap::default(); + deltas.insert(1, DegreeDelta::add_valid_edge(1, 2, 1.0)); + deltas.insert(2, DegreeDelta::add_valid_edge_incoming(1.0)); + let overlay = DegreeOverlaySnapshot::from_flat(deltas); + + write_segment_with_degree_overlay_and_secondary_indexes( + &seg_dir, + 1, + &mt, + None, + overlay.as_ref(), + &[], + ) + .unwrap(); + + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(reader.degree_delta_available()); + assert_eq!(reader.degree_delta(1).unwrap().out_degree, 1); + assert_eq!(reader.degree_delta(2).unwrap().in_degree, 1); + assert_eq!(reader.degree_delta(99).unwrap(), DegreeDelta::ZERO); + } + + #[test] + fn test_segment_reader_tolerates_missing_and_corrupt_degree_sidecar() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + write_segment_without_degree_sidecar_for_test(&seg_dir, 1, &mt, None).unwrap(); + + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.degree_delta_available()); + assert!(reader.get_node(1).unwrap().is_some()); + + std::fs::write(seg_dir.join(DEGREE_DELTA_FILENAME), b"not a degree sidecar").unwrap(); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(!reader.degree_delta_available()); + assert!(reader.get_node(1).unwrap().is_some()); + } + + #[test] + fn test_compaction_copy_writes_v10_manifest_and_root_identity() { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 7)), 3); + write_segment(&source_dir, 1, &mt, None).unwrap(); + let source = Arc::new(SegmentReader::open_unpinned_for_test(&source_dir, 1, None).unwrap()); + let reader = compact_copy_segment_for_test(source, &out_dir, 2, &[]); + + assert_ne!(reader.segment_data_id(), [0; 32]); + let manifest_bytes = fs::read(out_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + assert_eq!(manifest.segment_format_version, SEGMENT_FORMAT_VERSION); + assert_eq!(manifest.segment_id, 2); + assert_eq!(manifest.segment_data_id, reader.segment_data_id()); + assert_eq!( + manifest.build_kind, + crate::segment_components::SegmentComponentBuildKind::Compaction + ); + assert!(out_dir.join(PACKED_CORE_FILENAME).exists()); + assert_only_manifested_component_files(&out_dir); + for kind in [ + SegmentComponentKind::NodeRecords, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::NodeMetadata, + SegmentComponentKind::EdgeMetadata, + SegmentComponentKind::Tombstones, + SegmentComponentKind::KeyIndex, + SegmentComponentKind::NodeLabelIndex, + SegmentComponentKind::EdgeLabelIndex, + SegmentComponentKind::EdgeTripleIndex, + SegmentComponentKind::AdjOutIndex, + SegmentComponentKind::AdjOutPostings, + SegmentComponentKind::AdjInIndex, + SegmentComponentKind::AdjInPostings, + SegmentComponentKind::TimestampIndex, + SegmentComponentKind::EdgeWeightIndex, + SegmentComponentKind::EdgeUpdatedAtIndex, + SegmentComponentKind::EdgeValidFromIndex, + SegmentComponentKind::EdgeValidToIndex, + ] { + assert_component_handle_is_packed(&out_dir, kind); } + let node_meta = reader.node_meta_at(0).unwrap(); + let node_start = node_meta.data_offset as usize; + let node_end = node_start + node_meta.data_len as usize; + assert!(node_end <= reader.raw_nodes_mmap().len()); + assert_eq!(reader.raw_nodes_mmap()[node_start], 1); + assert_eq!( + u32::from_le_bytes( + reader.raw_nodes_mmap()[node_start + 1..node_start + 5] + .try_into() + .unwrap() + ), + node_meta.label_ids.single_label_id() + ); + let (_, edge_data_offset, edge_data_len, edge_from, edge_to, edge_label_id, ..) = + reader.edge_meta_at(0).unwrap(); + let edge_start = edge_data_offset as usize; + let edge_end = edge_start + edge_data_len as usize; + assert!(edge_end <= reader.raw_edges_mmap().len()); + assert_eq!( + u64::from_le_bytes( + reader.raw_edges_mmap()[edge_start..edge_start + 8] + .try_into() + .unwrap() + ), + edge_from + ); + assert_eq!( + u64::from_le_bytes( + reader.raw_edges_mmap()[edge_start + 8..edge_start + 16] + .try_into() + .unwrap() + ), + edge_to + ); + assert_eq!( + u32::from_le_bytes( + reader.raw_edges_mmap()[edge_start + 16..edge_start + 20] + .try_into() + .unwrap() + ), + edge_label_id + ); + } - let mut edge_metas = Vec::new(); - for index in 0..source.edge_meta_count() as usize { - let ( - edge_id, - data_offset, - data_len, - from, - to, - type_id, - updated_at, - weight, - valid_from, - valid_to, - last_write_seq, - ) = source.edge_meta_at(index).unwrap(); - edge_metas.push(CompactEdgeMeta { - edge_id, - new_data_offset: data_offset - edge_copy.orig_data_start + edge_copy.new_data_base, - data_len, - from, - to, - type_id, - updated_at, - weight, - valid_from, - valid_to, - last_write_seq, - }); + #[test] + fn test_v10_flush_writes_segment_manifest_and_root_identity() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 7)), 3); + + let info = write_segment(&seg_dir, 1, &mt, None).unwrap(); + assert_eq!(info.segment_format_version, SEGMENT_FORMAT_VERSION); + assert_ne!(info.segment_data_id, [0; 32]); + + let manifest_bytes = fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + assert_eq!(manifest.segment_format_version, SEGMENT_FORMAT_VERSION); + assert_eq!(manifest.segment_id, 1); + assert_eq!(manifest.segment_data_id, info.segment_data_id); + assert_eq!(manifest.node_count, 2); + assert_eq!(manifest.edge_count, 1); + + for kind in [ + SegmentComponentKind::NodeRecords, + SegmentComponentKind::EdgeRecords, + SegmentComponentKind::NodeMetadata, + SegmentComponentKind::EdgeMetadata, + SegmentComponentKind::Tombstones, + SegmentComponentKind::KeyIndex, + SegmentComponentKind::NodeLabelIndex, + SegmentComponentKind::EdgeLabelIndex, + SegmentComponentKind::EdgeTripleIndex, + SegmentComponentKind::AdjOutIndex, + SegmentComponentKind::AdjOutPostings, + SegmentComponentKind::AdjInIndex, + SegmentComponentKind::AdjInPostings, + SegmentComponentKind::TimestampIndex, + ] { + assert!( + manifest.components.iter().any(|record| record.kind == kind), + "missing required component {:?}", + kind + ); } - write_indexes_from_metadata_with_secondary_indexes( - out_segment_id, - out_dir, - &segments, - &node_metas, - &edge_metas, - None, - true, - secondary_indexes, + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(reader.get_node(1).unwrap().is_some()); + assert!(reader.get_edge(10).unwrap().is_some()); + } + + #[test] + fn test_finalize_compaction_segment_rejects_missing_required_component_before_manifest() { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + write_segment(&source_dir, 1, &mt, None).unwrap(); + + let manifest_bytes = + fs::read(source_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = decode_manifest_envelope(&manifest_bytes).unwrap(); + let mut records = manifest.components.clone(); + records.retain(|record| record.kind != SegmentComponentKind::KeyIndex); + + fs::create_dir_all(&out_dir).unwrap(); + let err = finalize_compaction_segment( + &out_dir, + 2, + manifest.node_count, + manifest.edge_count, + records, + ) + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains(SEGMENT_COMPONENT_MANIFEST_FILENAME)); + assert!(message.contains("KeyIndex")); + assert!(!out_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME).exists()); + } + + #[test] + fn test_finalize_segment_rejects_external_vector_source_truth_in_packed_output() { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); + + let mt = Memtable::new(); + let mut node = make_node(1, 1, "alice"); + node.dense_vector = Some(vec![0.1, 0.2, 0.3]); + mt.apply_op(&WalOp::UpsertNode(node), 1); + let dense_config = DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }; + write_segment(&source_dir, 1, &mt, Some(&dense_config)).unwrap(); + + let manifest_bytes = + fs::read(source_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = decode_manifest_envelope(&manifest_bytes).unwrap(); + let mut records = manifest.components.clone(); + let vector_meta = records + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::NodeVectorMetadata) + .expect("vector metadata record should exist"); + vector_meta.handle = ComponentHandleV1::ExternalFile { + relative_path: "external-node-vector-meta.dat".to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: vector_meta.payload_len, + }; + + fs::create_dir_all(&out_dir).unwrap(); + let err = finalize_compaction_segment( + &out_dir, + 2, + manifest.node_count, + manifest.edge_count, + records, + ) + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains(SEGMENT_COMPONENT_MANIFEST_FILENAME)); + assert!(message.contains("NodeVectorMetadata")); + assert!(message.contains("is not packed")); + assert!(!out_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME).exists()); + } + + #[test] + fn test_finalize_segment_rejects_external_packed_core_optional_in_packed_output() { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 1); + write_segment(&source_dir, 1, &mt, None).unwrap(); + + let manifest_bytes = + fs::read(source_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = decode_manifest_envelope(&manifest_bytes).unwrap(); + let mut records = manifest.components.clone(); + let edge_weight = records + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::EdgeWeightIndex) + .expect("edge weight index record should exist"); + edge_weight.handle = ComponentHandleV1::ExternalFile { + relative_path: "external-edge-weight-index.dat".to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: edge_weight.payload_len, + }; + + fs::create_dir_all(&out_dir).unwrap(); + let err = finalize_compaction_segment( + &out_dir, + 2, + manifest.node_count, + manifest.edge_count, + records, + ) + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains(SEGMENT_COMPONENT_MANIFEST_FILENAME)); + assert!(message.contains("EdgeWeightIndex")); + assert!(message.contains("not allowed as an external file")); + assert!(!out_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME).exists()); + } + + #[test] + fn test_optional_refresh_rejects_packed_core_component_kinds() { + let dir = tempfile::tempdir().unwrap(); + let err = refresh_optional_component_with_writer( + dir.path(), + SegmentComponentKind::EdgeWeightIndex, + "not-used.dat", + ComponentRequirement::Optional { + fallback: ComponentFallbackClass::MetadataScan, + }, + ComponentTrustClass::OptionalExactAccelerator, + Vec::new(), + component_fingerprint("test.bad_optional_refresh", &[]), + |writer| { + writer.write_all(b"should not be written")?; + Ok(()) + }, + ) + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("EdgeWeightIndex"), "got: {message}"); + assert!(message.contains("not eligible"), "got: {message}"); + assert!( + fs::read_dir(dir.path()).unwrap().next().is_none(), + "guard should reject before creating refresh files" + ); + } + + #[test] + fn test_finalize_segment_rejects_external_required_core_in_packed_output() { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + write_segment(&source_dir, 1, &mt, None).unwrap(); + + let manifest_bytes = + fs::read(source_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = decode_manifest_envelope(&manifest_bytes).unwrap(); + let mut records = manifest.components.clone(); + let node_records = records + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::NodeRecords) + .expect("node records component should exist"); + node_records.handle = ComponentHandleV1::ExternalFile { + relative_path: "external-node-records.dat".to_string(), + payload_offset: COMPONENT_IDENTITY_HEADER_LEN as u64, + payload_len: node_records.payload_len, + }; + + fs::create_dir_all(&out_dir).unwrap(); + let err = finalize_compaction_segment( + &out_dir, + 2, + manifest.node_count, + manifest.edge_count, + records, ) - .unwrap(); - SegmentReader::open(out_dir, out_segment_id, None).unwrap() + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains(SEGMENT_COMPONENT_MANIFEST_FILENAME)); + assert!(message.contains("NodeRecords")); + assert!(message.contains("is not packed")); + assert!(!out_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME).exists()); } - // --- encode_node_record / encode_edge_record --- - #[test] - fn test_encode_node_record_roundtrip() { - let node = make_node_with_props(42, 1, "alice"); - let mut buf = Vec::new(); - encode_node_record_into(&mut buf, &node).unwrap(); + fn test_finalize_segment_rejects_invalid_packed_core_contract_before_manifest() { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("seg_0001"); - // Verify structure (no id): type_id(4) + key_len(2) + key(5) + created(8) + updated(8) + weight(4) + props_len(4) + props(N) - assert!(buf.len() > 30 + 5); // minimum size with key "alice" + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 7)), 3); + write_segment(&source_dir, 1, &mt, None).unwrap(); + + let manifest_bytes = + fs::read(source_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = decode_manifest_envelope(&manifest_bytes).unwrap(); + let base_records = manifest.components.clone(); + + let assert_rejected = + |case_name: &str, records: Vec, expected: &str| { + let out_dir = dir.path().join(case_name); + fs::create_dir_all(&out_dir).unwrap(); + let err = finalize_compaction_segment( + &out_dir, + 2, + manifest.node_count, + manifest.edge_count, + records, + ) + .unwrap_err(); + let message = err.to_string(); + assert!( + message.contains(expected), + "expected error containing {expected:?}, got {message:?}" + ); + assert!(!out_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME).exists()); + }; - let type_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); - assert_eq!(type_id, 1); + let mut records = base_records.clone(); + let container = records + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::PackedSegmentContainer) + .expect("packed container record should exist"); + let ComponentHandleV1::ExternalFile { relative_path, .. } = &mut container.handle else { + panic!("packed container should be external"); + }; + *relative_path = "not-segment.core".to_string(); + assert_rejected( + "bad_container_path", + records, + "container path must be segment.core", + ); - let key_len = u16::from_le_bytes(buf[4..6].try_into().unwrap()) as usize; - assert_eq!(key_len, 5); + let mut records = base_records.clone(); + let node_records = records + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::NodeRecords) + .expect("node records component should exist"); + let ComponentHandleV1::PackedRange { + container_component_id, + .. + } = &mut node_records.handle + else { + panic!("node records should be packed"); + }; + *container_component_id = [42; 32]; + assert_rejected( + "wrong_container_id", + records, + "points at the wrong container", + ); - let key = std::str::from_utf8(&buf[6..6 + key_len]).unwrap(); - assert_eq!(key, "alice"); + let mut records = base_records.clone(); + let container_payload_len = records + .iter() + .find(|record| record.kind == SegmentComponentKind::PackedSegmentContainer) + .and_then(|record| match &record.handle { + ComponentHandleV1::ExternalFile { payload_len, .. } => Some(*payload_len), + ComponentHandleV1::PackedRange { .. } => None, + }) + .expect("packed container should have external payload length"); + let node_records = records + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::NodeRecords) + .expect("node records component should exist"); + let ComponentHandleV1::PackedRange { offset, .. } = &mut node_records.handle else { + panic!("node records should be packed"); + }; + *offset = container_payload_len; + assert_rejected( + "range_overflow", + records, + "exceeds segment.core payload length", + ); + + let mut records = base_records.clone(); + let node_offset = records + .iter() + .find(|record| record.kind == SegmentComponentKind::NodeRecords) + .and_then(|record| match &record.handle { + ComponentHandleV1::PackedRange { offset, .. } => Some(*offset), + ComponentHandleV1::ExternalFile { .. } => None, + }) + .expect("node records should have a packed offset"); + let edge_records = records + .iter_mut() + .find(|record| record.kind == SegmentComponentKind::EdgeRecords) + .expect("edge records component should exist"); + let ComponentHandleV1::PackedRange { offset, .. } = &mut edge_records.handle else { + panic!("edge records should be packed"); + }; + *offset = node_offset; + assert_rejected("overlap", records, "overlap"); } #[test] - fn test_encode_edge_record_roundtrip() { - let edge = make_edge(100, 1, 2, 10); - let mut buf = Vec::new(); - encode_edge_record_into(&mut buf, &edge).unwrap(); - - // No id in data section. Starts with from - let from = u64::from_le_bytes(buf[0..8].try_into().unwrap()); - assert_eq!(from, 1); + fn test_v10_reader_rejects_missing_local_manifest() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let to = u64::from_le_bytes(buf[8..16].try_into().unwrap()); - assert_eq!(to, 2); + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + write_segment(&seg_dir, 1, &mt, None).unwrap(); + fs::remove_file(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); - let type_id = u32::from_le_bytes(buf[16..20].try_into().unwrap()); - assert_eq!(type_id, 10); + let err = match SegmentReader::open_unpinned_for_test(&seg_dir, 1, None) { + Ok(_) => panic!("open should reject missing segment manifest"), + Err(error) => error, + }; + assert!(err.to_string().contains("missing segment_manifest.dat")); + assert!(err.to_string().contains("rebuild the database")); } - // --- write_nodes_dat --- - #[test] - fn test_write_nodes_dat_empty() { + fn test_v10_reader_rejects_required_component_trailing_bytes() { let dir = tempfile::tempdir().unwrap(); - let nodes = NodeIdMap::default(); - write_nodes_dat(dir.path(), &nodes).unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let data = fs::read(dir.path().join("nodes.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 0); - assert_eq!(data.len(), 8); // just the count + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + write_segment(&seg_dir, 1, &mt, None).unwrap(); + + let mut core = fs::OpenOptions::new() + .append(true) + .open(seg_dir.join(PACKED_CORE_FILENAME)) + .unwrap(); + core.write_all(&[0xAA]).unwrap(); + core.sync_all().unwrap(); + drop(core); + + let err = match SegmentReader::open_unpinned_for_test(&seg_dir, 1, None) { + Ok(_) => panic!("open should reject required component trailing bytes"), + Err(error) => error, + }; + assert!(err.to_string().contains("does not match file length")); } #[test] - fn test_write_nodes_dat_multiple() { + fn test_v10_reader_treats_optional_component_trailing_bytes_as_unavailable() { let dir = tempfile::tempdir().unwrap(); - let mut nodes = NodeIdMap::default(); - nodes.insert(3, make_node(3, 1, "charlie")); - nodes.insert(1, make_node(1, 1, "alice")); - nodes.insert(2, make_node(2, 1, "bob")); - - write_nodes_dat(dir.path(), &nodes).unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let data = fs::read(dir.path().join("nodes.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 3); + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 2); + mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 7)), 3); + write_segment(&seg_dir, 1, &mt, None).unwrap(); - // Index entries should be sorted by node_id - let idx_start = 8; - let id0 = u64::from_le_bytes(data[idx_start..idx_start + 8].try_into().unwrap()); - let id1 = u64::from_le_bytes(data[idx_start + 16..idx_start + 24].try_into().unwrap()); - let id2 = u64::from_le_bytes(data[idx_start + 32..idx_start + 40].try_into().unwrap()); - assert_eq!(id0, 1); - assert_eq!(id1, 2); - assert_eq!(id2, 3); + let mut planner_stats = fs::OpenOptions::new() + .append(true) + .open(seg_dir.join(PLANNER_STATS_FILENAME)) + .unwrap(); + planner_stats.write_all(&[0xBB]).unwrap(); + planner_stats.sync_all().unwrap(); + drop(planner_stats); - // Verify the offset of the first record leads to valid data - // Format v4: id is NOT in the record, first field is type_id (u32) - let offset0 = - u64::from_le_bytes(data[idx_start + 8..idx_start + 16].try_into().unwrap()) as usize; - let type_id = u32::from_le_bytes(data[offset0..offset0 + 4].try_into().unwrap()); - assert_eq!(type_id, 1); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); + assert!(reader.get_edge(10).unwrap().is_some()); } - // --- write_edges_dat --- - #[test] - fn test_write_edges_dat_empty() { + fn test_v10_reader_rejects_old_format_version_in_envelope() { let dir = tempfile::tempdir().unwrap(); - let edges = NodeIdMap::default(); - write_edges_dat(dir.path(), &edges).unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let data = fs::read(dir.path().join("edges.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 0); + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + write_segment(&seg_dir, 1, &mt, None).unwrap(); + + let manifest_path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + let mut data = fs::read(&manifest_path).unwrap(); + data[12..16].copy_from_slice(&9u32.to_le_bytes()); + fs::write(&manifest_path, &data).unwrap(); + + let err = match SegmentReader::open_unpinned_for_test(&seg_dir, 1, None) { + Ok(_) => panic!("open should reject old segment format"), + Err(error) => error, + }; + assert!( + err.to_string() + .contains("unsupported segment manifest version 9"), + "expected version rejection, got: {}", + err + ); } #[test] - fn test_write_edges_dat_multiple() { + fn test_v10_reader_rejects_future_format_version_in_envelope() { let dir = tempfile::tempdir().unwrap(); - let mut edges = NodeIdMap::default(); - edges.insert(2, make_edge(2, 1, 3, 10)); - edges.insert(1, make_edge(1, 1, 2, 10)); + let seg_dir = dir.path().join("seg_0001"); - write_edges_dat(dir.path(), &edges).unwrap(); + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); + write_segment(&seg_dir, 1, &mt, None).unwrap(); - let data = fs::read(dir.path().join("edges.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 2); + let manifest_path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + let mut data = fs::read(&manifest_path).unwrap(); + data[12..16].copy_from_slice(&(SEGMENT_FORMAT_VERSION + 1).to_le_bytes()); + fs::write(&manifest_path, &data).unwrap(); - // Index should be sorted: edge 1 then edge 2 - let idx_start = 8; - let eid0 = u64::from_le_bytes(data[idx_start..idx_start + 8].try_into().unwrap()); - let eid1 = u64::from_le_bytes(data[idx_start + 16..idx_start + 24].try_into().unwrap()); - assert_eq!(eid0, 1); - assert_eq!(eid1, 2); + let err = match SegmentReader::open_unpinned_for_test(&seg_dir, 1, None) { + Ok(_) => panic!("open should reject future segment format"), + Err(error) => error, + }; + assert!( + err.to_string() + .contains("unsupported segment manifest version"), + "expected version rejection, got: {}", + err + ); } - // --- write_adjacency_index --- + #[test] + fn test_write_segment_empty_memtable() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); - fn make_adj(edge_id: u64, type_id: u32, neighbor_id: u64, weight: f32) -> AdjEntry { - AdjEntry { - edge_id, - type_id, - neighbor_id, - weight, - valid_from: 1000, - valid_to: i64::MAX, + let mt = Memtable::new(); + let info = write_segment(&seg_dir, 1, &mt, None).unwrap(); + assert_eq!(info.node_count, 0); + assert_eq!(info.edge_count, 0); + + assert!(seg_dir.join(PACKED_CORE_FILENAME).exists()); + assert_only_manifested_component_files(&seg_dir); + for kind in [ + SegmentComponentKind::EdgeWeightIndex, + SegmentComponentKind::EdgeUpdatedAtIndex, + SegmentComponentKind::EdgeValidFromIndex, + SegmentComponentKind::EdgeValidToIndex, + ] { + let data = read_manifest_component_payload(&seg_dir, kind); + assert_eq!(u64::from_le_bytes(data[0..8].try_into().unwrap()), 0); } } - fn adj_map_from(node_id: u64, entries: Vec) -> NodeIdMap> { - let mut outer = NodeIdMap::default(); - let mut inner = NodeIdMap::default(); - for e in entries { - inner.insert(e.edge_id, e); + #[test] + fn test_packed_core_writer_alignment_digests_and_container_patch() { + let dir = tempfile::tempdir().unwrap(); + let mut writer = + PackedCoreWriter::create(dir.path(), 42, FLUSH_COMPONENT_GENERATION).unwrap(); + let (node_record, _) = writer + .write_component( + SegmentComponentKind::NodeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.nodes", &[]), + |sink| { + sink.write_all(b"abc")?; + Ok(()) + }, + ) + .unwrap(); + let (edge_record, _) = writer + .write_component( + SegmentComponentKind::EdgeRecords, + ComponentRequirement::Required, + ComponentTrustClass::PrimaryData, + Vec::new(), + component_fingerprint("flush.edges", &[]), + |sink| { + sink.write_all(b"defgh")?; + Ok(()) + }, + ) + .unwrap(); + + let records = writer.finish().unwrap(); + let container = records + .iter() + .find(|record| record.kind == SegmentComponentKind::PackedSegmentContainer) + .unwrap(); + let final_node = records + .iter() + .find(|record| record.kind == SegmentComponentKind::NodeRecords) + .unwrap(); + let final_edge = records + .iter() + .find(|record| record.kind == SegmentComponentKind::EdgeRecords) + .unwrap(); + + assert_eq!(final_node.component_id, node_record.component_id); + assert_eq!(final_edge.component_id, edge_record.component_id); + assert!(matches!( + final_node.handle, + ComponentHandleV1::PackedRange { + container_component_id, + offset: 0, + len: 3, + } if container_component_id == container.component_id + )); + assert!(matches!( + final_edge.handle, + ComponentHandleV1::PackedRange { + container_component_id, + offset: 8, + len: 5, + } if container_component_id == container.component_id + )); + assert_eq!(container.payload_len, 13); + + let mut node_digest = Sha256::new(); + node_digest.update(b"abc"); + assert_eq!( + final_node.payload_digest, + Some(node_digest.finalize().into()) + ); + let mut edge_digest = Sha256::new(); + edge_digest.update(b"defgh"); + assert_eq!( + final_edge.payload_digest, + Some(edge_digest.finalize().into()) + ); + let mut container_digest = Sha256::new(); + container_digest.update(b"abc"); + container_digest.update([0u8; 5]); + container_digest.update(b"defgh"); + assert_eq!( + container.payload_digest, + Some(container_digest.finalize().into()) + ); + + let payload = read_payload_file(&dir.path().join(PACKED_CORE_FILENAME)); + assert_eq!(&payload[0..3], b"abc"); + assert_eq!(&payload[3..8], &[0u8; 5]); + assert_eq!(&payload[8..13], b"defgh"); + } + + #[test] + fn test_edge_metadata_index_weight_encoding_ordering_and_nan() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let mt = Memtable::new(); + + let mut neg = make_edge(1, 1, 2, 10); + neg.weight = -1.0; + let mut neg_zero = make_edge(2, 1, 3, 10); + neg_zero.weight = -0.0; + let mut pos_zero = make_edge(3, 1, 4, 10); + pos_zero.weight = 0.0; + let mut pos = make_edge(4, 1, 5, 10); + pos.weight = 2.0; + let mut nan = make_edge(5, 1, 6, 10); + nan.weight = f32::NAN; + let mut inf = make_edge(6, 1, 7, 10); + inf.weight = f32::INFINITY; + let mut neg_inf = make_edge(7, 1, 8, 10); + neg_inf.weight = f32::NEG_INFINITY; + + for edge in [neg, neg_zero, pos_zero, pos, nan, inf, neg_inf] { + mt.apply_op(&WalOp::UpsertEdge(edge), 1); } - outer.insert(node_id, inner); - outer + write_segment(&seg_dir, 1, &mt, None).unwrap(); + + let entries = read_weight_index_entries(&seg_dir); + assert_eq!(entries.len(), 6); + assert!(!entries.iter().any(|entry| entry.2 == 5)); + assert_eq!( + crate::edge_metadata::encode_edge_weight_key(-0.0), + crate::edge_metadata::encode_edge_weight_key(0.0) + ); + let zero_key = crate::edge_metadata::encode_edge_weight_key(0.0).unwrap(); + assert!(entries.contains(&(10, zero_key, 2))); + assert!(entries.contains(&(10, zero_key, 3))); + let mut sorted = entries.clone(); + sorted.sort_unstable(); + assert_eq!(entries, sorted); } #[test] - fn test_write_adjacency_empty() { + fn test_edge_metadata_indexes_rebuilt_from_compaction_metadata() { let dir = tempfile::tempdir().unwrap(); - let adj: NodeIdMap> = NodeIdMap::default(); - write_adjacency_index(dir.path(), "adj_out", &adj).unwrap(); + let source_dir = dir.path().join("seg_0001"); + let out_dir = dir.path().join("seg_0002"); - let idx_data = fs::read(dir.path().join("adj_out.idx")).unwrap(); - let count = u64::from_le_bytes(idx_data[0..8].try_into().unwrap()); - assert_eq!(count, 0); + let mt = Memtable::new(); + let mut edge_a = make_edge(10, 1, 2, 5); + edge_a.weight = -0.0; + edge_a.updated_at = 300; + edge_a.valid_from = 10; + edge_a.valid_to = 100; + let mut edge_b = make_edge(11, 2, 3, 5); + edge_b.weight = 1.5; + edge_b.updated_at = 400; + edge_b.valid_from = 20; + edge_b.valid_to = 200; + mt.apply_op(&WalOp::UpsertEdge(edge_a), 1); + mt.apply_op(&WalOp::UpsertEdge(edge_b), 2); + + write_segment(&source_dir, 1, &mt, None).unwrap(); + let source = Arc::new(SegmentReader::open_unpinned_for_test(&source_dir, 1, None).unwrap()); + compact_copy_segment_for_test(source, &out_dir, 2, &[]); + + assert_eq!( + read_weight_index_entries(&source_dir), + read_weight_index_entries(&out_dir) + ); + assert_eq!( + read_i64_index_entries(&source_dir, SegmentComponentKind::EdgeUpdatedAtIndex), + read_i64_index_entries(&out_dir, SegmentComponentKind::EdgeUpdatedAtIndex) + ); + assert_eq!( + read_i64_index_entries(&source_dir, SegmentComponentKind::EdgeValidFromIndex), + read_i64_index_entries(&out_dir, SegmentComponentKind::EdgeValidFromIndex) + ); + assert_eq!( + read_i64_index_entries(&source_dir, SegmentComponentKind::EdgeValidToIndex), + read_i64_index_entries(&out_dir, SegmentComponentKind::EdgeValidToIndex) + ); } #[test] - fn test_write_adjacency_single_node() { + fn test_node_vector_prepare_emit_matches_packed_payloads() { + let mut nodes = NodeIdMap::default(); + + let mut node_1 = make_node(1, 1, "alice"); + node_1.dense_vector = Some(vec![0.1, 0.2, 0.3]); + node_1.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); + nodes.insert(node_1.id, node_1); + + let mut node_2 = make_node(2, 1, "bob"); + node_2.sparse_vector = Some(vec![(3, 2.5)]); + nodes.insert(node_2.id, node_2); + + let mut node_3 = make_node(3, 2, "carol"); + node_3.dense_vector = Some(vec![0.4, 0.5, 0.6]); + nodes.insert(node_3.id, node_3); + + let dense_config = DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }; + let ops = nodes + .values() + .cloned() + .map(WalOp::UpsertNode) + .collect::>(); + let (_dir, seg_dir) = write_packed_segment_from_ops(ops, Some(&dense_config)); + let node_records = + read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeRecords); + let node_data = read_record_spans_from_payload(&node_records); + let plan = prepare_node_vector_source_plan(&node_data, &nodes).unwrap(); + + let mut direct_meta = Vec::new(); + write_node_vector_meta_payload(&mut direct_meta, &plan).unwrap(); + let mut direct_dense = Vec::new(); + write_node_dense_vector_blob_payload(&mut direct_dense, &plan, &nodes).unwrap(); + let mut direct_sparse = Vec::new(); + write_node_sparse_vector_blob_payload(&mut direct_sparse, &plan, &nodes).unwrap(); + + assert_eq!( + read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeVectorMetadata), + direct_meta + ); + assert_eq!( + read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeDenseVectorBlob), + direct_dense + ); + assert_eq!( + read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeSparseVectorBlob), + direct_sparse + ); + + assert_eq!(plan.dense_points.len(), 2); + assert_eq!(plan.dense_points[0].node_id, 1); + assert_eq!(plan.dense_points[1].node_id, 3); + } + + #[test] + fn test_write_segment_with_vectors_writes_packed_vector_components() { let dir = tempfile::tempdir().unwrap(); - let adj = adj_map_from( - 1, - vec![ - make_adj(10, 1, 2, 0.5), - make_adj(11, 1, 3, 0.7), - make_adj(12, 2, 4, 1.0), + let seg_dir = dir.path().join("seg_0001"); + + let mt = Memtable::new(); + let mut node = make_node(1, 1, "alice"); + node.dense_vector = Some(vec![0.1, 0.2, 0.3]); + node.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); + mt.apply_op(&WalOp::UpsertNode(node), 0); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 0); + + let dense_config = DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }; + write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); + + assert_only_manifested_component_files(&seg_dir); + assert!(seg_dir + .join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME) + .exists()); + assert!(seg_dir + .join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME) + .exists()); + assert!(seg_dir + .join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME) + .exists()); + assert!(seg_dir + .join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME) + .exists()); + + let manifest_bytes = fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(); + let manifest = + crate::segment_components::decode_manifest_envelope(&manifest_bytes).unwrap(); + let record_for = |kind: SegmentComponentKind| { + manifest + .components + .iter() + .find(|record| record.kind == kind) + .unwrap_or_else(|| panic!("missing component {:?}", kind)) + }; + let node_records = record_for(SegmentComponentKind::NodeRecords); + let node_meta = record_for(SegmentComponentKind::NodeMetadata); + let tombstones = record_for(SegmentComponentKind::Tombstones); + let node_source = crate::segment_components::digest_source_group( + SegmentSourceGroupKind::NodeSource, + &[ + node_records.component_id, + node_meta.component_id, + tombstones.component_id, ], ); + let vector_meta = record_for(SegmentComponentKind::NodeVectorMetadata); + assert_component_handle_is_packed(&seg_dir, SegmentComponentKind::NodeVectorMetadata); + assert_component_handle_is_packed(&seg_dir, SegmentComponentKind::NodeDenseVectorBlob); + assert_component_handle_is_packed(&seg_dir, SegmentComponentKind::NodeSparseVectorBlob); + assert_component_handle_is_external(&seg_dir, SegmentComponentKind::DenseHnswMetadata); + assert_component_handle_is_external(&seg_dir, SegmentComponentKind::DenseHnswGraph); + assert_component_handle_is_external(&seg_dir, SegmentComponentKind::SparsePostingIndex); + assert_component_handle_is_external(&seg_dir, SegmentComponentKind::SparsePostings); + assert!(vector_meta.dependencies.iter().any(|dependency| { + matches!( + dependency, + ComponentDependencyV1::SourceGroup { group, group_id } + if *group == SegmentSourceGroupKind::NodeSource && *group_id == node_source + ) + })); + for kind in [ + SegmentComponentKind::NodeDenseVectorBlob, + SegmentComponentKind::NodeSparseVectorBlob, + ] { + let record = record_for(kind); + assert!(record.dependencies.iter().any(|dependency| { + matches!( + dependency, + ComponentDependencyV1::SourceGroup { group, group_id } + if *group == SegmentSourceGroupKind::NodeSource && *group_id == node_source + ) + })); + assert!(record.dependencies.iter().any(|dependency| { + matches!( + dependency, + ComponentDependencyV1::SourceComponent { kind, component_id } + if *kind == SegmentComponentKind::NodeVectorMetadata + && *component_id == vector_meta.component_id + ) + })); + } + } - write_adjacency_index(dir.path(), "adj_out", &adj).unwrap(); + #[test] + fn test_write_segment_with_sparse_only_vectors_skips_dense_hnsw() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let idx_data = fs::read(dir.path().join("adj_out.idx")).unwrap(); - let count = u64::from_le_bytes(idx_data[0..8].try_into().unwrap()); - // Node 1 has 2 type groups: type_id=1 (2 entries) and type_id=2 (1 entry) - assert_eq!(count, 2); + let mt = Memtable::new(); + let mut node = make_node(1, 1, "sparse"); + node.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); + mt.apply_op(&WalOp::UpsertNode(node), 0); - let dat_data = fs::read(dir.path().join("adj_out.dat")).unwrap(); - // Delta-encoded variable-length postings, much smaller than fixed-size. - // 3 postings with small ids/deltas → expect < 108 bytes (old fixed-size). - assert!(!dat_data.is_empty()); - assert!( - dat_data.len() < 108, - "delta encoding should be smaller than fixed 36-byte postings" - ); + let dense_config = DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }; + write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); + + assert_only_manifested_component_files(&seg_dir); + assert_component_handle_is_packed(&seg_dir, SegmentComponentKind::NodeVectorMetadata); + assert_component_handle_is_packed(&seg_dir, SegmentComponentKind::NodeSparseVectorBlob); + assert!(seg_dir + .join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME) + .exists()); + assert!(seg_dir + .join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME) + .exists()); + assert!(!seg_dir + .join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME) + .exists()); + assert!(!seg_dir + .join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME) + .exists()); } #[test] - fn test_write_adjacency_sorted_index() { - let dir = tempfile::tempdir().unwrap(); - let mut adj: NodeIdMap> = NodeIdMap::default(); - let mut m5 = NodeIdMap::default(); - m5.insert(10, make_adj(10, 1, 6, 0.5)); - adj.insert(5, m5); - let mut m1 = NodeIdMap::default(); - m1.insert(11, make_adj(11, 1, 2, 0.7)); - adj.insert(1, m1); - - write_adjacency_index(dir.path(), "adj_out", &adj).unwrap(); + fn test_sparse_posting_rebuild_rejects_overflowing_source_range() { + let mut node = make_node(1, 1, "alice"); + node.sparse_vector = Some(vec![(2, 1.5)]); + let (_source_dir_guard, source_dir) = + write_packed_segment_from_ops(vec![WalOp::UpsertNode(node)], None); + let source = Arc::new(SegmentReader::open_unpinned_for_test(&source_dir, 1, None).unwrap()); + let out_dir = tempfile::tempdir().unwrap(); + let metas = vec![CompactNodeMeta { + node_id: 1, + new_data_offset: 0, + data_len: 0, + label_ids: NodeLabelSet::single(1).unwrap(), + updated_at: 0, + weight: 1.0, + key_len: 0, + dense_vector_offset: 0, + dense_vector_len: 0, + sparse_vector_offset: u64::MAX, + sparse_vector_len: 1, + src_seg_idx: 0, + src_data_offset: 0, + last_write_seq: 0, + }]; + let source_groups = SegmentComponentSourceGroups { + node_source: [0; 32], + edge_source: [0; 32], + node_property_content_source: [0; 32], + node_property_hash_source: [0; 32], + edge_metadata_source: [0; 32], + degree_source: [0; 32], + dense_vector_source: [0; 32], + sparse_vector_source: [0; 32], + segment_data_id: [0; 32], + }; - let idx_data = fs::read(dir.path().join("adj_out.idx")).unwrap(); - let count = u64::from_le_bytes(idx_data[0..8].try_into().unwrap()); - assert_eq!(count, 2); + let err = write_sparse_posting_index_from_meta( + out_dir.path(), + 2, + &[source], + &metas, + source_groups, + ) + .unwrap_err(); - // First index entry should be node_id=1 (sorted) - let node_id_0 = u64::from_le_bytes(idx_data[8..16].try_into().unwrap()); - let node_id_1 = u64::from_le_bytes(idx_data[8 + 24..16 + 24].try_into().unwrap()); - assert_eq!(node_id_0, 1); - assert_eq!(node_id_1, 5); + assert!( + err.to_string().contains("sparse vector") && err.to_string().contains("overflow"), + "expected sparse vector overflow error, got: {err}" + ); } - // --- write_key_index --- - #[test] - fn test_write_key_index_empty() { - let dir = tempfile::tempdir().unwrap(); - let nodes = NodeIdMap::default(); - write_key_index(dir.path(), &nodes).unwrap(); - - let data = fs::read(dir.path().join("key_index.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 0); - assert_eq!(data.len(), 8); + fn test_segment_dir_paths() { + let db = Path::new("/tmp/mydb"); + assert_eq!( + segment_dir(db, 1), + PathBuf::from("/tmp/mydb/segments/seg_0001") + ); + assert_eq!( + segment_dir(db, 42), + PathBuf::from("/tmp/mydb/segments/seg_0042") + ); + assert_eq!( + segment_tmp_dir(db, 1), + PathBuf::from("/tmp/mydb/segments/seg_0001.tmp") + ); } #[test] - fn test_write_key_index_sorted_by_type_and_key() { - let dir = tempfile::tempdir().unwrap(); - let mut nodes = NodeIdMap::default(); - nodes.insert(1, make_node(1, 2, "zebra")); - nodes.insert(2, make_node(2, 1, "bob")); - nodes.insert(3, make_node(3, 1, "alice")); - - write_key_index(dir.path(), &nodes).unwrap(); + fn test_write_nodes_with_properties() { + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![WalOp::UpsertNode(make_node_with_props(1, 1, "alice"))], + None, + ); - let data = fs::read(dir.path().join("key_index.dat")).unwrap(); + let data = read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeRecords); let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 3); + assert_eq!(count, 1); - // Read offset table - let offsets: Vec = (0..3) - .map(|i| { - let start = 8 + i * 8; - u64::from_le_bytes(data[start..start + 8].try_into().unwrap()) - }) - .collect(); + // Offset should point to valid data. The id is NOT in the record. + // Layout: label_count(1) + label_ids + key_len(2) + key + timestamps(16) + weight(4) + props_len(4) + props + let offset = u64::from_le_bytes(data[16..24].try_into().unwrap()) as usize; + assert_eq!(data[offset], 1); + let label_id = u32::from_le_bytes(data[offset + 1..offset + 5].try_into().unwrap()); + assert_eq!(label_id, 1); - // First entry should be type_id=1, key="alice" - let off0 = offsets[0] as usize; - let type0 = u32::from_le_bytes(data[off0..off0 + 4].try_into().unwrap()); - let node0 = u64::from_le_bytes(data[off0 + 4..off0 + 12].try_into().unwrap()); - let klen0 = u16::from_le_bytes(data[off0 + 12..off0 + 14].try_into().unwrap()) as usize; - let key0 = std::str::from_utf8(&data[off0 + 14..off0 + 14 + klen0]).unwrap(); - assert_eq!(type0, 1); - assert_eq!(key0, "alice"); - assert_eq!(node0, 3); + // Properties should be serialized + let key_len = u16::from_le_bytes(data[offset + 5..offset + 7].try_into().unwrap()) as usize; + let props_len_offset = offset + 7 + key_len + 8 + 8 + 4; // skip key + timestamps + weight + let props_len = u32::from_le_bytes( + data[props_len_offset..props_len_offset + 4] + .try_into() + .unwrap(), + ) as usize; + assert!(props_len > 0); // Properties should be non-empty + } - // Second entry should be type_id=1, key="bob" - let off1 = offsets[1] as usize; - let type1 = u32::from_le_bytes(data[off1..off1 + 4].try_into().unwrap()); - let klen1 = u16::from_le_bytes(data[off1 + 12..off1 + 14].try_into().unwrap()) as usize; - let key1 = std::str::from_utf8(&data[off1 + 14..off1 + 14 + klen1]).unwrap(); - assert_eq!(type1, 1); - assert_eq!(key1, "bob"); + // --- Varint and sentinel encoding roundtrip tests --- - // Third entry should be type_id=2, key="zebra" - let off2 = offsets[2] as usize; - let type2 = u32::from_le_bytes(data[off2..off2 + 4].try_into().unwrap()); - assert_eq!(type2, 2); + #[test] + fn test_varint_roundtrip_zero() { + use crate::segment_reader::tests::read_varint_at_pub; + let mut buf = Vec::new(); + write_varint_to_vec(&mut buf, 0); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 0); + let (val, len) = read_varint_at_pub(&buf, 0); + assert_eq!(val, 0); + assert_eq!(len, 1); } - // --- write_tombstones --- + #[test] + fn test_varint_roundtrip_single_byte_max() { + use crate::segment_reader::tests::read_varint_at_pub; + let mut buf = Vec::new(); + write_varint_to_vec(&mut buf, 127); + assert_eq!(buf.len(), 1); + let (val, len) = read_varint_at_pub(&buf, 0); + assert_eq!(val, 127); + assert_eq!(len, 1); + } #[test] - fn test_write_tombstones_empty() { - let dir = tempfile::tempdir().unwrap(); - let dn = NodeIdMap::default(); - let de = NodeIdMap::default(); - write_tombstones(dir.path(), &dn, &de).unwrap(); + fn test_varint_roundtrip_two_byte_boundary() { + use crate::segment_reader::tests::read_varint_at_pub; + let mut buf = Vec::new(); + write_varint_to_vec(&mut buf, 128); + assert_eq!(buf.len(), 2); + let (val, len) = read_varint_at_pub(&buf, 0); + assert_eq!(val, 128); + assert_eq!(len, 2); + } - let data = fs::read(dir.path().join("tombstones.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 0); + #[test] + fn test_varint_roundtrip_u64_max() { + use crate::segment_reader::tests::read_varint_at_pub; + let mut buf = Vec::new(); + write_varint_to_vec(&mut buf, u64::MAX); + assert_eq!(buf.len(), 10); // ceil(64/7) = 10 bytes + let (val, len) = read_varint_at_pub(&buf, 0); + assert_eq!(val, u64::MAX); + assert_eq!(len, 10); } #[test] - fn test_write_tombstones_mixed() { - let dir = tempfile::tempdir().unwrap(); - let mut dn = NodeIdMap::default(); - dn.insert( - 5, - TombstoneEntry { - deleted_at: 1000, - last_write_seq: 0, - }, - ); - dn.insert( - 3, - TombstoneEntry { - deleted_at: 1001, - last_write_seq: 0, - }, - ); - let mut de = NodeIdMap::default(); - de.insert( - 10, - TombstoneEntry { - deleted_at: 2000, - last_write_seq: 0, - }, - ); + fn test_valid_to_sentinel_roundtrip() { + // i64::MAX encodes as 0 + let vt_max_enc = if i64::MAX == i64::MAX { + 0u64 + } else { + i64::MAX as u64 + 1 + }; + assert_eq!(vt_max_enc, 0); + let vt_max_dec = if vt_max_enc == 0 { + i64::MAX + } else { + (vt_max_enc - 1) as i64 + }; + assert_eq!(vt_max_dec, i64::MAX); - write_tombstones(dir.path(), &dn, &de).unwrap(); + // valid_to = 0 encodes as 1 + let vt_zero: i64 = 0; + let vt_zero_enc = if vt_zero == i64::MAX { + 0u64 + } else { + vt_zero as u64 + 1 + }; + assert_eq!(vt_zero_enc, 1); + let vt_zero_dec = if vt_zero_enc == 0 { + i64::MAX + } else { + (vt_zero_enc - 1) as i64 + }; + assert_eq!(vt_zero_dec, 0); - let data = fs::read(dir.path().join("tombstones.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 3); + // valid_to = 1000 encodes as 1001 + let vt_mid: i64 = 1000; + let vt_mid_enc = if vt_mid == i64::MAX { + 0u64 + } else { + vt_mid as u64 + 1 + }; + assert_eq!(vt_mid_enc, 1001); + let vt_mid_dec = if vt_mid_enc == 0 { + i64::MAX + } else { + (vt_mid_enc - 1) as i64 + }; + assert_eq!(vt_mid_dec, 1000); + } - // Each tombstone: 1 byte kind + 8 bytes id + 8 bytes deleted_at + 8 bytes last_write_seq = 25 bytes - // Node tombstones first (sorted: 3, 5), then edge tombstones (sorted: 10) - let entry_size = 25; - let off = 8; + // --- Packed metadata payload tests --- - assert_eq!(data[off], 0); // kind = node - let id0 = u64::from_le_bytes(data[off + 1..off + 9].try_into().unwrap()); - let ts0 = i64::from_le_bytes(data[off + 9..off + 17].try_into().unwrap()); - assert_eq!(id0, 3); - assert_eq!(ts0, 1001); + #[test] + fn test_packed_node_metadata_payload_roundtrip() { + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![ + WalOp::UpsertNode(make_node_with_props(1, 1, "alice")), + WalOp::UpsertNode(make_node(2, 2, "bob")), + ], + None, + ); + let node_records = + read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeRecords); + let node_data = read_record_spans_from_payload(&node_records); + assert_eq!(node_data.len(), 2); + // Sorted by id + assert_eq!(node_data[0].0, 1); + assert_eq!(node_data[1].0, 2); - assert_eq!(data[off + entry_size], 0); // kind = node - let id1 = u64::from_le_bytes( - data[off + entry_size + 1..off + entry_size + 9] - .try_into() - .unwrap(), + let meta = read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeMetadata); + let count = u64::from_le_bytes(meta[0..8].try_into().unwrap()); + assert_eq!(count, 2); + assert_eq!( + u16::from_le_bytes(meta[8..10].try_into().unwrap()), + NODE_META_FIXED_ENTRY_SIZE ); - let ts1 = i64::from_le_bytes( - data[off + entry_size + 9..off + entry_size + 17] - .try_into() - .unwrap(), + assert_eq!( + u16::from_le_bytes(meta[10..12].try_into().unwrap()), + NODE_META_LABEL_OFFSET_ENTRY_SIZE ); - assert_eq!(id1, 5); - assert_eq!(ts1, 1000); - - assert_eq!(data[off + 2 * entry_size], 1); // kind = edge - let id2 = u64::from_le_bytes( - data[off + 2 * entry_size + 1..off + 2 * entry_size + 9] - .try_into() - .unwrap(), + let fixed_entries_offset = u64::from_le_bytes(meta[16..24].try_into().unwrap()) as usize; + let label_offsets_offset = u64::from_le_bytes(meta[24..32].try_into().unwrap()) as usize; + let label_ids_offset = u64::from_le_bytes(meta[32..40].try_into().unwrap()) as usize; + let label_id_count = u64::from_le_bytes(meta[40..48].try_into().unwrap()); + assert_eq!(fixed_entries_offset, NODE_META_HEADER_SIZE as usize); + assert_eq!( + label_offsets_offset, + fixed_entries_offset + 2 * NODE_META_FIXED_ENTRY_SIZE as usize ); - let ts2 = i64::from_le_bytes( - data[off + 2 * entry_size + 9..off + 2 * entry_size + 17] - .try_into() - .unwrap(), + assert_eq!( + label_ids_offset, + label_offsets_offset + 3 * NODE_META_LABEL_OFFSET_ENTRY_SIZE as usize ); - assert_eq!(id2, 10); - assert_eq!(ts2, 2000); - } - - // --- write_segment (full pipeline) --- - - #[test] - fn test_write_segment_full() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_0001"); + assert_eq!(label_id_count, 2); - let mt = Memtable::new(); - mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 0); - mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 0); - mt.apply_op(&WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), 0); - mt.apply_op( - &WalOp::DeleteNode { - id: 99, - deleted_at: 9999, - }, - 0, + // Verify first entry fields (node_id=1) + let off = fixed_entries_offset; + let nid = u64::from_le_bytes(meta[off..off + 8].try_into().unwrap()); + assert_eq!(nid, 1); + let data_offset = u64::from_le_bytes(meta[off + 8..off + 16].try_into().unwrap()); + assert_eq!(data_offset, node_data[0].1); + let data_len = u32::from_le_bytes(meta[off + 16..off + 20].try_into().unwrap()); + assert_eq!(data_len, node_data[0].2); + let updated_at = i64::from_le_bytes(meta[off + 20..off + 28].try_into().unwrap()); + assert_eq!(updated_at, 1001); // make_node_with_props uses updated_at=1001 + let key_len = u16::from_le_bytes(meta[off + 32..off + 34].try_into().unwrap()); + assert_eq!(key_len, 5); // "alice" + let label_offset = label_offsets_offset; + assert_eq!( + u64::from_le_bytes(meta[label_offset..label_offset + 8].try_into().unwrap()), + 0 + ); + assert_eq!( + u64::from_le_bytes( + meta[label_offset + 8..label_offset + 16] + .try_into() + .unwrap() + ), + 1 + ); + assert_eq!( + u32::from_le_bytes( + meta[label_ids_offset..label_ids_offset + 4] + .try_into() + .unwrap() + ), + 1 ); - let info = write_segment(&seg_dir, 1, &mt, None).unwrap(); - assert_eq!(info.id, 1); - assert_eq!(info.node_count, 2); - assert_eq!(info.edge_count, 1); - - // Verify all files exist - assert!(seg_dir.join("nodes.dat").exists()); - assert!(seg_dir.join("edges.dat").exists()); - assert!(seg_dir.join("adj_out.idx").exists()); - assert!(seg_dir.join("adj_out.dat").exists()); - assert!(seg_dir.join("adj_in.idx").exists()); - assert!(seg_dir.join("adj_in.dat").exists()); - assert!(seg_dir.join("key_index.dat").exists()); - assert!(seg_dir.join("tombstones.dat").exists()); - assert!(seg_dir.join("format.ver").exists()); - assert!(seg_dir.join("node_type_index.dat").exists()); - assert!(seg_dir.join("edge_type_index.dat").exists()); - assert!(seg_dir.join("edge_triple_index.dat").exists()); - // V5 sidecar files - assert!(seg_dir.join("node_meta.dat").exists()); - assert!(seg_dir.join("edge_meta.dat").exists()); - assert!(!seg_dir.join("prop_index.dat").exists()); - assert!(!seg_dir.join("node_prop_hashes.dat").exists()); - assert!(!seg_dir.join(SECONDARY_INDEX_DIRNAME).exists()); - assert!(!seg_dir.join(NODE_VECTOR_META_FILENAME).exists()); - assert!(!seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME).exists()); - assert!(!seg_dir.join(NODE_SPARSE_VECTOR_BLOB_FILENAME).exists()); - assert!(!seg_dir - .join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME) - .exists()); - assert!(!seg_dir - .join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME) - .exists()); + // Second entry (node_id=2) + let off2 = fixed_entries_offset + NODE_META_FIXED_ENTRY_SIZE as usize; + let nid2 = u64::from_le_bytes(meta[off2..off2 + 8].try_into().unwrap()); + assert_eq!(nid2, 2); + let label_offset2 = label_offsets_offset + NODE_META_LABEL_OFFSET_ENTRY_SIZE as usize; + assert_eq!( + u64::from_le_bytes(meta[label_offset2..label_offset2 + 8].try_into().unwrap()), + 1 + ); + assert_eq!( + u64::from_le_bytes( + meta[label_offset2 + 8..label_offset2 + 16] + .try_into() + .unwrap() + ), + 2 + ); + assert_eq!( + u32::from_le_bytes( + meta[label_ids_offset + 4..label_ids_offset + 8] + .try_into() + .unwrap() + ), + 2 + ); + assert!(read_segment_component_manifest(&seg_dir) + .unwrap() + .components + .iter() + .all(|record| record.kind != SegmentComponentKind::NodePropertyHashMetadata)); } #[test] - fn test_write_segment_degree_sidecar_overlay_round_trip() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_0001"); - - let mt = Memtable::new(); - mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); - mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 2); - mt.apply_op(&WalOp::UpsertEdge(make_edge(1, 1, 2, 10)), 3); - - let mut deltas = NodeIdMap::default(); - deltas.insert(1, DegreeDelta::add_valid_edge(1, 2, 1.0)); - deltas.insert(2, DegreeDelta::add_valid_edge_incoming(1.0)); - let overlay = DegreeOverlaySnapshot::from_flat(deltas); - - write_segment_with_degree_overlay_and_secondary_indexes( - &seg_dir, - 1, - &mt, + fn test_packed_edge_metadata_payload_roundtrip() { + let (_dir, seg_dir) = write_packed_segment_from_ops( + vec![ + WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), + WalOp::UpsertEdge(make_edge(20, 3, 4, 7)), + ], None, - overlay.as_ref(), - &[], - ) - .unwrap(); + ); + let edge_records = + read_manifest_component_payload(&seg_dir, SegmentComponentKind::EdgeRecords); + let edge_data = read_record_spans_from_payload(&edge_records); + assert_eq!(edge_data.len(), 2); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); - assert!(reader.degree_delta_available()); - assert_eq!(reader.degree_delta(1).unwrap().out_degree, 1); - assert_eq!(reader.degree_delta(2).unwrap().in_degree, 1); - assert_eq!(reader.degree_delta(99).unwrap(), DegreeDelta::ZERO); + let meta = read_manifest_component_payload(&seg_dir, SegmentComponentKind::EdgeMetadata); + let count = u64::from_le_bytes(meta[0..8].try_into().unwrap()); + assert_eq!(count, 2); + + // Verify first entry (edge_id=10) + let off = 8; + let eid = u64::from_le_bytes(meta[off..off + 8].try_into().unwrap()); + assert_eq!(eid, 10); + let data_offset = u64::from_le_bytes(meta[off + 8..off + 16].try_into().unwrap()); + assert_eq!(data_offset, edge_data[0].1); + let data_len = u32::from_le_bytes(meta[off + 16..off + 20].try_into().unwrap()); + assert_eq!(data_len, edge_data[0].2); + let from = u64::from_le_bytes(meta[off + 20..off + 28].try_into().unwrap()); + assert_eq!(from, 1); + let to = u64::from_le_bytes(meta[off + 28..off + 36].try_into().unwrap()); + assert_eq!(to, 2); + let label_id = u32::from_le_bytes(meta[off + 36..off + 40].try_into().unwrap()); + assert_eq!(label_id, 5); + let valid_to = i64::from_le_bytes(meta[off + 60..off + 68].try_into().unwrap()); + assert_eq!(valid_to, i64::MAX); } #[test] - fn test_segment_reader_tolerates_missing_and_corrupt_degree_sidecar() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_0001"); - - let mt = Memtable::new(); - mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); - write_segment_without_degree_sidecar_for_test(&seg_dir, 1, &mt, None).unwrap(); + fn test_metadata_payloads_empty() { + let (_dir, seg_dir) = write_packed_segment_from_ops(Vec::new(), None); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); - assert!(!reader.degree_delta_available()); - assert!(reader.get_node(1).unwrap().is_some()); + let meta = read_manifest_component_payload(&seg_dir, SegmentComponentKind::NodeMetadata); + assert_eq!(u64::from_le_bytes(meta[0..8].try_into().unwrap()), 0); - std::fs::write(seg_dir.join(DEGREE_DELTA_FILENAME), b"not a degree sidecar").unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); - assert!(!reader.degree_delta_available()); - assert!(reader.get_node(1).unwrap().is_some()); + let emeta = read_manifest_component_payload(&seg_dir, SegmentComponentKind::EdgeMetadata); + assert_eq!(u64::from_le_bytes(emeta[0..8].try_into().unwrap()), 0); + assert!(read_segment_component_manifest(&seg_dir) + .unwrap() + .components + .iter() + .all(|record| record.kind != SegmentComponentKind::NodePropertyHashMetadata)); } #[test] - fn test_write_segment_empty_memtable() { + fn test_write_segment_with_declared_equality_sidecar() { let dir = tempfile::tempdir().unwrap(); let seg_dir = dir.path().join("seg_0001"); let mt = Memtable::new(); - let info = write_segment(&seg_dir, 1, &mt, None).unwrap(); - assert_eq!(info.node_count, 0); - assert_eq!(info.edge_count, 0); + let mut red_props = BTreeMap::new(); + red_props.insert("color".to_string(), PropValue::String("red".to_string())); + let mut green_props = BTreeMap::new(); + green_props.insert("color".to_string(), PropValue::String("green".to_string())); - // All files should still be created - assert!(seg_dir.join("nodes.dat").exists()); - assert!(seg_dir.join("edges.dat").exists()); - } + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), + key: "apple".to_string(), + props: red_props.clone(), + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 0, + ); + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id: 2, + label_ids: NodeLabelSet::single(1).unwrap(), + key: "berry".to_string(), + props: red_props, + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 0, + ); + mt.apply_op( + &WalOp::UpsertNode(NodeRecord { + id: 3, + label_ids: NodeLabelSet::single(1).unwrap(), + key: "lime".to_string(), + props: green_props, + created_at: 1000, + updated_at: 1001, + weight: 0.5, + dense_vector: None, + sparse_vector: None, + last_write_seq: 0, + }), + 0, + ); - #[test] - fn test_write_segment_with_vectors_writes_vector_sidecars() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_0001"); + let entry = SecondaryIndexManifestEntry { + index_id: 7, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, + }; + mt.register_secondary_index(&entry); - let mt = Memtable::new(); - let mut node = make_node(1, 1, "alice"); - node.dense_vector = Some(vec![0.1, 0.2, 0.3]); - node.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); - mt.apply_op(&WalOp::UpsertNode(node), 0); - mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "bob")), 0); + let info = write_segment_with_secondary_indexes( + &seg_dir, + 1, + &mt, + None, + std::slice::from_ref(&entry), + ) + .unwrap(); - let dense_config = DenseVectorConfig { - dimension: 3, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), - }; - write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); + assert_no_legacy_property_components(&seg_dir); + assert!(node_prop_eq_sidecar_path(&seg_dir, entry.index_id).exists()); + let manifest = crate::segment_components::decode_manifest_envelope( + &fs::read(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME)).unwrap(), + ) + .unwrap(); + assert!(manifest.components.iter().any(|record| { + record.kind + == SegmentComponentKind::NodePropertyEqualityIndex { + index_id: entry.index_id, + } + })); - assert!(seg_dir.join(NODE_VECTOR_META_FILENAME).exists()); - assert!(seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME).exists()); - assert!(seg_dir.join(NODE_SPARSE_VECTOR_BLOB_FILENAME).exists()); - assert!(seg_dir - .join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME) - .exists()); - assert!(seg_dir - .join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME) - .exists()); - assert!(seg_dir - .join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME) - .exists()); - assert!(seg_dir - .join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME) - .exists()); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(&entry)) + .unwrap(); + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let green_hash = hash_prop_value(&PropValue::String("green".to_string())); + + let mut reds = reader + .find_nodes_by_secondary_eq_index(entry.index_id, red_hash) + .unwrap(); + reds.sort_unstable(); + assert_eq!(reds, vec![1, 2]); + assert_eq!( + reader + .find_nodes_by_secondary_eq_index(entry.index_id, green_hash) + .unwrap(), + vec![3] + ); } #[test] - fn test_write_segment_with_sparse_only_vectors_skips_dense_hnsw() { + fn test_write_segment_with_declared_edge_property_sidecars() { let dir = tempfile::tempdir().unwrap(); let seg_dir = dir.path().join("seg_0001"); let mt = Memtable::new(); - let mut node = make_node(1, 1, "sparse"); - node.sparse_vector = Some(vec![(2, 1.5), (7, 0.25)]); - mt.apply_op(&WalOp::UpsertNode(node), 0); + let mut red_props = BTreeMap::new(); + red_props.insert("color".to_string(), PropValue::String("red".to_string())); + red_props.insert("score".to_string(), PropValue::Int(10)); + let mut blue_props = BTreeMap::new(); + blue_props.insert("color".to_string(), PropValue::String("blue".to_string())); + blue_props.insert("score".to_string(), PropValue::Int(20)); + let mut ignored_props = BTreeMap::new(); + ignored_props.insert("color".to_string(), PropValue::String("red".to_string())); + ignored_props.insert("score".to_string(), PropValue::Int(30)); + + let mut red_edge = make_edge(10, 1, 2, 1); + red_edge.props = red_props; + let mut blue_edge = make_edge(11, 1, 3, 1); + blue_edge.props = blue_props; + let mut ignored_edge = make_edge(12, 1, 4, 2); + ignored_edge.props = ignored_props; + mt.apply_op(&WalOp::UpsertEdge(red_edge), 1); + mt.apply_op(&WalOp::UpsertEdge(blue_edge), 2); + mt.apply_op(&WalOp::UpsertEdge(ignored_edge), 3); - let dense_config = DenseVectorConfig { - dimension: 3, - metric: DenseMetric::Cosine, - hnsw: HnswConfig::default(), + let eq_entry = SecondaryIndexManifestEntry { + index_id: 17, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, }; - write_segment(&seg_dir, 1, &mt, Some(&dense_config)).unwrap(); + let range_entry = SecondaryIndexManifestEntry { + index_id: 18, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Building, + last_error: None, + }; + mt.register_secondary_index(&eq_entry); + mt.register_secondary_index(&range_entry); + let indexes = vec![eq_entry.clone(), range_entry.clone()]; - assert!(seg_dir.join(NODE_VECTOR_META_FILENAME).exists()); - assert!(!seg_dir.join(NODE_DENSE_VECTOR_BLOB_FILENAME).exists()); - assert!(seg_dir.join(NODE_SPARSE_VECTOR_BLOB_FILENAME).exists()); - assert!(seg_dir - .join(crate::sparse_postings::SPARSE_POSTING_INDEX_FILENAME) - .exists()); - assert!(seg_dir - .join(crate::sparse_postings::SPARSE_POSTINGS_FILENAME) - .exists()); - assert!(!seg_dir - .join(crate::dense_hnsw::DENSE_HNSW_META_FILENAME) - .exists()); - assert!(!seg_dir - .join(crate::dense_hnsw::DENSE_HNSW_GRAPH_FILENAME) - .exists()); - } + let info = write_segment_with_secondary_indexes(&seg_dir, 1, &mt, None, &indexes).unwrap(); - #[test] - fn test_segment_dir_paths() { - let db = Path::new("/tmp/mydb"); + assert!(edge_prop_eq_sidecar_path(&seg_dir, eq_entry.index_id).exists()); + assert!(edge_prop_range_sidecar_path(&seg_dir, range_entry.index_id).exists()); + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &indexes).unwrap(); assert_eq!( - segment_dir(db, 1), - PathBuf::from("/tmp/mydb/segments/seg_0001") + reader.optional_component_availability( + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: eq_entry.index_id, + } + ), + crate::segment_components::ComponentAvailability::Available + ); + assert_eq!( + reader.optional_component_availability(SegmentComponentKind::EdgePropertyRangeIndex { + index_id: range_entry.index_id, + }), + crate::segment_components::ComponentAvailability::Available + ); + let manifest = read_segment_component_manifest(&seg_dir).unwrap(); + let eq_record = manifest + .components + .iter() + .find(|record| { + record.kind + == SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: eq_entry.index_id, + } + }) + .expect("edge equality sidecar record"); + assert!(matches!( + &eq_record.handle, + ComponentHandleV1::ExternalFile { .. } + )); + assert_eq!( + eq_record.build_fingerprint, + component_fingerprint("flush.edge_prop_eq", &[eq_entry.index_id]) + ); + assert!(eq_record.dependencies.iter().any(|dependency| { + matches!( + dependency, + ComponentDependencyV1::SourceGroup { group, .. } + if *group == SegmentSourceGroupKind::EdgeSource + ) + })); + assert!(eq_record.dependencies.iter().any(|dependency| { + matches!( + dependency, + ComponentDependencyV1::SecondaryIndexDeclaration { index_id, target_kind, .. } + if *index_id == eq_entry.index_id + && *target_kind + == crate::segment_components::SecondaryIndexTargetKindForComponents::Edge + ) + })); + assert!(manifest.components.iter().any(|record| { + record.kind + == SegmentComponentKind::EdgePropertyRangeIndex { + index_id: range_entry.index_id, + } + })); + let maintained = + maintained_secondary_index_ids_from_component_records(&manifest.components, &indexes); + assert!(maintained.equality_index_ids.contains(&eq_entry.index_id)); + assert!(maintained.range_index_ids.contains(&range_entry.index_id)); + + let eq_payload = read_manifest_component_payload( + &seg_dir, + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: eq_entry.index_id, + }, ); - assert_eq!( - segment_dir(db, 42), - PathBuf::from("/tmp/mydb/segments/seg_0042") + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let blue_hash = hash_prop_value(&PropValue::String("blue".to_string())); + let eq_count = u64::from_le_bytes(eq_payload[0..8].try_into().unwrap()) as usize; + let mut eq_groups = BTreeMap::new(); + for index in 0..eq_count { + let entry_off = 8 + index * 20; + let value_hash = + u64::from_le_bytes(eq_payload[entry_off..entry_off + 8].try_into().unwrap()); + let ids_offset = u64::from_le_bytes( + eq_payload[entry_off + 8..entry_off + 16] + .try_into() + .unwrap(), + ) as usize; + let id_count = u32::from_le_bytes( + eq_payload[entry_off + 16..entry_off + 20] + .try_into() + .unwrap(), + ) as usize; + let ids = (0..id_count) + .map(|id_index| { + let off = ids_offset + id_index * 8; + u64::from_le_bytes(eq_payload[off..off + 8].try_into().unwrap()) + }) + .collect::>(); + eq_groups.insert(value_hash, ids); + } + assert_eq!(eq_groups.get(&red_hash), Some(&vec![10])); + assert_eq!(eq_groups.get(&blue_hash), Some(&vec![11])); + + let range_payload = read_manifest_component_payload( + &seg_dir, + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: range_entry.index_id, + }, ); + let range_count = u64::from_le_bytes(range_payload[0..8].try_into().unwrap()) as usize; + let range_entries = (0..range_count) + .map(|index| { + let off = 8 + index * 16; + ( + u64::from_le_bytes(range_payload[off..off + 8].try_into().unwrap()), + u64::from_le_bytes(range_payload[off + 8..off + 16].try_into().unwrap()), + ) + }) + .collect::>(); assert_eq!( - segment_tmp_dir(db, 1), - PathBuf::from("/tmp/mydb/segments/seg_0001.tmp") + range_entries, + vec![(10u64 ^ (1u64 << 63), 10), (20u64 ^ (1u64 << 63), 11)] ); } #[test] - fn test_write_nodes_with_properties() { + fn test_compaction_rebuilds_declared_edge_property_sidecars() { let dir = tempfile::tempdir().unwrap(); - let mut nodes = NodeIdMap::default(); - nodes.insert(1, make_node_with_props(1, 1, "alice")); + let source_seg = dir.path().join("seg_0001"); + let compact_seg = dir.path().join("seg_0002"); - write_nodes_dat(dir.path(), &nodes).unwrap(); + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 2); + mt.apply_op(&WalOp::UpsertNode(make_node(3, 1, "c")), 3); - let data = fs::read(dir.path().join("nodes.dat")).unwrap(); - let count = u64::from_le_bytes(data[0..8].try_into().unwrap()); - assert_eq!(count, 1); + let mut red_props = BTreeMap::new(); + red_props.insert("color".to_string(), PropValue::String("red".to_string())); + red_props.insert("score".to_string(), PropValue::Int(10)); + let mut blue_props = BTreeMap::new(); + blue_props.insert("color".to_string(), PropValue::String("blue".to_string())); + blue_props.insert("score".to_string(), PropValue::Int(20)); + let mut ignored_props = BTreeMap::new(); + ignored_props.insert("color".to_string(), PropValue::String("red".to_string())); + ignored_props.insert("score".to_string(), PropValue::Int(30)); + + let mut red_edge = make_edge(10, 1, 2, 1); + red_edge.props = red_props; + let mut blue_edge = make_edge(11, 1, 3, 1); + blue_edge.props = blue_props; + let mut ignored_edge = make_edge(12, 1, 3, 2); + ignored_edge.props = ignored_props; + mt.apply_op(&WalOp::UpsertEdge(red_edge), 4); + mt.apply_op(&WalOp::UpsertEdge(blue_edge), 5); + mt.apply_op(&WalOp::UpsertEdge(ignored_edge), 6); - // Offset should point to valid data - // Format v4: id NOT in record. Layout: type_id(4) + key_len(2) + key + timestamps(16) + weight(4) + props_len(4) + props - let offset = u64::from_le_bytes(data[16..24].try_into().unwrap()) as usize; - let type_id = u32::from_le_bytes(data[offset..offset + 4].try_into().unwrap()); - assert_eq!(type_id, 1); + let eq_entry = SecondaryIndexManifestEntry { + index_id: 117, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, + }; + let range_entry = SecondaryIndexManifestEntry { + index_id: 118, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Building, + last_error: None, + }; + mt.register_secondary_index(&eq_entry); + mt.register_secondary_index(&range_entry); + let indexes = vec![eq_entry.clone(), range_entry.clone()]; - // Properties should be serialized - let key_len = u16::from_le_bytes(data[offset + 4..offset + 6].try_into().unwrap()) as usize; - let props_len_offset = offset + 6 + key_len + 8 + 8 + 4; // skip key + timestamps + weight - let props_len = u32::from_le_bytes( - data[props_len_offset..props_len_offset + 4] - .try_into() - .unwrap(), - ) as usize; - assert!(props_len > 0); // Properties should be non-empty - } + let source_info = + write_segment_with_secondary_indexes(&source_seg, 1, &mt, None, &indexes).unwrap(); + let source = Arc::new( + SegmentReader::open_with_info(&source_seg, &source_info, None, &indexes).unwrap(), + ); + let compact_reader = compact_copy_segment_for_test(source, &compact_seg, 2, &indexes); - // --- Varint and sentinel encoding roundtrip tests --- + assert_eq!( + compact_reader.optional_component_availability( + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: eq_entry.index_id, + } + ), + crate::segment_components::ComponentAvailability::Available + ); + assert_eq!( + compact_reader.optional_component_availability( + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: range_entry.index_id, + } + ), + crate::segment_components::ComponentAvailability::Available + ); - #[test] - fn test_varint_roundtrip_zero() { - use crate::segment_reader::tests::read_varint_at_pub; - let mut buf = Vec::new(); - write_varint_to_vec(&mut buf, 0); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 0); - let (val, len) = read_varint_at_pub(&buf, 0); - assert_eq!(val, 0); - assert_eq!(len, 1); - } + let manifest = read_segment_component_manifest(&compact_seg).unwrap(); + let eq_record = manifest + .components + .iter() + .find(|record| { + record.kind + == SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: eq_entry.index_id, + } + }) + .expect("compacted edge equality sidecar record"); + assert_eq!( + eq_record.build_fingerprint, + component_fingerprint("compaction.edge_prop_eq", &[eq_entry.index_id]) + ); + assert!(eq_record.dependencies.iter().any(|dependency| { + matches!( + dependency, + ComponentDependencyV1::SourceGroup { group, .. } + if *group == SegmentSourceGroupKind::EdgeSource + ) + })); + + let range_record = manifest + .components + .iter() + .find(|record| { + record.kind + == SegmentComponentKind::EdgePropertyRangeIndex { + index_id: range_entry.index_id, + } + }) + .expect("compacted edge range sidecar record"); + assert_eq!( + range_record.build_fingerprint, + component_fingerprint("compaction.edge_prop_range", &[range_entry.index_id]) + ); - #[test] - fn test_varint_roundtrip_single_byte_max() { - use crate::segment_reader::tests::read_varint_at_pub; - let mut buf = Vec::new(); - write_varint_to_vec(&mut buf, 127); - assert_eq!(buf.len(), 1); - let (val, len) = read_varint_at_pub(&buf, 0); - assert_eq!(val, 127); - assert_eq!(len, 1); - } + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let blue_hash = hash_prop_value(&PropValue::String("blue".to_string())); + let groups = read_secondary_eq_groups( + &compact_seg, + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: eq_entry.index_id, + }, + ); + assert_eq!(groups.get(&red_hash), Some(&vec![10])); + assert_eq!(groups.get(&blue_hash), Some(&vec![11])); - #[test] - fn test_varint_roundtrip_two_byte_boundary() { - use crate::segment_reader::tests::read_varint_at_pub; - let mut buf = Vec::new(); - write_varint_to_vec(&mut buf, 128); - assert_eq!(buf.len(), 2); - let (val, len) = read_varint_at_pub(&buf, 0); - assert_eq!(val, 128); - assert_eq!(len, 2); + let range_entries = read_secondary_range_entries( + &compact_seg, + SegmentComponentKind::EdgePropertyRangeIndex { + index_id: range_entry.index_id, + }, + ); + assert_eq!( + range_entries, + vec![(10u64 ^ (1u64 << 63), 10), (20u64 ^ (1u64 << 63), 11)] + ); } #[test] - fn test_varint_roundtrip_u64_max() { - use crate::segment_reader::tests::read_varint_at_pub; - let mut buf = Vec::new(); - write_varint_to_vec(&mut buf, u64::MAX); - assert_eq!(buf.len(), 10); // ceil(64/7) = 10 bytes - let (val, len) = read_varint_at_pub(&buf, 0); - assert_eq!(val, u64::MAX); - assert_eq!(len, 10); - } + fn test_background_build_edge_sidecar_record_is_reader_compatible() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); - #[test] - fn test_valid_to_sentinel_roundtrip() { - // i64::MAX encodes as 0 - let vt_max_enc = if i64::MAX == i64::MAX { - 0u64 - } else { - i64::MAX as u64 + 1 - }; - assert_eq!(vt_max_enc, 0); - let vt_max_dec = if vt_max_enc == 0 { - i64::MAX - } else { - (vt_max_enc - 1) as i64 - }; - assert_eq!(vt_max_dec, i64::MAX); + let mt = Memtable::new(); + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + let mut edge = make_edge(10, 1, 2, 1); + edge.props = props; + mt.apply_op(&WalOp::UpsertEdge(edge), 1); - // valid_to = 0 encodes as 1 - let vt_zero: i64 = 0; - let vt_zero_enc = if vt_zero == i64::MAX { - 0u64 - } else { - vt_zero as u64 + 1 - }; - assert_eq!(vt_zero_enc, 1); - let vt_zero_dec = if vt_zero_enc == 0 { - i64::MAX - } else { - (vt_zero_enc - 1) as i64 + let info = write_segment(&seg_dir, 1, &mt, None).unwrap(); + let entry = SecondaryIndexManifestEntry { + index_id: 217, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, }; - assert_eq!(vt_zero_dec, 0); + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let groups = BTreeMap::from([(red_hash, vec![10])]); + publish_edge_prop_eq_sidecar_component(&seg_dir, &entry, &groups).unwrap(); - // valid_to = 1000 encodes as 1001 - let vt_mid: i64 = 1000; - let vt_mid_enc = if vt_mid == i64::MAX { - 0u64 - } else { - vt_mid as u64 + 1 - }; - assert_eq!(vt_mid_enc, 1001); - let vt_mid_dec = if vt_mid_enc == 0 { - i64::MAX - } else { - (vt_mid_enc - 1) as i64 - }; - assert_eq!(vt_mid_dec, 1000); - } + let manifest = read_segment_component_manifest(&seg_dir).unwrap(); + assert_eq!( + manifest.build_kind, + SegmentComponentBuildKind::OptionalRefresh + ); + let record = manifest + .components + .iter() + .find(|record| { + record.kind + == SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + } + }) + .expect("background-built edge equality sidecar record"); + assert_eq!( + record.build_fingerprint, + component_fingerprint("build.edge_prop_eq", &[entry.index_id]) + ); - // --- V5 sidecar tests --- + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(&entry)) + .unwrap(); + assert_eq!( + reader.optional_component_availability( + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: entry.index_id, + } + ), + crate::segment_components::ComponentAvailability::Available + ); + } #[test] - fn test_write_node_meta_roundtrip() { + fn test_optional_refresh_manifest_keeps_flush_edge_sidecar_available() { let dir = tempfile::tempdir().unwrap(); - let mut nodes = NodeIdMap::default(); - nodes.insert(1, make_node_with_props(1, 1, "alice")); - nodes.insert(2, make_node(2, 2, "bob")); - - let node_data = write_nodes_dat(dir.path(), &nodes).unwrap(); - assert_eq!(node_data.len(), 2); - // Sorted by id - assert_eq!(node_data[0].0, 1); - assert_eq!(node_data[1].0, 2); - - write_node_meta(dir.path(), &node_data, &nodes).unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let meta = fs::read(dir.path().join("node_meta.dat")).unwrap(); - let count = u64::from_le_bytes(meta[0..8].try_into().unwrap()); - assert_eq!(count, 2); + let mt = Memtable::new(); + let mut node_props = BTreeMap::new(); + node_props.insert("color".to_string(), PropValue::String("red".to_string())); + let mut node = make_node(1, 1, "a"); + node.props = node_props; + mt.apply_op(&WalOp::UpsertNode(node), 1); + + let mut edge_props = BTreeMap::new(); + edge_props.insert("color".to_string(), PropValue::String("red".to_string())); + let mut edge = make_edge(10, 1, 1, 1); + edge.props = edge_props; + mt.apply_op(&WalOp::UpsertEdge(edge), 2); + + let node_entry = SecondaryIndexManifestEntry { + index_id: 501, + target: SecondaryIndexTarget::NodeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, + }; + let edge_entry = SecondaryIndexManifestEntry { + index_id: 502, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, + }; + mt.register_secondary_index(&node_entry); + mt.register_secondary_index(&edge_entry); + let indexes = vec![node_entry.clone(), edge_entry.clone()]; - // Verify first entry fields (node_id=1) - let off = 8; - let nid = u64::from_le_bytes(meta[off..off + 8].try_into().unwrap()); - assert_eq!(nid, 1); - let data_offset = u64::from_le_bytes(meta[off + 8..off + 16].try_into().unwrap()); - assert_eq!(data_offset, node_data[0].1); - let data_len = u32::from_le_bytes(meta[off + 16..off + 20].try_into().unwrap()); - assert_eq!(data_len, node_data[0].2); - let type_id = u32::from_le_bytes(meta[off + 20..off + 24].try_into().unwrap()); - assert_eq!(type_id, 1); - let updated_at = i64::from_le_bytes(meta[off + 24..off + 32].try_into().unwrap()); - assert_eq!(updated_at, 1001); // make_node_with_props uses updated_at=1001 - let key_len = u16::from_le_bytes(meta[off + 36..off + 38].try_into().unwrap()); - assert_eq!(key_len, 5); // "alice" + let info = write_segment_with_secondary_indexes(&seg_dir, 1, &mt, None, &indexes).unwrap(); + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + let node_groups = BTreeMap::from([(red_hash, vec![1])]); + publish_node_prop_eq_sidecar_component(&seg_dir, &node_entry, &node_groups).unwrap(); - // CP2 stops emitting legacy property hash metadata for new segments. - let prop_hash_count = u32::from_le_bytes(meta[off + 46..off + 50].try_into().unwrap()); - assert_eq!(prop_hash_count, 0); + let manifest = read_segment_component_manifest(&seg_dir).unwrap(); + assert_eq!( + manifest.build_kind, + SegmentComponentBuildKind::OptionalRefresh + ); + let edge_record = manifest + .components + .iter() + .find(|record| { + record.kind + == SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: edge_entry.index_id, + } + }) + .expect("flush-built edge sidecar survives optional refresh"); + assert_eq!( + edge_record.build_fingerprint, + component_fingerprint("flush.edge_prop_eq", &[edge_entry.index_id]) + ); - // Second entry (node_id=2) - let off2 = 8 + 60; // NODE_META_ENTRY_SIZE = 60 - let nid2 = u64::from_le_bytes(meta[off2..off2 + 8].try_into().unwrap()); - assert_eq!(nid2, 2); - let type_id2 = u32::from_le_bytes(meta[off2 + 20..off2 + 24].try_into().unwrap()); - assert_eq!(type_id2, 2); - let prop_hash_count2 = u32::from_le_bytes(meta[off2 + 46..off2 + 50].try_into().unwrap()); - assert_eq!(prop_hash_count2, 0); - assert!(!dir.path().join("node_prop_hashes.dat").exists()); + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &indexes).unwrap(); + assert_eq!( + reader.optional_component_availability( + SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: edge_entry.index_id, + } + ), + crate::segment_components::ComponentAvailability::Available + ); } #[test] - fn test_write_edge_meta_roundtrip() { + fn test_edge_drop_cleanup_paths_include_generated_sidecars() { let dir = tempfile::tempdir().unwrap(); - let mut edges = NodeIdMap::default(); - edges.insert(10, make_edge(10, 1, 2, 5)); - edges.insert(20, make_edge(20, 3, 4, 7)); - - let edge_data = write_edges_dat(dir.path(), &edges).unwrap(); - assert_eq!(edge_data.len(), 2); - - write_edge_meta(dir.path(), &edge_data, &edges).unwrap(); + let seg_dir = dir.path().join("seg_0001"); - let meta = fs::read(dir.path().join("edge_meta.dat")).unwrap(); - let count = u64::from_le_bytes(meta[0..8].try_into().unwrap()); - assert_eq!(count, 2); + let mt = Memtable::new(); + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String("red".to_string())); + props.insert("score".to_string(), PropValue::Int(10)); + let mut edge = make_edge(10, 1, 2, 1); + edge.props = props; + mt.apply_op(&WalOp::UpsertEdge(edge), 1); - // Verify first entry (edge_id=10) - let off = 8; - let eid = u64::from_le_bytes(meta[off..off + 8].try_into().unwrap()); - assert_eq!(eid, 10); - let data_offset = u64::from_le_bytes(meta[off + 8..off + 16].try_into().unwrap()); - assert_eq!(data_offset, edge_data[0].1); - let data_len = u32::from_le_bytes(meta[off + 16..off + 20].try_into().unwrap()); - assert_eq!(data_len, edge_data[0].2); - let from = u64::from_le_bytes(meta[off + 20..off + 28].try_into().unwrap()); - assert_eq!(from, 1); - let to = u64::from_le_bytes(meta[off + 28..off + 36].try_into().unwrap()); - assert_eq!(to, 2); - let type_id = u32::from_le_bytes(meta[off + 36..off + 40].try_into().unwrap()); - assert_eq!(type_id, 5); - let valid_to = i64::from_le_bytes(meta[off + 60..off + 68].try_into().unwrap()); - assert_eq!(valid_to, i64::MAX); - } + let eq_entry = SecondaryIndexManifestEntry { + index_id: 317, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, + }; + let range_entry = SecondaryIndexManifestEntry { + index_id: 318, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Building, + last_error: None, + }; + mt.register_secondary_index(&eq_entry); + mt.register_secondary_index(&range_entry); + let indexes = vec![eq_entry.clone(), range_entry.clone()]; + write_segment_with_secondary_indexes(&seg_dir, 1, &mt, None, &indexes).unwrap(); - #[test] - fn test_sidecars_empty() { - let dir = tempfile::tempdir().unwrap(); - let nodes = NodeIdMap::default(); - let edges = NodeIdMap::default(); - let node_data = write_nodes_dat(dir.path(), &nodes).unwrap(); - let edge_data = write_edges_dat(dir.path(), &edges).unwrap(); - write_sidecars(dir.path(), &node_data, &edge_data, &nodes, &edges).unwrap(); + let sidecar_dir = secondary_indexes_dir(&seg_dir); + let generated_path = + sidecar_dir.join(format!("edge_prop_eq_{}.g42.dat", eq_entry.index_id)); + let refresh_path = sidecar_dir.join(format!( + ".edge_prop_eq_{}.refresh_tmp.42", + eq_entry.index_id + )); + std::fs::write(&generated_path, b"generated").unwrap(); + std::fs::write(&refresh_path, b"refresh").unwrap(); + + let mut cleanup_paths = + remove_secondary_index_component_records(&seg_dir, &eq_entry).unwrap(); + cleanup_paths.extend(secondary_index_sidecar_paths_for_entry(&seg_dir, &eq_entry)); + cleanup_paths.sort(); + cleanup_paths.dedup(); + for path in cleanup_paths { + let _ = std::fs::remove_file(path); + } - let meta = fs::read(dir.path().join("node_meta.dat")).unwrap(); - assert_eq!(u64::from_le_bytes(meta[0..8].try_into().unwrap()), 0); + assert!(!edge_prop_eq_sidecar_path(&seg_dir, eq_entry.index_id).exists()); + assert!(!generated_path.exists()); + assert!(!refresh_path.exists()); + assert!(edge_prop_range_sidecar_path(&seg_dir, range_entry.index_id).exists()); - let emeta = fs::read(dir.path().join("edge_meta.dat")).unwrap(); - assert_eq!(u64::from_le_bytes(emeta[0..8].try_into().unwrap()), 0); - assert!(!dir.path().join("node_prop_hashes.dat").exists()); + let manifest = read_segment_component_manifest(&seg_dir).unwrap(); + assert!(!manifest.components.iter().any(|record| { + record.kind + == SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: eq_entry.index_id, + } + })); + assert!(manifest.components.iter().any(|record| { + record.kind + == SegmentComponentKind::EdgePropertyRangeIndex { + index_id: range_entry.index_id, + } + })); } #[test] - fn test_write_segment_with_declared_equality_sidecar() { - let dir = tempfile::tempdir().unwrap(); - let seg_dir = dir.path().join("seg_0001"); - - let mt = Memtable::new(); - let mut red_props = BTreeMap::new(); - red_props.insert("color".to_string(), PropValue::String("red".to_string())); - let mut green_props = BTreeMap::new(); - green_props.insert("color".to_string(), PropValue::String("green".to_string())); - - mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 1, - type_id: 1, - key: "apple".to_string(), - props: red_props.clone(), - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), - 0, - ); - mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 2, - type_id: 1, - key: "berry".to_string(), - props: red_props, - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), - 0, - ); - mt.apply_op( - &WalOp::UpsertNode(NodeRecord { - id: 3, - type_id: 1, - key: "lime".to_string(), - props: green_props, - created_at: 1000, - updated_at: 1001, - weight: 0.5, - dense_vector: None, - sparse_vector: None, - last_write_seq: 0, - }), - 0, - ); - - let entry = SecondaryIndexManifestEntry { - index_id: 7, + fn test_maintained_secondary_ids_match_target_kind() { + let node_entry = SecondaryIndexManifestEntry { + index_id: 417, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Building, + last_error: None, + }; + let edge_entry = SecondaryIndexManifestEntry { + index_id: 417, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 1, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, state: SecondaryIndexState::Building, last_error: None, }; - mt.register_secondary_index(&entry); - write_segment_with_secondary_indexes(&seg_dir, 1, &mt, None, std::slice::from_ref(&entry)) - .unwrap(); + let node_record = test_component_record(SegmentComponentKind::NodePropertyEqualityIndex { + index_id: 417, + }); + let edge_record = test_component_record(SegmentComponentKind::EdgePropertyEqualityIndex { + index_id: 417, + }); - assert!(!seg_dir.join("prop_index.dat").exists()); - assert!(!seg_dir.join("node_prop_hashes.dat").exists()); - assert!(node_prop_eq_sidecar_path(&seg_dir, entry.index_id).exists()); + let maintained = maintained_secondary_index_ids_from_component_records( + std::slice::from_ref(&node_record), + std::slice::from_ref(&edge_entry), + ); + assert!(!maintained.equality_index_ids.contains(&edge_entry.index_id)); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); - let red_hash = hash_prop_value(&PropValue::String("red".to_string())); - let green_hash = hash_prop_value(&PropValue::String("green".to_string())); + let maintained = maintained_secondary_index_ids_from_component_records( + std::slice::from_ref(&edge_record), + std::slice::from_ref(&node_entry), + ); + assert!(!maintained.equality_index_ids.contains(&node_entry.index_id)); - let mut reds = reader - .find_nodes_by_secondary_eq_index(entry.index_id, red_hash) - .unwrap(); - reds.sort_unstable(); - assert_eq!(reds, vec![1, 2]); - assert_eq!( - reader - .find_nodes_by_secondary_eq_index(entry.index_id, green_hash) - .unwrap(), - vec![3] + let maintained = maintained_secondary_index_ids_from_component_records( + std::slice::from_ref(&edge_record), + std::slice::from_ref(&edge_entry), ); + assert!(maintained.equality_index_ids.contains(&edge_entry.index_id)); } #[test] @@ -3559,7 +8673,7 @@ mod tests { let eq_entry = SecondaryIndexManifestEntry { index_id: 71, target: SecondaryIndexTarget::NodeProperty { - type_id: 7, + label_id: 7, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -3569,7 +8683,7 @@ mod tests { let range_entry = SecondaryIndexManifestEntry { index_id: 72, target: SecondaryIndexTarget::NodeProperty { - type_id: 7, + label_id: 7, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Range { @@ -3582,22 +8696,31 @@ mod tests { mt.register_secondary_index(&range_entry); let indexes = vec![eq_entry, range_entry]; - write_segment_with_secondary_indexes(&seg_dir, 1, &mt, None, &indexes).unwrap(); + let info = write_segment_with_secondary_indexes(&seg_dir, 1, &mt, None, &indexes).unwrap(); assert!(seg_dir .join(crate::planner_stats::PLANNER_STATS_FILENAME) .exists()); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = SegmentReader::open_with_info(&seg_dir, &info, None, &indexes).unwrap(); let stats = reader.planner_stats().expect("planner stats should load"); + let reference_stats = crate::planner_stats::build_flush_stats( + 1, + &seg_dir, + &mt.nodes(), + &mt.edges(), + &indexes, + ) + .unwrap(); + assert_eq!(stats, &reference_stats); assert_eq!(stats.node_count, 3); assert_eq!(stats.edge_count, 2); assert!(stats.general_property_stats_complete); assert_eq!(stats.node_id_sample, vec![1, 2, 3]); assert_eq!( stats - .type_stats + .node_label_stats .iter() - .map(|type_stats| (type_stats.type_id, type_stats.node_count)) + .map(|node_label_stats| (node_label_stats.label_id, node_label_stats.node_count)) .collect::>(), vec![(7, 2), (8, 1)] ); @@ -3605,7 +8728,7 @@ mod tests { let color_stats = stats .property_stats .iter() - .find(|prop| prop.type_id == 7 && prop.prop_key == "color") + .find(|prop| prop.label_id == 7 && prop.prop_key == "color") .unwrap(); assert_eq!( color_stats.tracked_reason, @@ -3637,7 +8760,7 @@ mod tests { .iter() .find(|stats| { stats.direction == crate::planner_stats::PlannerStatsDirection::Outgoing - && stats.edge_type_id == Some(5) + && stats.edge_label_id == Some(5) }) .unwrap(); assert_eq!(outgoing.source_node_count, 1); @@ -3645,6 +8768,113 @@ mod tests { assert_eq!(outgoing.max_fanout, 2); } + #[test] + fn test_edge_property_declared_planner_stats_survive_flush_compaction_and_reopen() { + let source_dir = tempfile::tempdir().unwrap(); + let source_seg = source_dir.path().join("seg_0001"); + let compact_dir = tempfile::tempdir().unwrap(); + let compact_seg = compact_dir.path().join("seg_0002"); + + let mt = Memtable::new(); + mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "a")), 1); + mt.apply_op(&WalOp::UpsertNode(make_node(2, 1, "b")), 2); + mt.apply_op(&WalOp::UpsertNode(make_node(3, 1, "c")), 3); + for (id, color, score) in [(10, "red", 10), (11, "red", 20), (12, "blue", 30)] { + let mut props = BTreeMap::new(); + props.insert("color".to_string(), PropValue::String(color.to_string())); + props.insert("score".to_string(), PropValue::Int(score)); + let mut edge = make_edge(id, 1, 2, 4); + edge.props = props; + mt.apply_op(&WalOp::UpsertEdge(edge), id); + } + let mut ignored_props = BTreeMap::new(); + ignored_props.insert("color".to_string(), PropValue::String("red".to_string())); + ignored_props.insert("score".to_string(), PropValue::Int(40)); + let mut ignored_edge = make_edge(20, 2, 3, 5); + ignored_edge.props = ignored_props; + mt.apply_op(&WalOp::UpsertEdge(ignored_edge), 20); + + let eq_entry = SecondaryIndexManifestEntry { + index_id: 171, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 4, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_entry = SecondaryIndexManifestEntry { + index_id: 172, + target: SecondaryIndexTarget::EdgeProperty { + label_id: 4, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + mt.register_secondary_index(&eq_entry); + mt.register_secondary_index(&range_entry); + let indexes = vec![eq_entry.clone(), range_entry.clone()]; + + let source_info = + write_segment_with_secondary_indexes(&source_seg, 1, &mt, None, &indexes).unwrap(); + let source_reader = Arc::new( + SegmentReader::open_with_info(&source_seg, &source_info, None, &indexes).unwrap(), + ); + let flush_stats = source_reader.planner_stats().unwrap(); + let reference_stats = crate::planner_stats::build_flush_stats( + 1, + &source_seg, + &mt.nodes(), + &mt.edges(), + &indexes, + ) + .unwrap(); + assert_eq!(flush_stats, &reference_stats); + + let equality = flush_stats + .equality_index_stats + .iter() + .find(|stats| stats.index_id == eq_entry.index_id) + .expect("edge equality stats"); + assert_eq!(equality.target_label_id, 4); + assert_eq!(equality.prop_key, "color"); + assert_eq!(equality.total_postings, 3); + assert_eq!(equality.value_group_count, 2); + assert_eq!(equality.max_group_postings, 2); + assert!(equality.sidecar_present_at_build); + + let range = flush_stats + .range_index_stats + .iter() + .find(|stats| stats.index_id == range_entry.index_id) + .expect("edge range stats"); + assert_eq!(range.target_label_id, 4); + assert_eq!(range.prop_key, "score"); + assert_eq!(range.total_entries, 3); + assert!(range.sidecar_present_at_build); + + let compact_reader = + compact_copy_segment_for_test(source_reader.clone(), &compact_seg, 2, &indexes); + let compact_stats = compact_reader.planner_stats().unwrap(); + assert_eq!( + compact_stats.build_kind, + crate::planner_stats::PlannerStatsBuildKind::Compaction + ); + assert_eq!( + compact_stats.equality_index_stats, + flush_stats.equality_index_stats + ); + assert_eq!( + compact_stats.range_index_stats, + flush_stats.range_index_stats + ); + } + #[test] fn test_planner_stats_sidecar_is_deterministic_for_same_segment_contents() { let mut props = BTreeMap::new(); @@ -3694,7 +8924,7 @@ mod tests { let declared = SecondaryIndexManifestEntry { index_id: 91, target: SecondaryIndexTarget::NodeProperty { - type_id: 1, + label_id: 1, prop_key: "zz_declared".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -3703,7 +8933,7 @@ mod tests { }; mt.register_secondary_index(&declared); - write_segment_with_secondary_indexes( + let info = write_segment_with_secondary_indexes( &seg_dir, 1, &mt, @@ -3711,14 +8941,16 @@ mod tests { std::slice::from_ref(&declared), ) .unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = + SegmentReader::open_with_info(&seg_dir, &info, None, std::slice::from_ref(&declared)) + .unwrap(); let stats = reader.planner_stats().unwrap(); - let type_one_props: Vec<_> = stats + let label_one_props: Vec<_> = stats .property_stats .iter() - .filter(|prop| prop.type_id == 1) + .filter(|prop| prop.label_id == 1) .collect(); - let general_count = type_one_props + let general_count = label_one_props .iter() .filter(|prop| { prop.tracked_reason @@ -3727,9 +8959,9 @@ mod tests { .count(); assert_eq!( general_count, - crate::planner_stats::PLANNER_STATS_MAX_PROPERTY_KEYS_PER_TYPE + crate::planner_stats::PLANNER_STATS_MAX_PROPERTY_KEYS_PER_LABEL ); - let declared_stats = type_one_props + let declared_stats = label_one_props .iter() .find(|prop| prop.prop_key == "zz_declared") .unwrap(); @@ -3747,7 +8979,7 @@ mod tests { let mt = Memtable::new(); let mut first_props = BTreeMap::new(); - for idx in 0..crate::planner_stats::PLANNER_STATS_MAX_PROPERTY_KEYS_PER_TYPE * 4 { + for idx in 0..crate::planner_stats::PLANNER_STATS_MAX_PROPERTY_KEYS_PER_LABEL * 4 { first_props.insert(format!("one_off_{:04}", idx), PropValue::UInt(idx as u64)); } mt.apply_op( @@ -3770,12 +9002,12 @@ mod tests { } write_segment(&seg_dir, 1, &mt, None).unwrap(); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); let stats = reader.planner_stats().unwrap(); let late_hot = stats .property_stats .iter() - .find(|prop| prop.type_id == 1 && prop.prop_key == "zz_late_hot") + .find(|prop| prop.label_id == 1 && prop.prop_key == "zz_late_hot") .expect("late frequent property should be tracked"); assert_eq!( late_hot.tracked_reason, @@ -3786,7 +9018,7 @@ mod tests { } #[test] - fn test_planner_stats_declared_index_for_absent_type_stays_available() { + fn test_planner_stats_declared_index_for_absent_label_stays_available() { let source_dir = tempfile::tempdir().unwrap(); let source_seg = source_dir.path().join("seg_0001"); let compact_dir = tempfile::tempdir().unwrap(); @@ -3803,7 +9035,7 @@ mod tests { let absent_declared = SecondaryIndexManifestEntry { index_id: 101, target: SecondaryIndexTarget::NodeProperty { - type_id: 99, + label_id: 99, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -3812,7 +9044,7 @@ mod tests { }; mt.register_secondary_index(&absent_declared); - write_segment_with_secondary_indexes( + let source_info = write_segment_with_secondary_indexes( &source_seg, 1, &mt, @@ -3820,12 +9052,20 @@ mod tests { std::slice::from_ref(&absent_declared), ) .unwrap(); - let source_reader = Arc::new(SegmentReader::open(&source_seg, 1, None).unwrap()); + let source_reader = Arc::new( + SegmentReader::open_with_info( + &source_seg, + &source_info, + None, + std::slice::from_ref(&absent_declared), + ) + .unwrap(), + ); let flush_stats = source_reader.planner_stats().unwrap(); assert!(flush_stats .property_stats .iter() - .all(|prop| prop.type_id != 99)); + .all(|prop| prop.label_id != 99)); let equality = flush_stats .equality_index_stats .iter() @@ -3839,7 +9079,7 @@ mod tests { assert!(compact_stats .property_stats .iter() - .all(|prop| prop.type_id != 99)); + .all(|prop| prop.label_id != 99)); let equality = compact_stats .equality_index_stats .iter() @@ -3878,7 +9118,7 @@ mod tests { let eq_entry = SecondaryIndexManifestEntry { index_id: 81, target: SecondaryIndexTarget::NodeProperty { - type_id: 9, + label_id: 9, prop_key: "color".to_string(), }, kind: SecondaryIndexKind::Equality, @@ -3888,7 +9128,7 @@ mod tests { let range_entry = SecondaryIndexManifestEntry { index_id: 82, target: SecondaryIndexTarget::NodeProperty { - type_id: 9, + label_id: 9, prop_key: "score".to_string(), }, kind: SecondaryIndexKind::Range { @@ -3901,8 +9141,11 @@ mod tests { mt.register_secondary_index(&range_entry); let indexes = vec![eq_entry, range_entry]; - write_segment_with_secondary_indexes(&source_seg, 1, &mt, None, &indexes).unwrap(); - let source_reader = Arc::new(SegmentReader::open(&source_seg, 1, None).unwrap()); + let source_info = + write_segment_with_secondary_indexes(&source_seg, 1, &mt, None, &indexes).unwrap(); + let source_reader = Arc::new( + SegmentReader::open_with_info(&source_seg, &source_info, None, &indexes).unwrap(), + ); let compact_reader = compact_copy_segment_for_test(source_reader.clone(), &compact_seg, 2, &indexes); @@ -3914,7 +9157,7 @@ mod tests { ); assert!(compact_stats.general_property_stats_complete); assert_eq!(compact_stats.general_property_sampled_node_count, 3); - assert_eq!(compact_stats.type_stats, flush_stats.type_stats); + assert_eq!(compact_stats.node_label_stats, flush_stats.node_label_stats); assert_eq!(compact_stats.timestamp_stats, flush_stats.timestamp_stats); assert_eq!(compact_stats.property_stats, flush_stats.property_stats); assert_eq!( @@ -3929,6 +9172,188 @@ mod tests { assert_eq!(compact_stats.node_id_sample, flush_stats.node_id_sample); } + #[test] + fn test_multi_label_compaction_rebuilds_label_scoped_indexes_like_flush() { + let source_dir = tempfile::tempdir().unwrap(); + let source_seg = source_dir.path().join("seg_0001"); + let compact_dir = tempfile::tempdir().unwrap(); + let compact_seg = compact_dir.path().join("seg_0002"); + + let mt = Memtable::new(); + let mut props_one = BTreeMap::new(); + props_one.insert("color".to_string(), PropValue::String("green".to_string())); + props_one.insert("score".to_string(), PropValue::Int(10)); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(1, &[1], "one", props_one, 100)), + 1, + ); + + let mut props_two = BTreeMap::new(); + props_two.insert("color".to_string(), PropValue::String("red".to_string())); + props_two.insert("score".to_string(), PropValue::Int(20)); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(2, &[2, 3], "two", props_two, 200)), + 2, + ); + + let labels_ten = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; + let mut props_ten = BTreeMap::new(); + props_ten.insert("color".to_string(), PropValue::String("blue".to_string())); + props_ten.insert("score".to_string(), PropValue::Int(30)); + mt.apply_op( + &WalOp::UpsertNode(make_node_with_labels(3, &labels_ten, "ten", props_ten, 300)), + 3, + ); + + let eq_entry = SecondaryIndexManifestEntry { + index_id: 901, + target: SecondaryIndexTarget::NodeProperty { + label_id: 3, + prop_key: "color".to_string(), + }, + kind: SecondaryIndexKind::Equality, + state: SecondaryIndexState::Ready, + last_error: None, + }; + let range_entry = SecondaryIndexManifestEntry { + index_id: 902, + target: SecondaryIndexTarget::NodeProperty { + label_id: 12, + prop_key: "score".to_string(), + }, + kind: SecondaryIndexKind::Range { + domain: SecondaryIndexRangeDomain::Int, + }, + state: SecondaryIndexState::Ready, + last_error: None, + }; + mt.register_secondary_index(&eq_entry); + mt.register_secondary_index(&range_entry); + let indexes = vec![eq_entry.clone(), range_entry.clone()]; + + let source_info = + write_segment_with_secondary_indexes(&source_seg, 1, &mt, None, &indexes).unwrap(); + let source_reader = Arc::new( + SegmentReader::open_with_info(&source_seg, &source_info, None, &indexes).unwrap(), + ); + let compact_reader = + compact_copy_segment_for_test(source_reader.clone(), &compact_seg, 2, &indexes); + + for (node_id, key, updated_at, labels) in [ + (1, "one", 100, vec![1]), + (2, "two", 200, vec![2, 3]), + (3, "ten", 300, labels_ten.to_vec()), + ] { + let source_node = source_reader.get_node(node_id).unwrap().unwrap(); + let compact_node = compact_reader.get_node(node_id).unwrap().unwrap(); + assert_eq!(compact_node.label_ids.as_slice(), labels.as_slice()); + assert_eq!(compact_node.label_ids, source_node.label_ids); + for label_id in labels { + assert_eq!( + source_reader + .node_by_key(label_id, key) + .unwrap() + .map(|node| node.id), + compact_reader + .node_by_key(label_id, key) + .unwrap() + .map(|node| node.id) + ); + assert_eq!( + source_reader.nodes_by_label_id(label_id).unwrap(), + compact_reader.nodes_by_label_id(label_id).unwrap() + ); + assert_eq!( + source_reader + .nodes_by_time_range(label_id, updated_at, updated_at) + .unwrap(), + compact_reader + .nodes_by_time_range(label_id, updated_at, updated_at) + .unwrap() + ); + } + } + assert!(compact_reader.node_by_key(1, "two").unwrap().is_none()); + assert!(compact_reader.node_by_key(9, "ten").unwrap().is_none()); + + let red_hash = hash_prop_value(&PropValue::String("red".to_string())); + assert_eq!( + compact_reader + .find_nodes_by_secondary_eq_index(eq_entry.index_id, red_hash) + .unwrap(), + vec![2] + ); + let encoded_score = + encode_range_prop_value(SecondaryIndexRangeDomain::Int, &PropValue::Int(30)).unwrap(); + assert_eq!( + compact_reader + .find_nodes_by_secondary_range_index_if_present( + range_entry.index_id, + Some((encoded_score, true)), + Some((encoded_score, true)), + None, + ) + .unwrap(), + Some(vec![(encoded_score, 3)]) + ); + + let flush_stats = source_reader.planner_stats().unwrap(); + let compact_stats = compact_reader.planner_stats().unwrap(); + let flush_label_counts = flush_stats + .node_label_stats + .iter() + .map(|stats| (stats.label_id, stats.node_count)) + .collect::>(); + let expected_label_counts = ([(1, 1), (2, 1), (3, 1)]) + .into_iter() + .chain(labels_ten.into_iter().map(|label_id| (label_id, 1))) + .collect::>(); + assert_eq!(flush_stats.node_count, 3); + assert_eq!(flush_label_counts, expected_label_counts); + assert_eq!( + flush_stats + .node_label_stats + .iter() + .map(|stats| stats.node_count) + .sum::(), + 13 + ); + for node_label_stats in &flush_stats.node_label_stats { + let timestamp_stats = flush_stats + .timestamp_stats + .iter() + .find(|stats| stats.label_id == node_label_stats.label_id) + .unwrap(); + assert_eq!(timestamp_stats.count, node_label_stats.node_count); + } + let color_declared_property = flush_stats + .property_stats + .iter() + .find(|stats| stats.label_id == 3 && stats.prop_key == "color") + .unwrap(); + assert_eq!(color_declared_property.present_count, 1); + let score_declared_property = flush_stats + .property_stats + .iter() + .find(|stats| stats.label_id == 12 && stats.prop_key == "score") + .unwrap(); + assert_eq!(score_declared_property.present_count, 1); + assert_eq!(flush_stats.equality_index_stats[0].total_postings, 1); + assert_eq!(flush_stats.range_index_stats[0].total_entries, 1); + + assert_eq!(compact_stats.node_label_stats, flush_stats.node_label_stats); + assert_eq!(compact_stats.timestamp_stats, flush_stats.timestamp_stats); + assert_eq!(compact_stats.property_stats, flush_stats.property_stats); + assert_eq!( + compact_stats.equality_index_stats, + flush_stats.equality_index_stats + ); + assert_eq!( + compact_stats.range_index_stats, + flush_stats.range_index_stats + ); + } + #[test] fn test_compaction_planner_stats_marks_general_property_decode_budget() { let source_dir = tempfile::tempdir().unwrap(); @@ -3952,7 +9377,8 @@ mod tests { ); } write_segment(&source_seg, 1, &mt, None).unwrap(); - let source_reader = Arc::new(SegmentReader::open(&source_seg, 1, None).unwrap()); + let source_reader = + Arc::new(SegmentReader::open_unpinned_for_test(&source_seg, 1, None).unwrap()); let compact_reader = compact_copy_segment_for_test(source_reader, &compact_seg, 2, &[]); let stats = compact_reader.planner_stats().unwrap(); @@ -3962,13 +9388,13 @@ mod tests { let sampled = stats .property_stats .iter() - .find(|prop| prop.type_id == 1 && prop.prop_key == "sampled") + .find(|prop| prop.label_id == 1 && prop.prop_key == "sampled") .unwrap(); assert_eq!(sampled.present_count, 1024); } #[test] - fn test_planner_stats_write_failure_does_not_block_segment_publish() { + fn test_planner_stats_final_tmp_collision_does_not_block_segment_publish() { let dir = tempfile::tempdir().unwrap(); let seg_dir = dir.path().join("seg_0001"); std::fs::create_dir_all(seg_dir.join("planner_stats.tmp")).unwrap(); @@ -3977,13 +9403,13 @@ mod tests { mt.apply_op(&WalOp::UpsertNode(make_node(1, 1, "alice")), 1); let info = write_segment(&seg_dir, 1, &mt, None).unwrap(); assert_eq!(info.node_count, 1); - assert!(seg_dir.join("nodes.dat").exists()); - assert!(!seg_dir + assert!(seg_dir.join(PACKED_CORE_FILENAME).exists()); + assert!(seg_dir .join(crate::planner_stats::PLANNER_STATS_FILENAME) .exists()); - let reader = SegmentReader::open(&seg_dir, 1, None).unwrap(); + let reader = SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(); assert!(reader.get_node(1).unwrap().is_some()); - assert!(!reader.planner_stats_available()); + assert!(reader.planner_stats_available()); } } diff --git a/src/source_list.rs b/src/source_list.rs index 9eab0af..1a7cd0f 100644 --- a/src/source_list.rs +++ b/src/source_list.rs @@ -7,11 +7,15 @@ //! checks that consult all live sources in the correct order. Engine read paths //! delegate to `SourceList` instead of open-coding memtable + segment logic. +use crate::edge_metadata::{EdgeMetadataCandidate, RangeBoundFlags}; use crate::engine::ReadViewImmutableEpoch; use crate::error::EngineError; use crate::memtable::Memtable; use crate::segment_reader::SegmentReader; use crate::types::*; +use std::cmp::Reverse; +use std::collections::{BTreeMap, BinaryHeap}; +use std::ops::ControlFlow; use std::sync::Arc; /// Concrete borrowing struct over the three source layers. This is not a trait, @@ -23,6 +27,20 @@ pub struct SourceList<'a> { pub(crate) snapshot_seq: u64, } +pub(crate) enum LimitedEdgeIndexRead { + Ready(Vec), + TooBroad, + MissingSidecar, +} + +#[derive(Clone, Copy)] +struct MemtableEndpointLimit<'a> { + direction: Direction, + label_filter_ids: Option<&'a [u32]>, + snapshot_seq: u64, + limit: usize, +} + impl<'a> SourceList<'a> { /// Find multiple nodes by ID across all sources. /// @@ -115,6 +133,145 @@ impl<'a> SourceList<'a> { Ok(results) } + pub(crate) fn find_edge_metadata( + &self, + ids: &[u64], + ) -> Result>, EngineError> { + let mut results = vec![None; ids.len()]; + if ids.is_empty() { + return Ok(results); + } + + let mut remaining: Vec<(usize, u64)> = ids + .iter() + .enumerate() + .map(|(index, &id)| (index, id)) + .collect(); + + remaining.retain(|&(index, id)| { + if let Some(meta) = self.active.get_edge_metadata_at(id, self.snapshot_seq) { + results[index] = Some(meta); + false + } else { + !self.active.is_edge_deleted_at(id, self.snapshot_seq) + } + }); + + for epoch in self.immutable { + if remaining.is_empty() { + break; + } + remaining.retain(|&(index, id)| { + if let Some(meta) = epoch.memtable.get_edge_metadata_at(id, self.snapshot_seq) { + results[index] = Some(meta); + false + } else { + !epoch.memtable.is_edge_deleted_at(id, self.snapshot_seq) + } + }); + } + + if !remaining.is_empty() { + remaining.sort_unstable_by_key(|&(_, id)| id); + for seg in self.segments { + if remaining.is_empty() { + break; + } + remaining.retain(|&(_, id)| !seg.is_edge_deleted(id)); + if remaining.is_empty() { + break; + } + seg.get_edge_metadata_batch(&remaining, &mut results)?; + remaining.retain(|&(index, _)| results[index].is_none()); + } + } + + Ok(results) + } + + pub(crate) fn find_node_visibility_meta( + &self, + ids: &[u64], + ) -> Result, EngineError> { + let mut results = vec![NodeVisibilityState::Missing; ids.len()]; + if ids.is_empty() { + return Ok(results); + } + + let mut remaining: Vec<(usize, u64)> = ids + .iter() + .enumerate() + .map(|(index, &id)| (index, id)) + .collect(); + + remaining = self.active.batch_get_node_visibility_meta_at( + &remaining, + self.snapshot_seq, + &mut results, + ); + + for epoch in self.immutable { + if remaining.is_empty() { + break; + } + remaining = epoch.memtable.batch_get_node_visibility_meta_at( + &remaining, + self.snapshot_seq, + &mut results, + ); + } + + if !remaining.is_empty() { + remaining.sort_unstable_by_key(|&(_, id)| id); + let mut compact_lookups = Vec::new(); + let mut segment_results: Vec> = Vec::new(); + for seg in self.segments { + if remaining.is_empty() { + break; + } + + remaining.retain(|&(index, id)| { + if seg.is_node_deleted(id) { + results[index] = NodeVisibilityState::Deleted; + false + } else { + true + } + }); + if remaining.is_empty() { + break; + } + + compact_lookups.clear(); + compact_lookups.reserve(remaining.len()); + for (compact_index, &(_, id)) in remaining.iter().enumerate() { + compact_lookups.push((compact_index, id)); + } + segment_results.clear(); + segment_results.resize(remaining.len(), None); + + seg.get_node_meta_batch(&compact_lookups, &mut segment_results)?; + let mut compact_index = 0usize; + remaining.retain(|&(index, _)| { + let state = segment_results[compact_index]; + compact_index += 1; + if let Some((label_ids, updated_at, weight)) = state { + results[index] = NodeVisibilityState::Live(NodeVisibilityMeta { + label_ids, + updated_at, + weight, + }); + false + } else { + true + } + }); + } + } + + Ok(results) + } + /// Find a node by ID across all sources. Short-circuits on the first /// source that has an opinion (live record or tombstone). pub fn find_node(&self, id: u64) -> Result, EngineError> { @@ -177,7 +334,7 @@ impl<'a> SourceList<'a> { Ok(None) } - pub fn find_nodes_by_keys<'b>( + pub fn find_nodes_by_label_keys<'b>( &self, keys: &[(u32, &'b str)], ) -> Result>, EngineError> { @@ -188,117 +345,129 @@ impl<'a> SourceList<'a> { } let mut remaining: Vec<(usize, u32, &'b str)> = Vec::with_capacity(n); - for (i, &(type_id, key)) in keys.iter().enumerate() { - if let Some(node) = self.active.node_by_key_at(type_id, key, self.snapshot_seq) { + for (i, &(label_id, key)) in keys.iter().enumerate() { + if let Some(node) = self.active.node_by_key_at(label_id, key, self.snapshot_seq) { results[i] = Some(node); } else { - remaining.push((i, type_id, key)); + remaining.push((i, label_id, key)); } } + let mut candidates: Vec<(usize, u32, &'b str, u64)> = Vec::new(); for (epoch_idx, epoch) in self.immutable.iter().enumerate() { if remaining.is_empty() { break; } - remaining.retain(|&(i, type_id, key)| { + remaining.retain(|&(i, label_id, key)| { if let Some(node) = epoch .memtable - .node_by_key_at(type_id, key, self.snapshot_seq) + .node_by_key_at(label_id, key, self.snapshot_seq) { if self.is_node_tombstoned_above_immutable(node.id, epoch_idx) { return false; } - results[i] = Some(node); + candidates.push((i, label_id, key, node.id)); return false; } true }); } - if remaining.is_empty() { - return Ok(results); - } - - remaining.sort_unstable_by(|left, right| (left.1, left.2).cmp(&(right.1, right.2))); - - let mut deleted_above = self.active.collect_deleted_nodes_at(self.snapshot_seq); - for epoch in self.immutable { - deleted_above.extend(epoch.memtable.collect_deleted_nodes_at(self.snapshot_seq)); + if !remaining.is_empty() { + remaining.sort_unstable_by(|left, right| (left.1, left.2).cmp(&(right.1, right.2))); } - for (seg_idx, seg) in self.segments.iter().enumerate() { + for seg in self.segments { if remaining.is_empty() { break; } - let found = seg.resolve_keys_batch(&remaining, &mut results)?; + let resolved = seg.resolve_keys_to_ids(&remaining)?; + if !resolved.is_empty() { + let mut found = Vec::with_capacity(resolved.len()); + for (orig_idx, node_id) in resolved { + let (label_id, key) = keys[orig_idx]; + candidates.push((orig_idx, label_id, key, node_id)); + found.push(orig_idx); + } + found.sort_unstable(); + found.dedup(); + remaining.retain(|&(i, _, _)| found.binary_search(&i).is_err()); + } + } - for &orig_idx in &found { - if let Some(node) = results[orig_idx].as_ref() { - let tombstoned = deleted_above.contains(&node.id) - || self.segments[..seg_idx] - .iter() - .any(|segment| segment.is_node_deleted(node.id)); - if tombstoned { - results[orig_idx] = None; - } + if !candidates.is_empty() { + let mut candidate_ids: Vec = candidates + .iter() + .map(|&(_, _, _, node_id)| node_id) + .collect(); + candidate_ids.sort_unstable(); + candidate_ids.dedup(); + + let visibility = self.find_node_visibility_meta(&candidate_ids)?; + let mut candidate_labels_by_id: NodeIdMap> = NodeIdMap::default(); + for &(_, label_id, _, node_id) in &candidates { + let labels = candidate_labels_by_id.entry(node_id).or_default(); + if !labels.contains(&label_id) { + labels.push(label_id); + } + } + let mut visible_ids = Vec::new(); + let mut visible_positions = NodeIdMap::default(); + for (index, state) in visibility.into_iter().enumerate() { + let NodeVisibilityState::Live(meta) = state else { + continue; + }; + let node_id = candidate_ids[index]; + if candidate_labels_by_id.get(&node_id).is_some_and(|labels| { + labels + .iter() + .any(|&label_id| meta.label_ids.contains(label_id)) + }) { + visible_positions.insert(node_id, visible_ids.len()); + visible_ids.push(node_id); } } - if !found.is_empty() { - let mut found_mask = vec![false; n]; - for &idx in &found { - found_mask[idx] = true; + if !visible_ids.is_empty() { + let hydrated = self.find_nodes(&visible_ids)?; + for (orig_idx, label_id, key, node_id) in candidates { + let Some(&position) = visible_positions.get(&node_id) else { + continue; + }; + let Some(node) = hydrated[position].as_ref() else { + continue; + }; + if node.label_ids.contains(label_id) && node.key == key { + results[orig_idx] = Some(node.clone()); + } } - remaining.retain(|&(i, _, _)| !found_mask[i]); } } Ok(results) } - pub fn find_node_by_key( + pub fn find_node_by_label_key( &self, - type_id: u32, + label_id: u32, key: &str, ) -> Result, EngineError> { - if let Some(node) = self.active.node_by_key_at(type_id, key, self.snapshot_seq) { - return Ok(Some(node)); - } - - for (i, epoch) in self.immutable.iter().enumerate() { - if let Some(node) = epoch - .memtable - .node_by_key_at(type_id, key, self.snapshot_seq) - { - if self.is_node_tombstoned_above_immutable(node.id, i) { - return Ok(None); - } - return Ok(Some(node)); - } - } - - for (s, seg) in self.segments.iter().enumerate() { - if let Some(node) = seg.node_by_key(type_id, key)? { - if self.is_node_tombstoned_above_segment(node.id, s) { - return Ok(None); - } - return Ok(Some(node)); - } - } - - Ok(None) + Ok(self + .find_nodes_by_label_keys(&[(label_id, key)])? + .pop() + .flatten()) } pub fn find_edge_by_triple( &self, from: u64, to: u64, - type_id: u32, + label_id: u32, ) -> Result, EngineError> { if let Some(edge) = self .active - .edge_by_triple_at(from, to, type_id, self.snapshot_seq) + .edge_by_triple_at(from, to, label_id, self.snapshot_seq) { return Ok(Some(edge)); } @@ -307,7 +476,7 @@ impl<'a> SourceList<'a> { if let Some(edge) = epoch .memtable - .edge_by_triple_at(from, to, type_id, self.snapshot_seq) + .edge_by_triple_at(from, to, label_id, self.snapshot_seq) { if self.is_edge_tombstoned_above_immutable(edge.id, i) { return Ok(None); @@ -317,7 +486,7 @@ impl<'a> SourceList<'a> { } for (s, seg) in self.segments.iter().enumerate() { - if let Some(edge) = seg.edge_by_triple(from, to, type_id)? { + if let Some(edge) = seg.edge_by_triple(from, to, label_id)? { if self.is_edge_tombstoned_above_segment(edge.id, s) { return Ok(None); } @@ -328,56 +497,77 @@ impl<'a> SourceList<'a> { Ok(None) } - pub fn is_node_deleted(&self, id: u64) -> bool { - if self.active.get_node_at(id, self.snapshot_seq).is_some() { - return false; - } - if self.active.is_node_deleted_at(id, self.snapshot_seq) { - return true; - } - for epoch in self.immutable { - if epoch.memtable.get_node_at(id, self.snapshot_seq).is_some() { - return false; - } - if epoch.memtable.is_node_deleted_at(id, self.snapshot_seq) { - return true; - } + pub fn find_edges_by_triples( + &self, + triples: &[(u64, u64, u32)], + ) -> Result>, EngineError> { + let n = triples.len(); + let mut results = vec![None; n]; + if n == 0 { + return Ok(results); } - for seg in self.segments { - if seg.is_node_deleted(id) { - return true; + + let mut remaining: Vec<(usize, u64, u64, u32)> = triples + .iter() + .enumerate() + .map(|(index, &(from, to, label_id))| (index, from, to, label_id)) + .collect(); + + remaining = + self.active + .batch_edges_by_triples_at(&remaining, self.snapshot_seq, &mut results); + + for (epoch_idx, epoch) in self.immutable.iter().enumerate() { + if remaining.is_empty() { + break; } - if seg.has_node(id) { - return false; + let previous = remaining; + remaining = epoch.memtable.batch_edges_by_triples_at( + &previous, + self.snapshot_seq, + &mut results, + ); + for &(orig_idx, _, _, _) in &previous { + if let Some(edge) = results[orig_idx].as_ref() { + if self.is_edge_tombstoned_above_immutable(edge.id, epoch_idx) { + results[orig_idx] = None; + } + } } } - false - } - pub fn is_edge_deleted(&self, id: u64) -> bool { - if self.active.get_edge_at(id, self.snapshot_seq).is_some() { - return false; - } - if self.active.is_edge_deleted_at(id, self.snapshot_seq) { - return true; + if remaining.is_empty() { + return Ok(results); } - for epoch in self.immutable { - if epoch.memtable.get_edge_at(id, self.snapshot_seq).is_some() { - return false; - } - if epoch.memtable.is_edge_deleted_at(id, self.snapshot_seq) { - return true; + + remaining.sort_unstable_by(|left, right| { + (left.1, left.2, left.3).cmp(&(right.1, right.2, right.3)) + }); + + for (seg_idx, seg) in self.segments.iter().enumerate() { + if remaining.is_empty() { + break; } - } - for seg in self.segments { - if seg.is_edge_deleted(id) { - return true; + + let found = seg.resolve_triples_batch(&remaining, &mut results)?; + for &orig_idx in &found { + if let Some(edge) = results[orig_idx].as_ref() { + if self.is_edge_tombstoned_above_segment(edge.id, seg_idx) { + results[orig_idx] = None; + } + } } - if seg.has_edge(id) { - return false; + + if !found.is_empty() { + let mut found_mask = vec![false; n]; + for &idx in &found { + found_mask[idx] = true; + } + remaining.retain(|&(idx, _, _, _)| !found_mask[idx]); } } - false + + Ok(results) } pub fn collect_deleted_nodes(&self) -> NodeIdSet { @@ -406,81 +596,958 @@ impl<'a> SourceList<'a> { deleted } - fn is_node_tombstoned_above_immutable(&self, node_id: u64, imm_idx: usize) -> bool { - if self.active.is_node_deleted_at(node_id, self.snapshot_seq) { - return true; - } - self.immutable[..imm_idx].iter().any(|epoch| { - epoch - .memtable - .is_node_deleted_at(node_id, self.snapshot_seq) - }) + fn append_edge_matches(result: &mut Vec, mut matching_ids: Vec) { + matching_ids.sort_unstable(); + matching_ids.dedup(); + result.extend(matching_ids); } - fn is_node_tombstoned_above_segment(&self, node_id: u64, seg_idx: usize) -> bool { - if self.active.is_node_deleted_at(node_id, self.snapshot_seq) { - return true; + fn append_edge_matches_filtered( + result: &mut Vec, + mut matching_ids: Vec, + mut is_shadowed: impl FnMut(u64) -> bool, + ) { + matching_ids.retain(|&id| !is_shadowed(id)); + Self::append_edge_matches(result, matching_ids); + } + + fn push_edge_match_limited( + result: &mut Vec, + edge_id: u64, + limit: usize, + ) -> ControlFlow<()> { + if result.len() >= limit { + return ControlFlow::Break(()); } - for epoch in self.immutable { - if epoch - .memtable - .is_node_deleted_at(node_id, self.snapshot_seq) - { - return true; - } + result.push(edge_id); + if result.len() >= limit { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) } - self.segments[..seg_idx] - .iter() - .any(|seg| seg.is_node_deleted(node_id)) } - fn is_edge_tombstoned_above_immutable(&self, edge_id: u64, imm_idx: usize) -> bool { - if self.active.is_edge_deleted_at(edge_id, self.snapshot_seq) { - return true; - } - self.immutable[..imm_idx].iter().any(|epoch| { - epoch - .memtable - .is_edge_deleted_at(edge_id, self.snapshot_seq) - }) + fn finalize_edge_matches(mut result: Vec) -> Vec { + result.sort_unstable(); + result.dedup(); + result } - fn is_edge_tombstoned_above_segment(&self, edge_id: u64, seg_idx: usize) -> bool { - if self.active.is_edge_deleted_at(edge_id, self.snapshot_seq) { - return true; + fn append_memtable_endpoint_matches_limited( + result: &mut Vec, + memtable: &Memtable, + node_ids: &[u64], + params: MemtableEndpointLimit<'_>, + mut is_shadowed: impl FnMut(u64) -> bool, + ) { + let mut cursors = Vec::new(); + for &node_id in node_ids { + match params.direction { + Direction::Outgoing => cursors.push((node_id, true, None)), + Direction::Incoming => cursors.push((node_id, false, None)), + Direction::Both => { + cursors.push((node_id, true, None)); + cursors.push((node_id, false, None)); + } + } } - for epoch in self.immutable { - if epoch - .memtable - .is_edge_deleted_at(edge_id, self.snapshot_seq) - { - return true; + + let mut heap = BinaryHeap::new(); + for (index, cursor) in cursors.iter_mut().enumerate() { + let next = if cursor.1 { + memtable.next_visible_edge_from_endpoint_after( + cursor.0, + params.label_filter_ids, + params.snapshot_seq, + cursor.2, + ) + } else { + memtable.next_visible_edge_to_endpoint_after( + cursor.0, + params.label_filter_ids, + params.snapshot_seq, + cursor.2, + ) + }; + if let Some(edge_id) = next { + cursor.2 = Some(edge_id); + heap.push(Reverse((edge_id, index))); } } - self.segments[..seg_idx] - .iter() - .any(|seg| seg.is_edge_deleted(edge_id)) - } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::degree_cache::DegreeOverlaySnapshot; - use crate::memtable::Memtable; - use crate::types::WalOp; + let mut last_seen = None; + while let Some(Reverse((edge_id, cursor_index))) = heap.pop() { + let cursor = &mut cursors[cursor_index]; + let next = if cursor.1 { + memtable.next_visible_edge_from_endpoint_after( + cursor.0, + params.label_filter_ids, + params.snapshot_seq, + cursor.2, + ) + } else { + memtable.next_visible_edge_to_endpoint_after( + cursor.0, + params.label_filter_ids, + params.snapshot_seq, + cursor.2, + ) + }; + if let Some(next_id) = next { + cursor.2 = Some(next_id); + heap.push(Reverse((next_id, cursor_index))); + } - fn wrap_imm(mt: Memtable) -> ReadViewImmutableEpoch { - ReadViewImmutableEpoch { - epoch_id: 0, - wal_generation_id: 0, - memtable: Arc::new(mt), - degree_overlay: DegreeOverlaySnapshot::empty(), - in_flight: false, + if last_seen == Some(edge_id) { + continue; + } + last_seen = Some(edge_id); + if is_shadowed(edge_id) { + continue; + } + if Self::push_edge_match_limited(result, edge_id, params.limit).is_break() { + break; + } } } - fn sources_for<'a>( + fn append_segment_endpoint_matches_limited( + result: &mut Vec, + segment: &SegmentReader, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + limit: usize, + mut is_shadowed: impl FnMut(u64) -> bool, + ) -> Result<(), EngineError> { + if result.len() >= limit { + return Ok(()); + } + + let mut cursors = + segment.endpoint_adj_posting_cursors(node_ids, direction, label_filter_ids)?; + let mut heap = BinaryHeap::new(); + for (index, cursor) in cursors.iter_mut().enumerate() { + if let Some(edge_id) = segment.next_adj_posting_edge_id(cursor)? { + heap.push(Reverse((edge_id, index))); + } + } + + let mut last_seen = None; + while let Some(Reverse((edge_id, cursor_index))) = heap.pop() { + let cursor = &mut cursors[cursor_index]; + if let Some(next_id) = segment.next_adj_posting_edge_id(cursor)? { + heap.push(Reverse((next_id, cursor_index))); + } + + if last_seen == Some(edge_id) { + continue; + } + last_seen = Some(edge_id); + if is_shadowed(edge_id) { + continue; + } + if Self::push_edge_match_limited(result, edge_id, limit).is_break() { + break; + } + } + + Ok(()) + } + + fn optional_edge_index_or_scan( + sidecar_result: Result>, EngineError>, + scan: impl FnOnce() -> Result, EngineError>, + ) -> Result, EngineError> { + match sidecar_result { + Ok(Some(flow)) => Ok(flow), + Ok(None) | Err(EngineError::CorruptRecord(_)) => scan(), + Err(error) => Err(error), + } + } + + pub(crate) fn edge_ids_by_label_id(&self, label_id: u32) -> Result, EngineError> { + let mut result = Vec::new(); + + Self::append_edge_matches( + &mut result, + self.active + .visible_edges_by_label_id(label_id, self.snapshot_seq), + ); + for (index, epoch) in self.immutable.iter().enumerate() { + Self::append_edge_matches_filtered( + &mut result, + epoch + .memtable + .visible_edges_by_label_id(label_id, self.snapshot_seq), + |id| self.is_edge_shadowed_above_immutable(id, index), + ); + } + for (index, seg) in self.segments.iter().enumerate() { + Self::append_edge_matches_filtered( + &mut result, + seg.edges_by_label_id(label_id)?, + |id| self.is_edge_shadowed_above_segment(id, index), + ); + } + + result.sort_unstable(); + result.dedup(); + Ok(result) + } + + pub(crate) fn edge_ids_by_triple( + &self, + from: u64, + to: u64, + label_id: u32, + ) -> Result, EngineError> { + let mut result = Vec::new(); + + Self::append_edge_matches( + &mut result, + self.active + .edge_ids_by_triple_at(from, to, label_id, self.snapshot_seq), + ); + for (index, epoch) in self.immutable.iter().enumerate() { + Self::append_edge_matches_filtered( + &mut result, + epoch + .memtable + .edge_ids_by_triple_at(from, to, label_id, self.snapshot_seq), + |id| self.is_edge_shadowed_above_immutable(id, index), + ); + } + for (index, seg) in self.segments.iter().enumerate() { + Self::append_edge_matches_filtered( + &mut result, + seg.edge_ids_by_triple(from, to, label_id)?, + |id| self.is_edge_shadowed_above_segment(id, index), + ); + } + + result.sort_unstable(); + result.dedup(); + Ok(result) + } + + pub(crate) fn edge_ids_by_endpoints_limited( + &self, + node_ids: &[u64], + direction: Direction, + label_filter_ids: Option<&[u32]>, + limit: usize, + ) -> Result, EngineError> { + if node_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut sorted_node_ids = node_ids.to_vec(); + sorted_node_ids.sort_unstable(); + sorted_node_ids.dedup(); + + let mut result = Vec::new(); + + Self::append_memtable_endpoint_matches_limited( + &mut result, + self.active, + &sorted_node_ids, + MemtableEndpointLimit { + direction, + label_filter_ids, + snapshot_seq: self.snapshot_seq, + limit, + }, + |_| false, + ); + if result.len() >= limit { + return Ok(Self::finalize_edge_matches(result)); + } + + for (index, epoch) in self.immutable.iter().enumerate() { + Self::append_memtable_endpoint_matches_limited( + &mut result, + &epoch.memtable, + &sorted_node_ids, + MemtableEndpointLimit { + direction, + label_filter_ids, + snapshot_seq: self.snapshot_seq, + limit, + }, + |id| self.is_edge_shadowed_above_immutable(id, index), + ); + if result.len() >= limit { + return Ok(Self::finalize_edge_matches(result)); + } + } + + for (index, seg) in self.segments.iter().enumerate() { + Self::append_segment_endpoint_matches_limited( + &mut result, + seg, + &sorted_node_ids, + direction, + label_filter_ids, + limit, + |id| self.is_edge_shadowed_above_segment(id, index), + )?; + if result.len() >= limit { + return Ok(Self::finalize_edge_matches(result)); + } + } + + result.sort_unstable(); + result.dedup(); + Ok(result) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_weight_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Result, EngineError> { + self.edge_ids_by_weight_range_limited(label_id, bounds, usize::MAX) + } + + pub(crate) fn edge_ids_by_weight_range_limited( + &self, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Result, EngineError> { + let mut result = Vec::new(); + if self + .active + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::weight_matches_bounds(meta.weight, bounds) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + for (index, epoch) in self.immutable.iter().enumerate() { + if epoch + .memtable + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::weight_matches_bounds(meta.weight, bounds) + && !self.is_edge_shadowed_above_immutable(meta.edge_id, index) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + } + for (index, seg) in self.segments.iter().enumerate() { + let mut push = |edge_id| { + if self.is_edge_shadowed_above_segment(edge_id, index) { + ControlFlow::Continue(()) + } else { + Self::push_edge_match_limited(&mut result, edge_id, limit) + } + }; + let flow = Self::optional_edge_index_or_scan( + seg.for_each_edge_id_by_weight_range(label_id, bounds, &mut push), + || { + seg.for_each_edge_metadata(|meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::weight_matches_bounds(meta.weight, bounds) + { + push(meta.edge_id) + } else { + ControlFlow::Continue(()) + } + }) + }, + )?; + if flow.is_break() { + return Ok(Self::finalize_edge_matches(result)); + } + } + Ok(Self::finalize_edge_matches(result)) + } + + #[cfg(test)] + pub(crate) fn edge_ids_by_updated_at_range( + &self, + label_id: Option, + bounds: RangeBoundFlags, + ) -> Result, EngineError> { + self.edge_ids_by_updated_at_range_limited(label_id, bounds, usize::MAX) + } + + pub(crate) fn edge_ids_by_updated_at_range_limited( + &self, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Result, EngineError> { + let mut result = Vec::new(); + if self + .active + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.updated_at, bounds) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + for (index, epoch) in self.immutable.iter().enumerate() { + if epoch + .memtable + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.updated_at, bounds) + && !self.is_edge_shadowed_above_immutable(meta.edge_id, index) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + } + for (index, seg) in self.segments.iter().enumerate() { + let mut push = |edge_id| { + if self.is_edge_shadowed_above_segment(edge_id, index) { + ControlFlow::Continue(()) + } else { + Self::push_edge_match_limited(&mut result, edge_id, limit) + } + }; + let flow = Self::optional_edge_index_or_scan( + seg.for_each_edge_id_by_updated_at_range(label_id, bounds, &mut push), + || { + seg.for_each_edge_metadata(|meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.updated_at, bounds) + { + push(meta.edge_id) + } else { + ControlFlow::Continue(()) + } + }) + }, + )?; + if flow.is_break() { + return Ok(Self::finalize_edge_matches(result)); + } + } + Ok(Self::finalize_edge_matches(result)) + } + + pub(crate) fn edge_ids_by_valid_from_range_limited( + &self, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Result, EngineError> { + let mut result = Vec::new(); + if self + .active + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.valid_from, bounds) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + for (index, epoch) in self.immutable.iter().enumerate() { + if epoch + .memtable + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.valid_from, bounds) + && !self.is_edge_shadowed_above_immutable(meta.edge_id, index) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + } + for (index, seg) in self.segments.iter().enumerate() { + let mut push = |edge_id| { + if self.is_edge_shadowed_above_segment(edge_id, index) { + ControlFlow::Continue(()) + } else { + Self::push_edge_match_limited(&mut result, edge_id, limit) + } + }; + let flow = Self::optional_edge_index_or_scan( + seg.for_each_edge_id_by_valid_from_range(label_id, bounds, &mut push), + || { + seg.for_each_edge_metadata(|meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.valid_from, bounds) + { + push(meta.edge_id) + } else { + ControlFlow::Continue(()) + } + }) + }, + )?; + if flow.is_break() { + return Ok(Self::finalize_edge_matches(result)); + } + } + Ok(Self::finalize_edge_matches(result)) + } + + pub(crate) fn edge_ids_by_valid_to_range_limited( + &self, + label_id: Option, + bounds: RangeBoundFlags, + limit: usize, + ) -> Result, EngineError> { + let mut result = Vec::new(); + if self + .active + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.valid_to, bounds) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + for (index, epoch) in self.immutable.iter().enumerate() { + if epoch + .memtable + .for_each_edge_metadata_at(self.snapshot_seq, |meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.valid_to, bounds) + && !self.is_edge_shadowed_above_immutable(meta.edge_id, index) + { + Self::push_edge_match_limited(&mut result, meta.edge_id, limit) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return Ok(Self::finalize_edge_matches(result)); + } + } + for (index, seg) in self.segments.iter().enumerate() { + let mut push = |edge_id| { + if self.is_edge_shadowed_above_segment(edge_id, index) { + ControlFlow::Continue(()) + } else { + Self::push_edge_match_limited(&mut result, edge_id, limit) + } + }; + let flow = Self::optional_edge_index_or_scan( + seg.for_each_edge_id_by_valid_to_range(label_id, bounds, &mut push), + || { + seg.for_each_edge_metadata(|meta| { + if label_id.is_none_or(|target| meta.label_id == target) + && crate::edge_metadata::i64_matches_bounds(meta.valid_to, bounds) + { + push(meta.edge_id) + } else { + ControlFlow::Continue(()) + } + }) + }, + )?; + if flow.is_break() { + return Ok(Self::finalize_edge_matches(result)); + } + } + Ok(Self::finalize_edge_matches(result)) + } + + pub(crate) fn edge_ids_by_secondary_eq_hashes_limited_read( + &self, + index_id: u64, + value_hashes: &[u64], + limit: usize, + ) -> Result { + let mut result = Vec::new(); + let mut raw_remaining = limit; + + for &value_hash in value_hashes { + if raw_remaining == 0 { + return Ok(LimitedEdgeIndexRead::TooBroad); + } + let ids = self.active.find_secondary_eq_edges_by_hash_at_limited( + index_id, + value_hash, + self.snapshot_seq, + Some(raw_remaining), + ); + raw_remaining = raw_remaining.saturating_sub(ids.len()); + Self::append_edge_matches(&mut result, ids); + if result.len() >= limit { + return Ok(LimitedEdgeIndexRead::Ready(Self::finalize_edge_matches( + result, + ))); + } + } + + for (index, epoch) in self.immutable.iter().enumerate() { + for &value_hash in value_hashes { + if raw_remaining == 0 { + return Ok(LimitedEdgeIndexRead::TooBroad); + } + let ids = epoch.memtable.find_secondary_eq_edges_by_hash_at_limited( + index_id, + value_hash, + self.snapshot_seq, + Some(raw_remaining), + ); + raw_remaining = raw_remaining.saturating_sub(ids.len()); + Self::append_edge_matches_filtered(&mut result, ids, |id| { + self.is_edge_shadowed_above_immutable(id, index) + }); + if result.len() >= limit { + return Ok(LimitedEdgeIndexRead::Ready(Self::finalize_edge_matches( + result, + ))); + } + } + } + + for (index, seg) in self.segments.iter().enumerate() { + for &value_hash in value_hashes { + let mut posting_offset = 0usize; + loop { + if raw_remaining == 0 { + return Ok(LimitedEdgeIndexRead::TooBroad); + } + let raw_limit = raw_remaining.min(256); + let Some(chunk) = seg.edge_secondary_eq_posting_chunk_if_present( + index_id, + value_hash, + posting_offset, + raw_limit, + )? + else { + return Ok(LimitedEdgeIndexRead::MissingSidecar); + }; + raw_remaining = + raw_remaining.saturating_sub(chunk.next_offset - posting_offset); + posting_offset = chunk.next_offset; + Self::append_edge_matches_filtered(&mut result, chunk.ids, |id| { + self.is_edge_shadowed_above_segment(id, index) + }); + if result.len() >= limit { + return Ok(LimitedEdgeIndexRead::Ready(Self::finalize_edge_matches( + result, + ))); + } + if chunk.exhausted { + break; + } + } + } + } + + Ok(LimitedEdgeIndexRead::Ready(Self::finalize_edge_matches( + result, + ))) + } + + pub(crate) fn edge_ids_by_secondary_range_index_limited( + &self, + index_id: u64, + lower: Option<(u64, bool)>, + upper: Option<(u64, bool)>, + limit: usize, + ) -> Result>, EngineError> { + let mut result = Vec::new(); + for (_, edge_id) in self.active.visible_secondary_range_entries( + index_id, + lower, + upper, + None, + self.snapshot_seq, + ) { + if Self::push_edge_match_limited(&mut result, edge_id, limit).is_break() { + return Ok(Some(Self::finalize_edge_matches(result))); + } + } + + for (index, epoch) in self.immutable.iter().enumerate() { + for (_, edge_id) in epoch.memtable.visible_secondary_range_entries( + index_id, + lower, + upper, + None, + self.snapshot_seq, + ) { + if self.is_edge_shadowed_above_immutable(edge_id, index) { + continue; + } + if Self::push_edge_match_limited(&mut result, edge_id, limit).is_break() { + return Ok(Some(Self::finalize_edge_matches(result))); + } + } + } + + for (index, seg) in self.segments.iter().enumerate() { + let mut after = None; + loop { + if result.len() >= limit { + return Ok(Some(Self::finalize_edge_matches(result))); + } + let remaining = limit.saturating_sub(result.len()).min(256); + let Some(entries) = seg.find_edges_by_secondary_range_index_if_present_limited( + index_id, + lower, + upper, + after, + Some(remaining), + )? + else { + return Ok(None); + }; + if entries.is_empty() { + break; + } + after = entries.last().copied(); + for (_, edge_id) in entries { + if self.is_edge_shadowed_above_segment(edge_id, index) { + continue; + } + if Self::push_edge_match_limited(&mut result, edge_id, limit).is_break() { + return Ok(Some(Self::finalize_edge_matches(result))); + } + } + } + } + + Ok(Some(Self::finalize_edge_matches(result))) + } + + pub(crate) fn find_edge_properties( + &self, + ids: &[u64], + prop_keys: &[String], + ) -> Result>>, EngineError> { + let mut results = vec![None; ids.len()]; + if ids.is_empty() { + return Ok(results); + } + let mut remaining: Vec<(usize, u64)> = ids + .iter() + .enumerate() + .map(|(index, &id)| (index, id)) + .collect(); + + remaining.retain(|&(index, id)| { + if let Some(props) = self + .active + .edge_properties_at(id, prop_keys, self.snapshot_seq) + { + results[index] = Some(props); + false + } else { + !self.active.is_edge_deleted_at(id, self.snapshot_seq) + } + }); + + for epoch in self.immutable { + if remaining.is_empty() { + break; + } + remaining.retain(|&(index, id)| { + if let Some(props) = + epoch + .memtable + .edge_properties_at(id, prop_keys, self.snapshot_seq) + { + results[index] = Some(props); + false + } else { + !epoch.memtable.is_edge_deleted_at(id, self.snapshot_seq) + } + }); + } + + if !remaining.is_empty() { + remaining.sort_unstable_by_key(|&(_, id)| id); + for seg in self.segments { + if remaining.is_empty() { + break; + } + let mut next_remaining = Vec::new(); + for (index, id) in remaining { + if seg.is_edge_deleted(id) { + continue; + } + if let Some(props) = seg.edge_properties(id, prop_keys)? { + results[index] = Some(props); + } else { + next_remaining.push((index, id)); + } + } + remaining = next_remaining; + } + } + + Ok(results) + } + + fn is_node_tombstoned_above_immutable(&self, node_id: u64, imm_idx: usize) -> bool { + if self.active.is_node_deleted_at(node_id, self.snapshot_seq) { + return true; + } + self.immutable[..imm_idx].iter().any(|epoch| { + epoch + .memtable + .is_node_deleted_at(node_id, self.snapshot_seq) + }) + } + + fn is_edge_shadowed_above_immutable(&self, edge_id: u64, imm_idx: usize) -> bool { + if !matches!( + self.active + .edge_visibility_state_at(edge_id, self.snapshot_seq), + EdgeVisibilityState::Missing + ) { + return true; + } + self.immutable[..imm_idx].iter().any(|epoch| { + !matches!( + epoch + .memtable + .edge_visibility_state_at(edge_id, self.snapshot_seq), + EdgeVisibilityState::Missing + ) + }) + } + + fn is_edge_shadowed_above_segment(&self, edge_id: u64, seg_idx: usize) -> bool { + if !matches!( + self.active + .edge_visibility_state_at(edge_id, self.snapshot_seq), + EdgeVisibilityState::Missing + ) { + return true; + } + for epoch in self.immutable { + if !matches!( + epoch + .memtable + .edge_visibility_state_at(edge_id, self.snapshot_seq), + EdgeVisibilityState::Missing + ) { + return true; + } + } + self.segments[..seg_idx] + .iter() + .any(|seg| seg.has_edge(edge_id) || seg.is_edge_deleted(edge_id)) + } + + fn is_edge_tombstoned_above_immutable(&self, edge_id: u64, imm_idx: usize) -> bool { + if self.active.is_edge_deleted_at(edge_id, self.snapshot_seq) { + return true; + } + self.immutable[..imm_idx].iter().any(|epoch| { + epoch + .memtable + .is_edge_deleted_at(edge_id, self.snapshot_seq) + }) + } + + fn is_edge_tombstoned_above_segment(&self, edge_id: u64, seg_idx: usize) -> bool { + if self.active.is_edge_deleted_at(edge_id, self.snapshot_seq) { + return true; + } + for epoch in self.immutable { + if epoch + .memtable + .is_edge_deleted_at(edge_id, self.snapshot_seq) + { + return true; + } + } + self.segments[..seg_idx] + .iter() + .any(|seg| seg.is_edge_deleted(edge_id)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::degree_cache::DegreeOverlaySnapshot; + use crate::edge_metadata::EDGE_WEIGHT_INDEX_LOGICAL_NAME; + use crate::memtable::Memtable; + use crate::segment_components::{ + decode_manifest_envelope, encode_manifest_envelope, SegmentComponentKind, + SEGMENT_COMPONENT_MANIFEST_FILENAME, + }; + use crate::segment_writer::write_segment_without_degree_sidecar_for_test; + use crate::types::WalOp; + + fn wrap_imm(mt: Memtable) -> ReadViewImmutableEpoch { + ReadViewImmutableEpoch { + epoch_id: 0, + wal_generation_id: 0, + memtable: Arc::new(mt), + degree_overlay: DegreeOverlaySnapshot::empty(), + in_flight: false, + } + } + + fn remove_manifest_component_for_test(seg_dir: &std::path::Path, kind: SegmentComponentKind) { + let manifest_path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); + let data = std::fs::read(&manifest_path).unwrap(); + let mut manifest = decode_manifest_envelope(&data).unwrap(); + let original_len = manifest.components.len(); + manifest.components.retain(|record| record.kind != kind); + assert_ne!( + manifest.components.len(), + original_len, + "missing component {:?}", + kind + ); + std::fs::write(&manifest_path, encode_manifest_envelope(&manifest).unwrap()).unwrap(); + } + + fn sources_for<'a>( active: &'a Memtable, immutable: &'a [ReadViewImmutableEpoch], snapshot_seq: u64, @@ -493,11 +1560,11 @@ mod tests { } } - fn make_node(id: u64, key: &str, type_id: u32) -> NodeRecord { + fn make_node(id: u64, key: &str, label_id: u32) -> NodeRecord { NodeRecord { id, key: key.to_string(), - type_id, + label_ids: NodeLabelSet::single(label_id).unwrap(), props: Default::default(), created_at: 1000, updated_at: 1000, @@ -508,12 +1575,12 @@ mod tests { } } - fn make_edge(id: u64, from: u64, to: u64, type_id: u32) -> EdgeRecord { + fn make_edge(id: u64, from: u64, to: u64, label_id: u32) -> EdgeRecord { EdgeRecord { id, from, to, - type_id, + label_id, props: Default::default(), created_at: 1000, updated_at: 1000, @@ -550,7 +1617,6 @@ mod tests { let sources = sources_for(&mt, &[], 2); assert!(sources.find_node(1).unwrap().is_none()); - assert!(sources.is_node_deleted(1)); } #[test] @@ -569,7 +1635,7 @@ mod tests { } #[test] - fn test_find_node_by_key_snapshot_correct() { + fn test_find_node_by_label_key_snapshot_correct() { let active = Memtable::new(); active.apply_op(&WalOp::UpsertNode(make_node(1, "alice", 1)), 1); active.apply_op( @@ -582,18 +1648,18 @@ mod tests { active.apply_op(&WalOp::UpsertNode(make_node(2, "alice", 1)), 3); let old = sources_for(&active, &[], 1) - .find_node_by_key(1, "alice") + .find_node_by_label_key(1, "alice") .unwrap() .unwrap(); assert_eq!(old.id, 1); assert!(sources_for(&active, &[], 2) - .find_node_by_key(1, "alice") + .find_node_by_label_key(1, "alice") .unwrap() .is_none()); let new = sources_for(&active, &[], 3) - .find_node_by_key(1, "alice") + .find_node_by_label_key(1, "alice") .unwrap() .unwrap(); assert_eq!(new.id, 2); @@ -630,6 +1696,334 @@ mod tests { assert_eq!(new.id, 2); } + #[test] + fn test_edge_source_helpers_cover_active_and_frozen_memtables() { + let active = Memtable::new(); + let mut active_edge = make_edge(10, 1, 2, 5); + active_edge.weight = -0.0; + active_edge.updated_at = 100; + active.apply_op(&WalOp::UpsertEdge(active_edge), 2); + + let frozen = { + let mt = Memtable::new(); + let mut edge = make_edge(20, 2, 3, 5); + edge.weight = 0.0; + edge.updated_at = 200; + mt.apply_op(&WalOp::UpsertEdge(edge), 1); + mt + }; + let immutable = vec![wrap_imm(frozen)]; + let sources = sources_for(&active, &immutable, 2); + + assert_eq!(sources.edge_ids_by_label_id(5).unwrap(), vec![10, 20]); + assert_eq!( + sources + .edge_ids_by_endpoints_limited(&[2], Direction::Both, Some(&[5]), usize::MAX) + .unwrap(), + vec![10, 20] + ); + assert_eq!( + sources + .edge_ids_by_weight_range( + Some(5), + RangeBoundFlags::inclusive(Some(0.0), Some(0.0)), + ) + .unwrap(), + vec![10, 20] + ); + assert_eq!( + sources + .edge_ids_by_updated_at_range( + Some(5), + RangeBoundFlags::inclusive(Some(150), Some(250)), + ) + .unwrap(), + vec![20] + ); + } + + #[test] + fn test_find_edge_properties_projects_across_sources_without_full_hydration() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let segment_mt = Memtable::new(); + let mut shadowed = make_edge(10, 1, 2, 5); + shadowed.props.insert( + "a".to_string(), + PropValue::String("segment-old".to_string()), + ); + shadowed.props.insert("b".to_string(), PropValue::Int(1)); + segment_mt.apply_op(&WalOp::UpsertEdge(shadowed), 1); + let mut deleted = make_edge(30, 1, 3, 5); + deleted + .props + .insert("a".to_string(), PropValue::String("deleted".to_string())); + segment_mt.apply_op(&WalOp::UpsertEdge(deleted), 1); + let mut segment_only = make_edge(40, 1, 4, 5); + segment_only + .props + .insert("a".to_string(), PropValue::String("segment".to_string())); + segment_only + .props + .insert("b".to_string(), PropValue::Int(40)); + segment_only + .props + .insert("ignored".to_string(), PropValue::Bool(true)); + segment_mt.apply_op(&WalOp::UpsertEdge(segment_only), 1); + write_segment_without_degree_sidecar_for_test(&seg_dir, 1, &segment_mt, None).unwrap(); + let segments = vec![Arc::new( + SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(), + )]; + + let active = Memtable::new(); + let mut active_edge = make_edge(10, 1, 2, 5); + active_edge + .props + .insert("a".to_string(), PropValue::String("active".to_string())); + active_edge + .props + .insert("b".to_string(), PropValue::Int(10)); + active.apply_op(&WalOp::UpsertEdge(active_edge), 2); + active.apply_op( + &WalOp::DeleteEdge { + id: 30, + deleted_at: 3, + }, + 3, + ); + + let frozen = Memtable::new(); + let mut frozen_edge = make_edge(20, 2, 3, 5); + frozen_edge + .props + .insert("a".to_string(), PropValue::String("frozen".to_string())); + frozen_edge + .props + .insert("b".to_string(), PropValue::Int(20)); + frozen.apply_op(&WalOp::UpsertEdge(frozen_edge), 2); + let immutable = vec![wrap_imm(frozen)]; + + let sources = SourceList { + active: &active, + immutable: &immutable, + segments: &segments, + snapshot_seq: 3, + }; + let props = sources + .find_edge_properties( + &[10, 20, 30, 40, 999], + &["a".to_string(), "b".to_string(), "missing".to_string()], + ) + .unwrap(); + + assert_eq!( + props[0].as_ref().unwrap().get("a"), + Some(&PropValue::String("active".to_string())) + ); + assert_eq!( + props[0].as_ref().unwrap().get("b"), + Some(&PropValue::Int(10)) + ); + assert_eq!( + props[1].as_ref().unwrap().get("a"), + Some(&PropValue::String("frozen".to_string())) + ); + assert_eq!( + props[1].as_ref().unwrap().get("b"), + Some(&PropValue::Int(20)) + ); + assert!(props[2].is_none()); + assert_eq!( + props[3].as_ref().unwrap().get("a"), + Some(&PropValue::String("segment".to_string())) + ); + assert_eq!( + props[3].as_ref().unwrap().get("b"), + Some(&PropValue::Int(40)) + ); + assert!(!props[3].as_ref().unwrap().contains_key("ignored")); + assert!(props[4].is_none()); + } + + #[test] + fn test_edge_endpoint_limited_early_exit_returns_sorted_deduped_ids() { + let active = Memtable::new(); + active.apply_op(&WalOp::UpsertEdge(make_edge(30, 1, 2, 5)), 1); + active.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 3, 5)), 2); + active.apply_op(&WalOp::UpsertEdge(make_edge(20, 1, 1, 5)), 3); + + let sources = sources_for(&active, &[], 3); + assert_eq!( + sources + .edge_ids_by_endpoints_limited(&[1], Direction::Both, Some(&[5]), 1) + .unwrap(), + vec![10] + ); + } + + #[test] + fn test_segment_endpoint_limit_counts_only_unshadowed_unique_edges() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let segment_mt = Memtable::new(); + segment_mt.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 1, 5)), 1); + segment_mt.apply_op(&WalOp::UpsertEdge(make_edge(20, 2, 1, 5)), 1); + segment_mt.apply_op(&WalOp::UpsertEdge(make_edge(30, 3, 1, 5)), 1); + write_segment_without_degree_sidecar_for_test(&seg_dir, 1, &segment_mt, None).unwrap(); + let segments = vec![Arc::new( + SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(), + )]; + + let active = Memtable::new(); + active.apply_op(&WalOp::UpsertEdge(make_edge(10, 9, 9, 6)), 2); + let sources = SourceList { + active: &active, + immutable: &[], + segments: &segments, + snapshot_seq: 2, + }; + + assert_eq!( + sources + .edge_ids_by_endpoints_limited(&[1], Direction::Both, Some(&[5]), 2) + .unwrap(), + vec![20, 30] + ); + } + + #[test] + fn test_memtable_triple_source_returns_parallel_edges_from_adjacency() { + let active = Memtable::new(); + active.apply_op(&WalOp::UpsertEdge(make_edge(30, 1, 2, 5)), 1); + active.apply_op(&WalOp::UpsertEdge(make_edge(10, 1, 2, 5)), 2); + active.apply_op(&WalOp::UpsertEdge(make_edge(20, 1, 3, 5)), 3); + + let sources = sources_for(&active, &[], 3); + assert_eq!(sources.edge_ids_by_triple(1, 2, 5).unwrap(), vec![10, 30]); + } + + #[test] + fn test_node_visibility_meta_resolves_live_deleted_and_missing() { + let active = Memtable::new(); + active.apply_op(&WalOp::UpsertNode(make_node(1, "live", 7)), 1); + active.apply_op(&WalOp::UpsertNode(make_node(2, "deleted", 7)), 2); + active.apply_op( + &WalOp::DeleteNode { + id: 2, + deleted_at: 3, + }, + 3, + ); + + let sources = sources_for(&active, &[], 3); + let states = sources.find_node_visibility_meta(&[1, 2, 3]).unwrap(); + assert!(matches!( + states[0], + NodeVisibilityState::Live(meta) if meta.label_ids.as_slice() == [7] + )); + assert_eq!(states[1], NodeVisibilityState::Deleted); + assert_eq!(states[2], NodeVisibilityState::Missing); + } + + #[test] + fn test_edge_source_helpers_shadow_older_segment_versions() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let segment_mt = Memtable::new(); + let mut old = make_edge(10, 1, 2, 5); + old.weight = 1.0; + segment_mt.apply_op(&WalOp::UpsertEdge(old), 1); + write_segment_without_degree_sidecar_for_test(&seg_dir, 1, &segment_mt, None).unwrap(); + let segments = vec![Arc::new( + SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(), + )]; + + let active = Memtable::new(); + let mut newer = make_edge(10, 1, 2, 6); + newer.weight = 2.0; + active.apply_op(&WalOp::UpsertEdge(newer), 2); + let sources = SourceList { + active: &active, + immutable: &[], + segments: &segments, + snapshot_seq: 2, + }; + + assert_eq!(sources.edge_ids_by_label_id(5).unwrap(), Vec::::new()); + assert_eq!(sources.edge_ids_by_label_id(6).unwrap(), vec![10]); + assert_eq!( + sources + .edge_ids_by_weight_range(None, RangeBoundFlags::inclusive(Some(1.0), Some(1.0)),) + .unwrap(), + Vec::::new() + ); + assert_eq!( + sources + .edge_ids_by_weight_range(None, RangeBoundFlags::inclusive(Some(2.0), Some(2.0)),) + .unwrap(), + vec![10] + ); + } + + #[test] + fn test_edge_metadata_range_falls_back_when_optional_sidecar_missing() { + let dir = tempfile::tempdir().unwrap(); + let seg_dir = dir.path().join("seg_0001"); + let segment_mt = Memtable::new(); + let mut in_range = make_edge(10, 1, 2, 5); + in_range.weight = 1.0; + let mut out_of_range = make_edge(20, 1, 3, 5); + out_of_range.weight = 4.0; + segment_mt.apply_op(&WalOp::UpsertEdge(in_range), 1); + segment_mt.apply_op(&WalOp::UpsertEdge(out_of_range), 1); + write_segment_without_degree_sidecar_for_test(&seg_dir, 1, &segment_mt, None).unwrap(); + assert!(!seg_dir.join(EDGE_WEIGHT_INDEX_LOGICAL_NAME).exists()); + remove_manifest_component_for_test(&seg_dir, SegmentComponentKind::EdgeWeightIndex); + let segments = vec![Arc::new( + SegmentReader::open_unpinned_for_test(&seg_dir, 1, None).unwrap(), + )]; + + let active = Memtable::new(); + let sources = SourceList { + active: &active, + immutable: &[], + segments: &segments, + snapshot_seq: 1, + }; + + assert_eq!( + sources + .edge_ids_by_weight_range( + Some(5), + RangeBoundFlags::inclusive(Some(0.5), Some(2.0)), + ) + .unwrap(), + vec![10] + ); + } + + #[test] + fn test_optional_edge_index_or_scan_falls_back_on_corrupt_record() { + let mut scanned = false; + let flow = SourceList::optional_edge_index_or_scan( + Err(EngineError::CorruptRecord("bad optional index".into())), + || { + scanned = true; + Ok(ControlFlow::Break(())) + }, + ) + .unwrap(); + assert!(scanned); + assert!(flow.is_break()); + + let err = SourceList::optional_edge_index_or_scan( + Err(EngineError::InvalidOperation("hard failure".into())), + || Ok(ControlFlow::Continue(())), + ) + .unwrap_err(); + assert!(matches!(err, EngineError::InvalidOperation(_))); + } + #[test] fn test_find_nodes_batch_uses_snapshot_visibility() { let active = Memtable::new(); diff --git a/src/sparse_postings.rs b/src/sparse_postings.rs index 82515c2..43f9b4e 100644 --- a/src/sparse_postings.rs +++ b/src/sparse_postings.rs @@ -1,8 +1,12 @@ use crate::error::EngineError; use crate::types::NodeIdMap; use std::collections::BTreeMap; +#[cfg(test)] use std::fs::File; -use std::io::{BufWriter, Write}; +#[cfg(test)] +use std::io::BufWriter; +use std::io::Write; +#[cfg(test)] use std::path::Path; pub const SPARSE_POSTING_INDEX_FILENAME: &str = "sparse_posting_index.dat"; @@ -56,6 +60,35 @@ fn read_u64_at(data: &[u8], offset: usize) -> Result { Ok(u64::from_le_bytes(bytes.try_into().unwrap())) } +fn sparse_posting_index_count(index_data: &[u8]) -> Result { + if index_data.len() < 8 { + return Err(EngineError::CorruptRecord(format!( + "sparse posting index too short: {} bytes", + index_data.len() + ))); + } + + let count = usize::try_from(read_u64_at(index_data, 0)?).map_err(|_| { + EngineError::CorruptRecord("sparse posting index count exceeds addressable memory".into()) + })?; + let index_bytes = count + .checked_mul(SPARSE_POSTING_INDEX_ENTRY_SIZE) + .ok_or_else(|| EngineError::CorruptRecord("sparse posting index size overflow".into()))?; + let expected_len = 8usize + .checked_add(index_bytes) + .ok_or_else(|| EngineError::CorruptRecord("sparse posting index size overflow".into()))?; + if index_data.len() != expected_len { + return Err(EngineError::CorruptRecord(format!( + "sparse posting index size {} does not match expected {}", + index_data.len(), + expected_len + ))); + } + + Ok(count) +} + +#[cfg(test)] fn read_f32_at(data: &[u8], offset: usize) -> Result { let end = offset .checked_add(4) @@ -193,6 +226,7 @@ fn find_dimension( Ok(None) } +#[cfg(test)] pub(crate) fn write_sparse_posting_files( seg_dir: &Path, groups: &BTreeMap>, @@ -206,7 +240,25 @@ pub(crate) fn write_sparse_posting_files( let mut index_w = BufWriter::new(index_file); let mut postings_w = BufWriter::new(postings_file); - write_u64(&mut index_w, groups.len() as u64)?; + write_sparse_posting_files_to_writers(&mut index_w, &mut postings_w, groups)?; + + index_w.flush()?; + index_w.get_ref().sync_all()?; + postings_w.flush()?; + postings_w.get_ref().sync_all()?; + Ok(()) +} + +pub(crate) fn write_sparse_posting_files_to_writers( + index_w: &mut impl Write, + postings_w: &mut impl Write, + groups: &BTreeMap>, +) -> Result<(), EngineError> { + if groups.is_empty() { + return Ok(()); + } + + write_u64(index_w, groups.len() as u64)?; let mut data_offset = 0u64; for (&dimension_id, postings) in groups { @@ -217,9 +269,9 @@ pub(crate) fn write_sparse_posting_files( ))); } - write_u32(&mut index_w, dimension_id)?; - write_u64(&mut index_w, data_offset)?; - write_u32(&mut index_w, postings.len() as u32)?; + write_u32(index_w, dimension_id)?; + write_u64(index_w, data_offset)?; + write_u32(index_w, postings.len() as u32)?; for &(node_id, weight) in postings { if !weight.is_finite() { @@ -234,24 +286,25 @@ pub(crate) fn write_sparse_posting_files( dimension_id, node_id ))); } - write_u64(&mut postings_w, node_id)?; + write_u64(postings_w, node_id)?; postings_w.write_all(&weight.to_le_bytes())?; } - data_offset = data_offset - .checked_add(postings.len() as u64 * SPARSE_POSTING_ENTRY_SIZE as u64) + let posting_bytes = u64::try_from(postings.len()) + .ok() + .and_then(|len| len.checked_mul(SPARSE_POSTING_ENTRY_SIZE as u64)) .ok_or_else(|| { EngineError::CorruptRecord("sparse posting data offset overflow".into()) })?; + data_offset = data_offset.checked_add(posting_bytes).ok_or_else(|| { + EngineError::CorruptRecord("sparse posting data offset overflow".into()) + })?; } - index_w.flush()?; - index_w.get_ref().sync_all()?; - postings_w.flush()?; - postings_w.get_ref().sync_all()?; Ok(()) } +#[cfg(test)] pub(crate) fn validate_sparse_posting_files( index_data: &[u8], postings_data: &[u8], @@ -282,24 +335,7 @@ pub(crate) fn validate_sparse_posting_files( "segment has sparse posting files but no sparse vectors".into(), )); } - if index_data.len() < 8 { - return Err(EngineError::CorruptRecord(format!( - "sparse posting index too short: {} bytes", - index_data.len() - ))); - } - - let count = read_u64_at(index_data, 0)? as usize; - let expected_len = 8usize - .checked_add(count * SPARSE_POSTING_INDEX_ENTRY_SIZE) - .ok_or_else(|| EngineError::CorruptRecord("sparse posting index size overflow".into()))?; - if index_data.len() != expected_len { - return Err(EngineError::CorruptRecord(format!( - "sparse posting index size {} does not match expected {}", - index_data.len(), - expected_len - ))); - } + let count = sparse_posting_index_count(index_data)?; let mut prev_dimension = None; let mut next_offset = 0usize; @@ -384,6 +420,133 @@ pub(crate) fn validate_sparse_posting_files( Ok(()) } +pub(crate) fn validate_sparse_posting_files_for_open( + index_data: &[u8], + postings_data: &[u8], + sparse_vector_count: usize, +) -> Result<(), EngineError> { + if index_data.is_empty() { + if !postings_data.is_empty() { + return Err(EngineError::CorruptRecord( + "sparse postings data exists without sparse posting index".into(), + )); + } + return Ok(()); + } + + if postings_data.is_empty() { + return Err(EngineError::CorruptRecord( + "sparse posting index exists without sparse postings data".into(), + )); + } + if sparse_vector_count == 0 { + return Err(EngineError::CorruptRecord( + "segment has sparse posting files but no sparse vectors".into(), + )); + } + + let count = sparse_posting_index_count(index_data)?; + if count == 0 { + return Err(EngineError::CorruptRecord( + "sparse posting index has zero dimensions".into(), + )); + } + if !postings_data + .len() + .is_multiple_of(SPARSE_POSTING_ENTRY_SIZE) + { + return Err(EngineError::CorruptRecord(format!( + "sparse postings data length {} is not aligned to posting entry size {}", + postings_data.len(), + SPARSE_POSTING_ENTRY_SIZE + ))); + } + + Ok(()) +} + +pub(crate) fn validate_sparse_posting_index_shape_for_search( + index_data: &[u8], + postings_data: &[u8], +) -> Result<(), EngineError> { + if index_data.is_empty() { + if !postings_data.is_empty() { + return Err(EngineError::CorruptRecord( + "sparse postings data exists without sparse posting index".into(), + )); + } + return Ok(()); + } + if postings_data.is_empty() { + return Err(EngineError::CorruptRecord( + "sparse posting index exists without sparse postings data".into(), + )); + } + + let count = sparse_posting_index_count(index_data)?; + let mut prev_dimension = None; + let mut next_offset = 0usize; + for entry_index in 0..count { + let base = 8 + entry_index * SPARSE_POSTING_INDEX_ENTRY_SIZE; + let dimension_id = read_u32_at(index_data, base)?; + let offset = usize::try_from(read_u64_at(index_data, base + 4)?).map_err(|_| { + EngineError::CorruptRecord(format!( + "sparse posting dimension {} offset exceeds addressable memory", + dimension_id + )) + })?; + let posting_count = read_u32_at(index_data, base + 12)? as usize; + if posting_count == 0 { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimension {} has zero posting count", + dimension_id + ))); + } + if prev_dimension.is_some_and(|prev| prev >= dimension_id) { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimensions are not strictly increasing at {}", + dimension_id + ))); + } + if offset != next_offset { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimension {} offset {} does not match expected {}", + dimension_id, offset, next_offset + ))); + } + + let postings_len = posting_count + .checked_mul(SPARSE_POSTING_ENTRY_SIZE) + .ok_or_else(|| EngineError::CorruptRecord("sparse posting range overflow".into()))?; + let end = offset + .checked_add(postings_len) + .ok_or_else(|| EngineError::CorruptRecord("sparse posting end overflow".into()))?; + if end > postings_data.len() { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimension {} range [{}, {}) exceeds data length {}", + dimension_id, + offset, + end, + postings_data.len() + ))); + } + + prev_dimension = Some(dimension_id); + next_offset = end; + } + + if next_offset != postings_data.len() { + return Err(EngineError::CorruptRecord(format!( + "sparse postings data has trailing or unreferenced bytes: expected {}, got {}", + next_offset, + postings_data.len() + ))); + } + + Ok(()) +} + +#[cfg(test)] pub(crate) fn read_sparse_posting_groups( index_data: &[u8], postings_data: &[u8], @@ -439,6 +602,7 @@ pub(crate) fn accumulate_sparse_posting_scores( let posting_bytes = sparse_posting_bytes(postings_data, offset, posting_count as usize)?; let mut cursor = posting_bytes.as_ptr(); + let mut prev_node_id = None; for _ in 0..posting_count as usize { let (node_id, weight) = unsafe { let node_id = @@ -447,6 +611,8 @@ pub(crate) fn accumulate_sparse_posting_scores( u32::from_le(std::ptr::read_unaligned(cursor.add(8) as *const u32)); (node_id, f32::from_bits(weight_bits)) }; + validate_sparse_posting_entry(dimension_id, node_id, weight, prev_node_id)?; + prev_node_id = Some(node_id); *scores.entry(node_id).or_insert(0.0) += query_weight * weight; cursor = unsafe { cursor.add(SPARSE_POSTING_ENTRY_SIZE) }; } @@ -475,6 +641,7 @@ pub(crate) fn accumulate_sparse_posting_scores( let posting_bytes = sparse_posting_bytes(postings_data, offset, posting_count)?; let mut cursor = posting_bytes.as_ptr(); + let mut prev_node_id = None; for _ in 0..posting_count { let (node_id, weight) = unsafe { let node_id = @@ -484,6 +651,13 @@ pub(crate) fn accumulate_sparse_posting_scores( )); (node_id, f32::from_bits(weight_bits)) }; + validate_sparse_posting_entry( + index_dimension, + node_id, + weight, + prev_node_id, + )?; + prev_node_id = Some(node_id); *scores.entry(node_id).or_insert(0.0) += query_weight * weight; cursor = unsafe { cursor.add(SPARSE_POSTING_ENTRY_SIZE) }; } @@ -497,6 +671,33 @@ pub(crate) fn accumulate_sparse_posting_scores( Ok(()) } +fn validate_sparse_posting_entry( + dimension_id: u32, + node_id: u64, + weight: f32, + prev_node_id: Option, +) -> Result<(), EngineError> { + if !weight.is_finite() { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimension {} node {} has non-finite weight", + dimension_id, node_id + ))); + } + if weight < 0.0 { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimension {} node {} has negative weight", + dimension_id, node_id + ))); + } + if prev_node_id.is_some_and(|prev| prev >= node_id) { + return Err(EngineError::CorruptRecord(format!( + "sparse posting dimension {} node IDs are not strictly increasing at {}", + dimension_id, node_id + ))); + } + Ok(()) +} + pub(crate) fn sparse_dot_score(query: &[(u32, f32)], values: &[(u32, f32)]) -> f32 { let mut score = 0.0f32; let mut qi = 0usize; @@ -613,6 +814,23 @@ mod tests { } } + #[test] + fn test_validate_sparse_postings_rejects_index_length_overflow() { + let mut index = Vec::new(); + index.extend_from_slice(&u64::MAX.to_le_bytes()); + let data = vec![0]; + + match validate_sparse_posting_files(&index, &data, 1, true) { + Err(EngineError::CorruptRecord(message)) => { + assert!(message.contains("overflow") || message.contains("addressable")); + } + other => panic!( + "expected sparse posting index overflow error, got {:?}", + other + ), + } + } + #[test] fn test_validate_sparse_postings_rejects_non_monotonic_dimensions() { let (mut index, data) = valid_sparse_posting_files(); diff --git a/src/types.rs b/src/types.rs index 790810c..44884c6 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,8 +1,411 @@ use crate::error::EngineError; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt; use std::hash::{BuildHasherDefault, Hasher}; +pub(crate) const LABEL_TOKEN_SCHEMA_VERSION: u32 = 1; +#[allow(dead_code)] +pub(crate) const MAX_NODE_LABELS_PER_NODE: usize = 10; + +pub(crate) fn validate_label_token_name(name: &str) -> Result<(), EngineError> { + if name.is_empty() { + return Err(EngineError::InvalidOperation( + "label token name must not be empty".to_string(), + )); + } + if name.len() > 255 { + return Err(EngineError::InvalidOperation(format!( + "label token name must be at most 255 UTF-8 bytes, got {}", + name.len() + ))); + } + if name.trim_matches(char::is_whitespace).len() != name.len() { + return Err(EngineError::InvalidOperation( + "label token name must not contain leading or trailing whitespace".to_string(), + )); + } + if name + .chars() + .any(|ch| ch == '\0' || (ch.is_ascii() && ch.is_control())) + { + return Err(EngineError::InvalidOperation( + "label token name must not contain ASCII control characters or NUL".to_string(), + )); + } + Ok(()) +} + +#[allow(dead_code)] +pub(crate) fn validate_public_node_label_list<'a, I>(labels: I) -> Result<(), EngineError> +where + I: IntoIterator, +{ + ValidatedNodeLabelList::new(labels).map(|_| ()) +} + +#[allow(dead_code)] +#[derive(Clone, Copy)] +pub(crate) struct ValidatedNodeLabelList<'a> { + count: u8, + labels: [&'a str; MAX_NODE_LABELS_PER_NODE], +} + +#[allow(dead_code)] +impl<'a> ValidatedNodeLabelList<'a> { + pub(crate) fn new(labels: I) -> Result + where + I: IntoIterator, + { + let mut stored: [&'a str; MAX_NODE_LABELS_PER_NODE] = [""; MAX_NODE_LABELS_PER_NODE]; + let mut count = 0usize; + let mut total_count = 0usize; + for label in labels { + validate_label_token_name(label)?; + total_count += 1; + if count < MAX_NODE_LABELS_PER_NODE { + stored[count] = label; + count += 1; + } + } + if total_count == 0 { + return Err(EngineError::InvalidOperation( + "node label set must contain at least one label".to_string(), + )); + } + if total_count > MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::InvalidOperation(format!( + "node label set must contain at most {} labels", + MAX_NODE_LABELS_PER_NODE + ))); + } + for idx in 0..count { + if stored[..idx].iter().any(|&seen| seen == stored[idx]) { + return Err(EngineError::InvalidOperation(format!( + "node label set contains duplicate label '{}'", + stored[idx] + ))); + } + } + Ok(Self { + count: count as u8, + labels: stored, + }) + } + + #[inline] + pub(crate) fn len(&self) -> usize { + self.count as usize + } + + #[inline] + pub(crate) fn as_slice(&self) -> &[&'a str] { + &self.labels[..self.len()] + } +} + +#[allow(dead_code)] +pub(crate) fn validate_node_label_filter(filter: &NodeLabelFilter) -> Result<(), EngineError> { + validate_public_node_label_list(filter.labels.iter().map(String::as_str)) +} + +mod node_label_input_seal { + pub trait Sealed {} +} + +/// Converts accepted public Rust node-label inputs into the owned label payload +/// used by queued write requests. +pub trait IntoNodeLabels: node_label_input_seal::Sealed { + fn into_node_labels(self) -> Vec; +} + +impl node_label_input_seal::Sealed for &str {} + +impl IntoNodeLabels for &str { + fn into_node_labels(self) -> Vec { + vec![self.to_string()] + } +} + +impl node_label_input_seal::Sealed for String {} + +impl IntoNodeLabels for String { + fn into_node_labels(self) -> Vec { + vec![self] + } +} + +impl node_label_input_seal::Sealed for &String {} + +impl IntoNodeLabels for &String { + fn into_node_labels(self) -> Vec { + vec![self.clone()] + } +} + +impl node_label_input_seal::Sealed for &[&str] {} + +impl IntoNodeLabels for &[&str] { + fn into_node_labels(self) -> Vec { + self.iter().map(|label| (*label).to_string()).collect() + } +} + +impl node_label_input_seal::Sealed for &[String] {} + +impl IntoNodeLabels for &[String] { + fn into_node_labels(self) -> Vec { + self.to_vec() + } +} + +impl node_label_input_seal::Sealed for Vec {} + +impl IntoNodeLabels for Vec { + fn into_node_labels(self) -> Vec { + self + } +} + +impl node_label_input_seal::Sealed for &[&str; N] {} + +impl IntoNodeLabels for &[&str; N] { + fn into_node_labels(self) -> Vec { + self.as_slice().into_node_labels() + } +} + +impl node_label_input_seal::Sealed for &[String; N] {} + +impl IntoNodeLabels for &[String; N] { + fn into_node_labels(self) -> Vec { + self.as_slice().into_node_labels() + } +} + +#[doc(hidden)] +#[allow(dead_code)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub(crate) struct NodeLabelSet { + count: u8, + label_ids: [u32; MAX_NODE_LABELS_PER_NODE], +} + +#[allow(dead_code)] +impl NodeLabelSet { + pub(crate) fn single(label_id: u32) -> Result { + Self::from_canonical_ids(&[label_id]) + } + + pub(crate) fn from_label_ids(label_ids: I) -> Result + where + I: IntoIterator, + { + let mut ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + let mut count = 0usize; + for label_id in label_ids { + if count == MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::InvalidOperation(format!( + "node label set must contain at most {} labels", + MAX_NODE_LABELS_PER_NODE + ))); + } + ids[count] = label_id; + count += 1; + } + if count == 0 { + return Err(EngineError::InvalidOperation( + "node label set must contain at least one label".to_string(), + )); + } + ids[..count].sort_unstable(); + Self::from_sorted_prefix(ids, count) + } + + pub(crate) fn from_canonical_ids(label_ids: &[u32]) -> Result { + if label_ids.is_empty() { + return Err(EngineError::InvalidOperation( + "node label set must contain at least one label".to_string(), + )); + } + if label_ids.len() > MAX_NODE_LABELS_PER_NODE { + return Err(EngineError::InvalidOperation(format!( + "node label set must contain at most {} labels", + MAX_NODE_LABELS_PER_NODE + ))); + } + let mut ids = [0u32; MAX_NODE_LABELS_PER_NODE]; + for (idx, &label_id) in label_ids.iter().enumerate() { + ids[idx] = label_id; + if label_id == 0 { + return Err(EngineError::InvalidOperation( + "node label token ID 0 is reserved".to_string(), + )); + } + if idx > 0 && label_ids[idx - 1] >= label_id { + return Err(EngineError::InvalidOperation( + "node label IDs must be sorted ascending and unique".to_string(), + )); + } + } + Ok(Self { + count: label_ids.len() as u8, + label_ids: ids, + }) + } + + fn from_sorted_prefix( + label_ids: [u32; MAX_NODE_LABELS_PER_NODE], + count: usize, + ) -> Result { + for idx in 0..count { + if label_ids[idx] == 0 { + return Err(EngineError::InvalidOperation( + "node label token ID 0 is reserved".to_string(), + )); + } + if idx > 0 && label_ids[idx - 1] == label_ids[idx] { + return Err(EngineError::InvalidOperation(format!( + "node label set contains duplicate label ID {}", + label_ids[idx] + ))); + } + } + Ok(Self { + count: count as u8, + label_ids, + }) + } + + #[inline] + pub(crate) fn len(&self) -> usize { + self.count as usize + } + + #[inline] + pub(crate) fn as_slice(&self) -> &[u32] { + &self.label_ids[..self.len()] + } + + #[inline] + pub(crate) fn contains(&self, label_id: u32) -> bool { + self.as_slice().binary_search(&label_id).is_ok() + } + + #[inline] + pub(crate) fn contains_all(&self, required: &NodeLabelSet) -> bool { + required + .as_slice() + .iter() + .all(|&label_id| self.contains(label_id)) + } + + #[inline] + pub(crate) fn contains_any(&self, candidates: &NodeLabelSet) -> bool { + candidates + .as_slice() + .iter() + .any(|&label_id| self.contains(label_id)) + } + + #[inline] + pub(crate) fn single_label_id(&self) -> u32 { + debug_assert_eq!(self.len(), 1); + self.as_slice()[0] + } + + pub(crate) fn require_single_label_id(&self, context: &str) -> Result { + if self.len() == 1 { + Ok(self.single_label_id()) + } else { + Err(EngineError::InvalidOperation(format!( + "{context} currently supports exactly one node label, got {}", + self.len() + ))) + } + } +} + +impl fmt::Debug for NodeLabelSet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("NodeLabelSet") + .field(&self.as_slice()) + .finish() + } +} + +/// Match mode for public node-label filters. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LabelMatchMode { + Any, + All, +} + +/// Public node-label filter used by multi-label-capable APIs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeLabelFilter { + pub labels: Vec, + pub mode: LabelMatchMode, +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ResolvedNodeLabelFilter { + Unconstrained, + Empty { + mode: LabelMatchMode, + unknown_label_count: usize, + }, + LabelSet { + mode: LabelMatchMode, + label_ids: NodeLabelSet, + unknown_label_count: usize, + }, +} + +#[allow(dead_code)] +impl ResolvedNodeLabelFilter { + pub(crate) fn known( + mode: LabelMatchMode, + label_ids: NodeLabelSet, + unknown_label_count: usize, + ) -> Self { + Self::LabelSet { + mode, + label_ids, + unknown_label_count, + } + } + + pub(crate) fn empty(mode: LabelMatchMode, unknown_label_count: usize) -> Self { + Self::Empty { + mode, + unknown_label_count, + } + } + + #[inline] + pub(crate) fn mode(&self) -> Option { + match self { + Self::Unconstrained => None, + Self::Empty { mode, .. } | Self::LabelSet { mode, .. } => Some(*mode), + } + } + + #[inline] + pub(crate) fn label_ids(&self) -> Option { + match self { + Self::LabelSet { label_ids, .. } => Some(*label_ids), + Self::Unconstrained | Self::Empty { .. } => None, + } + } + + #[inline] + pub(crate) fn is_empty_constraint(&self) -> bool { + matches!(self, Self::Empty { .. }) + } +} + // --------------------------------------------------------------------------- // Identity hasher for engine-generated u64 IDs (node IDs, edge IDs). // @@ -68,6 +471,27 @@ pub type NodeIdMap = HashMap; /// A `HashSet` of node or edge IDs with identity hashing. pub type NodeIdSet = HashSet; +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct NodeVisibilityMeta { + pub(crate) label_ids: NodeLabelSet, + pub(crate) updated_at: i64, + pub(crate) weight: f32, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum NodeVisibilityState { + Live(NodeVisibilityMeta), + Deleted, + Missing, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EdgeVisibilityState { + Live, + Deleted, + Missing, +} + /// Property value types supported in node/edge properties. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum PropValue { @@ -138,7 +562,7 @@ impl Default for HnswConfig { } } -/// Configuration for the single DB-scoped dense vector space in Phase 19. +/// Configuration for the single DB-scoped dense vector space. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DenseVectorConfig { pub dimension: u32, @@ -176,7 +600,7 @@ pub struct VectorSearchScope { pub start_node_id: u64, pub max_depth: u32, pub direction: Direction, - pub edge_type_filter: Option>, + pub edge_label_filter: Option>, pub at_epoch: Option, } @@ -187,7 +611,7 @@ pub struct VectorSearchRequest { pub dense_query: Option, pub sparse_query: Option, pub k: usize, - pub type_filter: Option>, + pub label_filter: Option, pub ef_search: Option, pub scope: Option, pub dense_weight: Option, @@ -318,11 +742,12 @@ pub struct TombstoneEntry { pub last_write_seq: u64, } -/// A node record in the graph. +/// Internal numeric node record used by storage, WAL, indexes, and planners. +#[doc(hidden)] #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NodeRecord { +pub(crate) struct NodeRecord { pub id: u64, - pub type_id: u32, + pub label_ids: NodeLabelSet, pub key: String, pub props: BTreeMap, pub created_at: i64, @@ -336,13 +761,14 @@ pub struct NodeRecord { pub last_write_seq: u64, } -/// An edge record in the graph. +/// Internal numeric edge record used by storage, WAL, indexes, and planners. +#[doc(hidden)] #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EdgeRecord { +pub(crate) struct EdgeRecord { pub id: u64, pub from: u64, pub to: u64, - pub type_id: u32, + pub label_id: u32, pub props: BTreeMap, pub created_at: i64, pub updated_at: i64, @@ -355,6 +781,42 @@ pub struct EdgeRecord { pub last_write_seq: u64, } +/// Public, fully hydrated node record returned by core point-read APIs. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NodeView { + pub id: u64, + pub labels: Vec, + pub key: String, + pub props: BTreeMap, + pub created_at: i64, + pub updated_at: i64, + pub weight: f32, + pub dense_vector: Option, + pub sparse_vector: Option, +} + +/// Public, fully hydrated edge record returned by core point-read APIs. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EdgeView { + pub id: u64, + pub from: u64, + pub to: u64, + pub label: String, + pub props: BTreeMap, + pub created_at: i64, + pub updated_at: i64, + pub weight: f32, + pub valid_from: i64, + pub valid_to: i64, +} + +/// Public key lookup request for `get_nodes_by_keys`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeKeyQuery { + pub label: String, + pub key: String, +} + /// Request parameters for cursor-based pagination. /// /// Both fields are optional: @@ -384,7 +846,8 @@ pub struct PageResult { /// Request for planner-backed node queries. #[derive(Debug, Clone, PartialEq)] pub struct NodeQuery { - pub type_id: Option, + /// Optional node-label membership filter. + pub label_filter: Option, pub ids: Vec, pub keys: Vec, pub filter: Option, @@ -396,7 +859,7 @@ pub struct NodeQuery { impl Default for NodeQuery { fn default() -> Self { Self { - type_id: None, + label_filter: None, ids: Vec::new(), keys: Vec::new(), filter: None, @@ -454,7 +917,103 @@ pub struct QueryNodeIdsResult { /// Hydrated result for planner-backed node queries. #[derive(Debug, Clone)] pub struct QueryNodesResult { - pub items: Vec, + pub items: Vec, + pub next_cursor: Option, +} + +/// Request for planner-backed edge queries. +#[derive(Debug, Clone, PartialEq)] +pub struct EdgeQuery { + pub label: Option, + pub ids: Vec, + pub from_ids: Vec, + pub to_ids: Vec, + pub endpoint_ids: Vec, + pub filter: Option, + pub page: PageRequest, + pub order: EdgeQueryOrder, + pub allow_full_scan: bool, +} + +impl Default for EdgeQuery { + fn default() -> Self { + Self { + label: None, + ids: Vec::new(), + from_ids: Vec::new(), + to_ids: Vec::new(), + endpoint_ids: Vec::new(), + filter: None, + page: PageRequest::default(), + order: EdgeQueryOrder::EdgeIdAsc, + allow_full_scan: false, + } + } +} + +/// Recursive boolean filter supported by planner-backed edge queries. +#[derive(Debug, Clone, PartialEq)] +pub enum EdgeFilterExpr { + PropertyEquals { + key: String, + value: PropValue, + }, + PropertyIn { + key: String, + values: Vec, + }, + PropertyRange { + key: String, + lower: Option, + upper: Option, + }, + PropertyExists { + key: String, + }, + PropertyMissing { + key: String, + }, + WeightRange { + lower: Option, + upper: Option, + }, + UpdatedAtRange { + lower_ms: Option, + upper_ms: Option, + }, + ValidAt { + epoch_ms: i64, + }, + ValidFromRange { + lower_ms: Option, + upper_ms: Option, + }, + ValidToRange { + lower_ms: Option, + upper_ms: Option, + }, + And(Vec), + Or(Vec), + Not(Box), +} + +/// Result ordering for planner-backed edge queries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeQueryOrder { + EdgeIdAsc, +} + +/// ID-only result for planner-backed edge queries. +#[derive(Debug, Clone, PartialEq)] +pub struct QueryEdgeIdsResult { + pub edge_ids: Vec, + pub next_cursor: Option, +} + +/// Hydrated result for planner-backed edge queries. +#[derive(Debug, Clone)] +pub struct QueryEdgesResult { + pub edges: Vec, pub next_cursor: Option, } @@ -472,7 +1031,8 @@ pub struct GraphPatternQuery { #[derive(Debug, Clone, PartialEq)] pub struct NodePattern { pub alias: String, - pub type_id: Option, + /// Optional node-label membership filter. + pub label_filter: Option, pub ids: Vec, pub keys: Vec, pub filter: Option, @@ -485,22 +1045,8 @@ pub struct EdgePattern { pub from_alias: String, pub to_alias: String, pub direction: Direction, - pub type_filter: Option>, - pub property_predicates: Vec, -} - -/// Predicate supported as a bounded post-filter on expanded edges. -#[derive(Debug, Clone, PartialEq)] -pub enum EdgePostFilterPredicate { - PropertyEquals { - key: String, - value: PropValue, - }, - PropertyRange { - key: String, - lower: Option, - upper: Option, - }, + pub label_filter: Vec, + pub filter: Option, } /// Result ordering for planner-backed graph pattern queries. @@ -531,6 +1077,7 @@ pub struct QueryMatch { #[derive(Debug, Clone, PartialEq, Eq)] pub enum QueryPlanKind { NodeQuery, + EdgeQuery, PatternQuery, } @@ -541,6 +1088,34 @@ pub struct QueryPlan { pub root: QueryPlanNode, pub estimated_candidates: Option, pub warnings: Vec, + pub notes: Vec, + pub public_inputs: QueryPlanPublicInputs, +} + +/// Public names referenced by planner explain input normalization. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct QueryPlanPublicInputs { + pub node_labels: Vec, + pub edge_labels: Vec, +} + +/// One public node-label or edge-label name surfaced in explain output. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QueryPlanPublicName { + pub alias: Option, + pub name: String, + pub known: bool, + pub mode: Option, +} + +/// Non-warning explain notes for planner behavior that is expected and +/// correctness-relevant. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QueryPlanNote { + NodeLabelAnyDedupeBeforePagination, + NodeLabelAnyFinalVerification, + NodeLabelAllSupersetVerification, + StaleNodeLabelMembershipVerification, } /// Explain tree node for planner-backed queries. @@ -548,11 +1123,22 @@ pub struct QueryPlan { pub enum QueryPlanNode { ExplicitIds, KeyLookup, - NodeTypeIndex, + NodeLabelIndex, + NodeLabelAnyIndex, PropertyEqualityIndex, PropertyRangeIndex, TimestampIndex, AdjacencyExpansion, + ExplicitEdgeIds, + EdgeLabelIndex, + EdgeTripleIndex, + EdgeEndpointAdjacency, + EdgeWeightIndex, + EdgeUpdatedAtIndex, + EdgeValidityIndex, + EdgeMetadataScan, + EdgePropertyEqualityIndex, + EdgePropertyRangeIndex, Intersect { inputs: Vec, }, @@ -562,6 +1148,9 @@ pub enum QueryPlanNode { VerifyNodeFilter { input: Box, }, + VerifyEdgeFilter { + input: Box, + }, VerifyEdgePredicates { input: Box, }, @@ -569,8 +1158,14 @@ pub enum QueryPlanNode { anchor_alias: String, input: Box, }, - FallbackTypeScan, + PatternEdgeAnchor { + edge_alias: Option, + input: Box, + }, + FallbackNodeLabelScan, FallbackFullNodeScan, + FallbackEdgeLabelScan, + FallbackFullEdgeScan, EmptyResult, } @@ -590,6 +1185,8 @@ pub enum QueryPlanWarning { VerifyOnlyFilter, BooleanBranchFallback, PlanningProbeBudgetExceeded, + UnknownNodeLabel, + UnknownEdgeLabel, } /// Range domain for an optional secondary index declaration. @@ -607,10 +1204,15 @@ pub enum SecondaryIndexKind { Range { domain: SecondaryIndexRangeDomain }, } -/// Target for an optional secondary index declaration. +/// Diagnostic/internal target for an optional secondary index declaration. +/// +/// This is exposed only because raw manifest inspection is a diagnostic surface. +/// Ordinary property-index APIs use `NodePropertyIndexInfo` and +/// `EdgePropertyIndexInfo`, which expose labels and edge labels instead. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum SecondaryIndexTarget { - NodeProperty { type_id: u32, prop_key: String }, + NodeProperty { label_id: u32, prop_key: String }, + EdgeProperty { label_id: u32, prop_key: String }, } /// Lifecycle state for an optional secondary index declaration. @@ -622,6 +1224,9 @@ pub enum SecondaryIndexState { } /// Persisted manifest entry for an optional secondary index declaration. +/// +/// This raw manifest shape is diagnostic introspection. Ordinary public APIs +/// return nameful index info DTOs instead. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SecondaryIndexManifestEntry { pub index_id: u64, @@ -636,7 +1241,32 @@ pub struct SecondaryIndexManifestEntry { #[derive(Debug, Clone, PartialEq, Eq)] pub struct NodePropertyIndexInfo { pub index_id: u64, - pub type_id: u32, + pub label: String, + pub prop_key: String, + pub kind: SecondaryIndexKind, + pub state: SecondaryIndexState, + pub last_error: Option, +} + +/// User-facing diagnostic information about a node-label token. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeLabelInfo { + pub label: String, + pub label_id: u32, +} + +/// User-facing diagnostic information about an edge-label token. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EdgeLabelInfo { + pub label: String, + pub label_id: u32, +} + +/// User-facing information about an edge-property optional secondary index. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EdgePropertyIndexInfo { + pub index_id: u64, + pub label: String, pub prop_key: String, pub kind: SecondaryIndexKind, pub state: SecondaryIndexState, @@ -690,6 +1320,10 @@ pub(crate) enum WalOp { UpsertEdge(EdgeRecord), DeleteNode { id: u64, deleted_at: i64 }, DeleteEdge { id: u64, deleted_at: i64 }, + EnsureNodeLabel { label: String, label_id: u32 }, + EnsureEdgeLabel { label: String, label_id: u32 }, + BeginAtomicBatch { first_seq: u64, op_count: u32 }, + CommitAtomicBatch { first_seq: u64, op_count: u32 }, } /// Operation type tags for binary encoding. @@ -700,6 +1334,10 @@ pub(crate) enum OpTag { UpsertEdge = 2, DeleteNode = 3, DeleteEdge = 4, + EnsureNodeLabel = 5, + EnsureEdgeLabel = 6, + BeginAtomicBatch = 7, + CommitAtomicBatch = 8, } impl OpTag { @@ -709,6 +1347,10 @@ impl OpTag { 2 => Some(OpTag::UpsertEdge), 3 => Some(OpTag::DeleteNode), 4 => Some(OpTag::DeleteEdge), + 5 => Some(OpTag::EnsureNodeLabel), + 6 => Some(OpTag::EnsureEdgeLabel), + 7 => Some(OpTag::BeginAtomicBatch), + 8 => Some(OpTag::CommitAtomicBatch), _ => None, } } @@ -720,16 +1362,40 @@ pub struct SegmentInfo { pub id: u64, pub node_count: u64, pub edge_count: u64, + #[serde(default)] + pub segment_format_version: u32, + #[serde(default)] + pub segment_data_id: [u8; 32], } /// Manifest state: the atomic checkpoint of the database. +/// +/// This raw structure is exposed for explicit diagnostic introspection through +/// `DatabaseEngine::manifest()` and `manifest::load_manifest_readonly()`. +/// Ordinary graph APIs use named labels and edge labels and do not accept these +/// internal numeric token IDs as inputs. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ManifestState { pub version: u32, + /// Named node-label / edge-label token schema marker. + #[serde(default)] + pub label_token_schema_version: u32, + /// DB-scoped node-label catalog: public label -> internal label_id. + #[serde(default)] + pub node_label_tokens: BTreeMap, + /// DB-scoped edge-label catalog: public edge label -> internal label_id. + #[serde(default)] + pub edge_label_tokens: BTreeMap, + /// Next node-label token ID to allocate. + #[serde(default)] + pub next_node_label_id: u32, + /// Next edge-label token ID to allocate. + #[serde(default)] + pub next_edge_label_id: u32, pub segments: Vec, pub next_node_id: u64, pub next_edge_id: u64, - /// DB-scoped dense vector configuration for Phase 19. + /// DB-scoped dense vector configuration. #[serde(default)] pub dense_vector: Option, /// Named prune policies applied automatically during compaction. @@ -829,7 +1495,7 @@ pub struct CompactionStats { /// Input for batch node upsert (user-facing, no ID or timestamps). #[derive(Debug, Clone)] pub struct NodeInput { - pub type_id: u32, + pub labels: Vec, pub key: String, pub props: BTreeMap, pub weight: f32, @@ -910,7 +1576,7 @@ pub enum TxnLocalRef { #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum TxnNodeRef { Id(u64), - Key { type_id: u32, key: String }, + Key { label: String, key: String }, Local(TxnLocalRef), } @@ -921,7 +1587,7 @@ pub enum TxnEdgeRef { Triple { from: TxnNodeRef, to: TxnNodeRef, - type_id: u32, + label: String, }, Local(TxnLocalRef), } @@ -931,7 +1597,7 @@ pub enum TxnEdgeRef { pub enum TxnIntent { UpsertNode { alias: Option, - type_id: u32, + labels: Vec, key: String, options: UpsertNodeOptions, }, @@ -939,7 +1605,7 @@ pub enum TxnIntent { alias: Option, from: TxnNodeRef, to: TxnNodeRef, - type_id: u32, + label: String, options: UpsertEdgeOptions, }, DeleteNode { @@ -959,7 +1625,7 @@ pub enum TxnIntent { pub struct TxnNodeView { pub id: Option, pub local: Option, - pub type_id: u32, + pub labels: Vec, pub key: String, pub props: BTreeMap, pub created_at: Option, @@ -976,7 +1642,7 @@ pub struct TxnEdgeView { pub local: Option, pub from: TxnNodeRef, pub to: TxnNodeRef, - pub type_id: u32, + pub label: String, pub props: BTreeMap, pub created_at: Option, pub updated_at: Option, @@ -1025,8 +1691,8 @@ impl TxnCommitResult { pub struct NeighborOptions { /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only include edges of these types. Default: None (all types). - pub type_filter: Option>, + /// Only include edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, /// Maximum number of results. Default: None (unlimited). pub limit: Option, /// Point-in-time epoch for temporal filtering. Default: None (current time). @@ -1039,7 +1705,7 @@ impl Default for NeighborOptions { fn default() -> Self { Self { direction: Direction::Outgoing, - type_filter: None, + edge_label_filter: None, limit: None, at_epoch: None, decay_lambda: None, @@ -1057,8 +1723,8 @@ impl Default for NeighborOptions { pub struct DegreeOptions { /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only include edges of these types. Default: None (all types). - pub type_filter: Option>, + /// Only include edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, /// Point-in-time epoch for temporal filtering. Default: None (current time). pub at_epoch: Option, } @@ -1067,7 +1733,7 @@ impl Default for DegreeOptions { fn default() -> Self { Self { direction: Direction::Outgoing, - type_filter: None, + edge_label_filter: None, at_epoch: None, } } @@ -1083,8 +1749,8 @@ impl Default for DegreeOptions { pub struct TopKOptions { /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only include edges of these types. Default: None (all types). - pub type_filter: Option>, + /// Only include edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, /// Scoring mode for ranking. Default: Weight. pub scoring: ScoringMode, /// Point-in-time epoch for temporal filtering. Default: None (current time). @@ -1095,7 +1761,7 @@ impl Default for TopKOptions { fn default() -> Self { Self { direction: Direction::Outgoing, - type_filter: None, + edge_label_filter: None, scoring: ScoringMode::Weight, at_epoch: None, } @@ -1114,10 +1780,10 @@ pub struct TraverseOptions { pub min_depth: u32, /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only traverse edges of these types. Default: None (all types). - pub edge_type_filter: Option>, - /// Only emit nodes of these types. Default: None (all types). - pub node_type_filter: Option>, + /// Only traverse edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, + /// Only emit nodes matching this label filter. Default: None (all labels). + pub emit_node_label_filter: Option, /// Point-in-time epoch for temporal filtering. Default: None (current time). pub at_epoch: Option, /// Exponential decay lambda for depth-based scoring. Default: None. @@ -1133,8 +1799,8 @@ impl Default for TraverseOptions { Self { min_depth: 1, direction: Direction::Outgoing, - edge_type_filter: None, - node_type_filter: None, + edge_label_filter: None, + emit_node_label_filter: None, at_epoch: None, decay_lambda: None, limit: None, @@ -1153,8 +1819,10 @@ impl Default for TraverseOptions { pub struct SubgraphOptions { /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only traverse edges of these types. Default: None (all types). - pub edge_type_filter: Option>, + /// Only traverse edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, + /// Only include and expand through nodes matching this label filter. Default: None (all labels). + pub node_label_filter: Option, /// Point-in-time epoch for temporal filtering. Default: None (current time). pub at_epoch: Option, } @@ -1163,7 +1831,8 @@ impl Default for SubgraphOptions { fn default() -> Self { Self { direction: Direction::Outgoing, - edge_type_filter: None, + edge_label_filter: None, + node_label_filter: None, at_epoch: None, } } @@ -1179,8 +1848,8 @@ impl Default for SubgraphOptions { pub struct ShortestPathOptions { /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only traverse edges of these types. Default: None (all types). - pub type_filter: Option>, + /// Only traverse edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, /// Property key to use as edge weight (Dijkstra). Default: None (BFS hop count). pub weight_field: Option, /// Point-in-time epoch for temporal filtering. Default: None (current time). @@ -1195,7 +1864,7 @@ impl Default for ShortestPathOptions { fn default() -> Self { Self { direction: Direction::Outgoing, - type_filter: None, + edge_label_filter: None, weight_field: None, at_epoch: None, max_depth: None, @@ -1214,8 +1883,8 @@ impl Default for ShortestPathOptions { pub struct AllShortestPathsOptions { /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only traverse edges of these types. Default: None (all types). - pub type_filter: Option>, + /// Only traverse edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, /// Property key to use as edge weight (Dijkstra). Default: None (BFS hop count). pub weight_field: Option, /// Point-in-time epoch for temporal filtering. Default: None (current time). @@ -1232,7 +1901,7 @@ impl Default for AllShortestPathsOptions { fn default() -> Self { Self { direction: Direction::Outgoing, - type_filter: None, + edge_label_filter: None, weight_field: None, at_epoch: None, max_depth: None, @@ -1252,8 +1921,8 @@ impl Default for AllShortestPathsOptions { pub struct IsConnectedOptions { /// Edge direction. Default: Outgoing. pub direction: Direction, - /// Only traverse edges of these types. Default: None (all types). - pub type_filter: Option>, + /// Only traverse edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, /// Point-in-time epoch for temporal filtering. Default: None (current time). pub at_epoch: Option, /// Maximum search depth in hops. Default: None (unlimited). @@ -1264,7 +1933,7 @@ impl Default for IsConnectedOptions { fn default() -> Self { Self { direction: Direction::Outgoing, - type_filter: None, + edge_label_filter: None, at_epoch: None, max_depth: None, } @@ -1279,10 +1948,10 @@ impl Default for IsConnectedOptions { /// ``` #[derive(Debug, Clone, PartialEq, Default)] pub struct ComponentOptions { - /// Only traverse edges of these types. Default: None (all types). - pub edge_type_filter: Option>, - /// Only include nodes of these types. Default: None (all types). - pub node_type_filter: Option>, + /// Only traverse edges with these labels. Default: None (all labels). + pub edge_label_filter: Option>, + /// Only include nodes matching this label filter. Default: None (all labels). + pub node_label_filter: Option, /// Point-in-time epoch for temporal filtering. Default: None (current time). pub at_epoch: Option, } @@ -1292,7 +1961,7 @@ pub struct ComponentOptions { pub struct EdgeInput { pub from: u64, pub to: u64, - pub type_id: u32, + pub label: String, pub props: BTreeMap, pub weight: f32, /// Optional start of validity window. If None, defaults to created_at. @@ -1321,18 +1990,25 @@ pub struct PatchResult { pub edge_ids: Vec, } +/// User-facing information about a named prune policy. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PrunePolicyInfo { + pub name: String, + pub policy: PrunePolicy, +} + /// Policy for pruning (deleting) nodes that match all specified criteria. /// All fields are optional; when multiple are set, they combine with AND logic. /// At least one of `max_age_ms` or `max_weight` must be set. An empty policy -/// (or one with only `type_id`) is rejected to prevent accidental mass deletion. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// (or one with only `label`) is rejected to prevent accidental mass deletion. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PrunePolicy { /// Prune nodes whose `updated_at` is older than `now - max_age_ms`. pub max_age_ms: Option, /// Prune nodes whose `weight <= max_weight`. pub max_weight: Option, - /// Scope pruning to a single node type. If None, all types are eligible. - pub type_id: Option, + /// Scope pruning to a single node label. If None, all labels are eligible. + pub label: Option, } /// Result of a prune operation. @@ -1375,6 +2051,50 @@ pub struct DbStats { pub oldest_retained_wal_generation_id: u64, } +/// Result of an offline integrity scrub of all segments in the database. +#[derive(Debug, Clone)] +pub struct ScrubReport { + pub segments: Vec, + pub total_components_checked: u64, + pub total_components_ok: u64, + pub total_components_failed: u64, + pub total_bytes_digested: u64, + pub duration_ms: u64, +} + +/// Scrub result for a single segment. +#[derive(Debug, Clone)] +pub struct SegmentScrubResult { + pub segment_id: u64, + pub findings: Vec, + pub components_ok: u64, + pub bytes_digested: u64, +} + +/// A single problem detected during offline scrub. +#[derive(Debug, Clone)] +pub struct ComponentScrubFinding { + pub component_kind: String, + pub finding_type: ScrubFindingType, + pub detail: String, +} + +/// Classification of a scrub finding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScrubFindingType { + PayloadDigestMismatch, + ComponentIdMismatch, + DependencyDigestMismatch, + IdentityHeaderMismatch, + ContainerIdMismatch, + SegmentIdentityMismatch, + SemanticMismatch, + RangeOverflow, + RangeOverlap, + FileMissing, + IoError, +} + /// Direction for neighbor queries. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum Direction { @@ -1393,8 +2113,8 @@ pub struct NeighborEntry { pub node_id: u64, /// The edge connecting to this neighbor. pub edge_id: u64, - /// The edge's type_id. - pub edge_type_id: u32, + /// The edge label. + pub label: String, /// The edge weight. pub weight: f32, /// Start of validity window (epoch ms). 0 means always-valid. @@ -1403,6 +2123,17 @@ pub struct NeighborEntry { pub valid_to: i64, } +/// Internal numeric adjacency entry used by graph traversal/storage paths. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct NeighborRecord { + pub node_id: u64, + pub edge_id: u64, + pub edge_label_id: u32, + pub weight: f32, + pub valid_from: i64, + pub valid_to: i64, +} + /// A single BFS traversal hit emitted by `traverse()`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TraversalHit { @@ -1461,12 +2192,12 @@ pub struct ShortestPath { } /// An extracted subgraph: all nodes and edges reachable within N hops of a starting node. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Subgraph { /// All nodes in the subgraph (including the starting node). - pub nodes: Vec, + pub nodes: Vec, /// All edges connecting nodes in the subgraph discovered during traversal. - pub edges: Vec, + pub edges: Vec, } /// Options for Personalized PageRank computation. @@ -1484,8 +2215,8 @@ pub struct PprOptions { /// Residual stopping tolerance for approximate forward-push PPR. /// Default: 1e-5. Used only when `algorithm` is `ApproxForwardPush`. pub approx_residual_tolerance: f64, - /// Optional edge type filter. Only walk edges of these types. - pub edge_type_filter: Option>, + /// Optional edge label filter. Only walk edges with these labels. + pub edge_label_filter: Option>, /// Optional top-k cutoff on returned results. pub max_results: Option, } @@ -1505,7 +2236,7 @@ impl Default for PprOptions { max_iterations: 20, epsilon: 1e-6, approx_residual_tolerance: 1e-5, - edge_type_filter: None, + edge_label_filter: None, max_results: None, } } @@ -1540,10 +2271,10 @@ pub struct PprResult { /// Options for graph adjacency export. #[derive(Debug, Clone)] pub struct ExportOptions { - /// Only include nodes of these types. None means all types. - pub node_type_filter: Option>, - /// Only include edges of these types. None means all types. - pub edge_type_filter: Option>, + /// Only include nodes matching this label filter. None means all labels. + pub node_label_filter: Option, + /// Only include edges with these labels. None means all labels. + pub edge_label_filter: Option>, /// Include edge weights in the output. Default: true. pub include_weights: bool, } @@ -1551,22 +2282,36 @@ pub struct ExportOptions { impl Default for ExportOptions { fn default() -> Self { Self { - node_type_filter: None, - edge_type_filter: None, + node_label_filter: None, + edge_label_filter: None, include_weights: true, } } } -/// An exported edge: (from_node_id, to_node_id, edge_type_id, weight). -pub type ExportEdge = (u64, u64, u32, f32); +/// An exported edge. `edge_label_index` is local to the export's `edge_labels` table. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ExportEdge { + pub from: u64, + pub to: u64, + pub edge_label_index: u32, + pub weight: Option, +} /// Result of a graph adjacency export. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct AdjacencyExport { /// All live node IDs in the exported subgraph. pub node_ids: Vec, - /// All live edges: (from, to, type_id, weight). + /// Export-local node-label side table. + #[serde(default)] + pub node_labels: Vec, + /// Per-node label side-table indexes, aligned with `node_ids`. + #[serde(default)] + pub node_label_indexes: Vec>, + /// Export-local edge-label side table. + pub edge_labels: Vec, + /// All live edges, referencing `edge_labels` by export-local index. pub edges: Vec, } @@ -1714,10 +2459,79 @@ mod tests { assert_eq!(tag as u8, tag_val); } assert!(OpTag::from_u8(0).is_none()); - assert!(OpTag::from_u8(5).is_none()); + assert_eq!(OpTag::from_u8(5), Some(OpTag::EnsureNodeLabel)); + assert_eq!(OpTag::from_u8(6), Some(OpTag::EnsureEdgeLabel)); + assert_eq!(OpTag::from_u8(7), Some(OpTag::BeginAtomicBatch)); + assert_eq!(OpTag::from_u8(8), Some(OpTag::CommitAtomicBatch)); assert!(OpTag::from_u8(255).is_none()); } + #[test] + fn test_node_label_set_canonicalizes_distinct_ids() { + let set = NodeLabelSet::from_label_ids([7, 3, 5]).unwrap(); + assert_eq!(set.len(), 3); + assert_eq!(set.as_slice(), &[3, 5, 7]); + assert!(set.contains(5)); + assert!(!set.contains(4)); + assert!(set.contains_all(&NodeLabelSet::from_label_ids([3, 7]).unwrap())); + assert!(set.contains_any(&NodeLabelSet::from_label_ids([2, 7]).unwrap())); + assert!(!set.contains_any(&NodeLabelSet::from_label_ids([1, 2]).unwrap())); + assert_eq!(NodeLabelSet::single(9).unwrap().as_slice(), &[9]); + } + + #[test] + fn test_node_label_set_rejects_empty_duplicate_zero_and_too_many_ids() { + assert!(NodeLabelSet::from_label_ids([]).is_err()); + assert!(NodeLabelSet::from_label_ids([3, 3]).is_err()); + assert!(NodeLabelSet::from_label_ids([0]).is_err()); + assert!(NodeLabelSet::from_label_ids(1..=11).is_err()); + } + + #[test] + fn test_node_label_set_canonical_decoder_rejects_unsorted_and_duplicates() { + assert_eq!( + NodeLabelSet::from_canonical_ids(&[1, 3, 5]) + .unwrap() + .as_slice(), + &[1, 3, 5] + ); + assert!(NodeLabelSet::from_canonical_ids(&[]).is_err()); + assert!(NodeLabelSet::from_canonical_ids(&[2, 1]).is_err()); + assert!(NodeLabelSet::from_canonical_ids(&[2, 2]).is_err()); + assert!(NodeLabelSet::from_canonical_ids(&(1..=11).collect::>()).is_err()); + } + + #[test] + fn test_public_node_label_list_validation() { + validate_public_node_label_list(["Person", "Employee"]).unwrap(); + assert!(validate_public_node_label_list(std::iter::empty::<&str>()).is_err()); + assert!(validate_public_node_label_list(["Person", "Person"]).is_err()); + assert!(validate_public_node_label_list([" Person"]).is_err()); + assert!(validate_public_node_label_list([ + "L1", "L2", "L3", "L4", "L5", "L6", "L7", "L8", "L9", "L10", "L11" + ]) + .is_err()); + } + + #[test] + fn test_node_label_filter_validation_rejects_empty_and_duplicate_labels() { + validate_node_label_filter(&NodeLabelFilter { + labels: vec!["Person".to_string(), "Employee".to_string()], + mode: LabelMatchMode::All, + }) + .unwrap(); + assert!(validate_node_label_filter(&NodeLabelFilter { + labels: Vec::new(), + mode: LabelMatchMode::Any, + }) + .is_err()); + assert!(validate_node_label_filter(&NodeLabelFilter { + labels: vec!["Person".to_string(), "Person".to_string()], + mode: LabelMatchMode::Any, + }) + .is_err()); + } + #[test] fn test_direction_serde_roundtrip() { for dir in [Direction::Outgoing, Direction::Incoming, Direction::Both] { @@ -1732,7 +2546,7 @@ mod tests { let entry = NeighborEntry { node_id: 42, edge_id: 99, - edge_type_id: 7, + label: "FRIENDS_WITH".to_string(), weight: 0.75, valid_from: 1000, valid_to: i64::MAX, @@ -1746,16 +2560,25 @@ mod tests { fn test_manifest_state_serde() { let state = ManifestState { version: 1, + label_token_schema_version: LABEL_TOKEN_SCHEMA_VERSION, + node_label_tokens: BTreeMap::new(), + edge_label_tokens: BTreeMap::new(), + next_node_label_id: 1, + next_edge_label_id: 1, segments: vec![ SegmentInfo { id: 1, node_count: 100, edge_count: 200, + segment_format_version: 10, + segment_data_id: [1; 32], }, SegmentInfo { id: 2, node_count: 50, edge_count: 75, + segment_format_version: 10, + segment_data_id: [2; 32], }, ], next_node_id: 151, @@ -1873,7 +2696,7 @@ mod tests { fn test_neighbor_options_default() { let opts = NeighborOptions::default(); assert_eq!(opts.direction, Direction::Outgoing); - assert!(opts.type_filter.is_none()); + assert!(opts.edge_label_filter.is_none()); assert!(opts.limit.is_none()); assert!(opts.at_epoch.is_none()); assert!(opts.decay_lambda.is_none()); @@ -1883,7 +2706,7 @@ mod tests { fn test_degree_options_default() { let opts = DegreeOptions::default(); assert_eq!(opts.direction, Direction::Outgoing); - assert!(opts.type_filter.is_none()); + assert!(opts.edge_label_filter.is_none()); assert!(opts.at_epoch.is_none()); } @@ -1892,19 +2715,28 @@ mod tests { let opts = TraverseOptions::default(); assert_eq!(opts.min_depth, 1); assert_eq!(opts.direction, Direction::Outgoing); - assert!(opts.edge_type_filter.is_none()); - assert!(opts.node_type_filter.is_none()); + assert!(opts.edge_label_filter.is_none()); + assert!(opts.emit_node_label_filter.is_none()); assert!(opts.at_epoch.is_none()); assert!(opts.decay_lambda.is_none()); assert!(opts.limit.is_none()); assert!(opts.cursor.is_none()); } + #[test] + fn test_subgraph_options_default() { + let opts = SubgraphOptions::default(); + assert_eq!(opts.direction, Direction::Outgoing); + assert!(opts.edge_label_filter.is_none()); + assert!(opts.node_label_filter.is_none()); + assert!(opts.at_epoch.is_none()); + } + #[test] fn test_shortest_path_options_default() { let opts = ShortestPathOptions::default(); assert_eq!(opts.direction, Direction::Outgoing); - assert!(opts.type_filter.is_none()); + assert!(opts.edge_label_filter.is_none()); assert!(opts.weight_field.is_none()); assert!(opts.at_epoch.is_none()); assert!(opts.max_depth.is_none()); @@ -1914,8 +2746,8 @@ mod tests { #[test] fn test_component_options_default() { let opts = ComponentOptions::default(); - assert!(opts.edge_type_filter.is_none()); - assert!(opts.node_type_filter.is_none()); + assert!(opts.edge_label_filter.is_none()); + assert!(opts.node_label_filter.is_none()); assert!(opts.at_epoch.is_none()); } diff --git a/src/wal.rs b/src/wal.rs index c67d9a6..adbc5c2 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -2,12 +2,13 @@ use crate::encoding::encode_wal_op; use crate::encoding::{decode_wal_op, encode_wal_op_into}; use crate::error::EngineError; -use crate::types::WalOp; +use crate::types::{OpTag, WalOp}; use std::fs::{File, OpenOptions}; use std::io::{BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; -const WAL_FILENAME: &str = "data.wal"; +#[cfg(test)] +const WAL_FILENAME: &str = "wal_0.wal"; const WAL_MAGIC: [u8; 4] = *b"OVGR"; const WAL_VERSION: u32 = 3; const WAL_HEADER_SIZE: usize = 8; // WAL_MAGIC (4) + WAL_VERSION (4) @@ -18,6 +19,44 @@ const MAX_WAL_RECORD_SIZE: usize = 64 * 1024 * 1024; /// - len: byte length of payload (not including len or crc fields) /// - crc32: CRC-32 of payload bytes /// - payload: encoded WalOp +#[derive(Clone, Copy)] +struct WalReadAtomicBatch { + first_seq: u64, + op_count: u32, + ops_read: u32, +} + +impl WalReadAtomicBatch { + fn new(first_seq: u64, op_count: u32) -> Option { + if first_seq == 0 || op_count < 2 { + return None; + } + Some(Self { + first_seq, + op_count, + ops_read: 0, + }) + } + + fn push_normal_op(&mut self, seq: u64) -> bool { + if self.ops_read >= self.op_count { + return false; + } + let Some(expected_seq) = self.first_seq.checked_add(self.ops_read as u64) else { + return false; + }; + if seq != expected_seq { + return false; + } + self.ops_read += 1; + true + } + + fn matches_commit(&self, first_seq: u64, op_count: u32) -> bool { + self.first_seq == first_seq && self.op_count == op_count && self.ops_read == op_count + } +} + fn write_wal_header(writer: &mut impl Write) -> Result<(), EngineError> { writer.write_all(&WAL_MAGIC)?; writer.write_all(&WAL_VERSION.to_le_bytes())?; @@ -44,61 +83,12 @@ fn validate_wal_header(header: &[u8; WAL_HEADER_SIZE]) -> Result<(), EngineError /// Write-ahead log writer. Appends framed records to the WAL file. pub struct WalWriter { - path: PathBuf, writer: BufWriter, encode_buf: Vec, } impl WalWriter { - /// Open or create a WAL file for appending. - /// - /// If the file is new or empty, writes the WAL header (magic + version). - /// If the file already has data, validates the existing header. - pub fn open(db_dir: &Path) -> Result { - let path = db_dir.join(WAL_FILENAME); - - // Open a single file handle for both validation and append writes. - // read+create+append avoids a TOCTOU race between check and open. - let file = OpenOptions::new() - .create(true) - .read(true) - .append(true) - .open(&path)?; - - let file_len = file.metadata()?.len(); - let needs_header = if file_len == 0 { - true - } else if file_len < WAL_HEADER_SIZE as u64 { - return Err(EngineError::CorruptWal( - "WAL file too small for header".into(), - )); - } else { - // Validate existing header using the same file handle - let mut header = [0u8; WAL_HEADER_SIZE]; - (&file).read_exact(&mut header)?; - validate_wal_header(&header)?; - false - }; - - let mut writer = BufWriter::new(file); - - if needs_header { - write_wal_header(&mut writer)?; - writer.flush()?; - writer.get_ref().sync_all()?; - } - - Ok(WalWriter { - path, - writer, - encode_buf: Vec::new(), - }) - } - /// Open or create a WAL generation file for appending. - /// - /// Same as `open()` but uses `wal_generation_path(db_dir, gen_id)` instead - /// of the legacy `data.wal` filename. pub fn open_generation(db_dir: &Path, gen_id: u64) -> Result { let path = wal_generation_path(db_dir, gen_id); @@ -131,7 +121,6 @@ impl WalWriter { } Ok(WalWriter { - path, writer, encode_buf: Vec::new(), }) @@ -161,28 +150,77 @@ impl WalWriter { Ok(8 + total_payload) } + fn encode_frame_into( + &mut self, + seq: u64, + op: &WalOp, + batch_buf: &mut Vec, + ) -> Result<(), EngineError> { + encode_wal_op_into(op, &mut self.encode_buf)?; + let seq_bytes = seq.to_le_bytes(); + let total_payload = 8 + self.encode_buf.len(); + let len = total_payload as u32; + + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&seq_bytes); + hasher.update(&self.encode_buf); + let crc = hasher.finalize(); + + batch_buf.extend_from_slice(&len.to_le_bytes()); + batch_buf.extend_from_slice(&crc.to_le_bytes()); + batch_buf.extend_from_slice(&seq_bytes); + batch_buf.extend_from_slice(&self.encode_buf); + Ok(()) + } + /// Append multiple (engine_seq, WalOp) pairs as a single atomic buffer write. + /// Multi-op batches are wrapped in begin/commit control frames. /// All ops are pre-encoded before any I/O, so encoding failures /// don't leave partial data in the write buffer. /// Returns total bytes written (framing + payload). pub(crate) fn append_batch(&mut self, ops: &[(u64, WalOp)]) -> Result { - // Pre-encode all ops into a single contiguous buffer let mut batch_buf = Vec::new(); - for (seq, op) in ops { - encode_wal_op_into(op, &mut self.encode_buf)?; - let seq_bytes = seq.to_le_bytes(); - let total_payload = 8 + self.encode_buf.len(); - let len = total_payload as u32; - - let mut hasher = crc32fast::Hasher::new(); - hasher.update(&seq_bytes); - hasher.update(&self.encode_buf); - let crc = hasher.finalize(); - - batch_buf.extend_from_slice(&len.to_le_bytes()); - batch_buf.extend_from_slice(&crc.to_le_bytes()); - batch_buf.extend_from_slice(&seq_bytes); - batch_buf.extend_from_slice(&self.encode_buf); + match ops { + [] => {} + [(seq, op)] => self.encode_frame_into(*seq, op, &mut batch_buf)?, + _ => { + let first_seq = ops[0].0; + let op_count = u32::try_from(ops.len()).map_err(|_| { + EngineError::InvalidOperation( + "atomic WAL batch op_count exceeds u32::MAX".to_string(), + ) + })?; + if first_seq == 0 { + return Err(EngineError::InvalidOperation( + "atomic WAL batch first sequence must be nonzero".to_string(), + )); + } + for (idx, (seq, _)) in ops.iter().enumerate() { + let expected_seq = first_seq.checked_add(idx as u64).ok_or_else(|| { + EngineError::InvalidOperation( + "atomic WAL batch sequence range overflows u64".to_string(), + ) + })?; + if *seq != expected_seq { + return Err(EngineError::InvalidOperation( + "atomic WAL batch sequences must be contiguous".to_string(), + )); + } + } + let begin = WalOp::BeginAtomicBatch { + first_seq, + op_count, + }; + self.encode_frame_into(first_seq, &begin, &mut batch_buf)?; + for (seq, op) in ops { + self.encode_frame_into(*seq, op, &mut batch_buf)?; + } + let commit = WalOp::CommitAtomicBatch { + first_seq, + op_count, + }; + self.encode_frame_into(first_seq, &commit, &mut batch_buf)?; + } } let total = batch_buf.len(); @@ -191,8 +229,8 @@ impl WalWriter { Ok(total) } - /// Flush the WAL to disk. - pub fn flush(&mut self) -> Result<(), EngineError> { + #[cfg(test)] + pub(crate) fn flush(&mut self) -> Result<(), EngineError> { self.writer.flush()?; Ok(()) } @@ -203,168 +241,65 @@ impl WalWriter { self.writer.get_ref().sync_all()?; Ok(()) } - - /// Truncate the WAL file and re-write the header. - /// Re-opens the append writer after truncation. - pub fn truncate_and_reset(&mut self) -> Result<(), EngineError> { - self.writer.flush()?; - { - // Use a dedicated truncate handle so Windows does not rely on - // set_len() against an append-mode descriptor. - let file = OpenOptions::new() - .write(true) - .truncate(true) - .open(&self.path)?; - let mut writer = BufWriter::new(file); - write_wal_header(&mut writer)?; - writer.flush()?; - writer.get_ref().sync_all()?; - } - - // Re-open append writer for subsequent WAL appends. - let file = OpenOptions::new() - .create(true) - .read(true) - .append(true) - .open(&self.path)?; - self.writer = BufWriter::new(file); - Ok(()) - } - - /// Return the WAL file path. - pub fn path(&self) -> &Path { - &self.path - } } /// Write-ahead log reader. Reads framed records with CRC validation. pub struct WalReader { + #[cfg(test)] path: PathBuf, } +pub(crate) struct WalReadResult { + pub(crate) records: Vec<(u64, WalOp)>, + pub(crate) durable_len: u64, +} + impl WalReader { - pub fn new(db_dir: &Path) -> Self { + #[cfg(test)] + pub(crate) fn new(db_dir: &Path) -> Self { WalReader { - path: db_dir.join(WAL_FILENAME), + path: wal_generation_path(db_dir, 0), } } - /// Read all valid records from the WAL. Stops at EOF or first corrupt/truncated - /// record (which is treated as a crash boundary; the partial record is ignored). - /// - /// Validates the WAL header (magic + version) before reading records. - /// Returns `(engine_seq, WalOp)` pairs. Returns an error if the file is - /// not a valid OverGraph WAL. #[cfg(test)] pub(crate) fn read_all(&self) -> Result, EngineError> { - if !self.path.exists() { - return Ok(Vec::new()); - } - - let file = File::open(&self.path)?; - let file_len = file.metadata()?.len(); - if file_len == 0 { - return Ok(Vec::new()); - } - - if file_len < WAL_HEADER_SIZE as u64 { - return Err(EngineError::CorruptWal( - "WAL file too small for header".into(), - )); - } - - let mut reader = BufReader::new(file); - - // Validate header - let mut header = [0u8; WAL_HEADER_SIZE]; - reader - .read_exact(&mut header) - .map_err(EngineError::IoError)?; - validate_wal_header(&header)?; - - let mut ops = Vec::new(); - - loop { - // Read length - let mut len_buf = [0u8; 4]; - match reader.read_exact(&mut len_buf) { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(e) => return Err(EngineError::IoError(e)), - } - let payload_len = u32::from_le_bytes(len_buf) as usize; - - // Sanity check: reject zero-length or impossibly large records - if payload_len == 0 || payload_len > MAX_WAL_RECORD_SIZE { - break; - } - - // Read CRC - let mut crc_buf = [0u8; 4]; - match reader.read_exact(&mut crc_buf) { - Ok(()) => {} - Err(_) => break, // truncated - } - let stored_crc = u32::from_le_bytes(crc_buf); - - // Read payload - let mut payload = vec![0u8; payload_len]; - match reader.read_exact(&mut payload) { - Ok(()) => {} - Err(_) => break, // truncated - } - - // Validate CRC - if crc32fast::hash(&payload) != stored_crc { - // Corrupt record. Stop here (crash boundary) - break; - } - - // V3 frame: first 8 bytes are engine_seq, rest is WalOp - if payload.len() < 8 { - break; // truncated seq prefix - } - let engine_seq = u64::from_le_bytes(payload[..8].try_into().expect("8 bytes for seq")); - let walop_bytes = &payload[8..]; - - let recognized_tag = walop_bytes - .first() - .and_then(|tag| crate::types::OpTag::from_u8(*tag)) - .is_some(); - - // Decode the operation. Unknown op tags remain a crash boundary for - // garbage tail recovery, but recognized malformed records are hard - // corruption and must fail reopen. - match decode_wal_op(walop_bytes) { - Ok(op) => ops.push((engine_seq, op)), - Err(err) if recognized_tag => { - return Err(EngineError::CorruptWal(format!( - "failed to decode WAL record: {}", - err - ))); - } - Err(_) => break, - } - } - - Ok(ops) + Self::read_path(&self.path).map(|result| result.records) } /// Read all valid records from a WAL generation file. - /// Same as `read_all()` but reads from `wal_generation_path(db_dir, gen_id)`. + #[cfg(test)] pub(crate) fn read_generation( db_dir: &Path, gen_id: u64, ) -> Result, EngineError> { + Self::read_generation_recoverable(db_dir, gen_id).map(|result| result.records) + } + + /// Read all recoverable records plus the byte offset of the durable prefix. + pub(crate) fn read_generation_recoverable( + db_dir: &Path, + gen_id: u64, + ) -> Result { let path = wal_generation_path(db_dir, gen_id); + Self::read_path(&path) + } + + fn read_path(path: &Path) -> Result { if !path.exists() { - return Ok(Vec::new()); + return Ok(WalReadResult { + records: Vec::new(), + durable_len: 0, + }); } - let file = File::open(&path)?; + let file = File::open(path)?; let file_len = file.metadata()?.len(); if file_len == 0 { - return Ok(Vec::new()); + return Ok(WalReadResult { + records: Vec::new(), + durable_len: 0, + }); } if file_len < WAL_HEADER_SIZE as u64 { @@ -383,8 +318,12 @@ impl WalReader { validate_wal_header(&header)?; let mut ops = Vec::new(); + let mut open_batch: Option = None; + let mut pos = WAL_HEADER_SIZE as u64; + let mut durable_len = pos; loop { + let frame_start = pos; // Read length let mut len_buf = [0u8; 4]; match reader.read_exact(&mut len_buf) { @@ -397,6 +336,7 @@ impl WalReader { if payload_len == 0 || payload_len > MAX_WAL_RECORD_SIZE { break; } + let frame_end = frame_start + 8 + payload_len as u64; // Read CRC let mut crc_buf = [0u8; 4]; @@ -425,14 +365,63 @@ impl WalReader { let engine_seq = u64::from_le_bytes(payload[..8].try_into().expect("8 bytes for seq")); let walop_bytes = &payload[8..]; - let recognized_tag = walop_bytes - .first() - .and_then(|tag| crate::types::OpTag::from_u8(*tag)) - .is_some(); + let recognized_tag = walop_bytes.first().and_then(|tag| OpTag::from_u8(*tag)); match decode_wal_op(walop_bytes) { - Ok(op) => ops.push((engine_seq, op)), - Err(err) if recognized_tag => { + Ok(op) => match &op { + WalOp::BeginAtomicBatch { + first_seq, + op_count, + } => { + if open_batch.is_some() { + break; + } + let Some(batch) = WalReadAtomicBatch::new(*first_seq, *op_count) else { + break; + }; + open_batch = Some(batch); + ops.push((engine_seq, op)); + pos = frame_end; + } + WalOp::CommitAtomicBatch { + first_seq, + op_count, + } => { + let Some(batch) = open_batch else { + break; + }; + if !batch.matches_commit(*first_seq, *op_count) { + break; + } + open_batch = None; + ops.push((engine_seq, op)); + durable_len = frame_end; + pos = frame_end; + } + _ => { + if let Some(batch) = open_batch.as_mut() { + if !batch.push_normal_op(engine_seq) { + break; + } + } else { + durable_len = frame_end; + } + ops.push((engine_seq, op)); + pos = frame_end; + } + }, + Err(_) + if matches!( + recognized_tag, + Some(OpTag::BeginAtomicBatch | OpTag::CommitAtomicBatch) + ) => + { + break; + } + Err(_) if open_batch.is_some() && recognized_tag.is_some() => { + break; + } + Err(err) if recognized_tag.is_some() => { return Err(EngineError::CorruptWal(format!( "failed to decode WAL record: {}", err @@ -442,27 +431,11 @@ impl WalReader { } } - Ok(ops) - } - - /// Returns true if the WAL file exists. - pub fn exists(&self) -> bool { - self.path.exists() - } - - /// Return the WAL file path. - pub fn path(&self) -> &Path { - &self.path - } -} - -/// Delete the WAL file (used after successful flush/checkpoint). -pub fn remove_wal(db_dir: &Path) -> Result<(), EngineError> { - let path = db_dir.join(WAL_FILENAME); - if path.exists() { - std::fs::remove_file(&path)?; + Ok(WalReadResult { + records: ops, + durable_len, + }) } - Ok(()) } /// WAL generation file path: `wal_.wal` @@ -479,14 +452,18 @@ pub fn remove_wal_generation(db_dir: &Path, gen_id: u64) -> Result<(), EngineErr Ok(()) } -/// Truncate the WAL file and re-write the header (alternative to remove for reuse). -pub fn truncate_wal(db_dir: &Path) -> Result<(), EngineError> { - let path = db_dir.join(WAL_FILENAME); - if path.exists() { - let file = OpenOptions::new().write(true).truncate(true).open(&path)?; - let mut writer = BufWriter::new(&file); - write_wal_header(&mut writer)?; - writer.flush()?; +pub(crate) fn truncate_wal_generation_to( + db_dir: &Path, + gen_id: u64, + durable_len: u64, +) -> Result<(), EngineError> { + let path = wal_generation_path(db_dir, gen_id); + if !path.exists() { + return Ok(()); + } + let file = OpenOptions::new().write(true).open(&path)?; + if file.metadata()?.len() > durable_len { + file.set_len(durable_len)?; file.sync_all()?; } Ok(()) @@ -504,7 +481,7 @@ mod tests { props.insert("name".to_string(), PropValue::String(key.to_string())); WalOp::UpsertNode(NodeRecord { id, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: key.to_string(), props, created_at: 1000 * id as i64, @@ -521,7 +498,7 @@ mod tests { id, from, to, - type_id: 10, + label_id: 10, props: BTreeMap::new(), created_at: 2000 * id as i64, updated_at: 2000 * id as i64 + 1, @@ -535,7 +512,7 @@ mod tests { #[test] fn test_wal_write_and_read_single() { let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); let op = make_test_node(1, "user:alice"); writer.append(&op, 1).unwrap(); @@ -560,7 +537,7 @@ mod tests { #[test] fn test_wal_write_and_read_many() { let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); for i in 0..1000 { let op = make_test_node(i, &format!("node:{}", i)); @@ -613,7 +590,7 @@ mod tests { #[test] fn test_wal_mixed_operations() { let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); writer.append(&make_test_node(1, "alice"), 1).unwrap(); writer.append(&make_test_node(2, "bob"), 2).unwrap(); @@ -672,7 +649,7 @@ mod tests { #[test] fn test_wal_corrupt_tail_recovery() { let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); // Write 5 valid records for i in 0..5 { @@ -700,7 +677,7 @@ mod tests { #[test] fn test_wal_corrupt_crc_detection() { let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); // Write 3 valid records for i in 0..3 { @@ -731,59 +708,10 @@ mod tests { assert_eq!(ops.len(), 1); } - #[test] - fn test_wal_truncate() { - let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); - writer.append(&make_test_node(1, "test"), 1).unwrap(); - writer.flush().unwrap(); - drop(writer); - - // Verify data exists - let reader = WalReader::new(dir.path()); - assert_eq!(reader.read_all().unwrap().len(), 1); - - // Truncate (re-writes header) - truncate_wal(dir.path()).unwrap(); - - // Verify empty records but valid WAL - let reader = WalReader::new(dir.path()); - assert!(reader.read_all().unwrap().is_empty()); - - // Verify we can reopen a writer on the truncated file - let mut writer = WalWriter::open(dir.path()).unwrap(); - writer - .append(&make_test_node(2, "after_truncate"), 2) - .unwrap(); - writer.flush().unwrap(); - drop(writer); - - let reader = WalReader::new(dir.path()); - let records = reader.read_all().unwrap(); - assert_eq!(records.len(), 1); - match &records[0] { - (_, WalOp::UpsertNode(node)) => assert_eq!(node.key, "after_truncate"), - _ => panic!("expected UpsertNode"), - } - } - - #[test] - fn test_wal_remove() { - let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); - writer.append(&make_test_node(1, "test"), 1).unwrap(); - writer.flush().unwrap(); - drop(writer); - - assert!(dir.path().join(WAL_FILENAME).exists()); - remove_wal(dir.path()).unwrap(); - assert!(!dir.path().join(WAL_FILENAME).exists()); - } - #[test] fn test_wal_append_returns_size() { let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); let delete_op = WalOp::DeleteNode { id: 1, @@ -796,13 +724,27 @@ mod tests { writer.flush().unwrap(); } + #[test] + fn test_wal_append_batch_rejects_non_contiguous_sequences() { + let dir = TempDir::new().unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); + let err = writer + .append_batch(&[(1, make_test_node(1, "a")), (3, make_test_node(2, "b"))]) + .unwrap_err(); + assert!(matches!(err, EngineError::InvalidOperation(_))); + writer.flush().unwrap(); + + let reader = WalReader::new(dir.path()); + assert!(reader.read_all().unwrap().is_empty()); + } + #[test] fn test_wal_reopen_writer_and_append() { let dir = TempDir::new().unwrap(); // First session: write 3 records { - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); for i in 0..3 { writer .append(&make_test_node(i, &format!("s1:{}", i)), i + 1) @@ -813,7 +755,7 @@ mod tests { // Second session: reopen and append 2 more { - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); for i in 10..12 { writer .append(&make_test_node(i, &format!("s2:{}", i)), i + 1) @@ -873,7 +815,7 @@ mod tests { std::fs::write(&wal_path, b"BADMAGIC").unwrap(); // Writer should refuse to open - let result = WalWriter::open(dir.path()); + let result = WalWriter::open_generation(dir.path(), 0); assert!(result.is_err()); } @@ -883,7 +825,7 @@ mod tests { let wal_path = dir.path().join(WAL_FILENAME); // Create a WAL via writer - let writer = WalWriter::open(dir.path()).unwrap(); + let writer = WalWriter::open_generation(dir.path(), 0).unwrap(); drop(writer); // Verify the file starts with the header @@ -902,7 +844,7 @@ mod tests { let wal_path = dir.path().join(WAL_FILENAME); // Write 3 valid records via writer (header auto-added) - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); for i in 0..3 { writer .append(&make_test_node(i, &format!("n:{}", i)), i + 1) @@ -937,7 +879,7 @@ mod tests { let dir = TempDir::new().unwrap(); let wal_path = dir.path().join(WAL_FILENAME); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); writer.append(&make_test_node(1, "valid"), 1).unwrap(); writer.flush().unwrap(); drop(writer); @@ -972,7 +914,7 @@ mod tests { let dir = TempDir::new().unwrap(); let wal_path = dir.path().join(WAL_FILENAME); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); writer.append(&make_test_node(1, "valid"), 1).unwrap(); writer.flush().unwrap(); drop(writer); @@ -1031,7 +973,7 @@ mod tests { let wal_path = dir.path().join(WAL_FILENAME); // Write 2 valid records via writer (header auto-added) - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); writer.append(&make_test_node(1, "first"), 1).unwrap(); writer.append(&make_test_node(2, "second"), 2).unwrap(); writer.flush().unwrap(); diff --git a/src/wal_sync.rs b/src/wal_sync.rs index 16d6625..df88ed3 100644 --- a/src/wal_sync.rs +++ b/src/wal_sync.rs @@ -113,7 +113,7 @@ mod tests { props.insert("name".to_string(), PropValue::String(key.to_string())); WalOp::UpsertNode(NodeRecord { id, - type_id: 1, + label_ids: NodeLabelSet::single(1).unwrap(), key: key.to_string(), props, created_at: 1000 * id as i64, @@ -125,45 +125,10 @@ mod tests { }) } - #[test] - fn test_truncate_and_reset() { - let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); - - // Write some records - writer.append(&make_test_node(1, "a"), 1).unwrap(); - writer.append(&make_test_node(2, "b"), 2).unwrap(); - writer.sync().unwrap(); - - // Verify records exist - let reader = WalReader::new(dir.path()); - assert_eq!(reader.read_all().unwrap().len(), 2); - - // Truncate and reset - writer.truncate_and_reset().unwrap(); - - // Verify WAL is empty (just header) - let reader = WalReader::new(dir.path()); - assert!(reader.read_all().unwrap().is_empty()); - - // Write new records after reset - writer.append(&make_test_node(3, "c"), 3).unwrap(); - writer.sync().unwrap(); - drop(writer); - - let reader = WalReader::new(dir.path()); - let ops = reader.read_all().unwrap(); - assert_eq!(ops.len(), 1); - match &ops[0] { - (_, WalOp::UpsertNode(node)) => assert_eq!(node.key, "c"), - _ => panic!("expected UpsertNode"), - } - } - #[test] fn test_sync_thread_basic_operation() { let dir = TempDir::new().unwrap(); - let writer = WalWriter::open(dir.path()).unwrap(); + let writer = WalWriter::open_generation(dir.path(), 0).unwrap(); let state = WalSyncState { wal_writer: writer, @@ -211,7 +176,7 @@ mod tests { #[test] fn test_append_batch_returns_size() { let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); let ops = vec![ (1u64, make_test_node(1, "a")), @@ -221,11 +186,11 @@ mod tests { // Each record has: 4 (len) + 4 (crc) + payload assert!(total > 0); - // Verify both records can be read back + // Verify both records and their atomic-batch markers can be read back writer.sync().unwrap(); drop(writer); let reader = WalReader::new(dir.path()); - assert_eq!(reader.read_all().unwrap().len(), 2); + assert_eq!(reader.read_all().unwrap().len(), 4); } #[test] @@ -233,7 +198,7 @@ mod tests { // Verify that shutdown_sync_thread performs a final sync // even if the sync thread hasn't drained buffered_bytes yet. let dir = TempDir::new().unwrap(); - let mut writer = WalWriter::open(dir.path()).unwrap(); + let mut writer = WalWriter::open_generation(dir.path(), 0).unwrap(); // Append data directly (not through sync thread) let bytes = writer.append(&make_test_node(1, "pending"), 1).unwrap(); @@ -278,7 +243,7 @@ mod tests { fn test_poisoned_state_is_visible() { // Verify that once poisoned is set, it persists in the shared state. let dir = TempDir::new().unwrap(); - let writer = WalWriter::open(dir.path()).unwrap(); + let writer = WalWriter::open_generation(dir.path(), 0).unwrap(); let state = WalSyncState { wal_writer: writer, @@ -312,7 +277,7 @@ mod tests { fn test_multiple_sync_cycles_drain_all() { // Append data in three separate batches, verify all are synced. let dir = TempDir::new().unwrap(); - let writer = WalWriter::open(dir.path()).unwrap(); + let writer = WalWriter::open_generation(dir.path(), 0).unwrap(); let state = WalSyncState { wal_writer: writer, @@ -362,7 +327,7 @@ mod tests { fn test_sync_thread_shutdown_with_zero_buffered() { // Shutdown when no data has been written should be clean and immediate. let dir = TempDir::new().unwrap(); - let writer = WalWriter::open(dir.path()).unwrap(); + let writer = WalWriter::open_generation(dir.path(), 0).unwrap(); let state = WalSyncState { wal_writer: writer, diff --git a/tests/phase4_integration.rs b/tests/compaction_integration.rs similarity index 95% rename from tests/phase4_integration.rs rename to tests/compaction_integration.rs index 6e9b54f..349c017 100644 --- a/tests/phase4_integration.rs +++ b/tests/compaction_integration.rs @@ -4,6 +4,8 @@ use overgraph::{ use std::collections::BTreeMap; use tempfile::TempDir; +const COMPACTION_LABELS: [&str; 3] = ["Person", "Company", "Article"]; + /// Insert 5k nodes + 10k edges, flush, delete 30%, flush, compact, /// verify disk shrinks and queries return correct results. #[test] @@ -17,7 +19,7 @@ fn test_compaction_removes_deleted_records_and_shrinks_disk() { let mut node_ids = Vec::with_capacity(5_000); let batch: Vec = (0..5_000) .map(|i| overgraph::NodeInput { - type_id: (i % 3) as u32 + 1, // types 1..3 + labels: vec![COMPACTION_LABELS[i % COMPACTION_LABELS.len()].to_string()], key: format!("n:{}", i), props: { let mut p = BTreeMap::new(); @@ -29,7 +31,7 @@ fn test_compaction_removes_deleted_records_and_shrinks_disk() { sparse_vector: None, }) .collect(); - node_ids.extend(engine.batch_upsert_nodes(&batch).unwrap()); + node_ids.extend(engine.batch_upsert_nodes(batch.clone()).unwrap()); assert_eq!(node_ids.len(), 5_000); // --- Step 1b: Insert 10k edges (chain + cross-links) --- @@ -39,28 +41,28 @@ fn test_compaction_removes_deleted_records_and_shrinks_disk() { .map(|i| overgraph::EdgeInput { from: node_ids[i], to: node_ids[i + 1], - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - edge_ids.extend(engine.batch_upsert_edges(&chain).unwrap()); + edge_ids.extend(engine.batch_upsert_edges(chain.clone()).unwrap()); // Cross-links: 5001 more edges let cross: Vec = (0..5_001) .map(|i| overgraph::EdgeInput { from: node_ids[i % 5_000], to: node_ids[(i + 500) % 5_000], - type_id: 20, + label: "REFERENCES".to_string(), props: BTreeMap::new(), weight: 0.7, valid_from: None, valid_to: None, }) .collect(); - edge_ids.extend(engine.batch_upsert_edges(&cross).unwrap()); + edge_ids.extend(engine.batch_upsert_edges(cross.clone()).unwrap()); assert_eq!(edge_ids.len(), 10_000); // --- Flush to segment 1 --- @@ -192,18 +194,18 @@ fn test_compaction_removes_deleted_records_and_shrinks_disk() { ); } - // Type-filtered neighbors should still work + // Relationship-filtered neighbors should still work let chain_only = engine .neighbors( mid, &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) .unwrap(); for entry in &chain_only { - assert_eq!(entry.edge_type_id, 10); + assert_eq!(entry.label, "KNOWS"); } engine.close().unwrap(); @@ -240,7 +242,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { // Segment 1: nodes A, B, C with edges A->B, B->C let a = engine .upsert_node( - 1, + "Person", "alpha", UpsertNodeOptions { props: props(&[("v", 1)]), @@ -251,7 +253,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { .unwrap(); let b = engine .upsert_node( - 1, + "Person", "beta", UpsertNodeOptions { props: props(&[("v", 2)]), @@ -262,7 +264,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { .unwrap(); let c = engine .upsert_node( - 2, + "Company", "gamma", UpsertNodeOptions { props: props(&[("v", 3)]), @@ -272,13 +274,13 @@ fn test_reads_consistent_through_compaction_lifecycle() { ) .unwrap(); let e_ab = engine - .upsert_edge(a, b, 10, UpsertEdgeOptions::default()) + .upsert_edge(a, b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); let e_bc = engine .upsert_edge( b, c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.9, ..Default::default() @@ -290,7 +292,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { // Segment 2: update B's props, add node D, edge C->D, delete edge A->B let _ = engine .upsert_node( - 1, + "Person", "beta", UpsertNodeOptions { props: props(&[("v", 20)]), @@ -301,7 +303,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { .unwrap(); // update B let d = engine .upsert_node( - 3, + "Article", "delta", UpsertNodeOptions { props: props(&[("v", 4)]), @@ -314,7 +316,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { .upsert_edge( c, d, - 20, + "REFERENCES", UpsertEdgeOptions { weight: 0.8, ..Default::default() @@ -328,7 +330,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { engine.delete_node(c).unwrap(); let e = engine .upsert_node( - 1, + "Person", "epsilon", UpsertNodeOptions { props: props(&[("v", 5)]), @@ -341,7 +343,7 @@ fn test_reads_consistent_through_compaction_lifecycle() { .upsert_edge( d, e, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.7, ..Default::default() diff --git a/tests/phase1_integration.rs b/tests/engine_lifecycle_integration.rs similarity index 84% rename from tests/phase1_integration.rs rename to tests/engine_lifecycle_integration.rs index 9270f6c..e35355e 100644 --- a/tests/phase1_integration.rs +++ b/tests/engine_lifecycle_integration.rs @@ -26,7 +26,7 @@ fn edge_options(from: u64, to: u64) -> UpsertEdgeOptions { } #[test] -fn test_full_phase1_lifecycle() { +fn test_full_engine_lifecycle() { let dir = tempfile::TempDir::new().unwrap(); let db_path = dir.path().join("integration_db"); @@ -34,14 +34,14 @@ fn test_full_phase1_lifecycle() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); for i in 1..=100 { - let type_id = match i % 3 { - 0 => 1, - 1 => 2, - _ => 3, + let label = match i % 3 { + 0 => "Person", + 1 => "Company", + _ => "Article", }; let key = format!("key:{}", i); let id = engine - .upsert_node(type_id, &key, node_options(&key, 0.5 + (i as f32 * 0.01))) + .upsert_node(label, &key, node_options(&key, 0.5 + (i as f32 * 0.01))) .unwrap(); assert_eq!(id, i); } @@ -49,9 +49,9 @@ fn test_full_phase1_lifecycle() { for i in 1..=200 { let from = (i % 100) + 1; let to = ((i * 7) % 100) + 1; - let type_id = if i <= 100 { 10 } else { 20 }; + let label = if i <= 100 { "KNOWS" } else { "REFERENCES" }; let id = engine - .upsert_edge(from, to, type_id, edge_options(from, to)) + .upsert_edge(from, to, label, edge_options(from, to)) .unwrap(); assert_eq!(id, i); } @@ -75,7 +75,7 @@ fn test_full_phase1_lifecycle() { let node1 = engine.get_node(1).unwrap().unwrap(); assert_eq!(node1.key, "key:1"); - assert_eq!(node1.type_id, 2); + assert_eq!(node1.labels.as_slice(), ["Company"]); let node99 = engine.get_node(99).unwrap().unwrap(); assert_eq!(node99.key, "key:99"); @@ -85,12 +85,12 @@ fn test_full_phase1_lifecycle() { let edge1 = engine.get_edge(1).unwrap().unwrap(); assert_eq!(edge1.from, 2); assert_eq!(edge1.to, 8); - assert_eq!(edge1.type_id, 10); + assert_eq!(edge1.label, "KNOWS"); assert!(engine.get_edge(100).unwrap().is_none()); let edge150 = engine.get_edge(150).unwrap().unwrap(); - assert_eq!(edge150.type_id, 20); + assert_eq!(edge150.label, "REFERENCES"); let node_props = &engine.get_node(1).unwrap().unwrap().props; assert_eq!( @@ -130,10 +130,10 @@ fn test_wal_replay_last_write_wins() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); let id1 = engine - .upsert_node(1, "stable-key", node_options("original", 0.51)) + .upsert_node("Person", "stable-key", node_options("original", 0.51)) .unwrap(); let id2 = engine - .upsert_node(1, "stable-key", node_options("updated", 0.99)) + .upsert_node("Person", "stable-key", node_options("updated", 0.99)) .unwrap(); assert_eq!(id1, id2); @@ -148,7 +148,7 @@ fn test_wal_replay_last_write_wins() { Some(&PropValue::String("updated".to_string())) ); assert!((node.weight - 0.99).abs() < f32::EPSILON); - assert_eq!(engine.get_nodes_by_type(1).unwrap().len(), 1); + assert_eq!(engine.get_nodes_by_labels("Person").unwrap().len(), 1); engine.close().unwrap(); } } @@ -163,7 +163,7 @@ fn test_crash_recovery_without_close() { for i in 1..=20 { let key = format!("crash:{}", i); let id = engine - .upsert_node(1, &key, node_options(&key, 0.5 + (i as f32 * 0.01))) + .upsert_node("Person", &key, node_options(&key, 0.5 + (i as f32 * 0.01))) .unwrap(); assert_eq!(id, i); } diff --git a/tests/phase2_integration.rs b/tests/graph_query_integration.rs similarity index 89% rename from tests/phase2_integration.rs rename to tests/graph_query_integration.rs index 4b11d2e..859466c 100644 --- a/tests/phase2_integration.rs +++ b/tests/graph_query_integration.rs @@ -1,6 +1,6 @@ use overgraph::*; -/// Build a small graph (20 nodes, 50 edges, mixed types), +/// Build a small graph (20 nodes, 50 edges, mixed labels and relationships), /// verify all query patterns: get, neighbors, deletes. #[test] fn test_full_graph_query_patterns() { @@ -9,13 +9,17 @@ fn test_full_graph_query_patterns() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); - // Create 20 nodes across 3 types + // Create 20 nodes across 3 labels let mut node_ids = Vec::new(); for i in 0..20 { - let type_id = (i % 3) as u32 + 1; // types 1, 2, 3 + let label = match i % 3 { + 0 => "Person", + 1 => "Company", + _ => "Article", + }; let id = engine .upsert_node( - type_id, + label, &format!("node:{}", i), UpsertNodeOptions { weight: 0.5, @@ -27,16 +31,16 @@ fn test_full_graph_query_patterns() { } assert_eq!(engine.node_count().unwrap(), 20); - // Create 50 edges across 2 types - // Type 10: "knows" edges, chain pattern (0->1->2->...->19) - // Type 20: "references" edges, skip pattern (i->i+3) + // Create 50 edges across two relationships. + // KNOWS edges use a chain pattern (0->1->2->...->19). + // REFERENCES edges use skip and hub patterns. let mut edge_ids = Vec::new(); for i in 0..19 { let eid = engine .upsert_edge( node_ids[i], node_ids[i + 1], - 10, + "KNOWS", UpsertEdgeOptions::default(), ) .unwrap(); @@ -47,7 +51,7 @@ fn test_full_graph_query_patterns() { .upsert_edge( node_ids[i], node_ids[i + 3], - 20, + "REFERENCES", UpsertEdgeOptions { weight: 0.8, ..Default::default() @@ -62,7 +66,7 @@ fn test_full_graph_query_patterns() { .upsert_edge( node_ids[0], node_ids[i], - 20, + "REFERENCES", UpsertEdgeOptions { weight: 0.5, ..Default::default() @@ -88,12 +92,12 @@ fn test_full_graph_query_patterns() { .unwrap(); assert_eq!(out_0.len(), 16); - // Filter by type 10 ("knows"), node 0 has exactly 1 (->1) + // Filter by KNOWS, node 0 has exactly 1 (->1) let knows_0 = engine .neighbors( node_ids[0], &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) @@ -106,7 +110,7 @@ fn test_full_graph_query_patterns() { .neighbors( node_ids[5], &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) @@ -169,7 +173,7 @@ fn test_full_graph_query_patterns() { .neighbors( node_ids[0], &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) @@ -200,7 +204,7 @@ fn test_graph_state_survives_restart() { node_a = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.5, @@ -210,7 +214,7 @@ fn test_graph_state_survives_restart() { .unwrap(); node_b = engine .upsert_node( - 1, + "Person", "b", UpsertNodeOptions { weight: 0.6, @@ -220,7 +224,7 @@ fn test_graph_state_survives_restart() { .unwrap(); node_c = engine .upsert_node( - 2, + "Company", "c", UpsertNodeOptions { weight: 0.7, @@ -230,7 +234,7 @@ fn test_graph_state_survives_restart() { .unwrap(); node_d = engine .upsert_node( - 2, + "Company", "d", UpsertNodeOptions { weight: 0.8, @@ -240,13 +244,13 @@ fn test_graph_state_survives_restart() { .unwrap(); edge_ab = engine - .upsert_edge(node_a, node_b, 10, UpsertEdgeOptions::default()) + .upsert_edge(node_a, node_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); edge_ac = engine .upsert_edge( node_a, node_c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.9, ..Default::default() @@ -257,7 +261,7 @@ fn test_graph_state_survives_restart() { .upsert_edge( node_b, node_c, - 20, + "REFERENCES", UpsertEdgeOptions { weight: 0.8, ..Default::default() @@ -268,7 +272,7 @@ fn test_graph_state_survives_restart() { .upsert_edge( node_c, node_d, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.7, ..Default::default() @@ -334,23 +338,23 @@ fn test_graph_state_survives_restart() { assert_eq!(inc_c.len(), 1); assert_eq!(inc_c[0].node_id, node_a); - // Type filter works after replay - let typed = engine + // Edge label filter works after replay + let label_filtered = engine .neighbors( node_a, &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) .unwrap(); - assert_eq!(typed.len(), 2); // both ab and ac are type 10 + assert_eq!(label_filtered.len(), 2); // both ab and ac are KNOWS // Upsert dedup still works after replay let engine = engine; // need mut for upsert let a_again = engine .upsert_node( - 1, + "Person", "a", UpsertNodeOptions { weight: 0.99, @@ -364,7 +368,7 @@ fn test_graph_state_survives_restart() { // New allocation doesn't collide let node_e = engine .upsert_node( - 1, + "Person", "e", UpsertNodeOptions { weight: 0.5, diff --git a/tests/identity_no_byte_check.rs b/tests/identity_no_byte_check.rs new file mode 100644 index 0000000..b8c1e12 --- /dev/null +++ b/tests/identity_no_byte_check.rs @@ -0,0 +1,175 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +const PROHIBITED_PRODUCTION_SYMBOLS: &[&str] = &[ + "ComponentChecksumVerifier", + "CheckedComponentRange", + "ComponentBlockChecksumSpec", + "COMPONENT_CHECKSUM_BLOCK_SIZE", + "checked_slice", + "checked_all", + "block_checksum", + "checksum_table", + "verify_block", + "OGCHK", + "read_path_counters", +]; + +const PROHIBITED_STANDALONE_CORE_FILENAMES: &[&str] = &[ + "nodes.dat", + "edges.dat", + "node_meta.dat", + "edge_meta.dat", + "tombstones.dat", + "key_index.dat", + "node_label_index.dat", + "edge_label_index.dat", + "edge_triple_index.dat", + "adj_out.idx", + "adj_out.dat", + "adj_in.idx", + "adj_in.dat", + "timestamp_index.dat", + "node_vector_meta.dat", + "node_dense_vectors.dat", + "node_sparse_vectors.dat", + "edge_weight_index.dat", + "edge_updated_at_index.dat", + "edge_valid_from_index.dat", + "edge_valid_to_index.dat", +]; + +#[test] +fn production_sources_do_not_contain_component_byte_checking() { + let failures = scan_production_sources_for(PROHIBITED_PRODUCTION_SYMBOLS); + + assert!( + failures.is_empty(), + "prohibited checked-read/component checksum symbols found in production sources:\n{}", + failures.join("\n") + ); +} + +#[test] +fn production_sources_do_not_create_standalone_core_component_files() { + let failures = scan_production_sources_for(PROHIBITED_STANDALONE_CORE_FILENAMES); + + assert!( + failures.is_empty(), + "prohibited standalone packed-core component filenames found in production sources:\n{}", + failures.join("\n") + ); +} + +fn scan_production_sources_for(needles: &[&str]) -> Vec { + let repo = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let src = repo.join("src"); + let mut failures = Vec::new(); + scan_dir(&src, needles, &mut failures); + failures +} + +fn scan_dir(dir: &Path, needles: &[&str], failures: &mut Vec) { + for entry in fs::read_dir(dir).expect("read source directory") { + let entry = entry.expect("read source entry"); + let path = entry.path(); + if path.is_dir() { + if path.ends_with(Path::new("src/engine/tests")) { + continue; + } + scan_dir(&path, needles, failures); + continue; + } + + if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { + scan_file(&path, needles, failures); + } + } +} + +fn scan_file(path: &Path, needles: &[&str], failures: &mut Vec) { + let content = fs::read_to_string(path).expect("read source file"); + for (line_no, line) in production_lines(&content).enumerate() { + for needle in needles { + if line.contains(needle) { + failures.push(format!( + "{}:{} contains `{}`", + path.display(), + line_no + 1, + needle + )); + } + } + } +} + +#[test] +fn scrub_not_called_from_normal_open_read_query_paths() { + let repo = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let src = repo.join("src"); + let scrub_call_sites = &["scrub_database", "crate::scrub::"]; + + let mut failures = Vec::new(); + scan_dir_for_scrub(&src, scrub_call_sites, &mut failures); + + assert!( + failures.is_empty(), + "scrub functions must not be called from normal engine paths:\n{}", + failures.join("\n") + ); +} + +fn scan_dir_for_scrub(dir: &Path, needles: &[&str], failures: &mut Vec) { + for entry in fs::read_dir(dir).expect("read source directory") { + let entry = entry.expect("read source entry"); + let path = entry.path(); + if path.is_dir() { + scan_dir_for_scrub(&path, needles, failures); + continue; + } + + if path.extension().and_then(|ext| ext.to_str()) != Some("rs") { + continue; + } + + let path_str = path.to_string_lossy(); + if path_str.ends_with("scrub.rs") { + continue; + } + + let content = fs::read_to_string(&path).expect("read source file"); + let in_scrub_method = std::cell::Cell::new(false); + for (line_no, line) in production_lines(&content).enumerate() { + let trimmed = line.trim(); + if trimmed.starts_with("pub fn scrub(") { + in_scrub_method.set(true); + } else if in_scrub_method.get() && trimmed == "}" { + in_scrub_method.set(false); + continue; + } + if in_scrub_method.get() { + continue; + } + for needle in needles { + if line.contains(needle) { + failures.push(format!( + "{}:{} contains `{}`", + path.display(), + line_no + 1, + needle, + )); + } + } + } + } +} + +fn production_lines(content: &str) -> impl Iterator { + let mut in_test_tail = false; + content.lines().filter(move |line| { + if line.trim_start().starts_with("#[cfg(test)]") { + in_test_tail = true; + } + !in_test_tail + }) +} diff --git a/tests/inspect_integration.rs b/tests/inspect_integration.rs index 6354f50..549dc40 100644 --- a/tests/inspect_integration.rs +++ b/tests/inspect_integration.rs @@ -50,16 +50,20 @@ fn test_inspect_with_data() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); for i in 0..10 { - db.upsert_node(1, &format!("node_{}", i), UpsertNodeOptions::default()) - .unwrap(); + db.upsert_node( + "Person", + &format!("node_{}", i), + UpsertNodeOptions::default(), + ) + .unwrap(); } let n1 = db - .upsert_node(2, "a", UpsertNodeOptions::default()) + .upsert_node("Company", "a", UpsertNodeOptions::default()) .unwrap(); let n2 = db - .upsert_node(2, "b", UpsertNodeOptions::default()) + .upsert_node("Company", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(n1, n2, 1, UpsertEdgeOptions::default()) + db.upsert_edge(n1, n2, "RELATES_TO", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -98,14 +102,22 @@ fn test_inspect_multiple_segments() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); for i in 0..5 { - db.upsert_node(1, &format!("batch1_{}", i), UpsertNodeOptions::default()) - .unwrap(); + db.upsert_node( + "Person", + &format!("batch1_{}", i), + UpsertNodeOptions::default(), + ) + .unwrap(); } db.flush().unwrap(); for i in 0..3 { - db.upsert_node(1, &format!("batch2_{}", i), UpsertNodeOptions::default()) - .unwrap(); + db.upsert_node( + "Person", + &format!("batch2_{}", i), + UpsertNodeOptions::default(), + ) + .unwrap(); } db.flush().unwrap(); @@ -141,7 +153,7 @@ fn test_inspect_with_prune_policies() { PrunePolicy { max_age_ms: Some(86_400_000), max_weight: Some(0.1), - type_id: Some(3), + label: Some("Article".to_string()), }, ) .unwrap(); @@ -159,7 +171,7 @@ fn test_inspect_with_prune_policies() { assert!(stdout.contains("old_memories")); assert!(stdout.contains("max_age=86400000ms")); assert!(stdout.contains("max_weight=0.1")); - assert!(stdout.contains("type_id=3")); + assert!(stdout.contains("label=Article")); } #[test] @@ -175,8 +187,12 @@ fn test_inspect_json_with_data() { let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); for i in 0..5 { - db.upsert_node(1, &format!("node_{}", i), UpsertNodeOptions::default()) - .unwrap(); + db.upsert_node( + "Person", + &format!("node_{}", i), + UpsertNodeOptions::default(), + ) + .unwrap(); } db.flush().unwrap(); db.close().unwrap(); @@ -207,6 +223,48 @@ fn test_inspect_json_with_data() { assert!(parsed["segments"][0]["size_bytes"].as_u64().unwrap() > 0); } +#[test] +fn test_inspect_json_with_multi_label_data() { + let dir = TempDir::new().unwrap(); + + let opts = DbOptions { + create_if_missing: true, + wal_sync_mode: WalSyncMode::Immediate, + compact_after_n_flushes: 0, + ..DbOptions::default() + }; + let db = DatabaseEngine::open(dir.path(), &opts).unwrap(); + + db.upsert_node("Person", "person-only", UpsertNodeOptions::default()) + .unwrap(); + db.upsert_node( + &["Person", "Employee"], + "person-employee", + UpsertNodeOptions::default(), + ) + .unwrap(); + db.flush().unwrap(); + db.close().unwrap(); + + let output = Command::new(inspect_binary()) + .args(["--json", dir.path().to_str().unwrap()]) + .output() + .expect("failed to run overgraph-inspect"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "inspect --json failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let parsed: serde_json::Value = + serde_json::from_str(&stdout).expect("output should be valid JSON"); + assert_eq!(parsed["segment_count"], 1); + assert_eq!(parsed["total_nodes"], 2); + assert_eq!(parsed["segments"][0]["node_count"], 2); +} + #[test] fn test_inspect_json_uninitialized() { let dir = TempDir::new().unwrap(); diff --git a/tests/named_graph_api_integration.rs b/tests/named_graph_api_integration.rs new file mode 100644 index 0000000..a2360d0 --- /dev/null +++ b/tests/named_graph_api_integration.rs @@ -0,0 +1,921 @@ +use overgraph::{ + AllShortestPathsOptions, ComponentOptions, DatabaseEngine, DbOptions, DegreeOptions, + DenseMetric, DenseVectorConfig, Direction, ExportOptions, HnswConfig, IsConnectedOptions, + LabelMatchMode, NeighborOptions, NodeLabelFilter, PageRequest, PprOptions, PropValue, + PrunePolicy, ShortestPathOptions, SubgraphOptions, TopKOptions, TraverseOptions, + UpsertEdgeOptions, UpsertNodeOptions, VectorSearchMode, VectorSearchRequest, VectorSearchScope, +}; +use std::collections::{BTreeMap, HashSet}; +use std::path::Path; +use tempfile::TempDir; + +#[derive(Debug, Clone, Copy)] +struct GraphIds { + alice: u64, + bob: u64, + carol: u64, + acme: u64, + doc: u64, +} + +fn node_options(weight: f32, sparse_vector: Option>) -> UpsertNodeOptions { + let mut props = BTreeMap::new(); + props.insert("weight".to_string(), PropValue::Float(weight as f64)); + UpsertNodeOptions { + props, + weight, + sparse_vector, + ..Default::default() + } +} + +fn weighted_edge(weight: f32) -> UpsertEdgeOptions { + UpsertEdgeOptions { + weight, + ..Default::default() + } +} + +fn named_node_label_filter(labels: &[&str], mode: LabelMatchMode) -> NodeLabelFilter { + NodeLabelFilter { + labels: labels.iter().map(|label| (*label).to_string()).collect(), + mode, + } +} + +fn open_graph(path: &Path) -> (DatabaseEngine, GraphIds) { + let engine = DatabaseEngine::open(path, &DbOptions::default()).unwrap(); + let alice = engine + .upsert_node("Person", "alice", node_options(1.0, Some(vec![(1, 1.0)]))) + .unwrap(); + let bob = engine + .upsert_node("Person", "bob", node_options(1.0, Some(vec![(1, 0.9)]))) + .unwrap(); + let carol = engine + .upsert_node("Person", "carol", node_options(1.0, Some(vec![(2, 1.0)]))) + .unwrap(); + let acme = engine + .upsert_node("Company", "acme", node_options(1.0, Some(vec![(1, 0.7)]))) + .unwrap(); + let doc = engine + .upsert_node("Document", "doc", node_options(1.0, Some(vec![(1, 0.8)]))) + .unwrap(); + + engine + .upsert_edge(alice, bob, "KNOWS", weighted_edge(3.0)) + .unwrap(); + engine + .upsert_edge(bob, carol, "KNOWS", weighted_edge(2.0)) + .unwrap(); + engine + .upsert_edge(alice, acme, "WORKS_AT", weighted_edge(5.0)) + .unwrap(); + let deleted = engine + .upsert_edge(alice, doc, "KNOWS", weighted_edge(7.0)) + .unwrap(); + engine.delete_edge(deleted).unwrap(); + + ( + engine, + GraphIds { + alice, + bob, + carol, + acme, + doc, + }, + ) +} + +fn ids(items: impl IntoIterator, f: impl Fn(T) -> u64) -> HashSet { + items.into_iter().map(f).collect() +} + +fn hit_ids(hits: &[(u64, f64)]) -> HashSet { + hits.iter().map(|(id, _)| *id).collect() +} + +#[test] +fn named_edge_filters_cover_neighbors_degrees_paths_and_pagination() { + let dir = TempDir::new().unwrap(); + let (engine, graph) = open_graph(&dir.path().join("db")); + + let known = Some(vec!["KNOWS".to_string()]); + let mixed = Some(vec!["MISSING_EDGE".to_string(), "KNOWS".to_string()]); + let all_unknown = Some(vec!["MISSING_EDGE".to_string()]); + let empty = Some(Vec::new()); + + let known_neighbors = engine + .neighbors( + graph.alice, + &NeighborOptions { + edge_label_filter: known.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(known_neighbors.len(), 1); + assert_eq!(known_neighbors[0].node_id, graph.bob); + assert_eq!(known_neighbors[0].label, "KNOWS"); + + let mixed_neighbors = engine + .neighbors( + graph.alice, + &NeighborOptions { + edge_label_filter: mixed.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(mixed_neighbors, known_neighbors); + + assert!(engine + .neighbors( + graph.alice, + &NeighborOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap() + .is_empty()); + + let unconstrained_neighbors = engine + .neighbors( + graph.alice, + &NeighborOptions { + edge_label_filter: empty.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + ids(&unconstrained_neighbors, |entry| entry.node_id), + HashSet::from([graph.bob, graph.acme]) + ); + + let first_page = engine + .neighbors_paged( + graph.alice, + &NeighborOptions { + edge_label_filter: empty.clone(), + ..Default::default() + }, + &PageRequest { + limit: Some(1), + after: None, + }, + ) + .unwrap(); + assert_eq!(first_page.items.len(), 1); + assert!(first_page.next_cursor.is_some()); + let second_page = engine + .neighbors_paged( + graph.alice, + &NeighborOptions { + edge_label_filter: empty.clone(), + ..Default::default() + }, + &PageRequest { + limit: Some(10), + after: first_page.next_cursor, + }, + ) + .unwrap(); + assert_eq!(second_page.items.len(), 1); + + let top = engine + .top_k_neighbors( + graph.alice, + 1, + &TopKOptions { + edge_label_filter: known.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(top.len(), 1); + assert_eq!(top[0].label, "KNOWS"); + + let known_degree = DegreeOptions { + edge_label_filter: known.clone(), + ..Default::default() + }; + let mixed_degree = DegreeOptions { + edge_label_filter: mixed.clone(), + ..Default::default() + }; + let unknown_degree = DegreeOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }; + let empty_degree = DegreeOptions { + edge_label_filter: empty.clone(), + ..Default::default() + }; + assert_eq!(engine.degree(graph.alice, &known_degree).unwrap(), 1); + assert_eq!(engine.degree(graph.alice, &mixed_degree).unwrap(), 1); + assert_eq!(engine.degree(graph.alice, &unknown_degree).unwrap(), 0); + assert_eq!(engine.degree(graph.alice, &empty_degree).unwrap(), 2); + assert_eq!( + engine.sum_edge_weights(graph.alice, &known_degree).unwrap(), + 3.0 + ); + assert_eq!( + engine.avg_edge_weight(graph.alice, &known_degree).unwrap(), + Some(3.0) + ); + assert_eq!( + engine + .degrees(&[graph.alice, graph.bob], &known_degree) + .unwrap() + .get(&graph.alice) + .copied(), + Some(1) + ); + assert!(engine + .degrees(&[graph.alice, graph.bob], &unknown_degree) + .unwrap() + .is_empty()); + + let path = engine + .shortest_path( + graph.alice, + graph.carol, + &ShortestPathOptions { + edge_label_filter: known.clone(), + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + assert_eq!(path.nodes, vec![graph.alice, graph.bob, graph.carol]); + assert!(engine + .shortest_path( + graph.alice, + graph.carol, + &ShortestPathOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap() + .is_none()); + assert!(engine + .is_connected( + graph.alice, + graph.carol, + &IsConnectedOptions { + edge_label_filter: mixed.clone(), + ..Default::default() + }, + ) + .unwrap()); + assert!(!engine + .is_connected( + graph.alice, + graph.carol, + &IsConnectedOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap()); + assert_eq!( + engine + .all_shortest_paths( + graph.alice, + graph.carol, + &AllShortestPathsOptions { + edge_label_filter: mixed.clone(), + ..Default::default() + }, + ) + .unwrap() + .len(), + 1 + ); + assert!(engine + .all_shortest_paths( + graph.alice, + graph.carol, + &AllShortestPathsOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap() + .is_empty()); + + let zero_hop = engine + .shortest_path( + graph.alice, + graph.alice, + &ShortestPathOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + assert_eq!(zero_hop.nodes, vec![graph.alice]); + assert!(zero_hop.edges.is_empty()); + assert_eq!(zero_hop.total_cost, 0.0); + let all_zero_hop = engine + .all_shortest_paths( + graph.alice, + graph.alice, + &AllShortestPathsOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(all_zero_hop.len(), 1); + assert_eq!(all_zero_hop[0].nodes, vec![graph.alice]); + assert!(all_zero_hop[0].edges.is_empty()); + assert!(engine + .is_connected( + graph.alice, + graph.alice, + &IsConnectedOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap()); + + let traversal = engine + .traverse( + graph.alice, + 2, + &TraverseOptions { + edge_label_filter: mixed.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + ids(&traversal.items, |hit| hit.node_id), + HashSet::from([graph.bob, graph.carol]) + ); + assert!(engine + .traverse( + graph.alice, + 2, + &TraverseOptions { + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap() + .items + .is_empty()); + let start_only = engine + .traverse( + graph.alice, + 2, + &TraverseOptions { + min_depth: 0, + edge_label_filter: all_unknown.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(start_only.items.len(), 1); + assert_eq!(start_only.items[0].node_id, graph.alice); + assert_eq!(start_only.items[0].depth, 0); + assert_eq!(start_only.items[0].via_edge_id, None); + let emitted_people = engine + .traverse( + graph.alice, + 1, + &TraverseOptions { + edge_label_filter: empty, + emit_node_label_filter: Some(named_node_label_filter( + &["Person"], + LabelMatchMode::Any, + )), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + ids(&emitted_people.items, |hit| hit.node_id), + HashSet::from([graph.bob]) + ); + + assert!(engine + .neighbors( + graph.alice, + &NeighborOptions { + edge_label_filter: Some(vec!["".to_string()]), + ..Default::default() + }, + ) + .is_err()); + assert_eq!(engine.get_edge_label_id("MISSING_EDGE").unwrap(), None); + assert_eq!(engine.get_node_label_id("MissingLabel").unwrap(), None); + + engine.close().unwrap(); +} + +#[test] +fn named_filters_cover_components_ppr_subgraph_export_and_prune() { + let dir = TempDir::new().unwrap(); + let (engine, graph) = open_graph(&dir.path().join("db")); + + let knows = Some(vec!["KNOWS".to_string()]); + let mixed_knows = Some(vec!["KNOWS".to_string(), "MISSING_EDGE".to_string()]); + let missing_edge = Some(vec!["MISSING_EDGE".to_string()]); + + let known_components = engine + .connected_components(&ComponentOptions { + edge_label_filter: mixed_knows.clone(), + ..Default::default() + }) + .unwrap(); + assert_eq!(known_components[&graph.alice], known_components[&graph.bob]); + assert_eq!(known_components[&graph.bob], known_components[&graph.carol]); + assert_eq!(known_components[&graph.acme], graph.acme); + + let singleton_components = engine + .connected_components(&ComponentOptions { + edge_label_filter: missing_edge.clone(), + ..Default::default() + }) + .unwrap(); + assert_eq!(singleton_components[&graph.alice], graph.alice); + assert_eq!(singleton_components[&graph.bob], graph.bob); + + let person_components = engine + .connected_components(&ComponentOptions { + node_label_filter: Some(named_node_label_filter(&["Person"], LabelMatchMode::Any)), + ..Default::default() + }) + .unwrap(); + assert!(person_components.contains_key(&graph.alice)); + assert!(!person_components.contains_key(&graph.acme)); + assert!(engine + .connected_components(&ComponentOptions { + node_label_filter: Some(named_node_label_filter( + &["MissingLabel"], + LabelMatchMode::Any, + )), + ..Default::default() + }) + .unwrap() + .is_empty()); + + assert_eq!( + engine + .component_of( + graph.alice, + &ComponentOptions { + edge_label_filter: knows.clone(), + ..Default::default() + }, + ) + .unwrap(), + vec![graph.alice, graph.bob, graph.carol] + ); + assert_eq!( + engine + .component_of( + graph.alice, + &ComponentOptions { + edge_label_filter: missing_edge.clone(), + ..Default::default() + }, + ) + .unwrap(), + vec![graph.alice] + ); + assert_eq!( + engine + .component_of( + graph.alice, + &ComponentOptions { + node_label_filter: Some(named_node_label_filter( + &["MissingLabel"], + LabelMatchMode::Any, + )), + ..Default::default() + }, + ) + .unwrap(), + Vec::::new() + ); + + let ppr = engine + .personalized_pagerank( + &[graph.alice], + &PprOptions { + edge_label_filter: mixed_knows.clone(), + max_results: Some(10), + ..Default::default() + }, + ) + .unwrap(); + assert!(!hit_ids(&ppr.scores).contains(&graph.acme)); + let ppr_unknown = engine + .personalized_pagerank( + &[graph.alice], + &PprOptions { + edge_label_filter: missing_edge.clone(), + max_results: Some(10), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(hit_ids(&ppr_unknown.scores), HashSet::from([graph.alice])); + + let subgraph = engine + .extract_subgraph( + graph.alice, + 2, + &SubgraphOptions { + edge_label_filter: mixed_knows, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + ids(&subgraph.nodes, |node| node.id), + HashSet::from([graph.alice, graph.bob, graph.carol]) + ); + assert!(subgraph + .nodes + .iter() + .all(|node| node.labels.as_slice() == ["Person"])); + assert!(subgraph.edges.iter().all(|edge| edge.label == "KNOWS")); + + let empty_subgraph = engine + .extract_subgraph( + graph.alice, + 2, + &SubgraphOptions { + edge_label_filter: missing_edge.clone(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(empty_subgraph.nodes.len(), 1); + assert_eq!(empty_subgraph.nodes[0].id, graph.alice); + assert!(empty_subgraph.edges.is_empty()); + + let export = engine + .export_adjacency(&ExportOptions { + node_label_filter: Some(named_node_label_filter( + &["Person", "MissingLabel"], + LabelMatchMode::Any, + )), + edge_label_filter: knows, + include_weights: true, + }) + .unwrap(); + assert_eq!(export.edge_labels, vec!["KNOWS".to_string()]); + assert!(export.edges.iter().all(|edge| edge.edge_label_index == 0)); + assert!(export.edges.iter().all(|edge| edge.weight.is_some())); + assert!(export.edges.iter().all(|edge| edge.to != graph.doc)); + + let empty_export = engine + .export_adjacency(&ExportOptions { + node_label_filter: Some(named_node_label_filter( + &["MissingLabel"], + LabelMatchMode::Any, + )), + ..Default::default() + }) + .unwrap(); + assert!(empty_export.node_ids.is_empty()); + assert!(empty_export.edge_labels.is_empty()); + assert!(empty_export.edges.is_empty()); + assert!(engine + .export_adjacency(&ExportOptions { + node_label_filter: Some(named_node_label_filter( + &["MissingLabel"], + LabelMatchMode::Any, + )), + edge_label_filter: Some(vec!["".to_string()]), + ..Default::default() + }) + .is_err()); + + let edge_empty_export = engine + .export_adjacency(&ExportOptions { + edge_label_filter: missing_edge.clone(), + include_weights: false, + ..Default::default() + }) + .unwrap(); + assert!(!edge_empty_export.node_ids.is_empty()); + assert!(edge_empty_export.edge_labels.is_empty()); + assert!(edge_empty_export.edges.is_empty()); + + let pruned = engine + .upsert_node("Person", "pruned", node_options(0.1, Some(vec![(1, 0.2)]))) + .unwrap(); + engine + .upsert_edge(graph.alice, pruned, "KNOWS", weighted_edge(9.0)) + .unwrap(); + assert_eq!( + engine + .degree( + graph.alice, + &DegreeOptions { + edge_label_filter: Some(vec!["KNOWS".to_string()]), + ..Default::default() + }, + ) + .unwrap(), + 2 + ); + engine + .set_prune_policy( + "low_weight_people", + PrunePolicy { + max_age_ms: None, + max_weight: Some(0.5), + label: Some("Person".to_string()), + }, + ) + .unwrap(); + let visible_after_policy = engine + .neighbors( + graph.alice, + &NeighborOptions { + edge_label_filter: Some(vec!["KNOWS".to_string()]), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + ids(&visible_after_policy, |entry| entry.node_id), + HashSet::from([graph.bob]) + ); + assert_eq!(engine.get_edge_label_id("MISSING_EDGE").unwrap(), None); + assert_eq!(engine.get_node_label_id("MissingLabel").unwrap(), None); + + engine.close().unwrap(); +} + +#[test] +fn named_graph_filters_survive_flush_compaction_and_reopen() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("db"); + { + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let alice = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let bob = engine + .upsert_node("Person", "bob", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_edge(alice, bob, "KNOWS", weighted_edge(1.0)) + .unwrap(); + engine.flush().unwrap(); + + let acme = engine + .upsert_node("Company", "acme", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_edge(alice, acme, "WORKS_AT", weighted_edge(2.0)) + .unwrap(); + engine.flush().unwrap(); + engine.compact().unwrap(); + engine.close().unwrap(); + } + + let reopened = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let alice = reopened + .get_node_by_key("Person", "alice") + .unwrap() + .unwrap() + .id; + let neighbors = reopened + .neighbors( + alice, + &NeighborOptions { + edge_label_filter: Some(vec!["KNOWS".to_string(), "MISSING_EDGE".to_string()]), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(neighbors.len(), 1); + assert_eq!(neighbors[0].label, "KNOWS"); + + let export = reopened + .export_adjacency(&ExportOptions { + edge_label_filter: Some(Vec::new()), + include_weights: false, + ..Default::default() + }) + .unwrap(); + assert_eq!( + export.edge_labels.iter().cloned().collect::>(), + HashSet::from(["KNOWS".to_string(), "WORKS_AT".to_string()]) + ); + assert!(export.edges.iter().all(|edge| edge.weight.is_none())); + assert_eq!(reopened.get_edge_label_id("MISSING_EDGE").unwrap(), None); + reopened.close().unwrap(); +} + +#[test] +fn vector_search_uses_named_label_and_scope_filters_but_returns_id_scores() { + let dir = TempDir::new().unwrap(); + let (engine, graph) = open_graph(&dir.path().join("db")); + + let request = |label_filter, scope| VectorSearchRequest { + mode: VectorSearchMode::Sparse, + dense_query: None, + sparse_query: Some(vec![(1, 1.0)]), + k: 10, + label_filter, + ef_search: None, + scope, + dense_weight: None, + sparse_weight: None, + fusion_mode: None, + }; + + let person_hits = engine + .vector_search(&request( + Some(named_node_label_filter( + &["Person", "MissingLabel"], + LabelMatchMode::Any, + )), + None, + )) + .unwrap(); + assert_eq!( + ids(&person_hits, |hit| hit.node_id), + HashSet::from([graph.alice, graph.bob]) + ); + + assert!(engine + .vector_search(&request( + Some(named_node_label_filter( + &["MissingLabel"], + LabelMatchMode::Any + )), + None, + )) + .unwrap() + .is_empty()); + assert!(engine + .vector_search(&request( + Some(named_node_label_filter( + &["MissingLabel"], + LabelMatchMode::Any + )), + Some(VectorSearchScope { + start_node_id: graph.alice, + max_depth: 1, + direction: Direction::Outgoing, + edge_label_filter: Some(vec!["".to_string()]), + at_epoch: None, + }), + )) + .is_err()); + + let unconstrained_hits = engine.vector_search(&request(None, None)).unwrap(); + let unconstrained_ids = ids(&unconstrained_hits, |hit| hit.node_id); + assert!(unconstrained_ids.contains(&graph.doc)); + assert!(unconstrained_ids.contains(&graph.acme)); + assert!(engine + .vector_search(&request( + Some(named_node_label_filter(&[], LabelMatchMode::Any)), + None, + )) + .is_err()); + + let knows_scope = Some(VectorSearchScope { + start_node_id: graph.alice, + max_depth: 1, + direction: Direction::Outgoing, + edge_label_filter: Some(vec!["KNOWS".to_string(), "MISSING_EDGE".to_string()]), + at_epoch: None, + }); + let scoped_hits = engine.vector_search(&request(None, knows_scope)).unwrap(); + assert_eq!( + ids(&scoped_hits, |hit| hit.node_id), + HashSet::from([graph.alice, graph.bob]) + ); + + let unknown_scope = Some(VectorSearchScope { + start_node_id: graph.alice, + max_depth: 1, + direction: Direction::Outgoing, + edge_label_filter: Some(vec!["MISSING_EDGE".to_string()]), + at_epoch: None, + }); + let start_only = engine.vector_search(&request(None, unknown_scope)).unwrap(); + assert_eq!( + ids(&start_only, |hit| hit.node_id), + HashSet::from([graph.alice]) + ); + + let empty_scope = Some(VectorSearchScope { + start_node_id: graph.alice, + max_depth: 1, + direction: Direction::Outgoing, + edge_label_filter: Some(Vec::new()), + at_epoch: None, + }); + let empty_scope_ids = ids( + &engine.vector_search(&request(None, empty_scope)).unwrap(), + |hit| hit.node_id, + ); + assert!(empty_scope_ids.contains(&graph.alice)); + assert!(empty_scope_ids.contains(&graph.bob)); + assert!(empty_scope_ids.contains(&graph.acme)); + assert!(!empty_scope_ids.contains(&graph.doc)); + assert_eq!(engine.get_edge_label_id("MISSING_EDGE").unwrap(), None); + assert_eq!(engine.get_node_label_id("MissingLabel").unwrap(), None); + + engine.close().unwrap(); +} + +#[test] +fn vector_search_unknown_label_filter_does_not_hide_invalid_shape() { + let dir = TempDir::new().unwrap(); + let opts = DbOptions { + dense_vector: Some(DenseVectorConfig { + dimension: 3, + metric: DenseMetric::Cosine, + hnsw: HnswConfig::default(), + }), + ..Default::default() + }; + let engine = DatabaseEngine::open(dir.path(), &opts).unwrap(); + let unknown_label = Some(named_node_label_filter( + &["MissingLabel"], + LabelMatchMode::Any, + )); + + let dense_request = |dense_query, ef_search| VectorSearchRequest { + mode: VectorSearchMode::Dense, + dense_query, + sparse_query: None, + k: 5, + label_filter: unknown_label.clone(), + ef_search, + scope: None, + dense_weight: None, + sparse_weight: None, + fusion_mode: None, + }; + + let err = engine + .vector_search(&dense_request(None, None)) + .unwrap_err(); + assert!(err.to_string().contains("requires dense_query")); + + let err = engine + .vector_search(&dense_request(Some(vec![0.1, 0.2]), None)) + .unwrap_err(); + assert!(err + .to_string() + .contains("does not match configured dimension")); + + let err = engine + .vector_search(&dense_request(Some(vec![0.1, 0.2, 0.3]), Some(0))) + .unwrap_err(); + assert!(err.to_string().contains("ef_search must be > 0")); + + let err = engine + .vector_search(&VectorSearchRequest { + mode: VectorSearchMode::Sparse, + dense_query: None, + sparse_query: Some(vec![(1, -1.0)]), + k: 5, + label_filter: unknown_label.clone(), + ef_search: None, + scope: None, + dense_weight: None, + sparse_weight: None, + fusion_mode: None, + }) + .unwrap_err(); + assert!(err + .to_string() + .contains("sparse vector weights must be non-negative")); + + let err = engine + .vector_search(&VectorSearchRequest { + mode: VectorSearchMode::Hybrid, + dense_query: None, + sparse_query: None, + k: 5, + label_filter: unknown_label, + ef_search: None, + scope: None, + dense_weight: None, + sparse_weight: None, + fusion_mode: None, + }) + .unwrap_err(); + assert!(err.to_string().contains("requires at least one")); +} diff --git a/tests/named_query_api_integration.rs b/tests/named_query_api_integration.rs new file mode 100644 index 0000000..ec40726 --- /dev/null +++ b/tests/named_query_api_integration.rs @@ -0,0 +1,376 @@ +use overgraph::{ + DatabaseEngine, DbOptions, Direction, EdgePattern, EdgeQuery, GraphPatternQuery, + LabelMatchMode, NodeFilterExpr, NodeLabelFilter, NodePattern, NodeQuery, PatternOrder, + PropValue, QueryPlanPublicName, QueryPlanWarning, UpsertEdgeOptions, UpsertNodeOptions, +}; +use std::collections::BTreeMap; +use tempfile::TempDir; + +fn props(entries: &[(&str, PropValue)]) -> BTreeMap { + entries + .iter() + .map(|(key, value)| ((*key).to_string(), value.clone())) + .collect() +} + +fn node_options(entries: &[(&str, PropValue)]) -> UpsertNodeOptions { + UpsertNodeOptions { + props: props(entries), + ..Default::default() + } +} + +fn edge_options(entries: &[(&str, PropValue)]) -> UpsertEdgeOptions { + UpsertEdgeOptions { + props: props(entries), + ..Default::default() + } +} + +#[test] +fn explain_plans_surface_public_names_without_token_ids() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let alice = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let acme = engine + .upsert_node("Company", "acme", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_edge(alice, acme, "WORKS_AT", UpsertEdgeOptions::default()) + .unwrap(); + + let node_plan = engine + .explain_node_query(&NodeQuery { + label_filter: Some(NodeLabelFilter { + labels: vec!["Person".to_string()], + mode: LabelMatchMode::All, + }), + ..Default::default() + }) + .unwrap(); + assert_eq!( + node_plan.public_inputs.node_labels, + vec![QueryPlanPublicName { + alias: None, + name: "Person".to_string(), + known: true, + mode: Some(LabelMatchMode::All), + }] + ); + assert!(node_plan.public_inputs.edge_labels.is_empty()); + + let edge_plan = engine + .explain_edge_query(&EdgeQuery { + label: Some("WORKS_AT".to_string()), + from_ids: vec![alice], + ..Default::default() + }) + .unwrap(); + assert!(edge_plan.public_inputs.node_labels.is_empty()); + assert_eq!( + edge_plan.public_inputs.edge_labels, + vec![QueryPlanPublicName { + alias: None, + name: "WORKS_AT".to_string(), + known: true, + mode: None, + }] + ); + + let pattern_plan = engine + .explain_pattern_query(&GraphPatternQuery { + nodes: vec![ + NodePattern { + alias: "p".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec!["Person".to_string()], + mode: LabelMatchMode::All, + }), + ids: Vec::new(), + keys: Vec::new(), + filter: None, + }, + NodePattern { + alias: "c".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec!["Company".to_string()], + mode: LabelMatchMode::All, + }), + ids: Vec::new(), + keys: Vec::new(), + filter: None, + }, + ], + edges: vec![EdgePattern { + alias: Some("e".to_string()), + from_alias: "p".to_string(), + to_alias: "c".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["WORKS_AT".to_string(), "MISSING".to_string()], + filter: None, + }], + at_epoch: None, + limit: 10, + order: PatternOrder::AnchorThenAliasesAsc, + }) + .unwrap(); + assert_eq!( + pattern_plan.public_inputs.node_labels, + vec![ + QueryPlanPublicName { + alias: Some("p".to_string()), + name: "Person".to_string(), + known: true, + mode: Some(LabelMatchMode::All), + }, + QueryPlanPublicName { + alias: Some("c".to_string()), + name: "Company".to_string(), + known: true, + mode: Some(LabelMatchMode::All), + }, + ] + ); + assert_eq!( + pattern_plan.public_inputs.edge_labels, + vec![ + QueryPlanPublicName { + alias: Some("e".to_string()), + name: "WORKS_AT".to_string(), + known: true, + mode: None, + }, + QueryPlanPublicName { + alias: Some("e".to_string()), + name: "MISSING".to_string(), + known: false, + mode: None, + }, + ] + ); + assert!(pattern_plan + .warnings + .contains(&QueryPlanWarning::UnknownEdgeLabel)); + + let debug = format!("{:?}", pattern_plan.public_inputs); + assert!(!debug.contains(concat!("type", "_id"))); + assert!(!debug.contains(concat!("type", "Id"))); +} + +#[test] +fn planner_queries_use_public_names_and_hydrate_views() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + + let alice = engine + .upsert_node( + "Person", + "alice", + node_options(&[("status", PropValue::String("active".to_string()))]), + ) + .unwrap(); + engine + .upsert_node( + "Person", + "bob", + node_options(&[("status", PropValue::String("inactive".to_string()))]), + ) + .unwrap(); + let acme = engine + .upsert_node("Company", "acme", UpsertNodeOptions::default()) + .unwrap(); + let works_at = engine + .upsert_edge( + alice, + acme, + "WORKS_AT", + edge_options(&[("role", PropValue::String("engineer".to_string()))]), + ) + .unwrap(); + + let node_query = NodeQuery { + label_filter: Some(NodeLabelFilter { + labels: vec!["Person".to_string()], + mode: LabelMatchMode::All, + }), + filter: Some(NodeFilterExpr::PropertyEquals { + key: "status".to_string(), + value: PropValue::String("active".to_string()), + }), + ..Default::default() + }; + assert_eq!( + engine.query_node_ids(&node_query).unwrap().items, + vec![alice] + ); + let nodes = engine.query_nodes(&node_query).unwrap(); + assert_eq!(nodes.items.len(), 1); + assert_eq!(nodes.items[0].labels.as_slice(), ["Person"]); + assert_eq!(nodes.items[0].id, alice); + + let edge_query = EdgeQuery { + label: Some("WORKS_AT".to_string()), + from_ids: vec![alice], + ..Default::default() + }; + assert_eq!( + engine.query_edge_ids(&edge_query).unwrap().edge_ids, + vec![works_at] + ); + let edges = engine.query_edges(&edge_query).unwrap(); + assert_eq!(edges.edges.len(), 1); + assert_eq!(edges.edges[0].label, "WORKS_AT"); + assert_eq!(edges.edges[0].id, works_at); +} + +#[test] +fn planner_unknown_names_are_read_only_empty_constraints() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let alice = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let bob = engine + .upsert_node("Person", "bob", UpsertNodeOptions::default()) + .unwrap(); + engine + .upsert_edge(alice, bob, "KNOWS", UpsertEdgeOptions::default()) + .unwrap(); + + let missing_node_query = NodeQuery { + label_filter: Some(NodeLabelFilter { + labels: vec!["Missing".to_string()], + mode: LabelMatchMode::All, + }), + ..Default::default() + }; + assert!(engine + .query_node_ids(&missing_node_query) + .unwrap() + .items + .is_empty()); + let node_plan = engine.explain_node_query(&missing_node_query).unwrap(); + assert!(node_plan + .warnings + .contains(&QueryPlanWarning::UnknownNodeLabel)); + assert_eq!(engine.get_node_label_id("Missing").unwrap(), None); + + let missing_edge_query = EdgeQuery { + label: Some("MISSING".to_string()), + from_ids: vec![alice], + ..Default::default() + }; + assert!(engine + .query_edge_ids(&missing_edge_query) + .unwrap() + .edge_ids + .is_empty()); + let edge_plan = engine.explain_edge_query(&missing_edge_query).unwrap(); + assert!(edge_plan + .warnings + .contains(&QueryPlanWarning::UnknownEdgeLabel)); + assert_eq!(engine.get_edge_label_id("MISSING").unwrap(), None); + + let invalid = NodeQuery { + label_filter: Some(NodeLabelFilter { + labels: vec![" Missing".to_string()], + mode: LabelMatchMode::All, + }), + ids: vec![alice], + ..Default::default() + }; + assert!(engine.query_node_ids(&invalid).is_err()); +} + +#[test] +fn graph_pattern_resolves_named_filters_without_changing_bindings() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("db"); + let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); + let alice = engine + .upsert_node("Person", "alice", UpsertNodeOptions::default()) + .unwrap(); + let acme = engine + .upsert_node("Company", "acme", UpsertNodeOptions::default()) + .unwrap(); + let works_at = engine + .upsert_edge(alice, acme, "WORKS_AT", UpsertEdgeOptions::default()) + .unwrap(); + + let base = GraphPatternQuery { + nodes: vec![ + NodePattern { + alias: "p".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec!["Person".to_string()], + mode: LabelMatchMode::All, + }), + ids: Vec::new(), + keys: Vec::new(), + filter: None, + }, + NodePattern { + alias: "c".to_string(), + label_filter: Some(NodeLabelFilter { + labels: vec!["Company".to_string()], + mode: LabelMatchMode::All, + }), + ids: Vec::new(), + keys: Vec::new(), + filter: None, + }, + ], + edges: vec![EdgePattern { + alias: Some("e".to_string()), + from_alias: "p".to_string(), + to_alias: "c".to_string(), + direction: Direction::Outgoing, + label_filter: vec!["WORKS_AT".to_string(), "MISSING".to_string()], + filter: None, + }], + at_epoch: None, + limit: 10, + order: PatternOrder::AnchorThenAliasesAsc, + }; + + let result = engine.query_pattern(&base).unwrap(); + assert_eq!(result.matches.len(), 1); + assert_eq!(result.matches[0].nodes["p"], alice); + assert_eq!(result.matches[0].nodes["c"], acme); + assert_eq!(result.matches[0].edges["e"], works_at); + + let mut all_unknown = base.clone(); + all_unknown.edges[0].label_filter = vec!["MISSING".to_string()]; + assert!(engine + .query_pattern(&all_unknown) + .unwrap() + .matches + .is_empty()); + assert!(engine + .explain_pattern_query(&all_unknown) + .unwrap() + .warnings + .contains(&QueryPlanWarning::UnknownEdgeLabel)); + + let mut unknown_node = base; + unknown_node.nodes[0].label_filter = Some(NodeLabelFilter { + labels: vec!["Missing".to_string()], + mode: LabelMatchMode::All, + }); + assert!(engine + .query_pattern(&unknown_node) + .unwrap() + .matches + .is_empty()); + assert!(engine + .explain_pattern_query(&unknown_node) + .unwrap() + .warnings + .contains(&QueryPlanWarning::UnknownNodeLabel)); +} diff --git a/tests/robustness_integration.rs b/tests/robustness_integration.rs index a69e33e..fec486c 100644 --- a/tests/robustness_integration.rs +++ b/tests/robustness_integration.rs @@ -5,6 +5,8 @@ use overgraph::{ use std::collections::BTreeMap; use tempfile::TempDir; +const LARGE_SCALE_LABELS: [&str; 5] = ["Person", "Company", "Article", "Topic", "Project"]; + fn make_props(key: &str, val: &str) -> BTreeMap { let mut m = BTreeMap::new(); m.insert(key.to_string(), PropValue::String(val.to_string())); @@ -30,12 +32,12 @@ fn test_crash_recovery_wal_replay() { let node_b; let edge_ab; - // Phase 1: write data, flush some to segment, leave some in WAL only + // Step 1: write data, flush some to segment, leave some in WAL only { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); node_a = db .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: make_props("role", "admin"), @@ -45,7 +47,7 @@ fn test_crash_recovery_wal_replay() { .unwrap(); node_b = db .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { props: make_props("role", "user"), @@ -55,7 +57,7 @@ fn test_crash_recovery_wal_replay() { ) .unwrap(); edge_ab = db - .upsert_edge(node_a, node_b, 10, UpsertEdgeOptions::default()) + .upsert_edge(node_a, node_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // Flush to segment @@ -63,7 +65,7 @@ fn test_crash_recovery_wal_replay() { // Write more data that stays in WAL only db.upsert_node( - 2, + "Company", "charlie", UpsertNodeOptions { props: make_props("role", "viewer"), @@ -73,7 +75,7 @@ fn test_crash_recovery_wal_replay() { ) .unwrap(); db.upsert_node( - 2, + "Company", "diana", UpsertNodeOptions { props: make_props("role", "editor"), @@ -87,7 +89,7 @@ fn test_crash_recovery_wal_replay() { // (The Drop impl will attempt cleanup but WAL data should be durable) } - // Phase 2: reopen and verify everything + // Step 2: reopen and verify everything { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); @@ -107,13 +109,13 @@ fn test_crash_recovery_wal_replay() { assert_eq!(edge.to, node_b); // WAL-only data should be recovered - let charlie = db.get_node_by_key(2, "charlie").unwrap(); + let charlie = db.get_node_by_key("Company", "charlie").unwrap(); assert!( charlie.is_some(), "WAL-only node 'charlie' should be recovered" ); - let diana = db.get_node_by_key(2, "diana").unwrap(); + let diana = db.get_node_by_key("Company", "diana").unwrap(); assert!(diana.is_some(), "WAL-only node 'diana' should be recovered"); // Neighbors should work across recovered data @@ -143,12 +145,12 @@ fn test_crash_recovery_with_deletes() { { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); node_a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); node_b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); - db.upsert_edge(node_a, node_b, 10, UpsertEdgeOptions::default()) + db.upsert_edge(node_a, node_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); db.flush().unwrap(); @@ -208,7 +210,7 @@ fn test_large_scale_100k_nodes() { let chunk_end = (chunk_start + chunk_size).min(total_nodes); let batch: Vec = (chunk_start..chunk_end) .map(|i| NodeInput { - type_id: (i % 5 + 1) as u32, + labels: vec![LARGE_SCALE_LABELS[i % LARGE_SCALE_LABELS.len()].to_string()], key: format!("node-{}", i), props: { let mut m = BTreeMap::new(); @@ -221,7 +223,7 @@ fn test_large_scale_100k_nodes() { }) .collect(); - let ids = db.batch_upsert_nodes(&batch).unwrap(); + let ids = db.batch_upsert_nodes(batch.clone()).unwrap(); all_ids.extend_from_slice(&ids); // Flush every other chunk to create segments @@ -238,7 +240,7 @@ fn test_large_scale_100k_nodes() { .map(|i| overgraph::EdgeInput { from: all_ids[i], to: all_ids[i + 1], - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, @@ -246,7 +248,7 @@ fn test_large_scale_100k_nodes() { }) .collect(); - db.batch_upsert_edges(&edge_batch).unwrap(); + db.batch_upsert_edges(edge_batch.clone()).unwrap(); db.flush().unwrap(); // Compact all segments @@ -259,13 +261,15 @@ fn test_large_scale_100k_nodes() { let spot = db.get_node(all_ids[50_000]).unwrap().unwrap(); assert_eq!(spot.key, "node-50000"); - // Type query - let type_1_count = db.count_nodes_by_type(1).unwrap(); - assert_eq!(type_1_count, 20_000); // 100k / 5 types + // Label query + let person_count = db.count_nodes_by_labels("Person").unwrap(); + assert_eq!(person_count, 20_000); // 100k / 5 labels // Find nodes - let found = db.find_nodes(3, "idx", &PropValue::Int(42)).unwrap(); - assert!(!found.is_empty() || 42 % 5 + 1 != 3); // only found if type matches + let found = db + .find_nodes("Article", "idx", &PropValue::Int(42)) + .unwrap(); + assert!(!found.is_empty()); // node 42 uses the Article label // Bulk read let sample_ids = &all_ids[0..100]; @@ -306,11 +310,11 @@ fn test_engine_manifest_corruption_recovery() { // Write data and flush to create a manifest { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); - db.upsert_node(1, "a", UpsertNodeOptions::default()) + db.upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); // Write a second manifest version so manifest.prev exists - db.upsert_node(1, "b", UpsertNodeOptions::default()) + db.upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); db.close().unwrap(); @@ -327,7 +331,7 @@ fn test_engine_manifest_corruption_recovery() { // The engine should open successfully (recovered from prev) // We may lose the second flush's manifest entry, but the WAL // should replay and recover the data - let node_a = db.get_node_by_key(1, "a").unwrap(); + let node_a = db.get_node_by_key("Person", "a").unwrap(); assert!(node_a.is_some(), "node 'a' should be recoverable"); db.close().unwrap(); } @@ -355,7 +359,7 @@ fn test_engine_wal_truncated_record_recovery() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); node_a = db .upsert_node( - 1, + "Person", "valid_node", UpsertNodeOptions { props: make_props("k", "v"), @@ -410,23 +414,23 @@ fn test_temporal_edges_cross_source() { let db = DatabaseEngine::open(&db_path, &opts).unwrap(); let a = db - .upsert_node(1, "a", UpsertNodeOptions::default()) + .upsert_node("Person", "a", UpsertNodeOptions::default()) .unwrap(); let b = db - .upsert_node(1, "b", UpsertNodeOptions::default()) + .upsert_node("Person", "b", UpsertNodeOptions::default()) .unwrap(); let c = db - .upsert_node(1, "c", UpsertNodeOptions::default()) + .upsert_node("Person", "c", UpsertNodeOptions::default()) .unwrap(); let d = db - .upsert_node(1, "d", UpsertNodeOptions::default()) + .upsert_node("Person", "d", UpsertNodeOptions::default()) .unwrap(); // Edge in segment: A->B valid [1000, 5000) db.upsert_edge( a, b, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(1000), valid_to: Some(5000), @@ -440,7 +444,7 @@ fn test_temporal_edges_cross_source() { db.upsert_edge( a, c, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(3000), valid_to: Some(9000), @@ -453,7 +457,7 @@ fn test_temporal_edges_cross_source() { db.upsert_edge( a, d, - 10, + "KNOWS", UpsertEdgeOptions { valid_from: Some(0), ..Default::default() @@ -504,7 +508,7 @@ fn test_temporal_edges_cross_source() { assert!(ids.contains(&d), "D (always-valid) should be visible"); // Compact and re-verify - db.upsert_node(1, "filler", UpsertNodeOptions::default()) + db.upsert_node("Person", "filler", UpsertNodeOptions::default()) .unwrap(); db.flush().unwrap(); db.compact().unwrap(); diff --git a/tests/scrub_integration.rs b/tests/scrub_integration.rs new file mode 100644 index 0000000..b293bb1 --- /dev/null +++ b/tests/scrub_integration.rs @@ -0,0 +1,261 @@ +use overgraph::{DatabaseEngine, DbOptions, NodeInput, ScrubFindingType, UpsertEdgeOptions}; +use std::collections::BTreeMap; +use tempfile::TempDir; + +fn open_test_db(dir: &std::path::Path) -> DatabaseEngine { + let opts = DbOptions { + compact_after_n_flushes: 0, + ..DbOptions::default() + }; + let db = DatabaseEngine::open(dir, &opts).unwrap(); + db +} + +fn populate_and_flush(db: &DatabaseEngine) { + let nodes: Vec = (0..10) + .map(|i| NodeInput { + labels: vec!["Person".to_string()], + key: format!("node_{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + let ids = db.batch_upsert_nodes(nodes.clone()).unwrap(); + + for i in 0..5 { + db.upsert_edge( + ids[i], + ids[i + 5], + "RELATES_TO", + UpsertEdgeOptions::default(), + ) + .unwrap(); + } + + db.flush().unwrap(); +} + +#[test] +fn test_scrub_healthy_database_no_findings() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + populate_and_flush(&db); + + let report = db.scrub().unwrap(); + assert_eq!(report.segments.len(), 1); + assert_eq!(report.total_components_failed, 0); + assert!(report.total_components_ok > 0); + assert!(report.total_components_checked > 0); + for seg in &report.segments { + assert!( + seg.findings.is_empty(), + "unexpected findings: {:?}", + seg.findings + ); + } +} + +#[test] +fn test_scrub_healthy_multi_label_database_no_findings() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + let nodes: Vec = (0..6) + .map(|i| NodeInput { + labels: vec![ + "Person".to_string(), + if i % 2 == 0 { "Researcher" } else { "Reviewer" }.to_string(), + ], + key: format!("node_{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + db.batch_upsert_nodes(nodes).unwrap(); + db.flush().unwrap(); + + let report = db.scrub().unwrap(); + assert_eq!(report.segments.len(), 1); + assert_eq!(report.total_components_failed, 0); + for seg in &report.segments { + assert!( + seg.findings.is_empty(), + "unexpected findings: {:?}", + seg.findings + ); + } +} + +#[test] +fn test_scrub_detects_packed_range_corruption() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + populate_and_flush(&db); + db.close().unwrap(); + + let seg_dir = db_path.join("segments").join("seg_0001"); + let core_path = seg_dir.join("segment.core"); + let mut data = std::fs::read(&core_path).unwrap(); + let corrupt_offset = data.len() / 2; + data[corrupt_offset] ^= 0xFF; + std::fs::write(&core_path, &data).unwrap(); + + let db = open_test_db(&db_path); + let report = db.scrub().unwrap(); + assert!(report.total_components_failed > 0); + + let has_digest_mismatch = report + .segments + .iter() + .flat_map(|s| &s.findings) + .any(|f| f.finding_type == ScrubFindingType::PayloadDigestMismatch); + assert!( + has_digest_mismatch, + "expected PayloadDigestMismatch, got: {:?}", + report.segments[0].findings + ); +} + +#[test] +fn test_scrub_detects_external_payload_corruption() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + populate_and_flush(&db); + db.close().unwrap(); + + let seg_dir = db_path.join("segments").join("seg_0001"); + let sidecar_path = find_external_sidecar(&seg_dir) + .expect("test precondition: expected at least one external sidecar after flush with edges"); + let mut data = std::fs::read(&sidecar_path).unwrap(); + assert!( + data.len() > 192, + "test precondition: external sidecar must have identity header + payload" + ); + data[193] ^= 0xFF; + std::fs::write(&sidecar_path, &data).unwrap(); + + let db = open_test_db(&db_path); + let report = db.scrub().unwrap(); + let has_mismatch = report.segments.iter().flat_map(|s| &s.findings).any(|f| { + f.finding_type == ScrubFindingType::PayloadDigestMismatch + || f.finding_type == ScrubFindingType::IdentityHeaderMismatch + }); + assert!( + has_mismatch, + "expected corruption finding, got: {:?}", + report.segments[0].findings + ); +} + +#[test] +fn test_scrub_detects_identity_header_tamper() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + populate_and_flush(&db); + db.close().unwrap(); + + let seg_dir = db_path.join("segments").join("seg_0001"); + let sidecar_path = find_external_sidecar(&seg_dir) + .expect("test precondition: expected at least one external sidecar after flush with edges"); + let mut data = std::fs::read(&sidecar_path).unwrap(); + assert!( + data.len() >= 192, + "test precondition: external sidecar must have identity header" + ); + data[16] ^= 0xFF; + std::fs::write(&sidecar_path, &data).unwrap(); + + let db = open_test_db(&db_path); + let report = db.scrub().unwrap(); + let has_header_mismatch = report + .segments + .iter() + .flat_map(|s| &s.findings) + .any(|f| f.finding_type == ScrubFindingType::IdentityHeaderMismatch); + assert!( + has_header_mismatch, + "expected IdentityHeaderMismatch, got: {:?}", + report.segments[0].findings + ); +} + +#[test] +fn test_scrub_handles_missing_segment_gracefully() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + populate_and_flush(&db); + + let report_before = db.scrub().unwrap(); + assert_eq!(report_before.total_components_failed, 0); + + db.close().unwrap(); + + let seg_dir = db_path.join("segments").join("seg_0001"); + std::fs::remove_dir_all(&seg_dir).unwrap(); + + let db = open_test_db(&db_path); + let report = db.scrub().unwrap(); + assert!(report.total_components_failed > 0); + + let has_missing = report + .segments + .iter() + .flat_map(|s| &s.findings) + .any(|f| f.finding_type == ScrubFindingType::FileMissing); + assert!(has_missing, "expected FileMissing finding"); +} + +#[test] +fn test_scrub_parallel_multiple_segments() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("testdb"); + let db = open_test_db(&db_path); + + for batch in 0..3 { + let nodes: Vec = (0..5) + .map(|i| NodeInput { + labels: vec!["Person".to_string()], + key: format!("batch{batch}_node_{i}"), + props: BTreeMap::new(), + weight: 1.0, + dense_vector: None, + sparse_vector: None, + }) + .collect(); + db.batch_upsert_nodes(nodes.clone()).unwrap(); + db.flush().unwrap(); + } + + let report = db.scrub().unwrap(); + assert_eq!(report.segments.len(), 3); + assert_eq!(report.total_components_failed, 0); + assert!(report.total_components_ok >= 3); + + let segment_ids: Vec = report.segments.iter().map(|s| s.segment_id).collect(); + assert_eq!(segment_ids.len(), 3); + assert!(segment_ids.contains(&1)); + assert!(segment_ids.contains(&2)); + assert!(segment_ids.contains(&3)); +} + +fn find_external_sidecar(seg_dir: &std::path::Path) -> Option { + let entries = std::fs::read_dir(seg_dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name != "segment_manifest.dat" && name != "segment.core" && path.is_file() { + return Some(path); + } + } + } + None +} diff --git a/tests/phase5_integration.rs b/tests/secondary_index_integration.rs similarity index 60% rename from tests/phase5_integration.rs rename to tests/secondary_index_integration.rs index d4d5c96..fda7d16 100644 --- a/tests/phase5_integration.rs +++ b/tests/secondary_index_integration.rs @@ -2,8 +2,8 @@ use overgraph::{DatabaseEngine, DbOptions, NodeInput, PropValue, UpsertEdgeOptio use std::collections::BTreeMap; use tempfile::TempDir; -/// Insert 1000 nodes across 5 types with varied properties across -/// multiple segments, verify find_nodes, nodes_by_type, edges_by_type work +/// Insert 1000 nodes across five labels with varied properties across +/// multiple segments, verify find_nodes and nodes_by_labels work /// correctly through flush, compact, and reopen cycles. #[test] fn test_secondary_indexes_across_flush_compact_reopen() { @@ -19,17 +19,22 @@ fn test_secondary_indexes_across_flush_compact_reopen() { let mut all_node_ids = Vec::new(); - // Create 500 nodes across 5 types with "category" property - let categories = ["alpha", "beta", "gamma", "delta", "epsilon"]; + // Create 500 nodes across five labels with a paired "category" property. + let label_categories = [ + ("Person", "alpha"), + ("Company", "beta"), + ("Article", "gamma"), + ("Topic", "delta"), + ("Project", "epsilon"), + ]; let batch1: Vec = (0..500) .map(|i| { - let type_id = (i % 5) as u32 + 1; - let cat = categories[i % 5]; + let (label, cat) = label_categories[i % label_categories.len()]; let mut props = BTreeMap::new(); props.insert("category".to_string(), PropValue::String(cat.to_string())); props.insert("index".to_string(), PropValue::Int(i as i64)); NodeInput { - type_id, + labels: vec![label.to_string()], key: format!("node:{}", i), props, weight: 0.5, @@ -38,17 +43,19 @@ fn test_secondary_indexes_across_flush_compact_reopen() { } }) .collect(); - let ids1 = engine.batch_upsert_nodes(&batch1).unwrap(); + let ids1 = engine.batch_upsert_nodes(batch1.clone()).unwrap(); all_node_ids.extend_from_slice(&ids1); - // Add edges: chain within each type + // Add edges only when adjacent generated nodes happen to share a label. for i in 0..499 { - if batch1[i].type_id == batch1[i + 1].type_id { + let (label, _) = label_categories[i % label_categories.len()]; + let (next_label, _) = label_categories[(i + 1) % label_categories.len()]; + if label == next_label { engine .upsert_edge( ids1[i], ids1[i + 1], - batch1[i].type_id, + "RELATED_TO", UpsertEdgeOptions::default(), ) .unwrap(); @@ -62,8 +69,7 @@ fn test_secondary_indexes_across_flush_compact_reopen() { // ---- Step 2: Add more nodes, flush to segment 2 ---- let batch2: Vec = (500..1000) .map(|i| { - let type_id = (i % 5) as u32 + 1; - let cat = categories[i % 5]; + let (label, cat) = label_categories[i % label_categories.len()]; let mut props = BTreeMap::new(); props.insert("category".to_string(), PropValue::String(cat.to_string())); props.insert("index".to_string(), PropValue::Int(i as i64)); @@ -75,7 +81,7 @@ fn test_secondary_indexes_across_flush_compact_reopen() { ); } NodeInput { - type_id, + labels: vec![label.to_string()], key: format!("node:{}", i), props, weight: 0.6, @@ -84,7 +90,7 @@ fn test_secondary_indexes_across_flush_compact_reopen() { } }) .collect(); - let ids2 = engine.batch_upsert_nodes(&batch2).unwrap(); + let ids2 = engine.batch_upsert_nodes(batch2.clone()).unwrap(); all_node_ids.extend_from_slice(&ids2); // Flush to segment 2 @@ -93,46 +99,44 @@ fn test_secondary_indexes_across_flush_compact_reopen() { // ---- Step 3: Verify indexes across two segments ---- - // nodes_by_type: each type should have 200 nodes (1000/5) - for type_id in 1..=5 { - let by_type = engine.nodes_by_type(type_id).unwrap(); + // nodes_by_labels: each label should have 200 nodes (1000/5) + for &(label, _) in &label_categories { + let by_label = engine.nodes_by_labels(label).unwrap(); assert_eq!( - by_type.len(), + by_label.len(), 200, - "type {} should have 200 nodes, got {}", - type_id, - by_type.len() + "label '{}' should have 200 nodes, got {}", + label, + by_label.len() ); } - // find_nodes: each (type, category) combo has 200 nodes - // Type 1 has category="alpha", type 2 has "beta", etc. - for (idx, cat) in categories.iter().enumerate() { - let type_id = idx as u32 + 1; + // find_nodes: each paired (label, category) combo has 200 nodes. + for &(label, cat) in &label_categories { let found = engine - .find_nodes(type_id, "category", &PropValue::String(cat.to_string())) + .find_nodes(label, "category", &PropValue::String(cat.to_string())) .unwrap(); assert_eq!( found.len(), 200, - "type {} category '{}' should have 200 nodes, got {}", - type_id, + "label '{}' category '{}' should have 200 nodes, got {}", + label, cat, found.len() ); } // find_nodes with status=active: 250 nodes (500..1000 step 2 = 250), - // distributed across 5 types = 50 per type - for type_id in 1..=5u32 { + // distributed across five labels = 50 per label. + for &(label, _) in &label_categories { let active = engine - .find_nodes(type_id, "status", &PropValue::String("active".to_string())) + .find_nodes(label, "status", &PropValue::String("active".to_string())) .unwrap(); assert_eq!( active.len(), 50, - "type {} active should have 50 nodes, got {}", - type_id, + "label '{}' active should have 50 nodes, got {}", + label, active.len() ); } @@ -156,29 +160,28 @@ fn test_secondary_indexes_across_flush_compact_reopen() { // ---- Step 5: Verify indexes correct after compaction ---- - // The deleted nodes were distributed across 5 types: 20 per type deleted - for type_id in 1..=5 { - let by_type = engine.nodes_by_type(type_id).unwrap(); + // The deleted nodes were distributed across five labels: 20 per label deleted. + for &(label, _) in &label_categories { + let by_label = engine.nodes_by_labels(label).unwrap(); assert_eq!( - by_type.len(), + by_label.len(), 180, - "after compact, type {} should have 180 nodes, got {}", - type_id, - by_type.len() + "after compact, label '{}' should have 180 nodes, got {}", + label, + by_label.len() ); } // find_nodes by category after compaction - for (idx, cat) in categories.iter().enumerate() { - let type_id = idx as u32 + 1; + for &(label, cat) in &label_categories { let found = engine - .find_nodes(type_id, "category", &PropValue::String(cat.to_string())) + .find_nodes(label, "category", &PropValue::String(cat.to_string())) .unwrap(); assert_eq!( found.len(), 180, - "after compact, type {} category '{}' should have 180, got {}", - type_id, + "after compact, label '{}' category '{}' should have 180, got {}", + label, cat, found.len() ); @@ -186,15 +189,15 @@ fn test_secondary_indexes_across_flush_compact_reopen() { // Active status nodes: the first 100 deleted were all from batch1 (0..500), // which had no "status" property. So all 250 active nodes should survive. - for type_id in 1..=5u32 { + for &(label, _) in &label_categories { let active = engine - .find_nodes(type_id, "status", &PropValue::String("active".to_string())) + .find_nodes(label, "status", &PropValue::String("active".to_string())) .unwrap(); assert_eq!( active.len(), 50, - "after compact, type {} active should still have 50, got {}", - type_id, + "after compact, label '{}' active should still have 50, got {}", + label, active.len() ); } @@ -205,32 +208,31 @@ fn test_secondary_indexes_across_flush_compact_reopen() { let engine = DatabaseEngine::open(&db_path, &DbOptions::default()).unwrap(); // Same checks after reopen - for type_id in 1..=5 { + for &(label, _) in &label_categories { assert_eq!( - engine.nodes_by_type(type_id).unwrap().len(), + engine.nodes_by_labels(label).unwrap().len(), 180, - "after reopen, type {} should have 180 nodes", - type_id + "after reopen, label '{}' should have 180 nodes", + label ); } - for (idx, cat) in categories.iter().enumerate() { - let type_id = idx as u32 + 1; + for &(label, cat) in &label_categories { let found = engine - .find_nodes(type_id, "category", &PropValue::String(cat.to_string())) + .find_nodes(label, "category", &PropValue::String(cat.to_string())) .unwrap(); assert_eq!( found.len(), 180, - "after reopen, type {} category '{}' should have 180", - type_id, + "after reopen, label '{}' category '{}' should have 180", + label, cat ); } // Verify a specific node from batch2 is still there with correct props let sample = engine.get_node(ids2[0]).unwrap().unwrap(); - assert_eq!(sample.type_id, 1); // 500 % 5 + 1 = 1 + assert_eq!(sample.labels.as_slice(), ["Person"]); assert_eq!( sample.props.get("category"), Some(&PropValue::String("alpha".to_string())) @@ -240,13 +242,17 @@ fn test_secondary_indexes_across_flush_compact_reopen() { // Verify a deleted node is gone assert!(engine.get_node(all_node_ids[0]).unwrap().is_none()); - // No false positives: wrong type+category combo returns empty + // No false positives: wrong label+category combo returns empty assert!(engine - .find_nodes(1, "category", &PropValue::String("beta".to_string())) + .find_nodes("Person", "category", &PropValue::String("beta".to_string()),) .unwrap() .is_empty()); assert!(engine - .find_nodes(2, "category", &PropValue::String("alpha".to_string())) + .find_nodes( + "Company", + "category", + &PropValue::String("alpha".to_string()), + ) .unwrap() .is_empty()); diff --git a/tests/phase3_integration.rs b/tests/segment_reopen_integration.rs similarity index 90% rename from tests/phase3_integration.rs rename to tests/segment_reopen_integration.rs index 223dd3a..8d24a64 100644 --- a/tests/phase3_integration.rs +++ b/tests/segment_reopen_integration.rs @@ -5,6 +5,8 @@ use overgraph::{ use std::collections::BTreeMap; use tempfile::TempDir; +const LARGE_GRAPH_LABELS: [&str; 5] = ["Person", "Company", "Article", "Topic", "Project"]; + /// Large-scale insert, flush, more writes, cross-source queries. #[test] fn test_large_graph_with_flush_and_cross_source_queries() { @@ -17,7 +19,7 @@ fn test_large_graph_with_flush_and_cross_source_queries() { let mut node_ids = Vec::with_capacity(10_000); let batch: Vec = (0..10_000) .map(|i| overgraph::NodeInput { - type_id: (i % 5) as u32 + 1, // types 1..5 + labels: vec![LARGE_GRAPH_LABELS[i % LARGE_GRAPH_LABELS.len()].to_string()], key: format!("node:{}", i), props: { let mut p = BTreeMap::new(); @@ -29,7 +31,7 @@ fn test_large_graph_with_flush_and_cross_source_queries() { sparse_vector: None, }) .collect(); - node_ids.extend(engine.batch_upsert_nodes(&batch).unwrap()); + node_ids.extend(engine.batch_upsert_nodes(batch.clone()).unwrap()); assert_eq!(node_ids.len(), 10_000); // --- Batch 1: 20k edges (chain + cross-links) --- @@ -39,28 +41,28 @@ fn test_large_graph_with_flush_and_cross_source_queries() { .map(|i| overgraph::EdgeInput { from: node_ids[i], to: node_ids[i + 1], - type_id: 10, + label: "KNOWS".to_string(), props: BTreeMap::new(), weight: 1.0, valid_from: None, valid_to: None, }) .collect(); - edge_ids.extend(engine.batch_upsert_edges(&chain_edges).unwrap()); + edge_ids.extend(engine.batch_upsert_edges(chain_edges.clone()).unwrap()); // Cross-link edges: 10,001 wrapping edges (edge_uniqueness=off, one dup is fine) let cross_edges: Vec = (0..10_001) .map(|i| overgraph::EdgeInput { from: node_ids[i % 10_000], to: node_ids[(i + 100) % 10_000], - type_id: 20, + label: "REFERENCES".to_string(), props: BTreeMap::new(), weight: 0.8, valid_from: None, valid_to: None, }) .collect(); - edge_ids.extend(engine.batch_upsert_edges(&cross_edges).unwrap()); + edge_ids.extend(engine.batch_upsert_edges(cross_edges.clone()).unwrap()); assert_eq!(edge_ids.len(), 20_000); // --- Force flush --- @@ -71,7 +73,7 @@ fn test_large_graph_with_flush_and_cross_source_queries() { // --- Batch 2: 500 more nodes + 1000 edges in memtable --- let batch2: Vec = (10_000..10_500) .map(|i| overgraph::NodeInput { - type_id: 6, + labels: vec!["Session".to_string()], key: format!("node:{}", i), props: BTreeMap::new(), weight: 0.7, @@ -79,21 +81,21 @@ fn test_large_graph_with_flush_and_cross_source_queries() { sparse_vector: None, }) .collect(); - let new_ids = engine.batch_upsert_nodes(&batch2).unwrap(); + let new_ids = engine.batch_upsert_nodes(batch2.clone()).unwrap(); assert_eq!(new_ids.len(), 500); let new_edges: Vec = (0..1000) .map(|i| overgraph::EdgeInput { from: new_ids[i % 500], to: node_ids[i % 10_000], // link new -> old (cross-source) - type_id: 30, + label: "LINKS_TO".to_string(), props: BTreeMap::new(), weight: 0.6, valid_from: None, valid_to: None, }) .collect(); - engine.batch_upsert_edges(&new_edges).unwrap(); + engine.batch_upsert_edges(new_edges.clone()).unwrap(); // --- Cross-source queries --- @@ -112,12 +114,12 @@ fn test_large_graph_with_flush_and_cross_source_queries() { .unwrap(); assert!(out_500.len() >= 2); // at least chain(->501) + cross-link(->600) - // 4. Type-filtered neighbors from segment + // 4. Relationship-filtered neighbors from segment let chain_only = engine .neighbors( node_ids[500], &NeighborOptions { - type_filter: Some(vec![10]), + edge_label_filter: Some(vec!["KNOWS".to_string()]), ..Default::default() }, ) @@ -176,7 +178,7 @@ fn test_flush_close_reopen_reads_from_segments() { // Build a small graph node_a = engine .upsert_node( - 1, + "Person", "alice", UpsertNodeOptions { props: { @@ -192,7 +194,7 @@ fn test_flush_close_reopen_reads_from_segments() { node_b = engine .upsert_node( - 1, + "Person", "bob", UpsertNodeOptions { weight: 0.5, @@ -202,7 +204,7 @@ fn test_flush_close_reopen_reads_from_segments() { .unwrap(); node_c = engine .upsert_node( - 2, + "Company", "charlie", UpsertNodeOptions { weight: 0.6, @@ -212,13 +214,13 @@ fn test_flush_close_reopen_reads_from_segments() { .unwrap(); edge_ab = engine - .upsert_edge(node_a, node_b, 10, UpsertEdgeOptions::default()) + .upsert_edge(node_a, node_b, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); edge_bc = engine .upsert_edge( node_b, node_c, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.8, ..Default::default() @@ -236,7 +238,7 @@ fn test_flush_close_reopen_reads_from_segments() { // Add post-flush data (stays in WAL for replay on reopen) let _node_d = engine .upsert_node( - 1, + "Person", "dave", UpsertNodeOptions { weight: 0.4, @@ -319,7 +321,7 @@ fn test_multi_segment_survives_reopen() { // Segment 1 id_a = engine .upsert_node( - 1, + "Person", "alpha", UpsertNodeOptions { weight: 0.5, @@ -328,14 +330,14 @@ fn test_multi_segment_survives_reopen() { ) .unwrap(); engine - .upsert_edge(id_a, id_a, 10, UpsertEdgeOptions::default()) + .upsert_edge(id_a, id_a, "KNOWS", UpsertEdgeOptions::default()) .unwrap(); // self-loop engine.flush().unwrap(); // Segment 2 id_b = engine .upsert_node( - 1, + "Person", "beta", UpsertNodeOptions { weight: 0.6, @@ -347,7 +349,7 @@ fn test_multi_segment_survives_reopen() { .upsert_edge( id_a, id_b, - 10, + "KNOWS", UpsertEdgeOptions { weight: 0.9, ..Default::default() @@ -359,7 +361,7 @@ fn test_multi_segment_survives_reopen() { // Memtable (will be WAL on reopen) id_c = engine .upsert_node( - 1, + "Person", "gamma", UpsertNodeOptions { weight: 0.7, @@ -371,7 +373,7 @@ fn test_multi_segment_survives_reopen() { .upsert_edge( id_b, id_c, - 20, + "REFERENCES", UpsertEdgeOptions { weight: 0.8, ..Default::default() diff --git a/tools/bench/python_connector_benchmark.py b/tools/bench/python_connector_benchmark.py index a631fb8..441480e 100755 --- a/tools/bench/python_connector_benchmark.py +++ b/tools/bench/python_connector_benchmark.py @@ -209,37 +209,37 @@ def traverse_deep_branching(fanout: int) -> tuple[int, int, int]: def build_depth_two_traversal_graph(db: OverGraph, cfg: dict[str, Any]) -> int: - two_hop_nodes = [{"type_id": 1, "key": "root"}] + two_hop_nodes = [node_input("Person", "root")] for i in range(cfg["two_hop_mid"]): - two_hop_nodes.append({"type_id": 1, "key": f"m-{i}"}) + two_hop_nodes.append(node_input("Person", f"m-{i}")) for j in range(cfg["two_hop_leaves_per_mid"]): - two_hop_nodes.append({"type_id": 1, "key": f"l-{i}-{j}"}) + two_hop_nodes.append(node_input("Person", f"l-{i}-{j}")) two_hop_ids = db.batch_upsert_nodes(two_hop_nodes) root = two_hop_ids[0] mid_stride = 1 + cfg["two_hop_leaves_per_mid"] two_hop_edges = [] for i in range(cfg["two_hop_mid"]): mid_id = two_hop_ids[1 + i * mid_stride] - two_hop_edges.append({"from_id": root, "to_id": mid_id, "type_id": 1, "weight": 1.0}) + two_hop_edges.append({"from_id": root, "to_id": mid_id, "label": "LINKS_TO", "weight": 1.0}) for j in range(cfg["two_hop_leaves_per_mid"]): leaf_id = two_hop_ids[1 + i * mid_stride + 1 + j] - two_hop_edges.append({"from_id": mid_id, "to_id": leaf_id, "type_id": 1, "weight": 1.0}) + two_hop_edges.append({"from_id": mid_id, "to_id": leaf_id, "label": "LINKS_TO", "weight": 1.0}) db.batch_upsert_edges(two_hop_edges) return root def build_deep_traversal_graph(db: OverGraph, fanout: int) -> tuple[int, tuple[int, int, int]]: level1, level2, level3 = traverse_deep_branching(fanout) - nodes = [{"type_id": 1, "key": "root"}] + nodes = [node_input("Person", "root")] for i in range(level1): - nodes.append({"type_id": 11, "key": f"lvl1-{i}"}) + nodes.append(node_input("LevelOne", f"lvl1-{i}")) for i in range(level1): for j in range(level2): - nodes.append({"type_id": 2 if (i + j) % 2 == 0 else 3, "key": f"lvl2-{i}-{j}"}) + nodes.append(node_input("Company" if (i + j) % 2 == 0 else "Document", f"lvl2-{i}-{j}")) for i in range(level1): for j in range(level2): for k in range(level3): - nodes.append({"type_id": 2 if (i + j + k) % 2 == 0 else 3, "key": f"lvl3-{i}-{j}-{k}"}) + nodes.append(node_input("Company" if (i + j + k) % 2 == 0 else "Document", f"lvl3-{i}-{j}-{k}")) ids = db.batch_upsert_nodes(nodes) root = ids[0] level1_offset = 1 @@ -248,14 +248,14 @@ def build_deep_traversal_graph(db: OverGraph, fanout: int) -> tuple[int, tuple[i edges = [] for i in range(level1): lvl1_id = ids[level1_offset + i] - edges.append({"from_id": root, "to_id": lvl1_id, "type_id": 1, "weight": 1.0}) + edges.append({"from_id": root, "to_id": lvl1_id, "label": "LINKS_TO", "weight": 1.0}) for j in range(level2): lvl2_idx = i * level2 + j lvl2_id = ids[level2_offset + lvl2_idx] - edges.append({"from_id": lvl1_id, "to_id": lvl2_id, "type_id": 1, "weight": 1.0}) + edges.append({"from_id": lvl1_id, "to_id": lvl2_id, "label": "LINKS_TO", "weight": 1.0}) for k in range(level3): lvl3_idx = lvl2_idx * level3 + k - edges.append({"from_id": lvl2_id, "to_id": ids[level3_offset + lvl3_idx], "type_id": 1, "weight": 1.0}) + edges.append({"from_id": lvl2_id, "to_id": ids[level3_offset + lvl3_idx], "label": "LINKS_TO", "weight": 1.0}) db.batch_upsert_edges(edges) return root, (level1, level2, level3) @@ -292,16 +292,29 @@ def bench_sparse_vector(dim_count: int, nnz: int, seed: int) -> list[tuple[int, return [(d, 1.0 - i * 0.05) for i, d in enumerate(dims)] +def node_input(label: str, key: str, **fields: Any) -> dict[str, Any]: + return {"labels": [label], "key": key, **fields} + + +def node_label_filter(label: str) -> dict[str, Any]: + return {"labels": [label], "mode": "all"} + + def pack_node_batch(nodes: list[dict[str, Any]]) -> bytes: """Pack node dicts using the Python connector binary wire format.""" - buf = bytearray(struct.pack(" None: raise RuntimeError(f"timed out waiting for property index {index_id} to become ready") +def wait_for_edge_property_index_ready(db: OverGraph, index_id: int) -> None: + deadline = time.monotonic() + 10.0 + while time.monotonic() < deadline: + if any(info.index_id == index_id and info.state == "ready" for info in db.list_edge_property_indexes()): + return + time.sleep(0.01) + raise RuntimeError(f"timed out waiting for edge property index {index_id} to become ready") + + def query_benchmark_layout(preload_nodes: int) -> dict[str, int]: segments = 1 if preload_nodes >= 2 else 0 segment_nodes = 0 if segments == 0 else max(1, preload_nodes // (segments + 1)) @@ -337,22 +359,18 @@ def query_benchmark_layout(preload_nodes: int) -> dict[str, int]: def query_bench_nodes(start: int, count: int) -> list[dict[str, Any]]: return [ - { - "type_id": 1, - "key": f"q-{i}", - "props": query_bench_props(i), - } + node_input("Person", f"q-{i}", props=query_bench_props(i)) for i in range(start, start + count) ] def build_query_benchmark_db(path: Path, preload_nodes: int) -> tuple[OverGraph, dict[str, int]]: db = OverGraph.open(str(path)) - status = db.ensure_node_property_index(1, "status", "equality") + status = db.ensure_node_property_index("Person", "status", "equality") wait_for_property_index_ready(db, status.index_id) - tier = db.ensure_node_property_index(1, "tier", "equality") + tier = db.ensure_node_property_index("Person", "tier", "equality") wait_for_property_index_ready(db, tier.index_id) - score = db.ensure_node_property_index(1, "score", "range", domain="int") + score = db.ensure_node_property_index("Person", "score", "range", domain="int") wait_for_property_index_ready(db, score.index_id) layout = query_benchmark_layout(preload_nodes) @@ -365,6 +383,62 @@ def build_query_benchmark_db(path: Path, preload_nodes: int) -> tuple[OverGraph, return db, layout +def build_edge_query_benchmark_db(path: Path, preload_edges: int) -> tuple[OverGraph, dict[str, int], int]: + db = OverGraph.open(str(path)) + source_count = 1 + target_count = max(1, preload_edges) + nodes = [node_input("Person", f"edge-source-{i}") for i in range(source_count)] + nodes.extend(node_input("Company", f"edge-target-{i}") for i in range(target_count)) + ids = db.batch_upsert_nodes(nodes) + source_ids = ids[:source_count] + target_ids = ids[source_count:] + source_id = source_ids[0] + segments = 1 if preload_edges >= 2 else 0 + segment_edges = 0 if segments == 0 else max(1, preload_edges // 2) + memtable_tail_edges = max(0, preload_edges - segment_edges) + + def make_edges(start: int, count: int) -> list[dict[str, Any]]: + edges = [] + for i in range(start, start + count): + edges.append( + { + "from_id": source_ids[i % source_count], + "to_id": target_ids[i % len(target_ids)], + "label": "WORKS_AT", + "props": {"role": "lead" if i % 10 == 0 else "member", "score": i % 100}, + "weight": 2.0 if i % 2 == 0 else 0.5, + } + ) + return edges + + if segment_edges > 0: + db.batch_upsert_edges(make_edges(0, segment_edges)) + db.flush() + if memtable_tail_edges > 0: + db.batch_upsert_edges(make_edges(segment_edges, memtable_tail_edges)) + return ( + db, + { + "segments": segments, + "segment_edges": segment_edges, + "memtable_tail_edges": memtable_tail_edges, + }, + source_id, + ) + + +def build_indexed_edge_query_benchmark_db( + path: Path, + preload_edges: int, +) -> tuple[OverGraph, dict[str, int], int]: + db, layout, source_id = build_edge_query_benchmark_db(path, preload_edges) + role = db.ensure_edge_property_index("WORKS_AT", "role", "equality") + wait_for_edge_property_index_ready(db, role.index_id) + score = db.ensure_edge_property_index("WORKS_AT", "score", "range", domain="int") + wait_for_edge_property_index_ready(db, score.index_id) + return db, layout, source_id + + def push_query_scenarios( args: argparse.Namespace, scenario_contract: dict[str, Any], @@ -381,10 +455,12 @@ def push_query_scenarios( s = run_bench( lambda _i: db.query_node_ids( { - "type_id": 1, - "where": { - "status": {"eq": "active"}, - "tier": {"eq": "gold"}, + "label_filter": node_label_filter("Person"), + "filter": { + "and": [ + {"property": "status", "eq": "active"}, + {"property": "tier", "eq": "gold"}, + ], }, "limit": limit, } @@ -400,7 +476,7 @@ def push_query_scenarios( s, iter_cfg, { - "type_id": 1, + "label": "Person", "preload_nodes": preload_nodes, "segments": layout["segments"], "segment_nodes": layout["segment_nodes"], @@ -413,16 +489,224 @@ def push_query_scenarios( ) db.close() + scenario_id = "S-QUERY-005" + iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) + db, layout, source_id = build_indexed_edge_query_benchmark_db( + tmp_root / "query-edge-ids-property-indexed-equality", preload_nodes + ) + s = run_bench( + lambda _i: db.query_edge_ids( + { + "label": "WORKS_AT", + "from_ids": [source_id], + "filter": {"property": "role", "eq": "lead"}, + "limit": limit, + } + ), + iter_cfg["warmup"], + iter_cfg["iters"], + ) + scenarios.append( + scenario( + scenario_id, + "query_edge_ids_property_indexed_equality", + "query", + s, + iter_cfg, + { + "label": "WORKS_AT", + "preload_edges": preload_nodes, + "segments": layout["segments"], + "segment_edges": layout["segment_edges"], + "memtable_tail_edges": layout["memtable_tail_edges"], + "filter": "role_eq_lead", + "limit": limit, + }, + scenario_comparability(scenario_contract, scenario_id), + ) + ) + db.close() + + scenario_id = "S-QUERY-006" + iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) + db, layout, source_id = build_indexed_edge_query_benchmark_db( + tmp_root / "query-edge-ids-property-indexed-range", preload_nodes + ) + s = run_bench( + lambda _i: db.query_edge_ids( + { + "label": "WORKS_AT", + "from_ids": [source_id], + "filter": {"property": "score", "gte": 90}, + "limit": limit, + } + ), + iter_cfg["warmup"], + iter_cfg["iters"], + ) + scenarios.append( + scenario( + scenario_id, + "query_edge_ids_property_indexed_range", + "query", + s, + iter_cfg, + { + "label": "WORKS_AT", + "preload_edges": preload_nodes, + "segments": layout["segments"], + "segment_edges": layout["segment_edges"], + "memtable_tail_edges": layout["memtable_tail_edges"], + "filter": "score_gte_90", + "limit": limit, + }, + scenario_comparability(scenario_contract, scenario_id), + ) + ) + db.close() + + scenario_id = "S-QUERY-007" + iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) + db, layout, _source_id = build_indexed_edge_query_benchmark_db( + tmp_root / "query-pattern-edge-property-anchor-indexed", preload_nodes + ) + s = run_bench( + lambda _i: db.query_pattern( + { + "nodes": [ + {"alias": "source", "label_filter": node_label_filter("Person")}, + {"alias": "target", "label_filter": node_label_filter("Company")}, + ], + "edges": [ + { + "alias": "edge", + "from_alias": "source", + "to_alias": "target", + "direction": "outgoing", + "label_filter": ["WORKS_AT"], + "filter": {"property": "role", "eq": "lead"}, + } + ], + "limit": limit, + } + ), + iter_cfg["warmup"], + iter_cfg["iters"], + ) + scenarios.append( + scenario( + scenario_id, + "query_pattern_edge_property_anchor_indexed", + "query", + s, + iter_cfg, + { + "label": "WORKS_AT", + "preload_edges": preload_nodes, + "segments": layout["segments"], + "segment_edges": layout["segment_edges"], + "memtable_tail_edges": layout["memtable_tail_edges"], + "filter": "role_eq_lead", + "limit": limit, + }, + scenario_comparability(scenario_contract, scenario_id), + ) + ) + db.close() + + scenario_id = "S-QUERY-003" + iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) + db, layout, source_id = build_edge_query_benchmark_db( + tmp_root / "query-edge-ids-endpoint-metadata", preload_nodes + ) + s = run_bench( + lambda _i: db.query_edge_ids( + { + "label": "WORKS_AT", + "from_ids": [source_id], + "filter": {"weight": {"gte": 1.0}}, + "limit": limit, + } + ), + iter_cfg["warmup"], + iter_cfg["iters"], + ) + scenarios.append( + scenario( + scenario_id, + "query_edge_ids_endpoint_metadata", + "query", + s, + iter_cfg, + { + "label": "WORKS_AT", + "preload_edges": preload_nodes, + "segments": layout["segments"], + "segment_edges": layout["segment_edges"], + "memtable_tail_edges": layout["memtable_tail_edges"], + "filter": "weight_gte_1", + "limit": limit, + }, + scenario_comparability(scenario_contract, scenario_id), + ) + ) + db.close() + + scenario_id = "S-QUERY-004" + iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) + db, layout, source_id = build_edge_query_benchmark_db( + tmp_root / "query-edges-endpoint-property-hydrated", preload_nodes + ) + s = run_bench( + lambda _i: db.query_edges( + { + "label": "WORKS_AT", + "from_ids": [source_id], + "filter": { + "and": [ + {"weight": {"gte": 1.0}}, + {"property": "role", "eq": "lead"}, + ] + }, + "limit": limit, + } + ), + iter_cfg["warmup"], + iter_cfg["iters"], + ) + scenarios.append( + scenario( + scenario_id, + "query_edges_endpoint_property_hydrated", + "query", + s, + iter_cfg, + { + "label": "WORKS_AT", + "preload_edges": preload_nodes, + "segments": layout["segments"], + "segment_edges": layout["segment_edges"], + "memtable_tail_edges": layout["memtable_tail_edges"], + "filter": "weight_gte_1_and_role_eq_lead", + "limit": limit, + }, + scenario_comparability(scenario_contract, scenario_id), + ) + ) + db.close() + scenario_id = "S-QUERY-002" iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db, layout = build_query_benchmark_db(tmp_root / "query-nodes-hydrated-intersected", preload_nodes) s = run_bench( lambda _i: db.query_nodes( { - "type_id": 1, - "where": { - "status": {"eq": "active"}, - "score": {"gte": 50}, + "label_filter": node_label_filter("Person"), + "filter": { + "and": [ + {"property": "status", "eq": "active"}, + {"property": "score", "gte": 50}, + ], }, "limit": limit, } @@ -438,7 +722,7 @@ def push_query_scenarios( s, iter_cfg, { - "type_id": 1, + "label": "Person", "preload_nodes": preload_nodes, "segments": layout["segments"], "segment_nodes": layout["segment_nodes"], @@ -488,7 +772,7 @@ def main() -> int: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "crud-upsert-node")) s = run_bench( - lambda i: db.upsert_node(1, f"node-{i}", props={"idx": i}, weight=1.0), + lambda i: db.upsert_node("Person", f"node-{i}", props={"idx": i}, weight=1.0), iter_cfg["warmup"], iter_cfg["iters"], growth=True, @@ -500,7 +784,7 @@ def main() -> int: "crud", s, iter_cfg, - {"type_id": 1, "with_props": True, "weight": 1.0}, + {"label": "Person", "with_props": True, "weight": 1.0}, scenario_comparability(scenario_contract, scenario_id), ) ) @@ -511,10 +795,10 @@ def main() -> int: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "crud-upsert-edge")) node_ids = db.batch_upsert_nodes( - [{"type_id": 1, "key": f"e-{i}"} for i in range(iter_cfg["warmup"] + iter_cfg["iters"] + 1)] + [node_input("Person", f"e-{i}") for i in range(iter_cfg["warmup"] + iter_cfg["iters"] + 1)] ) s = run_bench( - lambda i: db.upsert_edge(node_ids[i], node_ids[i + 1], 1, weight=1.0), + lambda i: db.upsert_edge(node_ids[i], node_ids[i + 1], "LINKS_TO", weight=1.0), iter_cfg["warmup"], iter_cfg["iters"], growth=True, @@ -526,7 +810,7 @@ def main() -> int: "crud", s, iter_cfg, - {"edge_type_id": 1, "weight": 1.0}, + {"label": "LINKS_TO", "weight": 1.0}, scenario_comparability(scenario_contract, scenario_id), ) ) @@ -539,7 +823,7 @@ def main() -> int: s = run_bench( lambda i: db.batch_upsert_nodes( [ - {"type_id": 1, "key": f"bn-{i}-{j}", "props": {"idx": j}, "weight": 1.0} + node_input("Person", f"bn-{i}-{j}", props={"idx": j}, weight=1.0) for j in range(cfg["batch_nodes"]) ] ), @@ -553,7 +837,7 @@ def main() -> int: "batch", s, iter_cfg, - {"batch_nodes": cfg["batch_nodes"], "type_id": 1, "with_props": True}, + {"batch_nodes": cfg["batch_nodes"], "label": "Person", "with_props": True}, scenario_comparability(scenario_contract, scenario_id), cfg["batch_nodes"], ) @@ -567,7 +851,7 @@ def main() -> int: def run_batch_binary(i: int) -> None: nodes = [ - {"type_id": 1, "key": f"bb-{i}-{j}", "props": {"idx": j}, "weight": 1.0} + node_input("Person", f"bb-{i}-{j}", props={"idx": j}, weight=1.0) for j in range(cfg["batch_nodes"]) ] db.batch_upsert_nodes_binary(pack_node_batch(nodes)) @@ -592,7 +876,7 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "crud-get-node")) ids = db.batch_upsert_nodes( - [{"type_id": 1, "key": f"gn-{i}", "props": {"idx": i}} for i in range(cfg["get_node_nodes"])] + [node_input("Person", f"gn-{i}", props={"idx": i}) for i in range(cfg["get_node_nodes"])] ) s = run_bench(lambda i: db.get_node(ids[i % len(ids)]), iter_cfg["warmup"], iter_cfg["iters"]) scenarios.append( @@ -612,9 +896,9 @@ def run_batch_binary(i: int) -> None: scenario_id = "S-CRUD-004" iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "crud-upsert-node-fixed")) - db.upsert_node(1, "fixed-node", props={"idx": 0}, weight=1.0) + db.upsert_node("Person", "fixed-node", props={"idx": 0}, weight=1.0) s = run_bench( - lambda i: db.upsert_node(1, "fixed-node", props={"idx": i}, weight=1.0), + lambda i: db.upsert_node("Person", "fixed-node", props={"idx": i}, weight=1.0), iter_cfg["warmup"], iter_cfg["iters"], ) @@ -625,7 +909,7 @@ def run_batch_binary(i: int) -> None: "crud", s, iter_cfg, - {"type_id": 1, "with_props": True, "weight": 1.0, "fixed_key": True}, + {"label": "Person", "with_props": True, "weight": 1.0, "fixed_key": True}, scenario_comparability(scenario_contract, scenario_id), ) ) @@ -635,10 +919,10 @@ def run_batch_binary(i: int) -> None: scenario_id = "S-CRUD-005" iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "crud-upsert-edge-fixed"), edge_uniqueness=True) - node_a = db.upsert_node(1, "fixed-a") - node_b = db.upsert_node(1, "fixed-b") + node_a = db.upsert_node("Person", "fixed-a") + node_b = db.upsert_node("Person", "fixed-b") s = run_bench( - lambda _i: db.upsert_edge(node_a, node_b, 1, weight=1.0), + lambda _i: db.upsert_edge(node_a, node_b, "LINKS_TO", weight=1.0), iter_cfg["warmup"], iter_cfg["iters"], ) @@ -649,7 +933,7 @@ def run_batch_binary(i: int) -> None: "crud", s, iter_cfg, - {"edge_type_id": 1, "weight": 1.0, "edge_uniqueness": True, "fixed_triple": True}, + {"label": "LINKS_TO", "weight": 1.0, "edge_uniqueness": True, "fixed_triple": True}, scenario_comparability(scenario_contract, scenario_id), ) ) @@ -660,12 +944,12 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "trav-neighbors")) nb_node_ids = db.batch_upsert_nodes( - [{"type_id": 1, "key": "hub"}] - + [{"type_id": 1, "key": f"n-{i}"} for i in range(cfg["fanout"])] + [node_input("Person", "hub")] + + [node_input("Person", f"n-{i}") for i in range(cfg["fanout"])] ) hub = nb_node_ids[0] db.batch_upsert_edges([ - {"from_id": hub, "to_id": nb_node_ids[1 + i], "type_id": 1, "weight": 1.0} + {"from_id": hub, "to_id": nb_node_ids[1 + i], "label": "LINKS_TO", "weight": 1.0} for i in range(cfg["fanout"]) ]) s = run_bench(lambda _i: db.neighbors(hub, direction="outgoing"), iter_cfg["warmup"], iter_cfg["iters"]) @@ -733,7 +1017,7 @@ def run_batch_binary(i: int) -> None: "layout": "memtable", "min_depth": 1, "max_depth": 3, - "node_type_filter": None, + "node_label_filter": None, "branching": list(branching), }, scenario_comparability(scenario_contract, scenario_id), @@ -764,7 +1048,7 @@ def run_batch_binary(i: int) -> None: "layout": "segment", "min_depth": 1, "max_depth": 3, - "node_type_filter": None, + "node_label_filter": None, "branching": list(branching), }, scenario_comparability(scenario_contract, scenario_id), @@ -783,7 +1067,7 @@ def run_batch_binary(i: int) -> None: min_depth=1, max_depth=3, direction="outgoing", - node_type_filter=[2], + emit_node_label_filter=node_label_filter("Company"), ), iter_cfg["warmup"], iter_cfg["iters"], @@ -800,7 +1084,7 @@ def run_batch_binary(i: int) -> None: "layout": "memtable", "min_depth": 1, "max_depth": 3, - "node_type_filter": [2], + "node_label_filter": ["Company"], "branching": list(branching), }, scenario_comparability(scenario_contract, scenario_id), @@ -820,7 +1104,7 @@ def run_batch_binary(i: int) -> None: min_depth=1, max_depth=3, direction="outgoing", - node_type_filter=[2], + emit_node_label_filter=node_label_filter("Company"), ), iter_cfg["warmup"], iter_cfg["iters"], @@ -837,7 +1121,7 @@ def run_batch_binary(i: int) -> None: "layout": "segment", "min_depth": 1, "max_depth": 3, - "node_type_filter": [2], + "node_label_filter": ["Company"], "branching": list(branching), }, scenario_comparability(scenario_contract, scenario_id), @@ -850,12 +1134,12 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "trav-degree")) deg_node_ids = db.batch_upsert_nodes( - [{"type_id": 1, "key": "hub"}] - + [{"type_id": 1, "key": f"d-{i}"} for i in range(cfg["fanout"])] + [node_input("Person", "hub")] + + [node_input("Person", f"d-{i}") for i in range(cfg["fanout"])] ) hub = deg_node_ids[0] db.batch_upsert_edges([ - {"from_id": hub, "to_id": deg_node_ids[1 + i], "type_id": 1, "weight": 1.0} + {"from_id": hub, "to_id": deg_node_ids[1 + i], "label": "LINKS_TO", "weight": 1.0} for i in range(cfg["fanout"]) ]) s = run_bench(lambda _i: db.degree(hub, direction="outgoing"), iter_cfg["warmup"], iter_cfg["iters"]) @@ -877,10 +1161,10 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "trav-degrees")) # Batch all nodes: hubs first, then fanout nodes for each hub - all_degree_nodes = [{"type_id": 1, "key": f"hub-{h}"} for h in range(cfg["batch_nodes"])] + all_degree_nodes = [node_input("Person", f"hub-{h}") for h in range(cfg["batch_nodes"])] for h in range(cfg["batch_nodes"]): for i in range(cfg["fanout"]): - all_degree_nodes.append({"type_id": 1, "key": f"dt-{h}-{i}"}) + all_degree_nodes.append(node_input("Person", f"dt-{h}-{i}")) all_degree_ids = db.batch_upsert_nodes(all_degree_nodes) hub_ids = all_degree_ids[: cfg["batch_nodes"]] # Batch all edges: each hub connects to its fanout nodes @@ -892,7 +1176,7 @@ def run_batch_binary(i: int) -> None: degree_edges.append({ "from_id": hub_id, "to_id": all_degree_ids[fanout_start + i], - "type_id": 1, + "label": "LINKS_TO", "weight": 1.0, }) db.batch_upsert_edges(degree_edges) @@ -918,7 +1202,7 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "trav-shortest-path")) sp_nodes = [ - {"type_id": 1, "key": f"sp-{i}", "weight": 1.0} for i in range(cfg["shortest_path_nodes"]) + node_input("Person", f"sp-{i}", weight=1.0) for i in range(cfg["shortest_path_nodes"]) ] sp_ids = db.batch_upsert_nodes(sp_nodes) offsets = cfg["shortest_path_edge_offsets"] @@ -927,8 +1211,8 @@ def run_batch_binary(i: int) -> None: from_id = sp_ids[i] to1 = sp_ids[(i + offsets[0]) % len(sp_ids)] to2 = sp_ids[(i + offsets[1]) % len(sp_ids)] - sp_edges.append({"from_id": from_id, "to_id": to1, "type_id": 1, "weight": 1.0}) - sp_edges.append({"from_id": from_id, "to_id": to2, "type_id": 1, "weight": 1.0}) + sp_edges.append({"from_id": from_id, "to_id": to1, "label": "LINKS_TO", "weight": 1.0}) + sp_edges.append({"from_id": from_id, "to_id": to2, "label": "LINKS_TO", "weight": 1.0}) db.batch_upsert_edges(sp_edges) sp_from = sp_ids[0] sp_to = sp_ids[len(sp_ids) // 2] @@ -960,7 +1244,7 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "trav-is-connected")) ic_nodes = [ - {"type_id": 1, "key": f"ic-{i}", "weight": 1.0} for i in range(cfg["shortest_path_nodes"]) + node_input("Person", f"ic-{i}", weight=1.0) for i in range(cfg["shortest_path_nodes"]) ] ic_ids = db.batch_upsert_nodes(ic_nodes) offsets = cfg["shortest_path_edge_offsets"] @@ -969,8 +1253,8 @@ def run_batch_binary(i: int) -> None: from_id = ic_ids[i] to1 = ic_ids[(i + offsets[0]) % len(ic_ids)] to2 = ic_ids[(i + offsets[1]) % len(ic_ids)] - ic_edges.append({"from_id": from_id, "to_id": to1, "type_id": 1, "weight": 1.0}) - ic_edges.append({"from_id": from_id, "to_id": to2, "type_id": 1, "weight": 1.0}) + ic_edges.append({"from_id": from_id, "to_id": to1, "label": "LINKS_TO", "weight": 1.0}) + ic_edges.append({"from_id": from_id, "to_id": to2, "label": "LINKS_TO", "weight": 1.0}) db.batch_upsert_edges(ic_edges) ic_from = ic_ids[0] ic_to = ic_ids[len(ic_ids) // 2] @@ -1001,13 +1285,13 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "adv-top-k")) tk_node_ids = db.batch_upsert_nodes( - [{"type_id": 1, "key": "hub"}] - + [{"type_id": 1, "key": f"tk-{i}"} for i in range(cfg["top_k_candidates"])] + [node_input("Person", "hub")] + + [node_input("Person", f"tk-{i}") for i in range(cfg["top_k_candidates"])] ) hub = tk_node_ids[0] tk_candidate_ids = tk_node_ids[1:] db.batch_upsert_edges([ - {"from_id": hub, "to_id": tk_candidate_ids[i], "type_id": 1, "weight": 1.0 + ((i % 100) / 10.0)} + {"from_id": hub, "to_id": tk_candidate_ids[i], "label": "LINKS_TO", "weight": 1.0 + ((i % 100) / 10.0)} for i in range(cfg["top_k_candidates"]) ]) s = run_bench( @@ -1038,13 +1322,13 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "adv-time-range")) db.batch_upsert_nodes([ - {"type_id": 1, "key": f"tr-{i}", "props": {"idx": i}, "weight": 1.0} + node_input("Person", f"tr-{i}", props={"idx": i}, weight=1.0) for i in range(cfg["time_range_nodes"]) ]) from_ms = cfg["time_range_from_ms"] to_ms = int(time.time() * 1000) + cfg["time_range_window_ms"] s = run_bench( - lambda _i: db.find_nodes_by_time_range(1, from_ms, to_ms), + lambda _i: db.find_nodes_by_time_range("Person", from_ms, to_ms), iter_cfg["warmup"], iter_cfg["iters"], ) @@ -1056,7 +1340,7 @@ def run_batch_binary(i: int) -> None: s, iter_cfg, { - "type_id": 1, + "label": "Person", "preload_nodes": cfg["time_range_nodes"], "from_ms": cfg["time_range_from_ms"], "to_ms_window": cfg["time_range_window_ms"], @@ -1070,13 +1354,13 @@ def run_batch_binary(i: int) -> None: scenario_id = "S-ADV-004" iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "adv-ppr")) - ids = db.batch_upsert_nodes([{"type_id": 1, "key": f"ppr-{i}"} for i in range(cfg["ppr_nodes"])]) + ids = db.batch_upsert_nodes([node_input("Person", f"ppr-{i}") for i in range(cfg["ppr_nodes"])]) ppr_edges = [] for i, from_id in enumerate(ids): to1 = ids[(i + cfg["ppr_edge_offsets"][0]) % len(ids)] to2 = ids[(i + cfg["ppr_edge_offsets"][1]) % len(ids)] - ppr_edges.append({"from_id": from_id, "to_id": to1, "type_id": 1, "weight": 1.0}) - ppr_edges.append({"from_id": from_id, "to_id": to2, "type_id": 1, "weight": 0.7}) + ppr_edges.append({"from_id": from_id, "to_id": to1, "label": "LINKS_TO", "weight": 1.0}) + ppr_edges.append({"from_id": from_id, "to_id": to2, "label": "LINKS_TO", "weight": 0.7}) db.batch_upsert_edges(ppr_edges) seed = ids[0] s = run_bench( @@ -1113,14 +1397,14 @@ def run_batch_binary(i: int) -> None: iter_cfg = scenario_iterations(args.warmup, args.iters, scenario_contract, scenario_id) db = OverGraph.open(str(tmp_root / "adv-export")) ids = db.batch_upsert_nodes( - [{"type_id": 1, "key": f"ex-{i}"} for i in range(cfg["export_nodes"])] + [node_input("Person", f"ex-{i}") for i in range(cfg["export_nodes"])] ) export_edges = [] for i in range(cfg["export_edges"]): from_id = ids[i % len(ids)] to_id = ids[(i * 13 + 7) % len(ids)] if from_id != to_id: - export_edges.append({"from_id": from_id, "to_id": to_id, "type_id": 1, "weight": 1.0}) + export_edges.append({"from_id": from_id, "to_id": to_id, "label": "LINKS_TO", "weight": 1.0}) db.batch_upsert_edges(export_edges) s = run_bench( lambda _i: db.export_adjacency(include_weights=cfg["include_weights_on_export"]), @@ -1151,7 +1435,7 @@ def run_batch_binary(i: int) -> None: def run_flush(i: int) -> None: nodes = [ - {"type_id": 1, "key": f"fl-{i}-{j}", "props": {"idx": j}, "weight": 1.0} + node_input("Person", f"fl-{i}-{j}", props={"idx": j}, weight=1.0) for j in range(cfg["flush_nodes_per_iter"]) ] node_ids = db.batch_upsert_nodes(nodes) @@ -1161,7 +1445,7 @@ def run_flush(i: int) -> None: { "from_id": node_ids[j], "to_id": node_ids[j + 1], - "type_id": 1, + "label": "LINKS_TO", "weight": 1.0, } ) @@ -1198,14 +1482,14 @@ def run_flush(i: int) -> None: for i in range(cfg["vector_nodes"]): seed = 1729 * (i + 1) vec_nodes.append( - { - "type_id": 1, - "key": f"v-{i}", - "dense_vector": bench_dense_vector(cfg["vector_dim"], seed), - "sparse_vector": bench_sparse_vector( + node_input( + "Person", + f"v-{i}", + dense_vector=bench_dense_vector(cfg["vector_dim"], seed), + sparse_vector=bench_sparse_vector( cfg["vector_sparse_dims"], cfg["vector_nnz"], seed + 0xCAFE ), - } + ) ) db.batch_upsert_nodes(vec_nodes) db.flush() diff --git a/tools/bench/txn_node_benchmark.mjs b/tools/bench/txn_node_benchmark.mjs index 1723ea3..0b6ae65 100644 --- a/tools/bench/txn_node_benchmark.mjs +++ b/tools/bench/txn_node_benchmark.mjs @@ -33,7 +33,7 @@ function runBench(fn, warmup, iters) { function txnOps(batch, nodeCount, edgeCount) { const ops = []; for (let i = 0; i < nodeCount; i++) { - ops.push({ op: 'upsertNode', alias: `n${i}`, typeId: 1, key: `txn:${batch}:n:${i}` }); + ops.push({ op: 'upsertNode', alias: `n${i}`, label: 'Person', key: `txn:${batch}:n:${i}` }); } for (let i = 0; i < edgeCount; i++) { ops.push({ @@ -41,7 +41,7 @@ function txnOps(batch, nodeCount, edgeCount) { alias: `e${i}`, from: { local: `n${i % nodeCount}` }, to: { local: `n${(i + 1) % nodeCount}` }, - typeId: 7, + label: 'WORKS_AT', }); } return ops; @@ -72,10 +72,10 @@ function main() { scenarios.push({ scenario_id: 'S-TXN-002', stats: runBench((i) => { - db.upsertNode(9, 'conflict', { props: { i } }); + db.upsertNode('Person', 'conflict', { props: { i } }); const txn = db.beginWriteTxn(); - txn.upsertNode(9, 'conflict', { props: { txn: i } }); - db.upsertNode(9, 'conflict', { props: { other: i } }); + txn.upsertNode('Person', 'conflict', { props: { txn: i } }); + db.upsertNode('Person', 'conflict', { props: { other: i } }); try { txn.commit(); throw new Error('expected conflict'); diff --git a/tools/bench/txn_python_benchmark.py b/tools/bench/txn_python_benchmark.py index 8f7d265..c439fea 100644 --- a/tools/bench/txn_python_benchmark.py +++ b/tools/bench/txn_python_benchmark.py @@ -40,7 +40,7 @@ def run_bench(fn, warmup: int, iters: int) -> dict[str, float]: def txn_ops(batch: int, node_count: int, edge_count: int) -> list[dict]: ops = [ - {"op": "upsert_node", "alias": f"n{i}", "type_id": 1, "key": f"txn:{batch}:n:{i}"} + {"op": "upsert_node", "alias": f"n{i}", "label": "Person", "key": f"txn:{batch}:n:{i}"} for i in range(node_count) ] ops.extend( @@ -49,7 +49,7 @@ def txn_ops(batch: int, node_count: int, edge_count: int) -> list[dict]: "alias": f"e{i}", "from": {"local": f"n{i % node_count}"}, "to": {"local": f"n{(i + 1) % node_count}"}, - "type_id": 7, + "label": "WORKS_AT", } for i in range(edge_count) ) @@ -107,10 +107,10 @@ def commit_ops(db: OverGraph, i: int, nodes: int, edges: int) -> None: def conflict(db: OverGraph, i: int) -> None: - db.upsert_node(9, "conflict", props={"i": i}) + db.upsert_node("Person", "conflict", props={"i": i}) txn = db.begin_write_txn() - txn.upsert_node(9, "conflict", props={"txn": i}) - db.upsert_node(9, "conflict", props={"other": i}) + txn.upsert_node("Person", "conflict", props={"txn": i}) + db.upsert_node("Person", "conflict", props={"other": i}) try: txn.commit() raise AssertionError("expected conflict") From 9bae57c4437a6b8aa0493bb44fb7a3bf5787c5ed Mon Sep 17 00:00:00 2001 From: Brandon Hensley Date: Tue, 19 May 2026 22:11:02 -0500 Subject: [PATCH 2/3] Fix Windows mmap-safe corruption tests --- benches/core_ops.rs | 2 +- src/engine/tests/graph_ops.rs | 8 ++-- src/engine/tests/label_catalog.rs | 4 +- src/engine/tests/lifecycle.rs | 20 ++++---- src/engine/tests/query_planner.rs | 34 +++++++++---- src/engine/tests/read.rs | 23 ++++++--- src/engine/tests/wal_atomic.rs | 2 +- src/engine/txn.rs | 2 +- src/engine/write.rs | 16 +++---- src/lib.rs | 22 ++++----- src/memtable.rs | 2 +- src/planner_stats.rs | 2 +- src/scrub.rs | 79 +++++++++++++++++++------------ src/segment_reader.rs | 56 +++++++++++++++------- src/segment_writer.rs | 6 +-- tests/scrub_integration.rs | 3 +- 16 files changed, 171 insertions(+), 110 deletions(-) diff --git a/benches/core_ops.rs b/benches/core_ops.rs index 0ca8799..8933193 100644 --- a/benches/core_ops.rs +++ b/benches/core_ops.rs @@ -2997,7 +2997,7 @@ fn bench_batch_get_by_keys(c: &mut Criterion) { let keys: Vec<(u32, String)> = (0..1000).map(|i| (1u32, format!("key_{:04}", i))).collect(); for (tid, k) in &keys { engine - .upsert_node(&bench_node_label(*tid), k, UpsertNodeOptions::default()) + .upsert_node(bench_node_label(*tid), k, UpsertNodeOptions::default()) .unwrap(); } engine.flush().unwrap(); diff --git a/src/engine/tests/graph_ops.rs b/src/engine/tests/graph_ops.rs index c14a2d8..9017a43 100644 --- a/src/engine/tests/graph_ops.rs +++ b/src/engine/tests/graph_ops.rs @@ -5369,7 +5369,7 @@ fn graph_node_label_filter(names: &[&str], mode: LabelMatchMode) -> NodeLabelFil assert_eq!(node_ids, NodeIdSet::from_iter([a, b, d])); assert_eq!(sg.edges.len(), 2); // All edges should be label 1 - assert!(sg.edges.iter().all(|e| e.label == "RELATES_TO".to_string())); + assert!(sg.edges.iter().all(|e| e.label == "RELATES_TO")); engine.close().unwrap(); } @@ -6947,7 +6947,7 @@ fn graph_node_label_filter(names: &[&str], mode: LabelMatchMode) -> NodeLabelFil txn.upsert_edge( TxnNodeRef::Id(a), TxnNodeRef::Id(b), - &"KNOWS".to_string(), + "KNOWS", UpsertEdgeOptions { weight: 2.5, ..Default::default() @@ -6989,7 +6989,7 @@ fn graph_node_label_filter(names: &[&str], mode: LabelMatchMode) -> NodeLabelFil .upsert_edge( TxnNodeRef::Id(a), TxnNodeRef::Id(b), - &"KNOWS".to_string(), + "KNOWS", UpsertEdgeOptions::default(), ) .unwrap(); @@ -7005,7 +7005,7 @@ fn graph_node_label_filter(names: &[&str], mode: LabelMatchMode) -> NodeLabelFil .upsert_edge( TxnNodeRef::Id(a), TxnNodeRef::Id(b), - &"KNOWS".to_string(), + "KNOWS", UpsertEdgeOptions { weight: 2.0, ..Default::default() diff --git a/src/engine/tests/label_catalog.rs b/src/engine/tests/label_catalog.rs index cded743..6cf0324 100644 --- a/src/engine/tests/label_catalog.rs +++ b/src/engine/tests/label_catalog.rs @@ -562,7 +562,7 @@ fn test_label_resolution_plan_caches_distinct_node_labels_per_request() { let label_ids = plan .resolve_node_label_ids_for_request( - ["Person", "Person", "Company", "Company", "Person"].into_iter(), + ["Person", "Person", "Company", "Company", "Person"], ) .unwrap(); @@ -737,7 +737,7 @@ fn test_label_resolution_plan_caches_distinct_edge_labels_per_request() { let label_ids = plan .resolve_edge_label_ids_for_request( - ["KNOWS", "KNOWS", "WORKS_AT", "WORKS_AT", "KNOWS"].into_iter(), + ["KNOWS", "KNOWS", "WORKS_AT", "WORKS_AT", "KNOWS"], ) .unwrap(); diff --git a/src/engine/tests/lifecycle.rs b/src/engine/tests/lifecycle.rs index 455e794..fe4c311 100644 --- a/src/engine/tests/lifecycle.rs +++ b/src/engine/tests/lifecycle.rs @@ -3063,7 +3063,7 @@ fn build_clean_compaction_fixture(engine: &mut DatabaseEngine) -> (Vec, Vec let id = next_node_id; next_node_id += 1; let created_at = 1_000 + (seg as i64 * 100) + (i as i64 * 2); - write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { + write_internal_wal_op(engine, &WalOp::UpsertNode(NodeRecord { id, label_ids: NodeLabelSet::single(1).unwrap(), key: format!("s{}_n{}", seg, i), @@ -3084,7 +3084,7 @@ fn build_clean_compaction_fixture(engine: &mut DatabaseEngine) -> (Vec, Vec let eid = next_edge_id; next_edge_id += 1; let created_at = 5_000 + (seg as i64 * 100) + (i as i64 * 2); - write_internal_wal_op(&engine, &WalOp::UpsertEdge(EdgeRecord { + write_internal_wal_op(engine, &WalOp::UpsertEdge(EdgeRecord { id: eid, from: seg_node_ids[i], to: seg_node_ids[i + 1], @@ -3156,10 +3156,8 @@ fn build_vector_compaction_and_flush_fixture( ..compact_node.clone() }; - write_internal_wal_op(&compact_engine, &WalOp::UpsertNode(compact_node)) - .unwrap(); - write_internal_wal_op(&flush_engine, &WalOp::UpsertNode(flush_node)) - .unwrap(); + write_internal_wal_op(compact_engine, &WalOp::UpsertNode(compact_node)).unwrap(); + write_internal_wal_op(flush_engine, &WalOp::UpsertNode(flush_node)).unwrap(); compact_seg_ids.push(node_id); flush_seg_ids.push(node_id); @@ -3189,10 +3187,8 @@ fn build_vector_compaction_and_flush_fixture( ..compact_edge.clone() }; - write_internal_wal_op(&compact_engine, &WalOp::UpsertEdge(compact_edge)) - .unwrap(); - write_internal_wal_op(&flush_engine, &WalOp::UpsertEdge(flush_edge)) - .unwrap(); + write_internal_wal_op(compact_engine, &WalOp::UpsertEdge(compact_edge)).unwrap(); + write_internal_wal_op(flush_engine, &WalOp::UpsertEdge(flush_edge)).unwrap(); compact_edge_ids.push(edge_id); flush_edge_ids.push(edge_id); @@ -6838,12 +6834,12 @@ fn test_backpressure_invalidate_edge() { "EDGE_LABEL_19", "REPORTS_TO", ]; - for i in 0..20 { + for label in edge_labels { let eid = engine .upsert_edge( n1, n2, - edge_labels[i], + label, UpsertEdgeOptions { weight: 0.5, ..Default::default() diff --git a/src/engine/tests/query_planner.rs b/src/engine/tests/query_planner.rs index 54a2f83..8b0a2ad 100644 --- a/src/engine/tests/query_planner.rs +++ b/src/engine/tests/query_planner.rs @@ -75,6 +75,18 @@ fn corrupt_planner_stats_for_segment(db_path: &std::path::Path, segment_id: u64) std::fs::write(stats_path, b"corrupt planner stats").unwrap(); } +fn write_test_bytes_at(path: &std::path::Path, offset: u64, bytes: &[u8]) { + use std::io::{Seek, SeekFrom, Write}; + + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start(offset)).unwrap(); + file.write_all(bytes).unwrap(); + file.sync_all().unwrap(); +} + fn insert_query_node( engine: &DatabaseEngine, label: &str, @@ -732,17 +744,19 @@ fn edge_query_metadata_sidecar_unavailable_falls_back_at_engine_level() { crate::segment_components::ComponentHandleV1::ExternalFile { relative_path, .. - } => std::fs::write(seg_dir.join(relative_path), b"corrupt metadata sidecar") - .unwrap(), + } => write_test_bytes_at( + &seg_dir.join(relative_path), + 0, + b"corrupt metadata sidecar", + ), crate::segment_components::ComponentHandleV1::PackedRange { offset, .. } => { let core_path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); - let mut core = std::fs::read(&core_path).unwrap(); + let core = std::fs::read(&core_path).unwrap(); let header = crate::segment_components::decode_identity_header(&core).unwrap(); let start = header.payload_offset as usize + *offset as usize; - core[start..start + 8].copy_from_slice(&u64::MAX.to_le_bytes()); - std::fs::write(core_path, core).unwrap(); + write_test_bytes_at(&core_path, start as u64, &u64::MAX.to_le_bytes()); } } } @@ -3797,7 +3811,7 @@ fn oracle_query_ids( fn set_query_node_updated_at(engine: &DatabaseEngine, node_id: u64, updated_at: i64) { let node = internal_node_record(engine, node_id).unwrap().unwrap(); - write_internal_wal_op(&engine, &WalOp::UpsertNode(NodeRecord { + write_internal_wal_op(engine, &WalOp::UpsertNode(NodeRecord { created_at: updated_at, updated_at, ..node @@ -3807,7 +3821,7 @@ fn set_query_node_updated_at(engine: &DatabaseEngine, node_id: u64, updated_at: fn set_query_edge_props(engine: &DatabaseEngine, edge_id: u64, props: BTreeMap) { let edge = internal_edge_record(engine, edge_id).unwrap().unwrap(); - write_internal_wal_op(&engine, &WalOp::UpsertEdge(EdgeRecord { props, ..edge })) + write_internal_wal_op(engine, &WalOp::UpsertEdge(EdgeRecord { props, ..edge })) .unwrap(); } @@ -3938,7 +3952,7 @@ fn planned_pattern_anchor_and_edge_aliases( let (_guard, published) = engine.runtime.published_snapshot().unwrap(); let normalized = published .view - .normalize_pattern_query(&query) + .normalize_pattern_query(query) .unwrap(); let planned = published .view @@ -3971,7 +3985,7 @@ fn planned_pattern_anchor_sort_and_edge_aliases( let (_guard, published) = engine.runtime.published_snapshot().unwrap(); let normalized = published .view - .normalize_pattern_query(&query) + .normalize_pattern_query(query) .unwrap(); let planned = published .view @@ -4657,7 +4671,7 @@ fn test_node_query_any_dedupes_before_pagination_and_hydrates_final_page() { let employee = insert_query_node(&engine, "Employee", "employee", &[], 1.0); let both_b = insert_query_node_with_labels(&engine, &["Person", "Employee"], "both-b", &[], 1.0); - let expected = vec![both_a, person, employee, both_b]; + let expected = [both_a, person, employee, both_b]; let mut query = query_label_filter(&["Person", "Employee"], LabelMatchMode::Any); query.page = PageRequest { diff --git a/src/engine/tests/read.rs b/src/engine/tests/read.rs index b2bb833..3108090 100644 --- a/src/engine/tests/read.rs +++ b/src/engine/tests/read.rs @@ -12791,19 +12791,30 @@ fn assert_vector_hits_match(actual: &[VectorHit], expected: &[VectorHit]) { } fn rewrite_segment_component_payload_for_test(path: &Path, rewrite: impl FnOnce(&mut [u8])) { - let mut data = std::fs::read(path).unwrap(); - if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN + use std::io::{Seek, SeekFrom, Write}; + + let data = std::fs::read(path).unwrap(); + let range = if data.len() >= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC { let header = crate::segment_components::decode_identity_header(&data).unwrap(); let start = header.payload_offset as usize; let end = start + header.payload_len as usize; - rewrite(&mut data[start..end]); + start..end } else { - rewrite(&mut data); - } - std::fs::write(path, data).unwrap(); + 0..data.len() + }; + let mut payload = data[range.clone()].to_vec(); + rewrite(&mut payload); + + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start(range.start as u64)).unwrap(); + file.write_all(&payload).unwrap(); + file.sync_all().unwrap(); } #[test] diff --git a/src/engine/tests/wal_atomic.rs b/src/engine/tests/wal_atomic.rs index 52d3451..1f109dd 100644 --- a/src/engine/tests/wal_atomic.rs +++ b/src/engine/tests/wal_atomic.rs @@ -154,7 +154,7 @@ fn edge_op(id: u64, from: u64, to: u64, label_id: u32) -> WalOp { id, from, to, - label_id: label_id, + label_id, props: BTreeMap::new(), created_at: id as i64, updated_at: id as i64, diff --git a/src/engine/txn.rs b/src/engine/txn.rs index 34ccde1..11e0ce1 100644 --- a/src/engine/txn.rs +++ b/src/engine/txn.rs @@ -2076,7 +2076,7 @@ impl EngineCore { id, from: from_id, to: to_id, - label_id: label_id, + label_id, props: options.props.clone(), created_at, updated_at: now, diff --git a/src/engine/write.rs b/src/engine/write.rs index 0f7bb66..a5908a0 100644 --- a/src/engine/write.rs +++ b/src/engine/write.rs @@ -464,7 +464,7 @@ impl EngineCore { let ops = if should_create { vec![WalOp::EnsureEdgeLabel { label: label.to_string(), - label_id: label_id, + label_id, }] } else { Vec::new() @@ -813,7 +813,7 @@ impl EngineCore { id, from, to, - label_id: label_id, + label_id, props: options.props.clone(), created_at, updated_at: now, @@ -827,7 +827,7 @@ impl EngineCore { if should_create_token { ops.push(WalOp::EnsureEdgeLabel { label: label.to_string(), - label_id: label_id, + label_id, }); } ops.push(WalOp::UpsertEdge(edge)); @@ -952,7 +952,7 @@ impl EngineCore { id, from: input.from, to: input.to, - label_id: label_id, + label_id, props: input.props.clone(), created_at, updated_at: now, @@ -1178,7 +1178,7 @@ impl EngineCore { id, from: input.from, to: input.to, - label_id: label_id, + label_id, props: input.props.clone(), created_at, updated_at: now, @@ -1783,7 +1783,7 @@ impl EngineCore { if let Some(existing) = manifest.secondary_indexes.iter_mut().find(|entry| { entry.target == SecondaryIndexTarget::EdgeProperty { - label_id: label_id, + label_id, prop_key: prop_key.clone(), } && entry.kind == kind @@ -1799,7 +1799,7 @@ impl EngineCore { let entry = SecondaryIndexManifestEntry { index_id: manifest.next_secondary_index_id, target: SecondaryIndexTarget::EdgeProperty { - label_id: label_id, + label_id, prop_key: prop_key.clone(), }, kind: kind.clone(), @@ -1858,7 +1858,7 @@ impl EngineCore { let idx = manifest.secondary_indexes.iter().position(|entry| { entry.target == SecondaryIndexTarget::EdgeProperty { - label_id: label_id, + label_id, prop_key: prop_key.clone(), } && entry.kind == kind diff --git a/src/lib.rs b/src/lib.rs index 49c4889..54470aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -298,17 +298,17 @@ mod public_api_boundary_tests { } for required in [ - concat!("Edge", "LabelInfo"), - concat!("pub label: String"), - concat!("pub edge_label_filter: Option>"), - concat!("pub edge_label_index: u32"), - concat!("pub fn ensure_edge_label"), - concat!("pub fn get_edge_label_id"), - concat!("pub fn get_edge_label("), - concat!("pub fn list_edge_labels"), - concat!("pub fn edges_by_label"), - concat!("pub fn get_edges_by_label"), - concat!("pub fn count_edges_by_label"), + "EdgeLabelInfo", + "pub label: String", + "pub edge_label_filter: Option>", + "pub edge_label_index: u32", + "pub fn ensure_edge_label", + "pub fn get_edge_label_id", + "pub fn get_edge_label(", + "pub fn list_edge_labels", + "pub fn edges_by_label", + "pub fn get_edges_by_label", + "pub fn count_edges_by_label", ] { assert!( lib.contains(required) || types.contains(required) || engine.contains(required), diff --git a/src/memtable.rs b/src/memtable.rs index 856ff56..84dbcd8 100644 --- a/src/memtable.rs +++ b/src/memtable.rs @@ -3537,7 +3537,7 @@ mod tests { id, from, to, - label_id: label_id, + label_id, props: BTreeMap::new(), created_at: 2000, updated_at: 2001, diff --git a/src/planner_stats.rs b/src/planner_stats.rs index 458ed74..c633c71 100644 --- a/src/planner_stats.rs +++ b/src/planner_stats.rs @@ -4894,7 +4894,7 @@ mod tests { SecondaryIndexManifestEntry { index_id, target: SecondaryIndexTarget::EdgeProperty { - label_id: label_id, + label_id, prop_key: prop_key.to_string(), }, kind: SecondaryIndexKind::Equality, diff --git a/src/scrub.rs b/src/scrub.rs index 50e919f..66b1456 100644 --- a/src/scrub.rs +++ b/src/scrub.rs @@ -1378,6 +1378,13 @@ mod tests { std::fs::write(seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME), data).unwrap(); } + fn write_test_bytes_at(path: &Path, offset: usize, bytes: &[u8]) { + let mut file = OpenOptions::new().write(true).open(path).unwrap(); + file.seek(SeekFrom::Start(offset as u64)).unwrap(); + file.write_all(bytes).unwrap(); + file.sync_all().unwrap(); + } + fn any_finding(report: &ScrubReport, finding_type: ScrubFindingType) -> bool { report .segments @@ -1595,11 +1602,10 @@ mod tests { let manifest = read_segment_manifest(&seg_dir); let node_records_payload_start = packed_node_records_payload_start(&manifest); let core_path = seg_dir.join(PACKED_CORE_FILENAME); - let mut core = std::fs::read(&core_path).unwrap(); + let core = std::fs::read(&core_path).unwrap(); let first_record_offset = read_test_u64(&core, node_records_payload_start + 8 + 8) as usize; let first_label_id_offset = node_records_payload_start + first_record_offset + 1; - core[first_label_id_offset..first_label_id_offset + 4].copy_from_slice(&0u32.to_le_bytes()); - std::fs::write(&core_path, core).unwrap(); + write_test_bytes_at(&core_path, first_label_id_offset, &0u32.to_le_bytes()); let report = db.scrub().unwrap(); assert!( @@ -1616,13 +1622,15 @@ mod tests { let manifest = read_segment_manifest(&seg_dir); let node_records_payload_start = packed_node_records_payload_start(&manifest); let core_path = seg_dir.join(PACKED_CORE_FILENAME); - let mut core = std::fs::read(&core_path).unwrap(); + let core = std::fs::read(&core_path).unwrap(); let first_offset_pos = node_records_payload_start + 8 + 8; let second_offset_pos = node_records_payload_start + 8 + 16 + 8; let second_record_offset = read_test_u64(&core, second_offset_pos); - core[first_offset_pos..first_offset_pos + 8] - .copy_from_slice(&second_record_offset.to_le_bytes()); - std::fs::write(&core_path, core).unwrap(); + write_test_bytes_at( + &core_path, + first_offset_pos, + &second_record_offset.to_le_bytes(), + ); let report = db.scrub().unwrap(); assert!( @@ -1659,7 +1667,7 @@ mod tests { let node_label_index_start = packed_component_payload_start(&manifest, SegmentComponentKind::NodeLabelIndex); let core_path = seg_dir.join(PACKED_CORE_FILENAME); - let mut core = std::fs::read(&core_path).unwrap(); + let core = std::fs::read(&core_path).unwrap(); let label_count = read_test_u64(&core, node_label_index_start); assert!( label_count >= 2, @@ -1667,9 +1675,11 @@ mod tests { ); let first_posting_offset = read_test_u64(&core, node_label_index_start + 8 + 4); let second_posting_offset_pos = node_label_index_start + 8 + 16 + 4; - core[second_posting_offset_pos..second_posting_offset_pos + 8] - .copy_from_slice(&first_posting_offset.to_le_bytes()); - std::fs::write(&core_path, core).unwrap(); + write_test_bytes_at( + &core_path, + second_posting_offset_pos, + &first_posting_offset.to_le_bytes(), + ); let report = db.scrub().unwrap(); assert!( @@ -1691,10 +1701,11 @@ mod tests { let node_records_payload_start = packed_component_payload_start(&manifest, SegmentComponentKind::NodeRecords); let core_path = seg_dir.join(PACKED_CORE_FILENAME); - let mut core = std::fs::read(&core_path).unwrap(); - core[node_records_payload_start..node_records_payload_start + 8] - .copy_from_slice(&u64::MAX.to_le_bytes()); - std::fs::write(&core_path, core).unwrap(); + write_test_bytes_at( + &core_path, + node_records_payload_start, + &u64::MAX.to_le_bytes(), + ); let report = db.scrub().unwrap(); assert!( @@ -1716,10 +1727,11 @@ mod tests { let node_metadata_payload_start = packed_component_payload_start(&manifest, SegmentComponentKind::NodeMetadata); let core_path = seg_dir.join(PACKED_CORE_FILENAME); - let mut core = std::fs::read(&core_path).unwrap(); - core[node_metadata_payload_start..node_metadata_payload_start + 8] - .copy_from_slice(&u64::MAX.to_le_bytes()); - std::fs::write(&core_path, core).unwrap(); + write_test_bytes_at( + &core_path, + node_metadata_payload_start, + &u64::MAX.to_le_bytes(), + ); let report = db.scrub().unwrap(); assert!( @@ -1800,11 +1812,13 @@ mod tests { let sidecar_path = seg_dir .join("secondary_indexes") .join(format!("node_prop_eq_{eq_index_id}.dat")); - let mut sidecar = std::fs::read(&sidecar_path).unwrap(); + let sidecar = std::fs::read(&sidecar_path).unwrap(); let group_payload_offset = read_test_u64(&sidecar, payload_offset + 16) as usize; - sidecar[payload_offset + group_payload_offset..payload_offset + group_payload_offset + 8] - .copy_from_slice(&999_999u64.to_le_bytes()); - std::fs::write(&sidecar_path, sidecar).unwrap(); + write_test_bytes_at( + &sidecar_path, + payload_offset + group_payload_offset, + &999_999u64.to_le_bytes(), + ); let report = db.scrub().unwrap(); assert!( @@ -1833,7 +1847,7 @@ mod tests { let sidecar_path = seg_dir .join("secondary_indexes") .join(format!("node_prop_eq_{eq_index_id}.dat")); - let mut sidecar = std::fs::read(&sidecar_path).unwrap(); + let sidecar = std::fs::read(&sidecar_path).unwrap(); let id_count_offset = payload_offset + 24; let id_count = u32::from_le_bytes( sidecar[id_count_offset..id_count_offset + 4] @@ -1844,9 +1858,11 @@ mod tests { id_count > 1, "test precondition: expected at least two node IDs in the first equality group" ); - sidecar[id_count_offset..id_count_offset + 4] - .copy_from_slice(&(id_count - 1).to_le_bytes()); - std::fs::write(&sidecar_path, sidecar).unwrap(); + write_test_bytes_at( + &sidecar_path, + id_count_offset, + &(id_count - 1).to_le_bytes(), + ); let report = db.scrub().unwrap(); assert!( @@ -1875,10 +1891,11 @@ mod tests { let sidecar_path = seg_dir .join("secondary_indexes") .join(format!("node_prop_range_{range_index_id}.dat")); - let mut sidecar = std::fs::read(&sidecar_path).unwrap(); - sidecar[payload_offset + 16..payload_offset + 24] - .copy_from_slice(&999_999u64.to_le_bytes()); - std::fs::write(&sidecar_path, sidecar).unwrap(); + write_test_bytes_at( + &sidecar_path, + payload_offset + 16, + &999_999u64.to_le_bytes(), + ); let report = db.scrub().unwrap(); assert!( diff --git a/src/segment_reader.rs b/src/segment_reader.rs index 15d85cd..77a1d2e 100644 --- a/src/segment_reader.rs +++ b/src/segment_reader.rs @@ -5132,7 +5132,7 @@ impl SegmentReader { edge_id, from, to, - label_id: label_id, + label_id, updated_at, weight, valid_from, @@ -8934,7 +8934,7 @@ fn decode_edge_at(data: &[u8], offset: usize, id: u64) -> Result= crate::segment_components::COMPONENT_IDENTITY_HEADER_LEN && data[0..crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC.len()] == crate::segment_components::COMPONENT_IDENTITY_HEADER_MAGIC @@ -9072,15 +9074,23 @@ pub(crate) mod tests { } else { 0..data.len() }; - rewrite(&mut data[range]); - std::fs::write(path, data).unwrap(); + let mut payload = data[range.clone()].to_vec(); + rewrite(&mut payload); + + let mut file = std::fs::OpenOptions::new().write(true).open(path).unwrap(); + file.seek(SeekFrom::Start(range.start as u64)).unwrap(); + file.write_all(&payload).unwrap(); + file.sync_all().unwrap(); } fn tamper_envelope_format_version(seg_dir: &std::path::Path, version: u32) { + use std::io::{Seek, SeekFrom, Write}; + let path = seg_dir.join(SEGMENT_COMPONENT_MANIFEST_FILENAME); - let mut data = std::fs::read(&path).unwrap(); - data[12..16].copy_from_slice(&version.to_le_bytes()); - std::fs::write(&path, data).unwrap(); + let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + file.seek(SeekFrom::Start(12)).unwrap(); + file.write_all(&version.to_le_bytes()).unwrap(); + file.sync_all().unwrap(); } fn read_segment_manifest_for_test(seg_dir: &std::path::Path) -> SegmentComponentManifestV1 { @@ -9119,6 +9129,8 @@ pub(crate) mod tests { kind: SegmentComponentKind, rewrite: impl FnOnce(&mut [u8]), ) { + use std::io::{Seek, SeekFrom, Write}; + let manifest = read_segment_manifest_for_test(seg_dir); let record = manifest .components @@ -9128,21 +9140,31 @@ pub(crate) mod tests { match &record.handle { ComponentHandleV1::ExternalFile { relative_path, .. } => { let path = seg_dir.join(relative_path); - let mut data = std::fs::read(&path).unwrap(); + let data = std::fs::read(&path).unwrap(); let header = crate::segment_components::decode_identity_header(&data).unwrap(); let start = header.payload_offset as usize; let end = start + header.payload_len as usize; - rewrite(&mut data[start..end]); - std::fs::write(path, data).unwrap(); + let mut payload = data[start..end].to_vec(); + rewrite(&mut payload); + + let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + file.seek(SeekFrom::Start(start as u64)).unwrap(); + file.write_all(&payload).unwrap(); + file.sync_all().unwrap(); } ComponentHandleV1::PackedRange { offset, len, .. } => { let path = seg_dir.join(crate::segment_components::PACKED_CORE_FILENAME); - let mut data = std::fs::read(&path).unwrap(); + let data = std::fs::read(&path).unwrap(); let header = crate::segment_components::decode_identity_header(&data).unwrap(); let start = header.payload_offset as usize + *offset as usize; let end = start + *len as usize; - rewrite(&mut data[start..end]); - std::fs::write(path, data).unwrap(); + let mut payload = data[start..end].to_vec(); + rewrite(&mut payload); + + let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + file.seek(SeekFrom::Start(start as u64)).unwrap(); + file.write_all(&payload).unwrap(); + file.sync_all().unwrap(); } } } @@ -9298,7 +9320,7 @@ pub(crate) mod tests { id, from, to, - label_id: label_id, + label_id, props: BTreeMap::new(), created_at: 2000, updated_at: 2001, @@ -11587,7 +11609,9 @@ pub(crate) mod tests { let err = expect_engine_error(SegmentReader::open_unpinned_for_test(&seg_dir, 1, None)); assert!( - err.contains("No such file") || err.contains("segment.core"), + err.contains("No such file") + || err.contains("cannot find the file") + || err.contains("segment.core"), "got: {err}" ); } diff --git a/src/segment_writer.rs b/src/segment_writer.rs index 07f7c97..d4d29c1 100644 --- a/src/segment_writer.rs +++ b/src/segment_writer.rs @@ -5731,7 +5731,7 @@ mod tests { id, from, to, - label_id: label_id, + label_id, props: BTreeMap::new(), created_at: 2000, updated_at: 2001, @@ -6105,7 +6105,7 @@ mod tests { data_len, from, to, - label_id: label_id, + label_id, updated_at, weight, valid_from, @@ -6288,7 +6288,7 @@ mod tests { fn make_adj(edge_id: u64, label_id: u32, neighbor_id: u64, weight: f32) -> AdjEntry { AdjEntry { edge_id, - label_id: label_id, + label_id, neighbor_id, weight, valid_from: 1000, diff --git a/tests/scrub_integration.rs b/tests/scrub_integration.rs index b293bb1..95ede78 100644 --- a/tests/scrub_integration.rs +++ b/tests/scrub_integration.rs @@ -7,8 +7,7 @@ fn open_test_db(dir: &std::path::Path) -> DatabaseEngine { compact_after_n_flushes: 0, ..DbOptions::default() }; - let db = DatabaseEngine::open(dir, &opts).unwrap(); - db + DatabaseEngine::open(dir, &opts).unwrap() } fn populate_and_flush(db: &DatabaseEngine) { From 5d5e0e3f0eda985f48fd56158ceb60481f5a9c0f Mon Sep 17 00:00:00 2001 From: Brandon Hensley Date: Tue, 19 May 2026 22:23:54 -0500 Subject: [PATCH 3/3] Fix Windows scrub integration corruption tests --- src/planner_stats.rs | 21 ++++++++------------- tests/scrub_integration.rs | 36 ++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/planner_stats.rs b/src/planner_stats.rs index c633c71..6374184 100644 --- a/src/planner_stats.rs +++ b/src/planner_stats.rs @@ -22,13 +22,13 @@ use std::fs; use std::fs::File; #[cfg(test)] use std::io::Read; -#[cfg(test)] +#[cfg(all(test, unix))] use std::io::Write; use std::path::Path; use std::sync::Arc; pub(crate) const PLANNER_STATS_FILENAME: &str = "planner_stats.dat"; -#[cfg(test)] +#[cfg(all(test, unix))] const PLANNER_STATS_TMP_FILENAME: &str = "planner_stats.tmp"; const PLANNER_STATS_MAGIC: [u8; 8] = *b"OGPST01\0"; pub(crate) const PLANNER_STATS_FORMAT_VERSION: u32 = 1; @@ -3025,7 +3025,7 @@ fn build_minimal_targeted_refresh_stats( }) } -#[cfg(test)] +#[cfg(all(test, unix))] pub(crate) fn write_planner_stats_sidecar_atomic( seg_dir: &Path, stats: SegmentPlannerStatsV1, @@ -3046,7 +3046,7 @@ pub(crate) fn write_planner_stats_sidecar_atomic( Ok(PlannerStatsWriteOutcome::Written) } -#[cfg(test)] +#[cfg(all(test, unix))] fn write_planner_stats_sidecar_atomic_cleanup_on_error( seg_dir: &Path, stats: SegmentPlannerStatsV1, @@ -4594,20 +4594,15 @@ fn read_optional_component_payload(path: &Path) -> Result>, Engin Ok(Some(data)) } -#[cfg(test)] +#[cfg(all(test, unix))] fn cleanup_stats_tmp(seg_dir: &Path) { let _ = fs::remove_file(seg_dir.join(PLANNER_STATS_TMP_FILENAME)); } -#[cfg(test)] +#[cfg(all(test, unix))] fn fsync_dir(dir: &Path) -> Result<(), EngineError> { - #[cfg(not(target_os = "windows"))] - { - let d = File::open(dir)?; - d.sync_all()?; - } - #[cfg(target_os = "windows")] - let _ = dir; + let d = File::open(dir)?; + d.sync_all()?; Ok(()) } diff --git a/tests/scrub_integration.rs b/tests/scrub_integration.rs index 95ede78..d9e8320 100644 --- a/tests/scrub_integration.rs +++ b/tests/scrub_integration.rs @@ -1,5 +1,6 @@ use overgraph::{DatabaseEngine, DbOptions, NodeInput, ScrubFindingType, UpsertEdgeOptions}; use std::collections::BTreeMap; +use std::io::{Read, Seek, SeekFrom, Write}; use tempfile::TempDir; fn open_test_db(dir: &std::path::Path) -> DatabaseEngine { @@ -36,6 +37,21 @@ fn populate_and_flush(db: &DatabaseEngine) { db.flush().unwrap(); } +fn flip_file_byte(path: &std::path::Path, offset: u64) { + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start(offset)).unwrap(); + let mut byte = [0u8; 1]; + file.read_exact(&mut byte).unwrap(); + byte[0] ^= 0xFF; + file.seek(SeekFrom::Start(offset)).unwrap(); + file.write_all(&byte).unwrap(); + file.sync_all().unwrap(); +} + #[test] fn test_scrub_healthy_database_no_findings() { let dir = TempDir::new().unwrap(); @@ -100,10 +116,8 @@ fn test_scrub_detects_packed_range_corruption() { let seg_dir = db_path.join("segments").join("seg_0001"); let core_path = seg_dir.join("segment.core"); - let mut data = std::fs::read(&core_path).unwrap(); - let corrupt_offset = data.len() / 2; - data[corrupt_offset] ^= 0xFF; - std::fs::write(&core_path, &data).unwrap(); + let corrupt_offset = std::fs::metadata(&core_path).unwrap().len() / 2; + flip_file_byte(&core_path, corrupt_offset); let db = open_test_db(&db_path); let report = db.scrub().unwrap(); @@ -132,13 +146,12 @@ fn test_scrub_detects_external_payload_corruption() { let seg_dir = db_path.join("segments").join("seg_0001"); let sidecar_path = find_external_sidecar(&seg_dir) .expect("test precondition: expected at least one external sidecar after flush with edges"); - let mut data = std::fs::read(&sidecar_path).unwrap(); + let len = std::fs::metadata(&sidecar_path).unwrap().len(); assert!( - data.len() > 192, + len > 192, "test precondition: external sidecar must have identity header + payload" ); - data[193] ^= 0xFF; - std::fs::write(&sidecar_path, &data).unwrap(); + flip_file_byte(&sidecar_path, 193); let db = open_test_db(&db_path); let report = db.scrub().unwrap(); @@ -164,13 +177,12 @@ fn test_scrub_detects_identity_header_tamper() { let seg_dir = db_path.join("segments").join("seg_0001"); let sidecar_path = find_external_sidecar(&seg_dir) .expect("test precondition: expected at least one external sidecar after flush with edges"); - let mut data = std::fs::read(&sidecar_path).unwrap(); + let len = std::fs::metadata(&sidecar_path).unwrap().len(); assert!( - data.len() >= 192, + len >= 192, "test precondition: external sidecar must have identity header" ); - data[16] ^= 0xFF; - std::fs::write(&sidecar_path, &data).unwrap(); + flip_file_byte(&sidecar_path, 16); let db = open_test_db(&db_path); let report = db.scrub().unwrap();