diff --git a/docs/unified_index_format.md b/docs/unified_index_format.md index 5a5142562b..6a77cdaa92 100644 --- a/docs/unified_index_format.md +++ b/docs/unified_index_format.md @@ -193,7 +193,7 @@ Replaces today's `_labels_map.txt` + `_labels_to_medoids.txt`. One row per disti `label_integer` is always written as a 4-byte little-endian unsigned integer, independent of the build-time `LabelT` template parameter (`uint16_t` values are zero-extended). This makes the on-disk dictionary self-describing and uniform across writer instantiations. Row count is implicit: read rows until `label_dictionary_len` bytes are consumed. -If `header.universal_label != 0`, the dictionary MAY contain a row whose `label_integer` matches it; otherwise the universal label has no explicit dictionary entry. +If `header.universal_label != 0`, the dictionary MUST contain a row whose `label_integer` matches it. That row supplies the universal label's filtered-search medoid. #### 2.4.2 Per-point labels @@ -268,6 +268,16 @@ This change is encapsulated in a single helper (`node_read_window(N)`) so the bu --- +### 3.3 Filtered search compatibility + +Unified-index search preserves the legacy invalid-label behavior: + +- Unknown label strings are ignored when at least one requested label resolves. +- If no requested label resolves and no universal label is configured, search completes successfully with an empty result and reports `QueryStats::label_valid == false`. +- If no requested label resolves and a universal label is configured with its required dictionary medoid, search reports `label_valid == false` but performs an ANN search using that universal label and medoid. +- A filtered index missing the resolved label medoid, including the universal medoid, is treated as invalid data and returns an empty successful result. +- Supplying filter labels to an unfiltered index remains an error. + ## 4. Build Path (informative) `build_unified_index` reuses the existing pipeline (preprocess → optional PQ training → `build_merged_vamana_index`) up to the point where the legacy code would write separate files or call `create_disk_layout`. From there: diff --git a/include/percentile_stats.h b/include/percentile_stats.h index 5ff563afac..211648cda5 100644 --- a/include/percentile_stats.h +++ b/include/percentile_stats.h @@ -33,6 +33,7 @@ struct QueryStats unsigned n_cmps = 0; // # cmps unsigned n_cache_hits = 0; // # cache_hits unsigned n_hops = 0; // # search hops + bool label_valid = true; // whether at least one requested filter label was valid }; struct TableStats diff --git a/include/unified_index.h b/include/unified_index.h index 0c5f17aef4..b2be42a9b2 100644 --- a/include/unified_index.h +++ b/include/unified_index.h @@ -40,16 +40,18 @@ struct UnifiedLoadContext struct UnifiedSearchContext { // ---- Inputs ---- - const void *query = nullptr; // typed by caller as const T* + const void *query = nullptr; // typed by caller as const T* size_t K = 10; uint32_t L = 100; - // Filter labels as user-facing strings. Required non-empty if the loaded - // index has labels; required empty otherwise. The index converts strings - // to internal label ints per its encoding. + // Filter labels as user-facing strings. Required empty for an unfiltered + // index. A filtered index ignores unknown labels; if none resolve, it + // returns an empty successful result unless a universal label is available, + // in which case it performs a universal-label search. QueryStats::label_valid + // reports whether at least one requested label resolved. std::vector filter_labels; - std::optional beam_width; // SSD-only - std::optional io_limit; // SSD-only - std::function rerank_fn; // SSD-only + std::optional beam_width; // SSD-only + std::optional io_limit; // SSD-only + std::function rerank_fn; // SSD-only // ---- Outputs (caller-allocated, length >= K) ---- uint64_t *indices = nullptr; @@ -90,7 +92,7 @@ DISKANN_DLLEXPORT std::unique_ptr make_unified_index_memory(const // Factory: open a unified file in disk-resident (SSD) mode. The supplied // AlignedFileReader is handed to the constructed unified_index_ssd. -DISKANN_DLLEXPORT std::unique_ptr make_unified_index_ssd( - std::shared_ptr reader, const UnifiedLoadContext &ctx); +DISKANN_DLLEXPORT std::unique_ptr make_unified_index_ssd(std::shared_ptr reader, + const UnifiedLoadContext &ctx); } // namespace diskann diff --git a/include/unified_label_data.h b/include/unified_label_data.h index 99fd141b27..70f3486119 100644 --- a/include/unified_label_data.h +++ b/include/unified_label_data.h @@ -20,6 +20,17 @@ namespace diskann class UnifiedIndexReader; +struct filter_resolution_result +{ + // True when at least one requested label string resolved through the + // dictionary. Callers use this to report invalid-label telemetry. + bool label_valid = false; + + // True when search can proceed through a resolved label or the universal + // label. False means the caller should return a successful empty result. + bool should_search = false; +}; + // --------------------------------------------------------------------------- // Abstract base for the label-data trio. // Owns the shared, encoding-independent state (label dictionary, universal @@ -68,16 +79,14 @@ class unified_label_data_base bool get_converted_label(const std::string &s, uint32_t &out) const; // Resolve filter label strings to their internal label ints AND per-label - // medoids in a single dictionary probe per string. out_label_ints[i] and - // out_medoids[i] both correspond to filter_label_strings[i]; both vectors - // are caller-owned, cleared, then filled in lockstep. Throws ANNException - // on an unknown label string. The unified format stores exactly one medoid - // per label, packed in the same dictionary row as the label int, so the - // search path gets the proxy input (label int) and the init-id seed - // (medoid) from one map lookup instead of two. - void resolve_filters(const std::vector &filter_label_strings, - std::vector &out_label_ints, - std::vector &out_medoids) const; + // medoids in a single dictionary probe per string. Unknown labels are + // ignored, matching the legacy filtered-search path. If no requested label + // resolves and a universal label with a dictionary medoid exists, both are + // inserted into the outputs. Otherwise the caller returns a successful + // empty result. + filter_resolution_result resolve_filters(const std::vector &filter_label_strings, + std::vector &out_label_ints, + std::vector &out_medoids) const; // Append every per-label entry-point medoid (the unified format stores // exactly one per label) to `out`. Used to seed SSD cache priming so that @@ -92,20 +101,21 @@ class unified_label_data_base // and is shared with init-id seeding). The returned proxy borrows internal // storage of `this` -- lifetime must not exceed `this`. No external scratch // is needed; the concrete proxy owns any per-query scratch it requires. - virtual std::unique_ptr make_match_proxy( - const std::vector &filter_label_ints) = 0; + virtual std::unique_ptr make_match_proxy(const std::vector &filter_label_ints) = 0; protected: // Derived classes load their encoding-specific region(s) after the base // has parsed the shared dictionary. virtual void load_encoding(UnifiedIndexReader &r, const UnifiedIndexHeader &h, uint64_t npts) = 0; - // Helper: convert strings -> uint32 label ints via dictionary; throws on unknown. + // Helper: parse the string -> {label int, medoid} dictionary. void parse_dictionary(const std::vector &dict_bytes); bool _has_labels = false; bool _use_universal_label = false; uint32_t _universal_label = 0; + bool _has_universal_medoid = false; + uint32_t _universal_medoid = 0; // Dictionary row: label string -> {internal label int, per-label medoid}. // Both fields come from the same on-disk dictionary entry (see @@ -135,8 +145,7 @@ class unified_label_data_bitmask final : public unified_label_data_base return _bitmask_buf._buf.size() * sizeof(std::uint64_t); } - std::unique_ptr make_match_proxy( - const std::vector &filter_label_ints) override; + std::unique_ptr make_match_proxy(const std::vector &filter_label_ints) override; simple_bitmask_buf &bitmask_buf() { @@ -169,8 +178,7 @@ class unified_label_data_integer final : public unified_label_data_base return _label_vector.get_memory_usage(); } - std::unique_ptr make_match_proxy( - const std::vector &filter_label_ints) override; + std::unique_ptr make_match_proxy(const std::vector &filter_label_ints) override; integer_label_vector &label_vector() { @@ -191,8 +199,7 @@ class unified_label_data_integer final : public unified_label_data_base // Factory: peeks at `h.label_encoding`, constructs the correct derived class, // runs load(), and returns it. Returns nullptr when the header carries no // labels (HAS_LABELS flag unset or encoding == None). -std::unique_ptr make_unified_label_data(UnifiedIndexReader &r, - const UnifiedIndexHeader &h, - uint64_t npts); +std::unique_ptr make_unified_label_data(UnifiedIndexReader &r, const UnifiedIndexHeader &h, + uint64_t npts); } // namespace diskann diff --git a/src/unified_index_base.cpp b/src/unified_index_base.cpp index 9806929425..7ac26dd44c 100644 --- a/src/unified_index_base.cpp +++ b/src/unified_index_base.cpp @@ -9,8 +9,7 @@ namespace diskann { -template -unified_index_base::unified_index_base(diskann::Metric metric) : _metric(metric) +template unified_index_base::unified_index_base(diskann::Metric metric) : _metric(metric) { } @@ -26,8 +25,7 @@ template void unified_index_base::validate_header(const UnifiedI throw ANNException("unified_index_base: data_type mismatch with T", -1, __FUNCSIG__, __FILE__, __LINE__); } -template -void unified_index_base::validate_search_context(const UnifiedSearchContext &ctx) const +template void unified_index_base::validate_search_context(const UnifiedSearchContext &ctx) const { if (ctx.query == nullptr) throw ANNException("UnifiedSearchContext: query == nullptr", -1, __FUNCSIG__, __FILE__, __LINE__); @@ -41,9 +39,6 @@ void unified_index_base::validate_search_context(const UnifiedSearchContext & const bool filtered = has_labels(); const bool has_filters = !ctx.filter_labels.empty(); - if (filtered && !has_filters) - throw ANNException("UnifiedSearchContext: filter_labels must be non-empty for a filtered index", -1, - __FUNCSIG__, __FILE__, __LINE__); if (!filtered && has_filters) throw ANNException("UnifiedSearchContext: filter_labels must be empty for a non-filtered index", -1, __FUNCSIG__, __FILE__, __LINE__); diff --git a/src/unified_index_memory.cpp b/src/unified_index_memory.cpp index 2e9a37dbc1..f0ea416485 100644 --- a/src/unified_index_memory.cpp +++ b/src/unified_index_memory.cpp @@ -41,8 +41,7 @@ template unified_index_memory::~unified_index_memory() } } -template -void unified_index_memory::load_storage(UnifiedIndexReader &r, const UnifiedLoadContext &ctx) +template void unified_index_memory::load_storage(UnifiedIndexReader &r, const UnifiedLoadContext &ctx) { // Build the resident node store. auto store = std::make_unique>(); @@ -91,17 +90,18 @@ template void unified_index_memory::init_scratch_pool(uint32_t n std::vector empty_sellers; for (uint32_t i = 0; i < num_threads; ++i) { - auto *s = new InMemQueryScratch(search_l, search_l, R, maxc, dim, aligned_dim, alignment_factor, - empty_sellers, - /*init_pq_scratch=*/false, bitmask_size); + auto *s = + new InMemQueryScratch(search_l, search_l, R, maxc, dim, aligned_dim, alignment_factor, empty_sellers, + /*init_pq_scratch=*/false, bitmask_size); _query_scratch.push(s); } } template -std::pair unified_index_memory::iterate_to_fixed_point( - InMemQueryScratch *scratch, uint32_t L, const T *query, const std::vector &init_ids, - filter_match_proxy *match_proxy) +std::pair unified_index_memory::iterate_to_fixed_point(InMemQueryScratch *scratch, uint32_t L, + const T *query, + const std::vector &init_ids, + filter_match_proxy *match_proxy) { auto *store = static_cast *>(this->_store.get()); const uint64_t aligned_dim = this->_header.aligned_dim; @@ -124,9 +124,8 @@ std::pair unified_index_memory::iterate_to_fixed_point( { if (inserted_into_pool_bs.size() < total_num_points) { - auto resize_size = 2 * total_num_points > MAX_POINTS_FOR_USING_BITSET - ? MAX_POINTS_FOR_USING_BITSET - : 2 * total_num_points; + auto resize_size = + 2 * total_num_points > MAX_POINTS_FOR_USING_BITSET ? MAX_POINTS_FOR_USING_BITSET : 2 * total_num_points; inserted_into_pool_bs.resize(resize_size); } } @@ -206,6 +205,38 @@ std::pair unified_index_memory::iterate_to_fixed_point( template void unified_index_memory::search_impl(UnifiedSearchContext &ctx) { + // Resolve filters before borrowing scratch so an all-invalid request + // without a universal label can return a successful empty result cheaply. + std::unique_ptr proxy; + thread_local std::vector init_ids; + init_ids.clear(); + if (ctx.stats != nullptr) + { + ctx.stats->label_valid = true; + } + if (this->_labels && this->_labels->has_labels()) + { + thread_local std::vector filter_label_ints; + const auto resolution = this->_labels->resolve_filters(ctx.filter_labels, filter_label_ints, init_ids); + if (ctx.stats != nullptr) + { + ctx.stats->label_valid = resolution.label_valid; + } + if (!resolution.should_search) + { + std::fill_n(ctx.indices, ctx.K, std::numeric_limits::max()); + std::fill_n(ctx.distances, ctx.K, std::numeric_limits::max()); + if (ctx.stats != nullptr) + { + ctx.stats->n_hops = 0; + ctx.stats->n_cmps = 0; + ctx.stats->n_ios = 0; + } + return; + } + proxy = this->_labels->make_match_proxy(filter_label_ints); + } + // Borrow per-thread scratch. ScratchStoreManager blocks until a scratch // object is available, so when more queries arrive concurrently than the // pool holds, callers wait for a free slot rather than failing. The pool is @@ -231,26 +262,6 @@ template void unified_index_memory::search_impl(UnifiedSearchCon std::memcpy(aligned_query, ctx.query, dim * sizeof(T)); } - // Build the label match proxy if the index is filtered, resolving the - // filter label strings once: resolve_filters yields both the internal - // label ints (for the proxy) and the per-label medoid seed ids (init_ids - // below) from a single dictionary probe per label. - // - // init_ids / filter_label_ints are thread_local: search_impl runs once per - // query on a pooled thread, so reusing these buffers across queries avoids a - // per-call heap allocation (mirrors unified_index_ssd::cached_beam_search). - // init_ids must be cleared up front because the unfiltered branch below - // relies on it being empty when no filter is applied. - std::unique_ptr proxy; - thread_local std::vector init_ids; - init_ids.clear(); - if (this->_labels && this->_labels->has_labels()) - { - thread_local std::vector filter_label_ints; - this->_labels->resolve_filters(ctx.filter_labels, filter_label_ints, init_ids); - proxy = this->_labels->make_match_proxy(filter_label_ints); - } - // Seed init_ids. Aligned with unified_index_ssd::cached_beam_search: // - Unfiltered: pick the single closest medoid from _medoids by // full-vector L2 (memory has all coords resident, so we don't need the @@ -260,9 +271,9 @@ template void unified_index_memory::search_impl(UnifiedSearchCon // seeding dramatically improves recall on filtered search because the // global start node may not lie within any filter-label cluster. auto *store = static_cast *>(this->_store.get()); - if (init_ids.empty()) + if (proxy == nullptr) { - // Unfiltered path -- pick closest of the global medoid set. + // Unfiltered path -- pick the closest global medoid. uint32_t best_id = _medoids.empty() ? _start : _medoids[0]; float best_dist = std::numeric_limits::max(); for (uint32_t mid : _medoids) @@ -277,6 +288,20 @@ template void unified_index_memory::search_impl(UnifiedSearchCon } init_ids.push_back(best_id); } + else if (init_ids.empty()) + { + // A valid filtered index supplies a medoid for every label, including + // the universal label. Missing init IDs indicate invalid index data. + std::fill_n(ctx.indices, ctx.K, std::numeric_limits::max()); + std::fill_n(ctx.distances, ctx.K, std::numeric_limits::max()); + if (ctx.stats != nullptr) + { + ctx.stats->n_hops = 0; + ctx.stats->n_cmps = 0; + ctx.stats->n_ios = 0; + } + return; + } auto [hops, cmps] = iterate_to_fixed_point(scratch, ctx.L, aligned_query, init_ids, proxy.get()); diff --git a/src/unified_index_ssd.cpp b/src/unified_index_ssd.cpp index f5aff7ed2d..97f57d3bbd 100644 --- a/src/unified_index_ssd.cpp +++ b/src/unified_index_ssd.cpp @@ -25,7 +25,6 @@ namespace diskann { - template unified_index_ssd::unified_index_ssd(std::shared_ptr reader, diskann::Metric metric) : unified_index_base(metric), _reader(std::move(reader)), _thread_data(nullptr) @@ -46,8 +45,7 @@ template unified_index_ssd::~unified_index_ssd() } } -template -void unified_index_ssd::load_storage(UnifiedIndexReader &r, const UnifiedLoadContext &ctx) +template void unified_index_ssd::load_storage(UnifiedIndexReader &r, const UnifiedLoadContext &ctx) { const UnifiedIndexHeader &h = r.header(); if (!(h.flags & HAS_PQ)) @@ -138,7 +136,7 @@ void unified_index_ssd::load_storage(UnifiedIndexReader &r, const UnifiedLoad SSDThreadData *tdata = manager.scratch_space(); NodeFetchScratch fetch_scratch; fetch_scratch.attach_borrowed(tdata->ctx, tdata->scratch.sector_scratch, - defaults::MAX_N_SECTOR_READS * defaults::SECTOR_LEN); + defaults::MAX_N_SECTOR_READS * defaults::SECTOR_LEN); std::vector cached_list; ssd_store->cache_bfs_levels(seed_ids, ctx.num_nodes_to_cache, cached_list, fetch_scratch); } @@ -210,7 +208,7 @@ template void unified_index_ssd::use_medoids_data_as_centroids() SSDThreadData *tdata = manager.scratch_space(); NodeFetchScratch scratch; scratch.attach_borrowed(tdata->ctx, tdata->scratch.sector_scratch, - defaults::MAX_N_SECTOR_READS * defaults::SECTOR_LEN); + defaults::MAX_N_SECTOR_READS * defaults::SECTOR_LEN); std::vector ids(1); std::vector> views; @@ -237,14 +235,48 @@ template void unified_index_ssd::search_impl(UnifiedSearchContex template void unified_index_ssd::cached_beam_search(const T *query, uint64_t K, uint64_t L, uint64_t *indices, float *distances, uint32_t beam_width, - const std::vector &filter_label_strings, - uint32_t io_limit, QueryStats *stats, - DebugTraversalInfo * /*debug_info*/) + const std::vector &filter_label_strings, uint32_t io_limit, + QueryStats *stats, DebugTraversalInfo * /*debug_info*/) { const uint64_t aligned_dim = this->_header.aligned_dim; const uint64_t dim = this->_header.dim; auto *ssd_store = static_cast *>(this->_store.get()); + // Resolve filters before doing query preprocessing or IO. Unknown labels + // are ignored. If none resolve, return an empty successful result unless a + // universal label is available, in which case search with that label. + std::unique_ptr proxy; + thread_local std::vector filter_label_ints; + thread_local std::vector filter_init_ids; + filter_label_ints.clear(); + filter_init_ids.clear(); + if (stats != nullptr) + { + stats->label_valid = true; + } + if (this->_labels && this->_labels->has_labels()) + { + const auto resolution = + this->_labels->resolve_filters(filter_label_strings, filter_label_ints, filter_init_ids); + if (stats != nullptr) + { + stats->label_valid = resolution.label_valid; + } + if (!resolution.should_search) + { + std::fill_n(indices, K, std::numeric_limits::max()); + std::fill_n(distances, K, std::numeric_limits::max()); + if (stats != nullptr) + { + stats->n_hops = 0; + stats->n_cmps = 0; + stats->n_ios = 0; + } + return; + } + proxy = this->_labels->make_match_proxy(filter_label_ints); + } + // Borrow per-thread scratch. ScratchStoreManager> manager(_thread_data); SSDThreadData *tdata = manager.scratch_space(); @@ -282,21 +314,6 @@ void unified_index_ssd::cached_beam_search(const T *query, uint64_t K, uint64 std::vector &full_retset = tdata->scratch.full_retset; tsl::robin_set &visited = tdata->scratch.visited; - // Build filter proxy if applicable. Resolve the filter label strings a - // single time here: resolve_filters returns both the internal label ints - // (consumed by make_match_proxy) and the per-label medoid seed ids - // (consumed by the filtered-seeding branch below), so the label dictionary - // is probed once per label instead of once for the proxy and again for the - // init ids. - std::unique_ptr proxy; - thread_local std::vector filter_label_ints; - thread_local std::vector filter_init_ids; - if (this->_labels && this->_labels->has_labels()) - { - this->_labels->resolve_filters(filter_label_strings, filter_label_ints, filter_init_ids); - proxy = this->_labels->make_match_proxy(filter_label_ints); - } - // Seed retset. Branches on filtered vs. unfiltered, mirroring the legacy // PQFlashIndex::cached_beam_search (src/pq_flash_index.cpp:1329-1377). // @@ -310,12 +327,13 @@ void unified_index_ssd::cached_beam_search(const T *query, uint64_t K, uint64 // above (one dictionary probe shared with the match proxy). if (proxy == nullptr) { + // Unfiltered path -- pick the closest global medoid. uint32_t best_medoid = _medoids.empty() ? 0 : _medoids[0]; float best_medoid_dist = std::numeric_limits::max(); for (size_t i = 0; i < _medoids.size(); ++i) { float d = _dist_cmp_float->compare(query_float, _centroid_data + i * aligned_dim, - static_cast(aligned_dim)); + static_cast(aligned_dim)); if (d < best_medoid_dist) { best_medoid_dist = d; @@ -326,6 +344,20 @@ void unified_index_ssd::cached_beam_search(const T *query, uint64_t K, uint64 retset.insert(Neighbor(best_medoid, dist_scratch[0])); visited.insert(best_medoid); } + else if (filter_init_ids.empty()) + { + // A valid filtered index supplies a medoid for every label, including + // the universal label. Missing init IDs indicate invalid index data. + std::fill_n(indices, K, std::numeric_limits::max()); + std::fill_n(distances, K, std::numeric_limits::max()); + if (stats != nullptr) + { + stats->n_hops = 0; + stats->n_cmps = 0; + stats->n_ios = 0; + } + return; + } else { // filter_init_ids was populated by resolve_filters above: one medoid @@ -349,7 +381,7 @@ void unified_index_ssd::cached_beam_search(const T *query, uint64_t K, uint64 // (MAX_N_SECTOR_READS * SECTOR_LEN), allocated once at load time. NodeFetchScratch fetch_scratch; fetch_scratch.attach_borrowed(tdata->ctx, tdata->scratch.sector_scratch, - defaults::MAX_N_SECTOR_READS * defaults::SECTOR_LEN); + defaults::MAX_N_SECTOR_READS * defaults::SECTOR_LEN); // Per-thread scratch reused across hops and across calls. Capacity grows // once to the worst-case beam_width that this thread ever sees. thread_local std::vector beam_ids; @@ -379,8 +411,7 @@ void unified_index_ssd::cached_beam_search(const T *query, uint64_t K, uint64 { const uint32_t id = static_cast(beam_ids[bi]); const NodeView &view = beam_views[bi]; - const float exact_d = _dist_cmp->compare(aligned_query_T, view.coords, - static_cast(aligned_dim)); + const float exact_d = _dist_cmp->compare(aligned_query_T, view.coords, static_cast(aligned_dim)); full_retset.push_back(Neighbor(id, exact_d)); // PQ-rank neighbors and admit unvisited ones to the search frontier. @@ -407,8 +438,8 @@ void unified_index_ssd::cached_beam_search(const T *query, uint64_t K, uint64 for (size_t i = 0; i < out_count; ++i) { indices[i] = static_cast(full_retset[i].id); - distances[i] = (this->_metric == diskann::Metric::INNER_PRODUCT) ? -full_retset[i].distance - : full_retset[i].distance; + distances[i] = + (this->_metric == diskann::Metric::INNER_PRODUCT) ? -full_retset[i].distance : full_retset[i].distance; } for (size_t i = out_count; i < K; ++i) { diff --git a/src/unified_label_data.cpp b/src/unified_label_data.cpp index 6566e75003..0561647f7a 100644 --- a/src/unified_label_data.cpp +++ b/src/unified_label_data.cpp @@ -26,14 +26,16 @@ void unified_label_data_base::load(UnifiedIndexReader &r, const UnifiedIndexHead _has_labels = false; _use_universal_label = false; _universal_label = 0; + _has_universal_medoid = false; + _universal_medoid = 0; _label_map.clear(); if ((h.flags & HAS_LABELS) == 0 || h.label_encoding == LabelEncoding::None) { // Factory shouldn't construct a derived label-data object in this // case; throw if it happens through some other path. - throw ANNException("unified_label_data_base::load called on a header with no labels", -1, __FUNCSIG__, - __FILE__, __LINE__); + throw ANNException("unified_label_data_base::load called on a header with no labels", -1, __FUNCSIG__, __FILE__, + __LINE__); } _has_labels = true; @@ -50,6 +52,15 @@ void unified_label_data_base::load(UnifiedIndexReader &r, const UnifiedIndexHead { _use_universal_label = true; _universal_label = static_cast(h.universal_label); + for (const auto &entry : _label_map) + { + if (entry.second.label_int == _universal_label) + { + _has_universal_medoid = true; + _universal_medoid = entry.second.medoid; + break; + } + } } } @@ -97,27 +108,42 @@ bool unified_label_data_base::get_converted_label(const std::string &s, uint32_t return true; } -void unified_label_data_base::resolve_filters(const std::vector &filter_label_strings, - std::vector &out_label_ints, - std::vector &out_medoids) const +filter_resolution_result unified_label_data_base::resolve_filters(const std::vector &filter_label_strings, + std::vector &out_label_ints, + std::vector &out_medoids) const { out_label_ints.clear(); out_medoids.clear(); out_label_ints.reserve(filter_label_strings.size()); out_medoids.reserve(filter_label_strings.size()); + + filter_resolution_result result; for (const auto &s : filter_label_strings) { auto it = _label_map.find(s); if (it == _label_map.end()) { - throw ANNException(std::string("unified_label_data: unknown filter label string: ") + s, -1, __FUNCSIG__, - __FILE__, __LINE__); + continue; } + + result.label_valid = true; // Single probe yields both the label int (for the match proxy) and the // per-label medoid (for init-id seeding). out_label_ints.push_back(it->second.label_int); out_medoids.push_back(it->second.medoid); } + + if (!result.label_valid && _use_universal_label) + { + out_label_ints.push_back(_universal_label); + if (_has_universal_medoid) + { + out_medoids.push_back(_universal_medoid); + } + } + + result.should_search = result.label_valid || (_use_universal_label && _has_universal_medoid); + return result; } void unified_label_data_base::collect_label_medoids(std::vector &out) const @@ -183,8 +209,8 @@ void unified_label_data_integer::load_encoding(UnifiedIndexReader &r, const Unif } if (h.per_point_labels_len % sizeof(uint32_t) != 0) { - throw ANNException("unified_label_data_integer: labels region size is not a uint32 multiple", -1, - __FUNCSIG__, __FILE__, __LINE__); + throw ANNException("unified_label_data_integer: labels region size is not a uint32 multiple", -1, __FUNCSIG__, + __FILE__, __LINE__); } const size_t total_labels = h.per_point_labels_len / sizeof(uint32_t); diff --git a/tests/unified_index_tests.cpp b/tests/unified_index_tests.cpp index de897f18b3..9966eb1237 100644 --- a/tests/unified_index_tests.cpp +++ b/tests/unified_index_tests.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -158,12 +159,12 @@ std::vector build_dict_bytes(const std::vector> &dict_entries, - const std::vector> &per_point_label_ints, - uint64_t total_labels, uint32_t universal_label) + const std::vector> &per_point_label_ints, uint64_t total_labels, + uint32_t universal_label) { UnifiedIndexWriter w(path); - const uint64_t dim = 4; - const uint64_t aligned_dim = 4; + const uint64_t dim = 8; + const uint64_t aligned_dim = 8; const uint32_t max_degree = 4; w.begin(npts, dim, aligned_dim, max_degree, DataTypeTag::Float, MetricTag::L2, /*start_node=*/0); @@ -195,15 +196,15 @@ void write_bitmask_labels_file(const std::string &path, uint64_t npts, } const uint64_t bitmap_bytes_len = bitmask.size() * sizeof(uint64_t); w.write_labels_bitmask(total_labels, universal_label, dict_bytes.data(), dict_bytes.size(), bitmask.data(), - bitmap_bytes_len); + bitmap_bytes_len); w.finalize(); } void write_integer_labels_file(const std::string &path, uint64_t npts, const std::vector> &dict_entries, - const std::vector> &per_point_label_ints, - uint64_t total_labels, uint32_t universal_label) + const std::vector> &per_point_label_ints, uint64_t total_labels, + uint32_t universal_label) { UnifiedIndexWriter w(path); const uint64_t dim = 4; @@ -234,7 +235,7 @@ void write_integer_labels_file(const std::string &path, uint64_t npts, flat.insert(flat.end(), per_point_label_ints[i].begin(), per_point_label_ints[i].end()); } w.write_labels_integer(total_labels, universal_label, dict_bytes.data(), dict_bytes.size(), flat.data(), - flat.size() * sizeof(uint32_t), offsets.data()); + flat.size() * sizeof(uint32_t), offsets.data()); w.finalize(); } @@ -299,8 +300,7 @@ inline std::vector brute_force_topk(const std::vector pairs.size()) K = pairs.size(); std::vector out(K); @@ -402,12 +402,11 @@ BOOST_AUTO_TEST_CASE(bitmask_load_and_match_proxy) // (simple_bitmask_full_val::merge_bitmask_val) indexes word 99/64 == 1 of a // single-word query buffer and corrupts the heap. const std::vector> dict = { - {"red", 10u, 0u}, {"green", 11u, 1u}, {"blue", 12u, 2u}}; - const std::vector> per_point = { - {10u}, // point 0 -> red - {11u, 12u}, // point 1 -> green, blue - {12u}, // point 2 -> blue - {10u, 11u}}; // point 3 -> red, green + {"red", 10u, 0u}, {"green", 11u, 1u}, {"blue", 12u, 2u}, {"universal", 99u, 3u}}; + const std::vector> per_point = {{10u}, // point 0 -> red + {11u, 12u}, // point 1 -> green, blue + {12u}, // point 2 -> blue + {10u, 11u, 99u}}; // point 3 -> red, green, universal write_bitmask_labels_file(sf.path, npts, dict, per_point, /*total_labels=*/100, /*universal_label=*/99); UnifiedIndexReader r(sf.path); @@ -415,7 +414,7 @@ BOOST_AUTO_TEST_CASE(bitmask_load_and_match_proxy) BOOST_REQUIRE(labels != nullptr); BOOST_CHECK_EQUAL(static_cast(labels->encoding()), static_cast(LabelEncoding::Bitmask)); BOOST_CHECK(labels->has_labels()); - BOOST_CHECK_EQUAL(labels->num_labels(), 3u); + BOOST_CHECK_EQUAL(labels->num_labels(), 4u); BOOST_CHECK(labels->is_valid_label("red")); BOOST_CHECK(labels->is_valid_label("green")); BOOST_CHECK(!labels->is_valid_label("yellow")); @@ -432,27 +431,27 @@ BOOST_AUTO_TEST_CASE(bitmask_load_and_match_proxy) BOOST_CHECK_EQUAL(blue_medoids[0], 2u); // collect_label_medoids appends every label's entry-point medoid (one per - // label). Dict above maps red/green/blue -> medoids 0/1/2. It appends (does - // not clear), so a pre-existing element must be preserved. Order is - // unspecified (hash-map iteration), so sort before comparing. + // label), including the universal label. It appends (does not clear), so a + // pre-existing element must be preserved. Order is unspecified (hash-map + // iteration), so sort before comparing. std::vector all_medoids = {42u}; labels->collect_label_medoids(all_medoids); - BOOST_REQUIRE_EQUAL(all_medoids.size(), 4u); + BOOST_REQUIRE_EQUAL(all_medoids.size(), 5u); std::sort(all_medoids.begin(), all_medoids.end()); - const std::vector expected_medoids = {0u, 1u, 2u, 42u}; + const std::vector expected_medoids = {0u, 1u, 2u, 3u, 42u}; BOOST_CHECK_EQUAL_COLLECTIONS(all_medoids.begin(), all_medoids.end(), expected_medoids.begin(), expected_medoids.end()); - // Match-proxy: filter "blue" -> matches points 1, 2, but NOT 0, 3. + // Match-proxy: filter "blue" -> matches points 1, 2 and universal point 3. auto proxy = labels->make_match_proxy(blue_ints); BOOST_REQUIRE(proxy != nullptr); BOOST_CHECK(!proxy->contain_filtered_label(0)); BOOST_CHECK(proxy->contain_filtered_label(1)); BOOST_CHECK(proxy->contain_filtered_label(2)); - BOOST_CHECK(!proxy->contain_filtered_label(3)); + BOOST_CHECK(proxy->contain_filtered_label(3)); } -BOOST_AUTO_TEST_CASE(bitmask_unknown_label_string_throws) +BOOST_AUTO_TEST_CASE(bitmask_unknown_label_without_universal_is_not_searchable) { ScopedFile sf(tmp_path("bitmask_unknown")); write_bitmask_labels_file(sf.path, /*npts=*/2, @@ -462,7 +461,56 @@ BOOST_AUTO_TEST_CASE(bitmask_unknown_label_string_throws) auto labels = make_unified_label_data(r, r.header(), r.header().npts); BOOST_REQUIRE(labels != nullptr); std::vector ints, medoids; - BOOST_CHECK_THROW(labels->resolve_filters({"nonexistent"}, ints, medoids), ANNException); + const auto resolution = labels->resolve_filters({"nonexistent"}, ints, medoids); + BOOST_CHECK(!resolution.label_valid); + BOOST_CHECK(!resolution.should_search); + BOOST_CHECK(ints.empty()); + BOOST_CHECK(medoids.empty()); +} + +BOOST_AUTO_TEST_CASE(bitmask_unknown_label_with_universal_uses_universal) +{ + ScopedFile sf(tmp_path("bitmask_unknown_universal")); + write_bitmask_labels_file(sf.path, /*npts=*/2, + /*dict=*/{{"a", 1u, 1u}, {"universal", 2u, 0u}}, + /*per_point=*/{{2u}, {1u}}, /*total_labels=*/3, + /*universal_label=*/2); + UnifiedIndexReader r(sf.path); + auto labels = make_unified_label_data(r, r.header(), r.header().npts); + BOOST_REQUIRE(labels != nullptr); + + std::vector ints, medoids; + const auto resolution = labels->resolve_filters({"nonexistent"}, ints, medoids); + BOOST_CHECK(!resolution.label_valid); + BOOST_CHECK(resolution.should_search); + BOOST_REQUIRE_EQUAL(ints.size(), 1u); + BOOST_CHECK_EQUAL(ints[0], 2u); + BOOST_REQUIRE_EQUAL(medoids.size(), 1u); + BOOST_CHECK_EQUAL(medoids[0], 0u); + + auto proxy = labels->make_match_proxy(ints); + BOOST_REQUIRE(proxy != nullptr); + BOOST_CHECK(proxy->contain_filtered_label(0)); + BOOST_CHECK(!proxy->contain_filtered_label(1)); +} + +BOOST_AUTO_TEST_CASE(bitmask_missing_universal_medoid_is_not_searchable) +{ + ScopedFile sf(tmp_path("bitmask_missing_universal_medoid")); + write_bitmask_labels_file(sf.path, /*npts=*/2, + /*dict=*/{{"a", 1u, 1u}}, /*per_point=*/{{2u}, {1u}}, /*total_labels=*/3, + /*universal_label=*/2); + UnifiedIndexReader r(sf.path); + auto labels = make_unified_label_data(r, r.header(), r.header().npts); + BOOST_REQUIRE(labels != nullptr); + + std::vector ints, medoids; + const auto resolution = labels->resolve_filters({"nonexistent"}, ints, medoids); + BOOST_CHECK(!resolution.label_valid); + BOOST_CHECK(!resolution.should_search); + BOOST_REQUIRE_EQUAL(ints.size(), 1u); + BOOST_CHECK_EQUAL(ints[0], 2u); + BOOST_CHECK(medoids.empty()); } BOOST_AUTO_TEST_SUITE_END() @@ -478,16 +526,16 @@ BOOST_AUTO_TEST_CASE(integer_load_and_match_proxy) ScopedFile sf(tmp_path("integer")); const uint64_t npts = 4; const std::vector> dict = { - {"red", 10u, 0u}, {"green", 11u, 1u}, {"blue", 12u, 2u}}; - const std::vector> per_point = {{10u}, {11u, 12u}, {12u}, {10u, 11u}}; - write_integer_labels_file(sf.path, npts, dict, per_point, /*total_labels=*/13, /*universal_label=*/99); + {"red", 10u, 0u}, {"green", 11u, 1u}, {"blue", 12u, 2u}, {"universal", 99u, 3u}}; + const std::vector> per_point = {{10u}, {11u, 12u}, {12u}, {10u, 11u, 99u}}; + write_integer_labels_file(sf.path, npts, dict, per_point, /*total_labels=*/100, /*universal_label=*/99); UnifiedIndexReader r(sf.path); auto labels = make_unified_label_data(r, r.header(), r.header().npts); BOOST_REQUIRE(labels != nullptr); BOOST_CHECK_EQUAL(static_cast(labels->encoding()), static_cast(LabelEncoding::Integer)); BOOST_CHECK(labels->has_labels()); - BOOST_CHECK_EQUAL(labels->num_labels(), 3u); + BOOST_CHECK_EQUAL(labels->num_labels(), 4u); std::vector blue_ints, blue_medoids; labels->resolve_filters({"blue"}, blue_ints, blue_medoids); @@ -496,7 +544,7 @@ BOOST_AUTO_TEST_CASE(integer_load_and_match_proxy) BOOST_CHECK(!proxy->contain_filtered_label(0)); BOOST_CHECK(proxy->contain_filtered_label(1)); BOOST_CHECK(proxy->contain_filtered_label(2)); - BOOST_CHECK(!proxy->contain_filtered_label(3)); + BOOST_CHECK(proxy->contain_filtered_label(3)); } BOOST_AUTO_TEST_CASE(integer_match_proxy_sorts_filter_labels) @@ -524,13 +572,13 @@ BOOST_AUTO_TEST_CASE(integer_match_proxy_sorts_filter_labels) labels->resolve_filters({"green", "red"}, ints, medoids); auto proxy = labels->make_match_proxy(ints); BOOST_REQUIRE(proxy != nullptr); - BOOST_CHECK(proxy->contain_filtered_label(0)); // red -- the case the sort fixes - BOOST_CHECK(proxy->contain_filtered_label(1)); // green + BOOST_CHECK(proxy->contain_filtered_label(0)); // red -- the case the sort fixes + BOOST_CHECK(proxy->contain_filtered_label(1)); // green BOOST_CHECK(!proxy->contain_filtered_label(2)); // blue only - BOOST_CHECK(proxy->contain_filtered_label(3)); // red + green + BOOST_CHECK(proxy->contain_filtered_label(3)); // red + green } -BOOST_AUTO_TEST_CASE(integer_unknown_label_string_throws) +BOOST_AUTO_TEST_CASE(integer_unknown_label_without_universal_is_not_searchable) { ScopedFile sf(tmp_path("integer_unknown")); write_integer_labels_file(sf.path, /*npts=*/2, @@ -540,7 +588,11 @@ BOOST_AUTO_TEST_CASE(integer_unknown_label_string_throws) auto labels = make_unified_label_data(r, r.header(), r.header().npts); BOOST_REQUIRE(labels != nullptr); std::vector ints, medoids; - BOOST_CHECK_THROW(labels->resolve_filters({"nonexistent"}, ints, medoids), ANNException); + const auto resolution = labels->resolve_filters({"nonexistent"}, ints, medoids); + BOOST_CHECK(!resolution.label_valid); + BOOST_CHECK(!resolution.should_search); + BOOST_CHECK(ints.empty()); + BOOST_CHECK(medoids.empty()); } BOOST_AUTO_TEST_SUITE_END() @@ -554,8 +606,7 @@ BOOST_AUTO_TEST_SUITE(unified_label_data_factory_tests) BOOST_AUTO_TEST_CASE(factory_no_labels_returns_null) { ScopedFile sf(tmp_path("no_labels")); - write_minimal_unified_file(sf.path, /*npts=*/2, /*dim=*/4, /*aligned_dim=*/4, /*max_degree=*/4, - DataTypeTag::Float); + write_minimal_unified_file(sf.path, /*npts=*/2, /*dim=*/4, /*aligned_dim=*/4, /*max_degree=*/4, DataTypeTag::Float); // write_minimal_unified_file doesn't call finalize. Do it here. { // Re-open as writer? No -- minimal helper writes nothing post-medoids. @@ -680,8 +731,7 @@ BOOST_AUTO_TEST_CASE(factory_ssd_dispatches_on_data_type) ScopedFile sf(tmp_path("ssd_disp")); { UnifiedIndexWriter w(sf.path); - w.begin(/*npts=*/2, /*dim=*/aligned_dim, aligned_dim, /*max_degree=*/4, DataTypeTag::Float, MetricTag::L2, - 0); + w.begin(/*npts=*/2, /*dim=*/aligned_dim, aligned_dim, /*max_degree=*/4, DataTypeTag::Float, MetricTag::L2, 0); w.begin_graph_region(); std::vector coord(aligned_dim, 0.0f); const uint32_t nb = 0; @@ -708,8 +758,7 @@ BOOST_AUTO_TEST_CASE(factory_ssd_dispatches_on_data_type) // Either the SSD index class flags the missing PQ ("requires HAS_PQ"), // or the file fails to open for some other reason. Either way the // throw should originate from our SSD code path. - BOOST_CHECK(msg.find("HAS_PQ") != std::string::npos || - msg.find("unified_index_ssd") != std::string::npos || + BOOST_CHECK(msg.find("HAS_PQ") != std::string::npos || msg.find("unified_index_ssd") != std::string::npos || msg.find("unified_node_store_ssd") != std::string::npos); } BOOST_CHECK(threw); @@ -807,6 +856,78 @@ BOOST_AUTO_TEST_CASE(memory_search_validates_K_and_L) BOOST_CHECK_THROW(idx->search(sctx), ANNException); } +BOOST_AUTO_TEST_CASE(memory_invalid_label_returns_empty_success) +{ + ScopedFile sf(tmp_path("memory_invalid_label")); + write_bitmask_labels_file(sf.path, /*npts=*/2, + /*dict=*/{{"a", 1u, 0u}}, /*per_point=*/{{1u}, {}}, /*total_labels=*/2, + /*universal_label=*/0); + + UnifiedLoadContext load_ctx; + load_ctx.path = sf.path; + load_ctx.num_threads = 1; + load_ctx.search_l = 4; + auto idx = make_unified_index_memory(load_ctx); + BOOST_REQUIRE(idx != nullptr); + + std::vector query(8, 0.0f); + std::vector out_ids(2, 0); + std::vector out_dists(2, 0.0f); + QueryStats stats; + + UnifiedSearchContext search_ctx; + search_ctx.query = query.data(); + search_ctx.K = 2; + search_ctx.L = 4; + search_ctx.filter_labels = {"missing"}; + search_ctx.indices = out_ids.data(); + search_ctx.distances = out_dists.data(); + search_ctx.stats = &stats; + + BOOST_CHECK_NO_THROW(idx->search(search_ctx)); + BOOST_CHECK(!stats.label_valid); + for (size_t i = 0; i < out_ids.size(); ++i) + { + BOOST_CHECK_EQUAL(out_ids[i], std::numeric_limits::max()); + BOOST_CHECK_EQUAL(out_dists[i], std::numeric_limits::max()); + } +} + +BOOST_AUTO_TEST_CASE(memory_invalid_label_with_universal_searches) +{ + ScopedFile sf(tmp_path("memory_invalid_label_universal")); + write_bitmask_labels_file(sf.path, /*npts=*/2, + /*dict=*/{{"a", 1u, 1u}, {"universal", 2u, 0u}}, + /*per_point=*/{{2u}, {1u}}, /*total_labels=*/3, + /*universal_label=*/2); + + UnifiedLoadContext load_ctx; + load_ctx.path = sf.path; + load_ctx.num_threads = 1; + load_ctx.search_l = 4; + auto idx = make_unified_index_memory(load_ctx); + BOOST_REQUIRE(idx != nullptr); + + std::vector query(8, 0.0f); + std::vector out_ids(1, std::numeric_limits::max()); + std::vector out_dists(1, std::numeric_limits::max()); + QueryStats stats; + + UnifiedSearchContext search_ctx; + search_ctx.query = query.data(); + search_ctx.K = 1; + search_ctx.L = 4; + search_ctx.filter_labels = {"missing"}; + search_ctx.indices = out_ids.data(); + search_ctx.distances = out_dists.data(); + search_ctx.stats = &stats; + + BOOST_CHECK_NO_THROW(idx->search(search_ctx)); + BOOST_CHECK(!stats.label_valid); + BOOST_CHECK_EQUAL(out_ids[0], 0u); + BOOST_CHECK_LT(out_dists[0], std::numeric_limits::max()); +} + BOOST_AUTO_TEST_SUITE_END() // =========================================================================== @@ -1072,7 +1193,7 @@ BOOST_AUTO_TEST_CASE(builder_with_pq_emits_ssd_loadable_file) ctx.L = 32; ctx.alpha = 1.2f; ctx.num_threads = 1; - ctx.pq_dim = 4; // PQ-compress 16-dim into 4 chunks + ctx.pq_dim = 4; // PQ-compress 16-dim into 4 chunks ctx.pq_sampling_rate = 1.0; // train on full data (tiny set) unified_index_builder builder; @@ -1201,8 +1322,7 @@ struct ScopedLegacyMemFiles }; // Fraction of `a`'s top-K ids that also appear in `b`'s top-K (set overlap). -template -double topk_overlap(const std::vector &a, const std::vector &b) +template double topk_overlap(const std::vector &a, const std::vector &b) { std::unordered_set bs; for (auto id : b) @@ -1267,8 +1387,8 @@ void make_legacy_ssd_from_index(Index &idx, const std // 2) Train PQ once. These files feed BOTH the legacy PQFlashIndex and (via // slurp) the unified file, so both sides use byte-identical codes. - diskann::generate_quantized_data(data_file, pq_pivots, pq_codes, metric, /*p_val=*/1.0, - num_pq_chunks, /*use_opq=*/false, /*codebook_prefix=*/""); + diskann::generate_quantized_data(data_file, pq_pivots, pq_codes, metric, /*p_val=*/1.0, num_pq_chunks, + /*use_opq=*/false, /*codebook_prefix=*/""); // 3) Pack coords + adjacency into the sector-aligned legacy disk index. diskann::create_disk_layout(data_file, mem_index, disk_index); @@ -1724,10 +1844,8 @@ BOOST_AUTO_TEST_CASE(memory_filtered_parity_legacy_vs_unified) { // save_path_prefix == legacy prefix so the labels_map + label files that // build/save emit all co-locate for the subsequent legacy load(). - IndexFilterParams fp = IndexFilterParamsBuilder() - .with_label_file(rawlabels_sf.path) - .with_save_path_prefix(legacy.prefix) - .build(); + IndexFilterParams fp = + IndexFilterParamsBuilder().with_label_file(rawlabels_sf.path).with_save_path_prefix(legacy.prefix).build(); idx.build(data_sf.path, npts, fp); } @@ -1935,10 +2053,8 @@ BOOST_AUTO_TEST_CASE(ssd_filtered_parity_legacy_vs_unified) /*concurrent_consolidate=*/false, /*pq_dist_build=*/false, /*num_pq_chunks=*/0, /*use_opq=*/false, /*filtered_index=*/true); { - IndexFilterParams fp = IndexFilterParamsBuilder() - .with_label_file(rawlabels_sf.path) - .with_save_path_prefix(legacy.prefix) - .build(); + IndexFilterParams fp = + IndexFilterParamsBuilder().with_label_file(rawlabels_sf.path).with_save_path_prefix(legacy.prefix).build(); idx.build(data_sf.path, npts, fp); } @@ -2029,8 +2145,95 @@ BOOST_AUTO_TEST_CASE(ssd_filtered_parity_legacy_vs_unified) // so with the shared graph + PQ the results should be highly similar. BOOST_CHECK_GE(avg_overlap, 0.90); + // An unknown label without a universal label is a successful empty search + // and is reported through QueryStats for the caller's invalid-label counter. + std::vector invalid_query(dim); + for (uint32_t j = 0; j < dim; ++j) + invalid_query[j] = det_float(/*seed=*/333, /*i=*/0, j); + std::vector invalid_ids(K, 0); + std::vector invalid_dists(K, 0.0f); + QueryStats invalid_stats; + UnifiedSearchContext invalid_ctx; + invalid_ctx.query = invalid_query.data(); + invalid_ctx.K = K; + invalid_ctx.L = search_L; + invalid_ctx.indices = invalid_ids.data(); + invalid_ctx.distances = invalid_dists.data(); + invalid_ctx.beam_width = beam; + invalid_ctx.filter_labels = {"missing"}; + invalid_ctx.stats = &invalid_stats; + BOOST_CHECK_NO_THROW(uidx->search(invalid_ctx)); + BOOST_CHECK(!invalid_stats.label_valid); + for (size_t i = 0; i < K; ++i) + { + BOOST_CHECK_EQUAL(invalid_ids[i], std::numeric_limits::max()); + BOOST_CHECK_EQUAL(invalid_dists[i], std::numeric_limits::max()); + } + uidx.reset(); ureader->close(); + + // Reuse the same unified index and designate existing label "1" as the + // universal label. Unknown-label search must now run and return only points + // carrying that universal label while still reporting label_valid=false. + ScopedFile universal_sf(tmp_path("parity_fssd_universal")); + { + std::ifstream src(unified_sf.path, std::ios::binary); + std::ofstream dst(universal_sf.path, std::ios::binary | std::ios::trunc); + BOOST_REQUIRE(src.is_open()); + BOOST_REQUIRE(dst.is_open()); + dst << src.rdbuf(); + } + { + std::fstream file(universal_sf.path, std::ios::binary | std::ios::in | std::ios::out); + BOOST_REQUIRE(file.is_open()); + UnifiedIndexHeader header{}; + file.read(reinterpret_cast(&header), sizeof(header)); + BOOST_REQUIRE(file.good()); + const uint32_t universal_label = idx.get_converted_label("1"); + BOOST_REQUIRE_NE(universal_label, 0u); + header.universal_label = universal_label; + file.seekp(0, std::ios::beg); + file.write(reinterpret_cast(&header), sizeof(header)); + BOOST_REQUIRE(file.good()); + } + + UnifiedLoadContext universal_load_ctx; + universal_load_ctx.path = universal_sf.path; + universal_load_ctx.num_threads = 1; + universal_load_ctx.search_l = search_L; + auto universal_reader = make_reader(); + auto universal_idx = make_unified_index_ssd(universal_reader, universal_load_ctx); + BOOST_REQUIRE(universal_idx != nullptr); + + std::vector universal_ids(K, std::numeric_limits::max()); + std::vector universal_dists(K, std::numeric_limits::max()); + QueryStats universal_stats; + UnifiedSearchContext universal_ctx; + universal_ctx.query = invalid_query.data(); + universal_ctx.K = K; + universal_ctx.L = search_L; + universal_ctx.indices = universal_ids.data(); + universal_ctx.distances = universal_dists.data(); + universal_ctx.beam_width = beam; + universal_ctx.filter_labels = {"missing"}; + universal_ctx.stats = &universal_stats; + BOOST_CHECK_NO_THROW(universal_idx->search(universal_ctx)); + BOOST_CHECK(!universal_stats.label_valid); + + size_t universal_result_count = 0; + for (uint64_t id : universal_ids) + { + if (id != std::numeric_limits::max()) + { + ++universal_result_count; + BOOST_CHECK(point_has_label(label_sets, static_cast(id), "1")); + } + } + BOOST_CHECK_GT(universal_result_count, 0u); + + universal_idx.reset(); + universal_reader->close(); } BOOST_AUTO_TEST_SUITE_END() @@ -2061,8 +2264,8 @@ BOOST_AUTO_TEST_CASE(memory_stats_unfiltered) // Memory keeps the full graph region resident, so total > 0 and is the sum // of the parts. BOOST_CHECK_GT(st.total_mem_usage, 0u); - BOOST_CHECK_EQUAL(st.total_mem_usage, st.node_mem_usage + st.graph_mem_usage + st.label_mem_usage + - st.tag_memory_usage); + BOOST_CHECK_EQUAL(st.total_mem_usage, + st.node_mem_usage + st.graph_mem_usage + st.label_mem_usage + st.tag_memory_usage); } BOOST_AUTO_TEST_CASE(ssd_stats_pq_codes) @@ -2097,8 +2300,8 @@ BOOST_AUTO_TEST_CASE(ssd_stats_pq_codes) // SSD node_mem_usage == resident PQ codes == npts * n_chunks; graph on disk. BOOST_CHECK_EQUAL(st.node_mem_usage, static_cast(npts) * pq_dim); BOOST_CHECK_EQUAL(st.graph_mem_usage, 0u); - BOOST_CHECK_EQUAL(st.total_mem_usage, st.node_mem_usage + st.graph_mem_usage + st.label_mem_usage + - st.tag_memory_usage); + BOOST_CHECK_EQUAL(st.total_mem_usage, + st.node_mem_usage + st.graph_mem_usage + st.label_mem_usage + st.tag_memory_usage); idx.reset(); reader->close();