From 8cbcfe7b68a978d8fb07716fac2997b1a9817815 Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Fri, 26 Jun 2026 10:58:23 +0200 Subject: [PATCH 1/9] Add layered FTS search capabilities ngrams, fuzzy search, incremental sidecar index maintenance --- README.md | 38 +- src/fts_extension.cpp | 2 + src/fts_indexing.cpp | 598 +++++++++++++++++++++++++- test/sql/fts/test_layered_search.test | 129 ++++++ 4 files changed, 754 insertions(+), 13 deletions(-) create mode 100644 test/sql/fts/test_layered_search.test diff --git a/README.md b/README.md index d3a8a76..af7d13e 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The extension adds two `PRAGMA` statements to DuckDB: one to create, and one to create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', stopwords = 'english', ignore = '(\\.|[^a-z])+', strip_accents = 1, lower = 1, overwrite = 0, - cluster_terms = 0) + incremental = 0, cluster_terms = 0, layered_search = 0) ``` `PRAGMA` that creates a FTS index for the specified table. @@ -40,7 +40,9 @@ create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', | `strip_accents` | `BOOLEAN` | Whether to remove accents (e.g., convert `á` to `a`). Defaults to `1` | | `lower` | `BOOLEAN` | Whether to convert all text to lowercase. Defaults to `1` | | `overwrite` | `BOOLEAN` | Whether to overwrite an existing index on a table. Defaults to `0` | +| `incremental` | `BOOLEAN` | Whether to keep the index in sync for subsequent `INSERT` and `DELETE` statements using triggers. Defaults to `0` | | `cluster_terms` | `BOOLEAN` | Whether to physically order the generated `terms` table by `termid`, `fieldid`, and `docid`. This can improve query-time pruning for direct reads from the FTS tables. Defaults to `0` | +| `layered_search` | `BOOLEAN` | Whether to build a dictionary trigram sidecar and layered BM25 search macros for exact, prefix, substring, and fuzzy query expansion. This implies `cluster_terms`. Defaults to `0` | @@ -76,6 +78,34 @@ When an index is built, this retrieval macro is created that can be used to sear | `b` | `DOUBLE` | Parameter _b_ in the Okapi BM25 retrieval model. Defaults to `0.75` | | `conjunctive` | `BOOLEAN` | Whether to make the query conjunctive i.e., all terms in the query string must be present in order for a document to be retrieved | +### Layered BM25 Search + +```python +search_layered_bm25(query_string, fields := NULL, top_k := 50, k := 1.2, + b := 0.75, term_limit := 32, max_df_ratio := 0.15, + max_df := 50000, enable_prefix := true, + enable_substring := true, enable_fuzzy := true, + enable_short_fuzzy := true, expand_exact_terms := false) + +match_layered_bm25(input_id, query_string, fields := NULL, k := 1.2, + b := 0.75, term_limit := 32, max_df_ratio := 0.15, + max_df := 50000, enable_prefix := true, + enable_substring := true, enable_fuzzy := true, + enable_short_fuzzy := true, expand_exact_terms := false) +``` + +When `layered_search` is enabled, the extension builds two additional +dictionary sidecar tables, `term_stats` and `term_grams`, plus a +length-clustered copy of the dictionary metadata. These tables grow with the +term dictionary rather than with the document corpus and are used to expand +query terms before scoring over the standard FTS `terms` table. + +`search_layered_bm25` returns `docname`, `score`, and `rank`. +`match_layered_bm25` is the scalar form for use against an input table row. +Exact terms are always included. Prefix, substring, and fuzzy alternatives can +be toggled with the corresponding parameters. Numeric query terms are searched +exactly but are excluded from trigram/fuzzy expansion. + ### `stem` Function ```python @@ -160,8 +190,10 @@ ORDER BY score DESC; |---------------------|------------------------------------------------------------|------:| | doc2 | The cat is a domestic species of small carnivorous mammal. | 0.0 | -> Warning The FTS index will not update automatically when input table changes. -> A workaround of this limitation can be recreating the index to refresh. +> Warning Without `incremental = true`, the FTS index will not update +> automatically when the input table changes. With `incremental = true`, the +> index and layered sidecar are maintained for `INSERT` and `DELETE` +> statements on tables that support triggers. ## Stemmers diff --git a/src/fts_extension.cpp b/src/fts_extension.cpp index 03a3163..41ff9e5 100644 --- a/src/fts_extension.cpp +++ b/src/fts_extension.cpp @@ -71,6 +71,8 @@ static void LoadInternal(ExtensionLoader &loader) { create_fts_index_func.named_parameters["incremental"] = LogicalType::BOOLEAN; create_fts_index_func.named_parameters["cluster_terms"] = LogicalType::BOOLEAN; + create_fts_index_func.named_parameters["layered_search"] = + LogicalType::BOOLEAN; auto drop_fts_index_func = PragmaFunction::PragmaCall( "drop_fts_index", FTSIndexing::DropFTSIndexQuery, {LogicalType::VARCHAR}); diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index e506f07..ae49d3b 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -64,10 +64,36 @@ static vector GetFTSDeleteTriggerNames(const QualifiedName &qname) { prefix + "30_dict_prune", prefix + "40_stats"}; } +static vector GetFTSLayeredInsertTriggerNames( + const QualifiedName &qname) { + auto prefix = StringUtil::Format("__fts_%s_ai_", GetFTSSchemaName(qname)); + return {prefix + "15_term_stats", + prefix + "16_term_stats_by_len", + prefix + "17_term_grams", + prefix + "35_term_stats_df", + prefix + "36_term_stats_by_len_df"}; +} + +static vector GetFTSLayeredDeleteTriggerNames( + const QualifiedName &qname) { + auto prefix = StringUtil::Format("__fts_%s_ad_", GetFTSSchemaName(qname)); + return {prefix + "23_term_stats_df", + prefix + "24_term_stats_by_len_df", + prefix + "25_term_grams_prune", + prefix + "26_term_stats_by_len_prune", + prefix + "27_term_stats_prune"}; +} + static vector GetFTSTriggerNames(const QualifiedName &qname) { auto result = GetFTSInsertTriggerNames(qname); + auto layered_insert_triggers = GetFTSLayeredInsertTriggerNames(qname); + result.insert(result.end(), layered_insert_triggers.begin(), + layered_insert_triggers.end()); auto delete_triggers = GetFTSDeleteTriggerNames(qname); result.insert(result.end(), delete_triggers.begin(), delete_triggers.end()); + auto layered_delete_triggers = GetFTSLayeredDeleteTriggerNames(qname); + result.insert(result.end(), layered_delete_triggers.begin(), + layered_delete_triggers.end()); return result; } @@ -80,6 +106,16 @@ static string GetFTSBuildDictTable(const QualifiedName &qname) { return SQLIdentifier::ToString("__fts_build_dict_" + GetFTSSchemaName(qname)); } +static string GetFTSTermStatsTermIndex(const QualifiedName &qname) { + return SQLIdentifier::ToString("__fts_" + GetFTSSchemaName(qname) + + "_term_stats_term_idx"); +} + +static string GetFTSTermGramsGramIndex(const QualifiedName &qname) { + return SQLIdentifier::ToString("__fts_" + GetFTSSchemaName(qname) + + "_term_grams_gram_idx"); +} + static bool TableExists(ClientContext &context, const QualifiedName &qname) { return Catalog::GetEntry( context, qname.Catalog(), qname.Schema(), qname.Name(), @@ -311,6 +347,55 @@ static string IndexTablesScript(const string &input_id, return result; } +static string LayeredSidecarScript(const QualifiedName &qname) { + // clang-format off + string result = R"( + CREATE TABLE %fts_schema%.term_stats AS + SELECT termid, + term, + df, + length(term)::BIGINT AS term_len, + greatest(length(term) - 2, 0)::BIGINT AS gram_count + FROM %fts_schema%.dict + WHERE term <> ''; + + CREATE TABLE %fts_schema%.term_stats_by_len AS + SELECT termid, + term, + df, + term_len, + gram_count + FROM %fts_schema%.term_stats + ORDER BY term_len, + df, + termid; + + CREATE TABLE %fts_schema%.term_grams AS + SELECT 'g' || lower(hex(substr(term, i, 3))) AS gram, + termid + FROM %fts_schema%.term_stats, + range(1, gram_count + 1) AS r(i) + WHERE gram_count > 0 + AND NOT regexp_full_match(term, '[0-9]+') + ORDER BY gram, + termid; + + CREATE INDEX %term_stats_term_index% ON %fts_schema%.term_stats(term); + CREATE INDEX %term_grams_gram_index% ON %fts_schema%.term_grams(gram); + + ANALYZE %fts_schema%.term_stats; + ANALYZE %fts_schema%.term_stats_by_len; + ANALYZE %fts_schema%.term_grams; + )"; + // clang-format on + + result = StringUtil::Replace(result, "%term_stats_term_index%", + GetFTSTermStatsTermIndex(qname)); + result = StringUtil::Replace(result, "%term_grams_gram_index%", + GetFTSTermGramsGramIndex(qname)); + return result; +} + static string MatchMacroScript() { // clang-format off return R"( @@ -381,6 +466,295 @@ static string MatchMacroScript() { // clang-format on } +static string LayeredSearchMacroScript() { + // clang-format off + return R"( + CREATE MACRO %fts_schema%.search_layered_bm25(query_string, fields := NULL, top_k := 50, k := 1.2, b := 0.75, term_limit := 32, max_df_ratio := 0.15, max_df := 50000, enable_prefix := true, enable_substring := true, enable_fuzzy := true, enable_short_fuzzy := true, expand_exact_terms := false) AS TABLE + WITH params(term_limit, max_df_ratio, max_df, enable_prefix, enable_substring, enable_fuzzy, enable_short_fuzzy, expand_exact_terms) AS ( + SELECT term_limit::BIGINT, + max_df_ratio::DOUBLE, + max_df::BIGINT, + enable_prefix::BOOLEAN, + enable_substring::BOOLEAN, + enable_fuzzy::BOOLEAN, + enable_short_fuzzy::BOOLEAN, + expand_exact_terms::BOOLEAN + ), + df_cap(max_df) AS ( + SELECT least(params.max_df, ceil(stats.num_docs * params.max_df_ratio)::BIGINT) + FROM %fts_schema%.stats AS stats + CROSS JOIN params + ), + raw_tokens AS ( + SELECT DISTINCT stem(unnest(%fts_schema%.tokenize(query_string)), '%stemmer%') AS query_term + ), + query_tokens AS ( + SELECT query_term, + length(query_term)::BIGINT AS query_len, + greatest(length(query_term) - 2, 0)::BIGINT AS query_gram_count, + regexp_full_match(query_term, '[0-9]+') AS is_numeric + FROM raw_tokens + WHERE query_term IS NOT NULL + AND query_term <> '' + ), + exact_terms AS ( + SELECT query_tokens.query_term, + term_stats.termid, + term_stats.term, + term_stats.df, + 1.0::DOUBLE AS expansion_weight, + 'exact' AS match_type + FROM query_tokens + JOIN %fts_schema%.term_stats AS term_stats + ON term_stats.term = query_tokens.query_term + ), + expansion_tokens AS ( + SELECT query_tokens.* + FROM query_tokens + LEFT JOIN exact_terms + ON exact_terms.query_term = query_tokens.query_term + CROSS JOIN params + WHERE params.expand_exact_terms + OR exact_terms.termid IS NULL + ), + query_grams AS ( + SELECT query_term, + 'g' || lower(hex(substr(query_term, i, 3))) AS gram + FROM expansion_tokens, + range(1, query_gram_count + 1) AS r(i) + WHERE query_gram_count > 0 + AND NOT is_numeric + ), + gram_candidates AS ( + SELECT query_grams.query_term, + term_grams.termid, + count(*)::BIGINT AS matching_grams + FROM query_grams + JOIN %fts_schema%.term_grams AS term_grams + ON term_grams.gram = query_grams.gram + JOIN %fts_schema%.term_stats AS term_stats + ON term_stats.termid = term_grams.termid + CROSS JOIN df_cap + WHERE term_stats.df <= df_cap.max_df + GROUP BY query_grams.query_term, + term_grams.termid + ), + gram_expansions AS ( + SELECT gram_candidates.query_term, + term_stats.termid, + term_stats.term, + term_stats.df, + CASE + WHEN params.enable_prefix + AND starts_with(term_stats.term, expansion_tokens.query_term) + AND term_stats.term <> expansion_tokens.query_term + THEN 0.85::DOUBLE + WHEN params.enable_substring + AND contains(term_stats.term, expansion_tokens.query_term) + AND term_stats.term <> expansion_tokens.query_term + AND gram_candidates.matching_grams = expansion_tokens.query_gram_count + THEN 0.75::DOUBLE + WHEN params.enable_fuzzy + AND expansion_tokens.query_len >= 3 + AND gram_candidates.matching_grams >= greatest(1, expansion_tokens.query_gram_count - 1) + AND abs(term_stats.term_len - expansion_tokens.query_len) <= CASE + WHEN expansion_tokens.query_len <= 4 THEN 1 + WHEN expansion_tokens.query_len <= 8 THEN 2 + ELSE 3 + END + AND damerau_levenshtein(expansion_tokens.query_term, term_stats.term) <= CASE + WHEN expansion_tokens.query_len <= 4 THEN 1 + WHEN expansion_tokens.query_len <= 8 THEN 2 + ELSE 3 + END + THEN greatest( + 0.25::DOUBLE, + 1.0::DOUBLE - ( + damerau_levenshtein(expansion_tokens.query_term, term_stats.term)::DOUBLE + / greatest(expansion_tokens.query_len, term_stats.term_len)::DOUBLE + ) + ) + ELSE NULL + END AS expansion_weight, + CASE + WHEN params.enable_prefix + AND starts_with(term_stats.term, expansion_tokens.query_term) + AND term_stats.term <> expansion_tokens.query_term + THEN 'prefix' + WHEN params.enable_substring + AND contains(term_stats.term, expansion_tokens.query_term) + AND term_stats.term <> expansion_tokens.query_term + AND gram_candidates.matching_grams = expansion_tokens.query_gram_count + THEN 'substring' + WHEN params.enable_fuzzy + AND expansion_tokens.query_len >= 3 + AND gram_candidates.matching_grams >= greatest(1, expansion_tokens.query_gram_count - 1) + AND abs(term_stats.term_len - expansion_tokens.query_len) <= CASE + WHEN expansion_tokens.query_len <= 4 THEN 1 + WHEN expansion_tokens.query_len <= 8 THEN 2 + ELSE 3 + END + AND damerau_levenshtein(expansion_tokens.query_term, term_stats.term) <= CASE + WHEN expansion_tokens.query_len <= 4 THEN 1 + WHEN expansion_tokens.query_len <= 8 THEN 2 + ELSE 3 + END + THEN 'fuzzy' + ELSE NULL + END AS match_type + FROM gram_candidates + JOIN expansion_tokens + ON expansion_tokens.query_term = gram_candidates.query_term + JOIN %fts_schema%.term_stats AS term_stats + ON term_stats.termid = gram_candidates.termid + CROSS JOIN params + ), + short_fuzzy_expansions AS ( + SELECT expansion_tokens.query_term, + term_stats.termid, + term_stats.term, + term_stats.df, + greatest( + 0.25::DOUBLE, + 1.0::DOUBLE - ( + damerau_levenshtein(expansion_tokens.query_term, term_stats.term)::DOUBLE + / greatest(expansion_tokens.query_len, term_stats.term_len)::DOUBLE + ) + ) AS expansion_weight, + 'fuzzy' AS match_type + FROM expansion_tokens + JOIN %fts_schema%.term_stats_by_len AS term_stats + ON term_stats.term_len BETWEEN expansion_tokens.query_len - 1 AND expansion_tokens.query_len + 1 + CROSS JOIN params + CROSS JOIN df_cap + WHERE params.enable_fuzzy + AND params.enable_short_fuzzy + AND expansion_tokens.query_len BETWEEN 3 AND 5 + AND NOT expansion_tokens.is_numeric + AND term_stats.df <= df_cap.max_df + AND term_stats.term <> expansion_tokens.query_term + AND damerau_levenshtein(expansion_tokens.query_term, term_stats.term) <= 1 + ), + limited_expansions AS ( + SELECT *, + row_number() OVER ( + PARTITION BY query_term + ORDER BY expansion_weight DESC, + df ASC, + length(term) ASC, + term ASC, + termid ASC + ) AS expansion_rank + FROM ( + SELECT * + FROM gram_expansions + WHERE expansion_weight IS NOT NULL + UNION ALL + SELECT * + FROM short_fuzzy_expansions + ) AS expansions + ), + selected_terms AS ( + SELECT query_term, + termid, + any_value(term) AS term, + any_value(match_type) AS match_type, + max(expansion_weight) AS expansion_weight + FROM ( + SELECT * + FROM exact_terms + UNION ALL + SELECT query_term, + termid, + term, + df, + expansion_weight, + match_type + FROM limited_expansions + CROSS JOIN params + WHERE expansion_rank <= params.term_limit + ) AS terms + GROUP BY query_term, + termid + ), + fieldids AS ( + SELECT fieldid + FROM %fts_schema%.fields + WHERE CASE WHEN fields IS NULL THEN true ELSE field IN (SELECT * FROM (SELECT unnest(string_split(fields, ','))) AS fsq) END + ), + term_tf AS ( + SELECT selected_terms.termid, + terms.docid, + max(selected_terms.expansion_weight) AS expansion_weight, + count(*) AS tf + FROM selected_terms + JOIN %fts_schema%.terms AS terms + ON terms.termid = selected_terms.termid + WHERE terms.fieldid IN (SELECT fieldid FROM fieldids) + GROUP BY selected_terms.termid, + terms.docid + ), + scores AS ( + SELECT docs.name AS docname, + sum( + term_tf.expansion_weight + * log(((((stats.num_docs - dict.df) + 0.5) / (dict.df + 0.5)) + 1)) + * ( + (term_tf.tf * (k + 1)) + / ( + term_tf.tf + + ( + k + * ((1 - b) + (b * (docs.len / stats.avgdl))) + ) + ) + ) + ) AS score + FROM term_tf + JOIN %fts_schema%.docs AS docs + ON docs.docid = term_tf.docid + JOIN %fts_schema%.dict AS dict + ON dict.termid = term_tf.termid + CROSS JOIN %fts_schema%.stats AS stats + GROUP BY docs.name + ), + ranked AS ( + SELECT docname, + score, + row_number() OVER (ORDER BY score DESC, docname) AS rank + FROM scores + ) + SELECT docname, + score, + rank + FROM ranked + WHERE top_k IS NULL + OR rank <= top_k + ORDER BY rank; + + CREATE MACRO %fts_schema%.match_layered_bm25(input_id, query_string, fields := NULL, k := 1.2, b := 0.75, term_limit := 32, max_df_ratio := 0.15, max_df := 50000, enable_prefix := true, enable_substring := true, enable_fuzzy := true, enable_short_fuzzy := true, expand_exact_terms := false) AS ( + SELECT score + FROM %fts_schema%.search_layered_bm25( + query_string, + fields := fields, + top_k := NULL::BIGINT, + k := k, + b := b, + term_limit := term_limit, + max_df_ratio := max_df_ratio, + max_df := max_df, + enable_prefix := enable_prefix, + enable_substring := enable_substring, + enable_fuzzy := enable_fuzzy, + enable_short_fuzzy := enable_short_fuzzy, + expand_exact_terms := expand_exact_terms + ) AS hits + WHERE hits.docname = input_id + ); + )"; + // clang-format on +} + static string DropFTSTriggersScript(const QualifiedName &qname) { vector statements; auto input_table = GetQualifiedTableName(qname); @@ -400,9 +774,186 @@ static string IncrementalIndexSetupScript(const QualifiedName &qname) { SQLIdentifier::ToString(index_name)); } +static string LayeredInsertNewTermsTriggerScript(const QualifiedName &qname) { + // clang-format off + string result = R"( + CREATE TRIGGER %trigger_15_term_stats% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + INSERT INTO %fts_schema%.term_stats (termid, term, df, term_len, gram_count) + SELECT d.termid, + d.term, + d.df, + length(d.term)::BIGINT AS term_len, + greatest(length(d.term) - 2, 0)::BIGINT AS gram_count + FROM %fts_schema%.dict AS d + WHERE d.term <> '' + AND NOT EXISTS ( + SELECT 1 + FROM %fts_schema%.term_stats AS ts + WHERE ts.termid = d.termid + ) + ORDER BY d.termid; + + CREATE TRIGGER %trigger_16_term_stats_by_len% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + INSERT INTO %fts_schema%.term_stats_by_len (termid, term, df, term_len, gram_count) + SELECT ts.termid, + ts.term, + ts.df, + ts.term_len, + ts.gram_count + FROM %fts_schema%.term_stats AS ts + WHERE NOT EXISTS ( + SELECT 1 + FROM %fts_schema%.term_stats_by_len AS tsl + WHERE tsl.termid = ts.termid + ) + ORDER BY ts.term_len, + ts.df, + ts.termid; + + CREATE TRIGGER %trigger_17_term_grams% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + INSERT INTO %fts_schema%.term_grams (gram, termid) + WITH grams AS ( + SELECT 'g' || lower(hex(substr(ts.term, i, 3))) AS gram, + ts.termid + FROM %fts_schema%.term_stats AS ts, + range(1, ts.gram_count + 1) AS r(i) + WHERE ts.gram_count > 0 + AND NOT regexp_full_match(ts.term, '[0-9]+') + ) + SELECT grams.gram, + grams.termid + FROM grams + WHERE NOT EXISTS ( + SELECT 1 + FROM %fts_schema%.term_grams AS tg + WHERE tg.termid = grams.termid + AND tg.gram = grams.gram + ) + ORDER BY grams.gram, + grams.termid; + )"; + // clang-format on + + auto trigger_names = GetFTSLayeredInsertTriggerNames(qname); + result = StringUtil::Replace(result, "%trigger_15_term_stats%", + SQLIdentifier::ToString(trigger_names[0])); + result = StringUtil::Replace(result, "%trigger_16_term_stats_by_len%", + SQLIdentifier::ToString(trigger_names[1])); + result = StringUtil::Replace(result, "%trigger_17_term_grams%", + SQLIdentifier::ToString(trigger_names[2])); + result = StringUtil::Replace(result, "%input_table%", + GetQualifiedTableName(qname)); + return result; +} + +static string LayeredInsertDFTriggerScript(const QualifiedName &qname) { + // clang-format off + string result = R"( + CREATE TRIGGER %trigger_35_term_stats_df% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.term_stats AS ts + SET df = d.df + FROM %fts_schema%.dict AS d + WHERE ts.termid = d.termid; + + CREATE TRIGGER %trigger_36_term_stats_by_len_df% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.term_stats_by_len AS ts + SET df = d.df + FROM %fts_schema%.dict AS d + WHERE ts.termid = d.termid; + )"; + // clang-format on + + auto trigger_names = GetFTSLayeredInsertTriggerNames(qname); + result = StringUtil::Replace(result, "%trigger_35_term_stats_df%", + SQLIdentifier::ToString(trigger_names[3])); + result = StringUtil::Replace(result, "%trigger_36_term_stats_by_len_df%", + SQLIdentifier::ToString(trigger_names[4])); + result = StringUtil::Replace(result, "%input_table%", + GetQualifiedTableName(qname)); + return result; +} + +static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname) { + // clang-format off + string result = R"( + CREATE TRIGGER %trigger_23_term_stats_df% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.term_stats AS ts + SET df = d.df + FROM %fts_schema%.dict AS d + WHERE ts.termid = d.termid; + + CREATE TRIGGER %trigger_24_term_stats_by_len_df% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.term_stats_by_len AS ts + SET df = d.df + FROM %fts_schema%.dict AS d + WHERE ts.termid = d.termid; + + CREATE TRIGGER %trigger_25_term_grams_prune% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + DELETE FROM %fts_schema%.term_grams + WHERE termid IN ( + SELECT termid + FROM %fts_schema%.dict + WHERE df = 0 + ); + + CREATE TRIGGER %trigger_26_term_stats_by_len_prune% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + DELETE FROM %fts_schema%.term_stats_by_len + WHERE termid IN ( + SELECT termid + FROM %fts_schema%.dict + WHERE df = 0 + ); + + CREATE TRIGGER %trigger_27_term_stats_prune% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + DELETE FROM %fts_schema%.term_stats + WHERE termid IN ( + SELECT termid + FROM %fts_schema%.dict + WHERE df = 0 + ); + )"; + // clang-format on + + auto trigger_names = GetFTSLayeredDeleteTriggerNames(qname); + result = StringUtil::Replace(result, "%trigger_23_term_stats_df%", + SQLIdentifier::ToString(trigger_names[0])); + result = StringUtil::Replace(result, "%trigger_24_term_stats_by_len_df%", + SQLIdentifier::ToString(trigger_names[1])); + result = StringUtil::Replace(result, "%trigger_25_term_grams_prune%", + SQLIdentifier::ToString(trigger_names[2])); + result = StringUtil::Replace(result, "%trigger_26_term_stats_by_len_prune%", + SQLIdentifier::ToString(trigger_names[3])); + result = StringUtil::Replace(result, "%trigger_27_term_stats_prune%", + SQLIdentifier::ToString(trigger_names[4])); + result = StringUtil::Replace(result, "%input_table%", + GetQualifiedTableName(qname)); + return result; +} + static string InsertTriggerScript(const QualifiedName &qname, const string &input_id, - const vector &input_values) { + const vector &input_values, + bool layered_search) { // clang-format off string docs_new_docs_cte = R"( fts_new_docs AS ( @@ -492,6 +1043,8 @@ static string InsertTriggerScript(const QualifiedName &qname, 0 AS df FROM new_terms; + %layered_insert_new_terms_triggers% + CREATE TRIGGER %trigger_20_terms% AFTER INSERT ON %input_table% REFERENCING NEW TABLE AS fts_new_rows FOR EACH STATEMENT @@ -501,7 +1054,8 @@ static string InsertTriggerScript(const QualifiedName &qname, ss.fieldid, d.termid FROM stemmed_stopped AS ss - JOIN %fts_schema%.dict AS d ON ss.term = d.term; + JOIN %fts_schema%.dict AS d ON ss.term = d.term + %insert_terms_order_by%; CREATE TRIGGER %trigger_30_dict_df% AFTER INSERT ON %input_table% REFERENCING NEW TABLE AS fts_new_rows @@ -518,6 +1072,8 @@ static string InsertTriggerScript(const QualifiedName &qname, ) AS inserted_df_delta WHERE d.termid = inserted_df_delta.termid; + %layered_insert_df_triggers% + CREATE TRIGGER %trigger_40_stats% AFTER INSERT ON %input_table% REFERENCING NEW TABLE AS fts_new_rows FOR EACH STATEMENT @@ -557,8 +1113,17 @@ static string InsertTriggerScript(const QualifiedName &qname, StringUtil::Replace(result, "%union_fields_query%", StringUtil::Join(tokenize_fields, " UNION ALL ")); result = StringUtil::Replace(result, "%token_ctes%", token_ctes); + result = StringUtil::Replace( + result, "%layered_insert_new_terms_triggers%", + layered_search ? LayeredInsertNewTermsTriggerScript(qname) : ""); + result = StringUtil::Replace( + result, "%layered_insert_df_triggers%", + layered_search ? LayeredInsertDFTriggerScript(qname) : ""); + result = StringUtil::Replace( + result, "%insert_terms_order_by%", + layered_search ? "ORDER BY d.termid, ss.fieldid, ss.docid" : ""); - auto trigger_names = GetFTSTriggerNames(qname); + auto trigger_names = GetFTSInsertTriggerNames(qname); result = StringUtil::Replace(result, "%trigger_00_docs%", SQLIdentifier::ToString(trigger_names[0])); result = StringUtil::Replace(result, "%trigger_10_dict_insert%", @@ -577,7 +1142,8 @@ static string InsertTriggerScript(const QualifiedName &qname, } static string DeleteTriggerScript(const QualifiedName &qname, - const string &input_id) { + const string &input_id, + bool layered_search) { // clang-format off string result = R"( CREATE TRIGGER %trigger_00_dict_df% AFTER DELETE ON %input_table% @@ -611,6 +1177,8 @@ static string DeleteTriggerScript(const QualifiedName &qname, DELETE FROM %fts_schema%.docs WHERE name IN (SELECT %input_id% FROM fts_old_rows); + %layered_delete_after_df_triggers% + CREATE TRIGGER %trigger_30_dict_prune% AFTER DELETE ON %input_table% REFERENCING OLD TABLE AS fts_old_rows FOR EACH STATEMENT @@ -637,6 +1205,9 @@ static string DeleteTriggerScript(const QualifiedName &qname, SQLIdentifier::ToString(trigger_names[3])); result = StringUtil::Replace(result, "%trigger_40_stats%", SQLIdentifier::ToString(trigger_names[4])); + result = StringUtil::Replace( + result, "%layered_delete_after_df_triggers%", + layered_search ? LayeredDeleteAfterDFTriggerScript(qname) : ""); result = StringUtil::Replace(result, "%input_table%", GetQualifiedTableName(qname)); result = StringUtil::Replace(result, "%input_id%", @@ -653,7 +1224,8 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname, const vector &input_values, const string &stemmer, const string &stopwords, const string &ignore, bool strip_accents, - bool lower, bool incremental, bool cluster_terms) { + bool lower, bool incremental, bool cluster_terms, + bool layered_search) { string result; if (TableExists(context, qname) && SupportsFTSTriggers(context, qname)) { result += DropFTSTriggersScript(qname); @@ -665,10 +1237,15 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname, GetFTSBuildTermsTable(qname), GetFTSBuildDictTable(qname), cluster_terms); result += MatchMacroScript(); + if (layered_search) { + result += LayeredSidecarScript(qname); + result += LayeredSearchMacroScript(); + } if (incremental) { result += IncrementalIndexSetupScript(qname); - result += InsertTriggerScript(qname, input_id, input_values); - result += DeleteTriggerScript(qname, input_id); + result += InsertTriggerScript(qname, input_id, input_values, + layered_search); + result += DeleteTriggerScript(qname, input_id, layered_search); } string fts_schema = GetFTSSchema(qname); @@ -737,7 +1314,8 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, const bool lower = get_bool("lower", true); const bool overwrite = get_bool("overwrite", false); const bool incremental = get_bool("incremental", false); - const bool cluster_terms = get_bool("cluster_terms", false); + const bool layered_search = get_bool("layered_search", false); + const bool cluster_terms = get_bool("cluster_terms", false) || layered_search; if (stopwords != "english" && stopwords != "none") { auto sw_qname = GetQualifiedName(context, stopwords); @@ -801,14 +1379,14 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, throw InvalidInputException("incremental FTS indexes require the document " "id column to be NOT NULL"); } - if (incremental && cluster_terms) { + if (incremental && cluster_terms && !layered_search) { throw InvalidInputException("cluster_terms cannot be combined with " "incremental FTS indexes"); } return IndexingScript(context, qname, doc_id, doc_values, stemmer, stopwords, ignore, strip_accents, lower, incremental, - cluster_terms); + cluster_terms, layered_search); } } // namespace duckdb diff --git a/test/sql/fts/test_layered_search.test b/test/sql/fts/test_layered_search.test new file mode 100644 index 0000000..a90d6bd --- /dev/null +++ b/test/sql/fts/test_layered_search.test @@ -0,0 +1,129 @@ +# name: test/sql/fts/test_layered_search.test +# description: Layered BM25 search sidecar and incremental maintenance +# group: [fts] + +require skip_reload + +require fts + +require no_alternative_verify + +statement ok +CREATE TABLE documents(id VARCHAR NOT NULL, body VARCHAR, author VARCHAR) + +statement ok +INSERT INTO documents VALUES + ('doc1', 'quacking quacking quacking', 'Hannes'), + ('doc2', 'barking barking barking barking', 'Mark'), + ('doc3', 'meowing meowing meowing meowing meowing 999', 'Laurens') + +statement ok +PRAGMA create_fts_index('documents', 'id', 'body', 'author', stemmer='none', stopwords='none', ignore='(\\.|[^a-z0-9])+', layered_search=true, incremental=true) + +query I +SELECT count(*) FROM fts_main_documents.term_stats +---- +7 + +query I +SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END FROM fts_main_documents.term_grams +---- +1 + +query I +SELECT count(*) +FROM ( + SELECT termid, lag(termid) OVER () AS previous_termid + FROM fts_main_documents.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND termid < previous_termid +---- +0 + +query I +SELECT docname +FROM fts_main_documents.search_layered_bm25('quack', fields := 'body', enable_short_fuzzy := false) +---- +doc1 + +query I +SELECT id +FROM ( + SELECT *, + fts_main_documents.match_layered_bm25(id, 'quack', fields := 'body', enable_short_fuzzy := false) AS score + FROM documents +) sq +WHERE score IS NOT NULL +ORDER BY score DESC +---- +doc1 + +query I +SELECT docname +FROM fts_main_documents.search_layered_bm25('mark', fields := 'author', enable_short_fuzzy := false) +---- +doc2 + +query I +SELECT docname +FROM fts_main_documents.search_layered_bm25('999', fields := 'body', enable_short_fuzzy := false) +---- +doc3 + +statement ok +INSERT INTO documents VALUES ('doc4', 'chirping chirping', 'Mark') + +query II +SELECT term, df +FROM fts_main_documents.term_stats +WHERE term IN ('chirping', 'mark') +ORDER BY term +---- +chirping 1 +mark 2 + +query I +SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END +FROM fts_main_documents.term_grams AS tg +JOIN fts_main_documents.term_stats AS ts USING (termid) +WHERE ts.term = 'chirping' +---- +1 + +query I +SELECT docname +FROM fts_main_documents.search_layered_bm25('chirp', fields := 'body', enable_short_fuzzy := false) +---- +doc4 + +statement ok +DELETE FROM documents WHERE id = 'doc3' + +query I +SELECT count(*) FROM fts_main_documents.term_stats WHERE term = 'meowing' +---- +0 + +query I +SELECT count(*) +FROM fts_main_documents.search_layered_bm25('meow', fields := 'body', enable_short_fuzzy := false) +---- +0 + +statement ok +CREATE TABLE plain_documents(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO plain_documents VALUES ('doc1', 'alpha beta') + +statement ok +PRAGMA create_fts_index('plain_documents', 'id', 'body', stemmer='none', stopwords='none') + +query I +SELECT count(*) +FROM information_schema.tables +WHERE table_schema = 'fts_main_plain_documents' + AND table_name = 'term_stats' +---- +0 From 209a6e2efde856d7a5c042c57863bfccc3dfbc8d Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Fri, 26 Jun 2026 11:13:38 +0200 Subject: [PATCH 2/9] Cleanup code a bit and add more tests --- src/fts_indexing.cpp | 195 +++++++++++++++++++------- test/sql/fts/test_layered_search.test | 158 +++++++++++++++++++++ 2 files changed, 306 insertions(+), 47 deletions(-) diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index ae49d3b..76cf27f 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -64,23 +64,19 @@ static vector GetFTSDeleteTriggerNames(const QualifiedName &qname) { prefix + "30_dict_prune", prefix + "40_stats"}; } -static vector GetFTSLayeredInsertTriggerNames( - const QualifiedName &qname) { +static vector +GetFTSLayeredInsertTriggerNames(const QualifiedName &qname) { auto prefix = StringUtil::Format("__fts_%s_ai_", GetFTSSchemaName(qname)); - return {prefix + "15_term_stats", - prefix + "16_term_stats_by_len", - prefix + "17_term_grams", - prefix + "35_term_stats_df", + return {prefix + "15_term_stats", prefix + "16_term_stats_by_len", + prefix + "17_term_grams", prefix + "35_term_stats_df", prefix + "36_term_stats_by_len_df"}; } -static vector GetFTSLayeredDeleteTriggerNames( - const QualifiedName &qname) { +static vector +GetFTSLayeredDeleteTriggerNames(const QualifiedName &qname) { auto prefix = StringUtil::Format("__fts_%s_ad_", GetFTSSchemaName(qname)); - return {prefix + "23_term_stats_df", - prefix + "24_term_stats_by_len_df", - prefix + "25_term_grams_prune", - prefix + "26_term_stats_by_len_prune", + return {prefix + "23_term_stats_df", prefix + "24_term_stats_by_len_df", + prefix + "25_term_grams_prune", prefix + "26_term_stats_by_len_prune", prefix + "27_term_stats_prune"}; } @@ -466,7 +462,7 @@ static string MatchMacroScript() { // clang-format on } -static string LayeredSearchMacroScript() { +static string LayeredSearchTableMacroScript() { // clang-format off return R"( CREATE MACRO %fts_schema%.search_layered_bm25(query_string, fields := NULL, top_k := 50, k := 1.2, b := 0.75, term_limit := 32, max_df_ratio := 0.15, max_df := 50000, enable_prefix := true, enable_substring := true, enable_fuzzy := true, enable_short_fuzzy := true, expand_exact_terms := false) AS TABLE @@ -731,7 +727,13 @@ static string LayeredSearchMacroScript() { WHERE top_k IS NULL OR rank <= top_k ORDER BY rank; + )"; + // clang-format on +} +static string LayeredMatchMacroScript() { + // clang-format off + return R"( CREATE MACRO %fts_schema%.match_layered_bm25(input_id, query_string, fields := NULL, k := 1.2, b := 0.75, term_limit := 32, max_df_ratio := 0.15, max_df := 50000, enable_prefix := true, enable_substring := true, enable_fuzzy := true, enable_short_fuzzy := true, expand_exact_terms := false) AS ( SELECT score FROM %fts_schema%.search_layered_bm25( @@ -755,6 +757,10 @@ static string LayeredSearchMacroScript() { // clang-format on } +static string LayeredSearchMacroScript() { + return LayeredSearchTableMacroScript() + LayeredMatchMacroScript(); +} + static string DropFTSTriggersScript(const QualifiedName &qname) { vector statements; auto input_table = GetQualifiedTableName(qname); @@ -774,37 +780,64 @@ static string IncrementalIndexSetupScript(const QualifiedName &qname) { SQLIdentifier::ToString(index_name)); } -static string LayeredInsertNewTermsTriggerScript(const QualifiedName &qname) { +static string LayeredAffectedTermIdsSubquery(const string &token_ctes) { + // clang-format off + return token_ctes + R"( + SELECT DISTINCT d.termid + FROM stemmed_stopped AS ss + JOIN %fts_schema%.dict AS d ON ss.term = d.term + )"; + // clang-format on +} + +static string LayeredInsertNewTermsTriggerScript(const QualifiedName &qname, + const string &token_ctes) { // clang-format off string result = R"( CREATE TRIGGER %trigger_15_term_stats% AFTER INSERT ON %input_table% REFERENCING NEW TABLE AS fts_new_rows FOR EACH STATEMENT INSERT INTO %fts_schema%.term_stats (termid, term, df, term_len, gram_count) - SELECT d.termid, - d.term, - d.df, - length(d.term)::BIGINT AS term_len, - greatest(length(d.term) - 2, 0)::BIGINT AS gram_count - FROM %fts_schema%.dict AS d - WHERE d.term <> '' + %token_ctes%, + affected_terms AS ( + SELECT DISTINCT d.termid, + d.term, + d.df + FROM stemmed_stopped AS ss + JOIN %fts_schema%.dict AS d ON ss.term = d.term + WHERE d.term <> '' + ) + SELECT affected_terms.termid, + affected_terms.term, + affected_terms.df, + length(affected_terms.term)::BIGINT AS term_len, + greatest(length(affected_terms.term) - 2, 0)::BIGINT AS gram_count + FROM affected_terms + WHERE affected_terms.term <> '' AND NOT EXISTS ( SELECT 1 FROM %fts_schema%.term_stats AS ts - WHERE ts.termid = d.termid + WHERE ts.termid = affected_terms.termid ) - ORDER BY d.termid; + ORDER BY affected_terms.termid; CREATE TRIGGER %trigger_16_term_stats_by_len% AFTER INSERT ON %input_table% REFERENCING NEW TABLE AS fts_new_rows FOR EACH STATEMENT INSERT INTO %fts_schema%.term_stats_by_len (termid, term, df, term_len, gram_count) + %token_ctes%, + affected_terms AS ( + SELECT DISTINCT d.termid + FROM stemmed_stopped AS ss + JOIN %fts_schema%.dict AS d ON ss.term = d.term + ) SELECT ts.termid, ts.term, ts.df, ts.term_len, ts.gram_count FROM %fts_schema%.term_stats AS ts + JOIN affected_terms USING (termid) WHERE NOT EXISTS ( SELECT 1 FROM %fts_schema%.term_stats_by_len AS tsl @@ -818,12 +851,20 @@ static string LayeredInsertNewTermsTriggerScript(const QualifiedName &qname) { REFERENCING NEW TABLE AS fts_new_rows FOR EACH STATEMENT INSERT INTO %fts_schema%.term_grams (gram, termid) - WITH grams AS ( + %token_ctes%, + affected_terms AS ( + SELECT DISTINCT d.termid + FROM stemmed_stopped AS ss + JOIN %fts_schema%.dict AS d ON ss.term = d.term + ), + grams AS ( SELECT 'g' || lower(hex(substr(ts.term, i, 3))) AS gram, ts.termid FROM %fts_schema%.term_stats AS ts, + affected_terms AS affected_terms, range(1, ts.gram_count + 1) AS r(i) - WHERE ts.gram_count > 0 + WHERE ts.termid = affected_terms.termid + AND ts.gram_count > 0 AND NOT regexp_full_match(ts.term, '[0-9]+') ) SELECT grams.gram, @@ -849,10 +890,12 @@ static string LayeredInsertNewTermsTriggerScript(const QualifiedName &qname) { SQLIdentifier::ToString(trigger_names[2])); result = StringUtil::Replace(result, "%input_table%", GetQualifiedTableName(qname)); + result = StringUtil::Replace(result, "%token_ctes%", token_ctes); return result; } -static string LayeredInsertDFTriggerScript(const QualifiedName &qname) { +static string LayeredInsertDFTriggerScript(const QualifiedName &qname, + const string &token_ctes) { // clang-format off string result = R"( CREATE TRIGGER %trigger_35_term_stats_df% AFTER INSERT ON %input_table% @@ -861,7 +904,10 @@ static string LayeredInsertDFTriggerScript(const QualifiedName &qname) { UPDATE %fts_schema%.term_stats AS ts SET df = d.df FROM %fts_schema%.dict AS d - WHERE ts.termid = d.termid; + WHERE ts.termid = d.termid + AND ts.termid IN ( + %affected_termids% + ); CREATE TRIGGER %trigger_36_term_stats_by_len_df% AFTER INSERT ON %input_table% REFERENCING NEW TABLE AS fts_new_rows @@ -869,10 +915,14 @@ static string LayeredInsertDFTriggerScript(const QualifiedName &qname) { UPDATE %fts_schema%.term_stats_by_len AS ts SET df = d.df FROM %fts_schema%.dict AS d - WHERE ts.termid = d.termid; + WHERE ts.termid = d.termid + AND ts.termid IN ( + %affected_termids% + ); )"; // clang-format on + auto affected_termids = LayeredAffectedTermIdsSubquery(token_ctes); auto trigger_names = GetFTSLayeredInsertTriggerNames(qname); result = StringUtil::Replace(result, "%trigger_35_term_stats_df%", SQLIdentifier::ToString(trigger_names[3])); @@ -880,10 +930,12 @@ static string LayeredInsertDFTriggerScript(const QualifiedName &qname) { SQLIdentifier::ToString(trigger_names[4])); result = StringUtil::Replace(result, "%input_table%", GetQualifiedTableName(qname)); + result = StringUtil::Replace(result, "%affected_termids%", affected_termids); return result; } -static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname) { +static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname, + const string &token_ctes) { // clang-format off string result = R"( CREATE TRIGGER %trigger_23_term_stats_df% AFTER DELETE ON %input_table% @@ -892,7 +944,10 @@ static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname) { UPDATE %fts_schema%.term_stats AS ts SET df = d.df FROM %fts_schema%.dict AS d - WHERE ts.termid = d.termid; + WHERE ts.termid = d.termid + AND ts.termid IN ( + %affected_termids% + ); CREATE TRIGGER %trigger_24_term_stats_by_len_df% AFTER DELETE ON %input_table% REFERENCING OLD TABLE AS fts_old_rows @@ -900,16 +955,22 @@ static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname) { UPDATE %fts_schema%.term_stats_by_len AS ts SET df = d.df FROM %fts_schema%.dict AS d - WHERE ts.termid = d.termid; + WHERE ts.termid = d.termid + AND ts.termid IN ( + %affected_termids% + ); CREATE TRIGGER %trigger_25_term_grams_prune% AFTER DELETE ON %input_table% REFERENCING OLD TABLE AS fts_old_rows FOR EACH STATEMENT DELETE FROM %fts_schema%.term_grams WHERE termid IN ( - SELECT termid - FROM %fts_schema%.dict - WHERE df = 0 + SELECT d.termid + FROM %fts_schema%.dict AS d + WHERE d.df = 0 + AND d.termid IN ( + %affected_termids% + ) ); CREATE TRIGGER %trigger_26_term_stats_by_len_prune% AFTER DELETE ON %input_table% @@ -917,9 +978,12 @@ static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname) { FOR EACH STATEMENT DELETE FROM %fts_schema%.term_stats_by_len WHERE termid IN ( - SELECT termid - FROM %fts_schema%.dict - WHERE df = 0 + SELECT d.termid + FROM %fts_schema%.dict AS d + WHERE d.df = 0 + AND d.termid IN ( + %affected_termids% + ) ); CREATE TRIGGER %trigger_27_term_stats_prune% AFTER DELETE ON %input_table% @@ -927,13 +991,17 @@ static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname) { FOR EACH STATEMENT DELETE FROM %fts_schema%.term_stats WHERE termid IN ( - SELECT termid - FROM %fts_schema%.dict - WHERE df = 0 + SELECT d.termid + FROM %fts_schema%.dict AS d + WHERE d.df = 0 + AND d.termid IN ( + %affected_termids% + ) ); )"; // clang-format on + auto affected_termids = LayeredAffectedTermIdsSubquery(token_ctes); auto trigger_names = GetFTSLayeredDeleteTriggerNames(qname); result = StringUtil::Replace(result, "%trigger_23_term_stats_df%", SQLIdentifier::ToString(trigger_names[0])); @@ -947,6 +1015,7 @@ static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname) { SQLIdentifier::ToString(trigger_names[4])); result = StringUtil::Replace(result, "%input_table%", GetQualifiedTableName(qname)); + result = StringUtil::Replace(result, "%affected_termids%", affected_termids); return result; } @@ -1115,10 +1184,11 @@ static string InsertTriggerScript(const QualifiedName &qname, result = StringUtil::Replace(result, "%token_ctes%", token_ctes); result = StringUtil::Replace( result, "%layered_insert_new_terms_triggers%", - layered_search ? LayeredInsertNewTermsTriggerScript(qname) : ""); + layered_search ? LayeredInsertNewTermsTriggerScript(qname, token_ctes) + : ""); result = StringUtil::Replace( result, "%layered_insert_df_triggers%", - layered_search ? LayeredInsertDFTriggerScript(qname) : ""); + layered_search ? LayeredInsertDFTriggerScript(qname, token_ctes) : ""); result = StringUtil::Replace( result, "%insert_terms_order_by%", layered_search ? "ORDER BY d.termid, ss.fieldid, ss.docid" : ""); @@ -1143,8 +1213,27 @@ static string InsertTriggerScript(const QualifiedName &qname, static string DeleteTriggerScript(const QualifiedName &qname, const string &input_id, + const vector &input_values, bool layered_search) { // clang-format off + string tokenize_field_query = R"( + SELECT unnest(%fts_schema%.tokenize(fts_di.%input_value%)) AS w + FROM fts_old_rows AS fts_di + )"; + + string token_ctes = R"( + WITH tokenized AS ( + %union_fields_query% + ), + stemmed_stopped AS ( + SELECT stem(t.w, '%stemmer%') AS term + FROM tokenized AS t + WHERE t.w NOT NULL + AND t.w <> '' + AND t.w NOT IN (SELECT sw FROM %fts_schema%.stopwords) + ) + )"; + string result = R"( CREATE TRIGGER %trigger_00_dict_df% AFTER DELETE ON %input_table% REFERENCING OLD TABLE AS fts_old_rows @@ -1191,9 +1280,19 @@ static string DeleteTriggerScript(const QualifiedName &qname, UPDATE %fts_schema%.stats SET num_docs = (SELECT COUNT(docid) FROM %fts_schema%.docs), avgdl = (SELECT SUM(len) / COUNT(len) FROM %fts_schema%.docs); - )"; + )"; // clang-format on + vector tokenize_fields; + for (auto &input_value : input_values) { + auto query = StringUtil::Replace(tokenize_field_query, "%input_value%", + SQLIdentifier::ToString(input_value)); + tokenize_fields.push_back(query); + } + token_ctes = + StringUtil::Replace(token_ctes, "%union_fields_query%", + StringUtil::Join(tokenize_fields, " UNION ALL ")); + auto trigger_names = GetFTSDeleteTriggerNames(qname); result = StringUtil::Replace(result, "%trigger_00_dict_df%", SQLIdentifier::ToString(trigger_names[0])); @@ -1207,7 +1306,8 @@ static string DeleteTriggerScript(const QualifiedName &qname, SQLIdentifier::ToString(trigger_names[4])); result = StringUtil::Replace( result, "%layered_delete_after_df_triggers%", - layered_search ? LayeredDeleteAfterDFTriggerScript(qname) : ""); + layered_search ? LayeredDeleteAfterDFTriggerScript(qname, token_ctes) + : ""); result = StringUtil::Replace(result, "%input_table%", GetQualifiedTableName(qname)); result = StringUtil::Replace(result, "%input_id%", @@ -1243,9 +1343,10 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname, } if (incremental) { result += IncrementalIndexSetupScript(qname); - result += InsertTriggerScript(qname, input_id, input_values, - layered_search); - result += DeleteTriggerScript(qname, input_id, layered_search); + result += + InsertTriggerScript(qname, input_id, input_values, layered_search); + result += + DeleteTriggerScript(qname, input_id, input_values, layered_search); } string fts_schema = GetFTSSchema(qname); diff --git a/test/sql/fts/test_layered_search.test b/test/sql/fts/test_layered_search.test index a90d6bd..71a51ad 100644 --- a/test/sql/fts/test_layered_search.test +++ b/test/sql/fts/test_layered_search.test @@ -30,6 +30,14 @@ SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END FROM fts_main_documents.term_gra ---- 1 +query I +SELECT count(*) +FROM fts_main_documents.term_grams AS tg +JOIN fts_main_documents.term_stats AS ts USING (termid) +WHERE ts.term = '999' +---- +0 + query I SELECT count(*) FROM ( @@ -74,6 +82,9 @@ doc3 statement ok INSERT INTO documents VALUES ('doc4', 'chirping chirping', 'Mark') +statement ok +INSERT INTO documents VALUES ('doc5', 'singing', 'Marker') + query II SELECT term, df FROM fts_main_documents.term_stats @@ -83,6 +94,21 @@ ORDER BY term chirping 1 mark 2 +query I rowsort +SELECT docname +FROM fts_main_documents.search_layered_bm25('mark', fields := 'author', enable_short_fuzzy := false, expand_exact_terms := false) +---- +doc2 +doc4 + +query I rowsort +SELECT docname +FROM fts_main_documents.search_layered_bm25('mark', fields := 'author', enable_short_fuzzy := false, expand_exact_terms := true) +---- +doc2 +doc4 +doc5 + query I SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END FROM fts_main_documents.term_grams AS tg @@ -105,6 +131,14 @@ SELECT count(*) FROM fts_main_documents.term_stats WHERE term = 'meowing' ---- 0 +query I +SELECT count(*) +FROM fts_main_documents.term_grams AS tg +LEFT JOIN fts_main_documents.term_stats AS ts USING (termid) +WHERE ts.termid IS NULL +---- +0 + query I SELECT count(*) FROM fts_main_documents.search_layered_bm25('meow', fields := 'body', enable_short_fuzzy := false) @@ -127,3 +161,127 @@ WHERE table_schema = 'fts_main_plain_documents' AND table_name = 'term_stats' ---- 0 + +statement ok +CREATE TABLE static_layered_documents(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO static_layered_documents VALUES ('doc1', 'alpha alpha') + +statement ok +PRAGMA create_fts_index('static_layered_documents', 'id', 'body', stemmer='none', stopwords='none', layered_search=true) + +query I +SELECT count(*) +FROM information_schema.tables +WHERE table_schema = 'fts_main_static_layered_documents' + AND table_name = 'term_stats' +---- +1 + +query I +SELECT docname +FROM fts_main_static_layered_documents.search_layered_bm25('alpha', enable_short_fuzzy := false) +---- +doc1 + +statement ok +INSERT INTO static_layered_documents VALUES ('doc2', 'beta') + +query I +SELECT count(*) +FROM fts_main_static_layered_documents.search_layered_bm25('beta', enable_short_fuzzy := false) +---- +0 + +statement ok +CREATE TABLE lifecycle_docs(id VARCHAR NOT NULL, body VARCHAR) + +statement ok +INSERT INTO lifecycle_docs VALUES ('doc1', 'alpha beta') + +statement ok +PRAGMA create_fts_index('lifecycle_docs', 'id', 'body', stemmer='none', stopwords='none', layered_search=true, incremental=true) + +query I +SELECT count(*) +FROM information_schema.tables +WHERE table_schema = 'fts_main_lifecycle_docs' + AND table_name = 'term_stats' +---- +1 + +query I +SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END +FROM duckdb_triggers() +WHERE trigger_name LIKE '__fts_fts_main_lifecycle_docs_%term_stats%' +---- +1 + +statement ok +PRAGMA create_fts_index('lifecycle_docs', 'id', 'body', stemmer='none', stopwords='none', overwrite=true, incremental=true) + +query I +SELECT count(*) +FROM information_schema.tables +WHERE table_schema = 'fts_main_lifecycle_docs' + AND table_name = 'term_stats' +---- +0 + +query I +SELECT count(*) +FROM duckdb_triggers() +WHERE trigger_name LIKE '__fts_fts_main_lifecycle_docs_%term_stats%' +---- +0 + +statement ok +PRAGMA drop_fts_index('lifecycle_docs') + +query I +SELECT count(*) +FROM duckdb_schemas() +WHERE schema_name = 'fts_main_lifecycle_docs' +---- +0 + +query I +SELECT count(*) +FROM duckdb_triggers() +WHERE trigger_name LIKE '__fts_fts_main_lifecycle_docs_%' +---- +0 + +statement ok +CREATE TABLE "layered docs"(id VARCHAR NOT NULL, "bo""dy" VARCHAR) + +statement ok +INSERT INTO "layered docs" VALUES ('docq', 'quoted quoted') + +statement ok +PRAGMA create_fts_index('"layered docs"', 'id', 'bo"dy', stemmer='none', stopwords='none', layered_search=true, incremental=true) + +query I +SELECT docname +FROM "fts_main_layered docs".search_layered_bm25('quote', fields := 'bo"dy', enable_short_fuzzy := false) +---- +docq + +statement ok +ATTACH ':memory:' AS layered_attach + +statement ok +CREATE TABLE layered_attach.main.docs(id VARCHAR NOT NULL, body VARCHAR) + +statement ok +INSERT INTO layered_attach.main.docs VALUES ('doca', 'attached attached') + +statement ok +PRAGMA create_fts_index(layered_attach.main.docs, 'id', 'body', stemmer='none', stopwords='none', layered_search=true) + +query I +SELECT docname +FROM layered_attach.fts_main_docs.search_layered_bm25('attach', enable_short_fuzzy := false) +---- +doca From e17c544b492e5b2d8660401cc3f5945bec1b9f85 Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Mon, 29 Jun 2026 11:26:40 +0200 Subject: [PATCH 3/9] Fix stopword matching --- src/fts_indexing.cpp | 13 +++++- test/sql/fts/test_layered_search.test | 57 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index 76cf27f..f162248 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -482,14 +482,23 @@ static string LayeredSearchTableMacroScript() { CROSS JOIN params ), raw_tokens AS ( - SELECT DISTINCT stem(unnest(%fts_schema%.tokenize(query_string)), '%stemmer%') AS query_term + SELECT DISTINCT raw_token + FROM ( + SELECT unnest(%fts_schema%.tokenize(query_string)) AS raw_token + ) AS tokenized_query + WHERE raw_token IS NOT NULL + AND raw_token <> '' + AND raw_token NOT IN (SELECT sw FROM %fts_schema%.stopwords) ), query_tokens AS ( SELECT query_term, length(query_term)::BIGINT AS query_len, greatest(length(query_term) - 2, 0)::BIGINT AS query_gram_count, regexp_full_match(query_term, '[0-9]+') AS is_numeric - FROM raw_tokens + FROM ( + SELECT stem(raw_token, '%stemmer%') AS query_term + FROM raw_tokens + ) AS stemmed_tokens WHERE query_term IS NOT NULL AND query_term <> '' ), diff --git a/test/sql/fts/test_layered_search.test b/test/sql/fts/test_layered_search.test index 71a51ad..a11b063 100644 --- a/test/sql/fts/test_layered_search.test +++ b/test/sql/fts/test_layered_search.test @@ -285,3 +285,60 @@ SELECT docname FROM layered_attach.fts_main_docs.search_layered_bm25('attach', enable_short_fuzzy := false) ---- doca + +statement ok +CREATE TABLE english_stopword_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO english_stopword_docs VALUES ('doc1', 'theme theme'), ('doc2', 'the theme') + +statement ok +PRAGMA create_fts_index('english_stopword_docs', 'id', 'body', stemmer='none', stopwords='english', layered_search=true) + +query I +SELECT count(*) +FROM fts_main_english_stopword_docs.search_layered_bm25('the', enable_short_fuzzy := false) +---- +0 + +query I +SELECT count(*) +FROM fts_main_english_stopword_docs.search_layered_bm25('the theme', enable_short_fuzzy := false) +---- +2 + +statement ok +CREATE TABLE no_stopword_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO no_stopword_docs VALUES ('doc1', 'the theme') + +statement ok +PRAGMA create_fts_index('no_stopword_docs', 'id', 'body', stemmer='none', stopwords='none', layered_search=true) + +query I +SELECT docname +FROM fts_main_no_stopword_docs.search_layered_bm25('the', enable_short_fuzzy := false) +---- +doc1 + +statement ok +CREATE TABLE custom_stopwords(sw VARCHAR) + +statement ok +INSERT INTO custom_stopwords VALUES ('noise') + +statement ok +CREATE TABLE custom_stopword_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO custom_stopword_docs VALUES ('doc1', 'noisy noisy') + +statement ok +PRAGMA create_fts_index('custom_stopword_docs', 'id', 'body', stemmer='none', stopwords='custom_stopwords', layered_search=true) + +query I +SELECT count(*) +FROM fts_main_custom_stopword_docs.search_layered_bm25('noise', enable_short_fuzzy := false) +---- +0 From 522f567fce0265956635fcf971063a2cb4cad949 Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Mon, 29 Jun 2026 13:34:00 +0200 Subject: [PATCH 4/9] Fix FTS query token normalization edge cases Fix several query-side correctness issues in FTS macros: - filter query stopwords before stemming in match_bm25 - deduplicate layered query terms after stemming - deduplicate layered expansion candidates before applying term_limit - trim comma-separated fields filters in both BM25 macros --- src/fts_indexing.cpp | 100 ++++++++++++++++++++------ test/sql/fts/test_indexing.test_slow | 11 +++ test/sql/fts/test_layered_search.test | 51 +++++++++++++ 3 files changed, 140 insertions(+), 22 deletions(-) diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index f162248..d2c3972 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -396,13 +396,35 @@ static string MatchMacroScript() { // clang-format off return R"( CREATE MACRO %fts_schema%.match_bm25(docname, query_string, fields := NULL, k := 1.2, b := 0.75, conjunctive := false) AS ( - WITH tokens AS ( - SELECT DISTINCT stem(unnest(%fts_schema%.tokenize(query_string)), '%stemmer%') AS t + WITH raw_tokens AS ( + SELECT DISTINCT raw_token + FROM ( + SELECT unnest(%fts_schema%.tokenize(query_string)) AS raw_token + ) AS tokenized_query + WHERE raw_token IS NOT NULL + AND raw_token <> '' + AND raw_token NOT IN (SELECT sw FROM %fts_schema%.stopwords) + ), + tokens AS ( + SELECT t + FROM ( + SELECT DISTINCT stem(raw_token, '%stemmer%') AS t + FROM raw_tokens + ) AS stemmed_tokens + WHERE t IS NOT NULL + AND t <> '' + ), + requested_fields AS ( + SELECT trim(field_name) AS field + FROM ( + SELECT unnest(string_split(fields, ',')) AS field_name + ) AS split_fields + WHERE trim(field_name) <> '' ), fieldids AS ( - SELECT fieldid - FROM %fts_schema%.fields - WHERE CASE WHEN fields IS NULL THEN 1 ELSE field IN (SELECT * FROM (SELECT UNNEST(string_split(fields, ','))) AS fsq) END + SELECT fts_fields.fieldid + FROM %fts_schema%.fields AS fts_fields + WHERE CASE WHEN fields IS NULL THEN 1 ELSE fts_fields.field IN (SELECT requested_fields.field FROM requested_fields) END ), qtermids AS ( SELECT termid @@ -490,18 +512,22 @@ static string LayeredSearchTableMacroScript() { AND raw_token <> '' AND raw_token NOT IN (SELECT sw FROM %fts_schema%.stopwords) ), - query_tokens AS ( - SELECT query_term, - length(query_term)::BIGINT AS query_len, - greatest(length(query_term) - 2, 0)::BIGINT AS query_gram_count, - regexp_full_match(query_term, '[0-9]+') AS is_numeric + stemmed_tokens AS ( + SELECT DISTINCT query_term FROM ( SELECT stem(raw_token, '%stemmer%') AS query_term FROM raw_tokens - ) AS stemmed_tokens + ) AS stemmed_query WHERE query_term IS NOT NULL AND query_term <> '' ), + query_tokens AS ( + SELECT query_term, + length(query_term)::BIGINT AS query_len, + greatest(length(query_term) - 2, 0)::BIGINT AS query_gram_count, + regexp_full_match(query_term, '[0-9]+') AS is_numeric + FROM stemmed_tokens + ), exact_terms AS ( SELECT query_tokens.query_term, term_stats.termid, @@ -640,6 +666,37 @@ static string LayeredSearchTableMacroScript() { AND term_stats.term <> expansion_tokens.query_term AND damerau_levenshtein(expansion_tokens.query_term, term_stats.term) <= 1 ), + expansion_candidates AS ( + SELECT * + FROM gram_expansions + WHERE expansion_weight IS NOT NULL + UNION ALL + SELECT * + FROM short_fuzzy_expansions + ), + deduped_expansions AS ( + SELECT query_term, + termid, + term, + df, + expansion_weight, + match_type + FROM ( + SELECT *, + row_number() OVER ( + PARTITION BY query_term, + termid + ORDER BY expansion_weight DESC, + df ASC, + length(term) ASC, + term ASC, + termid ASC, + match_type ASC + ) AS dedupe_rank + FROM expansion_candidates + ) AS ranked_candidates + WHERE dedupe_rank = 1 + ), limited_expansions AS ( SELECT *, row_number() OVER ( @@ -650,14 +707,7 @@ static string LayeredSearchTableMacroScript() { term ASC, termid ASC ) AS expansion_rank - FROM ( - SELECT * - FROM gram_expansions - WHERE expansion_weight IS NOT NULL - UNION ALL - SELECT * - FROM short_fuzzy_expansions - ) AS expansions + FROM deduped_expansions ), selected_terms AS ( SELECT query_term, @@ -683,9 +733,15 @@ static string LayeredSearchTableMacroScript() { termid ), fieldids AS ( - SELECT fieldid - FROM %fts_schema%.fields - WHERE CASE WHEN fields IS NULL THEN true ELSE field IN (SELECT * FROM (SELECT unnest(string_split(fields, ','))) AS fsq) END + SELECT fts_fields.fieldid + FROM %fts_schema%.fields AS fts_fields + WHERE CASE WHEN fields IS NULL THEN true ELSE fts_fields.field IN ( + SELECT trim(field_name) AS field + FROM ( + SELECT unnest(string_split(fields, ',')) AS field_name + ) AS split_fields + WHERE trim(field_name) <> '' + ) END ), term_tf AS ( SELECT selected_terms.termid, diff --git a/test/sql/fts/test_indexing.test_slow b/test/sql/fts/test_indexing.test_slow index a19e088..4643b87 100644 --- a/test/sql/fts/test_indexing.test_slow +++ b/test/sql/fts/test_indexing.test_slow @@ -211,6 +211,12 @@ SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'mark laurens', fiel doc2 doc3 +query I +SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'mark laurens', fields := 'body, author') AS score FROM documents) sq WHERE score IS NOT NULL ORDER BY score DESC +---- +doc2 +doc3 + # if we don't search any fields, we won't get any results query I SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'hannes mark laurens', fields := '') AS score FROM documents) sq WHERE score IS NOT NULL ORDER BY score DESC @@ -265,6 +271,11 @@ doc1 QUÁCKING+QUÁCKING+QUÁCKING Hannes doc2 BÁRKING+BÁRKING+BÁRKING+BÁRKING Mark doc3 MÉOWING+MÉOWING+MÉOWING+MÉOWING+MÉOWING+999 Laurens +query I +SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'the mark', conjunctive := true) AS score FROM documents) sq WHERE score IS NOT NULL ORDER BY score DESC +---- +doc2 + # document frequency remains document-based when terms repeat within documents statement ok CREATE TABLE df_documents(id VARCHAR, body VARCHAR) diff --git a/test/sql/fts/test_layered_search.test b/test/sql/fts/test_layered_search.test index a11b063..b0c5151 100644 --- a/test/sql/fts/test_layered_search.test +++ b/test/sql/fts/test_layered_search.test @@ -342,3 +342,54 @@ SELECT count(*) FROM fts_main_custom_stopword_docs.search_layered_bm25('noise', enable_short_fuzzy := false) ---- 0 + +statement ok +CREATE TABLE layered_stem_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO layered_stem_docs VALUES ('doc1', 'preconnectpost') + +statement ok +PRAGMA create_fts_index('layered_stem_docs', 'id', 'body', stemmer='porter', stopwords='none', layered_search=true) + +query I +SELECT docname +FROM fts_main_layered_stem_docs.search_layered_bm25( + 'connected connection', + enable_prefix := false, + enable_substring := true, + enable_fuzzy := false, + enable_short_fuzzy := false +) +---- +doc1 + +statement ok +CREATE TABLE layered_limit_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO layered_limit_docs VALUES ('doc1', 'quack'), ('doc2', 'quark') + +statement ok +PRAGMA create_fts_index('layered_limit_docs', 'id', 'body', stemmer='none', stopwords='none', layered_search=true) + +query I rowsort +SELECT docname +FROM fts_main_layered_limit_docs.search_layered_bm25( + 'quak', + term_limit := 2, + enable_prefix := true, + enable_substring := false, + enable_fuzzy := true, + enable_short_fuzzy := true +) +---- +doc1 +doc2 + +query I rowsort +SELECT docname +FROM fts_main_documents.search_layered_bm25('mark', fields := ' author ', enable_short_fuzzy := false) +---- +doc2 +doc4 From bf19cf7603f27c34f83eb07265f80db31b81365d Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Mon, 29 Jun 2026 13:40:21 +0200 Subject: [PATCH 5/9] Update duckdb to main and adapt to changes --- duckdb | 2 +- src/fts_indexing.cpp | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/duckdb b/duckdb index 464b647..b094486 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 464b647e189de96096dc3afae1ed62c1224adc63 +Subproject commit b0944869b2a7fde8e43acf1ea002d55517dec70a diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index d2c3972..3da9f69 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -20,9 +20,11 @@ static QualifiedName GetQualifiedName(ClientContext &context, const string &qname_str) { auto qname = QualifiedName::Parse(qname_str); if (qname.Schema() == INVALID_SCHEMA) { - qname.SchemaMutable() = + return QualifiedName( + qname.Catalog(), ClientData::Get(context).catalog_search_path->GetDefaultSchema( - qname.Catalog()); + qname.Catalog()), + qname.Name()); } return qname; } @@ -113,9 +115,9 @@ static string GetFTSTermGramsGramIndex(const QualifiedName &qname) { } static bool TableExists(ClientContext &context, const QualifiedName &qname) { - return Catalog::GetEntry( - context, qname.Catalog(), qname.Schema(), qname.Name(), - OnEntryNotFound::RETURN_NULL) != nullptr; + return Catalog::GetEntry(context, qname, + OnEntryNotFound::RETURN_NULL) != + nullptr; } static bool SupportsFTSTriggers(ClientContext &context, @@ -1455,8 +1457,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, const FunctionParameters ¶meters) { auto qname = GetQualifiedName(context, StringValue::Get(parameters.values[0])); - Catalog::GetEntry(context, qname.Catalog(), qname.Schema(), - qname.Name()); + Catalog::GetEntry(context, qname); // Named parameters auto get_string = [&](const string &name, const string &def) { @@ -1485,8 +1486,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, if (stopwords != "english" && stopwords != "none") { auto sw_qname = GetQualifiedName(context, stopwords); - Catalog::GetEntry(context, sw_qname.Catalog(), - sw_qname.Schema(), sw_qname.Name()); + Catalog::GetEntry(context, sw_qname); } const string fts_schema = GetFTSSchema(qname); @@ -1504,8 +1504,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, // Positional parameters: table, id column, value column(s) const string doc_id = StringValue::Get(parameters.values[1]); - auto &table = Catalog::GetEntry( - context, qname.Catalog(), qname.Schema(), qname.Name()); + auto &table = Catalog::GetEntry(context, qname); if (!table.ColumnExists(Identifier(doc_id))) { throw CatalogException("Table '%s.%s' does not have a column named '%s'!", qname.Schema().GetIdentifierName(), From 815838b55e58d0fef8f4be4db753c16ee6569947 Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Mon, 29 Jun 2026 13:43:54 +0200 Subject: [PATCH 6/9] Fix format --- src/fts_indexing.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index 3da9f69..f1d2474 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -115,9 +115,8 @@ static string GetFTSTermGramsGramIndex(const QualifiedName &qname) { } static bool TableExists(ClientContext &context, const QualifiedName &qname) { - return Catalog::GetEntry(context, qname, - OnEntryNotFound::RETURN_NULL) != - nullptr; + return Catalog::GetEntry( + context, qname, OnEntryNotFound::RETURN_NULL) != nullptr; } static bool SupportsFTSTriggers(ClientContext &context, From 600bf3eb38cf66958afa9d978a86cea98662d1db Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Thu, 2 Jul 2026 13:42:15 +0200 Subject: [PATCH 7/9] Update duckdb submodule --- duckdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duckdb b/duckdb index b094486..4ce2416 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit b0944869b2a7fde8e43acf1ea002d55517dec70a +Subproject commit 4ce2416bbf4996e191679c19316517ba03ce36f6 From fa1265b8b4e53c0cd16560f8ec4588a683aa8aa8 Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Thu, 2 Jul 2026 15:03:42 +0200 Subject: [PATCH 8/9] Add more description to README --- README.md | 184 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 165 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index af7d13e..3f2a674 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,8 @@ The extension adds two `PRAGMA` statements to DuckDB: one to create, and one to ```python create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', - stopwords = 'english', ignore = '(\\.|[^a-z])+', + stopwords = 'english', + ignore = "[0-9!@#$%^&*()_+={}\\[\\]:;<>,.?~\\\\/\\|''\"`-]+", strip_accents = 1, lower = 1, overwrite = 0, incremental = 0, cluster_terms = 0, layered_search = 0) ``` @@ -36,7 +37,7 @@ create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', | `input_values…` | `VARCHAR` | Column names of the text fields to be indexed (vararg), e.g., `'text_field_1'`, `'text_field_2'`, ..., `'text_field_N'`, or `'\*'` for all columns in input_table of type `VARCHAR` | | `stemmer` | `VARCHAR` | The type of stemmer to be used. One of `'arabic'`, `'armenian'`, `'basque'`, `'catalan'`, `'czech'`, `'danish'`, `'dutch'`, `'dutch_porter'`, `'english'`, `'esperanto'`, `'estonian'`, `'finnish'`, `'french'`, `'german'`, `'greek'`, `'hindi'`, `'hungarian'`, `'indonesian'`, `'irish'`, `'italian'`, `'lithuanian'`, `'nepali'`, `'norwegian'`, `'persian'`, `'polish'`, `'porter'`, `'portuguese'`, `'romanian'`, `'russian'`, `'serbian'`, `'sesotho'`, `'spanish'`, `'swedish'`, `'tamil'`, `'turkish'`, `'yiddish'`, or `'none'` if no stemming is to be used. Defaults to `'porter'` | | `stopwords` | `VARCHAR` | Qualified name of table containing a single `VARCHAR` column containing the desired stopwords, or `'none'` if no stopwords are to be used. Defaults to `'english'` for a pre-defined list of 571 English stopwords | -| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored. Defaults to `'(\\.|[^a-z])+'`, ignoring all escaped and non-alphabetic lowercase characters | +| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored. Defaults to a punctuation and digit pattern | | `strip_accents` | `BOOLEAN` | Whether to remove accents (e.g., convert `á` to `a`). Defaults to `1` | | `lower` | `BOOLEAN` | Whether to convert all text to lowercase. Defaults to `1` | | `overwrite` | `BOOLEAN` | Whether to overwrite an existing index on a table. Defaults to `0` | @@ -48,13 +49,28 @@ create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', This `PRAGMA` builds the index under a newly created schema. The schema will be named after the input table: if an index is created on table `'main.table_name'`, then the schema will be named `'fts_main_table_name'`. +By default, indexes are static snapshots. If the input table changes after +index creation, rebuild the index with `overwrite = true` or use +`incremental = true` when creating the index. Incremental indexes are maintained +with triggers for `INSERT` and `DELETE` statements. They require trigger +support, a document id column declared `NOT NULL` or `PRIMARY KEY`, and unique +document id values. Persistent databases must use storage version `v2.0.0` or +newer for incremental indexes. + +`cluster_terms = true` changes only the physical ordering of the generated +`terms` table. It cannot be combined with `incremental = true` unless +`layered_search = true`, because incremental inserts do not preserve the static +clustered layout. + ### `PRAGMA drop_fts_index` ```python drop_fts_index(input_table) ``` -Drops a FTS index for the specified table. +Drops a FTS index for the specified table. This removes the generated FTS +schema and any triggers used for incremental maintenance. Recreating an index +with `overwrite = true` performs the same cleanup before building the new index. | Name | Type | Description | @@ -73,10 +89,10 @@ When an index is built, this retrieval macro is created that can be used to sear |:--|:--|:----------| | `input_id` | `VARCHAR` | Column name of document identifier, e.g., `'document_identifier'` | | `query_string` | `VARCHAR` | The string to search the index for | -| `fields` | `VARCHAR` | Comma-separarated list of fields to search in, e.g., `'text_field_2, text_field_N'`. Defaults to `NULL` to search all indexed fields | +| `fields` | `VARCHAR` | Comma-separated list of fields to search in, e.g., `'text_field_2, text_field_N'`. Defaults to `NULL` to search all indexed fields | | `k` | `DOUBLE` | Parameter _k1_ in the Okapi BM25 retrieval model. Defaults to `1.2` | | `b` | `DOUBLE` | Parameter _b_ in the Okapi BM25 retrieval model. Defaults to `0.75` | -| `conjunctive` | `BOOLEAN` | Whether to make the query conjunctive i.e., all terms in the query string must be present in order for a document to be retrieved | +| `conjunctive` | `BOOLEAN` | Whether to make the query conjunctive, i.e., all query terms that remain after tokenization, stopword removal, and stemming must be present for a document to be retrieved | ### Layered BM25 Search @@ -94,17 +110,48 @@ match_layered_bm25(input_id, query_string, fields := NULL, k := 1.2, enable_short_fuzzy := true, expand_exact_terms := false) ``` -When `layered_search` is enabled, the extension builds two additional -dictionary sidecar tables, `term_stats` and `term_grams`, plus a -length-clustered copy of the dictionary metadata. These tables grow with the -term dictionary rather than with the document corpus and are used to expand -query terms before scoring over the standard FTS `terms` table. +When `layered_search` is enabled, the extension builds dictionary sidecar +tables used to expand query terms before scoring over the standard FTS `terms` +table. These sidecar tables grow with the term dictionary rather than with the +document corpus. -`search_layered_bm25` returns `docname`, `score`, and `rank`. -`match_layered_bm25` is the scalar form for use against an input table row. -Exact terms are always included. Prefix, substring, and fuzzy alternatives can -be toggled with the corresponding parameters. Numeric query terms are searched -exactly but are excluded from trigram/fuzzy expansion. +`search_layered_bm25` is a table macro returning `docname`, `score`, and +`rank`. `match_layered_bm25` is the scalar form for use against an input table +row. Both macros use the same tokenization, stopword removal, stemming, field +filtering, and BM25 parameters as the base FTS index. + + + +| Name | Type | Description | +|:--|:--|:----------| +| `input_id` | `VARCHAR` | Document identifier to score. Only used by `match_layered_bm25` | +| `query_string` | `VARCHAR` | The string to search the index for | +| `fields` | `VARCHAR` | Comma-separated list of indexed fields to search. Defaults to `NULL` to search all indexed fields | +| `top_k` | `BIGINT` | Maximum number of rows returned by `search_layered_bm25`. Defaults to `50`; use `NULL` to return all matches | +| `k` | `DOUBLE` | Parameter _k1_ in the Okapi BM25 retrieval model. Defaults to `1.2` | +| `b` | `DOUBLE` | Parameter _b_ in the Okapi BM25 retrieval model. Defaults to `0.75` | +| `term_limit` | `BIGINT` | Maximum number of expanded alternatives to include per query term. Defaults to `32` | +| `max_df_ratio` | `DOUBLE` | Maximum document-frequency ratio for terms considered during expansion. Defaults to `0.15` | +| `max_df` | `BIGINT` | Absolute document-frequency cap for terms considered during expansion. Defaults to `50000` | +| `enable_prefix` | `BOOLEAN` | Whether to include dictionary terms that start with a query term. Defaults to `true` | +| `enable_substring` | `BOOLEAN` | Whether to include dictionary terms that contain a query term. Defaults to `true` | +| `enable_fuzzy` | `BOOLEAN` | Whether to include Damerau-Levenshtein fuzzy alternatives. Defaults to `true` | +| `enable_short_fuzzy` | `BOOLEAN` | Whether to use a length-clustered path for short fuzzy alternatives. Defaults to `true` | +| `expand_exact_terms` | `BOOLEAN` | Whether to also expand a query term that already has an exact dictionary match. Defaults to `false` | + + + +Exact terms are always included in the candidate set. Prefix, substring, and +fuzzy alternatives are optional and receive lower expansion weights before BM25 +scoring. Numeric query terms are searched exactly and are excluded from +trigram/fuzzy expansion. Stopwords are removed before both exact matching and +expansion, so a query containing only stopwords returns no rows. + +Layered search can be static or incremental. With `layered_search = true` and +`incremental = false`, the sidecar tables are built once and later table +changes are not visible until the index is rebuilt. With both options enabled, +the sidecar tables are maintained together with the base FTS index for +`INSERT` and `DELETE`. ### `stem` Function @@ -190,10 +237,109 @@ ORDER BY score DESC; |---------------------|------------------------------------------------------------|------:| | doc2 | The cat is a domestic species of small carnivorous mammal. | 0.0 | -> Warning Without `incremental = true`, the FTS index will not update -> automatically when the input table changes. With `incremental = true`, the -> index and layered sidecar are maintained for `INSERT` and `DELETE` -> statements on tables that support triggers. +Build an incremental index when inserts and deletes should update the index +automatically. The document identifier must be non-null and unique: + +```sql +CREATE TABLE live_documents ( + document_identifier VARCHAR NOT NULL, + text_content VARCHAR +); + +INSERT INTO live_documents + VALUES ('doc1', 'quacking quacking'), + ('doc2', 'barking barking'); + +PRAGMA create_fts_index( + 'live_documents', + 'document_identifier', + 'text_content', + incremental = true +); + +INSERT INTO live_documents VALUES ('doc3', 'meowing'); + +SELECT document_identifier +FROM ( + SELECT *, fts_main_live_documents.match_bm25( + document_identifier, + 'meowing' + ) AS score + FROM live_documents +) sq +WHERE score IS NOT NULL; +``` + +Use layered search when query terms should match exact terms plus prefix, +substring, or typo-tolerant alternatives: + +```sql +CREATE TABLE animal_sounds ( + document_identifier VARCHAR NOT NULL, + text_content VARCHAR, + author VARCHAR +); + +INSERT INTO animal_sounds + VALUES ('doc1', 'quacking quacking', 'Hannes'), + ('doc2', 'barking barking', 'Mark'), + ('doc3', 'meowing meowing', 'Laurens'); + +PRAGMA create_fts_index( + 'animal_sounds', + 'document_identifier', + 'text_content', + 'author', + stemmer = 'none', + stopwords = 'none', + layered_search = true, + incremental = true +); + +SELECT docname, score, rank +FROM fts_main_animal_sounds.search_layered_bm25( + 'quack', + fields := 'text_content', + top_k := 10 +); +``` + +The scalar layered helper can be used in the same row-filtering pattern as +`match_bm25`: + +```sql +SELECT document_identifier, text_content, score +FROM ( + SELECT *, fts_main_animal_sounds.match_layered_bm25( + document_identifier, + 'mark', + fields := 'author', + enable_short_fuzzy := false + ) AS score + FROM animal_sounds +) sq +WHERE score IS NOT NULL +ORDER BY score DESC; +``` + +By default, exact dictionary matches are not further expanded. Set +`expand_exact_terms := true` to include alternatives for exact query terms: + +```sql +SELECT docname +FROM fts_main_animal_sounds.search_layered_bm25( + 'mark', + fields := 'author', + expand_exact_terms := true +); +``` + +> Warning Without `incremental = true`, the FTS index is a static snapshot and +> will not update automatically when the input table changes. Static layered +> indexes behave the same way: their sidecar tables are also rebuilt only when +> the index is rebuilt. With `incremental = true`, the index and layered +> sidecar are maintained for `INSERT` and `DELETE` statements on tables that +> support triggers. ## Stemmers From 89f9ea616e243ecf6e047de31144c25566f1c6d2 Mon Sep 17 00:00:00 2001 From: Denis Hirn Date: Thu, 2 Jul 2026 15:08:31 +0200 Subject: [PATCH 9/9] Fix duckdb-stable-deploy CI --- .github/workflows/MainDistributionPipeline.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index e75a401..619c96b 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -22,7 +22,9 @@ jobs: duckdb-stable-deploy: name: Deploy extension binaries - needs: duckdb-stable-build + needs: + - get-duckdb-version + - duckdb-stable-build uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main secrets: inherit with: @@ -37,4 +39,4 @@ jobs: duckdb_version: main ci_tools_version: main extension_name: fts - format_checks: 'format;tidy' \ No newline at end of file + format_checks: 'format;tidy'