diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml
index e75a401..619c96b 100644
--- a/.github/workflows/MainDistributionPipeline.yml
+++ b/.github/workflows/MainDistributionPipeline.yml
@@ -22,7 +22,9 @@ jobs:
duckdb-stable-deploy:
name: Deploy extension binaries
- needs: duckdb-stable-build
+ needs:
+ - get-duckdb-version
+ - duckdb-stable-build
uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
secrets: inherit
with:
@@ -37,4 +39,4 @@ jobs:
duckdb_version: main
ci_tools_version: main
extension_name: fts
- format_checks: 'format;tidy'
\ No newline at end of file
+ format_checks: 'format;tidy'
diff --git a/README.md b/README.md
index d3a8a76..3f2a674 100644
--- a/README.md
+++ b/README.md
@@ -20,9 +20,10 @@ The extension adds two `PRAGMA` statements to DuckDB: one to create, and one to
```python
create_fts_index(input_table, input_id, *input_values, stemmer = 'porter',
- stopwords = 'english', ignore = '(\\.|[^a-z])+',
+ stopwords = 'english',
+ ignore = "[0-9!@#$%^&*()_+={}\\[\\]:;<>,.?~\\\\/\\|''\"`-]+",
strip_accents = 1, lower = 1, overwrite = 0,
- cluster_terms = 0)
+ incremental = 0, cluster_terms = 0, layered_search = 0)
```
`PRAGMA` that creates a FTS index for the specified table.
@@ -36,23 +37,40 @@ create_fts_index(input_table, input_id, *input_values, stemmer = 'porter',
| `input_values…` | `VARCHAR` | Column names of the text fields to be indexed (vararg), e.g., `'text_field_1'`, `'text_field_2'`, ..., `'text_field_N'`, or `'\*'` for all columns in input_table of type `VARCHAR` |
| `stemmer` | `VARCHAR` | The type of stemmer to be used. One of `'arabic'`, `'armenian'`, `'basque'`, `'catalan'`, `'czech'`, `'danish'`, `'dutch'`, `'dutch_porter'`, `'english'`, `'esperanto'`, `'estonian'`, `'finnish'`, `'french'`, `'german'`, `'greek'`, `'hindi'`, `'hungarian'`, `'indonesian'`, `'irish'`, `'italian'`, `'lithuanian'`, `'nepali'`, `'norwegian'`, `'persian'`, `'polish'`, `'porter'`, `'portuguese'`, `'romanian'`, `'russian'`, `'serbian'`, `'sesotho'`, `'spanish'`, `'swedish'`, `'tamil'`, `'turkish'`, `'yiddish'`, or `'none'` if no stemming is to be used. Defaults to `'porter'` |
| `stopwords` | `VARCHAR` | Qualified name of table containing a single `VARCHAR` column containing the desired stopwords, or `'none'` if no stopwords are to be used. Defaults to `'english'` for a pre-defined list of 571 English stopwords |
-| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored. Defaults to `'(\\.|[^a-z])+'`, ignoring all escaped and non-alphabetic lowercase characters |
+| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored. Defaults to a punctuation and digit pattern |
| `strip_accents` | `BOOLEAN` | Whether to remove accents (e.g., convert `á` to `a`). Defaults to `1` |
| `lower` | `BOOLEAN` | Whether to convert all text to lowercase. Defaults to `1` |
| `overwrite` | `BOOLEAN` | Whether to overwrite an existing index on a table. Defaults to `0` |
+| `incremental` | `BOOLEAN` | Whether to keep the index in sync for subsequent `INSERT` and `DELETE` statements using triggers. Defaults to `0` |
| `cluster_terms` | `BOOLEAN` | Whether to physically order the generated `terms` table by `termid`, `fieldid`, and `docid`. This can improve query-time pruning for direct reads from the FTS tables. Defaults to `0` |
+| `layered_search` | `BOOLEAN` | Whether to build a dictionary trigram sidecar and layered BM25 search macros for exact, prefix, substring, and fuzzy query expansion. This implies `cluster_terms`. Defaults to `0` |
This `PRAGMA` builds the index under a newly created schema. The schema will be named after the input table: if an index is created on table `'main.table_name'`, then the schema will be named `'fts_main_table_name'`.
+By default, indexes are static snapshots. If the input table changes after
+index creation, rebuild the index with `overwrite = true` or use
+`incremental = true` when creating the index. Incremental indexes are maintained
+with triggers for `INSERT` and `DELETE` statements. They require trigger
+support, a document id column declared `NOT NULL` or `PRIMARY KEY`, and unique
+document id values. Persistent databases must use storage version `v2.0.0` or
+newer for incremental indexes.
+
+`cluster_terms = true` changes only the physical ordering of the generated
+`terms` table. It cannot be combined with `incremental = true` unless
+`layered_search = true`, because incremental inserts do not preserve the static
+clustered layout.
+
### `PRAGMA drop_fts_index`
```python
drop_fts_index(input_table)
```
-Drops a FTS index for the specified table.
+Drops a FTS index for the specified table. This removes the generated FTS
+schema and any triggers used for incremental maintenance. Recreating an index
+with `overwrite = true` performs the same cleanup before building the new index.
| Name | Type | Description |
@@ -71,10 +89,69 @@ When an index is built, this retrieval macro is created that can be used to sear
|:--|:--|:----------|
| `input_id` | `VARCHAR` | Column name of document identifier, e.g., `'document_identifier'` |
| `query_string` | `VARCHAR` | The string to search the index for |
-| `fields` | `VARCHAR` | Comma-separarated list of fields to search in, e.g., `'text_field_2, text_field_N'`. Defaults to `NULL` to search all indexed fields |
+| `fields` | `VARCHAR` | Comma-separated list of fields to search in, e.g., `'text_field_2, text_field_N'`. Defaults to `NULL` to search all indexed fields |
+| `k` | `DOUBLE` | Parameter _k1_ in the Okapi BM25 retrieval model. Defaults to `1.2` |
+| `b` | `DOUBLE` | Parameter _b_ in the Okapi BM25 retrieval model. Defaults to `0.75` |
+| `conjunctive` | `BOOLEAN` | Whether to make the query conjunctive, i.e., all query terms that remain after tokenization, stopword removal, and stemming must be present for a document to be retrieved |
+
+### Layered BM25 Search
+
+```python
+search_layered_bm25(query_string, fields := NULL, top_k := 50, k := 1.2,
+ b := 0.75, term_limit := 32, max_df_ratio := 0.15,
+ max_df := 50000, enable_prefix := true,
+ enable_substring := true, enable_fuzzy := true,
+ enable_short_fuzzy := true, expand_exact_terms := false)
+
+match_layered_bm25(input_id, query_string, fields := NULL, k := 1.2,
+ b := 0.75, term_limit := 32, max_df_ratio := 0.15,
+ max_df := 50000, enable_prefix := true,
+ enable_substring := true, enable_fuzzy := true,
+ enable_short_fuzzy := true, expand_exact_terms := false)
+```
+
+When `layered_search` is enabled, the extension builds dictionary sidecar
+tables used to expand query terms before scoring over the standard FTS `terms`
+table. These sidecar tables grow with the term dictionary rather than with the
+document corpus.
+
+`search_layered_bm25` is a table macro returning `docname`, `score`, and
+`rank`. `match_layered_bm25` is the scalar form for use against an input table
+row. Both macros use the same tokenization, stopword removal, stemming, field
+filtering, and BM25 parameters as the base FTS index.
+
+
+
+| Name | Type | Description |
+|:--|:--|:----------|
+| `input_id` | `VARCHAR` | Document identifier to score. Only used by `match_layered_bm25` |
+| `query_string` | `VARCHAR` | The string to search the index for |
+| `fields` | `VARCHAR` | Comma-separated list of indexed fields to search. Defaults to `NULL` to search all indexed fields |
+| `top_k` | `BIGINT` | Maximum number of rows returned by `search_layered_bm25`. Defaults to `50`; use `NULL` to return all matches |
| `k` | `DOUBLE` | Parameter _k1_ in the Okapi BM25 retrieval model. Defaults to `1.2` |
| `b` | `DOUBLE` | Parameter _b_ in the Okapi BM25 retrieval model. Defaults to `0.75` |
-| `conjunctive` | `BOOLEAN` | Whether to make the query conjunctive i.e., all terms in the query string must be present in order for a document to be retrieved |
+| `term_limit` | `BIGINT` | Maximum number of expanded alternatives to include per query term. Defaults to `32` |
+| `max_df_ratio` | `DOUBLE` | Maximum document-frequency ratio for terms considered during expansion. Defaults to `0.15` |
+| `max_df` | `BIGINT` | Absolute document-frequency cap for terms considered during expansion. Defaults to `50000` |
+| `enable_prefix` | `BOOLEAN` | Whether to include dictionary terms that start with a query term. Defaults to `true` |
+| `enable_substring` | `BOOLEAN` | Whether to include dictionary terms that contain a query term. Defaults to `true` |
+| `enable_fuzzy` | `BOOLEAN` | Whether to include Damerau-Levenshtein fuzzy alternatives. Defaults to `true` |
+| `enable_short_fuzzy` | `BOOLEAN` | Whether to use a length-clustered path for short fuzzy alternatives. Defaults to `true` |
+| `expand_exact_terms` | `BOOLEAN` | Whether to also expand a query term that already has an exact dictionary match. Defaults to `false` |
+
+
+
+Exact terms are always included in the candidate set. Prefix, substring, and
+fuzzy alternatives are optional and receive lower expansion weights before BM25
+scoring. Numeric query terms are searched exactly and are excluded from
+trigram/fuzzy expansion. Stopwords are removed before both exact matching and
+expansion, so a query containing only stopwords returns no rows.
+
+Layered search can be static or incremental. With `layered_search = true` and
+`incremental = false`, the sidecar tables are built once and later table
+changes are not visible until the index is rebuilt. With both options enabled,
+the sidecar tables are maintained together with the base FTS index for
+`INSERT` and `DELETE`.
### `stem` Function
@@ -160,8 +237,109 @@ ORDER BY score DESC;
|---------------------|------------------------------------------------------------|------:|
| doc2 | The cat is a domestic species of small carnivorous mammal. | 0.0 |
-> Warning The FTS index will not update automatically when input table changes.
-> A workaround of this limitation can be recreating the index to refresh.
+Build an incremental index when inserts and deletes should update the index
+automatically. The document identifier must be non-null and unique:
+
+```sql
+CREATE TABLE live_documents (
+ document_identifier VARCHAR NOT NULL,
+ text_content VARCHAR
+);
+
+INSERT INTO live_documents
+ VALUES ('doc1', 'quacking quacking'),
+ ('doc2', 'barking barking');
+
+PRAGMA create_fts_index(
+ 'live_documents',
+ 'document_identifier',
+ 'text_content',
+ incremental = true
+);
+
+INSERT INTO live_documents VALUES ('doc3', 'meowing');
+
+SELECT document_identifier
+FROM (
+ SELECT *, fts_main_live_documents.match_bm25(
+ document_identifier,
+ 'meowing'
+ ) AS score
+ FROM live_documents
+) sq
+WHERE score IS NOT NULL;
+```
+
+Use layered search when query terms should match exact terms plus prefix,
+substring, or typo-tolerant alternatives:
+
+```sql
+CREATE TABLE animal_sounds (
+ document_identifier VARCHAR NOT NULL,
+ text_content VARCHAR,
+ author VARCHAR
+);
+
+INSERT INTO animal_sounds
+ VALUES ('doc1', 'quacking quacking', 'Hannes'),
+ ('doc2', 'barking barking', 'Mark'),
+ ('doc3', 'meowing meowing', 'Laurens');
+
+PRAGMA create_fts_index(
+ 'animal_sounds',
+ 'document_identifier',
+ 'text_content',
+ 'author',
+ stemmer = 'none',
+ stopwords = 'none',
+ layered_search = true,
+ incremental = true
+);
+
+SELECT docname, score, rank
+FROM fts_main_animal_sounds.search_layered_bm25(
+ 'quack',
+ fields := 'text_content',
+ top_k := 10
+);
+```
+
+The scalar layered helper can be used in the same row-filtering pattern as
+`match_bm25`:
+
+```sql
+SELECT document_identifier, text_content, score
+FROM (
+ SELECT *, fts_main_animal_sounds.match_layered_bm25(
+ document_identifier,
+ 'mark',
+ fields := 'author',
+ enable_short_fuzzy := false
+ ) AS score
+ FROM animal_sounds
+) sq
+WHERE score IS NOT NULL
+ORDER BY score DESC;
+```
+
+By default, exact dictionary matches are not further expanded. Set
+`expand_exact_terms := true` to include alternatives for exact query terms:
+
+```sql
+SELECT docname
+FROM fts_main_animal_sounds.search_layered_bm25(
+ 'mark',
+ fields := 'author',
+ expand_exact_terms := true
+);
+```
+
+> Warning Without `incremental = true`, the FTS index is a static snapshot and
+> will not update automatically when the input table changes. Static layered
+> indexes behave the same way: their sidecar tables are also rebuilt only when
+> the index is rebuilt. With `incremental = true`, the index and layered
+> sidecar are maintained for `INSERT` and `DELETE` statements on tables that
+> support triggers.
## Stemmers
diff --git a/duckdb b/duckdb
index 464b647..4ce2416 160000
--- a/duckdb
+++ b/duckdb
@@ -1 +1 @@
-Subproject commit 464b647e189de96096dc3afae1ed62c1224adc63
+Subproject commit 4ce2416bbf4996e191679c19316517ba03ce36f6
diff --git a/src/fts_extension.cpp b/src/fts_extension.cpp
index 03a3163..41ff9e5 100644
--- a/src/fts_extension.cpp
+++ b/src/fts_extension.cpp
@@ -71,6 +71,8 @@ static void LoadInternal(ExtensionLoader &loader) {
create_fts_index_func.named_parameters["incremental"] = LogicalType::BOOLEAN;
create_fts_index_func.named_parameters["cluster_terms"] =
LogicalType::BOOLEAN;
+ create_fts_index_func.named_parameters["layered_search"] =
+ LogicalType::BOOLEAN;
auto drop_fts_index_func = PragmaFunction::PragmaCall(
"drop_fts_index", FTSIndexing::DropFTSIndexQuery, {LogicalType::VARCHAR});
diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp
index e506f07..f1d2474 100644
--- a/src/fts_indexing.cpp
+++ b/src/fts_indexing.cpp
@@ -20,9 +20,11 @@ static QualifiedName GetQualifiedName(ClientContext &context,
const string &qname_str) {
auto qname = QualifiedName::Parse(qname_str);
if (qname.Schema() == INVALID_SCHEMA) {
- qname.SchemaMutable() =
+ return QualifiedName(
+ qname.Catalog(),
ClientData::Get(context).catalog_search_path->GetDefaultSchema(
- qname.Catalog());
+ qname.Catalog()),
+ qname.Name());
}
return qname;
}
@@ -64,10 +66,32 @@ static vector GetFTSDeleteTriggerNames(const QualifiedName &qname) {
prefix + "30_dict_prune", prefix + "40_stats"};
}
+static vector
+GetFTSLayeredInsertTriggerNames(const QualifiedName &qname) {
+ auto prefix = StringUtil::Format("__fts_%s_ai_", GetFTSSchemaName(qname));
+ return {prefix + "15_term_stats", prefix + "16_term_stats_by_len",
+ prefix + "17_term_grams", prefix + "35_term_stats_df",
+ prefix + "36_term_stats_by_len_df"};
+}
+
+static vector
+GetFTSLayeredDeleteTriggerNames(const QualifiedName &qname) {
+ auto prefix = StringUtil::Format("__fts_%s_ad_", GetFTSSchemaName(qname));
+ return {prefix + "23_term_stats_df", prefix + "24_term_stats_by_len_df",
+ prefix + "25_term_grams_prune", prefix + "26_term_stats_by_len_prune",
+ prefix + "27_term_stats_prune"};
+}
+
static vector GetFTSTriggerNames(const QualifiedName &qname) {
auto result = GetFTSInsertTriggerNames(qname);
+ auto layered_insert_triggers = GetFTSLayeredInsertTriggerNames(qname);
+ result.insert(result.end(), layered_insert_triggers.begin(),
+ layered_insert_triggers.end());
auto delete_triggers = GetFTSDeleteTriggerNames(qname);
result.insert(result.end(), delete_triggers.begin(), delete_triggers.end());
+ auto layered_delete_triggers = GetFTSLayeredDeleteTriggerNames(qname);
+ result.insert(result.end(), layered_delete_triggers.begin(),
+ layered_delete_triggers.end());
return result;
}
@@ -80,10 +104,19 @@ static string GetFTSBuildDictTable(const QualifiedName &qname) {
return SQLIdentifier::ToString("__fts_build_dict_" + GetFTSSchemaName(qname));
}
+static string GetFTSTermStatsTermIndex(const QualifiedName &qname) {
+ return SQLIdentifier::ToString("__fts_" + GetFTSSchemaName(qname) +
+ "_term_stats_term_idx");
+}
+
+static string GetFTSTermGramsGramIndex(const QualifiedName &qname) {
+ return SQLIdentifier::ToString("__fts_" + GetFTSSchemaName(qname) +
+ "_term_grams_gram_idx");
+}
+
static bool TableExists(ClientContext &context, const QualifiedName &qname) {
return Catalog::GetEntry(
- context, qname.Catalog(), qname.Schema(), qname.Name(),
- OnEntryNotFound::RETURN_NULL) != nullptr;
+ context, qname, OnEntryNotFound::RETURN_NULL) != nullptr;
}
static bool SupportsFTSTriggers(ClientContext &context,
@@ -311,17 +344,88 @@ static string IndexTablesScript(const string &input_id,
return result;
}
+static string LayeredSidecarScript(const QualifiedName &qname) {
+ // clang-format off
+ string result = R"(
+ CREATE TABLE %fts_schema%.term_stats AS
+ SELECT termid,
+ term,
+ df,
+ length(term)::BIGINT AS term_len,
+ greatest(length(term) - 2, 0)::BIGINT AS gram_count
+ FROM %fts_schema%.dict
+ WHERE term <> '';
+
+ CREATE TABLE %fts_schema%.term_stats_by_len AS
+ SELECT termid,
+ term,
+ df,
+ term_len,
+ gram_count
+ FROM %fts_schema%.term_stats
+ ORDER BY term_len,
+ df,
+ termid;
+
+ CREATE TABLE %fts_schema%.term_grams AS
+ SELECT 'g' || lower(hex(substr(term, i, 3))) AS gram,
+ termid
+ FROM %fts_schema%.term_stats,
+ range(1, gram_count + 1) AS r(i)
+ WHERE gram_count > 0
+ AND NOT regexp_full_match(term, '[0-9]+')
+ ORDER BY gram,
+ termid;
+
+ CREATE INDEX %term_stats_term_index% ON %fts_schema%.term_stats(term);
+ CREATE INDEX %term_grams_gram_index% ON %fts_schema%.term_grams(gram);
+
+ ANALYZE %fts_schema%.term_stats;
+ ANALYZE %fts_schema%.term_stats_by_len;
+ ANALYZE %fts_schema%.term_grams;
+ )";
+ // clang-format on
+
+ result = StringUtil::Replace(result, "%term_stats_term_index%",
+ GetFTSTermStatsTermIndex(qname));
+ result = StringUtil::Replace(result, "%term_grams_gram_index%",
+ GetFTSTermGramsGramIndex(qname));
+ return result;
+}
+
static string MatchMacroScript() {
// clang-format off
return R"(
CREATE MACRO %fts_schema%.match_bm25(docname, query_string, fields := NULL, k := 1.2, b := 0.75, conjunctive := false) AS (
- WITH tokens AS (
- SELECT DISTINCT stem(unnest(%fts_schema%.tokenize(query_string)), '%stemmer%') AS t
+ WITH raw_tokens AS (
+ SELECT DISTINCT raw_token
+ FROM (
+ SELECT unnest(%fts_schema%.tokenize(query_string)) AS raw_token
+ ) AS tokenized_query
+ WHERE raw_token IS NOT NULL
+ AND raw_token <> ''
+ AND raw_token NOT IN (SELECT sw FROM %fts_schema%.stopwords)
+ ),
+ tokens AS (
+ SELECT t
+ FROM (
+ SELECT DISTINCT stem(raw_token, '%stemmer%') AS t
+ FROM raw_tokens
+ ) AS stemmed_tokens
+ WHERE t IS NOT NULL
+ AND t <> ''
+ ),
+ requested_fields AS (
+ SELECT trim(field_name) AS field
+ FROM (
+ SELECT unnest(string_split(fields, ',')) AS field_name
+ ) AS split_fields
+ WHERE trim(field_name) <> ''
),
fieldids AS (
- SELECT fieldid
- FROM %fts_schema%.fields
- WHERE CASE WHEN fields IS NULL THEN 1 ELSE field IN (SELECT * FROM (SELECT UNNEST(string_split(fields, ','))) AS fsq) END
+ SELECT fts_fields.fieldid
+ FROM %fts_schema%.fields AS fts_fields
+ WHERE CASE WHEN fields IS NULL THEN 1 ELSE fts_fields.field IN (SELECT requested_fields.field FROM requested_fields) END
),
qtermids AS (
SELECT termid
@@ -381,6 +485,348 @@ static string MatchMacroScript() {
// clang-format on
}
+static string LayeredSearchTableMacroScript() {
+ // clang-format off
+ return R"(
+ CREATE MACRO %fts_schema%.search_layered_bm25(query_string, fields := NULL, top_k := 50, k := 1.2, b := 0.75, term_limit := 32, max_df_ratio := 0.15, max_df := 50000, enable_prefix := true, enable_substring := true, enable_fuzzy := true, enable_short_fuzzy := true, expand_exact_terms := false) AS TABLE
+ WITH params(term_limit, max_df_ratio, max_df, enable_prefix, enable_substring, enable_fuzzy, enable_short_fuzzy, expand_exact_terms) AS (
+ SELECT term_limit::BIGINT,
+ max_df_ratio::DOUBLE,
+ max_df::BIGINT,
+ enable_prefix::BOOLEAN,
+ enable_substring::BOOLEAN,
+ enable_fuzzy::BOOLEAN,
+ enable_short_fuzzy::BOOLEAN,
+ expand_exact_terms::BOOLEAN
+ ),
+ df_cap(max_df) AS (
+ SELECT least(params.max_df, ceil(stats.num_docs * params.max_df_ratio)::BIGINT)
+ FROM %fts_schema%.stats AS stats
+ CROSS JOIN params
+ ),
+ raw_tokens AS (
+ SELECT DISTINCT raw_token
+ FROM (
+ SELECT unnest(%fts_schema%.tokenize(query_string)) AS raw_token
+ ) AS tokenized_query
+ WHERE raw_token IS NOT NULL
+ AND raw_token <> ''
+ AND raw_token NOT IN (SELECT sw FROM %fts_schema%.stopwords)
+ ),
+ stemmed_tokens AS (
+ SELECT DISTINCT query_term
+ FROM (
+ SELECT stem(raw_token, '%stemmer%') AS query_term
+ FROM raw_tokens
+ ) AS stemmed_query
+ WHERE query_term IS NOT NULL
+ AND query_term <> ''
+ ),
+ query_tokens AS (
+ SELECT query_term,
+ length(query_term)::BIGINT AS query_len,
+ greatest(length(query_term) - 2, 0)::BIGINT AS query_gram_count,
+ regexp_full_match(query_term, '[0-9]+') AS is_numeric
+ FROM stemmed_tokens
+ ),
+ exact_terms AS (
+ SELECT query_tokens.query_term,
+ term_stats.termid,
+ term_stats.term,
+ term_stats.df,
+ 1.0::DOUBLE AS expansion_weight,
+ 'exact' AS match_type
+ FROM query_tokens
+ JOIN %fts_schema%.term_stats AS term_stats
+ ON term_stats.term = query_tokens.query_term
+ ),
+ expansion_tokens AS (
+ SELECT query_tokens.*
+ FROM query_tokens
+ LEFT JOIN exact_terms
+ ON exact_terms.query_term = query_tokens.query_term
+ CROSS JOIN params
+ WHERE params.expand_exact_terms
+ OR exact_terms.termid IS NULL
+ ),
+ query_grams AS (
+ SELECT query_term,
+ 'g' || lower(hex(substr(query_term, i, 3))) AS gram
+ FROM expansion_tokens,
+ range(1, query_gram_count + 1) AS r(i)
+ WHERE query_gram_count > 0
+ AND NOT is_numeric
+ ),
+ gram_candidates AS (
+ SELECT query_grams.query_term,
+ term_grams.termid,
+ count(*)::BIGINT AS matching_grams
+ FROM query_grams
+ JOIN %fts_schema%.term_grams AS term_grams
+ ON term_grams.gram = query_grams.gram
+ JOIN %fts_schema%.term_stats AS term_stats
+ ON term_stats.termid = term_grams.termid
+ CROSS JOIN df_cap
+ WHERE term_stats.df <= df_cap.max_df
+ GROUP BY query_grams.query_term,
+ term_grams.termid
+ ),
+ gram_expansions AS (
+ SELECT gram_candidates.query_term,
+ term_stats.termid,
+ term_stats.term,
+ term_stats.df,
+ CASE
+ WHEN params.enable_prefix
+ AND starts_with(term_stats.term, expansion_tokens.query_term)
+ AND term_stats.term <> expansion_tokens.query_term
+ THEN 0.85::DOUBLE
+ WHEN params.enable_substring
+ AND contains(term_stats.term, expansion_tokens.query_term)
+ AND term_stats.term <> expansion_tokens.query_term
+ AND gram_candidates.matching_grams = expansion_tokens.query_gram_count
+ THEN 0.75::DOUBLE
+ WHEN params.enable_fuzzy
+ AND expansion_tokens.query_len >= 3
+ AND gram_candidates.matching_grams >= greatest(1, expansion_tokens.query_gram_count - 1)
+ AND abs(term_stats.term_len - expansion_tokens.query_len) <= CASE
+ WHEN expansion_tokens.query_len <= 4 THEN 1
+ WHEN expansion_tokens.query_len <= 8 THEN 2
+ ELSE 3
+ END
+ AND damerau_levenshtein(expansion_tokens.query_term, term_stats.term) <= CASE
+ WHEN expansion_tokens.query_len <= 4 THEN 1
+ WHEN expansion_tokens.query_len <= 8 THEN 2
+ ELSE 3
+ END
+ THEN greatest(
+ 0.25::DOUBLE,
+ 1.0::DOUBLE - (
+ damerau_levenshtein(expansion_tokens.query_term, term_stats.term)::DOUBLE
+ / greatest(expansion_tokens.query_len, term_stats.term_len)::DOUBLE
+ )
+ )
+ ELSE NULL
+ END AS expansion_weight,
+ CASE
+ WHEN params.enable_prefix
+ AND starts_with(term_stats.term, expansion_tokens.query_term)
+ AND term_stats.term <> expansion_tokens.query_term
+ THEN 'prefix'
+ WHEN params.enable_substring
+ AND contains(term_stats.term, expansion_tokens.query_term)
+ AND term_stats.term <> expansion_tokens.query_term
+ AND gram_candidates.matching_grams = expansion_tokens.query_gram_count
+ THEN 'substring'
+ WHEN params.enable_fuzzy
+ AND expansion_tokens.query_len >= 3
+ AND gram_candidates.matching_grams >= greatest(1, expansion_tokens.query_gram_count - 1)
+ AND abs(term_stats.term_len - expansion_tokens.query_len) <= CASE
+ WHEN expansion_tokens.query_len <= 4 THEN 1
+ WHEN expansion_tokens.query_len <= 8 THEN 2
+ ELSE 3
+ END
+ AND damerau_levenshtein(expansion_tokens.query_term, term_stats.term) <= CASE
+ WHEN expansion_tokens.query_len <= 4 THEN 1
+ WHEN expansion_tokens.query_len <= 8 THEN 2
+ ELSE 3
+ END
+ THEN 'fuzzy'
+ ELSE NULL
+ END AS match_type
+ FROM gram_candidates
+ JOIN expansion_tokens
+ ON expansion_tokens.query_term = gram_candidates.query_term
+ JOIN %fts_schema%.term_stats AS term_stats
+ ON term_stats.termid = gram_candidates.termid
+ CROSS JOIN params
+ ),
+ short_fuzzy_expansions AS (
+ SELECT expansion_tokens.query_term,
+ term_stats.termid,
+ term_stats.term,
+ term_stats.df,
+ greatest(
+ 0.25::DOUBLE,
+ 1.0::DOUBLE - (
+ damerau_levenshtein(expansion_tokens.query_term, term_stats.term)::DOUBLE
+ / greatest(expansion_tokens.query_len, term_stats.term_len)::DOUBLE
+ )
+ ) AS expansion_weight,
+ 'fuzzy' AS match_type
+ FROM expansion_tokens
+ JOIN %fts_schema%.term_stats_by_len AS term_stats
+ ON term_stats.term_len BETWEEN expansion_tokens.query_len - 1 AND expansion_tokens.query_len + 1
+ CROSS JOIN params
+ CROSS JOIN df_cap
+ WHERE params.enable_fuzzy
+ AND params.enable_short_fuzzy
+ AND expansion_tokens.query_len BETWEEN 3 AND 5
+ AND NOT expansion_tokens.is_numeric
+ AND term_stats.df <= df_cap.max_df
+ AND term_stats.term <> expansion_tokens.query_term
+ AND damerau_levenshtein(expansion_tokens.query_term, term_stats.term) <= 1
+ ),
+ expansion_candidates AS (
+ SELECT *
+ FROM gram_expansions
+ WHERE expansion_weight IS NOT NULL
+ UNION ALL
+ SELECT *
+ FROM short_fuzzy_expansions
+ ),
+ deduped_expansions AS (
+ SELECT query_term,
+ termid,
+ term,
+ df,
+ expansion_weight,
+ match_type
+ FROM (
+ SELECT *,
+ row_number() OVER (
+ PARTITION BY query_term,
+ termid
+ ORDER BY expansion_weight DESC,
+ df ASC,
+ length(term) ASC,
+ term ASC,
+ termid ASC,
+ match_type ASC
+ ) AS dedupe_rank
+ FROM expansion_candidates
+ ) AS ranked_candidates
+ WHERE dedupe_rank = 1
+ ),
+ limited_expansions AS (
+ SELECT *,
+ row_number() OVER (
+ PARTITION BY query_term
+ ORDER BY expansion_weight DESC,
+ df ASC,
+ length(term) ASC,
+ term ASC,
+ termid ASC
+ ) AS expansion_rank
+ FROM deduped_expansions
+ ),
+ selected_terms AS (
+ SELECT query_term,
+ termid,
+ any_value(term) AS term,
+ any_value(match_type) AS match_type,
+ max(expansion_weight) AS expansion_weight
+ FROM (
+ SELECT *
+ FROM exact_terms
+ UNION ALL
+ SELECT query_term,
+ termid,
+ term,
+ df,
+ expansion_weight,
+ match_type
+ FROM limited_expansions
+ CROSS JOIN params
+ WHERE expansion_rank <= params.term_limit
+ ) AS terms
+ GROUP BY query_term,
+ termid
+ ),
+ fieldids AS (
+ SELECT fts_fields.fieldid
+ FROM %fts_schema%.fields AS fts_fields
+ WHERE CASE WHEN fields IS NULL THEN true ELSE fts_fields.field IN (
+ SELECT trim(field_name) AS field
+ FROM (
+ SELECT unnest(string_split(fields, ',')) AS field_name
+ ) AS split_fields
+ WHERE trim(field_name) <> ''
+ ) END
+ ),
+ term_tf AS (
+ SELECT selected_terms.termid,
+ terms.docid,
+ max(selected_terms.expansion_weight) AS expansion_weight,
+ count(*) AS tf
+ FROM selected_terms
+ JOIN %fts_schema%.terms AS terms
+ ON terms.termid = selected_terms.termid
+ WHERE terms.fieldid IN (SELECT fieldid FROM fieldids)
+ GROUP BY selected_terms.termid,
+ terms.docid
+ ),
+ scores AS (
+ SELECT docs.name AS docname,
+ sum(
+ term_tf.expansion_weight
+ * log(((((stats.num_docs - dict.df) + 0.5) / (dict.df + 0.5)) + 1))
+ * (
+ (term_tf.tf * (k + 1))
+ / (
+ term_tf.tf
+ + (
+ k
+ * ((1 - b) + (b * (docs.len / stats.avgdl)))
+ )
+ )
+ )
+ ) AS score
+ FROM term_tf
+ JOIN %fts_schema%.docs AS docs
+ ON docs.docid = term_tf.docid
+ JOIN %fts_schema%.dict AS dict
+ ON dict.termid = term_tf.termid
+ CROSS JOIN %fts_schema%.stats AS stats
+ GROUP BY docs.name
+ ),
+ ranked AS (
+ SELECT docname,
+ score,
+ row_number() OVER (ORDER BY score DESC, docname) AS rank
+ FROM scores
+ )
+ SELECT docname,
+ score,
+ rank
+ FROM ranked
+ WHERE top_k IS NULL
+ OR rank <= top_k
+ ORDER BY rank;
+ )";
+ // clang-format on
+}
+
+static string LayeredMatchMacroScript() {
+ // clang-format off
+ return R"(
+ CREATE MACRO %fts_schema%.match_layered_bm25(input_id, query_string, fields := NULL, k := 1.2, b := 0.75, term_limit := 32, max_df_ratio := 0.15, max_df := 50000, enable_prefix := true, enable_substring := true, enable_fuzzy := true, enable_short_fuzzy := true, expand_exact_terms := false) AS (
+ SELECT score
+ FROM %fts_schema%.search_layered_bm25(
+ query_string,
+ fields := fields,
+ top_k := NULL::BIGINT,
+ k := k,
+ b := b,
+ term_limit := term_limit,
+ max_df_ratio := max_df_ratio,
+ max_df := max_df,
+ enable_prefix := enable_prefix,
+ enable_substring := enable_substring,
+ enable_fuzzy := enable_fuzzy,
+ enable_short_fuzzy := enable_short_fuzzy,
+ expand_exact_terms := expand_exact_terms
+ ) AS hits
+ WHERE hits.docname = input_id
+ );
+ )";
+ // clang-format on
+}
+
+static string LayeredSearchMacroScript() {
+ return LayeredSearchTableMacroScript() + LayeredMatchMacroScript();
+}
+
static string DropFTSTriggersScript(const QualifiedName &qname) {
vector statements;
auto input_table = GetQualifiedTableName(qname);
@@ -400,9 +846,249 @@ static string IncrementalIndexSetupScript(const QualifiedName &qname) {
SQLIdentifier::ToString(index_name));
}
+static string LayeredAffectedTermIdsSubquery(const string &token_ctes) {
+ // clang-format off
+ return token_ctes + R"(
+ SELECT DISTINCT d.termid
+ FROM stemmed_stopped AS ss
+ JOIN %fts_schema%.dict AS d ON ss.term = d.term
+ )";
+ // clang-format on
+}
+
+static string LayeredInsertNewTermsTriggerScript(const QualifiedName &qname,
+ const string &token_ctes) {
+ // clang-format off
+ string result = R"(
+ CREATE TRIGGER %trigger_15_term_stats% AFTER INSERT ON %input_table%
+ REFERENCING NEW TABLE AS fts_new_rows
+ FOR EACH STATEMENT
+ INSERT INTO %fts_schema%.term_stats (termid, term, df, term_len, gram_count)
+ %token_ctes%,
+ affected_terms AS (
+ SELECT DISTINCT d.termid,
+ d.term,
+ d.df
+ FROM stemmed_stopped AS ss
+ JOIN %fts_schema%.dict AS d ON ss.term = d.term
+ WHERE d.term <> ''
+ )
+ SELECT affected_terms.termid,
+ affected_terms.term,
+ affected_terms.df,
+ length(affected_terms.term)::BIGINT AS term_len,
+ greatest(length(affected_terms.term) - 2, 0)::BIGINT AS gram_count
+ FROM affected_terms
+ WHERE affected_terms.term <> ''
+ AND NOT EXISTS (
+ SELECT 1
+ FROM %fts_schema%.term_stats AS ts
+ WHERE ts.termid = affected_terms.termid
+ )
+ ORDER BY affected_terms.termid;
+
+ CREATE TRIGGER %trigger_16_term_stats_by_len% AFTER INSERT ON %input_table%
+ REFERENCING NEW TABLE AS fts_new_rows
+ FOR EACH STATEMENT
+ INSERT INTO %fts_schema%.term_stats_by_len (termid, term, df, term_len, gram_count)
+ %token_ctes%,
+ affected_terms AS (
+ SELECT DISTINCT d.termid
+ FROM stemmed_stopped AS ss
+ JOIN %fts_schema%.dict AS d ON ss.term = d.term
+ )
+ SELECT ts.termid,
+ ts.term,
+ ts.df,
+ ts.term_len,
+ ts.gram_count
+ FROM %fts_schema%.term_stats AS ts
+ JOIN affected_terms USING (termid)
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM %fts_schema%.term_stats_by_len AS tsl
+ WHERE tsl.termid = ts.termid
+ )
+ ORDER BY ts.term_len,
+ ts.df,
+ ts.termid;
+
+ CREATE TRIGGER %trigger_17_term_grams% AFTER INSERT ON %input_table%
+ REFERENCING NEW TABLE AS fts_new_rows
+ FOR EACH STATEMENT
+ INSERT INTO %fts_schema%.term_grams (gram, termid)
+ %token_ctes%,
+ affected_terms AS (
+ SELECT DISTINCT d.termid
+ FROM stemmed_stopped AS ss
+ JOIN %fts_schema%.dict AS d ON ss.term = d.term
+ ),
+ grams AS (
+ SELECT 'g' || lower(hex(substr(ts.term, i, 3))) AS gram,
+ ts.termid
+ FROM %fts_schema%.term_stats AS ts,
+ affected_terms AS affected_terms,
+ range(1, ts.gram_count + 1) AS r(i)
+ WHERE ts.termid = affected_terms.termid
+ AND ts.gram_count > 0
+ AND NOT regexp_full_match(ts.term, '[0-9]+')
+ )
+ SELECT grams.gram,
+ grams.termid
+ FROM grams
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM %fts_schema%.term_grams AS tg
+ WHERE tg.termid = grams.termid
+ AND tg.gram = grams.gram
+ )
+ ORDER BY grams.gram,
+ grams.termid;
+ )";
+ // clang-format on
+
+ auto trigger_names = GetFTSLayeredInsertTriggerNames(qname);
+ result = StringUtil::Replace(result, "%trigger_15_term_stats%",
+ SQLIdentifier::ToString(trigger_names[0]));
+ result = StringUtil::Replace(result, "%trigger_16_term_stats_by_len%",
+ SQLIdentifier::ToString(trigger_names[1]));
+ result = StringUtil::Replace(result, "%trigger_17_term_grams%",
+ SQLIdentifier::ToString(trigger_names[2]));
+ result = StringUtil::Replace(result, "%input_table%",
+ GetQualifiedTableName(qname));
+ result = StringUtil::Replace(result, "%token_ctes%", token_ctes);
+ return result;
+}
+
+static string LayeredInsertDFTriggerScript(const QualifiedName &qname,
+ const string &token_ctes) {
+ // clang-format off
+ string result = R"(
+ CREATE TRIGGER %trigger_35_term_stats_df% AFTER INSERT ON %input_table%
+ REFERENCING NEW TABLE AS fts_new_rows
+ FOR EACH STATEMENT
+ UPDATE %fts_schema%.term_stats AS ts
+ SET df = d.df
+ FROM %fts_schema%.dict AS d
+ WHERE ts.termid = d.termid
+ AND ts.termid IN (
+ %affected_termids%
+ );
+
+ CREATE TRIGGER %trigger_36_term_stats_by_len_df% AFTER INSERT ON %input_table%
+ REFERENCING NEW TABLE AS fts_new_rows
+ FOR EACH STATEMENT
+ UPDATE %fts_schema%.term_stats_by_len AS ts
+ SET df = d.df
+ FROM %fts_schema%.dict AS d
+ WHERE ts.termid = d.termid
+ AND ts.termid IN (
+ %affected_termids%
+ );
+ )";
+ // clang-format on
+
+ auto affected_termids = LayeredAffectedTermIdsSubquery(token_ctes);
+ auto trigger_names = GetFTSLayeredInsertTriggerNames(qname);
+ result = StringUtil::Replace(result, "%trigger_35_term_stats_df%",
+ SQLIdentifier::ToString(trigger_names[3]));
+ result = StringUtil::Replace(result, "%trigger_36_term_stats_by_len_df%",
+ SQLIdentifier::ToString(trigger_names[4]));
+ result = StringUtil::Replace(result, "%input_table%",
+ GetQualifiedTableName(qname));
+ result = StringUtil::Replace(result, "%affected_termids%", affected_termids);
+ return result;
+}
+
+static string LayeredDeleteAfterDFTriggerScript(const QualifiedName &qname,
+ const string &token_ctes) {
+ // clang-format off
+ string result = R"(
+ CREATE TRIGGER %trigger_23_term_stats_df% AFTER DELETE ON %input_table%
+ REFERENCING OLD TABLE AS fts_old_rows
+ FOR EACH STATEMENT
+ UPDATE %fts_schema%.term_stats AS ts
+ SET df = d.df
+ FROM %fts_schema%.dict AS d
+ WHERE ts.termid = d.termid
+ AND ts.termid IN (
+ %affected_termids%
+ );
+
+ CREATE TRIGGER %trigger_24_term_stats_by_len_df% AFTER DELETE ON %input_table%
+ REFERENCING OLD TABLE AS fts_old_rows
+ FOR EACH STATEMENT
+ UPDATE %fts_schema%.term_stats_by_len AS ts
+ SET df = d.df
+ FROM %fts_schema%.dict AS d
+ WHERE ts.termid = d.termid
+ AND ts.termid IN (
+ %affected_termids%
+ );
+
+ CREATE TRIGGER %trigger_25_term_grams_prune% AFTER DELETE ON %input_table%
+ REFERENCING OLD TABLE AS fts_old_rows
+ FOR EACH STATEMENT
+ DELETE FROM %fts_schema%.term_grams
+ WHERE termid IN (
+ SELECT d.termid
+ FROM %fts_schema%.dict AS d
+ WHERE d.df = 0
+ AND d.termid IN (
+ %affected_termids%
+ )
+ );
+
+ CREATE TRIGGER %trigger_26_term_stats_by_len_prune% AFTER DELETE ON %input_table%
+ REFERENCING OLD TABLE AS fts_old_rows
+ FOR EACH STATEMENT
+ DELETE FROM %fts_schema%.term_stats_by_len
+ WHERE termid IN (
+ SELECT d.termid
+ FROM %fts_schema%.dict AS d
+ WHERE d.df = 0
+ AND d.termid IN (
+ %affected_termids%
+ )
+ );
+
+ CREATE TRIGGER %trigger_27_term_stats_prune% AFTER DELETE ON %input_table%
+ REFERENCING OLD TABLE AS fts_old_rows
+ FOR EACH STATEMENT
+ DELETE FROM %fts_schema%.term_stats
+ WHERE termid IN (
+ SELECT d.termid
+ FROM %fts_schema%.dict AS d
+ WHERE d.df = 0
+ AND d.termid IN (
+ %affected_termids%
+ )
+ );
+ )";
+ // clang-format on
+
+ auto affected_termids = LayeredAffectedTermIdsSubquery(token_ctes);
+ auto trigger_names = GetFTSLayeredDeleteTriggerNames(qname);
+ result = StringUtil::Replace(result, "%trigger_23_term_stats_df%",
+ SQLIdentifier::ToString(trigger_names[0]));
+ result = StringUtil::Replace(result, "%trigger_24_term_stats_by_len_df%",
+ SQLIdentifier::ToString(trigger_names[1]));
+ result = StringUtil::Replace(result, "%trigger_25_term_grams_prune%",
+ SQLIdentifier::ToString(trigger_names[2]));
+ result = StringUtil::Replace(result, "%trigger_26_term_stats_by_len_prune%",
+ SQLIdentifier::ToString(trigger_names[3]));
+ result = StringUtil::Replace(result, "%trigger_27_term_stats_prune%",
+ SQLIdentifier::ToString(trigger_names[4]));
+ result = StringUtil::Replace(result, "%input_table%",
+ GetQualifiedTableName(qname));
+ result = StringUtil::Replace(result, "%affected_termids%", affected_termids);
+ return result;
+}
+
static string InsertTriggerScript(const QualifiedName &qname,
const string &input_id,
- const vector &input_values) {
+ const vector &input_values,
+ bool layered_search) {
// clang-format off
string docs_new_docs_cte = R"(
fts_new_docs AS (
@@ -492,6 +1178,8 @@ static string InsertTriggerScript(const QualifiedName &qname,
0 AS df
FROM new_terms;
+ %layered_insert_new_terms_triggers%
+
CREATE TRIGGER %trigger_20_terms% AFTER INSERT ON %input_table%
REFERENCING NEW TABLE AS fts_new_rows
FOR EACH STATEMENT
@@ -501,7 +1189,8 @@ static string InsertTriggerScript(const QualifiedName &qname,
ss.fieldid,
d.termid
FROM stemmed_stopped AS ss
- JOIN %fts_schema%.dict AS d ON ss.term = d.term;
+ JOIN %fts_schema%.dict AS d ON ss.term = d.term
+ %insert_terms_order_by%;
CREATE TRIGGER %trigger_30_dict_df% AFTER INSERT ON %input_table%
REFERENCING NEW TABLE AS fts_new_rows
@@ -518,6 +1207,8 @@ static string InsertTriggerScript(const QualifiedName &qname,
) AS inserted_df_delta
WHERE d.termid = inserted_df_delta.termid;
+ %layered_insert_df_triggers%
+
CREATE TRIGGER %trigger_40_stats% AFTER INSERT ON %input_table%
REFERENCING NEW TABLE AS fts_new_rows
FOR EACH STATEMENT
@@ -557,8 +1248,18 @@ static string InsertTriggerScript(const QualifiedName &qname,
StringUtil::Replace(result, "%union_fields_query%",
StringUtil::Join(tokenize_fields, " UNION ALL "));
result = StringUtil::Replace(result, "%token_ctes%", token_ctes);
+ result = StringUtil::Replace(
+ result, "%layered_insert_new_terms_triggers%",
+ layered_search ? LayeredInsertNewTermsTriggerScript(qname, token_ctes)
+ : "");
+ result = StringUtil::Replace(
+ result, "%layered_insert_df_triggers%",
+ layered_search ? LayeredInsertDFTriggerScript(qname, token_ctes) : "");
+ result = StringUtil::Replace(
+ result, "%insert_terms_order_by%",
+ layered_search ? "ORDER BY d.termid, ss.fieldid, ss.docid" : "");
- auto trigger_names = GetFTSTriggerNames(qname);
+ auto trigger_names = GetFTSInsertTriggerNames(qname);
result = StringUtil::Replace(result, "%trigger_00_docs%",
SQLIdentifier::ToString(trigger_names[0]));
result = StringUtil::Replace(result, "%trigger_10_dict_insert%",
@@ -577,8 +1278,28 @@ static string InsertTriggerScript(const QualifiedName &qname,
}
static string DeleteTriggerScript(const QualifiedName &qname,
- const string &input_id) {
+ const string &input_id,
+ const vector &input_values,
+ bool layered_search) {
// clang-format off
+ string tokenize_field_query = R"(
+ SELECT unnest(%fts_schema%.tokenize(fts_di.%input_value%)) AS w
+ FROM fts_old_rows AS fts_di
+ )";
+
+ string token_ctes = R"(
+ WITH tokenized AS (
+ %union_fields_query%
+ ),
+ stemmed_stopped AS (
+ SELECT stem(t.w, '%stemmer%') AS term
+ FROM tokenized AS t
+ WHERE t.w NOT NULL
+ AND t.w <> ''
+ AND t.w NOT IN (SELECT sw FROM %fts_schema%.stopwords)
+ )
+ )";
+
string result = R"(
CREATE TRIGGER %trigger_00_dict_df% AFTER DELETE ON %input_table%
REFERENCING OLD TABLE AS fts_old_rows
@@ -611,6 +1332,8 @@ static string DeleteTriggerScript(const QualifiedName &qname,
DELETE FROM %fts_schema%.docs
WHERE name IN (SELECT %input_id% FROM fts_old_rows);
+ %layered_delete_after_df_triggers%
+
CREATE TRIGGER %trigger_30_dict_prune% AFTER DELETE ON %input_table%
REFERENCING OLD TABLE AS fts_old_rows
FOR EACH STATEMENT
@@ -623,9 +1346,19 @@ static string DeleteTriggerScript(const QualifiedName &qname,
UPDATE %fts_schema%.stats
SET num_docs = (SELECT COUNT(docid) FROM %fts_schema%.docs),
avgdl = (SELECT SUM(len) / COUNT(len) FROM %fts_schema%.docs);
- )";
+ )";
// clang-format on
+ vector tokenize_fields;
+ for (auto &input_value : input_values) {
+ auto query = StringUtil::Replace(tokenize_field_query, "%input_value%",
+ SQLIdentifier::ToString(input_value));
+ tokenize_fields.push_back(query);
+ }
+ token_ctes =
+ StringUtil::Replace(token_ctes, "%union_fields_query%",
+ StringUtil::Join(tokenize_fields, " UNION ALL "));
+
auto trigger_names = GetFTSDeleteTriggerNames(qname);
result = StringUtil::Replace(result, "%trigger_00_dict_df%",
SQLIdentifier::ToString(trigger_names[0]));
@@ -637,6 +1370,10 @@ static string DeleteTriggerScript(const QualifiedName &qname,
SQLIdentifier::ToString(trigger_names[3]));
result = StringUtil::Replace(result, "%trigger_40_stats%",
SQLIdentifier::ToString(trigger_names[4]));
+ result = StringUtil::Replace(
+ result, "%layered_delete_after_df_triggers%",
+ layered_search ? LayeredDeleteAfterDFTriggerScript(qname, token_ctes)
+ : "");
result = StringUtil::Replace(result, "%input_table%",
GetQualifiedTableName(qname));
result = StringUtil::Replace(result, "%input_id%",
@@ -653,7 +1390,8 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname,
const vector &input_values,
const string &stemmer, const string &stopwords,
const string &ignore, bool strip_accents,
- bool lower, bool incremental, bool cluster_terms) {
+ bool lower, bool incremental, bool cluster_terms,
+ bool layered_search) {
string result;
if (TableExists(context, qname) && SupportsFTSTriggers(context, qname)) {
result += DropFTSTriggersScript(qname);
@@ -665,10 +1403,16 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname,
GetFTSBuildTermsTable(qname),
GetFTSBuildDictTable(qname), cluster_terms);
result += MatchMacroScript();
+ if (layered_search) {
+ result += LayeredSidecarScript(qname);
+ result += LayeredSearchMacroScript();
+ }
if (incremental) {
result += IncrementalIndexSetupScript(qname);
- result += InsertTriggerScript(qname, input_id, input_values);
- result += DeleteTriggerScript(qname, input_id);
+ result +=
+ InsertTriggerScript(qname, input_id, input_values, layered_search);
+ result +=
+ DeleteTriggerScript(qname, input_id, input_values, layered_search);
}
string fts_schema = GetFTSSchema(qname);
@@ -712,8 +1456,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context,
const FunctionParameters ¶meters) {
auto qname =
GetQualifiedName(context, StringValue::Get(parameters.values[0]));
- Catalog::GetEntry(context, qname.Catalog(), qname.Schema(),
- qname.Name());
+ Catalog::GetEntry(context, qname);
// Named parameters
auto get_string = [&](const string &name, const string &def) {
@@ -737,12 +1480,12 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context,
const bool lower = get_bool("lower", true);
const bool overwrite = get_bool("overwrite", false);
const bool incremental = get_bool("incremental", false);
- const bool cluster_terms = get_bool("cluster_terms", false);
+ const bool layered_search = get_bool("layered_search", false);
+ const bool cluster_terms = get_bool("cluster_terms", false) || layered_search;
if (stopwords != "english" && stopwords != "none") {
auto sw_qname = GetQualifiedName(context, stopwords);
- Catalog::GetEntry(context, sw_qname.Catalog(),
- sw_qname.Schema(), sw_qname.Name());
+ Catalog::GetEntry(context, sw_qname);
}
const string fts_schema = GetFTSSchema(qname);
@@ -760,8 +1503,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context,
// Positional parameters: table, id column, value column(s)
const string doc_id = StringValue::Get(parameters.values[1]);
- auto &table = Catalog::GetEntry(
- context, qname.Catalog(), qname.Schema(), qname.Name());
+ auto &table = Catalog::GetEntry(context, qname);
if (!table.ColumnExists(Identifier(doc_id))) {
throw CatalogException("Table '%s.%s' does not have a column named '%s'!",
qname.Schema().GetIdentifierName(),
@@ -801,14 +1543,14 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context,
throw InvalidInputException("incremental FTS indexes require the document "
"id column to be NOT NULL");
}
- if (incremental && cluster_terms) {
+ if (incremental && cluster_terms && !layered_search) {
throw InvalidInputException("cluster_terms cannot be combined with "
"incremental FTS indexes");
}
return IndexingScript(context, qname, doc_id, doc_values, stemmer, stopwords,
ignore, strip_accents, lower, incremental,
- cluster_terms);
+ cluster_terms, layered_search);
}
} // namespace duckdb
diff --git a/test/sql/fts/test_indexing.test_slow b/test/sql/fts/test_indexing.test_slow
index a19e088..4643b87 100644
--- a/test/sql/fts/test_indexing.test_slow
+++ b/test/sql/fts/test_indexing.test_slow
@@ -211,6 +211,12 @@ SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'mark laurens', fiel
doc2
doc3
+query I
+SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'mark laurens', fields := 'body, author') AS score FROM documents) sq WHERE score IS NOT NULL ORDER BY score DESC
+----
+doc2
+doc3
+
# if we don't search any fields, we won't get any results
query I
SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'hannes mark laurens', fields := '') AS score FROM documents) sq WHERE score IS NOT NULL ORDER BY score DESC
@@ -265,6 +271,11 @@ doc1 QUÁCKING+QUÁCKING+QUÁCKING Hannes
doc2 BÁRKING+BÁRKING+BÁRKING+BÁRKING Mark
doc3 MÉOWING+MÉOWING+MÉOWING+MÉOWING+MÉOWING+999 Laurens
+query I
+SELECT id FROM (SELECT *, fts_main_documents.match_bm25(id, 'the mark', conjunctive := true) AS score FROM documents) sq WHERE score IS NOT NULL ORDER BY score DESC
+----
+doc2
+
# document frequency remains document-based when terms repeat within documents
statement ok
CREATE TABLE df_documents(id VARCHAR, body VARCHAR)
diff --git a/test/sql/fts/test_layered_search.test b/test/sql/fts/test_layered_search.test
new file mode 100644
index 0000000..b0c5151
--- /dev/null
+++ b/test/sql/fts/test_layered_search.test
@@ -0,0 +1,395 @@
+# name: test/sql/fts/test_layered_search.test
+# description: Layered BM25 search sidecar and incremental maintenance
+# group: [fts]
+
+require skip_reload
+
+require fts
+
+require no_alternative_verify
+
+statement ok
+CREATE TABLE documents(id VARCHAR NOT NULL, body VARCHAR, author VARCHAR)
+
+statement ok
+INSERT INTO documents VALUES
+ ('doc1', 'quacking quacking quacking', 'Hannes'),
+ ('doc2', 'barking barking barking barking', 'Mark'),
+ ('doc3', 'meowing meowing meowing meowing meowing 999', 'Laurens')
+
+statement ok
+PRAGMA create_fts_index('documents', 'id', 'body', 'author', stemmer='none', stopwords='none', ignore='(\\.|[^a-z0-9])+', layered_search=true, incremental=true)
+
+query I
+SELECT count(*) FROM fts_main_documents.term_stats
+----
+7
+
+query I
+SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END FROM fts_main_documents.term_grams
+----
+1
+
+query I
+SELECT count(*)
+FROM fts_main_documents.term_grams AS tg
+JOIN fts_main_documents.term_stats AS ts USING (termid)
+WHERE ts.term = '999'
+----
+0
+
+query I
+SELECT count(*)
+FROM (
+ SELECT termid, lag(termid) OVER () AS previous_termid
+ FROM fts_main_documents.terms
+) ordered_terms
+WHERE previous_termid IS NOT NULL
+ AND termid < previous_termid
+----
+0
+
+query I
+SELECT docname
+FROM fts_main_documents.search_layered_bm25('quack', fields := 'body', enable_short_fuzzy := false)
+----
+doc1
+
+query I
+SELECT id
+FROM (
+ SELECT *,
+ fts_main_documents.match_layered_bm25(id, 'quack', fields := 'body', enable_short_fuzzy := false) AS score
+ FROM documents
+) sq
+WHERE score IS NOT NULL
+ORDER BY score DESC
+----
+doc1
+
+query I
+SELECT docname
+FROM fts_main_documents.search_layered_bm25('mark', fields := 'author', enable_short_fuzzy := false)
+----
+doc2
+
+query I
+SELECT docname
+FROM fts_main_documents.search_layered_bm25('999', fields := 'body', enable_short_fuzzy := false)
+----
+doc3
+
+statement ok
+INSERT INTO documents VALUES ('doc4', 'chirping chirping', 'Mark')
+
+statement ok
+INSERT INTO documents VALUES ('doc5', 'singing', 'Marker')
+
+query II
+SELECT term, df
+FROM fts_main_documents.term_stats
+WHERE term IN ('chirping', 'mark')
+ORDER BY term
+----
+chirping 1
+mark 2
+
+query I rowsort
+SELECT docname
+FROM fts_main_documents.search_layered_bm25('mark', fields := 'author', enable_short_fuzzy := false, expand_exact_terms := false)
+----
+doc2
+doc4
+
+query I rowsort
+SELECT docname
+FROM fts_main_documents.search_layered_bm25('mark', fields := 'author', enable_short_fuzzy := false, expand_exact_terms := true)
+----
+doc2
+doc4
+doc5
+
+query I
+SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END
+FROM fts_main_documents.term_grams AS tg
+JOIN fts_main_documents.term_stats AS ts USING (termid)
+WHERE ts.term = 'chirping'
+----
+1
+
+query I
+SELECT docname
+FROM fts_main_documents.search_layered_bm25('chirp', fields := 'body', enable_short_fuzzy := false)
+----
+doc4
+
+statement ok
+DELETE FROM documents WHERE id = 'doc3'
+
+query I
+SELECT count(*) FROM fts_main_documents.term_stats WHERE term = 'meowing'
+----
+0
+
+query I
+SELECT count(*)
+FROM fts_main_documents.term_grams AS tg
+LEFT JOIN fts_main_documents.term_stats AS ts USING (termid)
+WHERE ts.termid IS NULL
+----
+0
+
+query I
+SELECT count(*)
+FROM fts_main_documents.search_layered_bm25('meow', fields := 'body', enable_short_fuzzy := false)
+----
+0
+
+statement ok
+CREATE TABLE plain_documents(id VARCHAR, body VARCHAR)
+
+statement ok
+INSERT INTO plain_documents VALUES ('doc1', 'alpha beta')
+
+statement ok
+PRAGMA create_fts_index('plain_documents', 'id', 'body', stemmer='none', stopwords='none')
+
+query I
+SELECT count(*)
+FROM information_schema.tables
+WHERE table_schema = 'fts_main_plain_documents'
+ AND table_name = 'term_stats'
+----
+0
+
+statement ok
+CREATE TABLE static_layered_documents(id VARCHAR, body VARCHAR)
+
+statement ok
+INSERT INTO static_layered_documents VALUES ('doc1', 'alpha alpha')
+
+statement ok
+PRAGMA create_fts_index('static_layered_documents', 'id', 'body', stemmer='none', stopwords='none', layered_search=true)
+
+query I
+SELECT count(*)
+FROM information_schema.tables
+WHERE table_schema = 'fts_main_static_layered_documents'
+ AND table_name = 'term_stats'
+----
+1
+
+query I
+SELECT docname
+FROM fts_main_static_layered_documents.search_layered_bm25('alpha', enable_short_fuzzy := false)
+----
+doc1
+
+statement ok
+INSERT INTO static_layered_documents VALUES ('doc2', 'beta')
+
+query I
+SELECT count(*)
+FROM fts_main_static_layered_documents.search_layered_bm25('beta', enable_short_fuzzy := false)
+----
+0
+
+statement ok
+CREATE TABLE lifecycle_docs(id VARCHAR NOT NULL, body VARCHAR)
+
+statement ok
+INSERT INTO lifecycle_docs VALUES ('doc1', 'alpha beta')
+
+statement ok
+PRAGMA create_fts_index('lifecycle_docs', 'id', 'body', stemmer='none', stopwords='none', layered_search=true, incremental=true)
+
+query I
+SELECT count(*)
+FROM information_schema.tables
+WHERE table_schema = 'fts_main_lifecycle_docs'
+ AND table_name = 'term_stats'
+----
+1
+
+query I
+SELECT CASE WHEN count(*) > 0 THEN 1 ELSE 0 END
+FROM duckdb_triggers()
+WHERE trigger_name LIKE '__fts_fts_main_lifecycle_docs_%term_stats%'
+----
+1
+
+statement ok
+PRAGMA create_fts_index('lifecycle_docs', 'id', 'body', stemmer='none', stopwords='none', overwrite=true, incremental=true)
+
+query I
+SELECT count(*)
+FROM information_schema.tables
+WHERE table_schema = 'fts_main_lifecycle_docs'
+ AND table_name = 'term_stats'
+----
+0
+
+query I
+SELECT count(*)
+FROM duckdb_triggers()
+WHERE trigger_name LIKE '__fts_fts_main_lifecycle_docs_%term_stats%'
+----
+0
+
+statement ok
+PRAGMA drop_fts_index('lifecycle_docs')
+
+query I
+SELECT count(*)
+FROM duckdb_schemas()
+WHERE schema_name = 'fts_main_lifecycle_docs'
+----
+0
+
+query I
+SELECT count(*)
+FROM duckdb_triggers()
+WHERE trigger_name LIKE '__fts_fts_main_lifecycle_docs_%'
+----
+0
+
+statement ok
+CREATE TABLE "layered docs"(id VARCHAR NOT NULL, "bo""dy" VARCHAR)
+
+statement ok
+INSERT INTO "layered docs" VALUES ('docq', 'quoted quoted')
+
+statement ok
+PRAGMA create_fts_index('"layered docs"', 'id', 'bo"dy', stemmer='none', stopwords='none', layered_search=true, incremental=true)
+
+query I
+SELECT docname
+FROM "fts_main_layered docs".search_layered_bm25('quote', fields := 'bo"dy', enable_short_fuzzy := false)
+----
+docq
+
+statement ok
+ATTACH ':memory:' AS layered_attach
+
+statement ok
+CREATE TABLE layered_attach.main.docs(id VARCHAR NOT NULL, body VARCHAR)
+
+statement ok
+INSERT INTO layered_attach.main.docs VALUES ('doca', 'attached attached')
+
+statement ok
+PRAGMA create_fts_index(layered_attach.main.docs, 'id', 'body', stemmer='none', stopwords='none', layered_search=true)
+
+query I
+SELECT docname
+FROM layered_attach.fts_main_docs.search_layered_bm25('attach', enable_short_fuzzy := false)
+----
+doca
+
+statement ok
+CREATE TABLE english_stopword_docs(id VARCHAR, body VARCHAR)
+
+statement ok
+INSERT INTO english_stopword_docs VALUES ('doc1', 'theme theme'), ('doc2', 'the theme')
+
+statement ok
+PRAGMA create_fts_index('english_stopword_docs', 'id', 'body', stemmer='none', stopwords='english', layered_search=true)
+
+query I
+SELECT count(*)
+FROM fts_main_english_stopword_docs.search_layered_bm25('the', enable_short_fuzzy := false)
+----
+0
+
+query I
+SELECT count(*)
+FROM fts_main_english_stopword_docs.search_layered_bm25('the theme', enable_short_fuzzy := false)
+----
+2
+
+statement ok
+CREATE TABLE no_stopword_docs(id VARCHAR, body VARCHAR)
+
+statement ok
+INSERT INTO no_stopword_docs VALUES ('doc1', 'the theme')
+
+statement ok
+PRAGMA create_fts_index('no_stopword_docs', 'id', 'body', stemmer='none', stopwords='none', layered_search=true)
+
+query I
+SELECT docname
+FROM fts_main_no_stopword_docs.search_layered_bm25('the', enable_short_fuzzy := false)
+----
+doc1
+
+statement ok
+CREATE TABLE custom_stopwords(sw VARCHAR)
+
+statement ok
+INSERT INTO custom_stopwords VALUES ('noise')
+
+statement ok
+CREATE TABLE custom_stopword_docs(id VARCHAR, body VARCHAR)
+
+statement ok
+INSERT INTO custom_stopword_docs VALUES ('doc1', 'noisy noisy')
+
+statement ok
+PRAGMA create_fts_index('custom_stopword_docs', 'id', 'body', stemmer='none', stopwords='custom_stopwords', layered_search=true)
+
+query I
+SELECT count(*)
+FROM fts_main_custom_stopword_docs.search_layered_bm25('noise', enable_short_fuzzy := false)
+----
+0
+
+statement ok
+CREATE TABLE layered_stem_docs(id VARCHAR, body VARCHAR)
+
+statement ok
+INSERT INTO layered_stem_docs VALUES ('doc1', 'preconnectpost')
+
+statement ok
+PRAGMA create_fts_index('layered_stem_docs', 'id', 'body', stemmer='porter', stopwords='none', layered_search=true)
+
+query I
+SELECT docname
+FROM fts_main_layered_stem_docs.search_layered_bm25(
+ 'connected connection',
+ enable_prefix := false,
+ enable_substring := true,
+ enable_fuzzy := false,
+ enable_short_fuzzy := false
+)
+----
+doc1
+
+statement ok
+CREATE TABLE layered_limit_docs(id VARCHAR, body VARCHAR)
+
+statement ok
+INSERT INTO layered_limit_docs VALUES ('doc1', 'quack'), ('doc2', 'quark')
+
+statement ok
+PRAGMA create_fts_index('layered_limit_docs', 'id', 'body', stemmer='none', stopwords='none', layered_search=true)
+
+query I rowsort
+SELECT docname
+FROM fts_main_layered_limit_docs.search_layered_bm25(
+ 'quak',
+ term_limit := 2,
+ enable_prefix := true,
+ enable_substring := false,
+ enable_fuzzy := true,
+ enable_short_fuzzy := true
+)
+----
+doc1
+doc2
+
+query I rowsort
+SELECT docname
+FROM fts_main_documents.search_layered_bm25('mark', fields := ' author ', enable_short_fuzzy := false)
+----
+doc2
+doc4