From 63e770e780154c15fc3cdd16d35921b85dc794ce Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 30 Jun 2026 13:42:06 +0200 Subject: [PATCH 1/6] Allow incremental and clustered together --- .../fts/test_incremental_clustered_terms.test | 419 ++++++++++++++++++ 1 file changed, 419 insertions(+) create mode 100644 test/sql/fts/test_incremental_clustered_terms.test diff --git a/test/sql/fts/test_incremental_clustered_terms.test b/test/sql/fts/test_incremental_clustered_terms.test new file mode 100644 index 0000000..10ce5a2 --- /dev/null +++ b/test/sql/fts/test_incremental_clustered_terms.test @@ -0,0 +1,419 @@ +# name: test/sql/fts/test_incremental_clustered_terms.test +# description: Test clustered terms ordering with incremental FTS updates +# group: [fts] + +require fts + +require skip_reload + +require no_alternative_verify + +statement ok +CREATE TABLE clustered_docs(id VARCHAR NOT NULL, body VARCHAR) + +statement ok +INSERT INTO clustered_docs VALUES + ('doc1', 'alpha beta alpha'), + ('doc2', 'beta gamma'), + ('doc3', 'delta') + +statement ok +PRAGMA create_fts_index('clustered_docs', 'id', 'body', stemmer='none', stopwords='none', incremental=true, cluster_terms=true) + +query I +SELECT count(*) +FROM ( + SELECT termid, fieldid, docid, + lag(termid) OVER () AS previous_termid, + lag(fieldid) OVER () AS previous_fieldid, + lag(docid) OVER () AS previous_docid + FROM fts_main_clustered_docs.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND ( + termid < previous_termid + OR (termid = previous_termid AND fieldid < previous_fieldid) + OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) + ) +---- +0 + +statement ok +INSERT INTO clustered_docs VALUES + ('doc4', 'alpha aardvark aardvark'), + ('doc5', 'gamma beta beta') + +query I +SELECT count(*) +FROM ( + SELECT termid, fieldid, docid, + lag(termid) OVER () AS previous_termid, + lag(fieldid) OVER () AS previous_fieldid, + lag(docid) OVER () AS previous_docid + FROM fts_main_clustered_docs.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND ( + termid < previous_termid + OR (termid = previous_termid AND fieldid < previous_fieldid) + OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) + ) +---- +0 + +query II +SELECT term, df FROM fts_main_clustered_docs.dict ORDER BY term +---- +aardvark 1 +alpha 2 +beta 3 +delta 1 +gamma 2 + +query I +WITH recomputed AS ( + SELECT termid, count(DISTINCT docid) AS df + FROM fts_main_clustered_docs.terms + GROUP BY termid +), +diffs AS ( + SELECT COALESCE(d.termid, r.termid) AS termid + FROM fts_main_clustered_docs.dict AS d + FULL OUTER JOIN recomputed AS r USING (termid) + WHERE d.termid IS NULL + OR r.termid IS NULL + OR d.df IS DISTINCT FROM r.df +) +SELECT count(*) FROM diffs +---- +0 + +statement ok +DELETE FROM clustered_docs WHERE id IN ('doc3', 'doc4') + +query I +SELECT count(*) +FROM ( + SELECT termid, fieldid, docid, + lag(termid) OVER () AS previous_termid, + lag(fieldid) OVER () AS previous_fieldid, + lag(docid) OVER () AS previous_docid + FROM fts_main_clustered_docs.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND ( + termid < previous_termid + OR (termid = previous_termid AND fieldid < previous_fieldid) + OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) + ) +---- +0 + +query II +SELECT term, df FROM fts_main_clustered_docs.dict ORDER BY term +---- +alpha 1 +beta 3 +gamma 2 + +query I +SELECT count(*) +FROM fts_main_clustered_docs.terms AS t +LEFT JOIN fts_main_clustered_docs.docs AS d ON t.docid = d.docid +WHERE d.docid IS NULL +---- +0 + +query I +SELECT count(*) +FROM fts_main_clustered_docs.terms AS t +LEFT JOIN fts_main_clustered_docs.dict AS dict ON t.termid = dict.termid +WHERE dict.termid IS NULL +---- +0 + +statement ok +BEGIN TRANSACTION + +statement ok +DELETE FROM clustered_docs WHERE id = 'doc2' + +statement ok +INSERT INTO clustered_docs VALUES ('doc2', 'gamma zebra zebra') + +statement ok +COMMIT + +query II +SELECT term, df FROM fts_main_clustered_docs.dict ORDER BY term +---- +alpha 1 +beta 2 +gamma 2 +zebra 1 + +query I +SELECT count(*) +FROM ( + SELECT termid, fieldid, docid, + lag(termid) OVER () AS previous_termid, + lag(fieldid) OVER () AS previous_fieldid, + lag(docid) OVER () AS previous_docid + FROM fts_main_clustered_docs.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND ( + termid < previous_termid + OR (termid = previous_termid AND fieldid < previous_fieldid) + OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) + ) +---- +0 + +query I rowsort +SELECT id FROM ( + SELECT *, fts_main_clustered_docs.match_bm25(id, 'zebra') AS score + FROM clustered_docs +) sq +WHERE score IS NOT NULL +---- +doc2 + +query I +SELECT count(*) FROM ( + SELECT *, fts_main_clustered_docs.match_bm25(id, 'delta') AS score + FROM clustered_docs +) sq +WHERE score IS NOT NULL +---- +0 + +# Clustered incremental indexes expose `terms` as an ordered view backed by an +# internal storage table. Non-incremental clustered indexes keep `terms` as a +# physical table. +statement ok +CREATE TABLE clustered_static_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO clustered_static_docs VALUES ('doc1', 'alpha beta'), ('doc2', 'beta gamma') + +statement ok +PRAGMA create_fts_index('clustered_static_docs', 'id', 'body', stemmer='none', stopwords='none', cluster_terms=true) + +query I +SELECT count(*) FROM duckdb_views() +WHERE schema_name = 'fts_main_clustered_static_docs' + AND view_name = 'terms' +---- +0 + +query I +SELECT count(*) FROM duckdb_tables() +WHERE schema_name = 'fts_main_clustered_static_docs' + AND table_name = 'terms' +---- +1 + +query I +SELECT count(*) FROM duckdb_views() +WHERE schema_name = 'fts_main_clustered_docs' + AND view_name = 'terms' +---- +1 + +query I +SELECT count(*) FROM duckdb_tables() +WHERE schema_name = 'fts_main_clustered_docs' + AND table_name = '__terms_storage' +---- +1 + +# Dropping a clustered incremental index removes triggers, view, and backing +# storage through the FTS schema. +statement ok +CREATE TABLE drop_clustered_docs(id VARCHAR NOT NULL, body VARCHAR) + +statement ok +INSERT INTO drop_clustered_docs VALUES ('doc1', 'alpha beta') + +statement ok +PRAGMA create_fts_index('drop_clustered_docs', 'id', 'body', stemmer='none', stopwords='none', incremental=true, cluster_terms=true) + +query I +SELECT count(*) FROM duckdb_triggers() +WHERE trigger_name LIKE '__fts_fts_main_drop_clustered_docs_%' +---- +10 + +statement ok +PRAGMA drop_fts_index('drop_clustered_docs') + +query I +SELECT count(*) FROM duckdb_schemas() +WHERE schema_name = 'fts_main_drop_clustered_docs' +---- +0 + +query I +SELECT count(*) FROM duckdb_triggers() +WHERE trigger_name LIKE '__fts_fts_main_drop_clustered_docs_%' +---- +0 + +# Overwrite transitions should remove old trigger/view/table state cleanly. +statement ok +CREATE TABLE overwrite_clustered_docs(id VARCHAR NOT NULL, body VARCHAR) + +statement ok +INSERT INTO overwrite_clustered_docs VALUES ('doc1', 'alpha beta') + +statement ok +PRAGMA create_fts_index('overwrite_clustered_docs', 'id', 'body', stemmer='none', stopwords='none', incremental=true, cluster_terms=true) + +statement ok +PRAGMA create_fts_index('overwrite_clustered_docs', 'id', 'body', stemmer='none', stopwords='none', overwrite=true) + +query I +SELECT count(*) FROM duckdb_triggers() +WHERE trigger_name LIKE '__fts_fts_main_overwrite_clustered_docs_%' +---- +0 + +query II +SELECT + (SELECT count(*) FROM duckdb_tables() WHERE schema_name = 'fts_main_overwrite_clustered_docs' AND table_name = 'terms'), + (SELECT count(*) FROM duckdb_views() WHERE schema_name = 'fts_main_overwrite_clustered_docs' AND view_name = 'terms') +---- +1 0 + +statement ok +PRAGMA create_fts_index('overwrite_clustered_docs', 'id', 'body', stemmer='none', stopwords='none', overwrite=true, incremental=true) + +query III +SELECT + (SELECT count(*) FROM duckdb_tables() WHERE schema_name = 'fts_main_overwrite_clustered_docs' AND table_name = 'terms'), + (SELECT count(*) FROM duckdb_views() WHERE schema_name = 'fts_main_overwrite_clustered_docs' AND view_name = 'terms'), + (SELECT count(*) FROM duckdb_triggers() WHERE trigger_name LIKE '__fts_fts_main_overwrite_clustered_docs_%') +---- +1 0 10 + +statement ok +PRAGMA create_fts_index('overwrite_clustered_docs', 'id', 'body', stemmer='none', stopwords='none', overwrite=true, incremental=true, cluster_terms=true) + +query III +SELECT + (SELECT count(*) FROM duckdb_tables() WHERE schema_name = 'fts_main_overwrite_clustered_docs' AND table_name = '__terms_storage'), + (SELECT count(*) FROM duckdb_views() WHERE schema_name = 'fts_main_overwrite_clustered_docs' AND view_name = 'terms'), + (SELECT count(*) FROM duckdb_triggers() WHERE trigger_name LIKE '__fts_fts_main_overwrite_clustered_docs_%') +---- +1 1 10 + +statement ok +INSERT INTO overwrite_clustered_docs VALUES ('doc2', 'aardvark alpha') + +query I +SELECT count(*) +FROM ( + SELECT termid, fieldid, docid, + lag(termid) OVER () AS previous_termid, + lag(fieldid) OVER () AS previous_fieldid, + lag(docid) OVER () AS previous_docid + FROM fts_main_overwrite_clustered_docs.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND ( + termid < previous_termid + OR (termid = previous_termid AND fieldid < previous_fieldid) + OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) + ) +---- +0 + +# Multiple indexed columns should preserve field ids and order by term, field, +# then document. +statement ok +CREATE TABLE multi_field_docs(id VARCHAR NOT NULL, title VARCHAR, body VARCHAR) + +statement ok +INSERT INTO multi_field_docs VALUES + ('doc1', 'alpha', 'beta alpha'), + ('doc2', 'beta', 'alpha gamma') + +statement ok +PRAGMA create_fts_index('multi_field_docs', 'id', 'title', 'body', stemmer='none', stopwords='none', incremental=true, cluster_terms=true) + +statement ok +INSERT INTO multi_field_docs VALUES ('doc3', 'alpha', 'aardvark beta') + +query II rowsort +SELECT fieldid, field FROM fts_main_multi_field_docs.fields +---- +0 title +1 body + +query III rowsort +SELECT fields.field, dict.term, count(*) AS tf +FROM fts_main_multi_field_docs.terms AS terms +JOIN fts_main_multi_field_docs.fields AS fields ON fields.fieldid = terms.fieldid +JOIN fts_main_multi_field_docs.dict AS dict ON dict.termid = terms.termid +GROUP BY fields.field, dict.term +---- +body aardvark 1 +body alpha 2 +body beta 2 +body gamma 1 +title alpha 2 +title beta 1 + +query I +SELECT count(*) +FROM ( + SELECT termid, fieldid, docid, + lag(termid) OVER () AS previous_termid, + lag(fieldid) OVER () AS previous_fieldid, + lag(docid) OVER () AS previous_docid + FROM fts_main_multi_field_docs.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND ( + termid < previous_termid + OR (termid = previous_termid AND fieldid < previous_fieldid) + OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) + ) +---- +0 + +query I rowsort +SELECT id FROM ( + SELECT *, fts_main_multi_field_docs.match_bm25(id, 'aardvark', fields := 'body') AS score + FROM multi_field_docs +) sq +WHERE score IS NOT NULL +---- +doc3 + +# Stopword filtering should also apply to clustered incremental trigger writes. +statement ok +CREATE TABLE stopword_clustered_docs(id VARCHAR NOT NULL, body VARCHAR) + +statement ok +INSERT INTO stopword_clustered_docs VALUES ('doc1', 'alpha the beta') + +statement ok +PRAGMA create_fts_index('stopword_clustered_docs', 'id', 'body', stemmer='none', stopwords='english', incremental=true, cluster_terms=true) + +statement ok +INSERT INTO stopword_clustered_docs VALUES ('doc2', 'the gamma the') + +query I +SELECT count(*) FROM fts_main_stopword_clustered_docs.dict +WHERE term = 'the' +---- +0 + +query II +SELECT term, df FROM fts_main_stopword_clustered_docs.dict ORDER BY term +---- +alpha 1 +beta 1 +gamma 1 From c8b9d43969be4c27ba27a71f083c1b4d83cb481c Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 30 Jun 2026 13:42:26 +0200 Subject: [PATCH 2/6] Allow incremental and clustered together --- README.md | 5 +- extension_config.cmake | 2 + src/fts_indexing.cpp | 299 +++++++++++++++++- .../test_incremental_insert_edge_cases.test | 27 +- 4 files changed, 320 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d3a8a76..0f99f97 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The extension adds two `PRAGMA` statements to DuckDB: one to create, and one to create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', stopwords = 'english', ignore = '(\\.|[^a-z])+', strip_accents = 1, lower = 1, overwrite = 0, - cluster_terms = 0) + incremental = 0, cluster_terms = 0) ``` `PRAGMA` that creates a FTS index for the specified table. @@ -40,7 +40,8 @@ create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', | `strip_accents` | `BOOLEAN` | Whether to remove accents (e.g., convert `รก` to `a`). Defaults to `1` | | `lower` | `BOOLEAN` | Whether to convert all text to lowercase. Defaults to `1` | | `overwrite` | `BOOLEAN` | Whether to overwrite an existing index on a table. Defaults to `0` | -| `cluster_terms` | `BOOLEAN` | Whether to physically order the generated `terms` table by `termid`, `fieldid`, and `docid`. This can improve query-time pruning for direct reads from the FTS tables. Defaults to `0` | +| `incremental` | `BOOLEAN` | Whether to maintain the FTS index with triggers after inserts and deletes on the input table. Defaults to `0` | +| `cluster_terms` | `BOOLEAN` | Whether to order the generated `terms` relation by `termid`, `fieldid`, and `docid`. For non-incremental indexes this physically orders the `terms` table. With `incremental=1`, `terms` is exposed as an ordered view over internal incremental storage. Defaults to `0` | diff --git a/extension_config.cmake b/extension_config.cmake index 1c44a84..2b39738 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -1,5 +1,7 @@ # This file is included by DuckDB's build system. It specifies which extension to load +duckdb_extension_load(json) + # Extension from this repo duckdb_extension_load(fts SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index e506f07..99fef3a 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -58,16 +58,37 @@ static vector GetFTSInsertTriggerNames(const QualifiedName &qname) { prefix + "30_dict_df", prefix + "40_stats"}; } +static vector +GetFTSClusteredInsertTriggerNames(const QualifiedName &qname) { + auto prefix = StringUtil::Format("__fts_%s_ai_", GetFTSSchemaName(qname)); + return {prefix + "00_docs", prefix + "10_dict_insert", + prefix + "20_terms_store", prefix + "30_dict_df", + prefix + "40_stats"}; +} + static vector GetFTSDeleteTriggerNames(const QualifiedName &qname) { auto prefix = StringUtil::Format("__fts_%s_ad_", GetFTSSchemaName(qname)); return {prefix + "00_dict_df", prefix + "10_terms", prefix + "20_docs", prefix + "30_dict_prune", prefix + "40_stats"}; } +static vector +GetFTSClusteredDeleteTriggerNames(const QualifiedName &qname) { + auto prefix = StringUtil::Format("__fts_%s_ad_", GetFTSSchemaName(qname)); + return {prefix + "00_dict_df", prefix + "10_terms_store", + prefix + "20_docs", prefix + "30_dict_prune", prefix + "40_stats"}; +} + static vector GetFTSTriggerNames(const QualifiedName &qname) { auto result = GetFTSInsertTriggerNames(qname); auto delete_triggers = GetFTSDeleteTriggerNames(qname); result.insert(result.end(), delete_triggers.begin(), delete_triggers.end()); + auto clustered_insert_triggers = GetFTSClusteredInsertTriggerNames(qname); + result.insert(result.end(), clustered_insert_triggers.begin(), + clustered_insert_triggers.end()); + auto clustered_delete_triggers = GetFTSClusteredDeleteTriggerNames(qname); + result.insert(result.end(), clustered_delete_triggers.begin(), + clustered_delete_triggers.end()); return result; } @@ -80,6 +101,10 @@ static string GetFTSBuildDictTable(const QualifiedName &qname) { return SQLIdentifier::ToString("__fts_build_dict_" + GetFTSSchemaName(qname)); } +static string GetFTSTermsStorageTable() { + return SQLIdentifier::ToString("__terms_storage"); +} + static bool TableExists(ClientContext &context, const QualifiedName &qname) { return Catalog::GetEntry( context, qname.Catalog(), qname.Schema(), qname.Name(), @@ -400,6 +425,17 @@ static string IncrementalIndexSetupScript(const QualifiedName &qname) { SQLIdentifier::ToString(index_name)); } +static string ClusteredIncrementalIndexSetupScript() { + return StringUtil::Format( + "CREATE TABLE %%fts_schema%%.%s AS SELECT termid, docid, fieldid FROM " + "%%fts_schema%%.terms;\n" + "DROP TABLE %%fts_schema%%.terms;\n" + "CREATE VIEW %%fts_schema%%.terms AS SELECT termid, docid, fieldid FROM " + "%%fts_schema%%.%s ORDER BY termid, fieldid, docid;", + GetFTSTermsStorageTable(), + GetFTSTermsStorageTable()); +} + static string InsertTriggerScript(const QualifiedName &qname, const string &input_id, const vector &input_values) { @@ -576,6 +612,184 @@ static string InsertTriggerScript(const QualifiedName &qname, return result; } +static string ClusteredInsertTriggerScript(const QualifiedName &qname, + const string &input_id, + const vector &input_values) { + // clang-format off + string docs_new_docs_cte = R"( + fts_new_docs AS ( + SELECT COALESCE((SELECT max(docid) + 1 FROM %fts_schema%.docs), 0) + + row_number() OVER (ORDER BY fts_new_rows.%input_id%) - 1 AS docid, + fts_new_rows.%input_id% AS name, + %input_value_select_list% + FROM fts_new_rows + ) + )"; + + string token_new_docs_cte = R"( + fts_new_docs AS ( + SELECT (SELECT max(docid) - (SELECT count(*) FROM fts_new_rows) + 1 FROM %fts_schema%.docs) + + row_number() OVER (ORDER BY fts_new_rows.%input_id%) - 1 AS docid, + fts_new_rows.%input_id% AS name, + %input_value_select_list% + FROM fts_new_rows + ) + )"; + + string tokenize_field_query = R"( + SELECT unnest(%fts_schema%.tokenize(fts_ii.%input_value%)) AS w, + fts_ii.docid AS docid, + (SELECT fieldid FROM %fts_schema%.fields WHERE field = %input_value_string%) AS fieldid + FROM fts_new_docs AS fts_ii + )"; + + string token_ctes = R"( + WITH %new_docs_cte%, + tokenized AS ( + %union_fields_query% + ), + stemmed_stopped AS ( + SELECT stem(t.w, '%stemmer%') AS term, + t.docid AS docid, + t.fieldid AS fieldid + FROM tokenized AS t + WHERE t.w NOT NULL + AND t.w <> '' + AND t.w NOT IN (SELECT sw FROM %fts_schema%.stopwords) + ) + )"; + + string result = R"( + CREATE TRIGGER %trigger_00_docs% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + INSERT INTO %fts_schema%.docs (docid, name, len) + WITH %docs_new_docs_cte%, + tokenized AS ( + %union_fields_query% + ), + stemmed_stopped AS ( + SELECT stem(t.w, '%stemmer%') AS term, + t.docid AS docid, + t.fieldid AS fieldid + FROM tokenized AS t + WHERE t.w NOT NULL + AND t.w <> '' + AND t.w NOT IN (SELECT sw FROM %fts_schema%.stopwords) + ), + lengths AS ( + SELECT docid, count(term) AS len + FROM stemmed_stopped + GROUP BY docid + ) + SELECT nd.docid, + nd.name, + COALESCE(l.len, 0) AS len + FROM fts_new_docs AS nd + LEFT JOIN lengths AS l ON nd.docid = l.docid; + + CREATE TRIGGER %trigger_10_dict_insert% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + INSERT INTO %fts_schema%.dict (termid, term, df) + %token_ctes%, + new_terms AS ( + SELECT DISTINCT term + FROM stemmed_stopped + WHERE term NOT IN (SELECT term FROM %fts_schema%.dict) + ORDER BY term + ) + SELECT (SELECT COALESCE(max(termid) + 1, 0) FROM %fts_schema%.dict) + row_number() OVER () - 1 AS termid, + term, + 0 AS df + FROM new_terms; + + CREATE TRIGGER %trigger_20_terms_store% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + INSERT INTO %fts_schema%.%terms_storage_table% (termid, docid, fieldid) + %token_ctes% + SELECT d.termid, + ss.docid, + ss.fieldid + FROM stemmed_stopped AS ss + JOIN %fts_schema%.dict AS d ON ss.term = d.term; + + CREATE TRIGGER %trigger_30_dict_df% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.dict AS d + SET df = d.df + inserted_df_delta.df_delta + FROM ( + SELECT t.termid, + COUNT(DISTINCT t.docid) AS df_delta + FROM %fts_schema%.%terms_storage_table% AS t + JOIN %fts_schema%.docs AS docs ON t.docid = docs.docid + JOIN fts_new_rows AS new_rows ON docs.name = new_rows.%input_id% + GROUP BY t.termid + ) AS inserted_df_delta + WHERE d.termid = inserted_df_delta.termid; + + CREATE TRIGGER %trigger_40_stats% AFTER INSERT ON %input_table% + REFERENCING NEW TABLE AS fts_new_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.stats + SET num_docs = (SELECT COUNT(docid) FROM %fts_schema%.docs), + avgdl = (SELECT SUM(len) / COUNT(len) FROM %fts_schema%.docs); + )"; + // clang-format on + + vector input_value_selects; + vector tokenize_fields; + for (auto &input_value : input_values) { + input_value_selects.push_back(StringUtil::Format( + "fts_new_rows.%s AS %s", SQLIdentifier::ToString(input_value), + SQLIdentifier::ToString(input_value))); + auto query = StringUtil::Replace(tokenize_field_query, "%input_value%", + SQLIdentifier::ToString(input_value)); + query = StringUtil::Replace(query, "%input_value_string%", + SQLString::ToString(input_value)); + tokenize_fields.push_back(query); + } + + docs_new_docs_cte = StringUtil::Replace( + docs_new_docs_cte, "%input_value_select_list%", + StringUtil::Join(input_value_selects, ",\n ")); + token_new_docs_cte = StringUtil::Replace( + token_new_docs_cte, "%input_value_select_list%", + StringUtil::Join(input_value_selects, ",\n ")); + token_ctes = + StringUtil::Replace(token_ctes, "%new_docs_cte%", token_new_docs_cte); + token_ctes = + StringUtil::Replace(token_ctes, "%union_fields_query%", + StringUtil::Join(tokenize_fields, " UNION ALL ")); + result = + StringUtil::Replace(result, "%docs_new_docs_cte%", docs_new_docs_cte); + result = + StringUtil::Replace(result, "%union_fields_query%", + StringUtil::Join(tokenize_fields, " UNION ALL ")); + result = StringUtil::Replace(result, "%token_ctes%", token_ctes); + result = StringUtil::Replace(result, "%terms_storage_table%", + GetFTSTermsStorageTable()); + + auto trigger_names = GetFTSClusteredInsertTriggerNames(qname); + result = StringUtil::Replace(result, "%trigger_00_docs%", + SQLIdentifier::ToString(trigger_names[0])); + result = StringUtil::Replace(result, "%trigger_10_dict_insert%", + SQLIdentifier::ToString(trigger_names[1])); + result = StringUtil::Replace(result, "%trigger_20_terms_store%", + SQLIdentifier::ToString(trigger_names[2])); + result = StringUtil::Replace(result, "%trigger_30_dict_df%", + SQLIdentifier::ToString(trigger_names[3])); + result = StringUtil::Replace(result, "%trigger_40_stats%", + SQLIdentifier::ToString(trigger_names[4])); + result = StringUtil::Replace(result, "%input_table%", + GetQualifiedTableName(qname)); + result = StringUtil::Replace(result, "%input_id%", + SQLIdentifier::ToString(input_id)); + return result; +} + static string DeleteTriggerScript(const QualifiedName &qname, const string &input_id) { // clang-format off @@ -644,6 +858,76 @@ static string DeleteTriggerScript(const QualifiedName &qname, return result; } +static string ClusteredDeleteTriggerScript(const QualifiedName &qname, + const string &input_id) { + // clang-format off + string result = R"( + CREATE TRIGGER %trigger_00_dict_df% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.dict AS d + SET df = d.df - deleted_df_delta.df_delta + FROM ( + SELECT t.termid, + COUNT(DISTINCT t.docid) AS df_delta + FROM %fts_schema%.%terms_storage_table% AS t + JOIN %fts_schema%.docs AS docs ON t.docid = docs.docid + JOIN fts_old_rows AS old_rows ON docs.name = old_rows.%input_id% + GROUP BY t.termid + ) AS deleted_df_delta + WHERE d.termid = deleted_df_delta.termid; + + CREATE TRIGGER %trigger_10_terms_store% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + DELETE FROM %fts_schema%.%terms_storage_table% + WHERE docid IN ( + SELECT d.docid + FROM %fts_schema%.docs AS d + JOIN fts_old_rows AS old_rows ON d.name = old_rows.%input_id% + ); + + CREATE TRIGGER %trigger_20_docs% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + DELETE FROM %fts_schema%.docs + WHERE name IN (SELECT %input_id% FROM fts_old_rows); + + CREATE TRIGGER %trigger_30_dict_prune% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + DELETE FROM %fts_schema%.dict + WHERE df = 0; + + CREATE TRIGGER %trigger_40_stats% AFTER DELETE ON %input_table% + REFERENCING OLD TABLE AS fts_old_rows + FOR EACH STATEMENT + UPDATE %fts_schema%.stats + SET num_docs = (SELECT COUNT(docid) FROM %fts_schema%.docs), + avgdl = (SELECT SUM(len) / COUNT(len) FROM %fts_schema%.docs); + )"; + // clang-format on + + auto trigger_names = GetFTSClusteredDeleteTriggerNames(qname); + result = StringUtil::Replace(result, "%trigger_00_dict_df%", + SQLIdentifier::ToString(trigger_names[0])); + result = StringUtil::Replace(result, "%trigger_10_terms_store%", + SQLIdentifier::ToString(trigger_names[1])); + result = StringUtil::Replace(result, "%trigger_20_docs%", + SQLIdentifier::ToString(trigger_names[2])); + result = StringUtil::Replace(result, "%trigger_30_dict_prune%", + SQLIdentifier::ToString(trigger_names[3])); + result = StringUtil::Replace(result, "%trigger_40_stats%", + SQLIdentifier::ToString(trigger_names[4])); + result = StringUtil::Replace(result, "%terms_storage_table%", + GetFTSTermsStorageTable()); + result = StringUtil::Replace(result, "%input_table%", + GetQualifiedTableName(qname)); + result = StringUtil::Replace(result, "%input_id%", + SQLIdentifier::ToString(input_id)); + return result; +} + // --------------------------------------------------------------------------- // Coordinator: assembles all parts and substitutes cross-cutting placeholders // --------------------------------------------------------------------------- @@ -667,8 +951,14 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname, result += MatchMacroScript(); if (incremental) { result += IncrementalIndexSetupScript(qname); - result += InsertTriggerScript(qname, input_id, input_values); - result += DeleteTriggerScript(qname, input_id); + if (cluster_terms) { + result += ClusteredIncrementalIndexSetupScript(); + result += ClusteredInsertTriggerScript(qname, input_id, input_values); + result += ClusteredDeleteTriggerScript(qname, input_id); + } else { + result += InsertTriggerScript(qname, input_id, input_values); + result += DeleteTriggerScript(qname, input_id); + } } string fts_schema = GetFTSSchema(qname); @@ -801,11 +1091,6 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, throw InvalidInputException("incremental FTS indexes require the document " "id column to be NOT NULL"); } - if (incremental && cluster_terms) { - throw InvalidInputException("cluster_terms cannot be combined with " - "incremental FTS indexes"); - } - return IndexingScript(context, qname, doc_id, doc_values, stemmer, stopwords, ignore, strip_accents, lower, incremental, cluster_terms); diff --git a/test/sql/fts/test_incremental_insert_edge_cases.test b/test/sql/fts/test_incremental_insert_edge_cases.test index 43b0d0a..6677bb4 100644 --- a/test/sql/fts/test_incremental_insert_edge_cases.test +++ b/test/sql/fts/test_incremental_insert_edge_cases.test @@ -63,18 +63,37 @@ SELECT name, len FROM fts_main_primary_key_docs.docs ---- doc1 1 -# Clustered terms are a static physical layout; incremental inserts would append -# new terms after the clustered build and invalidate that ordering guarantee. +# Clustered incremental indexes keep the public terms relation ordered after +# writes. statement ok CREATE TABLE clustered_incremental_docs(id VARCHAR NOT NULL, body VARCHAR) statement ok INSERT INTO clustered_incremental_docs VALUES ('doc1', 'alpha beta') -statement error +statement ok PRAGMA create_fts_index('clustered_incremental_docs', 'id', 'body', stemmer='none', stopwords='none', incremental=true, cluster_terms=true) + +statement ok +INSERT INTO clustered_incremental_docs VALUES ('doc2', 'alpha aardvark') + +query I +SELECT count(*) +FROM ( + SELECT termid, fieldid, docid, + lag(termid) OVER () AS previous_termid, + lag(fieldid) OVER () AS previous_fieldid, + lag(docid) OVER () AS previous_docid + FROM fts_main_clustered_incremental_docs.terms +) ordered_terms +WHERE previous_termid IS NOT NULL + AND ( + termid < previous_termid + OR (termid = previous_termid AND fieldid < previous_fieldid) + OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) + ) ---- -cluster_terms cannot be combined with incremental FTS indexes +0 # Incremental indexes require unique document ids so DELETE triggers do not # remove multiple indexed documents for a single deleted source row. From cb7f456c87f07f36ef5dd68fe5640624f42d3539 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 30 Jun 2026 13:52:20 +0200 Subject: [PATCH 3/6] format fix --- src/fts_indexing.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index 99fef3a..e00a6d7 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -75,8 +75,8 @@ static vector GetFTSDeleteTriggerNames(const QualifiedName &qname) { static vector GetFTSClusteredDeleteTriggerNames(const QualifiedName &qname) { auto prefix = StringUtil::Format("__fts_%s_ad_", GetFTSSchemaName(qname)); - return {prefix + "00_dict_df", prefix + "10_terms_store", - prefix + "20_docs", prefix + "30_dict_prune", prefix + "40_stats"}; + return {prefix + "00_dict_df", prefix + "10_terms_store", prefix + "20_docs", + prefix + "30_dict_prune", prefix + "40_stats"}; } static vector GetFTSTriggerNames(const QualifiedName &qname) { @@ -432,8 +432,7 @@ static string ClusteredIncrementalIndexSetupScript() { "DROP TABLE %%fts_schema%%.terms;\n" "CREATE VIEW %%fts_schema%%.terms AS SELECT termid, docid, fieldid FROM " "%%fts_schema%%.%s ORDER BY termid, fieldid, docid;", - GetFTSTermsStorageTable(), - GetFTSTermsStorageTable()); + GetFTSTermsStorageTable(), GetFTSTermsStorageTable()); } static string InsertTriggerScript(const QualifiedName &qname, From 3fff0beae557daf35c0caf25934c0e34e6ae7800 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 30 Jun 2026 14:04:52 +0200 Subject: [PATCH 4/6] We don't want to always build json --- extension_config.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/extension_config.cmake b/extension_config.cmake index 2b39738..1c44a84 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -1,7 +1,5 @@ # This file is included by DuckDB's build system. It specifies which extension to load -duckdb_extension_load(json) - # Extension from this repo duckdb_extension_load(fts SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} From 567d034053c077d59772fe0cc1a7873f3f7ed14f Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 30 Jun 2026 14:47:28 +0200 Subject: [PATCH 5/6] Point duckdb to main --- duckdb | 2 +- src/fts_indexing.cpp | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/duckdb b/duckdb index 464b647..4ce2416 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 464b647e189de96096dc3afae1ed62c1224adc63 +Subproject commit 4ce2416bbf4996e191679c19316517ba03ce36f6 diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index e00a6d7..85ff609 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -19,10 +19,15 @@ namespace duckdb { static QualifiedName GetQualifiedName(ClientContext &context, const string &qname_str) { auto qname = QualifiedName::Parse(qname_str); - if (qname.Schema() == INVALID_SCHEMA) { - qname.SchemaMutable() = + if (qname.Schema().empty()) { + vector schema_path; + if (!qname.Catalog().empty()) { + schema_path.push_back(qname.Catalog()); + } + schema_path.push_back( ClientData::Get(context).catalog_search_path->GetDefaultSchema( - qname.Catalog()); + context, qname.Catalog())); + qname = qname.WithQualification(std::move(schema_path)); } return qname; } @@ -107,8 +112,7 @@ static string GetFTSTermsStorageTable() { static bool TableExists(ClientContext &context, const QualifiedName &qname) { return Catalog::GetEntry( - context, qname.Catalog(), qname.Schema(), qname.Name(), - OnEntryNotFound::RETURN_NULL) != nullptr; + context, qname, OnEntryNotFound::RETURN_NULL) != nullptr; } static bool SupportsFTSTriggers(ClientContext &context, @@ -1001,8 +1005,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, const FunctionParameters ¶meters) { auto qname = GetQualifiedName(context, StringValue::Get(parameters.values[0])); - Catalog::GetEntry(context, qname.Catalog(), qname.Schema(), - qname.Name()); + Catalog::GetEntry(context, qname); // Named parameters auto get_string = [&](const string &name, const string &def) { @@ -1030,8 +1033,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, if (stopwords != "english" && stopwords != "none") { auto sw_qname = GetQualifiedName(context, stopwords); - Catalog::GetEntry(context, sw_qname.Catalog(), - sw_qname.Schema(), sw_qname.Name()); + Catalog::GetEntry(context, sw_qname); } const string fts_schema = GetFTSSchema(qname); @@ -1049,8 +1051,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, // Positional parameters: table, id column, value column(s) const string doc_id = StringValue::Get(parameters.values[1]); - auto &table = Catalog::GetEntry( - context, qname.Catalog(), qname.Schema(), qname.Name()); + auto &table = Catalog::GetEntry(context, qname); if (!table.ColumnExists(Identifier(doc_id))) { throw CatalogException("Table '%s.%s' does not have a column named '%s'!", qname.Schema().GetIdentifierName(), From 3fa497cdb8bfa25cf773caf21fc2dbd30cbf06e6 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Thu, 16 Jul 2026 14:15:09 +0200 Subject: [PATCH 6/6] Remove the ORDER BY --- README.md | 5 +- src/fts_indexing.cpp | 2 +- .../fts/test_incremental_clustered_terms.test | 152 +++++------------- .../test_incremental_insert_edge_cases.test | 28 ++-- 4 files changed, 58 insertions(+), 129 deletions(-) diff --git a/README.md b/README.md index d5ffac4..7cbc574 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,8 @@ document id values. Persistent databases must use storage version `v2.0.0` or newer for incremental indexes. `cluster_terms = true` changes only the physical ordering of the generated -`terms` table. It cannot be combined with `incremental = true` unless -`layered_search = true`, because incremental inserts do not preserve the static -clustered layout. +`terms` table. For incremental indexes, the initial index build uses the +clustered layout, but trigger-appended rows are not globally reclustered. ### `PRAGMA drop_fts_index` diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index 6f77014..ee24146 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -880,7 +880,7 @@ static string ClusteredIncrementalIndexSetupScript() { "%%fts_schema%%.terms;\n" "DROP TABLE %%fts_schema%%.terms;\n" "CREATE VIEW %%fts_schema%%.terms AS SELECT termid, docid, fieldid FROM " - "%%fts_schema%%.%s ORDER BY termid, fieldid, docid;", + "%%fts_schema%%.%s;", GetFTSTermsStorageTable(), GetFTSTermsStorageTable()); } diff --git a/test/sql/fts/test_incremental_clustered_terms.test b/test/sql/fts/test_incremental_clustered_terms.test index 10ce5a2..d51e291 100644 --- a/test/sql/fts/test_incremental_clustered_terms.test +++ b/test/sql/fts/test_incremental_clustered_terms.test @@ -1,5 +1,5 @@ # name: test/sql/fts/test_incremental_clustered_terms.test -# description: Test clustered terms ordering with incremental FTS updates +# description: Test clustered terms maintenance with incremental FTS updates # group: [fts] require fts @@ -20,46 +20,27 @@ INSERT INTO clustered_docs VALUES statement ok PRAGMA create_fts_index('clustered_docs', 'id', 'body', stemmer='none', stopwords='none', incremental=true, cluster_terms=true) -query I -SELECT count(*) -FROM ( - SELECT termid, fieldid, docid, - lag(termid) OVER () AS previous_termid, - lag(fieldid) OVER () AS previous_fieldid, - lag(docid) OVER () AS previous_docid - FROM fts_main_clustered_docs.terms -) ordered_terms -WHERE previous_termid IS NOT NULL - AND ( - termid < previous_termid - OR (termid = previous_termid AND fieldid < previous_fieldid) - OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) - ) ----- -0 - statement ok INSERT INTO clustered_docs VALUES ('doc4', 'alpha aardvark aardvark'), ('doc5', 'gamma beta beta') -query I -SELECT count(*) -FROM ( - SELECT termid, fieldid, docid, - lag(termid) OVER () AS previous_termid, - lag(fieldid) OVER () AS previous_fieldid, - lag(docid) OVER () AS previous_docid - FROM fts_main_clustered_docs.terms -) ordered_terms -WHERE previous_termid IS NOT NULL - AND ( - termid < previous_termid - OR (termid = previous_termid AND fieldid < previous_fieldid) - OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) - ) +query III rowsort +SELECT docs.name, dict.term, count(*) AS tf +FROM fts_main_clustered_docs.terms AS terms +JOIN fts_main_clustered_docs.docs AS docs ON docs.docid = terms.docid +JOIN fts_main_clustered_docs.dict AS dict ON dict.termid = terms.termid +GROUP BY docs.name, dict.term ---- -0 +doc1 alpha 2 +doc1 beta 1 +doc2 beta 1 +doc2 gamma 1 +doc3 delta 1 +doc4 aardvark 2 +doc4 alpha 1 +doc5 beta 2 +doc5 gamma 1 query II SELECT term, df FROM fts_main_clustered_docs.dict ORDER BY term @@ -91,23 +72,19 @@ SELECT count(*) FROM diffs statement ok DELETE FROM clustered_docs WHERE id IN ('doc3', 'doc4') -query I -SELECT count(*) -FROM ( - SELECT termid, fieldid, docid, - lag(termid) OVER () AS previous_termid, - lag(fieldid) OVER () AS previous_fieldid, - lag(docid) OVER () AS previous_docid - FROM fts_main_clustered_docs.terms -) ordered_terms -WHERE previous_termid IS NOT NULL - AND ( - termid < previous_termid - OR (termid = previous_termid AND fieldid < previous_fieldid) - OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) - ) +query III rowsort +SELECT docs.name, dict.term, count(*) AS tf +FROM fts_main_clustered_docs.terms AS terms +JOIN fts_main_clustered_docs.docs AS docs ON docs.docid = terms.docid +JOIN fts_main_clustered_docs.dict AS dict ON dict.termid = terms.termid +GROUP BY docs.name, dict.term ---- -0 +doc1 alpha 2 +doc1 beta 1 +doc2 beta 1 +doc2 gamma 1 +doc5 beta 2 +doc5 gamma 1 query II SELECT term, df FROM fts_main_clustered_docs.dict ORDER BY term @@ -152,23 +129,19 @@ beta 2 gamma 2 zebra 1 -query I -SELECT count(*) -FROM ( - SELECT termid, fieldid, docid, - lag(termid) OVER () AS previous_termid, - lag(fieldid) OVER () AS previous_fieldid, - lag(docid) OVER () AS previous_docid - FROM fts_main_clustered_docs.terms -) ordered_terms -WHERE previous_termid IS NOT NULL - AND ( - termid < previous_termid - OR (termid = previous_termid AND fieldid < previous_fieldid) - OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) - ) +query III rowsort +SELECT docs.name, dict.term, count(*) AS tf +FROM fts_main_clustered_docs.terms AS terms +JOIN fts_main_clustered_docs.docs AS docs ON docs.docid = terms.docid +JOIN fts_main_clustered_docs.dict AS dict ON dict.termid = terms.termid +GROUP BY docs.name, dict.term ---- -0 +doc1 alpha 2 +doc1 beta 1 +doc2 gamma 1 +doc2 zebra 2 +doc5 beta 2 +doc5 gamma 1 query I rowsort SELECT id FROM ( @@ -188,9 +161,9 @@ WHERE score IS NOT NULL ---- 0 -# Clustered incremental indexes expose `terms` as an ordered view backed by an -# internal storage table. Non-incremental clustered indexes keep `terms` as a -# physical table. +# Clustered incremental indexes expose `terms` as a view backed by an internal +# storage table. Non-incremental clustered indexes keep `terms` as a physical +# table. statement ok CREATE TABLE clustered_static_docs(id VARCHAR, body VARCHAR) @@ -311,26 +284,7 @@ SELECT statement ok INSERT INTO overwrite_clustered_docs VALUES ('doc2', 'aardvark alpha') -query I -SELECT count(*) -FROM ( - SELECT termid, fieldid, docid, - lag(termid) OVER () AS previous_termid, - lag(fieldid) OVER () AS previous_fieldid, - lag(docid) OVER () AS previous_docid - FROM fts_main_overwrite_clustered_docs.terms -) ordered_terms -WHERE previous_termid IS NOT NULL - AND ( - termid < previous_termid - OR (termid = previous_termid AND fieldid < previous_fieldid) - OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) - ) ----- -0 - -# Multiple indexed columns should preserve field ids and order by term, field, -# then document. +# Multiple indexed columns should preserve field ids. statement ok CREATE TABLE multi_field_docs(id VARCHAR NOT NULL, title VARCHAR, body VARCHAR) @@ -365,24 +319,6 @@ body gamma 1 title alpha 2 title beta 1 -query I -SELECT count(*) -FROM ( - SELECT termid, fieldid, docid, - lag(termid) OVER () AS previous_termid, - lag(fieldid) OVER () AS previous_fieldid, - lag(docid) OVER () AS previous_docid - FROM fts_main_multi_field_docs.terms -) ordered_terms -WHERE previous_termid IS NOT NULL - AND ( - termid < previous_termid - OR (termid = previous_termid AND fieldid < previous_fieldid) - OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) - ) ----- -0 - query I rowsort SELECT id FROM ( SELECT *, fts_main_multi_field_docs.match_bm25(id, 'aardvark', fields := 'body') AS score diff --git a/test/sql/fts/test_incremental_insert_edge_cases.test b/test/sql/fts/test_incremental_insert_edge_cases.test index 6677bb4..8170770 100644 --- a/test/sql/fts/test_incremental_insert_edge_cases.test +++ b/test/sql/fts/test_incremental_insert_edge_cases.test @@ -63,7 +63,7 @@ SELECT name, len FROM fts_main_primary_key_docs.docs ---- doc1 1 -# Clustered incremental indexes keep the public terms relation ordered after +# Clustered incremental indexes keep the public terms relation updated after # writes. statement ok CREATE TABLE clustered_incremental_docs(id VARCHAR NOT NULL, body VARCHAR) @@ -77,23 +77,17 @@ PRAGMA create_fts_index('clustered_incremental_docs', 'id', 'body', stemmer='non statement ok INSERT INTO clustered_incremental_docs VALUES ('doc2', 'alpha aardvark') -query I -SELECT count(*) -FROM ( - SELECT termid, fieldid, docid, - lag(termid) OVER () AS previous_termid, - lag(fieldid) OVER () AS previous_fieldid, - lag(docid) OVER () AS previous_docid - FROM fts_main_clustered_incremental_docs.terms -) ordered_terms -WHERE previous_termid IS NOT NULL - AND ( - termid < previous_termid - OR (termid = previous_termid AND fieldid < previous_fieldid) - OR (termid = previous_termid AND fieldid = previous_fieldid AND docid < previous_docid) - ) +query III rowsort +SELECT docs.name, dict.term, count(*) AS tf +FROM fts_main_clustered_incremental_docs.terms AS terms +JOIN fts_main_clustered_incremental_docs.docs AS docs ON docs.docid = terms.docid +JOIN fts_main_clustered_incremental_docs.dict AS dict ON dict.termid = terms.termid +GROUP BY docs.name, dict.term ---- -0 +doc1 alpha 1 +doc1 beta 1 +doc2 aardvark 1 +doc2 alpha 1 # Incremental indexes require unique document ids so DELETE triggers do not # remove multiple indexed documents for a single deleted source row.