diff --git a/CMakeLists.txt b/CMakeLists.txt index d3a3beb..8c8beb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,11 +7,20 @@ project(FTSExtension) set(EXTENSION_BASE_FOLDER "src") set(SNOWBALL_BASE_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/third_party/snowball") +if(DEFINED DUCKDB_MODULE_BASE_DIR) + set(DUCKDB_SOURCE_BASE_FOLDER "${DUCKDB_MODULE_BASE_DIR}") +else() + set(DUCKDB_SOURCE_BASE_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/duckdb") +endif() +set(DUCKDB_ICU_BASE_FOLDER "${DUCKDB_SOURCE_BASE_FOLDER}/extension/icu") include_directories( ${EXTENSION_BASE_FOLDER}/include ${SNOWBALL_BASE_FOLDER}/include ${SNOWBALL_BASE_FOLDER}/libstemmer - ${SNOWBALL_BASE_FOLDER}/runtime) + ${SNOWBALL_BASE_FOLDER}/runtime + ${DUCKDB_SOURCE_BASE_FOLDER}/third_party/utf8proc/include + ${DUCKDB_ICU_BASE_FOLDER}/third_party/icu/common + ${DUCKDB_ICU_BASE_FOLDER}/third_party/icu/i18n) file(GLOB SNOWBALL_STEMMER_SOURCES CONFIGURE_DEPENDS ${SNOWBALL_BASE_FOLDER}/src_c/stem_*.c) @@ -27,6 +36,19 @@ build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) set(PARAMETERS "-warnings") build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES}) +if(TARGET duckdb_icu_common) + target_sources(${EXTENSION_NAME} PRIVATE $) + target_sources(${LOADABLE_EXTENSION_NAME} PRIVATE $) +endif() +if(TARGET duckdb_icu_common_unity) + target_sources(${EXTENSION_NAME} PRIVATE $) + target_sources(${LOADABLE_EXTENSION_NAME} PRIVATE $) +endif() +if(TARGET duckdb_icu_stubdata) + target_sources(${EXTENSION_NAME} PRIVATE $) + target_sources(${LOADABLE_EXTENSION_NAME} PRIVATE $) +endif() + install( TARGETS ${EXTENSION_NAME} EXPORT "${DUCKDB_EXPORT_SET}" diff --git a/README.md b/README.md index 08e6718..7e46d79 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ The extension adds two `PRAGMA` statements to DuckDB: one to create, and one to ```python create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', + tokenizer = 'regex', stopwords = 'english', ignore = "[0-9!@#$%^&*()_+={}\\[\\]:;<>,.?~\\\\/\\|''\"`-]+", strip_accents = 1, lower = 1, overwrite = 0, @@ -36,8 +37,9 @@ create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', | `input_id` | `VARCHAR` | Column name of document identifier, e.g., `'document_identifier'` | | `input_values…` | `VARCHAR` | Column names of the text fields to be indexed (vararg), e.g., `'text_field_1'`, `'text_field_2'`, ..., `'text_field_N'`, or `'\*'` for all columns in input_table of type `VARCHAR` | | `stemmer` | `VARCHAR` | The type of stemmer to be used. One of `'arabic'`, `'armenian'`, `'basque'`, `'catalan'`, `'czech'`, `'danish'`, `'dutch'`, `'dutch_porter'`, `'english'`, `'esperanto'`, `'estonian'`, `'finnish'`, `'french'`, `'german'`, `'greek'`, `'hindi'`, `'hungarian'`, `'indonesian'`, `'irish'`, `'italian'`, `'lithuanian'`, `'nepali'`, `'norwegian'`, `'persian'`, `'polish'`, `'porter'`, `'portuguese'`, `'romanian'`, `'russian'`, `'serbian'`, `'sesotho'`, `'spanish'`, `'swedish'`, `'tamil'`, `'turkish'`, `'yiddish'`, or `'none'` if no stemming is to be used. Defaults to `'porter'` | +| `tokenizer` | `VARCHAR` | Tokenizer to use. `'regex'` keeps the legacy regex-split behavior. `'opensearch_standard'` uses an OpenSearch/Lucene standard-tokenizer compatibility mode that splits Han, Hiragana, and Katakana into single-character tokens while preserving word runs for scripts such as Hebrew, Cyrillic, Arabic, Latin, and Hangul. Defaults to `'regex'` | | `stopwords` | `VARCHAR` | Qualified name of table containing a single `VARCHAR` column containing the desired stopwords, or `'none'` if no stopwords are to be used. Defaults to `'english'` for a pre-defined list of 571 English stopwords | -| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored. Defaults to a punctuation and digit pattern | +| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored by the `'regex'` tokenizer. Defaults to a punctuation and digit pattern | | `strip_accents` | `BOOLEAN` | Whether to remove accents (e.g., convert `á` to `a`). Defaults to `1` | | `lower` | `BOOLEAN` | Whether to convert all text to lowercase. Defaults to `1` | | `overwrite` | `BOOLEAN` | Whether to overwrite an existing index on a table. Defaults to `0` | diff --git a/extension_config.cmake b/extension_config.cmake index 7f3e03f..5aeb91c 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -4,6 +4,8 @@ if (LINK_FTS_STATICALLY) set(FTS_DONT_LINK "") endif() +duckdb_extension_load(icu) + # Extension from this repo duckdb_extension_load(fts SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} diff --git a/src/fts_config.py b/src/fts_config.py index bf59eae..c64d7f8 100644 --- a/src/fts_config.py +++ b/src/fts_config.py @@ -10,6 +10,9 @@ 'third_party/snowball/libstemmer', 'third_party/snowball/runtime', 'third_party/snowball/src_c', + 'third_party/utf8proc/include', + 'extension/icu/third_party/icu/common', + 'extension/icu/third_party/icu/i18n', ] ] # source files diff --git a/src/fts_extension.cpp b/src/fts_extension.cpp index 41ff9e5..5389788 100644 --- a/src/fts_extension.cpp +++ b/src/fts_extension.cpp @@ -3,14 +3,153 @@ #include "duckdb.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/common/vector/string_vector.hpp" +#include "duckdb/common/vector/vector_writer.hpp" #include "duckdb/function/pragma_function.hpp" #include "duckdb/function/scalar_function.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include "fts_indexing.hpp" #include "libstemmer.h" +#include "unicode/uchar.h" +#include "unicode/uscript.h" +#include "utf8proc_wrapper.hpp" namespace duckdb { +static bool IsOpenSearchStandardSingleTokenScript(UScriptCode script) { + return script == USCRIPT_HAN || script == USCRIPT_HIRAGANA; +} + +static bool IsOpenSearchStandardEmoji(UChar32 codepoint) { + return u_hasBinaryProperty(codepoint, UCHAR_EMOJI_PRESENTATION) || + u_hasBinaryProperty(codepoint, UCHAR_EXTENDED_PICTOGRAPHIC); +} + +static bool IsOpenSearchStandardIntraTokenPunctuation(UChar32 codepoint) { + return codepoint == '\'' || codepoint == 0x2019 || codepoint == '.' || + codepoint == '_'; +} + +static bool IsJapaneseProlongedSoundMark(UChar32 codepoint) { + return codepoint == 0x30FC || codepoint == 0xFF70; +} + +static bool IsOpenSearchStandardTokenChar(UChar32 codepoint, + UScriptCode script) { + if (u_isUWhiteSpace(codepoint) || u_ispunct(codepoint)) { + return false; + } + return u_isalnum(codepoint) || + u_hasBinaryProperty(codepoint, UCHAR_ALPHABETIC) || + script == USCRIPT_KATAKANA || IsJapaneseProlongedSoundMark(codepoint); +} + +static bool IsOpenSearchStandardContinuationChar(UChar32 codepoint, + UScriptCode script) { + return !IsOpenSearchStandardSingleTokenScript(script) && + !IsOpenSearchStandardEmoji(codepoint) && + IsOpenSearchStandardTokenChar(codepoint, script); +} + +template +static void TokenizeOpenSearchStandard(string_t input, LIST_WRITER &list) { + auto input_data = input.GetData(); + auto input_size = input.GetSize(); + idx_t token_start = 0; + idx_t token_size = 0; + + auto flush_token = [&]() { + if (token_size == 0) { + return; + } + list.WriteElement().WriteStringRef(string_t( + input_data + token_start, UnsafeNumericCast(token_size))); + token_size = 0; + }; + + auto decode_codepoint = [&](idx_t pos, UChar32 &codepoint, int &char_size, + UScriptCode &script) -> bool { + if (pos >= input_size) { + return false; + } + char_size = 0; + codepoint = Utf8Proc::UTF8ToCodepoint(input_data + pos, char_size, + input_size - pos); + if (char_size <= 0) { + return false; + } + UErrorCode status = U_ZERO_ERROR; + script = uscript_getScript(codepoint, &status); + return U_SUCCESS(status); + }; + + for (idx_t pos = 0; pos < input_size;) { + int char_size = 0; + UChar32 codepoint = 0; + UScriptCode script = USCRIPT_UNKNOWN; + if (!decode_codepoint(pos, codepoint, char_size, script)) { + flush_token(); + pos++; + continue; + } + + if (IsOpenSearchStandardSingleTokenScript(script) || + IsOpenSearchStandardEmoji(codepoint)) { + flush_token(); + list.WriteElement().WriteStringRef( + string_t(input_data + pos, UnsafeNumericCast(char_size))); + } else if (IsOpenSearchStandardIntraTokenPunctuation(codepoint) && + token_size > 0) { + UChar32 next_codepoint = 0; + int next_char_size = 0; + UScriptCode next_script = USCRIPT_UNKNOWN; + auto next_pos = pos + UnsafeNumericCast(char_size); + if (decode_codepoint(next_pos, next_codepoint, next_char_size, + next_script) && + IsOpenSearchStandardContinuationChar(next_codepoint, next_script)) { + token_size += UnsafeNumericCast(char_size); + } else { + flush_token(); + } + } else if (IsOpenSearchStandardTokenChar(codepoint, script)) { + if (token_size == 0) { + token_start = pos; + } + token_size += UnsafeNumericCast(char_size); + } else { + flush_token(); + } + pos += UnsafeNumericCast(char_size); + } + flush_token(); +} + +static void OpenSearchStandardTokenizeFunction(DataChunk &args, + ExpressionState &state, + Vector &result) { + auto input_entries = args.data[0].Values(); + + D_ASSERT(result.GetType().id() == LogicalTypeId::LIST); + result.SetVectorType(VectorType::FLAT_VECTOR); + + auto list_writer = + FlatVector::Writer>(result, args.size()); + for (idx_t i = 0; i < args.size(); i++) { + auto input_entry = input_entries[i]; + if (!input_entry.IsValid()) { + list_writer.WriteNull(); + continue; + } + auto list = list_writer.WriteDynamicList(); + TokenizeOpenSearchStandard(input_entry.GetValue(), list); + } + + StringVector::AddHeapReference(ListVector::GetChildMutable(result), + args.data[0]); +} + static void StemFunction(DataChunk &args, ExpressionState &state, Vector &result) { auto &input_vector = args.data[0]; @@ -57,11 +196,16 @@ static void LoadInternal(ExtensionLoader &loader) { ScalarFunction stem_func("stem", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, StemFunction); + ScalarFunction opensearch_standard_tokenize_func( + "fts_tokenize_opensearch_standard", {LogicalType::VARCHAR}, + LogicalType::LIST(LogicalType::VARCHAR), + OpenSearchStandardTokenizeFunction); auto create_fts_index_func = PragmaFunction::PragmaCall( "create_fts_index", FTSIndexing::CreateFTSIndexQuery, {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR); create_fts_index_func.named_parameters["stemmer"] = LogicalType::VARCHAR; + create_fts_index_func.named_parameters["tokenizer"] = LogicalType::VARCHAR; create_fts_index_func.named_parameters["stopwords"] = LogicalType::VARCHAR; create_fts_index_func.named_parameters["ignore"] = LogicalType::VARCHAR; create_fts_index_func.named_parameters["strip_accents"] = @@ -78,6 +222,7 @@ static void LoadInternal(ExtensionLoader &loader) { "drop_fts_index", FTSIndexing::DropFTSIndexQuery, {LogicalType::VARCHAR}); loader.RegisterFunction(stem_func); + loader.RegisterFunction(opensearch_standard_tokenize_func); loader.RegisterFunction(create_fts_index_func); loader.RegisterFunction(drop_fts_index_func); } diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index 37c28c5..5329cde 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -231,8 +231,7 @@ static string StopwordsScript(const string &stopwords) { return "INSERT INTO %fts_schema%.stopwords SELECT * FROM " + stopwords + ";"; } -static string TokenizeMacroScript(const string &ignore, bool strip_accents, - bool lower) { +static string NormalizeTokenInputExpression(bool strip_accents, bool lower) { string expr = "s::VARCHAR"; if (strip_accents) { expr = "strip_accents(" + expr + ")"; @@ -240,6 +239,16 @@ static string TokenizeMacroScript(const string &ignore, bool strip_accents, if (lower) { expr = "lower(" + expr + ")"; } + return expr; +} + +static string TokenizeMacroScript(const string &tokenizer, const string &ignore, + bool strip_accents, bool lower) { + auto expr = NormalizeTokenInputExpression(strip_accents, lower); + if (tokenizer == "opensearch_standard") { + expr = "fts_tokenize_opensearch_standard(" + expr + ")"; + return "CREATE MACRO %fts_schema%.tokenize(s) AS " + expr + ";"; + } auto delimiter = "(" + ignore + ")|\\s+"; expr = "string_split_regex(" + expr + ", $$" + delimiter + "$$)"; return "CREATE MACRO %fts_schema%.tokenize(s) AS " + expr + ";"; @@ -2053,16 +2062,16 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname, const string &input_id, const vector &input_values, const string &stemmer, const string &stopwords, - const string &ignore, bool strip_accents, - bool lower, bool incremental, bool cluster_terms, - bool layered_search) { + const string &tokenizer, const string &ignore, + bool strip_accents, bool lower, bool incremental, + bool cluster_terms, bool layered_search) { string result; if (TableExists(context, qname) && SupportsFTSTriggers(context, qname)) { result += DropFTSTriggersScript(qname); } result += SchemaSetupScript(); result += StopwordsScript(stopwords); - result += TokenizeMacroScript(ignore, strip_accents, lower); + result += TokenizeMacroScript(tokenizer, ignore, strip_accents, lower); result += IndexTablesScript( input_id, input_values, stemmer, stopwords, GetFTSBuildTermsTable(qname), GetFTSBuildDictTable(qname), GetFTSBuildRawDictTable(qname), @@ -2149,6 +2158,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, }; const string stemmer = get_string("stemmer", "porter"); + const string tokenizer = get_string("tokenizer", "regex"); const string stopwords = get_string("stopwords", "english"); const string ignore = get_string("ignore", "[0-9!@#$%^&*()_+={}\\[\\]:;<>,.?~\\\\/\\|''\"`-]+"); @@ -2159,6 +2169,13 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, const bool layered_search = get_bool("layered_search", false); const bool cluster_terms = get_bool("cluster_terms", false) || layered_search; + if (tokenizer != "regex" && tokenizer != "opensearch_standard") { + throw InvalidInputException( + "Unrecognized tokenizer '%s'. Supported tokenizers are: ['regex', " + "'opensearch_standard']", + tokenizer); + } + if (stopwords != "english" && stopwords != "none") { auto sw_qname = GetQualifiedName(context, stopwords); Catalog::GetEntry(context, sw_qname); @@ -2220,7 +2237,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, "id column to be NOT NULL"); } return IndexingScript(context, qname, doc_id, doc_values, stemmer, stopwords, - ignore, strip_accents, lower, incremental, + tokenizer, ignore, strip_accents, lower, incremental, cluster_terms, layered_search); } diff --git a/test/sql/fts/test_opensearch_standard_tokenizer.test b/test/sql/fts/test_opensearch_standard_tokenizer.test new file mode 100644 index 0000000..9ccce44 --- /dev/null +++ b/test/sql/fts/test_opensearch_standard_tokenizer.test @@ -0,0 +1,170 @@ +# name: test/sql/fts/test_opensearch_standard_tokenizer.test +# description: Test OpenSearch standard-tokenizer parity for multilingual FTS tokenization. +# group: [fts] + +require fts + +require no_alternative_verify + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower(strip_accents( + 'תל-אביב ביב Привет-мир مرحبا-بالعالم 東京大学に行く 北京大学 한국어 테스트 café' + )))) AS token +) +---- +תל +אביב +ביב +привет +мир +مرحبا +بالعالم +東 +京 +大 +学 +に +行 +く +北 +京 +大 +学 +한국어 +테스트 +cafe + +statement ok +CREATE TABLE unicode_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO unicode_docs VALUES ('doc1', 'תל-אביב ביב Привет-мир مرحبا-بالعالم 東京大学に行く 北京大学 한국어 테스트 café') + +statement ok +PRAGMA create_fts_index('unicode_docs', 'id', 'body', tokenizer='opensearch_standard', stemmer='none', stopwords='none') + +query I +SELECT count(*) +FROM fts_main_unicode_docs.dict +WHERE term IN ('東京大学に行く', '北京大学') +---- +0 + +query I +SELECT count(*) +FROM fts_main_unicode_docs.dict +WHERE term IN ('東', '京', '大', '学', 'に', '行', 'く', '北', '한국어', '테스트') +---- +10 + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + 'John''s co-op email@example.com foo_bar snake_case C++ C# node.js' + ))) AS token +) +---- +john's +co +op +email +example.com +foo_bar +snake_case +c +c +node.js + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard( + 'It’s l’amour rock’n’roll John''s' + )) AS token +) +---- +It’s +l’amour +rock’n’roll +John's + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + 'abc123 123abc 42 3.14 version-2 test_99 東京2024' + ))) AS token +) +---- +abc123 +123abc +42 +3.14 +version +2 +test_99 +東 +京 +2024 + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard( + 'ひらがな カタカナ ｶﾀｶﾅ ー タワー オープンサーチ' + )) AS token +) +---- +ひ +ら +が +な +カタカナ +ｶﾀｶﾅ +ー +タワー +オープンサーチ + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + 'hello🙂world #hashtag @user path/to/file 中文🙂test' + ))) AS token +) +---- +hello +🙂 +world +hashtag +user +path +to +file +中 +文 +🙂 +test + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + '한국어 테스트 서울대학교 K-pop 한글-테스트' + ))) AS token +) +---- +한국어 +테스트 +서울대학교 +k +pop +한글 +테스트 + +statement error +PRAGMA create_fts_index('unicode_docs', 'id', 'body', tokenizer='unknown', overwrite=true) +----