From b2c80029e775e2fb4b348743eb384df0597e6a98 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Thu, 9 Jul 2026 11:12:33 +0200 Subject: [PATCH 1/5] First implementation of UDF for tokenization with ICU --- CMakeLists.txt | 19 +++- README.md | 4 +- extension_config.cmake | 2 + src/fts_config.py | 3 + src/fts_extension.cpp | 97 +++++++++++++++++++ src/fts_indexing.cpp | 31 ++++-- .../test_opensearch_standard_tokenizer.test | 64 ++++++++++++ 7 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 test/sql/fts/test_opensearch_standard_tokenizer.test diff --git a/CMakeLists.txt b/CMakeLists.txt index d3a3beb..535c153 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,11 +7,15 @@ project(FTSExtension) set(EXTENSION_BASE_FOLDER "src") set(SNOWBALL_BASE_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/third_party/snowball") +set(DUCKDB_ICU_BASE_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/duckdb/extension/icu") include_directories( ${EXTENSION_BASE_FOLDER}/include ${SNOWBALL_BASE_FOLDER}/include ${SNOWBALL_BASE_FOLDER}/libstemmer - ${SNOWBALL_BASE_FOLDER}/runtime) + ${SNOWBALL_BASE_FOLDER}/runtime + ${CMAKE_CURRENT_SOURCE_DIR}/duckdb/third_party/utf8proc/include + ${DUCKDB_ICU_BASE_FOLDER}/third_party/icu/common + ${DUCKDB_ICU_BASE_FOLDER}/third_party/icu/i18n) file(GLOB SNOWBALL_STEMMER_SOURCES CONFIGURE_DEPENDS ${SNOWBALL_BASE_FOLDER}/src_c/stem_*.c) @@ -27,6 +31,19 @@ build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) set(PARAMETERS "-warnings") build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES}) +if(TARGET duckdb_icu_common) + target_sources(${EXTENSION_NAME} PRIVATE $) + target_sources(${LOADABLE_EXTENSION_NAME} PRIVATE $) +endif() +if(TARGET duckdb_icu_common_unity) + target_sources(${EXTENSION_NAME} PRIVATE $) + target_sources(${LOADABLE_EXTENSION_NAME} PRIVATE $) +endif() +if(TARGET duckdb_icu_stubdata) + target_sources(${EXTENSION_NAME} PRIVATE $) + target_sources(${LOADABLE_EXTENSION_NAME} PRIVATE $) +endif() + install( TARGETS ${EXTENSION_NAME} EXPORT "${DUCKDB_EXPORT_SET}" diff --git a/README.md b/README.md index 3f2a674..9dccdae 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ The extension adds two `PRAGMA` statements to DuckDB: one to create, and one to ```python create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', + tokenizer = 'regex', stopwords = 'english', ignore = "[0-9!@#$%^&*()_+={}\\[\\]:;<>,.?~\\\\/\\|''\"`-]+", strip_accents = 1, lower = 1, overwrite = 0, @@ -36,8 +37,9 @@ create_fts_index(input_table, input_id, *input_values, stemmer = 'porter', | `input_id` | `VARCHAR` | Column name of document identifier, e.g., `'document_identifier'` | | `input_values…` | `VARCHAR` | Column names of the text fields to be indexed (vararg), e.g., `'text_field_1'`, `'text_field_2'`, ..., `'text_field_N'`, or `'\*'` for all columns in input_table of type `VARCHAR` | | `stemmer` | `VARCHAR` | The type of stemmer to be used. One of `'arabic'`, `'armenian'`, `'basque'`, `'catalan'`, `'czech'`, `'danish'`, `'dutch'`, `'dutch_porter'`, `'english'`, `'esperanto'`, `'estonian'`, `'finnish'`, `'french'`, `'german'`, `'greek'`, `'hindi'`, `'hungarian'`, `'indonesian'`, `'irish'`, `'italian'`, `'lithuanian'`, `'nepali'`, `'norwegian'`, `'persian'`, `'polish'`, `'porter'`, `'portuguese'`, `'romanian'`, `'russian'`, `'serbian'`, `'sesotho'`, `'spanish'`, `'swedish'`, `'tamil'`, `'turkish'`, `'yiddish'`, or `'none'` if no stemming is to be used. Defaults to `'porter'` | +| `tokenizer` | `VARCHAR` | Tokenizer to use. `'regex'` keeps the legacy regex-split behavior. `'opensearch_standard'` uses an OpenSearch/Lucene standard-tokenizer compatibility mode that splits Han, Hiragana, and Katakana into single-character tokens while preserving word runs for scripts such as Hebrew, Cyrillic, Arabic, Latin, and Hangul. Defaults to `'regex'` | | `stopwords` | `VARCHAR` | Qualified name of table containing a single `VARCHAR` column containing the desired stopwords, or `'none'` if no stopwords are to be used. Defaults to `'english'` for a pre-defined list of 571 English stopwords | -| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored. Defaults to a punctuation and digit pattern | +| `ignore` | `VARCHAR` | Regular expression of patterns to be ignored by the `'regex'` tokenizer. Defaults to a punctuation and digit pattern | | `strip_accents` | `BOOLEAN` | Whether to remove accents (e.g., convert `á` to `a`). Defaults to `1` | | `lower` | `BOOLEAN` | Whether to convert all text to lowercase. Defaults to `1` | | `overwrite` | `BOOLEAN` | Whether to overwrite an existing index on a table. Defaults to `0` | diff --git a/extension_config.cmake b/extension_config.cmake index 7f3e03f..5aeb91c 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -4,6 +4,8 @@ if (LINK_FTS_STATICALLY) set(FTS_DONT_LINK "") endif() +duckdb_extension_load(icu) + # Extension from this repo duckdb_extension_load(fts SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} diff --git a/src/fts_config.py b/src/fts_config.py index bf59eae..c64d7f8 100644 --- a/src/fts_config.py +++ b/src/fts_config.py @@ -10,6 +10,9 @@ 'third_party/snowball/libstemmer', 'third_party/snowball/runtime', 'third_party/snowball/src_c', + 'third_party/utf8proc/include', + 'extension/icu/third_party/icu/common', + 'extension/icu/third_party/icu/i18n', ] ] # source files diff --git a/src/fts_extension.cpp b/src/fts_extension.cpp index 41ff9e5..643a039 100644 --- a/src/fts_extension.cpp +++ b/src/fts_extension.cpp @@ -3,14 +3,105 @@ #include "duckdb.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/common/vector/string_vector.hpp" +#include "duckdb/common/vector/vector_writer.hpp" #include "duckdb/function/pragma_function.hpp" #include "duckdb/function/scalar_function.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include "fts_indexing.hpp" #include "libstemmer.h" +#include "unicode/uchar.h" +#include "unicode/uscript.h" +#include "utf8proc_wrapper.hpp" namespace duckdb { +static bool IsOpenSearchStandardSingleTokenScript(UScriptCode script) { + return script == USCRIPT_HAN || script == USCRIPT_HIRAGANA || + script == USCRIPT_KATAKANA; +} + +static bool IsOpenSearchStandardTokenChar(UChar32 codepoint) { + if (u_isUWhiteSpace(codepoint) || u_ispunct(codepoint)) { + return false; + } + return u_isalnum(codepoint) || + u_hasBinaryProperty(codepoint, UCHAR_ALPHABETIC); +} + +template +static void TokenizeOpenSearchStandard(string_t input, LIST_WRITER &list) { + auto input_data = input.GetData(); + auto input_size = input.GetSize(); + idx_t token_start = 0; + idx_t token_size = 0; + + auto flush_token = [&]() { + if (token_size == 0) { + return; + } + list.WriteElement().WriteStringRef(string_t( + input_data + token_start, UnsafeNumericCast(token_size))); + token_size = 0; + }; + + for (idx_t pos = 0; pos < input_size;) { + int char_size = 0; + auto codepoint = + Utf8Proc::UTF8ToCodepoint(input_data + pos, char_size, input_size - pos); + D_ASSERT(char_size > 0); + + UErrorCode status = U_ZERO_ERROR; + auto script = uscript_getScript(codepoint, &status); + if (U_FAILURE(status)) { + flush_token(); + pos += UnsafeNumericCast(char_size); + continue; + } + + if (IsOpenSearchStandardSingleTokenScript(script)) { + flush_token(); + list.WriteElement().WriteStringRef(string_t( + input_data + pos, UnsafeNumericCast(char_size))); + } else if (IsOpenSearchStandardTokenChar(codepoint)) { + if (token_size == 0) { + token_start = pos; + } + token_size += UnsafeNumericCast(char_size); + } else { + flush_token(); + } + pos += UnsafeNumericCast(char_size); + } + flush_token(); +} + +static void OpenSearchStandardTokenizeFunction(DataChunk &args, + ExpressionState &state, + Vector &result) { + auto input_entries = args.data[0].Values(); + + D_ASSERT(result.GetType().id() == LogicalTypeId::LIST); + result.SetVectorType(VectorType::FLAT_VECTOR); + + auto list_writer = + FlatVector::Writer>(result, args.size()); + for (idx_t i = 0; i < args.size(); i++) { + auto input_entry = input_entries[i]; + if (!input_entry.IsValid()) { + list_writer.WriteNull(); + continue; + } + auto list = list_writer.WriteDynamicList(); + TokenizeOpenSearchStandard(input_entry.GetValue(), list); + } + + StringVector::AddHeapReference(ListVector::GetChildMutable(result), + args.data[0]); +} + static void StemFunction(DataChunk &args, ExpressionState &state, Vector &result) { auto &input_vector = args.data[0]; @@ -57,11 +148,16 @@ static void LoadInternal(ExtensionLoader &loader) { ScalarFunction stem_func("stem", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, StemFunction); + ScalarFunction opensearch_standard_tokenize_func( + "fts_tokenize_opensearch_standard", {LogicalType::VARCHAR}, + LogicalType::LIST(LogicalType::VARCHAR), + OpenSearchStandardTokenizeFunction); auto create_fts_index_func = PragmaFunction::PragmaCall( "create_fts_index", FTSIndexing::CreateFTSIndexQuery, {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR); create_fts_index_func.named_parameters["stemmer"] = LogicalType::VARCHAR; + create_fts_index_func.named_parameters["tokenizer"] = LogicalType::VARCHAR; create_fts_index_func.named_parameters["stopwords"] = LogicalType::VARCHAR; create_fts_index_func.named_parameters["ignore"] = LogicalType::VARCHAR; create_fts_index_func.named_parameters["strip_accents"] = @@ -78,6 +174,7 @@ static void LoadInternal(ExtensionLoader &loader) { "drop_fts_index", FTSIndexing::DropFTSIndexQuery, {LogicalType::VARCHAR}); loader.RegisterFunction(stem_func); + loader.RegisterFunction(opensearch_standard_tokenize_func); loader.RegisterFunction(create_fts_index_func); loader.RegisterFunction(drop_fts_index_func); } diff --git a/src/fts_indexing.cpp b/src/fts_indexing.cpp index f1d2474..5353981 100644 --- a/src/fts_indexing.cpp +++ b/src/fts_indexing.cpp @@ -187,8 +187,7 @@ static string StopwordsScript(const string &stopwords) { return "INSERT INTO %fts_schema%.stopwords SELECT * FROM " + stopwords + ";"; } -static string TokenizeMacroScript(const string &ignore, bool strip_accents, - bool lower) { +static string NormalizeTokenInputExpression(bool strip_accents, bool lower) { string expr = "s::VARCHAR"; if (strip_accents) { expr = "strip_accents(" + expr + ")"; @@ -196,6 +195,16 @@ static string TokenizeMacroScript(const string &ignore, bool strip_accents, if (lower) { expr = "lower(" + expr + ")"; } + return expr; +} + +static string TokenizeMacroScript(const string &tokenizer, const string &ignore, + bool strip_accents, bool lower) { + auto expr = NormalizeTokenInputExpression(strip_accents, lower); + if (tokenizer == "opensearch_standard") { + expr = "fts_tokenize_opensearch_standard(" + expr + ")"; + return "CREATE MACRO %fts_schema%.tokenize(s) AS " + expr + ";"; + } auto delimiter = "(" + ignore + ")|\\s+"; expr = "string_split_regex(" + expr + ", $$" + delimiter + "$$)"; return "CREATE MACRO %fts_schema%.tokenize(s) AS " + expr + ";"; @@ -1389,16 +1398,16 @@ static string IndexingScript(ClientContext &context, QualifiedName &qname, const string &input_id, const vector &input_values, const string &stemmer, const string &stopwords, - const string &ignore, bool strip_accents, - bool lower, bool incremental, bool cluster_terms, - bool layered_search) { + const string &tokenizer, const string &ignore, + bool strip_accents, bool lower, bool incremental, + bool cluster_terms, bool layered_search) { string result; if (TableExists(context, qname) && SupportsFTSTriggers(context, qname)) { result += DropFTSTriggersScript(qname); } result += SchemaSetupScript(); result += StopwordsScript(stopwords); - result += TokenizeMacroScript(ignore, strip_accents, lower); + result += TokenizeMacroScript(tokenizer, ignore, strip_accents, lower); result += IndexTablesScript(input_id, input_values, stemmer, stopwords, GetFTSBuildTermsTable(qname), GetFTSBuildDictTable(qname), cluster_terms); @@ -1473,6 +1482,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, }; const string stemmer = get_string("stemmer", "porter"); + const string tokenizer = get_string("tokenizer", "regex"); const string stopwords = get_string("stopwords", "english"); const string ignore = get_string("ignore", "[0-9!@#$%^&*()_+={}\\[\\]:;<>,.?~\\\\/\\|''\"`-]+"); @@ -1483,6 +1493,13 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, const bool layered_search = get_bool("layered_search", false); const bool cluster_terms = get_bool("cluster_terms", false) || layered_search; + if (tokenizer != "regex" && tokenizer != "opensearch_standard") { + throw InvalidInputException( + "Unrecognized tokenizer '%s'. Supported tokenizers are: ['regex', " + "'opensearch_standard']", + tokenizer); + } + if (stopwords != "english" && stopwords != "none") { auto sw_qname = GetQualifiedName(context, stopwords); Catalog::GetEntry(context, sw_qname); @@ -1549,7 +1566,7 @@ string FTSIndexing::CreateFTSIndexQuery(ClientContext &context, } return IndexingScript(context, qname, doc_id, doc_values, stemmer, stopwords, - ignore, strip_accents, lower, incremental, + tokenizer, ignore, strip_accents, lower, incremental, cluster_terms, layered_search); } diff --git a/test/sql/fts/test_opensearch_standard_tokenizer.test b/test/sql/fts/test_opensearch_standard_tokenizer.test new file mode 100644 index 0000000..581e19a --- /dev/null +++ b/test/sql/fts/test_opensearch_standard_tokenizer.test @@ -0,0 +1,64 @@ +# name: test/sql/fts/test_opensearch_standard_tokenizer.test +# description: Test OpenSearch standard-tokenizer parity for multilingual FTS tokenization. +# group: [fts] + +require fts + +require no_alternative_verify + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower(strip_accents( + 'תל-אביב ביב Привет-мир مرحبا-بالعالم 東京大学に行く 北京大学 한국어 테스트 café' + )))) AS token +) +---- +תל +אביב +ביב +привет +мир +مرحبا +بالعالم +東 +京 +大 +学 +に +行 +く +北 +京 +大 +学 +한국어 +테스트 +cafe + +statement ok +CREATE TABLE unicode_docs(id VARCHAR, body VARCHAR) + +statement ok +INSERT INTO unicode_docs VALUES ('doc1', 'תל-אביב ביב Привет-мир مرحبا-بالعالم 東京大学に行く 北京大学 한국어 테스트 café') + +statement ok +PRAGMA create_fts_index('unicode_docs', 'id', 'body', tokenizer='opensearch_standard', stemmer='none', stopwords='none') + +query I +SELECT count(*) +FROM fts_main_unicode_docs.dict +WHERE term IN ('東京大学に行く', '北京大学') +---- +0 + +query I +SELECT count(*) +FROM fts_main_unicode_docs.dict +WHERE term IN ('東', '京', '大', '学', 'に', '行', 'く', '北', '한국어', '테스트') +---- +10 + +statement error +PRAGMA create_fts_index('unicode_docs', 'id', 'body', tokenizer='unknown', overwrite=true) +---- From b254754d00db4819caba4bf7e9bfea9976d06b34 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Thu, 9 Jul 2026 11:48:20 +0200 Subject: [PATCH 2/5] Improve parity with opensearch --- src/fts_extension.cpp | 76 ++++++++++++--- .../test_opensearch_standard_tokenizer.test | 93 +++++++++++++++++++ 2 files changed, 155 insertions(+), 14 deletions(-) diff --git a/src/fts_extension.cpp b/src/fts_extension.cpp index 643a039..7a83ac1 100644 --- a/src/fts_extension.cpp +++ b/src/fts_extension.cpp @@ -19,16 +19,38 @@ namespace duckdb { static bool IsOpenSearchStandardSingleTokenScript(UScriptCode script) { - return script == USCRIPT_HAN || script == USCRIPT_HIRAGANA || - script == USCRIPT_KATAKANA; + return script == USCRIPT_HAN || script == USCRIPT_HIRAGANA; } -static bool IsOpenSearchStandardTokenChar(UChar32 codepoint) { +static bool IsOpenSearchStandardEmoji(UChar32 codepoint) { + return u_hasBinaryProperty(codepoint, UCHAR_EMOJI_PRESENTATION) || + u_hasBinaryProperty(codepoint, UCHAR_EXTENDED_PICTOGRAPHIC); +} + +static bool IsOpenSearchStandardIntraTokenPunctuation(UChar32 codepoint) { + return codepoint == '\'' || codepoint == '.' || codepoint == '_'; +} + +static bool IsJapaneseProlongedSoundMark(UChar32 codepoint) { + return codepoint == 0x30FC || codepoint == 0xFF70; +} + +static bool IsOpenSearchStandardTokenChar(UChar32 codepoint, + UScriptCode script) { if (u_isUWhiteSpace(codepoint) || u_ispunct(codepoint)) { return false; } return u_isalnum(codepoint) || - u_hasBinaryProperty(codepoint, UCHAR_ALPHABETIC); + u_hasBinaryProperty(codepoint, UCHAR_ALPHABETIC) || + script == USCRIPT_KATAKANA || + IsJapaneseProlongedSoundMark(codepoint); +} + +static bool IsOpenSearchStandardContinuationChar(UChar32 codepoint, + UScriptCode script) { + return !IsOpenSearchStandardSingleTokenScript(script) && + !IsOpenSearchStandardEmoji(codepoint) && + IsOpenSearchStandardTokenChar(codepoint, script); } template @@ -47,25 +69,51 @@ static void TokenizeOpenSearchStandard(string_t input, LIST_WRITER &list) { token_size = 0; }; - for (idx_t pos = 0; pos < input_size;) { - int char_size = 0; - auto codepoint = + auto decode_codepoint = [&](idx_t pos, UChar32 &codepoint, int &char_size, + UScriptCode &script) -> bool { + if (pos >= input_size) { + return false; + } + char_size = 0; + codepoint = Utf8Proc::UTF8ToCodepoint(input_data + pos, char_size, input_size - pos); - D_ASSERT(char_size > 0); - + if (char_size <= 0) { + return false; + } UErrorCode status = U_ZERO_ERROR; - auto script = uscript_getScript(codepoint, &status); - if (U_FAILURE(status)) { + script = uscript_getScript(codepoint, &status); + return U_SUCCESS(status); + }; + + for (idx_t pos = 0; pos < input_size;) { + int char_size = 0; + UChar32 codepoint = 0; + UScriptCode script = USCRIPT_UNKNOWN; + if (!decode_codepoint(pos, codepoint, char_size, script)) { flush_token(); - pos += UnsafeNumericCast(char_size); + pos++; continue; } - if (IsOpenSearchStandardSingleTokenScript(script)) { + if (IsOpenSearchStandardSingleTokenScript(script) || + IsOpenSearchStandardEmoji(codepoint)) { flush_token(); list.WriteElement().WriteStringRef(string_t( input_data + pos, UnsafeNumericCast(char_size))); - } else if (IsOpenSearchStandardTokenChar(codepoint)) { + } else if (IsOpenSearchStandardIntraTokenPunctuation(codepoint) && + token_size > 0) { + UChar32 next_codepoint = 0; + int next_char_size = 0; + UScriptCode next_script = USCRIPT_UNKNOWN; + auto next_pos = pos + UnsafeNumericCast(char_size); + if (decode_codepoint(next_pos, next_codepoint, next_char_size, + next_script) && + IsOpenSearchStandardContinuationChar(next_codepoint, next_script)) { + token_size += UnsafeNumericCast(char_size); + } else { + flush_token(); + } + } else if (IsOpenSearchStandardTokenChar(codepoint, script)) { if (token_size == 0) { token_start = pos; } diff --git a/test/sql/fts/test_opensearch_standard_tokenizer.test b/test/sql/fts/test_opensearch_standard_tokenizer.test index 581e19a..e61ea48 100644 --- a/test/sql/fts/test_opensearch_standard_tokenizer.test +++ b/test/sql/fts/test_opensearch_standard_tokenizer.test @@ -59,6 +59,99 @@ WHERE term IN ('東', '京', '大', '学', 'に', '行', 'く', '北', '한국 ---- 10 +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + 'John''s co-op email@example.com foo_bar snake_case C++ C# node.js' + ))) AS token +) +---- +john's +co +op +email +example.com +foo_bar +snake_case +c +c +node.js + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + 'abc123 123abc 42 3.14 version-2 test_99 東京2024' + ))) AS token +) +---- +abc123 +123abc +42 +3.14 +version +2 +test_99 +東 +京 +2024 + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard( + 'ひらがな カタカナ ｶﾀｶﾅ ー タワー オープンサーチ' + )) AS token +) +---- +ひ +ら +が +な +カタカナ +ｶﾀｶﾅ +ー +タワー +オープンサーチ + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + 'hello🙂world #hashtag @user path/to/file 中文🙂test' + ))) AS token +) +---- +hello +🙂 +world +hashtag +user +path +to +file +中 +文 +🙂 +test + +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard(lower( + '한국어 테스트 서울대학교 K-pop 한글-테스트' + ))) AS token +) +---- +한국어 +테스트 +서울대학교 +k +pop +한글 +테스트 + statement error PRAGMA create_fts_index('unicode_docs', 'id', 'body', tokenizer='unknown', overwrite=true) ---- From 9ef9762b6db60521e5f1e1712c19a5903bfeba04 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Fri, 10 Jul 2026 14:41:48 +0200 Subject: [PATCH 3/5] Format fix --- src/fts_extension.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/fts_extension.cpp b/src/fts_extension.cpp index 7a83ac1..5821a99 100644 --- a/src/fts_extension.cpp +++ b/src/fts_extension.cpp @@ -42,8 +42,7 @@ static bool IsOpenSearchStandardTokenChar(UChar32 codepoint, } return u_isalnum(codepoint) || u_hasBinaryProperty(codepoint, UCHAR_ALPHABETIC) || - script == USCRIPT_KATAKANA || - IsJapaneseProlongedSoundMark(codepoint); + script == USCRIPT_KATAKANA || IsJapaneseProlongedSoundMark(codepoint); } static bool IsOpenSearchStandardContinuationChar(UChar32 codepoint, @@ -75,8 +74,8 @@ static void TokenizeOpenSearchStandard(string_t input, LIST_WRITER &list) { return false; } char_size = 0; - codepoint = - Utf8Proc::UTF8ToCodepoint(input_data + pos, char_size, input_size - pos); + codepoint = Utf8Proc::UTF8ToCodepoint(input_data + pos, char_size, + input_size - pos); if (char_size <= 0) { return false; } @@ -98,8 +97,8 @@ static void TokenizeOpenSearchStandard(string_t input, LIST_WRITER &list) { if (IsOpenSearchStandardSingleTokenScript(script) || IsOpenSearchStandardEmoji(codepoint)) { flush_token(); - list.WriteElement().WriteStringRef(string_t( - input_data + pos, UnsafeNumericCast(char_size))); + list.WriteElement().WriteStringRef( + string_t(input_data + pos, UnsafeNumericCast(char_size))); } else if (IsOpenSearchStandardIntraTokenPunctuation(codepoint) && token_size > 0) { UChar32 next_codepoint = 0; From 106ba572a9568bf528565d3015c8d632a8161a76 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Thu, 16 Jul 2026 14:24:16 +0200 Subject: [PATCH 4/5] Handle U 2019 properly --- src/fts_extension.cpp | 3 ++- .../sql/fts/test_opensearch_standard_tokenizer.test | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/fts_extension.cpp b/src/fts_extension.cpp index 5821a99..5389788 100644 --- a/src/fts_extension.cpp +++ b/src/fts_extension.cpp @@ -28,7 +28,8 @@ static bool IsOpenSearchStandardEmoji(UChar32 codepoint) { } static bool IsOpenSearchStandardIntraTokenPunctuation(UChar32 codepoint) { - return codepoint == '\'' || codepoint == '.' || codepoint == '_'; + return codepoint == '\'' || codepoint == 0x2019 || codepoint == '.' || + codepoint == '_'; } static bool IsJapaneseProlongedSoundMark(UChar32 codepoint) { diff --git a/test/sql/fts/test_opensearch_standard_tokenizer.test b/test/sql/fts/test_opensearch_standard_tokenizer.test index e61ea48..9ccce44 100644 --- a/test/sql/fts/test_opensearch_standard_tokenizer.test +++ b/test/sql/fts/test_opensearch_standard_tokenizer.test @@ -78,6 +78,19 @@ c c node.js +query T +SELECT token +FROM ( + SELECT unnest(fts_tokenize_opensearch_standard( + 'It’s l’amour rock’n’roll John''s' + )) AS token +) +---- +It’s +l’amour +rock’n’roll +John's + query T SELECT token FROM ( From 61a3dabbb6b7bce40da3e133fdf6e6b19b50c730 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Thu, 16 Jul 2026 15:30:20 +0200 Subject: [PATCH 5/5] Use host DuckDB ICU headers --- CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 535c153..8c8beb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,13 +7,18 @@ project(FTSExtension) set(EXTENSION_BASE_FOLDER "src") set(SNOWBALL_BASE_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/third_party/snowball") -set(DUCKDB_ICU_BASE_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/duckdb/extension/icu") +if(DEFINED DUCKDB_MODULE_BASE_DIR) + set(DUCKDB_SOURCE_BASE_FOLDER "${DUCKDB_MODULE_BASE_DIR}") +else() + set(DUCKDB_SOURCE_BASE_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/duckdb") +endif() +set(DUCKDB_ICU_BASE_FOLDER "${DUCKDB_SOURCE_BASE_FOLDER}/extension/icu") include_directories( ${EXTENSION_BASE_FOLDER}/include ${SNOWBALL_BASE_FOLDER}/include ${SNOWBALL_BASE_FOLDER}/libstemmer ${SNOWBALL_BASE_FOLDER}/runtime - ${CMAKE_CURRENT_SOURCE_DIR}/duckdb/third_party/utf8proc/include + ${DUCKDB_SOURCE_BASE_FOLDER}/third_party/utf8proc/include ${DUCKDB_ICU_BASE_FOLDER}/third_party/icu/common ${DUCKDB_ICU_BASE_FOLDER}/third_party/icu/i18n)