Opensearch compatible tokenization #39
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,14 +3,152 @@ | |
| #include "duckdb.hpp" | ||
| #include "duckdb/common/exception.hpp" | ||
| #include "duckdb/common/string_util.hpp" | ||
| #include "duckdb/common/vector/flat_vector.hpp" | ||
| #include "duckdb/common/vector/list_vector.hpp" | ||
| #include "duckdb/common/vector/string_vector.hpp" | ||
| #include "duckdb/common/vector/vector_writer.hpp" | ||
| #include "duckdb/function/pragma_function.hpp" | ||
| #include "duckdb/function/scalar_function.hpp" | ||
| #include "duckdb/main/extension/extension_loader.hpp" | ||
| #include "fts_indexing.hpp" | ||
| #include "libstemmer.h" | ||
| #include "unicode/uchar.h" | ||
| #include "unicode/uscript.h" | ||
| #include "utf8proc_wrapper.hpp" | ||
|
|
||
| namespace duckdb { | ||
|
|
||
| static bool IsOpenSearchStandardSingleTokenScript(UScriptCode script) { | ||
| return script == USCRIPT_HAN || script == USCRIPT_HIRAGANA; | ||
| } | ||
|
|
||
| static bool IsOpenSearchStandardEmoji(UChar32 codepoint) { | ||
| return u_hasBinaryProperty(codepoint, UCHAR_EMOJI_PRESENTATION) || | ||
| u_hasBinaryProperty(codepoint, UCHAR_EXTENDED_PICTOGRAPHIC); | ||
| } | ||
|
|
||
| static bool IsOpenSearchStandardIntraTokenPunctuation(UChar32 codepoint) { | ||
| return codepoint == '\'' || codepoint == '.' || codepoint == '_'; | ||
| } | ||
|
Member
There was a problem hiding this comment. This only treats ASCII ' as intra-token punctuation. As a result, common text such as It’s and l’amour is split into It, s / l, amour, whereas OpenSearch’s standard tokenizer preserves It’s as one token. Could we implement the relevant UAX #29 apostrophe handling (at least U+2019), and add coverage for it? # name: test/temp.test
# group: [fts]
require fts
query T
SELECT token
FROM unnest(fts_tokenize_opensearch_standard('It’s')) AS t(token)
----
It’s |
||
|
|
||
| static bool IsJapaneseProlongedSoundMark(UChar32 codepoint) { | ||
| return codepoint == 0x30FC || codepoint == 0xFF70; | ||
| } | ||
|
|
||
| static bool IsOpenSearchStandardTokenChar(UChar32 codepoint, | ||
| UScriptCode script) { | ||
| if (u_isUWhiteSpace(codepoint) || u_ispunct(codepoint)) { | ||
| return false; | ||
| } | ||
| return u_isalnum(codepoint) || | ||
| u_hasBinaryProperty(codepoint, UCHAR_ALPHABETIC) || | ||
| script == USCRIPT_KATAKANA || IsJapaneseProlongedSoundMark(codepoint); | ||
| } | ||
|
|
||
| static bool IsOpenSearchStandardContinuationChar(UChar32 codepoint, | ||
| UScriptCode script) { | ||
| return !IsOpenSearchStandardSingleTokenScript(script) && | ||
| !IsOpenSearchStandardEmoji(codepoint) && | ||
| IsOpenSearchStandardTokenChar(codepoint, script); | ||
| } | ||
|
|
||
| template <class LIST_WRITER> | ||
| static void TokenizeOpenSearchStandard(string_t input, LIST_WRITER &list) { | ||
| auto input_data = input.GetData(); | ||
| auto input_size = input.GetSize(); | ||
| idx_t token_start = 0; | ||
| idx_t token_size = 0; | ||
|
|
||
| auto flush_token = [&]() { | ||
| if (token_size == 0) { | ||
| return; | ||
| } | ||
| list.WriteElement().WriteStringRef(string_t( | ||
| input_data + token_start, UnsafeNumericCast<uint32_t>(token_size))); | ||
| token_size = 0; | ||
| }; | ||
|
|
||
| auto decode_codepoint = [&](idx_t pos, UChar32 &codepoint, int &char_size, | ||
| UScriptCode &script) -> bool { | ||
| if (pos >= input_size) { | ||
| return false; | ||
| } | ||
| char_size = 0; | ||
| codepoint = Utf8Proc::UTF8ToCodepoint(input_data + pos, char_size, | ||
| input_size - pos); | ||
| if (char_size <= 0) { | ||
| return false; | ||
| } | ||
| UErrorCode status = U_ZERO_ERROR; | ||
| script = uscript_getScript(codepoint, &status); | ||
| return U_SUCCESS(status); | ||
| }; | ||
|
|
||
| for (idx_t pos = 0; pos < input_size;) { | ||
| int char_size = 0; | ||
| UChar32 codepoint = 0; | ||
| UScriptCode script = USCRIPT_UNKNOWN; | ||
| if (!decode_codepoint(pos, codepoint, char_size, script)) { | ||
| flush_token(); | ||
| pos++; | ||
| continue; | ||
| } | ||
|
|
||
| if (IsOpenSearchStandardSingleTokenScript(script) || | ||
| IsOpenSearchStandardEmoji(codepoint)) { | ||
| flush_token(); | ||
| list.WriteElement().WriteStringRef( | ||
| string_t(input_data + pos, UnsafeNumericCast<uint32_t>(char_size))); | ||
| } else if (IsOpenSearchStandardIntraTokenPunctuation(codepoint) && | ||
| token_size > 0) { | ||
| UChar32 next_codepoint = 0; | ||
| int next_char_size = 0; | ||
| UScriptCode next_script = USCRIPT_UNKNOWN; | ||
| auto next_pos = pos + UnsafeNumericCast<idx_t>(char_size); | ||
| if (decode_codepoint(next_pos, next_codepoint, next_char_size, | ||
| next_script) && | ||
| IsOpenSearchStandardContinuationChar(next_codepoint, next_script)) { | ||
| token_size += UnsafeNumericCast<idx_t>(char_size); | ||
| } else { | ||
| flush_token(); | ||
| } | ||
| } else if (IsOpenSearchStandardTokenChar(codepoint, script)) { | ||
| if (token_size == 0) { | ||
| token_start = pos; | ||
| } | ||
| token_size += UnsafeNumericCast<idx_t>(char_size); | ||
| } else { | ||
| flush_token(); | ||
| } | ||
| pos += UnsafeNumericCast<idx_t>(char_size); | ||
| } | ||
| flush_token(); | ||
| } | ||
|
|
||
| static void OpenSearchStandardTokenizeFunction(DataChunk &args, | ||
| ExpressionState &state, | ||
| Vector &result) { | ||
| auto input_entries = args.data[0].Values<string_t>(); | ||
|
|
||
| D_ASSERT(result.GetType().id() == LogicalTypeId::LIST); | ||
| result.SetVectorType(VectorType::FLAT_VECTOR); | ||
|
|
||
| auto list_writer = | ||
| FlatVector::Writer<VectorListType<string_t>>(result, args.size()); | ||
| for (idx_t i = 0; i < args.size(); i++) { | ||
| auto input_entry = input_entries[i]; | ||
| if (!input_entry.IsValid()) { | ||
| list_writer.WriteNull(); | ||
| continue; | ||
| } | ||
| auto list = list_writer.WriteDynamicList(); | ||
| TokenizeOpenSearchStandard(input_entry.GetValue(), list); | ||
| } | ||
|
|
||
| StringVector::AddHeapReference(ListVector::GetChildMutable(result), | ||
| args.data[0]); | ||
| } | ||
|
|
||
| static void StemFunction(DataChunk &args, ExpressionState &state, | ||
| Vector &result) { | ||
| auto &input_vector = args.data[0]; | ||
|
|
@@ -57,11 +195,16 @@ static void LoadInternal(ExtensionLoader &loader) { | |
|
|
||
| ScalarFunction stem_func("stem", {LogicalType::VARCHAR, LogicalType::VARCHAR}, | ||
| LogicalType::VARCHAR, StemFunction); | ||
| ScalarFunction opensearch_standard_tokenize_func( | ||
| "fts_tokenize_opensearch_standard", {LogicalType::VARCHAR}, | ||
| LogicalType::LIST(LogicalType::VARCHAR), | ||
| OpenSearchStandardTokenizeFunction); | ||
|
|
||
| auto create_fts_index_func = PragmaFunction::PragmaCall( | ||
| "create_fts_index", FTSIndexing::CreateFTSIndexQuery, | ||
| {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR); | ||
| create_fts_index_func.named_parameters["stemmer"] = LogicalType::VARCHAR; | ||
| create_fts_index_func.named_parameters["tokenizer"] = LogicalType::VARCHAR; | ||
| create_fts_index_func.named_parameters["stopwords"] = LogicalType::VARCHAR; | ||
| create_fts_index_func.named_parameters["ignore"] = LogicalType::VARCHAR; | ||
| create_fts_index_func.named_parameters["strip_accents"] = | ||
|
|
@@ -78,6 +221,7 @@ static void LoadInternal(ExtensionLoader &loader) { | |
| "drop_fts_index", FTSIndexing::DropFTSIndexQuery, {LogicalType::VARCHAR}); | ||
|
|
||
| loader.RegisterFunction(stem_func); | ||
| loader.RegisterFunction(opensearch_standard_tokenize_func); | ||
| loader.RegisterFunction(create_fts_index_func); | ||
| loader.RegisterFunction(drop_fts_index_func); | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This compiles against ICU headers from FTS’s pinned duckdb submodule, but below we link duckdb_icu_* object targets from the host DuckDB build. That mixes two DuckDB/ICU revisions with no compatibility guarantee. Can we source both headers and objects consistently from the host build (or consistently from the submodule)?