use bm25::{EmbedderBuilder, Language, LanguageMode};
fn main() {
let em = EmbedderBuilder::<u32>::with_avgdl(1.0)
.b(0.0)
.language_mode(LanguageMode::Fixed(Language::English))
.build();
dbg!(em.embed("§\t184.1240\tCarbon\tdioxide."));
dbg!(em.embed("1\t184.1240\tCarbon\tdioxide."));
dbg!(em.embed("\t184.1240\tCarbon\tdioxide."));
dbg!(em.embed("§ 184.1240 Carbon dioxide."));
dbg!(em.embed("184.1240 Carbon dioxide."));
}
results in
[src/main.rs:9:5] em.embed("§\t184.1240\tCarbon\tdioxide.") = Embedding(
[
TokenEmbedding {
index: 504642266,
value: 1.0,
},
],
)
[src/main.rs:10:5] em.embed("1\t184.1240\tCarbon\tdioxide.") = Embedding(
[
TokenEmbedding {
index: 2646367888,
value: 1.0,
},
TokenEmbedding {
index: 2840934837,
value: 1.0,
},
TokenEmbedding {
index: 196859388,
value: 1.0,
},
TokenEmbedding {
index: 2767204738,
value: 1.0,
},
],
)
[src/main.rs:11:5] em.embed("\t184.1240\tCarbon\tdioxide.") = Embedding(
[
TokenEmbedding {
index: 2840934837,
value: 1.0,
},
TokenEmbedding {
index: 196859388,
value: 1.0,
},
TokenEmbedding {
index: 2767204738,
value: 1.0,
},
],
)
[src/main.rs:12:5] em.embed("§ 184.1240 Carbon dioxide.") = Embedding(
[
TokenEmbedding {
index: 4201093589,
value: 1.0,
},
TokenEmbedding {
index: 2840934837,
value: 1.0,
},
TokenEmbedding {
index: 196859388,
value: 1.0,
},
TokenEmbedding {
index: 2767204738,
value: 1.0,
},
],
)
[src/main.rs:13:5] em.embed("184.1240 Carbon dioxide.") = Embedding(
[
TokenEmbedding {
index: 2840934837,
value: 1.0,
},
TokenEmbedding {
index: 196859388,
value: 1.0,
},
TokenEmbedding {
index: 2767204738,
value: 1.0,
},
],
)
What's happening:
- having
§ and tabs results in a single token
- replacing
§ with other character results in correct tokens (even with tabs)
- replacing tabs with spaces results in correct tokens (even with
§)
Is this expected?
results in
What's happening:
§and tabs results in a single token§with other character results in correct tokens (even with tabs)§)Is this expected?