Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -346,16 +346,16 @@ jobs:
echo "TODO: Test item" > test_mvp.txt

# Test basic search functionality
./target/release/semisearch search "TODO" --path ./test_mvp.txt | grep -q "TODO: Test item" || (echo "โŒ Basic search not working" && exit 1)
NO_COLOR=1 ./target/release/semisearch search "TODO" --path ./test_mvp.txt | grep -q "TODO: Test item" || (echo "โŒ Basic search not working" && exit 1)

# Test JSON output
./target/release/semisearch --advanced search "TODO" --path ./test_mvp.txt --format json | jq . > /dev/null || (echo "โŒ JSON output not working" && exit 1)

# Test regex mode
./target/release/semisearch --advanced search "TODO.*:" --path ./test_mvp.txt --mode regex | wc -l | grep -q -v "^0$" || (echo "โŒ Regex search not working" && exit 1)
NO_COLOR=1 ./target/release/semisearch --advanced search "TODO.*:" --path ./test_mvp.txt --mode regex | wc -l | grep -q -v "^0$" || (echo "โŒ Regex search not working" && exit 1)

# Test fuzzy mode (includes typo tolerance)
./target/release/semisearch --advanced search "TODO" --path ./test_mvp.txt --mode fuzzy | wc -l | grep -q -v "^0$" || (echo "โŒ Fuzzy search not working" && exit 1)
NO_COLOR=1 ./target/release/semisearch --advanced search "TODO" --path ./test_mvp.txt --mode fuzzy | wc -l | grep -q -v "^0$" || (echo "โŒ Fuzzy search not working" && exit 1)

# Cleanup
rm test_mvp.txt
Expand Down
6 changes: 6 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ rustc-hash = "1.1"
# Async runtime
tokio = { version = "1.0", features = ["full"] }

# Pattern management
lazy_static = "1.4"

# Dynamic loading for optional features
libloading = "0.8"
Expand All @@ -56,3 +58,7 @@ tfidf-only = [] # Legacy feature for minimal builds

# Build optimizations
[profile.release]



# Neural demo example was removed
23 changes: 18 additions & 5 deletions src/core/embedder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,11 @@ pub struct LocalEmbedder {
impl LocalEmbedder {
/// Create a new local embedder with neural capabilities
pub async fn new(config: EmbeddingConfig) -> Result<Self> {
Self::new_with_mode(config, false).await
}

/// Create a new local embedder with neural capabilities and mode control
pub async fn new_with_mode(config: EmbeddingConfig, _advanced_mode: bool) -> Result<Self> {
let capability = Self::detect_capabilities();

match capability {
Expand All @@ -82,7 +87,10 @@ impl LocalEmbedder {
// Try to initialize neural embeddings
match Self::initialize_neural_embedder(&config).await {
Ok((session, tokenizer)) => {
eprintln!("โœ… Neural embeddings initialized successfully");
// Only show success message in advanced mode [[memory:655345]]
if _advanced_mode {
eprintln!("โœ… Neural embeddings initialized successfully");
}
Ok(Self {
config,
session: Some(session),
Expand Down Expand Up @@ -151,6 +159,7 @@ impl LocalEmbedder {
eprintln!("๐Ÿ“ฅ Neural model missing, attempting download for semantic search...");
match Self::initialize_neural_embedder(&config).await {
Ok((session, tokenizer)) => {
// Neural embeddings message is controlled by caller context
eprintln!("โœ… Neural embeddings initialized successfully");
Ok(Self {
config,
Expand Down Expand Up @@ -249,7 +258,8 @@ impl LocalEmbedder {
let total_size = response.content_length().unwrap_or(0);

if total_size > 0 {
println!("๐Ÿ“ฆ Model size: {:.2} MB", total_size as f64 / 1_048_576.0);
let size_mb = total_size as f64 / 1_048_576.0;
println!("๐Ÿ“ฆ Model size: {size_mb:.2} MB");
}

// Download the entire content at once instead of streaming
Expand Down Expand Up @@ -1062,8 +1072,10 @@ mod tests {
println!(" ๐Ÿ”ค Model: {}", config.model_name);
println!(" ๐Ÿ“ Max sequence length: {}", config.max_length);
println!(" ๐Ÿ“ฆ Batch size: {}", config.batch_size);
println!(" ๐Ÿ–ฅ๏ธ Device: {:?}", config.device);
println!(" ๐Ÿ’พ Cache directory: {:?}", config.cache_dir);
let device = &config.device;
let cache_dir = &config.cache_dir;
println!(" ๐Ÿ–ฅ๏ธ Device: {device:?}");
println!(" ๐Ÿ’พ Cache directory: {cache_dir:?}");

println!("\n๐Ÿ” System Capability Detection:");
let capability = LocalEmbedder::detect_capabilities();
Expand All @@ -1089,7 +1101,8 @@ mod tests {
match LocalEmbedder::new(config).await {
Ok(embedder) => {
println!("โœ… LocalEmbedder created successfully!");
println!(" ๐Ÿ“Š Final capability: {:?}", embedder.capability());
let capability = embedder.capability();
println!(" ๐Ÿ“Š Final capability: {capability:?}");
println!(" ๐Ÿ“ Embedding dimension: {}", embedder.embedding_dim());
println!(" ๐Ÿงฎ Has vocabulary: {}", embedder.has_vocabulary());

Expand Down
Loading