From b1784a18c654c3da6387e81aee3168d665a0deaf Mon Sep 17 00:00:00 2001 From: ccmdi Date: Mon, 8 Dec 2025 15:51:05 -0500 Subject: [PATCH 01/16] feat: add lightweight llm wrapper instead of litellm to reduce import time --- obsidianki/ai/call.py | 353 ++++++++++++++++++++++++++++++++++++++++ obsidianki/ai/client.py | 24 +-- pyproject.toml | 1 - 3 files changed, 358 insertions(+), 20 deletions(-) create mode 100644 obsidianki/ai/call.py diff --git a/obsidianki/ai/call.py b/obsidianki/ai/call.py new file mode 100644 index 0000000..defaf58 --- /dev/null +++ b/obsidianki/ai/call.py @@ -0,0 +1,353 @@ +""" +lite_llm.py - Minimal LLM wrapper (~130ms import) + +Supports: OpenAI, Anthropic, Google (Gemini), DeepSeek +No streaming. Tool calling supported. +""" +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Union + +import httpx + +# Provider endpoints +ENDPOINTS = { + "openai": "https://api.openai.com/v1/chat/completions", + "anthropic": "https://api.anthropic.com/v1/messages", + "google": "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", + "deepseek": "https://api.deepseek.com/chat/completions", +} + +# Environment variable names for API keys +API_KEY_NAMES = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "google": "GEMINI_API_KEY", + "deepseek": "DEEPSEEK_API_KEY", +} + + +@dataclass +class Function: + name: str + arguments: str + + +@dataclass +class ToolCall: + id: str + type: str + function: Function + + +@dataclass +class Message: + role: str + content: Optional[str] = None + tool_calls: Optional[List[ToolCall]] = None + + +@dataclass +class Choice: + index: int + message: Message + finish_reason: Optional[str] = None + + +@dataclass +class Usage: + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + + +@dataclass +class ModelResponse: + id: str + object: str + created: int + model: str + choices: List[Choice] + usage: Usage = field(default_factory=Usage) + + +def _get_provider(model: str) -> tuple[str, str]: + """Extract provider and model name from model string like 'openai/gpt-4'""" + if "/" in model: + parts = model.split("/", 1) + provider = parts[0] + model_name = parts[1] + + # Handle nested paths like "gemini/gemini-2.5-pro" + if provider == "gemini": + provider = "google" + + return provider, model_name + + # Guess provider from model name + if model.startswith("gpt") or model.startswith("o1") or model.startswith("o3"): + return "openai", model + elif model.startswith("claude"): + return "anthropic", model + elif model.startswith("gemini"): + return "google", model + elif model.startswith("deepseek"): + return "deepseek", model + + raise ValueError(f"Cannot determine provider for model: {model}") + + +def _get_api_key(provider: str) -> str: + """Get API key from environment""" + key_name = API_KEY_NAMES.get(provider) + if not key_name: + raise ValueError(f"Unknown provider: {provider}") + + key = os.environ.get(key_name) + if not key: + raise ValueError(f"{key_name} not found in environment variables") + + return key + + +def _build_headers(provider: str, api_key: str) -> Dict[str, str]: + """Build request headers for each provider""" + if provider == "anthropic": + return { + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + } + elif provider == "google": + return { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + else: + # OpenAI-compatible (openai, deepseek) + return { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + +def _convert_tools_for_anthropic(tools: List[Dict]) -> List[Dict]: + """Convert OpenAI tool format to Anthropic format""" + anthropic_tools = [] + for tool in tools: + if tool.get("type") == "function": + func = tool["function"] + anthropic_tools.append({ + "name": func["name"], + "description": func.get("description", ""), + "input_schema": func.get("parameters", {"type": "object", "properties": {}}), + }) + return anthropic_tools + + +def _convert_tool_choice_for_anthropic(tool_choice: Union[str, Dict]) -> Dict: + """Convert OpenAI tool_choice to Anthropic format""" + if tool_choice == "auto": + return {"type": "auto"} + elif tool_choice == "required": + return {"type": "any"} + elif tool_choice == "none": + return {"type": "none"} + elif isinstance(tool_choice, dict): + # {"type": "function", "function": {"name": "..."}} + return {"type": "tool", "name": tool_choice["function"]["name"]} + return {"type": "auto"} + + +def _build_anthropic_request( + model: str, + messages: List[Dict], + tools: Optional[List[Dict]] = None, + tool_choice: Optional[Union[str, Dict]] = None, + max_tokens: int = 4096, + **kwargs +) -> Dict: + """Build Anthropic API request body""" + # Extract system message + system = None + chat_messages = [] + + for msg in messages: + if msg["role"] == "system": + system = msg["content"] + else: + chat_messages.append(msg) + + body: Dict[str, Any] = { + "model": model, + "messages": chat_messages, + "max_tokens": max_tokens, + } + + if system: + body["system"] = system + + if tools: + body["tools"] = _convert_tools_for_anthropic(tools) + if tool_choice: + body["tool_choice"] = _convert_tool_choice_for_anthropic(tool_choice) + + return body + + +def _parse_anthropic_response(response_json: Dict) -> ModelResponse: + """Convert Anthropic response to OpenAI-compatible ModelResponse""" + content_blocks = response_json.get("content", []) + + # Extract text content + text_content = None + tool_calls = [] + + for i, block in enumerate(content_blocks): + if block["type"] == "text": + text_content = block["text"] + elif block["type"] == "tool_use": + tool_calls.append(ToolCall( + id=block["id"], + type="function", + function=Function( + name=block["name"], + arguments=json.dumps(block["input"]), + ) + )) + + message = Message( + role="assistant", + content=text_content, + tool_calls=tool_calls if tool_calls else None, + ) + + usage_data = response_json.get("usage", {}) + usage = Usage( + prompt_tokens=usage_data.get("input_tokens", 0), + completion_tokens=usage_data.get("output_tokens", 0), + total_tokens=usage_data.get("input_tokens", 0) + usage_data.get("output_tokens", 0), + ) + + return ModelResponse( + id=response_json.get("id", ""), + object="chat.completion", + created=0, + model=response_json.get("model", ""), + choices=[Choice(index=0, message=message, finish_reason=response_json.get("stop_reason"))], + usage=usage, + ) + + +def _parse_openai_response(response_json: Dict) -> ModelResponse: + """Convert OpenAI-compatible response to ModelResponse""" + choices = [] + + for i, choice_data in enumerate(response_json.get("choices", [])): + msg_data = choice_data.get("message", {}) + + tool_calls = None + if msg_data.get("tool_calls"): + tool_calls = [ + ToolCall( + id=tc["id"], + type=tc["type"], + function=Function( + name=tc["function"]["name"], + arguments=tc["function"]["arguments"], + ) + ) + for tc in msg_data["tool_calls"] + ] + + message = Message( + role=msg_data.get("role", "assistant"), + content=msg_data.get("content"), + tool_calls=tool_calls, + ) + + choices.append(Choice( + index=i, + message=message, + finish_reason=choice_data.get("finish_reason"), + )) + + usage_data = response_json.get("usage", {}) + usage = Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ) + + return ModelResponse( + id=response_json.get("id", ""), + object=response_json.get("object", "chat.completion"), + created=response_json.get("created", 0), + model=response_json.get("model", ""), + choices=choices, + usage=usage, + ) + + +def completion( + model: str, + messages: List[Dict[str, str]], + tools: Optional[List[Dict]] = None, + tool_choice: Optional[Union[str, Dict]] = None, + max_tokens: int = 4096, + timeout: float = 120.0, + **kwargs +) -> ModelResponse: + """ + Unified completion API for multiple providers. + + Args: + model: Model identifier (e.g., "openai/gpt-4", "claude-sonnet-4-5", "gemini/gemini-2.5-pro") + messages: List of message dicts with 'role' and 'content' + tools: Optional list of tools in OpenAI format + tool_choice: Optional tool choice ("auto", "required", "none", or specific tool) + max_tokens: Maximum tokens in response + timeout: Request timeout in seconds + **kwargs: Additional provider-specific parameters + + Returns: + ModelResponse with OpenAI-compatible structure + """ + provider, model_name = _get_provider(model) + api_key = _get_api_key(provider) + endpoint = ENDPOINTS[provider] + headers = _build_headers(provider, api_key) + + if provider == "anthropic": + body = _build_anthropic_request( + model=model_name, + messages=messages, + tools=tools, + tool_choice=tool_choice, + max_tokens=max_tokens, + **kwargs + ) + else: + # OpenAI-compatible providers + body: Dict[str, Any] = { + "model": model_name, + "messages": messages, + "max_tokens": max_tokens, + } + if tools: + body["tools"] = tools + if tool_choice: + body["tool_choice"] = tool_choice + + with httpx.Client(timeout=timeout) as client: + response = client.post(endpoint, headers=headers, json=body) + response.raise_for_status() + response_json = response.json() + + if provider == "anthropic": + return _parse_anthropic_response(response_json) + else: + return _parse_openai_response(response_json) diff --git a/obsidianki/ai/client.py b/obsidianki/ai/client.py index a2832e4..bb9648f 100644 --- a/obsidianki/ai/client.py +++ b/obsidianki/ai/client.py @@ -1,8 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from litellm import completion - from litellm.types.utils import ModelResponse +from obsidianki.ai.call import completion +from obsidianki.ai.call import ModelResponse import json from typing import List, Dict, Optional, Union, cast @@ -149,22 +147,10 @@ def _call_llm( tool_choice: Union[str, Dict[str, object]], max_tokens: int = 8000 ) -> Optional[ModelResponse]: - """Unified LLM call using litellm""" + """Unified LLM call""" try: - from litellm import completion - from litellm.types.utils import ModelResponse - response = completion( - model=self.model, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt} - ], - tools=tools, - tool_choice=tool_choice, - max_tokens=max_tokens - ) - # We never use streaming, so response is always ModelResponse - return cast(ModelResponse, response) + response = completion(model=self.model, messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}], tools=tools, tool_choice=tool_choice, max_tokens=max_tokens) + return response except Exception as e: console.print(f"[red]ERROR:[/red] LLM call failed: {e}") return None diff --git a/pyproject.toml b/pyproject.toml index 9fddca6..9c90037 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ classifiers = [] dependencies = [ "requests>=2.25.0", "python-dotenv>=0.19.0", - "litellm>=1.8.0", "rich>=13.0.0", "urllib3>=1.26.0", "pygments>=2.10.0", From 2d5b3ddcab32ac85747daee26443b6645b032c1c Mon Sep 17 00:00:00 2001 From: ccmdi Date: Mon, 8 Dec 2025 15:57:23 -0500 Subject: [PATCH 02/16] fix: openai `max_completion_tokens` --- obsidianki/ai/call.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/obsidianki/ai/call.py b/obsidianki/ai/call.py index defaf58..e2c7230 100644 --- a/obsidianki/ai/call.py +++ b/obsidianki/ai/call.py @@ -332,10 +332,11 @@ def completion( ) else: # OpenAI-compatible providers + token_param = "max_completion_tokens" if provider == "openai" else "max_tokens" body: Dict[str, Any] = { "model": model_name, "messages": messages, - "max_tokens": max_tokens, + token_param: max_tokens, } if tools: body["tools"] = tools From 79936e8a146f90d03bfdd45679c32a2eb1ed8422 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Mon, 8 Dec 2025 15:57:48 -0500 Subject: [PATCH 03/16] fix: drop `DEDUPLICATE_VIA_DECK` warning in favor of user prompt --- obsidianki/cli/processors.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/obsidianki/cli/processors.py b/obsidianki/cli/processors.py index 2fba4b6..84be17e 100644 --- a/obsidianki/cli/processors.py +++ b/obsidianki/cli/processors.py @@ -126,9 +126,6 @@ def preprocess(args: argparse.Namespace): CONFIG.show_weights() console.print() - if args.query and not args.notes and CONFIG.deduplicate_via_deck: - console.print("[yellow]WARNING:[/yellow] DEDUPLICATE_VIA_DECK is experimental and may be expensive for large decks\n") - # Test connections if not OBSIDIAN.test_connection(): console.print("[red]ERROR:[/red] Cannot connect to Obsidian REST API") @@ -254,6 +251,11 @@ def preprocess(args: argparse.Namespace): deck_fronts = ANKI.get_card_fronts(CONFIG.deck) if deck_fronts: console.print(f"[dim]Found {len(deck_fronts)} existing cards in deck '{CONFIG.deck}' for deduplication[/dim]") + if len(deck_fronts) > 10: + from rich.prompt import Confirm + if not Confirm.ask(f"Are you sure you want to proceed?", default=False): + console.print("[red]ERROR:[/red] User cancelled") + return 1 previous_fronts = [deck_fronts] * len(notes) # Same fronts for all notes (just the query note) total_cards = 0 From bea8d7388d877a8963e628415573fd6936e06fab Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 09:42:20 -0500 Subject: [PATCH 04/16] feat: add optional vector db for deduplication --- obsidianki/ai/client.py | 194 +++++++++++++++++++++++++++++++++-- obsidianki/ai/tools.py | 16 ++- obsidianki/ai/vectors.py | 191 ++++++++++++++++++++++++++++++++++ obsidianki/cli/config.py | 5 +- obsidianki/cli/processors.py | 7 ++ pyproject.toml | 4 + 6 files changed, 407 insertions(+), 10 deletions(-) create mode 100644 obsidianki/ai/vectors.py diff --git a/obsidianki/ai/client.py b/obsidianki/ai/client.py index bb9648f..5cfc1a7 100644 --- a/obsidianki/ai/client.py +++ b/obsidianki/ai/client.py @@ -10,7 +10,7 @@ from obsidianki.cli.models import Note, Flashcard from obsidianki.ai.models import MODEL_MAP from obsidianki.ai.prompts import SYSTEM_PROMPT, QUERY_SYSTEM_PROMPT, TARGETED_SYSTEM_PROMPT, MULTI_TURN_DQL_AGENT_PROMPT -from obsidianki.ai.tools import FLASHCARD_TOOL, DQL_EXECUTION_TOOL, FINALIZE_SELECTION_TOOL +from obsidianki.ai.tools import FLASHCARD_TOOL, SUBMIT_FLASHCARDS_TOOL, DQL_EXECUTION_TOOL, FINALIZE_SELECTION_TOOL AI_RESULT_SET_SIZE = 20 @@ -212,6 +212,156 @@ def _extract_flashcards_from_response( console.print(f"[red]ERROR:[/red] Failed to parse flashcards: {e}") return [] + def _generate_with_vector_feedback( + self, + system_prompt: str, + user_prompt: str, + note: Note, + default_tags: Optional[List[str]] = None + ) -> List[Flashcard]: + """Generate flashcards with vector similarity feedback loop. + + Multi-turn conversation where: + 1. LLM proposes cards via create_flashcards + 2. System checks against vector DB, returns similarity feedback + 3. LLM can revise or call submit_flashcards to confirm + """ + from obsidianki.ai.vectors import get_vectors + + vectors = get_vectors() + threshold = CONFIG.vector_threshold or 0.85 + max_turns = CONFIG.vector_max_turns or 5 + + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt} + ] + + pending_cards: List[Dict] = [] + + for turn in range(max_turns): + try: + response = completion( + model=self.model, + messages=messages, + tools=[FLASHCARD_TOOL, SUBMIT_FLASHCARDS_TOOL], + tool_choice="required" if turn == 0 else "auto", + max_tokens=8000 + ) + + message = response.choices[0].message + + # Add assistant message to history + messages.append({ + "role": "assistant", + "content": message.content or "", + "tool_calls": message.tool_calls if hasattr(message, 'tool_calls') else None + }) + + if not hasattr(message, 'tool_calls') or not message.tool_calls: + # No tool call - shouldn't happen but handle gracefully + if pending_cards: + return self._convert_pending_to_flashcards(pending_cards, note, default_tags) + break + + tool_call = message.tool_calls[0] + tool_name = tool_call.function.name + + if tool_name == "create_flashcards": + args = json.loads(tool_call.function.arguments) + pending_cards = args.get("flashcards", []) + + # Check each card against vector DB + similar_matches = vectors.find_similar_batch( + [card.get("front", "") for card in pending_cards], + threshold + ) + + if similar_matches: + # Build feedback message + feedback_lines = [] + for idx, front, existing, score in similar_matches: + feedback_lines.append( + f"- Card {idx + 1}: \"{front[:50]}{'...' if len(front) > 50 else ''}\" " + f"≈ \"{existing[:50]}{'...' if len(existing) > 50 else ''}\" ({score:.0%})" + ) + + feedback = ( + f"Similar existing cards found:\n" + f"{chr(10).join(feedback_lines)}\n\n" + f"You may:\n" + f"1. Call create_flashcards again with revised cards that explore different angles\n" + f"2. Call submit_flashcards if you believe these are sufficiently distinct" + ) + console.print(f"[yellow]Vector feedback:[/yellow] {len(similar_matches)} similar card(s) found") + for line in feedback_lines: + console.print(f"[dim]{line}[/dim]") + else: + feedback = "No similar cards found in the database. Call submit_flashcards to confirm." + console.print("[green]Vector check:[/green] No similar cards found") + + messages.append({ + "tool_call_id": tool_call.id, + "role": "tool", + "name": tool_name, + "content": feedback + }) + + elif tool_name == "submit_flashcards": + # Finalize submission + messages.append({ + "tool_call_id": tool_call.id, + "role": "tool", + "name": tool_name, + "content": f"{len(pending_cards)} cards submitted." + }) + console.print(f"[green]Submitted:[/green] {len(pending_cards)} cards") + return self._convert_pending_to_flashcards(pending_cards, note, default_tags) + + except Exception as e: + console.print(f"[red]ERROR:[/red] Vector feedback loop failed: {e}") + if pending_cards: + return self._convert_pending_to_flashcards(pending_cards, note, default_tags) + return [] + + # Max turns reached - return whatever we have + if pending_cards: + console.print(f"[yellow]Max turns reached:[/yellow] Submitting {len(pending_cards)} pending cards") + return self._convert_pending_to_flashcards(pending_cards, note, default_tags) + + return [] + + def _convert_pending_to_flashcards( + self, + pending_cards: List[Dict], + note: Note, + default_tags: Optional[List[str]] = None + ) -> List[Flashcard]: + """Convert pending card dicts to Flashcard objects with processing.""" + flashcard_objects = [] + for card in pending_cards: + front_original = card.get('front', '') + back_original = card.get('back', '') + + # Process code blocks with syntax highlighting + front_processed = process_code_blocks(front_original, CONFIG.syntax_highlighting) + back_processed = process_code_blocks(back_original, CONFIG.syntax_highlighting) + + # Determine tags priority: card's tags > default_tags > note's tags + tags = card.get('tags') or default_tags or note.tags.copy() + + flashcard = Flashcard( + front=front_processed, + back=back_processed, + note=note, + tags=tags, + front_original=front_original, + back_original=back_original + ) + flashcard_objects.append(flashcard) + + return flashcard_objects + def generate_flashcards( self, note: Note, @@ -232,6 +382,15 @@ def generate_flashcards( Please analyze this note and {card_instruction} for the key information that would be valuable for spaced repetition learning.""" + # Use vector feedback loop if enabled + if CONFIG.vector_dedup: + return self._generate_with_vector_feedback( + system_prompt=SYSTEM_PROMPT, + user_prompt=user_prompt, + note=note + ) + + # Original single-shot behavior response = self._call_llm( system_prompt=SYSTEM_PROMPT, user_prompt=user_prompt, @@ -258,13 +417,6 @@ def generate_from_query( Please {card_instruction} to help someone learn about this topic. Focus on the most important concepts, definitions, and practical information related to this query.{difficulty_context}{dedup_context}{schema_context}""" - response = self._call_llm( - system_prompt=QUERY_SYSTEM_PROMPT, - user_prompt=user_prompt, - tools=[FLASHCARD_TOOL], - tool_choice=self._get_tool_choice("create_flashcards") - ) - # Create virtual Note object for query-based flashcards virtual_note = Note( path="query", @@ -274,6 +426,23 @@ def generate_from_query( size=0 ) + # Use vector feedback loop if enabled + if CONFIG.vector_dedup: + return self._generate_with_vector_feedback( + system_prompt=QUERY_SYSTEM_PROMPT, + user_prompt=user_prompt, + note=virtual_note, + default_tags=["query-generated"] + ) + + # Original single-shot behavior + response = self._call_llm( + system_prompt=QUERY_SYSTEM_PROMPT, + user_prompt=user_prompt, + tools=[FLASHCARD_TOOL], + tool_choice=self._get_tool_choice("create_flashcards") + ) + return self._extract_flashcards_from_response(response, virtual_note, default_tags=["query-generated"]) def generate_from_note_query(self, note: Note, query: str, target_cards: int, previous_fronts: List[str] | None = None, deck_examples: List[Dict[str, str]] | None = None) -> List[Flashcard]: @@ -296,6 +465,15 @@ def generate_from_note_query(self, note: Note, query: str, target_cards: int, pr Please analyze this note and extract information specifically related to the query "{query}". {card_instruction} only for information in the note that directly addresses or relates to this query.""" + # Use vector feedback loop if enabled + if CONFIG.vector_dedup: + return self._generate_with_vector_feedback( + system_prompt=TARGETED_SYSTEM_PROMPT, + user_prompt=user_prompt, + note=note + ) + + # Original single-shot behavior response = self._call_llm( system_prompt=TARGETED_SYSTEM_PROMPT, user_prompt=user_prompt, diff --git a/obsidianki/ai/tools.py b/obsidianki/ai/tools.py index 1996820..7bd1647 100644 --- a/obsidianki/ai/tools.py +++ b/obsidianki/ai/tools.py @@ -2,7 +2,7 @@ "type": "function", "function": { "name": "create_flashcards", - "description": "Create flashcards from note content with front (question) and back (answer)", + "description": "Propose flashcards from note content. In vector mode, returns similarity feedback before final submission.", "parameters": { "type": "object", "properties": { @@ -30,6 +30,20 @@ } } +# Submit tool for vector dedup mode - confirms flashcard submission after similarity review +SUBMIT_FLASHCARDS_TOOL: dict = { + "type": "function", + "function": { + "name": "submit_flashcards", + "description": "Confirm and submit the last proposed flashcards. Call this after reviewing similarity feedback to finalize submission.", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } +} + # DQL Execution Tool for multi-turn agent DQL_EXECUTION_TOOL: dict = { "type": "function", diff --git a/obsidianki/ai/vectors.py b/obsidianki/ai/vectors.py new file mode 100644 index 0000000..aaac879 --- /dev/null +++ b/obsidianki/ai/vectors.py @@ -0,0 +1,191 @@ +"""Vector-based semantic deduplication for flashcards. + +Uses ChromaDB for storage and sentence-transformers for local embeddings. +Provides a feedback loop where the LLM can revise cards based on similarity. +""" +from __future__ import annotations +import hashlib +from pathlib import Path +from typing import List, Optional, Tuple, TYPE_CHECKING + +if TYPE_CHECKING: + from chromadb.api.models.Collection import Collection + from chromadb import ClientAPI + +from obsidianki.cli.config import CONFIG_DIR, console + +VECTORS_DIR = CONFIG_DIR / "vectors" + + +class VectorStore: + """Lazy-loaded vector store for flashcard semantic deduplication.""" + + def __init__(self): + self._client: Optional[ClientAPI] = None + self._collection: Optional[Collection] = None + self._model: Optional[BaseEmbedder] = None + + @property + def collection(self) -> Collection: + """Lazy-load ChromaDB collection.""" + if self._collection is None: + try: + import chromadb + except ImportError: + raise ImportError( + "ChromaDB is required for vector deduplication. " + "Install with: pip install chromadb" + ) + + VECTORS_DIR.mkdir(parents=True, exist_ok=True) + self._client = chromadb.PersistentClient(path=str(VECTORS_DIR)) + self._collection = self._client.get_or_create_collection( + name="flashcards", + metadata={"hnsw:space": "cosine"} + ) + return self._collection + + @property + def model(self) -> BaseEmbedder: + """Lazy-load embedding model.""" + if self._model is None: + self._model = LocalEmbedder() + return self._model + + def add(self, fronts: List[str]) -> None: + """Index flashcard fronts.""" + if not fronts: + return + + # Filter out empty strings and duplicates + fronts = [f for f in fronts if f.strip()] + if not fronts: + return + + embeddings = self.model.embed(fronts) + ids = [self._hash(f) for f in fronts] + + # Upsert to handle duplicates + self.collection.upsert( + ids=ids, + embeddings=embeddings, + documents=fronts + ) + + def find_similar(self, front: str, threshold: float) -> Optional[Tuple[str, float]]: + """Find most similar existing card above threshold. + + Args: + front: The flashcard front text to check + threshold: Minimum cosine similarity (0-1) to consider a match + + Returns: + Tuple of (similar_front, similarity_score) or None if no match + """ + if self.collection.count() == 0: + return None + + # Don't match against itself + front_id = self._hash(front) + + results = self.collection.query( + query_embeddings=[self.model.embed([front])[0]], + n_results=2, # Get 2 in case first is itself + include=["documents", "distances"] + ) + + if not results["documents"] or not results["documents"][0]: + return None + + # Find best match that isn't the same card + for i, doc in enumerate(results["documents"][0]): + doc_id = self._hash(doc) + if doc_id == front_id: + continue + + # ChromaDB returns cosine distance, convert to similarity + distance = results["distances"][0][i] + similarity = 1 - distance + + if similarity >= threshold: + return (doc, similarity) + + return None + + def find_similar_batch( + self, + fronts: List[str], + threshold: float + ) -> List[Tuple[int, str, str, float]]: + """Check multiple fronts for similarity. + + Args: + fronts: List of flashcard front texts to check + threshold: Minimum cosine similarity to flag + + Returns: + List of (index, front, similar_existing, similarity) for matches only + """ + matches = [] + for i, front in enumerate(fronts): + similar = self.find_similar(front, threshold) + if similar: + existing, score = similar + matches.append((i, front, existing, score)) + return matches + + def count(self) -> int: + """Number of indexed cards.""" + return self.collection.count() + + def clear(self) -> None: + """Clear all indexed cards.""" + if self._client is not None: + self._client.delete_collection("flashcards") + self._collection = None + + def _hash(self, text: str) -> str: + """Generate stable ID for text.""" + return hashlib.sha256(text.encode()).hexdigest()[:16] + + +class BaseEmbedder: + """Base class for embedding providers.""" + + def embed(self, texts: List[str]) -> List[List[float]]: + raise NotImplementedError + + +class LocalEmbedder(BaseEmbedder): + """Local embeddings using sentence-transformers.""" + + def __init__(self): + self._model = None + + @property + def model(self): + if self._model is None: + try: + from sentence_transformers import SentenceTransformer + except ImportError: + raise ImportError( + "sentence-transformers is required for vector deduplication. " + "Install with: pip install sentence-transformers" + ) + self._model = SentenceTransformer('all-MiniLM-L6-v2') + return self._model + + def embed(self, texts: List[str]) -> List[List[float]]: + return self.model.encode(texts).tolist() + + +# Global lazy instance +_VECTORS: Optional[VectorStore] = None + + +def get_vectors() -> VectorStore: + """Get or create the global vector store.""" + global _VECTORS + if _VECTORS is None: + _VECTORS = VectorStore() + return _VECTORS diff --git a/obsidianki/cli/config.py b/obsidianki/cli/config.py index fe18eba..06a6e8b 100644 --- a/obsidianki/cli/config.py +++ b/obsidianki/cli/config.py @@ -38,7 +38,10 @@ "UPFRONT_BATCHING": False, # Process all notes in parallel instead of one-by-one "BATCH_SIZE_LIMIT": 20, # Maximum notes to process in batch mode "BATCH_CARD_LIMIT": 100, # Maximum total cards in batch mode - "MODEL": "Claude Sonnet 4.5" # AI model to use (Claude Sonnet 4, GPT-5, Gemini 3 Pro Preview, etc.) + "MODEL": "Claude Sonnet 4.5", # AI model to use (Claude Sonnet 4, GPT-5, Gemini 3 Pro Preview, etc.) + "VECTOR_DEDUP": False, # Enable vector-based semantic deduplication with feedback loop + "VECTOR_THRESHOLD": 0.85, # Similarity threshold (0-1) to flag as potential duplicate + "VECTOR_MAX_TURNS": 5, # Max revision attempts in vector feedback loop } class Config: diff --git a/obsidianki/cli/processors.py b/obsidianki/cli/processors.py index 84be17e..dae04c9 100644 --- a/obsidianki/cli/processors.py +++ b/obsidianki/cli/processors.py @@ -74,6 +74,13 @@ def postprocess(note: Note, flashcards: List[Flashcard], deck_name: str): if note.path != "query": #TODO flashcard_fronts = [fc.front for fc in cards_to_add[:successful_cards]] CONFIG.record_flashcards_created(note, successful_cards, flashcard_fronts) + + # Index new cards in vector store for semantic deduplication + if CONFIG.vector_dedup: + from obsidianki.ai.vectors import get_vectors + fronts_to_index = [fc.front_original for fc in cards_to_add[:successful_cards]] + get_vectors().add(fronts_to_index) + return successful_cards else: console.print(f"[red]ERROR:[/red] Failed to add cards to Anki for {note.filename}") diff --git a/pyproject.toml b/pyproject.toml index 9c90037..47d655e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,10 @@ dev = [ "pytest>=7.0.0", "pytest-mock>=3.10.0" ] +vectors = [ + "chromadb>=0.4.0", + "sentence-transformers>=2.0.0" +] [project.scripts] obsidianki = "obsidianki.main:main" From 24cebc7fd69917e953c8946488b5471712e2d545 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 09:48:00 -0500 Subject: [PATCH 05/16] feat: default obsidian search to jsonlogic instead of dql --- obsidianki/api/obsidian.py | 217 ++++++++++++++++++++++++------------- obsidianki/cli/models.py | 49 ++++++++- tests/test_models.py | 4 +- 3 files changed, 191 insertions(+), 79 deletions(-) diff --git a/obsidianki/api/obsidian.py b/obsidianki/api/obsidian.py index a836d40..b0a1988 100644 --- a/obsidianki/api/obsidian.py +++ b/obsidianki/api/obsidian.py @@ -1,7 +1,7 @@ import os import urllib3 from datetime import datetime, timedelta -from typing import List +from typing import List, Dict, Any, Optional from obsidianki.cli.config import console, CONFIG from obsidianki.cli.models import Note @@ -11,6 +11,7 @@ OBSIDIAN_TIMEOUT_LENGTH = 30 + class ObsidianAPI(BaseAPI): def __init__(self): super().__init__("https://127.0.0.1:27124", OBSIDIAN_TIMEOUT_LENGTH) @@ -24,42 +25,28 @@ def __init__(self): "Content-Type": "application/json" } - def _build_filters(self, search_folders=None) -> str: - """Build combined DQL filter conditions""" - filters = [] - - # Folder filter - if search_folders: - folder_conditions = [f'startswith(file.path, "{folder}/")' for folder in search_folders] - filters.append(f"({' OR '.join(folder_conditions)})") - - # Excluded tags filter - if CONFIG and CONFIG.excluded_tags: - exclude_conditions = [f'!contains(file.tags, "{tag}")' for tag in CONFIG.excluded_tags] - filters.append(f"({' AND '.join(exclude_conditions)})") - - return f"AND {' AND '.join(filters)}" if filters else "" - - def _build_base_query(self, extra_conditions="", sort_field="file.mtime", sort_order="ASC") -> str: - """Build standard DQL query structure""" - return f"""TABLE - file.name AS "filename", - file.path AS "path", - file.mtime AS "mtime", - file.size AS "size", - file.tags AS "tags" - FROM "" - WHERE {extra_conditions} - SORT {sort_field} {sort_order}""" + def search(self, query: Dict[str, Any]) -> List[Note]: + """Search notes using JsonLogic query - returns Note objects""" + headers = { + **self.headers, + "Content-Type": "application/vnd.olrapi.jsonlogic+json" + } - def _make_obsidian_request(self, endpoint: str, method: str = "GET", data: dict = {}): - """Make a request to the Obsidian REST API, ignoring SSL verification""" - url = f"{self.base_url}{endpoint}" - response = super()._make_request(method, url, json=data, verify=False) - return self._parse_response(response) + try: + url = f"{self.base_url}/search/" + response = super()._make_request("POST", url, headers=headers, json=query, verify=False) + results = self._parse_response(response) + + return [Note.from_jsonlogic_result(r) for r in results] + except Exception as e: + raise def dql(self, query: str) -> List[Note]: - """Search notes using Dataview DQL query - returns Note objects""" + """Search notes using Dataview DQL query - returns Note objects. + + Note: Requires the Dataview plugin to be installed in Obsidian. + Used primarily by agent mode (--agent) which generates DQL queries dynamically. + """ headers = { **self.headers, "Content-Type": "application/vnd.olrapi.dataview.dql+txt" @@ -70,40 +57,108 @@ def dql(self, query: str) -> List[Note]: response = super()._make_request("POST", url, headers=headers, data=query, verify=False) dict_results = self._parse_response(response) - return [Note.from_obsidian_result(result) for result in dict_results] + return [Note.from_dql_result(result) for result in dict_results] except Exception as e: raise + def _build_folder_filter(self, search_folders: Optional[List[str]] = None) -> Optional[Dict]: + """Build JsonLogic filter for folder restrictions""" + folders = search_folders or [] + if not folders: + return None + + if len(folders) == 1: + return {"glob": [f"{folders[0]}/*", {"var": "path"}]} + + return { + "or": [ + {"glob": [f"{folder}/*", {"var": "path"}]} + for folder in folders + ] + } + + def _build_excluded_tags_filter(self) -> Optional[Dict]: + """Build JsonLogic filter to exclude notes with certain tags""" + if not CONFIG or not CONFIG.excluded_tags: + return None + + # None of the excluded tags should be present + return { + "and": [ + {"!": {"in": [tag, {"var": "tags"}]}} + for tag in CONFIG.excluded_tags + ] + } + + def _combine_filters(self, *filters) -> Dict: + """Combine multiple JsonLogic filters with AND, returning full note object on match""" + valid_filters = [f for f in filters if f is not None] + + if not valid_filters: + # Match all - return full object + return {"var": ""} + + if len(valid_filters) == 1: + condition = valid_filters[0] + else: + condition = {"and": valid_filters} + + # Wrap in if to return full note object when condition matches + return { + "if": [ + condition, + {"var": ""}, # Return full note object on match + None # Return null (falsy) on no match + ] + } + + def _make_obsidian_request(self, endpoint: str, method: str = "GET", data: dict = {}): + """Make a request to the Obsidian REST API, ignoring SSL verification""" + url = f"{self.base_url}{endpoint}" + response = super()._make_request(method, url, json=data, verify=False) + return self._parse_response(response) + def get_old_notes(self, days: int, limit: int = 0) -> List[Note]: """Get notes older than specified days""" cutoff_date = datetime.now() - timedelta(days=days) - cutoff_str = cutoff_date.strftime("%Y-%m-%d") + cutoff_ms = int(cutoff_date.timestamp() * 1000) - filters = self._build_filters(CONFIG.search_folders) + query = self._combine_filters( + {"<": [{"var": "stat.mtime"}, cutoff_ms]}, + {">": [{"var": "stat.size"}, 100]}, + self._build_folder_filter(CONFIG.search_folders), + self._build_excluded_tags_filter() + ) - condition = f'file.mtime < date("{cutoff_str}") {filters}' - query = self._build_base_query(condition) + results = self.search(query) - if limit: - query += f"\nLIMIT {limit}" + if limit and len(results) > limit: + return results[:limit] - return self.dql(query) + return results def get_tagged_notes(self, tags: List[str], exclude_recent_days: int = 0) -> List[Note]: """Get notes with specific tags""" - tag_conditions = " OR ".join([f'contains(file.tags, "{tag}")' for tag in tags]) - filters = self._build_filters(CONFIG.search_folders) + # At least one of the tags should be present + tag_filter = { + "or": [ + {"in": [tag, {"var": "tags"}]} + for tag in tags + ] + } - condition = f'({tag_conditions})' + filters = [tag_filter] if exclude_recent_days > 0: cutoff_date = datetime.now() - timedelta(days=exclude_recent_days) - cutoff_str = cutoff_date.strftime("%Y-%m-%d") - condition += f' AND file.mtime < date("{cutoff_str}")' + cutoff_ms = int(cutoff_date.timestamp() * 1000) + filters.append({"<": [{"var": "stat.mtime"}, cutoff_ms]}) - condition += f' {filters}' + filters.append(self._build_folder_filter(CONFIG.search_folders)) + filters.append(self._build_excluded_tags_filter()) - return self.dql(self._build_base_query(condition)) + query = self._combine_filters(*filters) + return self.search(query) def get_note_content(self, note_path: str) -> str: """Get the content of a specific note""" @@ -115,11 +170,16 @@ def get_note_content(self, note_path: str) -> str: def sample_old_notes(self, days: int, limit: int = 0, bias_strength: float = 0.0, search_folders: List[str] = []) -> List[Note]: """Sample old notes with optional weighting""" cutoff_date = datetime.now() - timedelta(days=days) - cutoff_str = cutoff_date.strftime("%Y-%m-%d") - filters = self._build_filters(search_folders) + cutoff_ms = int(cutoff_date.timestamp() * 1000) - condition = f'file.mtime < date("{cutoff_str}") AND file.size > 100 {filters}' - all_notes = self.dql(self._build_base_query(condition)) + query = self._combine_filters( + {"<": [{"var": "stat.mtime"}, cutoff_ms]}, + {">": [{"var": "stat.size"}, 100]}, + self._build_folder_filter(search_folders), + self._build_excluded_tags_filter() + ) + + all_notes = self.search(query) if not all_notes: return [] @@ -132,7 +192,6 @@ def sample_old_notes(self, days: int, limit: int = 0, bias_strength: float = 0.0 if not limit or len(all_notes) <= limit: return all_notes - # Use weighted sampling by default (since we have global config) return self._weighted_sample(all_notes, limit, bias_strength) def _weighted_sample(self, notes: List[Note], limit: int, bias_strength: float = 0.0) -> List[Note]: @@ -153,7 +212,7 @@ def _weighted_sample(self, notes: List[Note], limit: int, bias_strength: float = chosen_idx = available_notes.index(chosen) sampled_notes.append(chosen) - + available_notes.pop(chosen_idx) available_weights.pop(chosen_idx) @@ -161,25 +220,33 @@ def _weighted_sample(self, notes: List[Note], limit: int, bias_strength: float = def find_by_pattern(self, pattern: str, sample_size: int = 0, bias_strength: float = 0.0, search_folders: List[str] = []) -> List[Note]: """Find notes by pattern""" - filters = self._build_filters(search_folders) - - # Build pattern condition + # Build pattern condition using glob if pattern.endswith('/*'): + # Directory pattern: frontend/* directory_path = pattern[:-2] - condition = f'startswith(file.path, "{directory_path}/")' + pattern_filter = {"glob": [f"{directory_path}/*", {"var": "path"}]} elif '*' in pattern: - if pattern.startswith('*'): - condition = f'endswith(file.path, "{pattern[1:]}")' - elif pattern.endswith('*'): - condition = f'startswith(file.path, "{pattern[:-1]}")' - else: - parts = [f'contains(file.path, "{part}")' for part in pattern.split('*') if part] - condition = ' AND '.join(parts) if parts else 'true' + # Glob pattern - convert to glob syntax + # Handle patterns like "react*", "*hooks", "react*hooks" + glob_pattern = pattern if pattern.endswith('*') or pattern.startswith('*') else f"*{pattern}*" + pattern_filter = {"glob": [glob_pattern, {"var": "path"}]} else: - condition = f'(file.path = "{pattern}" OR contains(file.name, "{pattern}"))' - - full_condition = f'{condition} AND file.size > 100 {filters}' - results = self.dql(self._build_base_query(full_condition)) + # Exact match or name contains + pattern_filter = { + "or": [ + {"===": [{"var": "path"}, pattern]}, + {"glob": [f"*{pattern}*", {"var": "basename"}]} + ] + } + + query = self._combine_filters( + pattern_filter, + {">": [{"var": "stat.size"}, 100]}, + self._build_folder_filter(search_folders), + self._build_excluded_tags_filter() + ) + + results = self.search(query) if not results: return [] @@ -200,10 +267,13 @@ def find_by_pattern(self, pattern: str, sample_size: int = 0, bias_strength: flo def find_by_name(self, note_name: str, search_folders: List[str]) -> Note | None: """Find note by name with partial matching""" - filters = self._build_filters(search_folders) + query = self._combine_filters( + {"glob": [f"*{note_name}*", {"var": "basename"}]}, + self._build_folder_filter(search_folders), + self._build_excluded_tags_filter() + ) - condition = f'contains(file.name, "{note_name}") {filters}' - results = self.dql(self._build_base_query(condition, sort_field="file.name")) + results = self.search(query) if not results: return None @@ -225,4 +295,3 @@ def test_connection(self) -> bool: return True except Exception: return False - diff --git a/obsidianki/cli/models.py b/obsidianki/cli/models.py index 94b3d21..ccd55cd 100644 --- a/obsidianki/cli/models.py +++ b/obsidianki/cli/models.py @@ -68,9 +68,9 @@ def to_obsidian_link_rich(self) -> str: return f"[link={self.to_obsidian_uri()}]{self.path}[/link]" @classmethod - def from_obsidian_result(cls, obsidian_result: Dict[str, Any], content: str = "") -> 'Note': - """Create Note from Obsidian API result format.""" - result = obsidian_result.get('result', obsidian_result) + def from_dql_result(cls, dql_result: Dict[str, Any], content: str = "") -> 'Note': + """Create Note from Obsidian API result format (DQL).""" + result = dql_result.get('result', dql_result) return cls( path=result['path'], filename=result['filename'], @@ -79,6 +79,49 @@ def from_obsidian_result(cls, obsidian_result: Dict[str, Any], content: str = "" size=result.get('size', 0) ) + @classmethod + def from_jsonlogic_result(cls, jsonlogic_result: Dict[str, Any], content: str = "") -> 'Note': + """Create Note from Obsidian API JsonLogic result format. + + JsonLogic response format: + { + "filename": "path/to/note.md", + "result": { ... full note object ... } + } + + The result contains the full note object with: + - path: full file path + - basename: filename without extension + - stat.mtime: modification time (ms) + - stat.size: file size (bytes) + - tags: array of tags + """ + filename = jsonlogic_result.get('filename', '') + result = jsonlogic_result.get('result', {}) + + # Handle case where result is the full note object + if isinstance(result, dict): + path = result.get('path', filename) + # Get filename from path or basename + name = path.split('/')[-1] if path else filename.split('/')[-1] + stat = result.get('stat', {}) + tags = result.get('tags', []) + size = stat.get('size', 0) + else: + # Fallback if result is just True or simple value + path = filename + name = filename.split('/')[-1] if filename else '' + tags = [] + size = 0 + + return cls( + path=path, + filename=name, + content=content or "", + tags=tags if tags else [], + size=size + ) + class NotePattern: """ diff --git a/tests/test_models.py b/tests/test_models.py index 19b6f45..9c6724d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -34,7 +34,7 @@ def test_note_from_obsidian_result(self): } # from_obsidian_result accepts optional content parameter - note = Note.from_obsidian_result(result, content="Content here") + note = Note.from_dql_result(result, content="Content here") assert note.filename == "My Note" assert note.path == "folder/my_note.md" @@ -288,7 +288,7 @@ def test_note_from_obsidian_result_with_defaults(self): } # Should handle missing optional fields with defaults - note = Note.from_obsidian_result(result, content="test") + note = Note.from_dql_result(result, content="test") assert note.filename == "Minimal" assert note.path == "minimal.md" assert note.content == "test" From ee9196df5ecca1d2eed9228ece05fde245f52083 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:06:11 -0500 Subject: [PATCH 06/16] feat: add vector command --- README.md | 45 +++++++ obsidianki/ai/client.py | 27 +++- obsidianki/ai/vectors.py | 30 ++++- obsidianki/cli/commands/__init__.py | 2 + obsidianki/cli/commands/vector_cmd.py | 171 ++++++++++++++++++++++++++ 5 files changed, 271 insertions(+), 4 deletions(-) create mode 100644 obsidianki/cli/commands/vector_cmd.py diff --git a/README.md b/README.md index 6556cd9..e689ffb 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,51 @@ oki template use programming # runs the above command as "oki --notes 'frontend/ | `batch_card_limit` | `100` | Max cards per batch | | `density_bias_strength` | `0.5` | Bias strength against over-processed notes (0-1) | | `search_folders` | `[]` | Limit processing to specific folders (array) | +| `vector_dedup` | `false` | Enable semantic deduplication via embeddings | +| `vector_threshold` | `0.85` | Similarity threshold for duplicate detection (0-1) | + +## Vector Deduplication + +Avoid generating semantically similar flashcards using local embeddings. + +### Install + +```bash +pip install obsidianki[vectors] +# or +uv tool install obsidianki --with chromadb --with sentence-transformers +``` + +### Enable + +```bash +oki config set vector_dedup true +``` + +### Index existing cards + +```bash +oki vector index # Index cards from default deck +oki vector index --deck "My Deck" # Index cards from specific deck +``` + +### Commands + +```bash +oki vector status # Show index stats +oki vector check "question text" # Check if similar card exists +oki vector clear # Clear the index +``` + +### How it works + +1. AI proposes flashcards via `create_flashcards` tool +2. Each card is checked against the vector database for semantic similarity +3. If similar cards exist, AI receives feedback: *"Card 2 is 91% similar to 'What is polymorphism?'"* +4. AI can revise or confirm via `submit_flashcards` tool +5. Accepted cards are indexed for future deduplication + +The vector database is stored in `~/.config/obsidianki/vectors/`. # MCP There is an [experimental MCP server](https://github.com/ccmdi/obsidianki-mcp) that runs Obsidianki as a subprocess. Useful if you want to generate flashcards from daily use with an LLM, such as if you ask questions back and forth and want to generate flashcards from that material. \ No newline at end of file diff --git a/obsidianki/ai/client.py b/obsidianki/ai/client.py index 5cfc1a7..763c08c 100644 --- a/obsidianki/ai/client.py +++ b/obsidianki/ai/client.py @@ -212,6 +212,22 @@ def _extract_flashcards_from_response( console.print(f"[red]ERROR:[/red] Failed to parse flashcards: {e}") return [] + def _serialize_tool_calls(self, tool_calls) -> Optional[List[Dict]]: + """Serialize tool calls to JSON-compatible format.""" + if not tool_calls: + return None + return [ + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.function.name, + "arguments": tc.function.arguments + } + } + for tc in tool_calls + ] + def _generate_with_vector_feedback( self, system_prompt: str, @@ -232,6 +248,13 @@ def _generate_with_vector_feedback( threshold = CONFIG.vector_threshold or 0.85 max_turns = CONFIG.vector_max_turns or 5 + # Show vector index status + index_count = vectors.count() + if index_count == 0: + console.print("[dim]Vector index empty - no similarity checks will match[/dim]") + else: + console.print(f"[dim]Vector index: {index_count} cards indexed[/dim]") + messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt} @@ -251,11 +274,11 @@ def _generate_with_vector_feedback( message = response.choices[0].message - # Add assistant message to history + # Add assistant message to history (serialize tool_calls to dict) messages.append({ "role": "assistant", "content": message.content or "", - "tool_calls": message.tool_calls if hasattr(message, 'tool_calls') else None + "tool_calls": self._serialize_tool_calls(message.tool_calls) if hasattr(message, 'tool_calls') else None }) if not hasattr(message, 'tool_calls') or not message.tool_calls: diff --git a/obsidianki/ai/vectors.py b/obsidianki/ai/vectors.py index aaac879..ded30fd 100644 --- a/obsidianki/ai/vectors.py +++ b/obsidianki/ai/vectors.py @@ -62,6 +62,7 @@ def add(self, fronts: List[str]) -> None: if not fronts: return + console.print(f"[dim]Indexing {len(fronts)} card(s) in vector store...[/dim]") embeddings = self.model.embed(fronts) ids = [self._hash(f) for f in fronts] @@ -71,6 +72,7 @@ def add(self, fronts: List[str]) -> None: embeddings=embeddings, documents=fronts ) + console.print(f"[dim]Vector index now has {self.count()} cards[/dim]") def find_similar(self, front: str, threshold: float) -> Optional[Tuple[str, float]]: """Find most similar existing card above threshold. @@ -161,22 +163,46 @@ class LocalEmbedder(BaseEmbedder): def __init__(self): self._model = None + self._loaded = False + + def _get_device(self) -> str: + """Detect best available device (cuda > mps > cpu).""" + try: + import torch + if torch.cuda.is_available(): + return "cuda" + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + return "mps" + except ImportError: + pass + return "cpu" @property def model(self): if self._model is None: try: + # Suppress noisy logging from transformers/torch + import logging + import warnings + logging.getLogger("sentence_transformers").setLevel(logging.WARNING) + logging.getLogger("transformers").setLevel(logging.WARNING) + warnings.filterwarnings("ignore", message=".*position_ids.*") + from sentence_transformers import SentenceTransformer except ImportError: raise ImportError( "sentence-transformers is required for vector deduplication. " "Install with: pip install sentence-transformers" ) - self._model = SentenceTransformer('all-MiniLM-L6-v2') + device = self._get_device() + if not self._loaded: + console.print(f"[dim]Loading embedding model ({device})...[/dim]") + self._model = SentenceTransformer('all-MiniLM-L6-v2', device=device) + self._loaded = True return self._model def embed(self, texts: List[str]) -> List[List[float]]: - return self.model.encode(texts).tolist() + return self.model.encode(texts, show_progress_bar=False).tolist() # Global lazy instance diff --git a/obsidianki/cli/commands/__init__.py b/obsidianki/cli/commands/__init__.py index 94943ff..e9552e2 100644 --- a/obsidianki/cli/commands/__init__.py +++ b/obsidianki/cli/commands/__init__.py @@ -7,6 +7,7 @@ from obsidianki.cli.commands.template_cmd import COMMAND as template_command from obsidianki.cli.commands.hide_cmd import COMMAND as hide_command from obsidianki.cli.commands.edit_cmd import COMMAND as edit_command +from obsidianki.cli.commands.vector_cmd import COMMAND as vector_command ALL_COMMANDS = [ config_command, @@ -16,4 +17,5 @@ template_command, hide_command, edit_command, + vector_command, ] diff --git a/obsidianki/cli/commands/vector_cmd.py b/obsidianki/cli/commands/vector_cmd.py new file mode 100644 index 0000000..85ba46e --- /dev/null +++ b/obsidianki/cli/commands/vector_cmd.py @@ -0,0 +1,171 @@ +"""Vector index management commands.""" +import argparse +from obsidianki.cli.config import console, CONFIG + + +def setup_parser(subparsers): + """Set up the vector subcommand parser.""" + vector_parser = subparsers.add_parser( + "vector", + help="Manage vector index for semantic deduplication" + ) + + vector_subparsers = vector_parser.add_subparsers(dest="vector_action") + + # vector index + index_parser = vector_subparsers.add_parser( + "index", + help="Index existing Anki cards into vector store" + ) + index_parser.add_argument( + "--deck", + type=str, + default=None, + help="Anki deck to index (defaults to configured deck)" + ) + + # vector status + vector_subparsers.add_parser( + "status", + help="Show vector index status" + ) + + # vector clear + vector_subparsers.add_parser( + "clear", + help="Clear the vector index" + ) + + # vector check + check_parser = vector_subparsers.add_parser( + "check", + help="Check if a question is similar to existing cards" + ) + check_parser.add_argument( + "question", + type=str, + help="Question text to check for similarity" + ) + + return vector_parser + + +def handler(args: argparse.Namespace): + """Handle vector subcommands.""" + if not CONFIG.vector_dedup: + console.print("[yellow]Vector deduplication is disabled.[/yellow]") + console.print("Enable with: [cyan]oki config set vector_dedup true[/cyan]") + return + + action = getattr(args, "vector_action", None) + + if action == "index": + _handle_index(args) + elif action == "status": + _handle_status() + elif action == "clear": + _handle_clear() + elif action == "check": + _handle_check(args) + else: + console.print("Usage: oki vector [index|status|clear|check]") + console.print("Run 'oki vector --help' for more information") + + +def _handle_index(args: argparse.Namespace): + """Index existing Anki cards into vector store.""" + from obsidianki.cli.services import ANKI + from obsidianki.ai.vectors import get_vectors + + deck = args.deck or CONFIG.deck + if not deck: + console.print("[red]ERROR:[/red] No deck specified. Use --deck or set default deck.") + return + + console.print(f"[cyan]Fetching cards from deck:[/cyan] {deck}") + + try: + fronts = ANKI.get_card_fronts(deck) + except Exception as e: + console.print(f"[red]ERROR:[/red] Failed to fetch cards from Anki: {e}") + return + + if not fronts: + console.print("[yellow]No cards found in deck.[/yellow]") + return + + console.print(f"[cyan]Found {len(fronts)} cards to index[/cyan]") + + vectors = get_vectors() + existing_count = vectors.count() + + # Index in batches for visibility + batch_size = 50 + for i in range(0, len(fronts), batch_size): + batch = fronts[i:i + batch_size] + vectors.add(batch) + console.print(f"[dim]Indexed {min(i + batch_size, len(fronts))}/{len(fronts)}[/dim]") + + new_count = vectors.count() + added = new_count - existing_count + console.print(f"[green]Done![/green] Added {added} new cards to vector index (total: {new_count})") + + +def _handle_status(): + """Show vector index status.""" + from obsidianki.ai.vectors import get_vectors + + vectors = get_vectors() + count = vectors.count() + + console.print(f"[cyan]Vector index status[/cyan]") + console.print(f" Cards indexed: {count}") + console.print(f" Threshold: {CONFIG.vector_threshold}") + console.print(f" Max turns: {CONFIG.vector_max_turns}") + + +def _handle_clear(): + """Clear the vector index.""" + from obsidianki.ai.vectors import get_vectors + + vectors = get_vectors() + count = vectors.count() + + if count == 0: + console.print("[yellow]Vector index is already empty.[/yellow]") + return + + vectors.clear() + console.print(f"[green]Cleared {count} cards from vector index.[/green]") + + +def _handle_check(args: argparse.Namespace): + """Check if a question is similar to existing cards.""" + from obsidianki.ai.vectors import get_vectors + + vectors = get_vectors() + question = args.question + threshold = CONFIG.vector_threshold or 0.85 + + if vectors.count() == 0: + console.print("[yellow]Vector index is empty. Run 'oki vector index' first.[/yellow]") + return + + console.print(f"[cyan]Checking:[/cyan] {question}") + console.print(f"[dim]Threshold: {threshold}[/dim]") + + result = vectors.find_similar(question, threshold) + + if result: + similar_front, score = result + console.print(f"\n[yellow]Similar card found ({score:.0%}):[/yellow]") + console.print(f" {similar_front}") + else: + console.print(f"\n[green]No similar cards found above {threshold:.0%} threshold.[/green]") + + +COMMAND = { + 'names': ['vector'], + 'setup_parser': setup_parser, + 'handler': handler +} From a15337b482a4e48930ef596de45c630e24d5b9f3 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:08:59 -0500 Subject: [PATCH 07/16] fix: jsonlogic search pattern --- obsidianki/api/obsidian.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/obsidianki/api/obsidian.py b/obsidianki/api/obsidian.py index b0a1988..e9feea3 100644 --- a/obsidianki/api/obsidian.py +++ b/obsidianki/api/obsidian.py @@ -232,12 +232,7 @@ def find_by_pattern(self, pattern: str, sample_size: int = 0, bias_strength: flo pattern_filter = {"glob": [glob_pattern, {"var": "path"}]} else: # Exact match or name contains - pattern_filter = { - "or": [ - {"===": [{"var": "path"}, pattern]}, - {"glob": [f"*{pattern}*", {"var": "basename"}]} - ] - } + pattern_filter = {"glob": [f"*{pattern}*", {"var": "path"}]} query = self._combine_filters( pattern_filter, @@ -268,7 +263,7 @@ def find_by_pattern(self, pattern: str, sample_size: int = 0, bias_strength: flo def find_by_name(self, note_name: str, search_folders: List[str]) -> Note | None: """Find note by name with partial matching""" query = self._combine_filters( - {"glob": [f"*{note_name}*", {"var": "basename"}]}, + {"glob": [f"*{note_name}*", {"var": "path"}]}, self._build_folder_filter(search_folders), self._build_excluded_tags_filter() ) From b36ead7881775aac1a020930bac953c0a02f9e84 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:09:22 -0500 Subject: [PATCH 08/16] test: migrate to new search api and disable vector mode in tests --- tests/test_advanced_features.py | 3 +- tests/test_api_obsidian.py | 252 ++++++++++++++++++-------------- tests/test_cli_basic.py | 3 +- tests/test_main_flow.py | 3 +- 4 files changed, 147 insertions(+), 114 deletions(-) diff --git a/tests/test_advanced_features.py b/tests/test_advanced_features.py index f5882cc..ff5de33 100644 --- a/tests/test_advanced_features.py +++ b/tests/test_advanced_features.py @@ -37,7 +37,8 @@ def mock_config(): patch.object(obsidianki.cli.config.CONFIG, 'tag_schema_file', tags_file), \ patch.object(obsidianki.cli.config.CONFIG, 'APPROVE_NOTES', False), \ patch.object(obsidianki.cli.config.CONFIG, 'APPROVE_CARDS', False), \ - patch.object(obsidianki.cli.config.CONFIG, 'UPFRONT_BATCHING', False): + patch.object(obsidianki.cli.config.CONFIG, 'UPFRONT_BATCHING', False), \ + patch.object(obsidianki.cli.config.CONFIG, 'vector_dedup', False): yield { 'config_dir': config_dir, 'env_file': env_file, diff --git a/tests/test_api_obsidian.py b/tests/test_api_obsidian.py index 6f535c1..27f3b9c 100644 --- a/tests/test_api_obsidian.py +++ b/tests/test_api_obsidian.py @@ -25,94 +25,116 @@ def test_init_without_api_key(self): ObsidianAPI() -class TestObsidianAPIBuildFilters: - """Test filter building""" +class TestObsidianAPIJsonLogicFilters: + """Test JsonLogic filter building""" @patch('obsidianki.api.obsidian.CONFIG', None) - def test_build_filters_no_filters(self): - """Test building filters with no conditions""" + def test_build_folder_filter_none(self): + """Test building folder filter with no folders""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): api = ObsidianAPI() - result = api._build_filters(None) - assert result == "" + result = api._build_folder_filter(None) + assert result is None @patch('obsidianki.api.obsidian.CONFIG', None) - def test_build_filters_with_folders(self): - """Test building filters with search folders""" + def test_build_folder_filter_empty(self): + """Test building folder filter with empty list""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): api = ObsidianAPI() - result = api._build_filters(['folder1', 'folder2']) - assert 'startswith(file.path, "folder1/")' in result - assert 'startswith(file.path, "folder2/")' in result - assert ' OR ' in result + result = api._build_folder_filter([]) + assert result is None + + @patch('obsidianki.api.obsidian.CONFIG', None) + def test_build_folder_filter_single(self): + """Test building folder filter with single folder""" + with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): + api = ObsidianAPI() + result = api._build_folder_filter(['folder1']) + assert result == {"glob": ["folder1/*", {"var": "path"}]} + + @patch('obsidianki.api.obsidian.CONFIG', None) + def test_build_folder_filter_multiple(self): + """Test building folder filter with multiple folders""" + with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): + api = ObsidianAPI() + result = api._build_folder_filter(['folder1', 'folder2']) + assert "or" in result + assert len(result["or"]) == 2 @patch('obsidianki.api.obsidian.CONFIG') - def test_build_filters_with_excluded_tags(self, mock_config): - """Test building filters with excluded tags""" + def test_build_excluded_tags_filter(self, mock_config): + """Test building excluded tags filter""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_config.excluded_tags = ['private', 'draft'] api = ObsidianAPI() - result = api._build_filters(None) - assert '!contains(file.tags, "private")' in result - assert '!contains(file.tags, "draft")' in result + result = api._build_excluded_tags_filter() + assert "and" in result + assert len(result["and"]) == 2 + @patch('obsidianki.api.obsidian.CONFIG') + def test_build_excluded_tags_filter_empty(self, mock_config): + """Test building excluded tags filter with no excluded tags""" + with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): + mock_config.excluded_tags = [] + api = ObsidianAPI() + result = api._build_excluded_tags_filter() + assert result is None -class TestObsidianAPIBuildQuery: - """Test DQL query building""" + @patch('obsidianki.api.obsidian.CONFIG', None) + def test_combine_filters_empty(self): + """Test combining filters with no conditions""" + with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): + api = ObsidianAPI() + result = api._combine_filters() + # Should return query that matches all (returns full object) + assert result == {"var": ""} - def test_build_base_query_default(self): - """Test building base query with defaults""" + @patch('obsidianki.api.obsidian.CONFIG', None) + def test_combine_filters_single(self): + """Test combining single filter""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): api = ObsidianAPI() - query = api._build_base_query() - assert 'TABLE' in query - assert 'file.name' in query - assert 'file.path' in query - assert 'file.mtime' in query - assert 'SORT file.mtime ASC' in query - - def test_build_base_query_custom_sort(self): - """Test building query with custom sort""" + condition = {">": [{"var": "stat.size"}, 100]} + result = api._combine_filters(condition) + # Should wrap in if statement + assert "if" in result + assert result["if"][0] == condition + + @patch('obsidianki.api.obsidian.CONFIG', None) + def test_combine_filters_multiple(self): + """Test combining multiple filters""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): api = ObsidianAPI() - query = api._build_base_query( - extra_conditions='file.size > 100', - sort_field='file.name', - sort_order='DESC' - ) - assert 'file.size > 100' in query - assert 'SORT file.name DESC' in query + cond1 = {">": [{"var": "stat.size"}, 100]} + cond2 = {"<": [{"var": "stat.mtime"}, 1234567890]} + result = api._combine_filters(cond1, cond2) + # Should wrap in if with and + assert "if" in result + assert "and" in result["if"][0] class TestObsidianAPIDQL: - """Test DQL query execution""" + """Test DQL query execution (for agent mode)""" @patch('obsidianki.api.obsidian.BaseAPI._make_request') def test_dql_success(self, mock_request): """Test successful DQL query""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_response = Mock() - mock_response.json.return_value = { - "data": { - "values": [ - ["Note 1", "path/note1.md", "2024-01-01", 100, ["tag1"]], - ["Note 2", "path/note2.md", "2024-01-02", 200, ["tag2"]] - ] - } - } mock_request.return_value = mock_response api = ObsidianAPI() - # Mock the _parse_response to return list of dicts with patch.object(api, '_parse_response') as mock_parse: mock_parse.return_value = [ { - "filename": "Note 1", - "path": "path/note1.md", - "mtime": "2024-01-01", - "size": 100, - "tags": ["tag1"] + "result": { + "filename": "Note 1", + "path": "path/note1.md", + "mtime": "2024-01-01", + "size": 100, + "tags": ["tag1"] + } } ] @@ -131,16 +153,47 @@ def test_dql_failure(self, mock_request): api.dql("INVALID QUERY") +class TestObsidianAPISearch: + """Test JsonLogic search""" + + @patch('obsidianki.api.obsidian.BaseAPI._make_request') + def test_search_success(self, mock_request): + """Test successful JsonLogic search""" + with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): + mock_response = Mock() + mock_request.return_value = mock_response + + api = ObsidianAPI() + + with patch.object(api, '_parse_response') as mock_parse: + mock_parse.return_value = [ + { + "filename": "path/note1.md", + "result": { + "path": "path/note1.md", + "basename": "note1", + "stat": {"mtime": 1234567890, "size": 100}, + "tags": ["tag1"] + } + } + ] + + results = api.search({">": [{"var": "stat.size"}, 50]}) + assert len(results) == 1 + assert isinstance(results[0], Note) + + class TestObsidianAPIGetOldNotes: """Test getting old notes""" - @patch.object(ObsidianAPI, 'dql') + @patch.object(ObsidianAPI, 'search') @patch('obsidianki.api.obsidian.CONFIG') - def test_get_old_notes_basic(self, mock_config, mock_dql): + def test_get_old_notes_basic(self, mock_config, mock_search): """Test getting old notes with basic parameters""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_config.search_folders = [] - mock_dql.return_value = [ + mock_config.excluded_tags = [] + mock_search.return_value = [ Note(path="old.md", filename="Old", content="test", tags=[], size=100) ] @@ -148,38 +201,37 @@ def test_get_old_notes_basic(self, mock_config, mock_dql): notes = api.get_old_notes(days=7, limit=10) assert len(notes) == 1 - mock_dql.assert_called_once() + mock_search.assert_called_once() - # Check the query contains date filter - call_args = mock_dql.call_args[0][0] - assert 'file.mtime <' in call_args - assert 'LIMIT 10' in call_args - - @patch.object(ObsidianAPI, 'dql') + @patch.object(ObsidianAPI, 'search') @patch('obsidianki.api.obsidian.CONFIG') - def test_get_old_notes_no_limit(self, mock_config, mock_dql): - """Test getting old notes without limit""" + def test_get_old_notes_with_limit(self, mock_config, mock_search): + """Test getting old notes respects limit""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_config.search_folders = [] - mock_dql.return_value = [] + mock_config.excluded_tags = [] + mock_search.return_value = [ + Note(path=f"note{i}.md", filename=f"Note{i}", content="", tags=[], size=100) + for i in range(20) + ] api = ObsidianAPI() - api.get_old_notes(days=30, limit=0) + notes = api.get_old_notes(days=30, limit=5) - call_args = mock_dql.call_args[0][0] - assert 'LIMIT' not in call_args + assert len(notes) == 5 class TestObsidianAPIGetTaggedNotes: """Test getting tagged notes""" - @patch.object(ObsidianAPI, 'dql') + @patch.object(ObsidianAPI, 'search') @patch('obsidianki.api.obsidian.CONFIG') - def test_get_tagged_notes_single_tag(self, mock_config, mock_dql): + def test_get_tagged_notes_single_tag(self, mock_config, mock_search): """Test getting notes with single tag""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_config.search_folders = [] - mock_dql.return_value = [ + mock_config.excluded_tags = [] + mock_search.return_value = [ Note(path="tagged.md", filename="Tagged", content="test", tags=["important"], size=100) ] @@ -188,59 +240,37 @@ def test_get_tagged_notes_single_tag(self, mock_config, mock_dql): notes = api.get_tagged_notes(["important"]) assert len(notes) == 1 - call_args = mock_dql.call_args[0][0] - assert 'contains(file.tags, "important")' in call_args + mock_search.assert_called_once() - @patch.object(ObsidianAPI, 'dql') + @patch.object(ObsidianAPI, 'search') @patch('obsidianki.api.obsidian.CONFIG') - def test_get_tagged_notes_multiple_tags(self, mock_config, mock_dql): + def test_get_tagged_notes_multiple_tags(self, mock_config, mock_search): """Test getting notes with multiple tags""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_config.search_folders = [] - mock_dql.return_value = [] + mock_config.excluded_tags = [] + mock_search.return_value = [] api = ObsidianAPI() api.get_tagged_notes(["tag1", "tag2", "tag3"]) - call_args = mock_dql.call_args[0][0] - assert 'contains(file.tags, "tag1")' in call_args - assert 'contains(file.tags, "tag2")' in call_args - assert ' OR ' in call_args - - @patch.object(ObsidianAPI, 'dql') - @patch('obsidianki.api.obsidian.CONFIG') - def test_get_tagged_notes_exclude_recent(self, mock_config, mock_dql): - """Test getting tagged notes excluding recent ones""" - with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): - mock_config.search_folders = [] - mock_dql.return_value = [] - - api = ObsidianAPI() - api.get_tagged_notes(["important"], exclude_recent_days=7) - - call_args = mock_dql.call_args[0][0] - assert 'file.mtime <' in call_args + mock_search.assert_called_once() + # Verify the query contains an 'or' for multiple tags + call_args = mock_search.call_args[0][0] + assert "if" in call_args class TestObsidianAPIEdgeCases: """Test edge cases""" + @patch.object(ObsidianAPI, 'search') @patch('obsidianki.api.obsidian.CONFIG') - def test_empty_search_folders(self, mock_config): - """Test with empty search folders list""" - with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): - mock_config.excluded_tags = [] - api = ObsidianAPI() - result = api._build_filters([]) - assert result == "" - - @patch.object(ObsidianAPI, 'dql') - @patch('obsidianki.api.obsidian.CONFIG') - def test_get_old_notes_empty_result(self, mock_config, mock_dql): + def test_get_old_notes_empty_result(self, mock_config, mock_search): """Test getting old notes with empty result""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_config.search_folders = [] - mock_dql.return_value = [] + mock_config.excluded_tags = [] + mock_search.return_value = [] api = ObsidianAPI() notes = api.get_old_notes(days=365) @@ -248,19 +278,19 @@ def test_get_old_notes_empty_result(self, mock_config, mock_dql): assert notes == [] assert len(notes) == 0 - @patch.object(ObsidianAPI, 'dql') + @patch.object(ObsidianAPI, 'search') @patch('obsidianki.api.obsidian.CONFIG') - def test_get_tagged_notes_empty_tags(self, mock_config, mock_dql): + def test_get_tagged_notes_empty_tags(self, mock_config, mock_search): """Test getting notes with empty tags list""" with patch.dict(os.environ, {'OBSIDIAN_API_KEY': 'test'}): mock_config.search_folders = [] - mock_dql.return_value = [] + mock_config.excluded_tags = [] + mock_search.return_value = [] api = ObsidianAPI() notes = api.get_tagged_notes([]) - # Should still make a query but with no tag conditions - mock_dql.assert_called_once() + mock_search.assert_called_once() if __name__ == "__main__": diff --git a/tests/test_cli_basic.py b/tests/test_cli_basic.py index 3ee63a1..9f72e77 100644 --- a/tests/test_cli_basic.py +++ b/tests/test_cli_basic.py @@ -77,7 +77,8 @@ def mock_config(): patch.object(obsidianki.cli.config.CONFIG, 'processing_history_file', history_file), \ patch.object(obsidianki.cli.config.CONFIG, 'processing_history', {}), \ patch.object(obsidianki.cli.config.CONFIG, 'APPROVE_NOTES', False), \ - patch.object(obsidianki.cli.config.CONFIG, 'APPROVE_CARDS', False): + patch.object(obsidianki.cli.config.CONFIG, 'APPROVE_CARDS', False), \ + patch.object(obsidianki.cli.config.CONFIG, 'vector_dedup', False): yield diff --git a/tests/test_main_flow.py b/tests/test_main_flow.py index 0e3ac8d..9bbaf9e 100644 --- a/tests/test_main_flow.py +++ b/tests/test_main_flow.py @@ -31,7 +31,8 @@ def mock_config(): patch.object(obsidianki.cli.config.CONFIG, 'processing_history', {}), \ patch.object(obsidianki.cli.config.CONFIG, 'APPROVE_NOTES', False), \ patch.object(obsidianki.cli.config.CONFIG, 'APPROVE_CARDS', False), \ - patch.object(obsidianki.cli.config.CONFIG, 'UPFRONT_BATCHING', False): + patch.object(obsidianki.cli.config.CONFIG, 'UPFRONT_BATCHING', False), \ + patch.object(obsidianki.cli.config.CONFIG, 'vector_dedup', False): yield From 100a95758af61f3f0b75ac21af3e421cec339776 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:27:31 -0500 Subject: [PATCH 09/16] feat: gemini/openai embeddings instead of local --- README.md | 18 +- obsidianki/ai/client.py | 18 +- obsidianki/ai/vectors.py | 291 ++++++++++++++------------ obsidianki/cli/commands/vector_cmd.py | 20 +- obsidianki/cli/config.py | 2 +- pyproject.toml | 7 +- 6 files changed, 193 insertions(+), 163 deletions(-) diff --git a/README.md b/README.md index e689ffb..c76547a 100644 --- a/README.md +++ b/README.md @@ -143,19 +143,11 @@ oki template use programming # runs the above command as "oki --notes 'frontend/ | `density_bias_strength` | `0.5` | Bias strength against over-processed notes (0-1) | | `search_folders` | `[]` | Limit processing to specific folders (array) | | `vector_dedup` | `false` | Enable semantic deduplication via embeddings | -| `vector_threshold` | `0.85` | Similarity threshold for duplicate detection (0-1) | +| `vector_threshold` | `0.7` | Similarity threshold for duplicate detection (0-1) | ## Vector Deduplication -Avoid generating semantically similar flashcards using local embeddings. - -### Install - -```bash -pip install obsidianki[vectors] -# or -uv tool install obsidianki --with chromadb --with sentence-transformers -``` +Avoid generating semantically similar flashcards using API embeddings (Gemini or OpenAI). ### Enable @@ -181,12 +173,12 @@ oki vector clear # Clear the index ### How it works 1. AI proposes flashcards via `create_flashcards` tool -2. Each card is checked against the vector database for semantic similarity -3. If similar cards exist, AI receives feedback: *"Card 2 is 91% similar to 'What is polymorphism?'"* +2. Each card is checked for semantic similarity against existing cards +3. If similar cards exist, AI receives feedback: *"Card 2 is 87% similar to 'What is polymorphism?'"* 4. AI can revise or confirm via `submit_flashcards` tool 5. Accepted cards are indexed for future deduplication -The vector database is stored in `~/.config/obsidianki/vectors/`. +Embeddings use Gemini (`GEMINI_API_KEY`) or OpenAI (`OPENAI_API_KEY`). The index is stored in `~/.config/obsidianki/vectors.json`. # MCP There is an [experimental MCP server](https://github.com/ccmdi/obsidianki-mcp) that runs Obsidianki as a subprocess. Useful if you want to generate flashcards from daily use with an LLM, such as if you ask questions back and forth and want to generate flashcards from that material. \ No newline at end of file diff --git a/obsidianki/ai/client.py b/obsidianki/ai/client.py index 763c08c..55b4b9d 100644 --- a/obsidianki/ai/client.py +++ b/obsidianki/ai/client.py @@ -301,13 +301,17 @@ def _generate_with_vector_feedback( ) if similar_matches: - # Build feedback message + # Build feedback message - now handles multiple matches per card feedback_lines = [] - for idx, front, existing, score in similar_matches: - feedback_lines.append( - f"- Card {idx + 1}: \"{front[:50]}{'...' if len(front) > 50 else ''}\" " - f"≈ \"{existing[:50]}{'...' if len(existing) > 50 else ''}\" ({score:.0%})" - ) + total_matches = 0 + for idx, front, matches in similar_matches: + front_preview = f"{front[:50]}{'...' if len(front) > 50 else ''}" + for existing, score in matches: + existing_preview = f"{existing[:50]}{'...' if len(existing) > 50 else ''}" + feedback_lines.append( + f"- Card {idx + 1}: \"{front_preview}\" ≈ \"{existing_preview}\" ({score:.0%})" + ) + total_matches += 1 feedback = ( f"Similar existing cards found:\n" @@ -316,7 +320,7 @@ def _generate_with_vector_feedback( f"1. Call create_flashcards again with revised cards that explore different angles\n" f"2. Call submit_flashcards if you believe these are sufficiently distinct" ) - console.print(f"[yellow]Vector feedback:[/yellow] {len(similar_matches)} similar card(s) found") + console.print(f"[yellow]Vector feedback:[/yellow] {total_matches} similar match(es) for {len(similar_matches)} card(s)") for line in feedback_lines: console.print(f"[dim]{line}[/dim]") else: diff --git a/obsidianki/ai/vectors.py b/obsidianki/ai/vectors.py index ded30fd..36023e2 100644 --- a/obsidianki/ai/vectors.py +++ b/obsidianki/ai/vectors.py @@ -1,150 +1,162 @@ """Vector-based semantic deduplication for flashcards. -Uses ChromaDB for storage and sentence-transformers for local embeddings. -Provides a feedback loop where the LLM can revise cards based on similarity. +Simple JSON storage + API embeddings. No heavy dependencies. """ from __future__ import annotations + import hashlib +import json +import os from pathlib import Path -from typing import List, Optional, Tuple, TYPE_CHECKING +from typing import List, Optional, Tuple -if TYPE_CHECKING: - from chromadb.api.models.Collection import Collection - from chromadb import ClientAPI +import httpx from obsidianki.cli.config import CONFIG_DIR, console -VECTORS_DIR = CONFIG_DIR / "vectors" +VECTORS_FILE = CONFIG_DIR / "vectors.json" + +# Embedding endpoints +GEMINI_BATCH_EMBED_URL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents" +OPENAI_EMBED_URL = "https://api.openai.com/v1/embeddings" + + +def cosine_similarity(a: List[float], b: List[float]) -> float: + """Pure Python cosine similarity.""" + dot = sum(x * y for x, y in zip(a, b)) + norm_a = sum(x * x for x in a) ** 0.5 + norm_b = sum(x * x for x in b) ** 0.5 + if norm_a == 0 or norm_b == 0: + return 0.0 + return dot / (norm_a * norm_b) class VectorStore: - """Lazy-loaded vector store for flashcard semantic deduplication.""" + """Simple vector store using JSON file + API embeddings.""" def __init__(self): - self._client: Optional[ClientAPI] = None - self._collection: Optional[Collection] = None - self._model: Optional[BaseEmbedder] = None + self._data: Optional[dict] = None + self._embedder: Optional[BaseEmbedder] = None + self._dims: Optional[int] = None @property - def collection(self) -> Collection: - """Lazy-load ChromaDB collection.""" - if self._collection is None: - try: - import chromadb - except ImportError: - raise ImportError( - "ChromaDB is required for vector deduplication. " - "Install with: pip install chromadb" - ) - - VECTORS_DIR.mkdir(parents=True, exist_ok=True) - self._client = chromadb.PersistentClient(path=str(VECTORS_DIR)) - self._collection = self._client.get_or_create_collection( - name="flashcards", - metadata={"hnsw:space": "cosine"} - ) - return self._collection + def data(self) -> dict: + """Lazy-load vector data from JSON file.""" + if self._data is None: + if VECTORS_FILE.exists(): + try: + with open(VECTORS_FILE) as f: + self._data = json.load(f) + # Check dimension compatibility + if self._data.get("_dims") and self._data["_dims"] != self._get_expected_dims(): + console.print(f"[yellow]Embedder changed. Clearing vector index.[/yellow]") + self._data = {"_dims": self._get_expected_dims()} + except (json.JSONDecodeError, KeyError): + self._data = {"_dims": self._get_expected_dims()} + else: + self._data = {"_dims": self._get_expected_dims()} + return self._data + + def _save(self) -> None: + """Save vector data to JSON file.""" + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + with open(VECTORS_FILE, 'w') as f: + json.dump(self._data, f) + + def _get_expected_dims(self) -> int: + """Get expected embedding dimensions based on current embedder.""" + if os.environ.get("GEMINI_API_KEY"): + return 768 + elif os.environ.get("OPENAI_API_KEY"): + return 1536 + return 0 # Unknown @property - def model(self) -> BaseEmbedder: - """Lazy-load embedding model.""" - if self._model is None: - self._model = LocalEmbedder() - return self._model + def embedder(self) -> BaseEmbedder: + """Get embedder - Gemini or OpenAI.""" + if self._embedder is None: + if os.environ.get("GEMINI_API_KEY"): + self._embedder = GeminiEmbedder() + elif os.environ.get("OPENAI_API_KEY"): + self._embedder = OpenAIEmbedder() + else: + raise ValueError( + "No API key for embeddings. Set GEMINI_API_KEY or OPENAI_API_KEY." + ) + return self._embedder def add(self, fronts: List[str]) -> None: """Index flashcard fronts.""" if not fronts: return - # Filter out empty strings and duplicates fronts = [f for f in fronts if f.strip()] if not fronts: return - console.print(f"[dim]Indexing {len(fronts)} card(s) in vector store...[/dim]") - embeddings = self.model.embed(fronts) - ids = [self._hash(f) for f in fronts] + console.print(f"[dim]Indexing {len(fronts)} card(s)...[/dim]") + embeddings = self.embedder.embed(fronts) - # Upsert to handle duplicates - self.collection.upsert( - ids=ids, - embeddings=embeddings, - documents=fronts - ) - console.print(f"[dim]Vector index now has {self.count()} cards[/dim]") + for front, embedding in zip(fronts, embeddings): + card_id = self._hash(front) + self.data[card_id] = {"text": front, "embedding": embedding} - def find_similar(self, front: str, threshold: float) -> Optional[Tuple[str, float]]: - """Find most similar existing card above threshold. + self.data["_dims"] = len(embeddings[0]) if embeddings else self._get_expected_dims() + self._save() + console.print(f"[dim]Vector index: {self.count()} cards[/dim]") - Args: - front: The flashcard front text to check - threshold: Minimum cosine similarity (0-1) to consider a match + def find_similar(self, front: str, threshold: float, limit: int = 5) -> List[Tuple[str, float]]: + """Find all similar existing cards above threshold. Returns: - Tuple of (similar_front, similarity_score) or None if no match + List of (similar_text, similarity_score) tuples, sorted by score descending """ - if self.collection.count() == 0: - return None + if self.count() == 0: + return [] - # Don't match against itself front_id = self._hash(front) + query_embedding = self.embedder.embed([front])[0] - results = self.collection.query( - query_embeddings=[self.model.embed([front])[0]], - n_results=2, # Get 2 in case first is itself - include=["documents", "distances"] - ) - - if not results["documents"] or not results["documents"][0]: - return None - - # Find best match that isn't the same card - for i, doc in enumerate(results["documents"][0]): - doc_id = self._hash(doc) - if doc_id == front_id: + matches = [] + for card_id, card_data in self.data.items(): + if card_id.startswith("_"): # Skip metadata + continue + if card_id == front_id: # Skip self continue - # ChromaDB returns cosine distance, convert to similarity - distance = results["distances"][0][i] - similarity = 1 - distance - + similarity = cosine_similarity(query_embedding, card_data["embedding"]) if similarity >= threshold: - return (doc, similarity) + matches.append((card_data["text"], similarity)) - return None + # Sort by similarity descending, limit results + matches.sort(key=lambda x: x[1], reverse=True) + return matches[:limit] def find_similar_batch( self, fronts: List[str], threshold: float - ) -> List[Tuple[int, str, str, float]]: + ) -> List[Tuple[int, str, List[Tuple[str, float]]]]: """Check multiple fronts for similarity. - Args: - fronts: List of flashcard front texts to check - threshold: Minimum cosine similarity to flag - Returns: - List of (index, front, similar_existing, similarity) for matches only + List of (index, front, [(similar_text, score), ...]) for cards with matches """ - matches = [] + results = [] for i, front in enumerate(fronts): - similar = self.find_similar(front, threshold) - if similar: - existing, score = similar - matches.append((i, front, existing, score)) - return matches + matches = self.find_similar(front, threshold) + if matches: + results.append((i, front, matches)) + return results def count(self) -> int: """Number of indexed cards.""" - return self.collection.count() + return len([k for k in self.data.keys() if not k.startswith("_")]) def clear(self) -> None: """Clear all indexed cards.""" - if self._client is not None: - self._client.delete_collection("flashcards") - self._collection = None + self._data = {"_dims": self._get_expected_dims()} + self._save() def _hash(self, text: str) -> str: """Generate stable ID for text.""" @@ -158,51 +170,70 @@ def embed(self, texts: List[str]) -> List[List[float]]: raise NotImplementedError -class LocalEmbedder(BaseEmbedder): - """Local embeddings using sentence-transformers.""" +class GeminiEmbedder(BaseEmbedder): + """Embeddings via Gemini API (batched).""" - def __init__(self): - self._model = None - self._loaded = False - - def _get_device(self) -> str: - """Detect best available device (cuda > mps > cpu).""" - try: - import torch - if torch.cuda.is_available(): - return "cuda" - elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): - return "mps" - except ImportError: - pass - return "cpu" + def __init__(self, dimensions: int = 768): + self.api_key = os.environ.get("GEMINI_API_KEY") + if not self.api_key: + raise ValueError("GEMINI_API_KEY not found") + self.dimensions = dimensions + self.model = "models/gemini-embedding-001" - @property - def model(self): - if self._model is None: - try: - # Suppress noisy logging from transformers/torch - import logging - import warnings - logging.getLogger("sentence_transformers").setLevel(logging.WARNING) - logging.getLogger("transformers").setLevel(logging.WARNING) - warnings.filterwarnings("ignore", message=".*position_ids.*") - - from sentence_transformers import SentenceTransformer - except ImportError: - raise ImportError( - "sentence-transformers is required for vector deduplication. " - "Install with: pip install sentence-transformers" - ) - device = self._get_device() - if not self._loaded: - console.print(f"[dim]Loading embedding model ({device})...[/dim]") - self._model = SentenceTransformer('all-MiniLM-L6-v2', device=device) - self._loaded = True - return self._model + def embed(self, texts: List[str]) -> List[List[float]]: + # Gemini batch limit is 100, chunk if needed + all_embeddings = [] + for i in range(0, len(texts), 100): + batch = texts[i:i + 100] + embeddings = self._embed_batch(batch) + all_embeddings.extend(embeddings) + return all_embeddings + + def _embed_batch(self, texts: List[str]) -> List[List[float]]: + url = f"{GEMINI_BATCH_EMBED_URL}?key={self.api_key}" + payload = { + "requests": [ + { + "model": self.model, + "content": {"parts": [{"text": text}]}, + "outputDimensionality": self.dimensions + } + for text in texts + ] + } + + with httpx.Client(timeout=60.0) as client: + response = client.post(url, json=payload) + response.raise_for_status() + data = response.json() + + return [item["values"] for item in data["embeddings"]] + + +class OpenAIEmbedder(BaseEmbedder): + """Embeddings via OpenAI API.""" + + def __init__(self, model: str = "text-embedding-3-small"): + self.api_key = os.environ.get("OPENAI_API_KEY") + if not self.api_key: + raise ValueError("OPENAI_API_KEY not found") + self.model = model def embed(self, texts: List[str]) -> List[List[float]]: - return self.model.encode(texts, show_progress_bar=False).tolist() + url = OPENAI_EMBED_URL + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + payload = {"model": self.model, "input": texts} + + with httpx.Client(timeout=30.0) as client: + response = client.post(url, headers=headers, json=payload) + response.raise_for_status() + data = response.json() + + sorted_data = sorted(data["data"], key=lambda x: x["index"]) + return [item["embedding"] for item in sorted_data] # Global lazy instance diff --git a/obsidianki/cli/commands/vector_cmd.py b/obsidianki/cli/commands/vector_cmd.py index 85ba46e..ecd51f7 100644 --- a/obsidianki/cli/commands/vector_cmd.py +++ b/obsidianki/cli/commands/vector_cmd.py @@ -46,6 +46,12 @@ def setup_parser(subparsers): type=str, help="Question text to check for similarity" ) + check_parser.add_argument( + "-t", "--threshold", + type=float, + default=None, + help="Custom similarity threshold (0-1), overrides config" + ) return vector_parser @@ -145,21 +151,21 @@ def _handle_check(args: argparse.Namespace): vectors = get_vectors() question = args.question - threshold = CONFIG.vector_threshold or 0.85 + threshold = args.threshold if args.threshold is not None else (CONFIG.vector_threshold or 0.7) if vectors.count() == 0: console.print("[yellow]Vector index is empty. Run 'oki vector index' first.[/yellow]") return console.print(f"[cyan]Checking:[/cyan] {question}") - console.print(f"[dim]Threshold: {threshold}[/dim]") + console.print(f"[dim]Threshold: {threshold:.0%}[/dim]") - result = vectors.find_similar(question, threshold) + matches = vectors.find_similar(question, threshold) - if result: - similar_front, score = result - console.print(f"\n[yellow]Similar card found ({score:.0%}):[/yellow]") - console.print(f" {similar_front}") + if matches: + console.print(f"\n[yellow]{len(matches)} similar card(s) found:[/yellow]") + for text, score in matches: + console.print(f" [{score:.0%}] {text}") else: console.print(f"\n[green]No similar cards found above {threshold:.0%} threshold.[/green]") diff --git a/obsidianki/cli/config.py b/obsidianki/cli/config.py index 06a6e8b..b70d3c2 100644 --- a/obsidianki/cli/config.py +++ b/obsidianki/cli/config.py @@ -40,7 +40,7 @@ "BATCH_CARD_LIMIT": 100, # Maximum total cards in batch mode "MODEL": "Claude Sonnet 4.5", # AI model to use (Claude Sonnet 4, GPT-5, Gemini 3 Pro Preview, etc.) "VECTOR_DEDUP": False, # Enable vector-based semantic deduplication with feedback loop - "VECTOR_THRESHOLD": 0.85, # Similarity threshold (0-1) to flag as potential duplicate + "VECTOR_THRESHOLD": 0.7, # Similarity threshold (0-1) to flag as potential duplicate "VECTOR_MAX_TURNS": 5, # Max revision attempts in vector feedback loop } diff --git a/pyproject.toml b/pyproject.toml index 47d655e..26835be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,8 @@ dependencies = [ "rich>=13.0.0", "urllib3>=1.26.0", "pygments>=2.10.0", - "questionary>=2.0.0" + "questionary>=2.0.0", + "httpx>=0.24.0" ] [project.optional-dependencies] @@ -26,10 +27,6 @@ dev = [ "pytest>=7.0.0", "pytest-mock>=3.10.0" ] -vectors = [ - "chromadb>=0.4.0", - "sentence-transformers>=2.0.0" -] [project.scripts] obsidianki = "obsidianki.main:main" From fd5742dfde15e702be43a41ef7e307f0ca395441 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:33:44 -0500 Subject: [PATCH 10/16] feat: add generation logs and make vectors that are too similar force retry --- obsidianki/ai/client.py | 131 +++++++++++++++++++++++++++++++++------ obsidianki/ai/vectors.py | 1 - 2 files changed, 111 insertions(+), 21 deletions(-) diff --git a/obsidianki/ai/client.py b/obsidianki/ai/client.py index 55b4b9d..5e8c035 100644 --- a/obsidianki/ai/client.py +++ b/obsidianki/ai/client.py @@ -3,9 +3,11 @@ from obsidianki.ai.call import ModelResponse import json +from datetime import datetime +from pathlib import Path from typing import List, Dict, Optional, Union, cast -from obsidianki.cli.config import console, CONFIG +from obsidianki.cli.config import console, CONFIG, CONFIG_DIR from obsidianki.cli.utils import process_code_blocks, strip_html from obsidianki.cli.models import Note, Flashcard from obsidianki.ai.models import MODEL_MAP @@ -13,6 +15,7 @@ from obsidianki.ai.tools import FLASHCARD_TOOL, SUBMIT_FLASHCARDS_TOOL, DQL_EXECUTION_TOOL, FINALIZE_SELECTION_TOOL AI_RESULT_SET_SIZE = 20 +LOGS_DIR = CONFIG_DIR / "logs" class FlashcardAI: def __init__(self): @@ -228,6 +231,43 @@ def _serialize_tool_calls(self, tool_calls) -> Optional[List[Dict]]: for tc in tool_calls ] + def _log_conversation( + self, + messages: List[Dict], + note: Optional[Note] = None, + flashcards: Optional[List[Flashcard]] = None, + mode: str = "generate" + ) -> None: + """Log conversation to file for debugging.""" + try: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + note_name = note.filename.replace(".md", "") if note else "query" + safe_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in note_name)[:30] + log_file = LOGS_DIR / f"{timestamp}_{safe_name}.json" + + log_data = { + "timestamp": datetime.now().isoformat(), + "model": self.model, + "mode": mode, + "note": note.path if note else None, + "messages": messages, + "result": { + "count": len(flashcards) if flashcards else 0, + "cards": [ + {"front": fc.front_original, "back": fc.back_original} + for fc in (flashcards or []) + ] + } + } + + with open(log_file, 'w', encoding='utf-8') as f: + json.dump(log_data, f, indent=2, ensure_ascii=False) + + except Exception as e: + console.print(f"[dim]Log write failed: {e}[/dim]") + def _generate_with_vector_feedback( self, system_prompt: str, @@ -252,8 +292,8 @@ def _generate_with_vector_feedback( index_count = vectors.count() if index_count == 0: console.print("[dim]Vector index empty - no similarity checks will match[/dim]") - else: - console.print(f"[dim]Vector index: {index_count} cards indexed[/dim]") + # else: + # console.print(f"[dim]Vector index: {index_count} cards indexed[/dim]") messages = [ {"role": "system", "content": system_prompt}, @@ -304,22 +344,29 @@ def _generate_with_vector_feedback( # Build feedback message - now handles multiple matches per card feedback_lines = [] total_matches = 0 + high_similarity = False for idx, front, matches in similar_matches: - front_preview = f"{front[:50]}{'...' if len(front) > 50 else ''}" for existing, score in matches: - existing_preview = f"{existing[:50]}{'...' if len(existing) > 50 else ''}" feedback_lines.append( - f"- Card {idx + 1}: \"{front_preview}\" ≈ \"{existing_preview}\" ({score:.0%})" + f"- Card {idx + 1} ({score:.0%} similar): \"{front}\" ≈ \"{existing}\"" ) total_matches += 1 - - feedback = ( - f"Similar existing cards found:\n" - f"{chr(10).join(feedback_lines)}\n\n" - f"You may:\n" - f"1. Call create_flashcards again with revised cards that explore different angles\n" - f"2. Call submit_flashcards if you believe these are sufficiently distinct" - ) + if score >= 0.85: + high_similarity = True + + if high_similarity: + instruction = ( + "Cards with ≥85% similarity are TOO SIMILAR and should NOT be submitted.\n" + "You MUST call create_flashcards again with substantially different questions." + ) + else: + instruction = ( + "You may:\n" + "1. Call create_flashcards again with revised cards that explore different angles\n" + "2. Call submit_flashcards if you believe these are sufficiently distinct" + ) + + feedback = f"Similar existing cards found:\n{chr(10).join(feedback_lines)}\n\n{instruction}" console.print(f"[yellow]Vector feedback:[/yellow] {total_matches} similar match(es) for {len(similar_matches)} card(s)") for line in feedback_lines: console.print(f"[dim]{line}[/dim]") @@ -343,19 +390,27 @@ def _generate_with_vector_feedback( "content": f"{len(pending_cards)} cards submitted." }) console.print(f"[green]Submitted:[/green] {len(pending_cards)} cards") - return self._convert_pending_to_flashcards(pending_cards, note, default_tags) + flashcards = self._convert_pending_to_flashcards(pending_cards, note, default_tags) + self._log_conversation(messages, note, flashcards, mode="vector_feedback") + return flashcards except Exception as e: console.print(f"[red]ERROR:[/red] Vector feedback loop failed: {e}") if pending_cards: - return self._convert_pending_to_flashcards(pending_cards, note, default_tags) + flashcards = self._convert_pending_to_flashcards(pending_cards, note, default_tags) + self._log_conversation(messages, note, flashcards, mode="vector_feedback_error") + return flashcards + self._log_conversation(messages, note, [], mode="vector_feedback_error") return [] # Max turns reached - return whatever we have if pending_cards: console.print(f"[yellow]Max turns reached:[/yellow] Submitting {len(pending_cards)} pending cards") - return self._convert_pending_to_flashcards(pending_cards, note, default_tags) + flashcards = self._convert_pending_to_flashcards(pending_cards, note, default_tags) + self._log_conversation(messages, note, flashcards, mode="vector_feedback_max_turns") + return flashcards + self._log_conversation(messages, note, [], mode="vector_feedback_empty") return [] def _convert_pending_to_flashcards( @@ -418,6 +473,10 @@ def generate_flashcards( ) # Original single-shot behavior + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt} + ] response = self._call_llm( system_prompt=SYSTEM_PROMPT, user_prompt=user_prompt, @@ -425,7 +484,15 @@ def generate_flashcards( tool_choice=self._get_tool_choice("create_flashcards") ) - return self._extract_flashcards_from_response(response, note) + flashcards = self._extract_flashcards_from_response(response, note) + if response and response.choices[0].message.tool_calls: + messages.append({ + "role": "assistant", + "content": response.choices[0].message.content or "", + "tool_calls": self._serialize_tool_calls(response.choices[0].message.tool_calls) + }) + self._log_conversation(messages, note, flashcards, mode="generate") + return flashcards def generate_from_query( self, @@ -463,6 +530,10 @@ def generate_from_query( ) # Original single-shot behavior + messages = [ + {"role": "system", "content": QUERY_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt} + ] response = self._call_llm( system_prompt=QUERY_SYSTEM_PROMPT, user_prompt=user_prompt, @@ -470,7 +541,15 @@ def generate_from_query( tool_choice=self._get_tool_choice("create_flashcards") ) - return self._extract_flashcards_from_response(response, virtual_note, default_tags=["query-generated"]) + flashcards = self._extract_flashcards_from_response(response, virtual_note, default_tags=["query-generated"]) + if response and response.choices[0].message.tool_calls: + messages.append({ + "role": "assistant", + "content": response.choices[0].message.content or "", + "tool_calls": self._serialize_tool_calls(response.choices[0].message.tool_calls) + }) + self._log_conversation(messages, virtual_note, flashcards, mode="query") + return flashcards def generate_from_note_query(self, note: Note, query: str, target_cards: int, previous_fronts: List[str] | None = None, deck_examples: List[Dict[str, str]] | None = None) -> List[Flashcard]: """Generate flashcards by extracting specific information from a note based on a query""" @@ -501,6 +580,10 @@ def generate_from_note_query(self, note: Note, query: str, target_cards: int, pr ) # Original single-shot behavior + messages = [ + {"role": "system", "content": TARGETED_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt} + ] response = self._call_llm( system_prompt=TARGETED_SYSTEM_PROMPT, user_prompt=user_prompt, @@ -508,7 +591,15 @@ def generate_from_note_query(self, note: Note, query: str, target_cards: int, pr tool_choice=self._get_tool_choice("create_flashcards") ) - return self._extract_flashcards_from_response(response, note) + flashcards = self._extract_flashcards_from_response(response, note) + if response and response.choices[0].message.tool_calls: + messages.append({ + "role": "assistant", + "content": response.choices[0].message.content or "", + "tool_calls": self._serialize_tool_calls(response.choices[0].message.tool_calls) + }) + self._log_conversation(messages, note, flashcards, mode="note_query") + return flashcards def find_with_agent(self, natural_request: str, sample_size: int | None = None, bias_strength: float | None = None) -> List[Note]: """Use multi-turn agent with tool calling to find notes via iterative DQL refinement""" diff --git a/obsidianki/ai/vectors.py b/obsidianki/ai/vectors.py index 36023e2..50cc8ca 100644 --- a/obsidianki/ai/vectors.py +++ b/obsidianki/ai/vectors.py @@ -103,7 +103,6 @@ def add(self, fronts: List[str]) -> None: self.data["_dims"] = len(embeddings[0]) if embeddings else self._get_expected_dims() self._save() - console.print(f"[dim]Vector index: {self.count()} cards[/dim]") def find_similar(self, front: str, threshold: float, limit: int = 5) -> List[Tuple[str, float]]: """Find all similar existing cards above threshold. From 99e62ff532823f95ec71a3085c0a6047e5cf3bc2 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:38:27 -0500 Subject: [PATCH 11/16] fix: deduplicate via history in targeted mode --- obsidianki/cli/processors.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/obsidianki/cli/processors.py b/obsidianki/cli/processors.py index dae04c9..842baa2 100644 --- a/obsidianki/cli/processors.py +++ b/obsidianki/cli/processors.py @@ -251,8 +251,11 @@ def preprocess(args: argparse.Namespace): console.print(f"[dim]Using {len(deck_examples)} example cards for schema enforcement[/dim]") previous_fronts = [] - if not args.query and args.notes and CONFIG.deduplicate_via_history: + if args.notes and CONFIG.deduplicate_via_history: previous_fronts = [note.get_previous_flashcard_fronts() for note in notes] + total_prev = sum(len(pf) for pf in previous_fronts) + if total_prev > 0: + console.print(f"[dim]{total_prev} previous card(s) loaded for this note[/dim]") elif args.query and not args.notes and CONFIG.deduplicate_via_deck: # For standalone query mode, use deck-based deduplication deck_fronts = ANKI.get_card_fronts(CONFIG.deck) From 51c92f28b828a2a49bb7793f5e4436dd7167f828 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:39:02 -0500 Subject: [PATCH 12/16] chore: add opus 4.5 and gemini 3 flash --- obsidianki/ai/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/obsidianki/ai/models.py b/obsidianki/ai/models.py index f229178..44f13d7 100644 --- a/obsidianki/ai/models.py +++ b/obsidianki/ai/models.py @@ -5,9 +5,9 @@ "model": "claude-sonnet-4-5", "key_name": "ANTHROPIC_API_KEY" }, - "Claude Opus 4": { + "Claude Opus 4.5": { "provider": "anthropic", - "model": "claude-opus-4-1", + "model": "claude-opus-4-5", "key_name": "ANTHROPIC_API_KEY", "url": "https://console.anthropic.com/" }, @@ -35,9 +35,9 @@ "key_name": "OPENAI_API_KEY", "url": "https://platform.openai.com/api-keys" }, - "Gemini 2.5 Flash": { + "Gemini 3 Flash": { "provider": "google", - "model": "gemini/gemini-2.5-flash", + "model": "gemini/gemini-3-flash", "key_name": "GEMINI_API_KEY", "url": "https://makersuite.google.com/app/apikey" }, From e30c7aa8531c751594b2f60b0e028cef2f9fc2fe Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:43:16 -0500 Subject: [PATCH 13/16] refactor: indent manager --- obsidianki/cli/config.py | 47 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/obsidianki/cli/config.py b/obsidianki/cli/config.py index b70d3c2..a6761b6 100644 --- a/obsidianki/cli/config.py +++ b/obsidianki/cli/config.py @@ -8,10 +8,55 @@ from typing import Dict, List, Union, Mapping, cast from dotenv import load_dotenv from rich.console import Console +from contextlib import contextmanager +class IndentedConsole: + """Console wrapper with scope-based indentation.""" -console = Console() + def __init__(self, base_console: Console, indent_str: str = " "): + self._console = base_console + self._indent_str = indent_str + self._level = 0 + + @property + def prefix(self) -> str: + """Current indentation prefix string.""" + return self._indent_str * self._level + + @contextmanager + def indent(self, levels: int = 1): + """Context manager to increase indentation.""" + self._level += levels + try: + yield + finally: + self._level -= levels + + def print(self, *args, **kwargs): + """Print with current indentation level.""" + if args: + first = args[0] + if isinstance(first, str): + args = (self.prefix + first,) + args[1:] + else: + self._console.print(self.prefix, end="") + self._console.print(*args, **kwargs) + + def input(self, prompt: str = "") -> str: + """Input with current indentation level.""" + return self._console.input(self.prefix + prompt) + + def status(self, message: str, **kwargs): + """Status spinner with current indentation level.""" + return self._console.status(self.prefix + message, **kwargs) + + def __getattr__(self, name): + """Delegate other methods to underlying console.""" + return getattr(self._console, name) + + +console = IndentedConsole(Console()) CONFIG_DIR = Path.home() / ".config" / "obsidianki" ENV_FILE = CONFIG_DIR / ".env" From a4ed736ac1983773c71377bd79793e8ce0015425 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:49:56 -0500 Subject: [PATCH 14/16] refactor: implement indentation scopes --- obsidianki/cli/interactive/approval.py | 26 ++++------ obsidianki/cli/processors.py | 70 +++++++++++++------------- 2 files changed, 46 insertions(+), 50 deletions(-) diff --git a/obsidianki/cli/interactive/approval.py b/obsidianki/cli/interactive/approval.py index d9150ef..ee7dcfc 100644 --- a/obsidianki/cli/interactive/approval.py +++ b/obsidianki/cli/interactive/approval.py @@ -38,10 +38,10 @@ def approve_note(note: Note) -> bool: metadata = f"[dim](W {weight:.2f} | T {total_cards})[/dim]" # Format: NOTE TITLE (W | D | T ) - console.print(f" [dim]Path: {note.to_obsidian_link_rich()} {metadata}[/dim]") + console.print(f"[dim]Path: {note.to_obsidian_link_rich()} {metadata}[/dim]") if weight == 0: - console.print(f" [yellow]WARNING:[/yellow] This note has 0 weight") + console.print(f"[yellow]WARNING:[/yellow] This note has 0 weight") def show_deck_breakdown(): """Display deck breakdown for the note.""" @@ -85,8 +85,8 @@ def get_deck_line_count(): def get_input_with_keyboard_listener(): """Custom input that listens for Ctrl+D to toggle deck breakdown.""" - # Build the prompt text - prompt_text = " Process this note? [magenta](y/n/hide)[/magenta]" + # Build the prompt text with current indent + prompt_text = f"{console.prefix}Process this note? [magenta](y/n/hide)[/magenta]" if has_deck_info: prompt_text += " [dim](Ctrl+D)[/dim]" @@ -235,14 +235,14 @@ def get_input_with_keyboard_listener(): if choice == "hide": CONFIG.hide_note(note.path) - console.print(f" [yellow]Note hidden permanently[/yellow]") + console.print(f"[yellow]Note hidden permanently[/yellow]") return False if choice in ["y", "n"]: return choice == "y" # Invalid input, re-prompt - console.print(f" [yellow]Invalid choice. Please enter y, n, or hide[/yellow]") + console.print(f"[yellow]Invalid choice. Please enter y, n, or hide[/yellow]") except KeyboardInterrupt: raise @@ -252,21 +252,15 @@ def get_input_with_keyboard_listener(): def approve_flashcard(flashcard: Flashcard) -> bool: """Ask user to approve Flashcard object before adding to Anki""" - from rich.console import Group - from rich.text import Text - front_clean = flashcard.get_clean_front() back_clean = flashcard.get_clean_back() - front_line = Padding(f"[cyan]Front:[/cyan] {front_clean}", (0, 0, 0, 3)) - back_line = Padding(f"[cyan]Back:[/cyan] {back_clean}", (0, 0, 0, 3)) - blank_line = Text("") - - console.print(Group(front_line, back_line, blank_line)) + console.print(f"[cyan]Front:[/cyan] {front_clean}") + console.print(f"[cyan]Back:[/cyan] {back_clean}") + console.print() try: - result = Confirm.ask(" Add this card to Anki?", default=True, console=console) - console.print() + result = Confirm.ask(f"{console.prefix}Add this card to Anki?", default=True, console=console._console) return result except KeyboardInterrupt: raise diff --git a/obsidianki/cli/processors.py b/obsidianki/cli/processors.py index 842baa2..243fd1a 100644 --- a/obsidianki/cli/processors.py +++ b/obsidianki/cli/processors.py @@ -17,26 +17,27 @@ def process(note: Note, args: argparse.Namespace, deck_examples: List[Dict[str, from obsidianki.cli.config import console note.ensure_content() - console.print(" ", end="") - # Generate flashcards if args.query and note.path == "query": # Standalone query mode - use direct query generation - flashcards = AI.generate_from_query(args.query, - target_cards=target_cards_per_note, - previous_fronts=previous_fronts, - deck_examples=deck_examples) + with console.status("Generating..."): + flashcards = AI.generate_from_query(args.query, + target_cards=target_cards_per_note, + previous_fronts=previous_fronts, + deck_examples=deck_examples) elif args.query: - console.print(f" [cyan]Extracting info for query:[/cyan] [bold]{args.query}[/bold]") - flashcards = AI.generate_from_note_query(note, args.query, - target_cards=target_cards_per_note, - previous_fronts=previous_fronts, - deck_examples=deck_examples) + console.print(f"[cyan]Extracting info for query:[/cyan] [bold]{args.query}[/bold]") + with console.status("Generating..."): + flashcards = AI.generate_from_note_query(note, args.query, + target_cards=target_cards_per_note, + previous_fronts=previous_fronts, + deck_examples=deck_examples) else: - flashcards = AI.generate_flashcards(note, - target_cards=target_cards_per_note, - previous_fronts=previous_fronts, - deck_examples=deck_examples) + with console.status("Generating..."): + flashcards = AI.generate_flashcards(note, + target_cards=target_cards_per_note, + previous_fronts=previous_fronts, + deck_examples=deck_examples) return flashcards @@ -54,8 +55,8 @@ def postprocess(note: Note, flashcards: List[Flashcard], deck_name: str): if CONFIG.approve_cards and approve_flashcard(flashcard): approved_flashcards.append(flashcard) elif CONFIG.print_cards: - console.print(f" [cyan]Front:[/cyan] {flashcard.front}") - console.print(f" [cyan]Back:[/cyan] {flashcard.back}") + console.print(f"[cyan]Front:[/cyan] {flashcard.front}") + console.print(f"[cyan]Back:[/cyan] {flashcard.back}") console.print() approved_flashcards.append(flashcard) except KeyboardInterrupt: @@ -326,29 +327,30 @@ def preprocess(args: argparse.Namespace): console.print(f"\n[blue]PROCESSING:[/blue] {note.filename}") - if CONFIG.approve_notes: + with console.indent(): + if CONFIG.approve_notes: + try: + if not approve_note(note): + continue + except KeyboardInterrupt: + console.print("\n[yellow]Operation cancelled by user[/yellow]") + return 0 + try: - if not approve_note(note): + flashcards = process(note, args, deck_examples, target_cards_per_note, previous_fronts[i-1] if previous_fronts else []) + console.print() + + if not flashcards: + console.print("[yellow]WARNING:[/yellow] No flashcards generated, skipping") continue + + cards_added = postprocess(note, flashcards, CONFIG.deck) + total_cards += cards_added + except KeyboardInterrupt: console.print("\n[yellow]Operation cancelled by user[/yellow]") return 0 - try: - flashcards = process(note, args, deck_examples, target_cards_per_note, previous_fronts[i-1] if previous_fronts else []) - console.print() # Clear the indented cursor line - - if not flashcards: - console.print(" [yellow]WARNING:[/yellow] No flashcards generated, skipping") - continue - - cards_added = postprocess(note, flashcards, CONFIG.deck) - total_cards += cards_added - - except KeyboardInterrupt: - console.print("\n[yellow]Operation cancelled by user[/yellow]") - return 0 - console.print("") console.print(Panel(f"[bold green]COMPLETE![/bold green] Added {total_cards}/{CONFIG.max_cards} flashcards to deck '{CONFIG.deck}'", style="green")) return 0 From 372a12dc501a7c981ab4a52ea5b691eba5359133 Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 10:54:10 -0500 Subject: [PATCH 15/16] feat: custom scope ai generation spinner and add provider colors --- obsidianki/ai/models.py | 13 ++++++++++--- obsidianki/cli/config.py | 24 ++++++++++++++++++++++-- obsidianki/cli/processors.py | 8 ++++---- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/obsidianki/ai/models.py b/obsidianki/ai/models.py index 44f13d7..16f5123 100644 --- a/obsidianki/ai/models.py +++ b/obsidianki/ai/models.py @@ -1,3 +1,10 @@ +PROVIDER_COLORS = { + "anthropic": "#D97757", # Anthropic orange + "google": "#4285F4", # Google blue + "openai": "#10A37F", # OpenAI green + "deepseek": "#536DFE", # DeepSeek purple-blue +} + MODEL_MAP = { "Claude Sonnet 4.5": { "provider": "anthropic", @@ -17,9 +24,9 @@ "key_name": "OPENAI_API_KEY", "url": "https://platform.openai.com/api-keys" }, - "Gemini 3": { + "Gemini 3 Pro": { "provider": "google", - "model": "gemini/gemini-2.5-pro", + "model": "gemini/gemini-3-pro-preview", "key_name": "GEMINI_API_KEY", "url": "https://makersuite.google.com/app/apikey" }, @@ -37,7 +44,7 @@ }, "Gemini 3 Flash": { "provider": "google", - "model": "gemini/gemini-3-flash", + "model": "gemini/gemini-3-flash-preview", "key_name": "GEMINI_API_KEY", "url": "https://makersuite.google.com/app/apikey" }, diff --git a/obsidianki/cli/config.py b/obsidianki/cli/config.py index a6761b6..4dfb6f9 100644 --- a/obsidianki/cli/config.py +++ b/obsidianki/cli/config.py @@ -48,8 +48,28 @@ def input(self, prompt: str = "") -> str: return self._console.input(self.prefix + prompt) def status(self, message: str, **kwargs): - """Status spinner with current indentation level.""" - return self._console.status(self.prefix + message, **kwargs) + """Status spinner with current indentation level, colored by AI provider.""" + from rich.live import Live + from rich.spinner import Spinner + from obsidianki.ai.models import MODEL_MAP, PROVIDER_COLORS + + # Get provider color from configured model + color = "white" + try: + model_name = getattr(CONFIG, 'model', '') + provider = MODEL_MAP.get(model_name, {}).get("provider") + color = PROVIDER_COLORS.get(provider, "white") + except: + pass + + # Include indent in spinner frames + base_frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] + indented_frames = [self.prefix + f for f in base_frames] + + spinner = Spinner("dots", text=message, style=color) + spinner.frames = indented_frames + + return Live(spinner, console=self._console, refresh_per_second=10, transient=True) def __getattr__(self, name): """Delegate other methods to underlying console.""" diff --git a/obsidianki/cli/processors.py b/obsidianki/cli/processors.py index 243fd1a..b0d93a6 100644 --- a/obsidianki/cli/processors.py +++ b/obsidianki/cli/processors.py @@ -9,15 +9,15 @@ from obsidianki.cli.models import Note, Flashcard, NotePattern from obsidianki.cli.services import OBSIDIAN, AI, ANKI from obsidianki.cli.utils import encode_path +from obsidianki.cli.config import console #TODO # deck_examples -> List[Flashcard] # previous_fronts -> List[Flashcard]? def process(note: Note, args: argparse.Namespace, deck_examples: List[Dict[str, str]], target_cards_per_note: int, previous_fronts: List[str]) -> List[Flashcard]: - from obsidianki.cli.config import console note.ensure_content() - # Generate flashcards + # Generate flashcards (console is already imported at module level) if args.query and note.path == "query": # Standalone query mode - use direct query generation with console.status("Generating..."): @@ -44,7 +44,7 @@ def process(note: Note, args: argparse.Namespace, deck_examples: List[Dict[str, def postprocess(note: Note, flashcards: List[Flashcard], deck_name: str): """Handle flashcard approval and Anki addition""" - from obsidianki.cli.config import console, CONFIG + from obsidianki.cli.config import CONFIG # Flashcard approval cards_to_add = flashcards @@ -91,7 +91,7 @@ def preprocess(args: argparse.Namespace): """ Entry point for flashcard generation. """ - from obsidianki.cli.config import console, CONFIG + from obsidianki.cli.config import CONFIG from rich.panel import Panel if args.mcp: From 8f2889e1da7d388322c08415d9d836780886609a Mon Sep 17 00:00:00 2001 From: ccmdi Date: Fri, 30 Jan 2026 11:06:54 -0500 Subject: [PATCH 16/16] test: update and add tests for vector/indent console --- tests/test_advanced_features.py | 87 --------- tests/test_indented_console.py | 204 +++++++++++++++++++++ tests/test_multi_provider.py | 16 +- tests/test_vectors.py | 308 ++++++++++++++++++++++++++++++++ 4 files changed, 520 insertions(+), 95 deletions(-) create mode 100644 tests/test_indented_console.py create mode 100644 tests/test_vectors.py diff --git a/tests/test_advanced_features.py b/tests/test_advanced_features.py index ff5de33..43b718c 100644 --- a/tests/test_advanced_features.py +++ b/tests/test_advanced_features.py @@ -308,93 +308,6 @@ def test_hide_note_during_approval(self, mock_services, mock_config): sampled_paths = [n.path for n in sampled] assert test_note.path not in sampled_paths, "Hidden note should not be sampled" - -class TestSemanticPadding: - """Test semantic padding/indentation for flashcard output""" - - def test_flashcard_output_has_padding(self, mock_services, mock_config): - """Test that flashcard output uses proper padding""" - from obsidianki.cli.interactive.approval import approve_flashcard - from obsidianki.cli.models import Flashcard, Note - from rich.console import Console - from io import StringIO - - # Create test flashcard - note = Note( - path="test.md", - filename="test.md", - content="test content", - tags=["test"], - size=100 - ) - flashcard = Flashcard( - front="What is Python?", - back="A programming language", - note=note, - front_original="What is Python?", - back_original="A programming language" - ) - - # Capture console output - string_io = StringIO() - test_console = Console(file=string_io, force_terminal=True, width=120) - - # Mock Confirm.ask to auto-approve - from rich.prompt import Confirm - with patch.object(Confirm, 'ask', return_value=True), \ - patch('obsidianki.cli.interactive.approval.console', test_console): - result = approve_flashcard(flashcard) - - output = string_io.getvalue() - - # Check that output contains Front and Back labels - assert "Front:" in output, "Should display Front label" - assert "Back:" in output, "Should display Back label" - assert "What is Python?" in output - assert "A programming language" in output - - def test_multiline_flashcard_maintains_indentation(self, mock_services, mock_config): - """Test that multiline flashcards maintain proper indentation""" - from obsidianki.cli.interactive.approval import approve_flashcard - from obsidianki.cli.models import Flashcard, Note - from rich.console import Console - from io import StringIO - - # Create flashcard with multiline content - note = Note( - path="test.md", - filename="test.md", - content="test", - tags=[], - size=100 - ) - - multiline_front = "What are the three pillars?\n1. First\n2. Second\n3. Third" - multiline_back = "Answer:\n- Point A\n- Point B\n- Point C" - - flashcard = Flashcard( - front=multiline_front, - back=multiline_back, - note=note, - front_original=multiline_front, - back_original=multiline_back - ) - - string_io = StringIO() - test_console = Console(file=string_io, force_terminal=True, width=120) - - from rich.prompt import Confirm - with patch.object(Confirm, 'ask', return_value=True), \ - patch('obsidianki.cli.interactive.approval.console', test_console): - approve_flashcard(flashcard) - output = string_io.getvalue() - - # Verify multiline content is present (check without color codes) - assert "First" in output - assert "Second" in output - assert "Point A" in output - - class TestTemplates: """Test template functionality""" diff --git a/tests/test_indented_console.py b/tests/test_indented_console.py new file mode 100644 index 0000000..c88f2b6 --- /dev/null +++ b/tests/test_indented_console.py @@ -0,0 +1,204 @@ +"""Tests for IndentedConsole.""" + +import pytest +from unittest.mock import MagicMock, patch +from io import StringIO + +from rich.console import Console +from obsidianki.cli.config import IndentedConsole + + +class TestIndentedConsole: + """Tests for the IndentedConsole class.""" + + @pytest.fixture + def console(self): + """Create an IndentedConsole for testing.""" + base = Console(file=StringIO(), force_terminal=True) + return IndentedConsole(base) + + def test_initial_level_is_zero(self, console): + """Initial indent level should be 0.""" + assert console._level == 0 + + def test_initial_prefix_is_empty(self, console): + """Initial prefix should be empty string.""" + assert console.prefix == "" + + def test_indent_increases_level(self, console): + """indent() should increase the level.""" + assert console._level == 0 + with console.indent(): + assert console._level == 1 + + def test_indent_decreases_on_exit(self, console): + """Level should decrease when exiting indent context.""" + with console.indent(): + assert console._level == 1 + assert console._level == 0 + + def test_nested_indent(self, console): + """Nested indents should accumulate.""" + assert console._level == 0 + with console.indent(): + assert console._level == 1 + with console.indent(): + assert console._level == 2 + with console.indent(): + assert console._level == 3 + assert console._level == 2 + assert console._level == 1 + assert console._level == 0 + + def test_indent_multiple_levels(self, console): + """indent() can increase by multiple levels at once.""" + with console.indent(levels=3): + assert console._level == 3 + assert console._level == 0 + + def test_prefix_reflects_level(self, console): + """Prefix should be indent_str repeated by level.""" + assert console.prefix == "" + with console.indent(): + assert console.prefix == " " + with console.indent(): + assert console.prefix == " " + + def test_custom_indent_string(self): + """Can use custom indent string.""" + base = Console(file=StringIO()) + console = IndentedConsole(base, indent_str="\t") + + assert console.prefix == "" + with console.indent(): + assert console.prefix == "\t" + with console.indent(): + assert console.prefix == "\t\t" + + def test_print_adds_prefix(self, console): + """print() should prepend the current prefix.""" + output = StringIO() + base = Console(file=output, force_terminal=False, width=200) + console = IndentedConsole(base) + + console.print("Level 0") + with console.indent(): + console.print("Level 1") + + result = output.getvalue() + assert "Level 0" in result + assert " Level 1" in result + + def test_print_with_no_args(self, console): + """print() with no args should still work.""" + # Should not raise + console.print() + + def test_indent_context_handles_exceptions(self, console): + """Level should decrease even if exception occurs.""" + try: + with console.indent(): + assert console._level == 1 + raise ValueError("Test error") + except ValueError: + pass + + assert console._level == 0 + + def test_input_adds_prefix(self): + """input() should prepend prefix to prompt.""" + base = MagicMock(spec=Console) + base.input = MagicMock(return_value="user input") + console = IndentedConsole(base) + + with console.indent(): + result = console.input("Enter: ") + + base.input.assert_called_once_with(" Enter: ") + assert result == "user input" + + def test_getattr_delegates_to_base(self): + """Unknown attributes should delegate to base console.""" + base = MagicMock(spec=Console) + base.some_method = MagicMock(return_value="result") + console = IndentedConsole(base) + + result = console.some_method("arg") + + base.some_method.assert_called_once_with("arg") + assert result == "result" + + +class TestIndentedConsoleStatus: + """Tests for the status spinner functionality.""" + + @pytest.fixture + def console(self): + """Create an IndentedConsole for testing.""" + base = Console(file=StringIO(), force_terminal=True) + return IndentedConsole(base) + + @patch('obsidianki.cli.config.CONFIG') + @patch('obsidianki.ai.models.MODEL_MAP', { + "Claude Sonnet 4.5": {"provider": "anthropic"} + }) + @patch('obsidianki.ai.models.PROVIDER_COLORS', { + "anthropic": "#D97757" + }) + def test_status_returns_live_context(self, mock_config, console): + """status() should return a Live context manager.""" + mock_config.model = "Claude Sonnet 4.5" + + status = console.status("Loading...") + + # Should be a Live instance (context manager) + assert hasattr(status, '__enter__') + assert hasattr(status, '__exit__') + + @patch('obsidianki.cli.config.CONFIG') + @patch('obsidianki.ai.models.MODEL_MAP', { + "Claude Sonnet 4.5": {"provider": "anthropic"} + }) + @patch('obsidianki.ai.models.PROVIDER_COLORS', { + "anthropic": "#D97757" + }) + def test_status_spinner_frames_include_indent(self, mock_config, console): + """Spinner frames should include the current indent prefix.""" + mock_config.model = "Claude Sonnet 4.5" + + with console.indent(): + status = console.status("Loading...") + # Access the spinner through the Live object + spinner = status.renderable + # Each frame should start with indent + for frame in spinner.frames: + assert frame.startswith(" ") + + @patch('obsidianki.cli.config.CONFIG') + @patch('obsidianki.ai.models.MODEL_MAP', { + "GPT-5": {"provider": "openai"} + }) + @patch('obsidianki.ai.models.PROVIDER_COLORS', { + "openai": "#10A37F" + }) + def test_status_color_from_provider(self, mock_config, console): + """Status spinner should use provider color.""" + mock_config.model = "GPT-5" + + status = console.status("Loading...") + spinner = status.renderable + + # Spinner style should be the provider color + assert spinner.style == "#10A37F" + + @patch('obsidianki.cli.config.CONFIG') + @patch('obsidianki.ai.models.MODEL_MAP', {}) + @patch('obsidianki.ai.models.PROVIDER_COLORS', {}) + def test_status_defaults_to_white(self, mock_config, console): + """Unknown model should default to white color.""" + mock_config.model = "Unknown Model" + + status = console.status("Loading...") + spinner = status.renderable + + assert spinner.style == "white" diff --git a/tests/test_multi_provider.py b/tests/test_multi_provider.py index ad84ab6..0c6d5b6 100644 --- a/tests/test_multi_provider.py +++ b/tests/test_multi_provider.py @@ -13,12 +13,12 @@ def test_model_map_has_expected_models(self): """Verify MODEL_MAP contains all expected models""" expected_models = [ "Claude Sonnet 4.5", - "Claude Opus 4", + "Claude Opus 4.5", "GPT-5", - "Gemini 3", + "Gemini 3 Pro", "GPT-4o", "GPT-4o Mini", - "Gemini 2.5 Flash", + "Gemini 3 Flash", "DeepSeek V3.1" ] @@ -36,7 +36,7 @@ def test_model_map_entries_have_required_fields(self): def test_anthropic_models_use_correct_provider(self): """Verify Anthropic models use 'anthropic' provider""" - claude_models = ["Claude Sonnet 4.5", "Claude Opus 4"] + claude_models = ["Claude Sonnet 4.5", "Claude Opus 4.5"] for model in claude_models: assert MODEL_MAP[model]["provider"] == "anthropic" @@ -52,7 +52,7 @@ def test_openai_models_use_correct_provider(self): def test_google_models_use_correct_provider(self): """Verify Google models use 'google' provider""" - google_models = ["Gemini 3", "Gemini 2.5 Flash"] + google_models = ["Gemini 3 Pro", "Gemini 3 Flash"] for model in google_models: assert MODEL_MAP[model]["provider"] == "google" @@ -85,8 +85,8 @@ def test_ai_client_respects_model_config(self): """Test that FlashcardAI uses model from CONFIG""" test_cases = [ ("GPT-5", "openai", "gpt-5"), - ("Claude Opus 4", "anthropic", "claude-opus-4-1"), - ("Gemini 3", "google", "gemini/gemini-2.5-pro"), + ("Claude Opus 4.5", "anthropic", "claude-opus-4-5"), + ("Gemini 3 Pro", "google", "gemini/gemini-3-pro-preview"), ] for model_name, expected_provider, expected_model in test_cases: @@ -114,7 +114,7 @@ def test_config_accepts_valid_model_names(self): """Test that config command accepts valid model names""" from obsidianki.cli.config import CONFIG - valid_models = ["GPT-5", "Claude Sonnet 4.5", "Gemini 2.5 Flash"] + valid_models = ["GPT-5", "Claude Sonnet 4.5", "Gemini 3 Flash"] for model in valid_models: assert model in MODEL_MAP, \ diff --git a/tests/test_vectors.py b/tests/test_vectors.py new file mode 100644 index 0000000..7c52847 --- /dev/null +++ b/tests/test_vectors.py @@ -0,0 +1,308 @@ +"""Tests for vector-based semantic deduplication.""" + +import json +import pytest +from unittest.mock import patch, MagicMock +from pathlib import Path + +from obsidianki.ai.vectors import ( + cosine_similarity, + VectorStore, + GeminiEmbedder, + OpenAIEmbedder, +) + + +class TestCosineSimilarity: + """Tests for the cosine_similarity function.""" + + def test_identical_vectors(self): + """Identical vectors should have similarity of 1.0.""" + vec = [1.0, 2.0, 3.0] + assert cosine_similarity(vec, vec) == pytest.approx(1.0) + + def test_orthogonal_vectors(self): + """Orthogonal vectors should have similarity of 0.0.""" + vec_a = [1.0, 0.0, 0.0] + vec_b = [0.0, 1.0, 0.0] + assert cosine_similarity(vec_a, vec_b) == pytest.approx(0.0) + + def test_opposite_vectors(self): + """Opposite vectors should have similarity of -1.0.""" + vec_a = [1.0, 2.0, 3.0] + vec_b = [-1.0, -2.0, -3.0] + assert cosine_similarity(vec_a, vec_b) == pytest.approx(-1.0) + + def test_similar_vectors(self): + """Similar vectors should have high similarity.""" + vec_a = [1.0, 2.0, 3.0] + vec_b = [1.1, 2.1, 3.1] + similarity = cosine_similarity(vec_a, vec_b) + assert similarity > 0.99 + + def test_zero_vector(self): + """Zero vector should return 0.0 similarity.""" + vec_a = [0.0, 0.0, 0.0] + vec_b = [1.0, 2.0, 3.0] + assert cosine_similarity(vec_a, vec_b) == 0.0 + + def test_different_magnitudes(self): + """Vectors with same direction but different magnitudes should be similar.""" + vec_a = [1.0, 2.0, 3.0] + vec_b = [2.0, 4.0, 6.0] # Same direction, 2x magnitude + assert cosine_similarity(vec_a, vec_b) == pytest.approx(1.0) + + +class TestVectorStore: + """Tests for the VectorStore class.""" + + @pytest.fixture + def mock_embedder(self): + """Create a mock embedder that returns predictable embeddings.""" + embedder = MagicMock() + # Return different embeddings based on input text + def embed_side_effect(texts): + embeddings = [] + for text in texts: + # Simple hash-based embedding for testing + hash_val = hash(text) % 1000 + embeddings.append([hash_val / 1000, (hash_val + 1) / 1000, (hash_val + 2) / 1000]) + return embeddings + embedder.embed.side_effect = embed_side_effect + return embedder + + @pytest.fixture + def vector_store(self, mock_embedder, tmp_path): + """Create a VectorStore with mocked embedder and temp storage.""" + with patch('obsidianki.ai.vectors.VECTORS_FILE', tmp_path / 'vectors.json'): + with patch('obsidianki.ai.vectors.CONFIG_DIR', tmp_path): + store = VectorStore() + store._embedder = mock_embedder + yield store + + def test_add_single_card(self, vector_store): + """Test adding a single card to the store.""" + vector_store.add(["What is Python?"]) + assert vector_store.count() == 1 + + def test_add_multiple_cards(self, vector_store): + """Test adding multiple cards.""" + vector_store.add(["Question 1", "Question 2", "Question 3"]) + assert vector_store.count() == 3 + + def test_add_empty_list(self, vector_store): + """Adding empty list should not change count.""" + vector_store.add([]) + assert vector_store.count() == 0 + + def test_add_filters_empty_strings(self, vector_store): + """Empty strings should be filtered out.""" + vector_store.add(["Valid question", "", " ", "Another valid"]) + assert vector_store.count() == 2 + + def test_find_similar_empty_store(self, vector_store): + """Finding similar in empty store should return empty list.""" + matches = vector_store.find_similar("Any question", threshold=0.5) + assert matches == [] + + def test_find_similar_returns_matches(self, vector_store): + """Test that find_similar returns matches above threshold.""" + # Add some cards + vector_store.add(["What is Python?", "What is Java?", "What is Rust?"]) + + # Mock embedder to return similar embedding for query + def query_embed(texts): + # Return embedding very similar to "What is Python?" + return [[hash("What is Python?") % 1000 / 1000 + 0.001, + (hash("What is Python?") % 1000 + 1) / 1000 + 0.001, + (hash("What is Python?") % 1000 + 2) / 1000 + 0.001]] + + vector_store._embedder.embed.side_effect = query_embed + + matches = vector_store.find_similar("What is Python?", threshold=0.5) + # Should find matches (exact behavior depends on mock) + assert isinstance(matches, list) + + def test_find_similar_respects_threshold(self, vector_store): + """High threshold should return fewer matches.""" + vector_store.add(["Question A", "Question B"]) + + # With threshold of 1.0, nothing should match (except exact) + matches = vector_store.find_similar("Something else", threshold=1.0) + assert len(matches) == 0 + + def test_find_similar_excludes_self(self, vector_store): + """Finding similar should not return the same card.""" + vector_store.add(["What is Python?"]) + + # Query with exact same text - should not match itself + vector_store._embedder.embed.return_value = [[0.5, 0.5, 0.5]] + vector_store.data[vector_store._hash("What is Python?")]["embedding"] = [0.5, 0.5, 0.5] + + matches = vector_store.find_similar("What is Python?", threshold=0.0) + # Should not include the exact same card + for text, score in matches: + assert text != "What is Python?" + + def test_find_similar_batch(self, vector_store): + """Test batch similarity checking.""" + vector_store.add(["Existing card 1", "Existing card 2"]) + + results = vector_store.find_similar_batch( + ["New card A", "New card B"], + threshold=0.0 + ) + assert isinstance(results, list) + + def test_clear(self, vector_store): + """Test clearing the store.""" + vector_store.add(["Card 1", "Card 2"]) + assert vector_store.count() == 2 + + vector_store.clear() + assert vector_store.count() == 0 + + # TODO: fix this + # def test_persistence(self, tmp_path): + # """Test that data persists to disk.""" + # vectors_file = tmp_path / 'vectors.json' + + # with patch.dict('os.environ', {'OPENAI_API_KEY': 'test-key'}): + # with patch('obsidianki.ai.vectors.VECTORS_FILE', vectors_file): + # with patch('obsidianki.ai.vectors.CONFIG_DIR', tmp_path): + # # Create store and add data + # store1 = VectorStore() + # store1._embedder = MagicMock() + # store1._embedder.embed.return_value = [[0.1, 0.2, 0.3]] + # store1.add(["Test card"]) + + # # Verify file was created + # assert vectors_file.exists() + + # # Create new store instance - should load data + # store2 = VectorStore() + # assert store2.count() == 1 + + def test_dimension_mismatch_clears_store(self, tmp_path): + """Store should clear if embedding dimensions change.""" + vectors_file = tmp_path / 'vectors.json' + + # Create initial data with 3 dimensions + initial_data = { + "_dims": 3, + "abc123": {"text": "Old card", "embedding": [0.1, 0.2, 0.3]} + } + vectors_file.parent.mkdir(parents=True, exist_ok=True) + with open(vectors_file, 'w') as f: + json.dump(initial_data, f) + + with patch('obsidianki.ai.vectors.VECTORS_FILE', vectors_file): + with patch('obsidianki.ai.vectors.CONFIG_DIR', tmp_path): + # Create store expecting different dimensions + with patch.dict('os.environ', {'OPENAI_API_KEY': 'test'}): + store = VectorStore() + # OpenAI expects 1536 dims, but stored data has 3 + # This should trigger a clear + _ = store.data # Access data to trigger load + + # Store should be cleared (only _dims key) + assert store.count() == 0 + + +class TestGeminiEmbedder: + """Tests for GeminiEmbedder.""" + + def test_requires_api_key(self): + """Should raise error without API key.""" + with patch.dict('os.environ', {}, clear=True): + with pytest.raises(ValueError, match="GEMINI_API_KEY"): + GeminiEmbedder() + + @patch('httpx.Client') + def test_embed_batch(self, mock_client_class): + """Test batch embedding.""" + mock_response = MagicMock() + mock_response.json.return_value = { + "embeddings": [ + {"values": [0.1, 0.2, 0.3]}, + {"values": [0.4, 0.5, 0.6]} + ] + } + mock_response.raise_for_status = MagicMock() + + mock_client = MagicMock() + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=False) + mock_client.post.return_value = mock_response + mock_client_class.return_value = mock_client + + with patch.dict('os.environ', {'GEMINI_API_KEY': 'test-key'}): + embedder = GeminiEmbedder() + result = embedder.embed(["Text 1", "Text 2"]) + + assert len(result) == 2 + assert result[0] == [0.1, 0.2, 0.3] + assert result[1] == [0.4, 0.5, 0.6] + + +class TestOpenAIEmbedder: + """Tests for OpenAIEmbedder.""" + + def test_requires_api_key(self): + """Should raise error without API key.""" + with patch.dict('os.environ', {}, clear=True): + with pytest.raises(ValueError, match="OPENAI_API_KEY"): + OpenAIEmbedder() + + @patch('httpx.Client') + def test_embed_batch(self, mock_client_class): + """Test batch embedding with OpenAI.""" + mock_response = MagicMock() + mock_response.json.return_value = { + "data": [ + {"index": 0, "embedding": [0.1, 0.2, 0.3]}, + {"index": 1, "embedding": [0.4, 0.5, 0.6]} + ] + } + mock_response.raise_for_status = MagicMock() + + mock_client = MagicMock() + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=False) + mock_client.post.return_value = mock_response + mock_client_class.return_value = mock_client + + with patch.dict('os.environ', {'OPENAI_API_KEY': 'test-key'}): + embedder = OpenAIEmbedder() + result = embedder.embed(["Text 1", "Text 2"]) + + assert len(result) == 2 + assert result[0] == [0.1, 0.2, 0.3] + assert result[1] == [0.4, 0.5, 0.6] + + @patch('httpx.Client') + def test_embed_sorts_by_index(self, mock_client_class): + """Test that results are sorted by index.""" + mock_response = MagicMock() + # Return out of order + mock_response.json.return_value = { + "data": [ + {"index": 1, "embedding": [0.4, 0.5, 0.6]}, + {"index": 0, "embedding": [0.1, 0.2, 0.3]} + ] + } + mock_response.raise_for_status = MagicMock() + + mock_client = MagicMock() + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=False) + mock_client.post.return_value = mock_response + mock_client_class.return_value = mock_client + + with patch.dict('os.environ', {'OPENAI_API_KEY': 'test-key'}): + embedder = OpenAIEmbedder() + result = embedder.embed(["Text 1", "Text 2"]) + + # Should be sorted by index + assert result[0] == [0.1, 0.2, 0.3] + assert result[1] == [0.4, 0.5, 0.6]