From 62eb32247568d9f18c719b960afc70e217a6f30d Mon Sep 17 00:00:00 2001 From: mBerasategui-ehu Date: Tue, 23 Jun 2026 21:09:11 +0200 Subject: [PATCH 1/2] feat: grep_rag implementation --- backend/lamb/completions/rag/grep_rag.py | 1165 +++++++++++++++++ .../src/lib/components/AssistantsList.svelte | 4 +- .../assistants/AssistantForm.svelte | 24 +- .../components/ConfigurationPanel.svelte | 14 +- .../components/RagOptionsPanel.svelte | 92 +- .../assistants/logic/assistantFormFetchers.js | 4 +- .../logic/assistantFormState.svelte.js | 31 +- .../assistants/logic/assistantFormSubmit.js | 12 +- frontend/svelte-app/src/lib/locales/ca.json | 25 + frontend/svelte-app/src/lib/locales/en.json | 25 + frontend/svelte-app/src/lib/locales/es.json | 25 + frontend/svelte-app/src/lib/locales/eu.json | 25 + .../src/lib/stores/assistantConfigStore.js | 2 +- .../src/lib/utils/ragProcessorHelpers.js | 20 + .../src/routes/assistants/+page.svelte | 37 +- 15 files changed, 1485 insertions(+), 20 deletions(-) create mode 100644 backend/lamb/completions/rag/grep_rag.py diff --git a/backend/lamb/completions/rag/grep_rag.py b/backend/lamb/completions/rag/grep_rag.py new file mode 100644 index 000000000..e0575e29f --- /dev/null +++ b/backend/lamb/completions/rag/grep_rag.py @@ -0,0 +1,1165 @@ +""" +Grep RAG — Complementary Grep-Based Search Layer + +A RAG processor that uses a small/nano LLM to drive iterative grep/egrep/ripgrep +searches across all files in the assistant's knowledge bases. Works alongside +any embedding-based RAG processor in two modes: + +- **hybrid**: Run both grep AND the underlying RAG in parallel, merge results. +- **primary**: Grep runs first. If no matches found → fall back to underlying RAG. + +The nano model chooses the search tool (grep/egrep/ripgrep), proposes regex patterns, +evaluates result quality, and decides when enough content has been found. +""" + +import asyncio +import json +import os +import re +from typing import Dict, Any, List, Optional, Tuple + +import requests + +from lamb.lamb_classes import Assistant +from lamb.completions.org_config_resolver import OrganizationConfigResolver +from lamb.logging_config import get_logger + +logger = get_logger(__name__, component="RAG") + +# ── Configuration defaults ────────────────────────────────────────────────── + +DEFAULT_CONFIG = { + "grep_mode": "hybrid", # "hybrid" | "primary" + "grep_fallback_rag": "simple_rag", # Underlying RAG to complement or fall back to + "grep_max_tries": 5, # Max search iterations (1 LLM call per try) + "grep_context_lines": 3, # Lines of context before/after each match + "grep_max_total_chars": 8000, # Max total characters sent to main LLM +} + +# ── Nano model prompts ────────────────────────────────────────────────────── + +NANO_SYSTEM_PROMPT = """You are a search assistant. Your job is to help find relevant +information across a collection of knowledge base documents. + +Given a user's question and the results of previous searches, respond with ONE of: + +Option 1 — Propose a new search: +TOOL: +PATTERN: +FLAGS: <-i -C N> +REASON: + +Option 2 — Declare done: +DONE: + +Rules: +1. On the first try, propose the most natural search for the question. +2. If a search finds content, READ the matched context preview in the history. + If the found text CONTAINS the information the user asked for → respond DONE. + Do NOT keep searching for "more details" when the answer is already present. + The main LLM will extract the answer from what you found. +3. If previous searches found NOTHING, try synonyms, related terms, broader/ + narrower terms, fix spelling, or TRANSLATE to the document's language — + this is the main reason retries exist. +4. LANGUAGE AWARENESS: If the user's question is in a different language than + the documents, TRANSLATE your search terms into the document's language. + Include terms in BOTH languages using | alternation when unsure. + Example: user asks "What are the prices?" in English but documents are in + Spanish → pattern: "precios|prices|tarifas|fees|costos|costs" +5. Choose the right tool: + - grep: simple word/phrase searches + - egrep: when you need alternation (|), grouping (), or word boundaries (\\b) + - ripgrep: for large multi-file searches where speed matters +6. Regex tips for egrep: + - Use | for alternatives: "late|missing|overdue" + - Use () for grouping: "grad(e|ing)" + - Use \\b for word boundaries: "\\bexam\\b" +7. Maximum one pattern per response. Keep it SIMPLE but include translations + when the user's language differs from the documents'. +8. The search runs recursively (-r) across ALL files by default.""" + + +NANO_USER_PROMPT_TEMPLATE = """Question: {question} +{doc_language} +Search history: +{history} + +What should I search for next?""" + + +# ── Main processor ────────────────────────────────────────────────────────── + +async def rag_processor( + messages: List[Dict[str, Any]], + assistant: Assistant = None, + request: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Grep RAG processor — nano LLM drives iterative grep across KB documents. + + Args: + messages: Conversation messages + assistant: Assistant object with metadata config + request: Original request dict + + Returns: + Dict with "context" (formatted grep results) and "sources" (metadata) + """ + logger.info("Using grep_rag processor with assistant: %s", + assistant.name if assistant else "None") + + if not assistant: + return {"context": "", "sources": []} + + # Parse config from assistant metadata + config = _parse_config(assistant) + mode = config.get("grep_mode", "hybrid") + fallback_rag = config.get("grep_fallback_rag", "simple_rag") + max_tries = int(config.get("grep_max_tries", 5)) + context_lines = int(config.get("grep_context_lines", 3)) + max_total_chars = int(config.get("grep_max_total_chars", 8000)) + + # Extract the user's question + user_question = _extract_user_question(messages) + if not user_question: + return {"context": "", "sources": []} + + logger.info("Grep RAG mode=%s fallback=%s question='%s...'", + mode, fallback_rag, user_question[:80]) + + # Resolve KB documents (file content + metadata for citations) + documents = await _resolve_kb_documents(assistant) + if not documents: + logger.warning("No KB documents found for assistant %s", assistant.id) + return await _run_fallback_rag(messages, assistant, request, fallback_rag) + + logger.info("Resolved %d documents for grep search", len(documents)) + + # Run the grep search loop + grep_result = await _run_grep_search( + user_question=user_question, + documents=documents, + max_tries=max_tries, + context_lines=context_lines, + max_total_chars=max_total_chars, + assistant_owner=assistant.owner, + ) + + if mode == "primary": + # Grep first, fall back to RAG only if zero matches + if grep_result["matches"]: + logger.info("Primary mode: grep found %d matches, using grep results", + len(grep_result["matches"])) + return _build_grep_response(grep_result["matches"], documents, max_total_chars) + else: + logger.info("Primary mode: grep found no matches, falling back to %s", + fallback_rag) + return await _run_fallback_rag(messages, assistant, request, fallback_rag) + + elif mode == "hybrid": + # Run RAG in parallel with grep (grep already completed above) + # Hybrid always uses simple_rag as the companion — grep handles precision, + # simple semantic coverage is all that's needed alongside it. + logger.info("Hybrid mode: running simple_rag in parallel") + rag_context = await _run_fallback_rag(messages, assistant, request, "simple_rag") + + if grep_result["matches"]: + grep_response = _build_grep_response( + grep_result["matches"], documents, max_total_chars + ) + return _merge_contexts(grep_response, rag_context, max_total_chars) + else: + # No grep matches, just return RAG results + logger.info("Hybrid mode: grep found no matches, using RAG only") + return rag_context + + else: + logger.warning("Unknown grep_mode '%s', falling back to %s", mode, fallback_rag) + return await _run_fallback_rag(messages, assistant, request, fallback_rag) + + +# ── Config parsing ────────────────────────────────────────────────────────── + +def _parse_config(assistant: Assistant) -> Dict[str, Any]: + """Parse grep_rag configuration from assistant metadata.""" + config = dict(DEFAULT_CONFIG) + try: + if assistant.metadata and assistant.metadata.strip(): + metadata = json.loads(assistant.metadata) + for key in DEFAULT_CONFIG: + if key in metadata: + config[key] = metadata[key] + except (json.JSONDecodeError, AttributeError) as e: + logger.warning("Failed to parse assistant metadata: %s", e) + return config + + +# ── Question extraction ───────────────────────────────────────────────────── + +def _extract_user_question(messages: List[Dict[str, Any]]) -> str: + """Extract the last user message from the conversation.""" + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content", "") + if isinstance(content, list): + # Multimodal: extract text parts + text_parts = [ + item.get("text", "") for item in content + if item.get("type") == "text" + ] + return " ".join(text_parts) + return content + return "" + + +# ── KB document resolution ────────────────────────────────────────────────── + +async def _resolve_kb_documents(assistant: Assistant) -> List[Dict[str, Any]]: + """ + Get all KB documents with their content and source metadata. + + For each KB collection attached to the assistant: + 1. List file registries via KB server API + 2. Fetch document content via KB server + 3. Return list of {file_path, content, metadata} + + Returns: + List of document dicts with keys: file_path, content, metadata + """ + if not hasattr(assistant, 'RAG_collections') or not assistant.RAG_collections: + return [] + + # Get KB server configuration + KB_SERVER_URL = None + KB_API_KEY = None + + try: + config_resolver = OrganizationConfigResolver(assistant.owner) + kb_config = config_resolver.get_knowledge_base_config() + if kb_config: + KB_SERVER_URL = kb_config.get("server_url") + KB_API_KEY = kb_config.get("api_token") + except Exception as e: + logger.warning("Error getting org KB config: %s", e) + + if not KB_SERVER_URL: + import config as app_config + KB_SERVER_URL = os.getenv('LAMB_KB_SERVER') + KB_API_KEY = os.getenv('LAMB_KB_SERVER_TOKEN') or getattr( + app_config, 'LAMB_BEARER_TOKEN', '' + ) + + if not KB_SERVER_URL: + logger.error("No KB server URL configured") + return [] + + headers = { + "Authorization": f"Bearer {KB_API_KEY}", + "Content-Type": "application/json", + } + + # Parse collection IDs + collections = [ + cid.strip() for cid in assistant.RAG_collections.split(',') if cid.strip() + ] + + documents = [] + + for collection_id in collections: + try: + # Get file list for this collection + files_url = f"{KB_SERVER_URL}/collections/{collection_id}/files" + resp = requests.get(files_url, headers=headers, timeout=30) + + if resp.status_code != 200: + logger.warning( + "Failed to list files for collection %s: %s", + collection_id, resp.status_code + ) + continue + + file_entries = resp.json() + + # Also query all chunks for full text + query_url = f"{KB_SERVER_URL}/collections/{collection_id}/query" + query_payload = { + "query_text": "", # Empty query to get all? Use top_k instead + "top_k": 500, # Get up to 500 chunks + "threshold": 0.0, + "plugin_params": {}, + } + + # Build a document map: file → full text + # For each file entry, we try to get its content + for entry in file_entries: + try: + # Try to get markdown content URL from processing_stats + md_url = None + stats = entry.get("processing_stats", {}) + if isinstance(stats, dict): + output_files = stats.get("output_files", {}) + md_url = output_files.get("markdown_url") + + # If we have a markdown URL, fetch the full text + content = "" + if md_url: + content = await _fetch_text_content(md_url, headers) + elif entry.get("file_url"): + content = await _fetch_text_content( + entry.get("file_url"), headers + ) + elif entry.get("markdown_preview"): + # Use the preview as fallback + content = entry.get("markdown_preview", "") + + if content: + documents.append({ + "file_path": entry.get("file_path", ""), + "original_filename": entry.get("original_filename", ""), + "content": content, + "metadata": entry, # Full file registry entry + "collection_id": collection_id, + }) + except Exception as e: + logger.debug("Error processing file entry: %s", e) + continue + + except Exception as e: + logger.warning("Error processing collection %s: %s", collection_id, e) + continue + + return documents + + +async def _fetch_text_content(url: str, headers: Dict[str, str]) -> str: + """Fetch text content from a URL. Handles absolute, relative, and localhost URLs.""" + try: + from urllib.parse import urlparse + + parsed = urlparse(url) + + # If URL points to localhost (frontend dev server :5173, backend :9099, + # or any other port), the Docker container can't reach it via localhost. + # Rewrite to use the KB server's internal Docker hostname instead. + if parsed.hostname in ("localhost", "127.0.0.1", "host.docker.internal"): + # Try reading from local filesystem first (dev without Docker) + url_path = parsed.path.lstrip("/") + for candidate in _local_file_candidates(url_path): + if os.path.isfile(candidate): + try: + with open(candidate, "r", encoding="utf-8") as f: + content = f.read() + if content and _is_text_content(content): + logger.debug("Read content from local file: %s", candidate) + return content + except (UnicodeDecodeError, OSError): + continue + + # Fallback: rewrite to KB server (kb:9090 inside Docker) + kb_base = os.getenv("LAMB_KB_SERVER", "http://kb:9090") + url = kb_base.rstrip("/") + parsed.path + logger.debug("Rewrote localhost URL → KB server: %s", url) + + # If URL is relative (starts with /), it's relative to KB server + elif url.startswith("/"): + kb_base = os.getenv("LAMB_KB_SERVER", "") + if kb_base: + url = kb_base.rstrip("/") + url + + resp = requests.get(url, headers=headers, timeout=30) + if resp.status_code == 200: + content = resp.text + if len(content) > 0 and _is_text_content(content): + return content + else: + logger.debug("Skipping non-text content from %s", url) + else: + logger.debug("Failed to fetch %s: %s", url, resp.status_code) + return "" + except Exception as e: + logger.debug("Error fetching %s: %s", url, e) + return "" + + +def _is_text_content(content: str) -> bool: + """Heuristic check if string looks like text (not binary).""" + if not content: + return False + # Check for null bytes (binary indicator) + if '\x00' in content[:1000]: + return False + # Check printable ratio + sample = content[:2000] + printable = sum(1 for c in sample if c.isprintable() or c in '\n\r\t') + return printable / max(len(sample), 1) > 0.8 + + +def _local_file_candidates(url_path: str) -> list: + """Generate candidate filesystem paths for a URL path like 'static/1/dbizi/xxx.txt'.""" + candidates = [] + # 1. Relative to CWD + candidates.append(url_path) + # 2. Project root (when LAMB_PROJECT_PATH is mounted) + project_path = os.getenv("LAMB_PROJECT_PATH", "") + if project_path: + candidates.append(os.path.join(project_path, url_path)) + # Also try kb-server as sibling + candidates.append(os.path.join( + project_path, "lamb-kb-server-stable", "backend", url_path + )) + # 3. Backend's own static/ directory + backend_static = os.path.join(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))), "static") + candidates.append(os.path.join(backend_static, url_path)) + # 4. Production Docker layout (/app) + if url_path.startswith("static/"): + candidates.append(os.path.join("/app", url_path)) + return candidates + + +# ── Language detection ────────────────────────────────────────────────────── + +# Common words unique to each language (high-frequency, low-overlap) +_LANG_MARKERS = { + "Spanish": [ + "que", "los", "las", "del", "una", "por", "para", "como", "está", + "más", "pero", "entre", "desde", "hasta", "porque", "cuando", + "también", "muy", "todo", "ción", "idad", "ente", "aron", "iendo", + ], + "English": [ + "the", "and", "that", "have", "for", "not", "with", "this", + "but", "from", "they", "been", "would", "there", "their", + "about", "which", "after", "could", "should", "tion", "ing ", + ], + "Catalan": [ + "amb", "dels", "una", "com", "més", "també", "tots", "cada", + "sense", "sobre", "entre", "després", "contra", "segons", + "mitjançant", "llarg", "següent", "forma", "qualsevol", + ], + "Basque": [ + "eta", "ere", "duten", "ditu", "arte", "dira", "baten", + "izan", "egin", "dago", "dutenak", "beraz", "orduan", + "bidez", "zehar", "gainean", "artean", "aurka", + ], + "French": [ + "que", "les", "des", "est", "pas", "dans", "nous", "aux", + "sont", "avec", "fait", "leur", "être", "avoir", "cette", + "comme", "plus", "bien", "ment", "tion", "eurs", + ], +} + + +def _detect_document_language(documents: List[Dict[str, Any]]) -> str: + """ + Detect the dominant language of the KB documents by sampling text + and counting language-specific marker words. + + Returns a human-readable string like "Spanish" or empty string if + documents are too short or detection is ambiguous. + """ + if not documents: + return "" + + # Concatenate a sample from all documents (up to ~5000 chars) + sample_parts = [] + total_chars = 0 + for doc in documents: + content = doc.get("content", "") + if content: + sample_parts.append(content[:2000]) + total_chars += len(content[:2000]) + if total_chars >= 5000: + break + + if not sample_parts: + return "" + + sample = " ".join(sample_parts).lower() + + # Count marker words for each language + scores = {} + for lang, markers in _LANG_MARKERS.items(): + score = 0 + for marker in markers: + score += sample.count(marker.lower()) + scores[lang] = score + + # Pick the language with the highest score, but require a minimum threshold + if not scores: + return "" + + best_lang = max(scores, key=scores.get) + best_score = scores[best_lang] + + # Require at least some marker hits to be confident + if best_score < 3: + return "" + + return best_lang + + +def _detect_question_language(question: str) -> str: + """ + Detect the language of the user's question using the same marker-word + approach. Returns language name or empty string. + """ + if not question or len(question) < 10: + return "" + + sample = question.lower() + scores = {} + for lang, markers in _LANG_MARKERS.items(): + score = 0 + for marker in markers: + score += sample.count(marker.lower()) + scores[lang] = score + + if not scores: + return "" + + best_lang = max(scores, key=scores.get) + if scores[best_lang] < 1: + return "" + + return best_lang + + +# ── Grep search loop ──────────────────────────────────────────────────────── + +async def _run_grep_search( + user_question: str, + documents: List[Dict[str, Any]], + max_tries: int, + context_lines: int, + max_total_chars: int, + assistant_owner: str, +) -> Dict[str, Any]: + """ + Run the iterative grep search loop driven by the nano model. + + Returns: + Dict with "matches" (list of match dicts) and "tries" (list of try records) + """ + from lamb.completions.small_fast_model_helper import ( + invoke_small_fast_model, + is_small_fast_model_configured, + ) + + all_matches = [] + tries = [] + already_tried = set() # Track (tool, pattern) to avoid duplicates + + # Detect document language so the nano model can translate search terms + doc_lang = _detect_document_language(documents) + if doc_lang: + logger.info("Detected document language: %s", doc_lang) + + for attempt in range(1, max_tries + 1): + # Build history for the nano model + history = _build_search_history(tries, all_matches) + + # Ask the nano model what to search + nano_response = await _ask_nano_model( + user_question=user_question, + history=history, + assistant_owner=assistant_owner, + doc_language=doc_lang, + ) + + if not nano_response: + logger.warning("Nano model returned no response on try %d", attempt) + break + + if nano_response.get("done"): + logger.info( + "Nano model declared DONE on try %d: %s", + attempt, nano_response.get("reason", "") + ) + tries.append({ + "attempt": attempt, + "done": True, + "reason": nano_response.get("reason", ""), + }) + break + + # Execute the search + tool = nano_response.get("tool", "grep") + pattern = nano_response.get("pattern", "") + reason = nano_response.get("reason", "") + + if not pattern: + logger.warning("Nano model returned empty pattern on try %d", attempt) + tries.append({ + "attempt": attempt, + "tool": tool, + "pattern": "", + "matches": 0, + "error": "Empty pattern", + }) + continue + + # Skip duplicate patterns + dedup_key = (tool, pattern) + if dedup_key in already_tried: + logger.debug("Skipping duplicate pattern: %s %s", tool, pattern) + tries.append({ + "attempt": attempt, + "tool": tool, + "pattern": pattern, + "matches": 0, + "reason": reason, + "skipped": True, + }) + continue + + already_tried.add(dedup_key) + + # Run the search across all documents + logger.info( + "Grep try %d/%d: %s '%s' — %s", + attempt, max_tries, tool, pattern, reason + ) + + matches = _search_across_documents( + documents=documents, + pattern=pattern, + tool=tool, + context_lines=context_lines, + ) + + tries.append({ + "attempt": attempt, + "tool": tool, + "pattern": pattern, + "matches": len(matches), + "reason": reason, + }) + + if matches: + all_matches.extend(matches) + logger.info("Try %d: found %d matches", attempt, len(matches)) + else: + logger.info("Try %d: no matches found", attempt) + + # Deduplicate matches + deduped = _deduplicate_matches(all_matches, context_lines) + + return { + "matches": deduped, + "tries": tries, + } + + +def _build_search_history( + tries: List[Dict[str, Any]], + matches: List[Dict[str, Any]], +) -> str: + """Build a human-readable search history for the nano model.""" + if not tries: + return "(No searches yet)" + + lines = [] + for t in tries: + if t.get("done"): + lines.append(f"Try {t['attempt']}: DONE — {t.get('reason', '')}") + elif t.get("skipped"): + lines.append( + f"Try {t['attempt']}: SKIPPED (duplicate) — " + f"{t.get('tool','')} '{t.get('pattern','')}'" + ) + elif t.get("error"): + lines.append( + f"Try {t['attempt']}: ERROR — {t.get('error','')}" + ) + else: + lines.append( + f"Try {t['attempt']}: {t.get('tool','')} '{t.get('pattern','')}' " + f"— {t.get('matches', 0)} matches — {t.get('reason', '')}" + ) + + # Add a preview of current matches + if matches: + lines.append(f"\nCurrently found {len(matches)} matches across all attempts.") + lines.append("Preview of found content:") + preview_chars = 0 + for m in matches[:5]: + snippet = m.get("context", "")[:300] + lines.append(f" [{m.get('source','?')}]: {snippet}...") + preview_chars += len(snippet) + if preview_chars > 1200: + lines.append(" ... (more matches)") + break + + # If we have a decent amount of content, nudge the nano model to consider stopping + if len(matches) >= 3: + lines.append( + f"\nYou have found {len(matches)} relevant sections. " + f"If this content answers the user's question, respond with DONE. " + f"Only keep searching if the answer is clearly incomplete." + ) + elif len(matches) >= 1: + lines.append( + "\nSome content was found. If it sufficiently answers the question, " + "respond with DONE. Otherwise propose a more targeted search." + ) + + return "\n".join(lines) + + +async def _ask_nano_model( + user_question: str, + history: str, + assistant_owner: str, + doc_language: str = "", +) -> Optional[Dict[str, Any]]: + """ + Ask the nano model what to search for next. + + Args: + user_question: The user's question + history: Formatted search history from previous tries + assistant_owner: Email of the assistant owner (for org config) + doc_language: Detected language of the KB documents (e.g. "Spanish") + + Returns parsed response dict or None on failure. + """ + from lamb.completions.small_fast_model_helper import ( + invoke_small_fast_model, + is_small_fast_model_configured, + ) + + if not is_small_fast_model_configured(assistant_owner): + # Fallback: do a single grep with the user's question keywords + logger.info("No small-fast-model configured, using fallback keyword search") + keywords = _extract_keywords(user_question) + return { + "done": False, + "tool": "grep", + "pattern": "|".join(re.escape(k) for k in keywords[:5]), + "reason": "Fallback keyword search (no nano model configured)", + } + + # Build language context line for the prompt + lang_line = "" + if doc_language: + question_lang = _detect_question_language(user_question) + if question_lang and question_lang != doc_language: + lang_line = ( + f"Documents are in {doc_language}. " + f"User asked in {question_lang}. " + f"Include search terms in BOTH languages using | alternation.\n" + ) + else: + lang_line = f"Documents are in {doc_language}.\n" + + user_prompt = NANO_USER_PROMPT_TEMPLATE.format( + question=user_question, + doc_language=lang_line, + history=history, + ) + + messages = [ + {"role": "system", "content": NANO_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ] + + try: + response = await invoke_small_fast_model( + messages=messages, + assistant_owner=assistant_owner, + stream=False, + ) + + # Extract text from response + text = _extract_response_text(response) + if not text: + return None + + logger.debug("Nano model response: %s", text[:200]) + return _parse_nano_response(text) + + except Exception as e: + logger.error("Error invoking nano model: %s", e) + return None + + +def _extract_response_text(response: Any) -> str: + """Extract text content from various LLM response formats.""" + if isinstance(response, str): + return response + if isinstance(response, dict): + # OpenAI format + choices = response.get("choices", []) + if choices: + return choices[0].get("message", {}).get("content", "") + # Anthropic format + content = response.get("content", []) + if isinstance(content, list): + return "".join( + c.get("text", "") for c in content if c.get("type") == "text" + ) + return "" + + +def _parse_nano_response(text: str) -> Optional[Dict[str, Any]]: + """Parse the nano model's structured response.""" + text = text.strip() + + # Check for DONE — accept multiple formats: + # "DONE: reason" / "DONE - reason" / "DONE" + # "I'm DONE" / "We are DONE" + # First-line "DONE" anywhere followed by reason + done_patterns = [ + r"^DONE\b[:\-]?\s*(.*)", # "DONE", "DONE: reason", "DONE - reason" + r"\bDONE\b[:\-]?\s*(.*)", # "I'm DONE", "We are DONE: ..." + ] + + for pattern in done_patterns: + m = re.match(pattern, text, re.IGNORECASE) + if m: + reason = m.group(1).strip() if m.group(1) else "" + logger.info("Nano model declared DONE: %s", reason if reason else "(no reason)") + return {"done": True, "reason": reason} + + # If the first line says DONE but the parser above didn't catch it + first_line = text.split("\n")[0].strip() + if re.match(r".*\bDONE\b", first_line, re.IGNORECASE): + # Extract anything after DONE on that line + done_idx = first_line.upper().find("DONE") + reason = first_line[done_idx + 4:].strip().lstrip(":-").strip() + logger.info("Nano model declared DONE (first-line match): %s", reason if reason else "(no reason)") + return {"done": True, "reason": reason} + + # Parse structured TOOL/PATTERN/FLAGS/REASON response + result = {"done": False} + lines = text.split("\n") + + for line in lines: + line = line.strip() + if line.upper().startswith("TOOL:"): + result["tool"] = line[5:].strip().lower() + elif line.upper().startswith("PATTERN:"): + result["pattern"] = line[8:].strip() + elif line.upper().startswith("FLAGS:"): + result["flags"] = line[6:].strip() + elif line.upper().startswith("REASON:"): + result["reason"] = line[7:].strip() + + # Fallback: try to extract pattern from simpler formats + if not result.get("pattern"): + # Look for a quoted pattern or regex-like content + quoted = re.findall(r'["\'](.+?)["\']', text) + if quoted: + result["pattern"] = quoted[0] + # Assume grep as default tool + if not result.get("tool"): + result["tool"] = "grep" + return result + + # Try to find pattern after common labels + for label in ["pattern:", "PATTERN:", "regex:", "REGEX:"]: + idx = text.lower().find(label.lower()) + if idx >= 0: + result["pattern"] = text[idx + len(label):].split("\n")[0].strip() + break + + if not result.get("pattern"): + logger.debug("Could not parse pattern from nano response: %s", text[:200]) + return None + + if not result.get("tool"): + result["tool"] = "grep" + + return result + + +def _extract_keywords(text: str, max_keywords: int = 5) -> List[str]: + """Extract meaningful keywords from user question for fallback search.""" + # Simple keyword extraction: split, remove short/common words, take unique + stop_words = { + "a", "an", "the", "is", "are", "was", "were", "be", "been", + "in", "on", "at", "to", "for", "of", "with", "by", "from", + "and", "or", "but", "not", "no", "yes", "what", "how", "when", + "where", "who", "why", "which", "do", "does", "did", "can", + "could", "will", "would", "should", "may", "might", "this", + "that", "these", "those", "it", "its", "i", "me", "my", "we", + "our", "you", "your", "he", "she", "they", "them", + } + words = re.findall(r'\b[a-zA-Z]{3,}\b', text.lower()) + keywords = [] + seen = set() + for w in words: + if w not in stop_words and w not in seen: + keywords.append(w) + seen.add(w) + if len(keywords) >= max_keywords: + break + return keywords + + +# ── In-memory search engine ───────────────────────────────────────────────── + +def _search_across_documents( + documents: List[Dict[str, Any]], + pattern: str, + tool: str, + context_lines: int, +) -> List[Dict[str, Any]]: + """ + Run regex search across all documents. + + Args: + documents: List of document dicts with 'content', 'file_path', 'original_filename' + pattern: Regex pattern from nano model + tool: "grep", "egrep", or "ripgrep" (all use Python re internally) + context_lines: Lines of context before/after each match + + Returns: + List of match dicts + """ + # Compile regex (case-insensitive by default, matching grep -i behavior) + try: + compiled = re.compile(pattern, re.IGNORECASE | re.MULTILINE) + except re.error as e: + logger.debug("Invalid regex pattern '%s': %s", pattern, e) + return [] + + all_matches = [] + + for doc in documents: + content = doc.get("content", "") + if not content: + continue + + lines = content.split("\n") + source_label = doc.get("original_filename") or doc.get("file_path", "unknown") + + for i, line in enumerate(lines): + if compiled.search(line): + start = max(0, i - context_lines) + end = min(len(lines), i + context_lines + 1) + + all_matches.append({ + "file_path": doc.get("file_path", ""), + "source": source_label, + "line_num": i + 1, + "matched_line": line.strip(), + "context": "\n".join(lines[start:end]), + "context_start": start, + "context_end": end, + "metadata": doc.get("metadata", {}), + "collection_id": doc.get("collection_id", ""), + }) + + return all_matches + + +# ── Deduplication ─────────────────────────────────────────────────────────── + +def _deduplicate_matches( + matches: List[Dict[str, Any]], + context_lines: int, +) -> List[Dict[str, Any]]: + """Deduplicate matches, merging overlapping context ranges per file.""" + if not matches: + return [] + + # Group by file_path + by_file: Dict[str, List[Dict[str, Any]]] = {} + for match in matches: + fp = match.get("file_path", "__unknown__") + by_file.setdefault(fp, []).append(match) + + merged = [] + for file_path, file_matches in by_file.items(): + # Sort by context_start + file_matches.sort(key=lambda m: m.get("context_start", 0)) + + file_merged = [dict(file_matches[0])] + for match in file_matches[1:]: + last = file_merged[-1] + # If this match's context overlaps with the last, merge + if match["context_start"] <= last["context_end"] + context_lines: + # Merge by extending context range + last["context_end"] = max(last["context_end"], match["context_end"]) + # Rebuild context from the merged range (approximate) + last["matched_line"] = ( + last.get("matched_line", "") + + " | " + + match.get("matched_line", "") + ) + else: + file_merged.append(dict(match)) + + merged.extend(file_merged) + + return merged + + +# ── Context formatting ────────────────────────────────────────────────────── + +def _build_grep_response( + matches: List[Dict[str, Any]], + documents: List[Dict[str, Any]], + max_total_chars: int, +) -> Dict[str, Any]: + """Build the final response dict from grep matches.""" + # Build file_path → metadata map for source resolution + docs_map = {d["file_path"]: d for d in documents} + + blocks = [] + sources = [] + total_chars = 0 + seen_sources = set() + + for match in matches: + meta = match.get("metadata", {}) + fp = match.get("file_path", "") + + # Resolve best source label + source_label = _best_source_label(meta, match.get("source", "Unknown")) + source_url = _best_source_url(meta) + + header = f"### {source_label}" + if source_url: + header += f" ({source_url})" + + body = match.get("context", "").strip() + block = f"{header}\n{body}" + + if total_chars + len(block) > max_total_chars: + break + + blocks.append(block) + total_chars += len(block) + + # Track sources + source_key = source_url or source_label + if source_key not in seen_sources: + sources.append({ + "title": source_label, + "url": source_url, + }) + seen_sources.add(source_key) + + context = "\n\n".join(blocks) if blocks else "" + + return { + "context": context, + "sources": sources, + } + + +def _best_source_label(metadata: Dict[str, Any], fallback: str) -> str: + """Resolve the best human-readable source label from metadata.""" + # Try various metadata fields in order of preference + if isinstance(metadata, dict): + # URL-ingested documents often have page_url + if metadata.get("page_url"): + return fallback # fallback usually has filename + # Check for original filename + if metadata.get("original_filename"): + return metadata["original_filename"] + # Check processing stats + stats = metadata.get("processing_stats", {}) + if isinstance(stats, dict): + output_files = stats.get("output_files", {}) + if output_files: + return fallback + + return fallback + + +def _best_source_url(metadata: Dict[str, Any]) -> str: + """Resolve the best source URL for citation.""" + if isinstance(metadata, dict): + stats = metadata.get("processing_stats", {}) + if isinstance(stats, dict): + output_files = stats.get("output_files", {}) + # Prefer markdown URL + if output_files.get("markdown_url"): + return output_files["markdown_url"] + if output_files.get("original_file_url"): + return output_files["original_file_url"] + + # Fall back to file_url + if metadata.get("file_url"): + return metadata["file_url"] + + return "" + + +# ── RAG fallback ──────────────────────────────────────────────────────────── + +async def _run_fallback_rag( + messages: List[Dict[str, Any]], + assistant: Assistant, + request: Optional[Dict[str, Any]], + fallback_rag: str, +) -> Dict[str, Any]: + """Run the specified fallback RAG processor.""" + import importlib + + try: + module = importlib.import_module( + f"lamb.completions.rag.{fallback_rag}" + ) + rag_func = getattr(module, "rag_processor") + + if asyncio.iscoroutinefunction(rag_func): + return await rag_func( + messages=messages, assistant=assistant, request=request + ) + else: + return rag_func( + messages=messages, assistant=assistant, request=request + ) + except Exception as e: + logger.error("Error running fallback RAG '%s': %s", fallback_rag, e) + return {"context": "", "sources": []} + + +# ── Context merging ───────────────────────────────────────────────────────── + +def _merge_contexts( + grep_response: Dict[str, Any], + rag_response: Dict[str, Any], + max_total_chars: int, +) -> Dict[str, Any]: + """Merge grep results with embedding RAG results.""" + rag_context = rag_response.get("context", "") if isinstance(rag_response, dict) else "" + grep_context = grep_response.get("context", "") if isinstance(grep_response, dict) else "" + + rag_sources = rag_response.get("sources", []) if isinstance(rag_response, dict) else [] + grep_sources = grep_response.get("sources", []) if isinstance(grep_response, dict) else [] + + # Combine contexts with a clear separator + parts = [] + total = 0 + + if rag_context: + parts.append(rag_context) + total += len(rag_context) + + if grep_context: + remaining = max_total_chars - total + if remaining > 200: # Only add grep if there's meaningful space + if parts: + parts.append("\n\n---\n\n## Exact Keyword Matches\n\n") + total += len(parts[-1]) + if len(grep_context) > remaining: + grep_context = grep_context[:remaining] + "\n\n[... results truncated ...]" + parts.append(grep_context) + + # Merge sources, deduplicating by URL + merged_sources = list(rag_sources) + seen_urls = {s.get("url", "") for s in rag_sources if s.get("url")} + for s in grep_sources: + if s.get("url") and s["url"] not in seen_urls: + merged_sources.append(s) + seen_urls.add(s["url"]) + + return { + "context": "".join(parts), + "sources": merged_sources, + } diff --git a/frontend/svelte-app/src/lib/components/AssistantsList.svelte b/frontend/svelte-app/src/lib/components/AssistantsList.svelte index 76d7748ab..1758916d3 100644 --- a/frontend/svelte-app/src/lib/components/AssistantsList.svelte +++ b/frontend/svelte-app/src/lib/components/AssistantsList.svelte @@ -557,8 +557,8 @@ - - {#if callback.rag_processor === 'simple_rag'} + + {#if callback.rag_processor === 'simple_rag' || callback.rag_processor === 'grep_rag'}
diff --git a/frontend/svelte-app/src/lib/components/assistants/AssistantForm.svelte b/frontend/svelte-app/src/lib/components/assistants/AssistantForm.svelte index aaa118e4c..92f52e592 100644 --- a/frontend/svelte-app/src/lib/components/assistants/AssistantForm.svelte +++ b/frontend/svelte-app/src/lib/components/assistants/AssistantForm.svelte @@ -6,7 +6,7 @@ import { get } from 'svelte/store'; import { createAssistant, updateAssistant } from '$lib/services/assistantService'; import { extractModelsFromConnectorData, selectModel } from './logic/assistantFormUtils.svelte.js'; - import { isKbBasedRag, isSingleFileRag, isRubricRag } from '$lib/utils/ragProcessorHelpers.js'; + import { isKbBasedRag, isSingleFileRag, isRubricRag, isGrepRag } from '$lib/utils/ragProcessorHelpers.js'; import { validateImportedAssistant } from './logic/importAssistantValidator.js'; import { createAssistantFormState, resetFormFieldsToDefaults, populateFormFields, revertToInitial, clearRagDependentState, handleFieldChange } from './logic/assistantFormState.svelte.js'; import { fetchKnowledgeBases, fetchRubricsList, fetchUserFiles } from './logic/assistantFormFetchers.js'; @@ -163,6 +163,11 @@ if (!form.rubricsFetchAttempted && !form.loadingRubrics) { tick().then(doFetchRubricsList); } + } else if (isGrepRag(form.selectedRagProcessor) && form.configInitialized) { + // Grep RAG searches across KB documents — fetch KBs + if (!form.kbFetchAttempted && !form.loadingKnowledgeBases) { + doFetchKnowledgeBases(); + } } else { // Clear KB state AND reset attempted flag if RAG processor changes away clearRagDependentState(form); @@ -315,14 +320,22 @@ // Populate RAG specific fields // FIX FOR ISSUE #96: Apply Load-Then-Select pattern for imports too - if (isKbBasedRag(form.selectedRagProcessor)) { - form.selectedFilePath = ''; // Clear file path if switching to simple RAG, context_aware_rag, or hierarchical_rag + if (isKbBasedRag(form.selectedRagProcessor) || isGrepRag(form.selectedRagProcessor)) { + form.selectedFilePath = ''; // Clear file path if switching to simple RAG, context_aware_rag, hierarchical_rag, or grep_rag // Fetch KBs BEFORE setting selections if (!form.kbFetchAttempted) { await doFetchKnowledgeBases(); // ✅ WAIT for KBs to load } // NOW set selections when KB list is ready form.selectedKnowledgeBases = parsedData.RAG_collections?.split(',').filter(Boolean) || []; + // Populate Grep RAG fields if applicable + if (isGrepRag(form.selectedRagProcessor)) { + form.grepMode = callbackData.grep_mode || 'hybrid'; + form.grepFallbackRag = callbackData.grep_fallback_rag || 'simple_rag'; + form.grepMaxTries = callbackData.grep_max_tries ?? 5; + form.grepContextLines = callbackData.grep_context_lines ?? 3; + form.grepMaxTotalChars = callbackData.grep_max_total_chars ?? 8000; + } } else if (isSingleFileRag(form.selectedRagProcessor)) { form.selectedKnowledgeBases = []; // Clear KBs if switching to single file RAG // Fetch files BEFORE setting selection @@ -459,6 +472,11 @@ fileError={form.fileError} onFilesChanged={() => doFetchUserFiles(true)} onchange={() => handleFieldChange(form)} + bind:grepMode={form.grepMode} + bind:grepFallbackRag={form.grepFallbackRag} + bind:grepMaxTries={form.grepMaxTries} + bind:grepContextLines={form.grepContextLines} + bind:grepMaxTotalChars={form.grepMaxTotalChars} />
diff --git a/frontend/svelte-app/src/lib/components/assistants/components/ConfigurationPanel.svelte b/frontend/svelte-app/src/lib/components/assistants/components/ConfigurationPanel.svelte index 9d0190847..2e4eb386b 100644 --- a/frontend/svelte-app/src/lib/components/assistants/components/ConfigurationPanel.svelte +++ b/frontend/svelte-app/src/lib/components/assistants/components/ConfigurationPanel.svelte @@ -33,7 +33,13 @@ loadingFiles = false, fileError = '', onFilesChanged, - onchange + onchange, + // Grep RAG fields + grepMode = $bindable('hybrid'), + grepFallbackRag = $bindable('simple_rag'), + grepMaxTries = $bindable(5), + grepContextLines = $bindable(3), + grepMaxTotalChars = $bindable(8000) } = $props(); let currentConnectorMetadata = $derived.by(() => { @@ -208,6 +214,12 @@ fileError={fileError} {formState} {onFilesChanged} + {ragProcessors} + bind:grepMode + bind:grepFallbackRag + bind:grepMaxTries + bind:grepContextLines + bind:grepMaxTotalChars /> {/if} diff --git a/frontend/svelte-app/src/lib/components/assistants/components/RagOptionsPanel.svelte b/frontend/svelte-app/src/lib/components/assistants/components/RagOptionsPanel.svelte index 45c4d4a15..ecd139ba3 100644 --- a/frontend/svelte-app/src/lib/components/assistants/components/RagOptionsPanel.svelte +++ b/frontend/svelte-app/src/lib/components/assistants/components/RagOptionsPanel.svelte @@ -1,7 +1,7 @@
@@ -66,4 +74,80 @@ {onFilesChanged} /> {/if} + {#if showGrepOptions} +
+
+ {$_('assistants.form.grepRag.sectionTitle', { default: 'Grep RAG Configuration' })} +
+ + +
+ + +

+ {$_('assistants.form.grepRag.mode.description', { default: 'How grep interacts with embedding-based RAG' })} +

+
+ + + {#if grepMode === 'primary'} +
+ + +

+ {$_('assistants.form.grepRag.fallbackRag.description', { default: 'Embedding RAG to fall back to when grep finds no matches' })} +

+
+ {/if} + + +
+ + +

+ {$_('assistants.form.grepRag.maxTries.description', { default: 'Maximum number of search iterations (1-10)' })} +

+
+ + +
+ + +

+ {$_('assistants.form.grepRag.contextLines.description', { default: 'Lines of context before/after each match (1-10)' })} +

+
+ + +
+ + +

+ {$_('assistants.form.grepRag.maxTotalChars.description', { default: 'Max total characters of grep results sent to main LLM (1000-32000)' })} +

+
+
+ {/if}
diff --git a/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormFetchers.js b/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormFetchers.js index 89981f950..9548fd3a8 100644 --- a/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormFetchers.js +++ b/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormFetchers.js @@ -7,7 +7,7 @@ import { getUserKnowledgeBases, getSharedKnowledgeBases } from '$lib/services/knowledgeBaseService'; import { fetchAccessibleRubrics } from '$lib/services/rubricService'; import { apiJson } from '$lib/services/apiClient'; -import { isKbBasedRag, isRubricRag } from '$lib/utils/ragProcessorHelpers.js'; +import { isKbBasedRag, isRubricRag, isGrepRag } from '$lib/utils/ragProcessorHelpers.js'; import { getAssistantMetadataObject } from '$lib/utils/assistantData'; /** @@ -16,7 +16,7 @@ import { getAssistantMetadataObject } from '$lib/utils/assistantData'; */ export async function fetchKnowledgeBases(form) { if (form.loadingKnowledgeBases || form.kbFetchAttempted) return; - if (!isKbBasedRag(form.selectedRagProcessor)) return; + if (!isKbBasedRag(form.selectedRagProcessor) && !isGrepRag(form.selectedRagProcessor)) return; form.loadingKnowledgeBases = true; form.knowledgeBaseError = ''; diff --git a/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormState.svelte.js b/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormState.svelte.js index 3c7a7f385..ee454ddd1 100644 --- a/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormState.svelte.js +++ b/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormState.svelte.js @@ -16,7 +16,7 @@ import { get } from 'svelte/store'; import { assistantConfigStore } from '$lib/stores/assistantConfigStore'; -import { isKbBasedRag, isSingleFileRag, isRubricRag, normalizeRagProcessor } from '$lib/utils/ragProcessorHelpers.js'; +import { isKbBasedRag, isSingleFileRag, isRubricRag, isGrepRag, normalizeRagProcessor } from '$lib/utils/ragProcessorHelpers.js'; import { loadRagPlaceholders, selectModel } from './assistantFormUtils.svelte.js'; import { getAssistantMetadataObject } from '$lib/utils/assistantData'; @@ -84,6 +84,18 @@ export function createAssistantFormState() { rubricError: '', rubricsFetchAttempted: false, + // --- Grep RAG state --- + /** @type {'hybrid' | 'primary'} */ + grepMode: 'hybrid', + /** @type {string} */ + grepFallbackRag: 'simple_rag', + /** @type {number} */ + grepMaxTries: 5, + /** @type {number} */ + grepContextLines: 3, + /** @type {number} */ + grepMaxTotalChars: 8000, + // --- UI / loading state --- formError: '', formLoading: false, @@ -187,6 +199,15 @@ export function populateFormFields(form, data, getAvailableModels, preserveDescr } } + // Grep RAG fields + if (isGrepRag(form.selectedRagProcessor)) { + form.grepMode = metadata?.grep_mode || 'hybrid'; + form.grepFallbackRag = metadata?.grep_fallback_rag || 'simple_rag'; + form.grepMaxTries = metadata?.grep_max_tries ?? 5; + form.grepContextLines = metadata?.grep_context_lines ?? 3; + form.grepMaxTotalChars = metadata?.grep_max_total_chars ?? 8000; + } + // Vision capability try { form.visionEnabled = metadata?.capabilities?.vision || false; @@ -235,4 +256,12 @@ export function clearRagDependentState(form) { form.selectedRubricId = ''; form.rubricFormat = 'markdown'; } + + if (!isGrepRag(form.selectedRagProcessor)) { + form.grepMode = 'hybrid'; + form.grepFallbackRag = 'simple_rag'; + form.grepMaxTries = 5; + form.grepContextLines = 3; + form.grepMaxTotalChars = 8000; + } } diff --git a/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormSubmit.js b/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormSubmit.js index 12765fca0..8fdeda2ab 100644 --- a/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormSubmit.js +++ b/frontend/svelte-app/src/lib/components/assistants/logic/assistantFormSubmit.js @@ -4,7 +4,7 @@ * Extracted from AssistantForm.svelte to enable isolated testing. */ -import { isKbBasedRag, isSingleFileRag, isRubricRag } from '$lib/utils/ragProcessorHelpers.js'; +import { isKbBasedRag, isSingleFileRag, isRubricRag, isGrepRag } from '$lib/utils/ragProcessorHelpers.js'; /** * Validates form data before submission. @@ -42,13 +42,21 @@ export function buildAssistantPayload(form) { metadataObj.rubric_format = form.rubricFormat; } + if (isGrepRag(form.selectedRagProcessor)) { + metadataObj.grep_mode = form.grepMode; + metadataObj.grep_fallback_rag = form.grepFallbackRag; + metadataObj.grep_max_tries = form.grepMaxTries; + metadataObj.grep_context_lines = form.grepContextLines; + metadataObj.grep_max_total_chars = form.grepMaxTotalChars; + } + return { name: form.name.trim(), description: form.description, system_prompt: form.system_prompt, prompt_template: form.prompt_template, RAG_Top_k: Number(form.RAG_Top_k) || 3, - RAG_collections: isKbBasedRag(form.selectedRagProcessor) ? form.selectedKnowledgeBases.join(',') : '', + RAG_collections: (isKbBasedRag(form.selectedRagProcessor) || isGrepRag(form.selectedRagProcessor)) ? form.selectedKnowledgeBases.join(',') : '', metadata: JSON.stringify(metadataObj), pre_retrieval_endpoint: '', post_retrieval_endpoint: '', diff --git a/frontend/svelte-app/src/lib/locales/ca.json b/frontend/svelte-app/src/lib/locales/ca.json index 486905f79..bf759c1cf 100644 --- a/frontend/svelte-app/src/lib/locales/ca.json +++ b/frontend/svelte-app/src/lib/locales/ca.json @@ -203,6 +203,31 @@ "error": "Error en carregar arxius:", "noneFound": "No s'han trobat arxius. Si us plau puja un arxiu.", "required": "Si us plau selecciona un arxiu" + }, + "grepRag": { + "sectionTitle": "Configuració de Grep RAG", + "mode": { + "label": "Mode", + "description": "Com interactua grep amb el RAG basat en embeddings", + "hybrid": "Híbrid (grep + RAG en paral·lel)", + "primary": "Principal (grep primer, RAG com a respatller)" + }, + "fallbackRag": { + "label": "RAG de Respatller", + "description": "RAG d'embeddings per complementar o com a respatller" + }, + "maxTries": { + "label": "Intents Màxims", + "description": "Nombre màxim d'iteracions de cerca (1-10)" + }, + "contextLines": { + "label": "Línies de Context", + "description": "Línies de context abans/després de cada coincidència (1-10)" + }, + "maxTotalChars": { + "label": "Caràcters Màxims", + "description": "Màxim total de caràcters de resultats enviats al LLM (1000-32000)" + } } }, "noDescription": "Sense descripció", diff --git a/frontend/svelte-app/src/lib/locales/en.json b/frontend/svelte-app/src/lib/locales/en.json index 789b12763..7c0c80a94 100644 --- a/frontend/svelte-app/src/lib/locales/en.json +++ b/frontend/svelte-app/src/lib/locales/en.json @@ -342,6 +342,31 @@ "error": "Error loading files:", "noneFound": "No files found. Please upload a file.", "required": "Please select a file" + }, + "grepRag": { + "sectionTitle": "Grep RAG Configuration", + "mode": { + "label": "Mode", + "description": "How grep interacts with embedding-based RAG", + "hybrid": "Hybrid (grep + RAG in parallel)", + "primary": "Primary (grep first, RAG fallback)" + }, + "fallbackRag": { + "label": "Fallback RAG", + "description": "Embedding RAG to complement or fall back to" + }, + "maxTries": { + "label": "Max Search Tries", + "description": "Maximum number of search iterations (1-10)" + }, + "contextLines": { + "label": "Context Lines", + "description": "Lines of context before/after each match (1-10)" + }, + "maxTotalChars": { + "label": "Max Result Characters", + "description": "Max total characters of grep results sent to main LLM (1000-32000)" + } } } }, diff --git a/frontend/svelte-app/src/lib/locales/es.json b/frontend/svelte-app/src/lib/locales/es.json index 0b09d394d..221abeaf7 100644 --- a/frontend/svelte-app/src/lib/locales/es.json +++ b/frontend/svelte-app/src/lib/locales/es.json @@ -203,6 +203,31 @@ "error": "Error al cargar archivos:", "noneFound": "No se encontraron archivos. Por favor sube un archivo.", "required": "Por favor selecciona un archivo" + }, + "grepRag": { + "sectionTitle": "Configuración de Grep RAG", + "mode": { + "label": "Modo", + "description": "Cómo interactúa grep con el RAG basado en embeddings", + "hybrid": "Híbrido (grep + RAG en paralelo)", + "primary": "Principal (grep primero, RAG como respaldo)" + }, + "fallbackRag": { + "label": "RAG de Respaldo", + "description": "RAG de embeddings para complementar o como respaldo" + }, + "maxTries": { + "label": "Intentos Máximos", + "description": "Número máximo de iteraciones de búsqueda (1-10)" + }, + "contextLines": { + "label": "Líneas de Contexto", + "description": "Líneas de contexto antes/después de cada coincidencia (1-10)" + }, + "maxTotalChars": { + "label": "Caracteres Máximos", + "description": "Máximo total de caracteres de resultados enviados al LLM (1000-32000)" + } } }, "noDescription": "Sin descripción", diff --git a/frontend/svelte-app/src/lib/locales/eu.json b/frontend/svelte-app/src/lib/locales/eu.json index 740440408..9caa6a9e6 100644 --- a/frontend/svelte-app/src/lib/locales/eu.json +++ b/frontend/svelte-app/src/lib/locales/eu.json @@ -353,6 +353,31 @@ "error": "Errorea fitxategiak kargatzean:", "noneFound": "Ez da fitxategirik aurkitu. Mesedez igo fitxategi bat.", "required": "Mesedez hautatu fitxategi bat" + }, + "grepRag": { + "sectionTitle": "Grep RAG Konfigurazioa", + "mode": { + "label": "Modua", + "description": "Grepek embedding-etan oinarritutako RAGarekin nola elkarreragiten duen", + "hybrid": "Hibridoa (grep + RAG paraleloan)", + "primary": "Nagusia (grep lehenik, RAG ordezko gisa)" + }, + "fallbackRag": { + "label": "Ordezko RAG", + "description": "Embedding RAG osagarri edo ordezko gisa" + }, + "maxTries": { + "label": "Gehienezko Saiakerak", + "description": "Bilaketa iterazioen gehienezko kopurua (1-10)" + }, + "contextLines": { + "label": "Testuinguru Lerroak", + "description": "Bat-etortze bakoitzaren aurretik/ondoren testuinguru lerroak (1-10)" + }, + "maxTotalChars": { + "label": "Gehienezko Karaktereak", + "description": "LLM-ra bidalitako grep emaitzen gehienezko karaktereak (1000-32000)" + } } } }, diff --git a/frontend/svelte-app/src/lib/stores/assistantConfigStore.js b/frontend/svelte-app/src/lib/stores/assistantConfigStore.js index c4a6469fe..ab40e5af9 100644 --- a/frontend/svelte-app/src/lib/stores/assistantConfigStore.js +++ b/frontend/svelte-app/src/lib/stores/assistantConfigStore.js @@ -84,7 +84,7 @@ function getFallbackCapabilities() { const capabilities = { prompt_processors: ['simple_augment'], connectors: {}, - rag_processors: ['no_rag', 'simple_rag', 'context_aware_rag', 'single_file_rag'] + rag_processors: ['no_rag', 'simple_rag', 'context_aware_rag', 'single_file_rag', 'grep_rag'] }; return capabilities; } diff --git a/frontend/svelte-app/src/lib/utils/ragProcessorHelpers.js b/frontend/svelte-app/src/lib/utils/ragProcessorHelpers.js index f2c9499e8..f289ab54c 100644 --- a/frontend/svelte-app/src/lib/utils/ragProcessorHelpers.js +++ b/frontend/svelte-app/src/lib/utils/ragProcessorHelpers.js @@ -19,6 +19,8 @@ export const RAG_TYPES = Object.freeze({ SINGLE_FILE: ['single_file_rag'], /** RAG processor that uses rubrics */ RUBRIC: ['rubric_rag'], + /** RAG processor that uses iterative grep across KB documents */ + GREP: ['grep_rag'], /** No RAG processing */ NONE: ['no_rag'] }); @@ -50,6 +52,15 @@ export function isRubricRag(processor) { return RAG_TYPES.RUBRIC.includes(processor); } +/** + * Returns true if the processor uses iterative grep across KB documents. + * @param {string} processor + * @returns {boolean} + */ +export function isGrepRag(processor) { + return RAG_TYPES.GREP.includes(processor); +} + /** * Returns true if no RAG processing is configured. * @param {string} processor @@ -68,6 +79,15 @@ export function hasRagOptions(processor) { return !!processor && !isNoRag(processor); } +/** + * Returns true if the processor is grep-based and needs grep-specific UI. + * @param {string} processor + * @returns {boolean} + */ +export function isGrepBasedRag(processor) { + return isGrepRag(processor); +} + /** * Normalizes RAG processor value from config (handles "no rag" -> "no_rag"). * @param {string} processor diff --git a/frontend/svelte-app/src/routes/assistants/+page.svelte b/frontend/svelte-app/src/routes/assistants/+page.svelte index f1e4cf8a3..c7db1067f 100644 --- a/frontend/svelte-app/src/routes/assistants/+page.svelte +++ b/frontend/svelte-app/src/routes/assistants/+page.svelte @@ -700,8 +700,8 @@ const callbackData = getAssistantMetadataObject(selectedAssistantData); ragProcessor = callbackData.rag_processor || ''; - if (ragProcessor !== 'simple_rag') { - console.log('Skipping KB fetch for detail view (not simple_rag)'); + if (ragProcessor !== 'simple_rag' && ragProcessor !== 'grep_rag') { + console.log('Skipping KB fetch for detail view (not simple_rag or grep_rag)'); accessibleKnowledgeBases = []; // Clear if not needed knowledgeBaseError = ''; kbFetchTriggered = true; // Mark as checked for this load @@ -1383,8 +1383,8 @@ {/if} - - {#if apiCallback.rag_processor === 'simple_rag'} + + {#if apiCallback.rag_processor === 'simple_rag' || apiCallback.rag_processor === 'grep_rag'}
{$_('assistants.form.knowledgeBases.label', { default: 'Knowledge Bases' })}
{#if loadingKnowledgeBases} @@ -1421,6 +1421,35 @@
{/if} + + + {#if apiCallback.rag_processor === 'grep_rag'} +
+
+ {$_('assistants.form.grepRag.sectionTitle', { default: 'Grep RAG Configuration' })} +
+
+ {$_('assistants.form.grepRag.mode.label', { default: 'Mode' })}: + {apiCallback.grep_mode === 'primary' ? $_('assistants.form.grepRag.mode.primary', { default: 'Primary' }) : $_('assistants.form.grepRag.mode.hybrid', { default: 'Hybrid' })} +
+
+ {$_('assistants.form.grepRag.fallbackRag.label', { default: 'Fallback RAG' })}: + {apiCallback.grep_fallback_rag || 'simple_rag'} +
+
+ {$_('assistants.form.grepRag.maxTries.label', { default: 'Max Tries' })}: + {apiCallback.grep_max_tries ?? 5} +
+
+ {$_('assistants.form.grepRag.contextLines.label', { default: 'Context Lines' })}: + {apiCallback.grep_context_lines ?? 3} +
+
+ {$_('assistants.form.grepRag.maxTotalChars.label', { default: 'Max Result Chars' })}: + {apiCallback.grep_max_total_chars ?? 8000} +
+
+ {/if} {/if} From 7dedc4382923de55c4fe2e798cd161524a2c6293 Mon Sep 17 00:00:00 2001 From: mBerasategui-ehu Date: Thu, 2 Jul 2026 12:43:32 +0200 Subject: [PATCH 2/2] feat: add end-to-end tests for grep_rag configuration and chat functionality --- testing/playwright/tests/grep_rag.spec.js | 690 ++++++++++++++++++++++ 1 file changed, 690 insertions(+) create mode 100644 testing/playwright/tests/grep_rag.spec.js diff --git a/testing/playwright/tests/grep_rag.spec.js b/testing/playwright/tests/grep_rag.spec.js new file mode 100644 index 000000000..e46d591fb --- /dev/null +++ b/testing/playwright/tests/grep_rag.spec.js @@ -0,0 +1,690 @@ +const { test, expect } = require("@playwright/test"); +const path = require("path"); +require("dotenv").config({ path: path.join(__dirname, ".env"), quiet: true }); + +/** + * Grep RAG Configuration & Chat Tests + * + * Covers the grep_rag assistant end-to-end flow: + * 1. Create a KB with ingest for grep_rag to search against + * 2. Create an assistant with grep_rag in hybrid mode + * 3. Configure all grep options (mode, max tries, context lines, max chars) + * 4. Select knowledge bases for grep_rag + * 5. Chat with the hybrid assistant — verify grep_rag responds at runtime + * 6. Verify grep settings persist on detail view and re-edit + * 7. Switch to primary mode, verify fallback RAG selector appears + * 8. Create a second assistant with grep_rag in primary mode + * 9. Chat with the primary mode assistant — verify grep_rag + fallback works + * 10. Verify primary mode settings persisted on re-edit + * 11. Cleanup: delete both assistants and the KB + * + * Prerequisites: + * - Logged in as admin via global-setup.js + * - Backend has grep_rag plugin available in backend/lamb/completions/rag/ + */ + +test.describe.serial("Grep RAG Configuration", () => { + const timestamp = Date.now(); + const kbName = `pw_grep_rag_kb_${timestamp}`; + const hybridAssistantName = `pw_grep_hybrid_${timestamp}`; + const primaryAssistantName = `pw_grep_primary_${timestamp}`; + + // ════════════════════════════════════════════════════════════════ + // Helper: Find an assistant row by name fragment + // ════════════════════════════════════════════════════════════════ + async function findAssistantRow(page, nameFragment) { + await page.goto("assistants"); + await page.waitForLoadState("networkidle"); + + const searchBox = page.locator('input[placeholder*="Search" i]'); + if (await searchBox.count()) { + await searchBox.fill(nameFragment); + await page.waitForTimeout(500); + } + + const row = page.locator(`tr:has-text("${nameFragment}")`).first(); + await expect(row).toBeVisible({ timeout: 30_000 }); + return row; + } + + // ════════════════════════════════════════════════════════════════ + // Helper: Open assistant detail view + // ════════════════════════════════════════════════════════════════ + async function openAssistantDetail(page, nameFragment) { + const row = await findAssistantRow(page, nameFragment); + + const viewButton = row.getByRole("button", { name: /view|edit/i }).first(); + if (await viewButton.count()) { + await viewButton.click(); + } else { + const nameButton = row + .getByRole("button", { name: new RegExp(nameFragment, "i") }) + .first(); + if (await nameButton.count()) { + await nameButton.click(); + } else { + await row.click(); + } + } + + await page.waitForLoadState("networkidle"); + await expect( + page.getByRole("heading", { name: /assistant/i }).first() + ).toBeVisible({ timeout: 30_000 }); + await expect( + page.getByRole("button", { name: /^properties$/i }).first() + ).toBeVisible({ timeout: 10_000 }); + } + + // ════════════════════════════════════════════════════════════════ + // Helper: Navigate to assistant create form and wait for it + // ════════════════════════════════════════════════════════════════ + async function openCreateForm(page) { + await page.goto("assistants?view=create"); + await page.waitForLoadState("networkidle"); + + const createButton = page + .getByRole("button", { name: /create assistant/i }) + .first(); + await expect(createButton).toBeVisible({ timeout: 10_000 }); + await createButton.click(); + + const form = page.locator("#assistant-form-main"); + await expect(form).toBeVisible({ timeout: 30_000 }); + return form; + } + + // ════════════════════════════════════════════════════════════════ + // Helper: Fill basic assistant fields + // ════════════════════════════════════════════════════════════════ + async function fillBasicFields(page, name, description, systemPrompt) { + await page.fill("#assistant-name", name); + await page.fill("#assistant-description", description); + await page.fill("#system-prompt", systemPrompt); + } + + // ════════════════════════════════════════════════════════════════ + // Helper: Select RAG processor from dropdown + // ════════════════════════════════════════════════════════════════ + async function selectRagProcessor(page, processorValue) { + const ragSelect = page.locator("#rag-processor"); + await expect(ragSelect).toBeVisible({ timeout: 10_000 }); + await ragSelect.selectOption(processorValue); + // Wait for dependent UI to render + await page.waitForTimeout(500); + } + + // ════════════════════════════════════════════════════════════════ + // Helper: Save assistant and wait for redirect to list. + // Returns the created assistant ID from the API response. + // ════════════════════════════════════════════════════════════════ + async function saveAssistant(page) { + let assistantId = null; + + const createRequest = page.waitForResponse((response) => { + if (response.request().method() !== "POST") return false; + try { + const url = new URL(response.url()); + return ( + url.pathname.endsWith("/assistant/create_assistant") && + response.status() >= 200 && + response.status() < 300 + ); + } catch { + return false; + } + }); + + const saveButton = page.locator( + 'button[type="submit"][form="assistant-form-main"]' + ); + await expect(saveButton).toBeEnabled({ timeout: 60_000 }); + + const form = page.locator("#assistant-form-main"); + const [response] = await Promise.all([ + createRequest, + form.evaluate((f) => f.requestSubmit()), + ]); + + // Extract assistant ID from the creation response + try { + const body = await response.json(); + assistantId = body?.id || body?.assistant?.id || body?.assistant_id || null; + } catch { + // Response may not be JSON; ID will be extracted from URL fallback + } + + await page.waitForURL(/\/assistants(\?.*)?$/, { timeout: 30_000 }); + return assistantId; + } + + // ════════════════════════════════════════════════════════════════ + // Helper: Select a knowledge base by name in the assistant form. + // Finds the label containing the KB name and checks its checkbox. + // ════════════════════════════════════════════════════════════════ + async function selectKnowledgeBase(page, kbName) { + // KB checkboxes are inside