Skip to content

feat: grep_rag implementation #428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mBerasategui-ehu wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

feat: grep_rag implementation #428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
62eb322
feat: grep_rag implementation
mBerasategui-ehu Jun 23, 2026
7dedc43
feat: add end-to-end tests for grep_rag configuration and chat functi…
mBerasategui-ehu Jul 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,165 changes: 1,165 additions & 0 deletions backend/lamb/completions/rag/grep_rag.py
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions frontend/svelte-app/src/lib/components/AssistantsList.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,8 +557,8 @@
<td class="px-6 py-2"></td> <!-- Empty cell to maintain table structure -->
</tr>

<!-- Conditional row for simple_rag details -->
{#if callback.rag_processor === 'simple_rag'}
<!-- Conditional row for simple_rag or grep_rag details -->
{#if callback.rag_processor === 'simple_rag' || callback.rag_processor === 'grep_rag'}
<tr class="bg-gray-50 border-b border-gray-200">
<td colspan="2" class="px-6 py-2 text-sm">
<div class="flex flex-wrap">
Expand Down
24 changes: 21 additions & 3 deletions frontend/svelte-app/src/lib/components/assistants/AssistantForm.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import { get } from 'svelte/store';
import { createAssistant, updateAssistant } from '$lib/services/assistantService';
import { extractModelsFromConnectorData, selectModel } from './logic/assistantFormUtils.svelte.js';
import { isKbBasedRag, isSingleFileRag, isRubricRag } from '$lib/utils/ragProcessorHelpers.js';
import { isKbBasedRag, isSingleFileRag, isRubricRag, isGrepRag } from '$lib/utils/ragProcessorHelpers.js';
import { validateImportedAssistant } from './logic/importAssistantValidator.js';
import { createAssistantFormState, resetFormFieldsToDefaults, populateFormFields, revertToInitial, clearRagDependentState, handleFieldChange } from './logic/assistantFormState.svelte.js';
import { fetchKnowledgeBases, fetchRubricsList, fetchUserFiles } from './logic/assistantFormFetchers.js';
Expand Down Expand Up @@ -163,6 +163,11 @@
if (!form.rubricsFetchAttempted && !form.loadingRubrics) {
tick().then(doFetchRubricsList);
}
} else if (isGrepRag(form.selectedRagProcessor) && form.configInitialized) {
// Grep RAG searches across KB documents — fetch KBs
if (!form.kbFetchAttempted && !form.loadingKnowledgeBases) {
doFetchKnowledgeBases();
}
} else {
// Clear KB state AND reset attempted flag if RAG processor changes away
clearRagDependentState(form);
Expand Down Expand Up @@ -315,14 +320,22 @@

// Populate RAG specific fields
// FIX FOR ISSUE #96: Apply Load-Then-Select pattern for imports too
if (isKbBasedRag(form.selectedRagProcessor)) {
form.selectedFilePath = ''; // Clear file path if switching to simple RAG, context_aware_rag, or hierarchical_rag
if (isKbBasedRag(form.selectedRagProcessor) || isGrepRag(form.selectedRagProcessor)) {
form.selectedFilePath = ''; // Clear file path if switching to simple RAG, context_aware_rag, hierarchical_rag, or grep_rag
// Fetch KBs BEFORE setting selections
if (!form.kbFetchAttempted) {
await doFetchKnowledgeBases(); // ✅ WAIT for KBs to load
}
// NOW set selections when KB list is ready
form.selectedKnowledgeBases = parsedData.RAG_collections?.split(',').filter(Boolean) || [];
// Populate Grep RAG fields if applicable
if (isGrepRag(form.selectedRagProcessor)) {
form.grepMode = callbackData.grep_mode || 'hybrid';
form.grepFallbackRag = callbackData.grep_fallback_rag || 'simple_rag';
form.grepMaxTries = callbackData.grep_max_tries ?? 5;
form.grepContextLines = callbackData.grep_context_lines ?? 3;
form.grepMaxTotalChars = callbackData.grep_max_total_chars ?? 8000;
}
} else if (isSingleFileRag(form.selectedRagProcessor)) {
form.selectedKnowledgeBases = []; // Clear KBs if switching to single file RAG
// Fetch files BEFORE setting selection
Expand Down Expand Up @@ -459,6 +472,11 @@
fileError={form.fileError}
onFilesChanged={() => doFetchUserFiles(true)}
onchange={() => handleFieldChange(form)}
bind:grepMode={form.grepMode}
bind:grepFallbackRag={form.grepFallbackRag}
bind:grepMaxTries={form.grepMaxTries}
bind:grepContextLines={form.grepContextLines}
bind:grepMaxTotalChars={form.grepMaxTotalChars}
/>
</div>
</div>
Expand Down
14 changes: 13 additions & 1 deletion frontend/svelte-app/src/lib/components/assistants/components/ConfigurationPanel.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,13 @@
loadingFiles = false,
fileError = '',
onFilesChanged,
onchange
onchange,
// Grep RAG fields
grepMode = $bindable('hybrid'),
grepFallbackRag = $bindable('simple_rag'),
grepMaxTries = $bindable(5),
grepContextLines = $bindable(3),
grepMaxTotalChars = $bindable(8000)
} = $props();

let currentConnectorMetadata = $derived.by(() => {
Expand Down Expand Up @@ -208,6 +214,12 @@
fileError={fileError}
{formState}
{onFilesChanged}
{ragProcessors}
bind:grepMode
bind:grepFallbackRag
bind:grepMaxTries
bind:grepContextLines
bind:grepMaxTotalChars
/>
{/if}
</fieldset>
92 changes: 88 additions & 4 deletions frontend/svelte-app/src/lib/components/assistants/components/RagOptionsPanel.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<!-- src/lib/components/assistants/RagOptionsPanel.svelte -->
<script>
import { _ } from '$lib/i18n';
import { isKbBasedRag, isSingleFileRag, isRubricRag } from '$lib/utils/ragProcessorHelpers.js';
import { isKbBasedRag, isSingleFileRag, isRubricRag, isGrepRag } from '$lib/utils/ragProcessorHelpers.js';
import KnowledgeBaseSelector from './KnowledgeBaseSelector.svelte';
import SingleFileSelector from './SingleFileSelector.svelte';

Expand All @@ -18,12 +18,20 @@
loadingFiles = false,
fileError = '',
formState,
onFilesChanged
onFilesChanged,
// Grep RAG fields
grepMode = $bindable('hybrid'),
grepFallbackRag = $bindable('simple_rag'),
grepMaxTries = $bindable(5),
grepContextLines = $bindable(3),
grepMaxTotalChars = $bindable(8000),
ragProcessors = []
} = $props();

let showKbSelector = $derived(isKbBasedRag(selectedRagProcessor));
let showKbSelector = $derived(isKbBasedRag(selectedRagProcessor) || isGrepRag(selectedRagProcessor));
let showFileSelector = $derived(isSingleFileRag(selectedRagProcessor));
let showTopK = $derived(isKbBasedRag(selectedRagProcessor));
let showTopK = $derived(isKbBasedRag(selectedRagProcessor) || isGrepRag(selectedRagProcessor));
let showGrepOptions = $derived(isGrepRag(selectedRagProcessor));
</script>

<div class="pt-4 border-t border-gray-200 space-y-4">
Expand Down Expand Up @@ -66,4 +74,80 @@
{onFilesChanged}
/>
{/if}
{#if showGrepOptions}
<div class="space-y-3 p-3 bg-gray-50 border border-gray-200 rounded-md">
<h5 class="text-sm font-semibold text-gray-700">
{$_('assistants.form.grepRag.sectionTitle', { default: 'Grep RAG Configuration' })}
</h5>

<!-- Mode -->
<div>
<label for="grep-mode" class="block text-sm font-medium text-gray-700">
{$_('assistants.form.grepRag.mode.label', { default: 'Mode' })}
</label>
<select id="grep-mode" bind:value={grepMode}
class="mt-1 block w-full px-3 py-2 text-sm border border-gray-300 rounded-md bg-white text-gray-900">
<option value="hybrid">{$_('assistants.form.grepRag.mode.hybrid', { default: 'Hybrid (grep + RAG in parallel)' })}</option>
<option value="primary">{$_('assistants.form.grepRag.mode.primary', { default: 'Primary (grep first, RAG fallback)' })}</option>
</select>
<p class="mt-1 text-xs text-gray-500">
{$_('assistants.form.grepRag.mode.description', { default: 'How grep interacts with embedding-based RAG' })}
</p>
</div>

<!-- Fallback RAG (only relevant in primary mode) -->
{#if grepMode === 'primary'}
<div>
<label for="grep-fallback-rag" class="block text-sm font-medium text-gray-700">
{$_('assistants.form.grepRag.fallbackRag.label', { default: 'Fallback RAG' })}
</label>
<select id="grep-fallback-rag" bind:value={grepFallbackRag}
class="mt-1 block w-full px-3 py-2 text-sm border border-gray-300 rounded-md bg-white text-gray-900">
{#each ragProcessors.filter(p => isKbBasedRag(p)) as processor}
<option value={processor}>{processor.replace(/_/g, ' ').replace(/\b\w/g, (l) => l.toUpperCase())}</option>
{/each}
</select>
<p class="mt-1 text-xs text-gray-500">
{$_('assistants.form.grepRag.fallbackRag.description', { default: 'Embedding RAG to fall back to when grep finds no matches' })}
</p>
</div>
{/if}

<!-- Max Tries -->
<div>
<label for="grep-max-tries" class="block text-sm font-medium text-gray-700">
{$_('assistants.form.grepRag.maxTries.label', { default: 'Max Search Tries' })}
</label>
<input type="number" id="grep-max-tries" bind:value={grepMaxTries} min="1" max="10"
class="mt-1 block w-24 px-3 py-2 text-sm border border-gray-300 rounded-md bg-white text-gray-900">
<p class="mt-1 text-xs text-gray-500">
{$_('assistants.form.grepRag.maxTries.description', { default: 'Maximum number of search iterations (1-10)' })}
</p>
</div>

<!-- Context Lines -->
<div>
<label for="grep-context-lines" class="block text-sm font-medium text-gray-700">
{$_('assistants.form.grepRag.contextLines.label', { default: 'Context Lines' })}
</label>
<input type="number" id="grep-context-lines" bind:value={grepContextLines} min="1" max="10"
class="mt-1 block w-24 px-3 py-2 text-sm border border-gray-300 rounded-md bg-white text-gray-900">
<p class="mt-1 text-xs text-gray-500">
{$_('assistants.form.grepRag.contextLines.description', { default: 'Lines of context before/after each match (1-10)' })}
</p>
</div>

<!-- Max Total Chars -->
<div>
<label for="grep-max-chars" class="block text-sm font-medium text-gray-700">
{$_('assistants.form.grepRag.maxTotalChars.label', { default: 'Max Result Characters' })}
</label>
<input type="number" id="grep-max-chars" bind:value={grepMaxTotalChars} min="1000" max="32000" step="1000"
class="mt-1 block w-32 px-3 py-2 text-sm border border-gray-300 rounded-md bg-white text-gray-900">
<p class="mt-1 text-xs text-gray-500">
{$_('assistants.form.grepRag.maxTotalChars.description', { default: 'Max total characters of grep results sent to main LLM (1000-32000)' })}
</p>
</div>
</div>
{/if}
</div>
4 changes: 2 additions & 2 deletions frontend/svelte-app/src/lib/components/assistants/logic/assistantFormFetchers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import { getUserKnowledgeBases, getSharedKnowledgeBases } from '$lib/services/knowledgeBaseService';
import { fetchAccessibleRubrics } from '$lib/services/rubricService';
import { apiJson } from '$lib/services/apiClient';
import { isKbBasedRag, isRubricRag } from '$lib/utils/ragProcessorHelpers.js';
import { isKbBasedRag, isRubricRag, isGrepRag } from '$lib/utils/ragProcessorHelpers.js';
import { getAssistantMetadataObject } from '$lib/utils/assistantData';

/**
Expand All @@ -16,7 +16,7 @@ import { getAssistantMetadataObject } from '$lib/utils/assistantData';
*/
export async function fetchKnowledgeBases(form) {
if (form.loadingKnowledgeBases || form.kbFetchAttempted) return;
if (!isKbBasedRag(form.selectedRagProcessor)) return;
if (!isKbBasedRag(form.selectedRagProcessor) && !isGrepRag(form.selectedRagProcessor)) return;

form.loadingKnowledgeBases = true;
form.knowledgeBaseError = '';
Expand Down
31 changes: 30 additions & 1 deletion frontend/svelte-app/src/lib/components/assistants/logic/assistantFormState.svelte.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import { get } from 'svelte/store';
import { assistantConfigStore } from '$lib/stores/assistantConfigStore';
import { isKbBasedRag, isSingleFileRag, isRubricRag, normalizeRagProcessor } from '$lib/utils/ragProcessorHelpers.js';
import { isKbBasedRag, isSingleFileRag, isRubricRag, isGrepRag, normalizeRagProcessor } from '$lib/utils/ragProcessorHelpers.js';
import { loadRagPlaceholders, selectModel } from './assistantFormUtils.svelte.js';
import { getAssistantMetadataObject } from '$lib/utils/assistantData';

Expand Down Expand Up @@ -84,6 +84,18 @@ export function createAssistantFormState() {
rubricError: '',
rubricsFetchAttempted: false,

// --- Grep RAG state ---
/** @type {'hybrid' | 'primary'} */
grepMode: 'hybrid',
/** @type {string} */
grepFallbackRag: 'simple_rag',
/** @type {number} */
grepMaxTries: 5,
/** @type {number} */
grepContextLines: 3,
/** @type {number} */
grepMaxTotalChars: 8000,

// --- UI / loading state ---
formError: '',
formLoading: false,
Expand Down Expand Up @@ -187,6 +199,15 @@ export function populateFormFields(form, data, getAvailableModels, preserveDescr
}
}

// Grep RAG fields
if (isGrepRag(form.selectedRagProcessor)) {
form.grepMode = metadata?.grep_mode || 'hybrid';
form.grepFallbackRag = metadata?.grep_fallback_rag || 'simple_rag';
form.grepMaxTries = metadata?.grep_max_tries ?? 5;
form.grepContextLines = metadata?.grep_context_lines ?? 3;
form.grepMaxTotalChars = metadata?.grep_max_total_chars ?? 8000;
}

// Vision capability
try {
form.visionEnabled = metadata?.capabilities?.vision || false;
Expand Down Expand Up @@ -235,4 +256,12 @@ export function clearRagDependentState(form) {
form.selectedRubricId = '';
form.rubricFormat = 'markdown';
}

if (!isGrepRag(form.selectedRagProcessor)) {
form.grepMode = 'hybrid';
form.grepFallbackRag = 'simple_rag';
form.grepMaxTries = 5;
form.grepContextLines = 3;
form.grepMaxTotalChars = 8000;
}
}
12 changes: 10 additions & 2 deletions frontend/svelte-app/src/lib/components/assistants/logic/assistantFormSubmit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* Extracted from AssistantForm.svelte to enable isolated testing.
*/

import { isKbBasedRag, isSingleFileRag, isRubricRag } from '$lib/utils/ragProcessorHelpers.js';
import { isKbBasedRag, isSingleFileRag, isRubricRag, isGrepRag } from '$lib/utils/ragProcessorHelpers.js';

/**
* Validates form data before submission.
Expand Down Expand Up @@ -42,13 +42,21 @@ export function buildAssistantPayload(form) {
metadataObj.rubric_format = form.rubricFormat;
}

if (isGrepRag(form.selectedRagProcessor)) {
metadataObj.grep_mode = form.grepMode;
metadataObj.grep_fallback_rag = form.grepFallbackRag;
metadataObj.grep_max_tries = form.grepMaxTries;
metadataObj.grep_context_lines = form.grepContextLines;
metadataObj.grep_max_total_chars = form.grepMaxTotalChars;
}

return {
name: form.name.trim(),
description: form.description,
system_prompt: form.system_prompt,
prompt_template: form.prompt_template,
RAG_Top_k: Number(form.RAG_Top_k) || 3,
RAG_collections: isKbBasedRag(form.selectedRagProcessor) ? form.selectedKnowledgeBases.join(',') : '',
RAG_collections: (isKbBasedRag(form.selectedRagProcessor) || isGrepRag(form.selectedRagProcessor)) ? form.selectedKnowledgeBases.join(',') : '',
metadata: JSON.stringify(metadataObj),
pre_retrieval_endpoint: '',
post_retrieval_endpoint: '',
Expand Down
25 changes: 25 additions & 0 deletions frontend/svelte-app/src/lib/locales/ca.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,31 @@
"error": "Error en carregar arxius:",
"noneFound": "No s'han trobat arxius. Si us plau puja un arxiu.",
"required": "Si us plau selecciona un arxiu"
},
"grepRag": {
"sectionTitle": "Configuració de Grep RAG",
"mode": {
"label": "Mode",
"description": "Com interactua grep amb el RAG basat en embeddings",
"hybrid": "Híbrid (grep + RAG en paral·lel)",
"primary": "Principal (grep primer, RAG com a respatller)"
},
"fallbackRag": {
"label": "RAG de Respatller",
"description": "RAG d'embeddings per complementar o com a respatller"
},
"maxTries": {
"label": "Intents Màxims",
"description": "Nombre màxim d'iteracions de cerca (1-10)"
},
"contextLines": {
"label": "Línies de Context",
"description": "Línies de context abans/després de cada coincidència (1-10)"
},
"maxTotalChars": {
"label": "Caràcters Màxims",
"description": "Màxim total de caràcters de resultats enviats al LLM (1000-32000)"
}
}
},
"noDescription": "Sense descripció",
Expand Down
25 changes: 25 additions & 0 deletions frontend/svelte-app/src/lib/locales/en.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,31 @@
"error": "Error loading files:",
"noneFound": "No files found. Please upload a file.",
"required": "Please select a file"
},
"grepRag": {
"sectionTitle": "Grep RAG Configuration",
"mode": {
"label": "Mode",
"description": "How grep interacts with embedding-based RAG",
"hybrid": "Hybrid (grep + RAG in parallel)",
"primary": "Primary (grep first, RAG fallback)"
},
"fallbackRag": {
"label": "Fallback RAG",
"description": "Embedding RAG to complement or fall back to"
},
"maxTries": {
"label": "Max Search Tries",
"description": "Maximum number of search iterations (1-10)"
},
"contextLines": {
"label": "Context Lines",
"description": "Lines of context before/after each match (1-10)"
},
"maxTotalChars": {
"label": "Max Result Characters",
"description": "Max total characters of grep results sent to main LLM (1000-32000)"
}
}
}
},
Expand Down
Loading