feat(app): maximize RAG context and improve chat UX (#118)

Author: comppaz

.changeset/rag-context-budget.md

@@ -0,0 +1,13 @@
+---
+"think-app": minor
+---
+
+Maximize RAG context utilization with dynamic budget system and improve chat UX
+
+- Add dynamic context budget that scales with model context window size
+- Widen distance thresholds proportionally when budget allows more content
+- Include AI-generated summaries as context overview/fallback for each memory
+- Subtract page content from budget in browser extension chat to prevent overflow
+- Rewrite system prompt for better grounding and natural source attribution
+- Fix missing bullet point markers in chat message markdown rendering
+- Remove hover blur effect from assistant chat message bubbles

app/src/components/ChatMessage.tsx

@@ -42,7 +42,7 @@ export function ChatMessage({ message }: ChatMessageProps) {
             "p-4 rounded-2xl",
             isUser
               ? "bg-primary text-primary-foreground"
-              : cn(glass.base, glass.hover)
+              : glass.base
           )}
         >
           {isUser ? (

app/src/index.css

@@ -153,9 +153,15 @@
   .chat-prose p:last-child {
     margin-bottom: 0;
   }
-  .chat-prose ul, .chat-prose ol {
+  .chat-prose ul {
     margin: 0.5em 0;
     padding-left: 1.5em;
+    list-style-type: disc;
+  }
+  .chat-prose ol {
+    margin: 0.5em 0;
+    padding-left: 1.5em;
+    list-style-type: decimal;
   }
   .chat-prose li {
     margin: 0.25em 0;

backend/app/native_messaging.py

@@ -14,11 +14,13 @@
 from .services.embeddings import (
     get_embedding, get_current_embedding_model,
     filter_memories_dynamically, format_memories_as_context,
+    compute_context_budget,
 )
 from .services.ai import (
-    chat, process_memory_async, process_conversation_title_async,
+    chat, get_model, process_memory_async, process_conversation_title_async,
     maybe_rewrite_query, generate_followup_suggestions,
 )
+from .models_info import get_context_window
 from .services.query import preprocess_query, extract_keywords, is_special_prompt, execute_special_handler
 from .events import event_manager, MemoryEvent, EventType
 
@@ -283,6 +285,14 @@ async def _chat_message(self, params: dict) -> dict:
         sources = []
         return_page_summary = None
 
+        # Compute context budget based on model and conversation history
+        model = get_model()
+        context_window = get_context_window(model)
+        # Account for page content that will also consume context space
+        page_content_chars = min(len(page_content), 8000) if page_content else 0
+        context_budget = compute_context_budget(context_window, history)
+        context_budget = max(2000, context_budget - page_content_chars)
+
         # Generate page summary for memory search (only on first message)
         if page_content and not page_summary:
             page_summary = await self._generate_page_summary(page_content, page_title)
@@ -333,7 +343,7 @@ async def _chat_message(self, params: dict) -> dict:
                     # Use model-specific thresholds for filtering
                     embedding_model = get_current_embedding_model()
                     filtered_memories = filter_memories_dynamically(
-                        memories, embedding_model=embedding_model
+                        memories, embedding_model=embedding_model, context_budget_chars=context_budget
                     )
 
                     # Build sources list for the response
@@ -349,7 +359,7 @@ async def _chat_message(self, params: dict) -> dict:
                     ]
 
                     # Format memories as context
-                    memories_context = format_memories_as_context(filtered_memories)
+                    memories_context = format_memories_as_context(filtered_memories, max_chars=context_budget)
                     if memories_context:
                         context_parts.append(memories_context)
 

backend/app/routes/chat.py

@@ -14,7 +14,7 @@
 from ..services.ai.query_rewriting import maybe_rewrite_query
 from ..services.ai.suggestions import get_quick_prompts, generate_followup_suggestions
 from ..services.query.special_handlers import is_special_prompt, execute_special_handler
-from ..services.embeddings.filtering import filter_memories_dynamically, format_memories_as_context
+from ..services.embeddings.filtering import filter_memories_dynamically, format_memories_as_context, compute_context_budget
 from ..db.search import search_similar_memories
 from ..schemas import ChatRequest
 from .. import config
@@ -66,6 +66,11 @@ async def _retrieve_context(
     sources = []
     attached_context = ""
 
+    # Compute context budget based on model and conversation history
+    model = get_model()
+    context_window = get_context_window(model)
+    total_budget = compute_context_budget(context_window, history)
+
     # Handle explicitly attached memories first
     if attached_memory_ids:
         attached_memories = []
@@ -80,7 +85,9 @@ async def _retrieve_context(
                 })
 
         if attached_memories:
-            attached_context = "## User's Selected Memory:\n" + format_memories_as_context(attached_memories)
+            # Give attached memories up to half the budget; RAG gets the other half
+            attached_budget = total_budget // 2
+            attached_context = "## User's Selected Memory:\n" + format_memories_as_context(attached_memories, max_chars=attached_budget)
             logger.info(f"Using {len(attached_memories)} attached memories as context")
 
     # Skip RAG for very short messages (< 10 chars) or when explicitly disabled
@@ -118,9 +125,13 @@ async def _retrieve_context(
             )
 
             if similar_memories:
+                # RAG budget: half if attached memories exist, full otherwise
+                rag_budget = total_budget // 2 if attached_context else total_budget
                 # Filter using dynamic threshold with model-specific thresholds
-                filtered_memories = filter_memories_dynamically(similar_memories, embedding_model=embedding_model)
-                context = format_memories_as_context(filtered_memories)
+                filtered_memories = filter_memories_dynamically(
+                    similar_memories, embedding_model=embedding_model, context_budget_chars=rag_budget
+                )
+                context = format_memories_as_context(filtered_memories, max_chars=rag_budget)
                 # Build sources list from filtered memories, avoiding duplicates with attached
                 attached_ids = {s["id"] for s in sources}
                 for m in filtered_memories:

backend/app/services/ai/client.py

@@ -5,11 +5,15 @@
 
 
 # Custom system prompt for Think
-SYSTEM_PROMPT = """You are Think, a friendly personal assistant with access to the user's saved memories and notes. You help them recall information, answer questions, and have natural conversations.
-
-When context from their memories is provided, use it naturally to inform your responses without explicitly mentioning "your saved article" or "your memories" - just incorporate the knowledge seamlessly.
-
-Keep responses conversational and concise. Be helpful and warm, like a knowledgeable friend."""
+SYSTEM_PROMPT = """You are Think, a helpful personal assistant. You help users recall and explore information from their saved content. 
+The content can be owned by the user (like notes and voice memos) or from third-party sources (like web pages, videos, and audio). 
+You use this information to answer questions, provide summaries, and assist with tasks.
+
+Guidelines:
+- Ground your answers in the provided context. If it doesn't contain enough information to answer, say so rather than guessing.
+- Be conversational and concise.
+- You can naturally reference sources (e.g. "from a saved article", "in one of your videos"), but never output the raw type tags like [web] or [video] in your responses.
+"""
 
 
 async def get_client() -> AsyncOpenAI:

backend/app/services/embeddings/__init__.py

@@ -1,4 +1,4 @@
 """Vector Search & Similarity services."""
 from .client import get_embedding, cosine_similarity, get_current_embedding_model
-from .filtering import filter_memories_dynamically, format_memories_as_context
+from .filtering import filter_memories_dynamically, format_memories_as_context, compute_context_budget
 from .jobs import job_manager, reembed_worker, JobStatus