Show calibrated evidence and useful uncertainty in chat

api/api/chatbot_service.py

@@ -293,6 +293,7 @@ def detect_image_context_card_ids(
     count_tokens,
     count_message_tokens,
     truncate_text,
+    truncate_text_preserving_suffix,
     truncate_conversation_history,
     truncate_document_content,
 )
@@ -691,14 +692,15 @@ def get_client_for_request(has_image: bool):
    - chamoru_info_dictionary
    - chamorro_english_dictionary_TOD
 
-2. **NEVER guess or hallucinate translations**
-   - If you don't see the word in a dictionary source, say: "I don't have that specific translation in my dictionary sources."
-   - DO NOT make up Chamorro words
-   - DO NOT use words from blog posts or articles as authoritative translations
+2. **Never present an unsupported translation as verified**
+   - If a dictionary source is attached, keep the translation and its citation faithful to that source.
+   - If no dictionary source matched, you may still give a plausible candidate when useful, but label it "Unverified best effort" and state the uncertainty plainly.
+   - DO NOT make up citations or use words from blog posts or articles as authoritative translations.
 
 3. **How to answer word translation questions:**
    ✅ CORRECT: "In Chamorro, 'listen' is **ekungok**. [Source: chamorro_english_dictionary_TOD]"
-   ❌ WRONG: Guessing or using non-dictionary content for single-word translations
+   ✅ CORRECT WHEN NO SOURCE MATCHES: "Unverified best effort: a plausible candidate is **...**, but I could not verify it in the available dictionaries."
+   ❌ WRONG: Presenting a guess or non-dictionary content as a verified translation
 
 4. **For contextual/cultural questions** (not single-word translations):
    - You may use all sources (blogs, articles, cultural content)
@@ -721,11 +723,14 @@ def get_client_for_request(has_image: bool):
 
 🔴 Para i tiningo' palåbra (word translations):
 - Usa HA' i diksionarion-måmi (dictionaries): revised_and_updated_chamorro_dictionary, chamoru_info_dictionary, chamorro_english_dictionary_TOD
-- MUNGA un adibina palåbra! (DO NOT guess words!)
+- Never present an unsupported candidate as a verified translation.
+- If no dictionary source matched and a credible candidate would help, you may
+  provide it only as an explicitly unverified best effort, with plain uncertainty.
+- If there is no credible candidate, say so and ask one focused question or
+  suggest a related source-backed lookup.
 
-Use governed Chamorro dictionary/canonical context for language claims. Munga un
-adibina pat un fa'tinas nuebu na tiningo'. Yanggen ti guaha sufisiente na prineba,
-na'fanmanungo' na ti siña un na'siguru.
+Use governed Chamorro dictionary/canonical context for language claims. Keep the
+entire response in Chamorro, including any uncertainty label or clarification.
 
 If you receive web search results, use them but respond in Chamorro only."""
     },
@@ -746,7 +751,11 @@ def get_client_for_request(has_image: bool):
 - chamoru_info_dictionary
 - chamorro_english_dictionary_TOD
 
-NEVER guess or make up Chamorro words. If unsure, say "I don't have that translation."
+Never present a guess as a verified Chamorro word. When no dictionary source
+matched, a plausible candidate may be offered only as an explicitly unverified
+best effort with the uncertainty stated plainly. If there is no credible
+candidate, say so and ask one focused question or suggest a related source-backed
+lookup.
 
 For abbreviations, literal translations, phrase variants, pronunciation, and
 usage claims, retrieve a governed source and state uncertainty when the evidence
@@ -758,10 +767,16 @@ def get_client_for_request(has_image: bool):
 NO_REFERENCE_GUARD = """
 
 NO GOVERNED REFERENCE WAS RETRIEVED FOR THIS REQUEST:
-- Do not add a translation, abbreviation expansion, pronunciation, etymology,
-  cultural/regional usage, or new example sentence from model memory.
-- Say that the requested accuracy-sensitive detail could not be verified from
-  the available references, and offer to help with a source-backed alternative.
+- Do not refuse solely because retrieval returned no match. Give the most useful
+  concise answer you can while preserving the selected response language.
+- For an accuracy-sensitive Chamorro translation, abbreviation, pronunciation,
+  etymology, cultural claim, regional-usage claim, or example sentence, clearly
+  label any plausible model-memory candidate as an unverified best effort.
+- Separate what you know from what you are inferring, and name material uncertainty.
+  Use plain certainty wording; never invent a percentage confidence.
+- Never invent a citation or imply that an unverified candidate came from a source.
+- If you do not have a credible candidate, say that directly and ask one focused
+  clarifying question or suggest a related source-backed lookup.
 """
 
 # Skill level modifiers - adjust response style based on user's experience
@@ -1311,6 +1326,7 @@ def early_cancelled_response(log_user_message: bool = True):
     # Check if we should use web search
     use_web, search_type = should_use_web_search(message)
     web_context = ""
+    web_results_used = False
 
     if use_web:
         # Check for cancellation before web search
@@ -1320,6 +1336,7 @@ def early_cancelled_response(log_user_message: bool = True):
         search_result = web_search(message, search_type=search_type, max_results=3)
         if search_result["success"] and search_result["results"]:
             web_context = format_search_results(search_result)
+            web_results_used = bool(web_context)
 
     # Check for cancellation before RAG search
     if is_message_cancelled(pending_id):
@@ -1380,18 +1397,21 @@ def early_cancelled_response(log_user_message: bool = True):
     elif not is_passage_translation(effective_translation_message) and not school_announcement:
         system_prompt += NO_REFERENCE_GUARD
 
-    # Add web search context if available
-    if web_context:
-        system_prompt += f"\n\n{web_context}"
-
     # Initialize token manager for this request
     token_manager = TokenManager(budget=TokenBudget(), model=LLM_MODEL_ID)
-
-    # Apply token limit to system prompt
-    system_prompt_tokens = count_tokens(system_prompt)
+
+    # Keep usable web results in the final model prompt even when general
+    # instructions and retrieved context exceed the system-prompt budget.
+    system_prompt_tokens = count_tokens(
+        system_prompt + (f"\n\n{web_context}" if web_context else "")
+    )
     if system_prompt_tokens > token_manager.budget.system_prompt:
         logger.warning(f"System prompt ({system_prompt_tokens} tokens) exceeds budget, truncating...")
-        system_prompt = truncate_text(system_prompt, token_manager.budget.system_prompt)
+    system_prompt = truncate_text_preserving_suffix(
+        system_prompt,
+        f"\n\n{web_context}" if web_context else "",
+        token_manager.budget.system_prompt,
+    )
 
     # Build conversation history
     history = [
@@ -1496,7 +1516,7 @@ def early_cancelled_response(log_user_message: bool = True):
 
         sources = []
         used_rag = False
-        use_web = False
+        web_results_used = False
 
     # Calculate response time
     response_time = time.time() - start_time
@@ -1518,7 +1538,7 @@ def early_cancelled_response(log_user_message: bool = True):
             mode=mode,
             sources=[],
             used_rag=used_rag,
-            used_web_search=use_web,
+            used_web_search=web_results_used,
             response_time=response_time,
             session_id=session_id,
             user_id=user_id,
@@ -1532,7 +1552,7 @@ def early_cancelled_response(log_user_message: bool = True):
             "response": "[Message was cancelled by user]",
             "sources": [],
             "used_rag": used_rag,
-            "used_web_search": use_web,
+            "used_web_search": web_results_used,
             "response_time": response_time,
             "cancelled": True
         }
@@ -1544,7 +1564,7 @@ def early_cancelled_response(log_user_message: bool = True):
         mode=mode,
         sources=formatted_sources,
         used_rag=used_rag,
-        used_web_search=use_web,
+        used_web_search=web_results_used,
         response_time=response_time,
         session_id=session_id,
         user_id=user_id,
@@ -1561,7 +1581,7 @@ def early_cancelled_response(log_user_message: bool = True):
         "response": response_text,
         "sources": formatted_sources,
         "used_rag": used_rag,
-        "used_web_search": use_web,
+        "used_web_search": web_results_used,
         "response_time": response_time,
         "cancelled": False
     }
@@ -1637,6 +1657,7 @@ def get_chatbot_response_stream(
     # Check if we should use web search
     use_web, search_type = should_use_web_search(message)
     web_context = ""
+    web_results_used = False
 
     if use_web:
         if is_message_cancelled(pending_id):
@@ -1646,6 +1667,7 @@ def get_chatbot_response_stream(
         search_result = web_search(message, search_type=search_type, max_results=3)
         if search_result["success"] and search_result["results"]:
             web_context = format_search_results(search_result)
+            web_results_used = bool(web_context)
 
     # Check for cancellation before RAG
     if is_message_cancelled(pending_id):
@@ -1707,15 +1729,17 @@ def get_chatbot_response_stream(
     elif not is_passage_translation(effective_translation_message) and not school_announcement:
         system_prompt += NO_REFERENCE_GUARD
 
-    # Add web search context if available
-    if web_context:
-        system_prompt += f"\n\n{web_context}"
-
     # Track token usage and apply limits
-    system_prompt_tokens = count_tokens(system_prompt)
+    system_prompt_tokens = count_tokens(
+        system_prompt + (f"\n\n{web_context}" if web_context else "")
+    )
     if system_prompt_tokens > token_manager.budget.system_prompt:
         logger.warning(f"System prompt ({system_prompt_tokens} tokens) exceeds budget ({token_manager.budget.system_prompt}), truncating...")
-        system_prompt = truncate_text(system_prompt, token_manager.budget.system_prompt)
+    system_prompt = truncate_text_preserving_suffix(
+        system_prompt,
+        f"\n\n{web_context}" if web_context else "",
+        token_manager.budget.system_prompt,
+    )
 
     # Build conversation history
     history = [{"role": "system", "content": system_prompt}]
@@ -1763,7 +1787,7 @@ def get_chatbot_response_stream(
         "type": "metadata",
         "sources": formatted_sources if should_show_sources else [],
         "used_rag": used_rag,
-        "used_web_search": use_web
+        "used_web_search": web_results_used
     }
 
     # Stream LLM response
@@ -1787,7 +1811,7 @@ def get_chatbot_response_stream(
             mode=mode,
             sources=[],
             used_rag=used_rag,
-            used_web_search=use_web,
+            used_web_search=web_results_used,
             response_time=time.time() - start_time,
             session_id=session_id,
             user_id=user_id,
@@ -1828,7 +1852,7 @@ def get_chatbot_response_stream(
                             mode=mode,
                             sources=formatted_sources,
                             used_rag=used_rag,
-                            used_web_search=use_web,
+                            used_web_search=web_results_used,
                             response_time=time.time() - start_time,
                             session_id=session_id,
                             user_id=user_id,
@@ -1913,7 +1937,7 @@ def get_chatbot_response_stream(
             mode=mode,
             sources=[],
             used_rag=used_rag,
-            used_web_search=use_web,
+            used_web_search=web_results_used,
             response_time=time.time() - start_time,
             session_id=session_id,
             user_id=user_id,
@@ -1937,7 +1961,7 @@ def get_chatbot_response_stream(
         mode=mode,
         sources=formatted_sources,
         used_rag=used_rag,
-        used_web_search=use_web,
+        used_web_search=web_results_used,
         response_time=response_time,
         session_id=session_id,
         user_id=user_id,

api/api/models.py

@@ -81,7 +81,7 @@ class ChatResponse(BaseModel):
     )
     used_web_search: bool = Field(
         default=False,
-        description="Whether web search was used"
+        description="Whether usable web search results informed the response"
     )
     response_time: Optional[float] = Field(
         None,

api/src/utils/token_manager.py

@@ -149,6 +149,8 @@ def truncate_text(text: str, max_tokens: int, model: str = "gpt-4o") -> str:
     Returns:
         Truncated text with "[truncated]" indicator if needed
     """
+    if max_tokens <= 0:
+        return ""
     if not text:
         return text
 
@@ -163,6 +165,11 @@ def truncate_text(text: str, max_tokens: int, model: str = "gpt-4o") -> str:
         # Leave room for truncation indicator
         truncation_indicator = "\n\n[... content truncated due to length ...]"
         indicator_tokens = count_tokens(truncation_indicator, model)
+
+        # For very small budgets the indicator itself would violate the limit.
+        # Preserve only as much original content as the caller allowed.
+        if indicator_tokens >= max_tokens:
+            return tokenizer.decode(tokens[:max_tokens])
 
         # Truncate tokens
         truncated_tokens = tokens[:max_tokens - indicator_tokens]
@@ -173,7 +180,48 @@ def truncate_text(text: str, max_tokens: int, model: str = "gpt-4o") -> str:
         logger.warning(f"Token-based truncation failed, using char estimate: {e}")
         # Fallback: ~4 chars per token
         max_chars = max_tokens * 4
-        return text[:max_chars] + "\n\n[... content truncated ...]"
+        return text[:max_chars]
+
+
+def truncate_text_preserving_suffix(
+    text: str,
+    suffix: str,
+    max_tokens: int,
+    model: str = "gpt-4o",
+) -> str:
+    """Fit text within a token budget while retaining a priority suffix.
+
+    The suffix receives up to one third of the budget when the combined text is
+    oversized. This is intended for evidence appended after general prompt
+    instructions, where prefix-only truncation would silently remove the
+    evidence while leaving the response metadata unchanged.
+    """
+    if max_tokens <= 0:
+        return ""
+
+    combined = text + suffix
+    if count_tokens(combined, model) <= max_tokens:
+        return combined
+    if not suffix:
+        return truncate_text(text, max_tokens, model)
+
+    suffix_budget = min(count_tokens(suffix, model), max(1, max_tokens // 3))
+    fitted_suffix = truncate_text(suffix, suffix_budget, model)
+    prefix_budget = max(0, max_tokens - count_tokens(fitted_suffix, model))
+    fitted_prefix = truncate_text(text, prefix_budget, model) if prefix_budget else ""
+    combined = fitted_prefix + fitted_suffix
+
+    # Token boundaries can change when two independently encoded strings are
+    # joined. Tighten only the prefix so the priority evidence remains present.
+    while count_tokens(combined, model) > max_tokens and prefix_budget > 0:
+        overflow = count_tokens(combined, model) - max_tokens
+        prefix_budget = max(0, prefix_budget - overflow)
+        fitted_prefix = truncate_text(text, prefix_budget, model) if prefix_budget else ""
+        combined = fitted_prefix + fitted_suffix
+
+    if count_tokens(combined, model) > max_tokens:
+        return truncate_text(fitted_suffix, max_tokens, model)
+    return combined
 
 
 def truncate_conversation_history(
@@ -624,4 +672,3 @@ def get_token_summary(self) -> dict:
             "remaining_for_response": self.tokens_remaining_for_response(),
             "budget_total": self.budget.total
         }
-

api/tests/test_prompt_token_budget.py

@@ -0,0 +1,40 @@
+from src.utils.token_manager import count_tokens, truncate_text, truncate_text_preserving_suffix
+
+
+def test_oversized_prompt_preserves_web_evidence_suffix() -> None:
+    base_prompt = "General assistant instruction. " * 500
+    web_evidence = "\n\nWEB SEARCH RESULTS\nCurrent Guam weather result: sunny."
+
+    fitted = truncate_text_preserving_suffix(base_prompt, web_evidence, max_tokens=120)
+
+    assert count_tokens(fitted) <= 120
+    assert "WEB SEARCH RESULTS" in fitted
+    assert "Current Guam weather result: sunny." in fitted
+    assert "General assistant instruction" in fitted
+
+
+def test_prompt_without_suffix_keeps_existing_prefix_truncation() -> None:
+    base_prompt = "General assistant instruction. " * 500
+
+    fitted = truncate_text_preserving_suffix(base_prompt, "", max_tokens=80)
+
+    assert count_tokens(fitted) <= 80
+    assert fitted.startswith("General assistant instruction")
+
+
+def test_oversized_suffix_respects_a_very_small_budget() -> None:
+    fitted = truncate_text_preserving_suffix(
+        "General assistant instruction. " * 50,
+        "WEB SEARCH RESULTS " * 50,
+        max_tokens=10,
+    )
+
+    assert count_tokens(fitted) <= 10
+    assert "WEB SEARCH RESULTS" in fitted
+
+
+def test_truncate_text_omits_indicator_when_it_cannot_fit() -> None:
+    fitted = truncate_text("evidence " * 100, max_tokens=3)
+
+    assert count_tokens(fitted) <= 3
+    assert "content truncated" not in fitted