diff --git a/api/api/chatbot_service.py b/api/api/chatbot_service.py index f9de5af..fa49724 100644 --- a/api/api/chatbot_service.py +++ b/api/api/chatbot_service.py @@ -293,6 +293,7 @@ def detect_image_context_card_ids( count_tokens, count_message_tokens, truncate_text, + truncate_text_preserving_suffix, truncate_conversation_history, truncate_document_content, ) @@ -691,14 +692,15 @@ def get_client_for_request(has_image: bool): - chamoru_info_dictionary - chamorro_english_dictionary_TOD -2. **NEVER guess or hallucinate translations** - - If you don't see the word in a dictionary source, say: "I don't have that specific translation in my dictionary sources." - - DO NOT make up Chamorro words - - DO NOT use words from blog posts or articles as authoritative translations +2. **Never present an unsupported translation as verified** + - If a dictionary source is attached, keep the translation and its citation faithful to that source. + - If no dictionary source matched, you may still give a plausible candidate when useful, but label it "Unverified best effort" and state the uncertainty plainly. + - DO NOT make up citations or use words from blog posts or articles as authoritative translations. 3. **How to answer word translation questions:** ✅ CORRECT: "In Chamorro, 'listen' is **ekungok**. [Source: chamorro_english_dictionary_TOD]" - ❌ WRONG: Guessing or using non-dictionary content for single-word translations + ✅ CORRECT WHEN NO SOURCE MATCHES: "Unverified best effort: a plausible candidate is **...**, but I could not verify it in the available dictionaries." + ❌ WRONG: Presenting a guess or non-dictionary content as a verified translation 4. **For contextual/cultural questions** (not single-word translations): - You may use all sources (blogs, articles, cultural content) @@ -721,11 +723,14 @@ def get_client_for_request(has_image: bool): 🔴 Para i tiningo' palåbra (word translations): - Usa HA' i diksionarion-måmi (dictionaries): revised_and_updated_chamorro_dictionary, chamoru_info_dictionary, chamorro_english_dictionary_TOD -- MUNGA un adibina palåbra! (DO NOT guess words!) +- Never present an unsupported candidate as a verified translation. +- If no dictionary source matched and a credible candidate would help, you may + provide it only as an explicitly unverified best effort, with plain uncertainty. +- If there is no credible candidate, say so and ask one focused question or + suggest a related source-backed lookup. -Use governed Chamorro dictionary/canonical context for language claims. Munga un -adibina pat un fa'tinas nuebu na tiningo'. Yanggen ti guaha sufisiente na prineba, -na'fanmanungo' na ti siña un na'siguru. +Use governed Chamorro dictionary/canonical context for language claims. Keep the +entire response in Chamorro, including any uncertainty label or clarification. If you receive web search results, use them but respond in Chamorro only.""" }, @@ -746,7 +751,11 @@ def get_client_for_request(has_image: bool): - chamoru_info_dictionary - chamorro_english_dictionary_TOD -NEVER guess or make up Chamorro words. If unsure, say "I don't have that translation." +Never present a guess as a verified Chamorro word. When no dictionary source +matched, a plausible candidate may be offered only as an explicitly unverified +best effort with the uncertainty stated plainly. If there is no credible +candidate, say so and ask one focused question or suggest a related source-backed +lookup. For abbreviations, literal translations, phrase variants, pronunciation, and usage claims, retrieve a governed source and state uncertainty when the evidence @@ -758,10 +767,16 @@ def get_client_for_request(has_image: bool): NO_REFERENCE_GUARD = """ NO GOVERNED REFERENCE WAS RETRIEVED FOR THIS REQUEST: -- Do not add a translation, abbreviation expansion, pronunciation, etymology, - cultural/regional usage, or new example sentence from model memory. -- Say that the requested accuracy-sensitive detail could not be verified from - the available references, and offer to help with a source-backed alternative. +- Do not refuse solely because retrieval returned no match. Give the most useful + concise answer you can while preserving the selected response language. +- For an accuracy-sensitive Chamorro translation, abbreviation, pronunciation, + etymology, cultural claim, regional-usage claim, or example sentence, clearly + label any plausible model-memory candidate as an unverified best effort. +- Separate what you know from what you are inferring, and name material uncertainty. + Use plain certainty wording; never invent a percentage confidence. +- Never invent a citation or imply that an unverified candidate came from a source. +- If you do not have a credible candidate, say that directly and ask one focused + clarifying question or suggest a related source-backed lookup. """ # Skill level modifiers - adjust response style based on user's experience @@ -1311,6 +1326,7 @@ def early_cancelled_response(log_user_message: bool = True): # Check if we should use web search use_web, search_type = should_use_web_search(message) web_context = "" + web_results_used = False if use_web: # Check for cancellation before web search @@ -1320,6 +1336,7 @@ def early_cancelled_response(log_user_message: bool = True): search_result = web_search(message, search_type=search_type, max_results=3) if search_result["success"] and search_result["results"]: web_context = format_search_results(search_result) + web_results_used = bool(web_context) # Check for cancellation before RAG search if is_message_cancelled(pending_id): @@ -1380,18 +1397,21 @@ def early_cancelled_response(log_user_message: bool = True): elif not is_passage_translation(effective_translation_message) and not school_announcement: system_prompt += NO_REFERENCE_GUARD - # Add web search context if available - if web_context: - system_prompt += f"\n\n{web_context}" - # Initialize token manager for this request token_manager = TokenManager(budget=TokenBudget(), model=LLM_MODEL_ID) - - # Apply token limit to system prompt - system_prompt_tokens = count_tokens(system_prompt) + + # Keep usable web results in the final model prompt even when general + # instructions and retrieved context exceed the system-prompt budget. + system_prompt_tokens = count_tokens( + system_prompt + (f"\n\n{web_context}" if web_context else "") + ) if system_prompt_tokens > token_manager.budget.system_prompt: logger.warning(f"System prompt ({system_prompt_tokens} tokens) exceeds budget, truncating...") - system_prompt = truncate_text(system_prompt, token_manager.budget.system_prompt) + system_prompt = truncate_text_preserving_suffix( + system_prompt, + f"\n\n{web_context}" if web_context else "", + token_manager.budget.system_prompt, + ) # Build conversation history history = [ @@ -1496,7 +1516,7 @@ def early_cancelled_response(log_user_message: bool = True): sources = [] used_rag = False - use_web = False + web_results_used = False # Calculate response time response_time = time.time() - start_time @@ -1518,7 +1538,7 @@ def early_cancelled_response(log_user_message: bool = True): mode=mode, sources=[], used_rag=used_rag, - used_web_search=use_web, + used_web_search=web_results_used, response_time=response_time, session_id=session_id, user_id=user_id, @@ -1532,7 +1552,7 @@ def early_cancelled_response(log_user_message: bool = True): "response": "[Message was cancelled by user]", "sources": [], "used_rag": used_rag, - "used_web_search": use_web, + "used_web_search": web_results_used, "response_time": response_time, "cancelled": True } @@ -1544,7 +1564,7 @@ def early_cancelled_response(log_user_message: bool = True): mode=mode, sources=formatted_sources, used_rag=used_rag, - used_web_search=use_web, + used_web_search=web_results_used, response_time=response_time, session_id=session_id, user_id=user_id, @@ -1561,7 +1581,7 @@ def early_cancelled_response(log_user_message: bool = True): "response": response_text, "sources": formatted_sources, "used_rag": used_rag, - "used_web_search": use_web, + "used_web_search": web_results_used, "response_time": response_time, "cancelled": False } @@ -1637,6 +1657,7 @@ def get_chatbot_response_stream( # Check if we should use web search use_web, search_type = should_use_web_search(message) web_context = "" + web_results_used = False if use_web: if is_message_cancelled(pending_id): @@ -1646,6 +1667,7 @@ def get_chatbot_response_stream( search_result = web_search(message, search_type=search_type, max_results=3) if search_result["success"] and search_result["results"]: web_context = format_search_results(search_result) + web_results_used = bool(web_context) # Check for cancellation before RAG if is_message_cancelled(pending_id): @@ -1707,15 +1729,17 @@ def get_chatbot_response_stream( elif not is_passage_translation(effective_translation_message) and not school_announcement: system_prompt += NO_REFERENCE_GUARD - # Add web search context if available - if web_context: - system_prompt += f"\n\n{web_context}" - # Track token usage and apply limits - system_prompt_tokens = count_tokens(system_prompt) + system_prompt_tokens = count_tokens( + system_prompt + (f"\n\n{web_context}" if web_context else "") + ) if system_prompt_tokens > token_manager.budget.system_prompt: logger.warning(f"System prompt ({system_prompt_tokens} tokens) exceeds budget ({token_manager.budget.system_prompt}), truncating...") - system_prompt = truncate_text(system_prompt, token_manager.budget.system_prompt) + system_prompt = truncate_text_preserving_suffix( + system_prompt, + f"\n\n{web_context}" if web_context else "", + token_manager.budget.system_prompt, + ) # Build conversation history history = [{"role": "system", "content": system_prompt}] @@ -1763,7 +1787,7 @@ def get_chatbot_response_stream( "type": "metadata", "sources": formatted_sources if should_show_sources else [], "used_rag": used_rag, - "used_web_search": use_web + "used_web_search": web_results_used } # Stream LLM response @@ -1787,7 +1811,7 @@ def get_chatbot_response_stream( mode=mode, sources=[], used_rag=used_rag, - used_web_search=use_web, + used_web_search=web_results_used, response_time=time.time() - start_time, session_id=session_id, user_id=user_id, @@ -1828,7 +1852,7 @@ def get_chatbot_response_stream( mode=mode, sources=formatted_sources, used_rag=used_rag, - used_web_search=use_web, + used_web_search=web_results_used, response_time=time.time() - start_time, session_id=session_id, user_id=user_id, @@ -1913,7 +1937,7 @@ def get_chatbot_response_stream( mode=mode, sources=[], used_rag=used_rag, - used_web_search=use_web, + used_web_search=web_results_used, response_time=time.time() - start_time, session_id=session_id, user_id=user_id, @@ -1937,7 +1961,7 @@ def get_chatbot_response_stream( mode=mode, sources=formatted_sources, used_rag=used_rag, - used_web_search=use_web, + used_web_search=web_results_used, response_time=response_time, session_id=session_id, user_id=user_id, diff --git a/api/api/models.py b/api/api/models.py index 5a31c77..abab7ba 100644 --- a/api/api/models.py +++ b/api/api/models.py @@ -81,7 +81,7 @@ class ChatResponse(BaseModel): ) used_web_search: bool = Field( default=False, - description="Whether web search was used" + description="Whether usable web search results informed the response" ) response_time: Optional[float] = Field( None, diff --git a/api/src/utils/token_manager.py b/api/src/utils/token_manager.py index ad5cd81..f8cf2ab 100644 --- a/api/src/utils/token_manager.py +++ b/api/src/utils/token_manager.py @@ -149,6 +149,8 @@ def truncate_text(text: str, max_tokens: int, model: str = "gpt-4o") -> str: Returns: Truncated text with "[truncated]" indicator if needed """ + if max_tokens <= 0: + return "" if not text: return text @@ -163,6 +165,11 @@ def truncate_text(text: str, max_tokens: int, model: str = "gpt-4o") -> str: # Leave room for truncation indicator truncation_indicator = "\n\n[... content truncated due to length ...]" indicator_tokens = count_tokens(truncation_indicator, model) + + # For very small budgets the indicator itself would violate the limit. + # Preserve only as much original content as the caller allowed. + if indicator_tokens >= max_tokens: + return tokenizer.decode(tokens[:max_tokens]) # Truncate tokens truncated_tokens = tokens[:max_tokens - indicator_tokens] @@ -173,7 +180,48 @@ def truncate_text(text: str, max_tokens: int, model: str = "gpt-4o") -> str: logger.warning(f"Token-based truncation failed, using char estimate: {e}") # Fallback: ~4 chars per token max_chars = max_tokens * 4 - return text[:max_chars] + "\n\n[... content truncated ...]" + return text[:max_chars] + + +def truncate_text_preserving_suffix( + text: str, + suffix: str, + max_tokens: int, + model: str = "gpt-4o", +) -> str: + """Fit text within a token budget while retaining a priority suffix. + + The suffix receives up to one third of the budget when the combined text is + oversized. This is intended for evidence appended after general prompt + instructions, where prefix-only truncation would silently remove the + evidence while leaving the response metadata unchanged. + """ + if max_tokens <= 0: + return "" + + combined = text + suffix + if count_tokens(combined, model) <= max_tokens: + return combined + if not suffix: + return truncate_text(text, max_tokens, model) + + suffix_budget = min(count_tokens(suffix, model), max(1, max_tokens // 3)) + fitted_suffix = truncate_text(suffix, suffix_budget, model) + prefix_budget = max(0, max_tokens - count_tokens(fitted_suffix, model)) + fitted_prefix = truncate_text(text, prefix_budget, model) if prefix_budget else "" + combined = fitted_prefix + fitted_suffix + + # Token boundaries can change when two independently encoded strings are + # joined. Tighten only the prefix so the priority evidence remains present. + while count_tokens(combined, model) > max_tokens and prefix_budget > 0: + overflow = count_tokens(combined, model) - max_tokens + prefix_budget = max(0, prefix_budget - overflow) + fitted_prefix = truncate_text(text, prefix_budget, model) if prefix_budget else "" + combined = fitted_prefix + fitted_suffix + + if count_tokens(combined, model) > max_tokens: + return truncate_text(fitted_suffix, max_tokens, model) + return combined def truncate_conversation_history( @@ -624,4 +672,3 @@ def get_token_summary(self) -> dict: "remaining_for_response": self.tokens_remaining_for_response(), "budget_total": self.budget.total } - diff --git a/api/tests/test_prompt_token_budget.py b/api/tests/test_prompt_token_budget.py new file mode 100644 index 0000000..f71b14d --- /dev/null +++ b/api/tests/test_prompt_token_budget.py @@ -0,0 +1,40 @@ +from src.utils.token_manager import count_tokens, truncate_text, truncate_text_preserving_suffix + + +def test_oversized_prompt_preserves_web_evidence_suffix() -> None: + base_prompt = "General assistant instruction. " * 500 + web_evidence = "\n\nWEB SEARCH RESULTS\nCurrent Guam weather result: sunny." + + fitted = truncate_text_preserving_suffix(base_prompt, web_evidence, max_tokens=120) + + assert count_tokens(fitted) <= 120 + assert "WEB SEARCH RESULTS" in fitted + assert "Current Guam weather result: sunny." in fitted + assert "General assistant instruction" in fitted + + +def test_prompt_without_suffix_keeps_existing_prefix_truncation() -> None: + base_prompt = "General assistant instruction. " * 500 + + fitted = truncate_text_preserving_suffix(base_prompt, "", max_tokens=80) + + assert count_tokens(fitted) <= 80 + assert fitted.startswith("General assistant instruction") + + +def test_oversized_suffix_respects_a_very_small_budget() -> None: + fitted = truncate_text_preserving_suffix( + "General assistant instruction. " * 50, + "WEB SEARCH RESULTS " * 50, + max_tokens=10, + ) + + assert count_tokens(fitted) <= 10 + assert "WEB SEARCH RESULTS" in fitted + + +def test_truncate_text_omits_indicator_when_it_cannot_fit() -> None: + fitted = truncate_text("evidence " * 100, max_tokens=3) + + assert count_tokens(fitted) <= 3 + assert "content truncated" not in fitted diff --git a/api/tests/test_system.py b/api/tests/test_system.py index 1113a48..acc51dd 100644 --- a/api/tests/test_system.py +++ b/api/tests/test_system.py @@ -51,6 +51,17 @@ def test_chat_model_registry_and_prompts_are_current_modules() -> None: "analysis_guidance, school_announcement, contextual_card_ids = (" ) == 2 assert "Do not add etymology, pronunciation, cultural-origin" in chatbot_source + assert "Do not refuse solely because retrieval returned no match" in chatbot_source + assert 'label it "Unverified best effort"' in chatbot_source + assert "never invent a percentage confidence" in chatbot_source + assert "Never invent a citation" in chatbot_source + assert "CORRECT WHEN NO SOURCE MATCHES" in chatbot_source + assert "Presenting a guess or non-dictionary content as a verified translation" in chatbot_source + assert "provide it only as an explicitly unverified best effort" in chatbot_source + assert "Keep the\nentire response in Chamorro" in chatbot_source + assert "If there is no credible candidate" in chatbot_source + assert chatbot_source.count("web_results_used = bool(web_context)") == 2 + assert '"used_web_search": web_results_used' in chatbot_source def test_crawler_inventory_is_present() -> None: diff --git a/web/src/components/Message.evidence.test.tsx b/web/src/components/Message.evidence.test.tsx new file mode 100644 index 0000000..4d11e65 --- /dev/null +++ b/web/src/components/Message.evidence.test.tsx @@ -0,0 +1,53 @@ +import { render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { Message } from './Message'; + +vi.mock('@clerk/clerk-react', () => ({ + useAuth: () => ({ getToken: vi.fn() }), +})); + +vi.mock('../hooks/useSpeech', () => ({ + useSpeech: () => ({ + speak: vi.fn(), + stop: vi.fn(), + extractChamorroText: (content: string) => content, + isSpeaking: false, + isSupported: false, + }), +})); + +describe('Message evidence disclosure', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('shows source-supported when citations are attached', () => { + render( + , + ); + + expect( + screen.getByRole('note', { name: 'Answer evidence: Source-supported' }), + ).toHaveTextContent('Check the citations below.'); + }); + + it('shows web-informed when current web context was used without RAG citations', () => { + render(); + + expect( + screen.getByRole('note', { name: 'Answer evidence: Web-informed' }), + ).toHaveTextContent('Current web results were used.'); + }); + + it('clearly marks an answer with no matched evidence as unverified', () => { + render(); + + expect( + screen.getByRole('note', { name: 'Answer evidence: Unverified best effort' }), + ).toHaveTextContent('No supporting source matched.'); + }); +}); diff --git a/web/src/components/Message.tsx b/web/src/components/Message.tsx index 079b638..4c411b8 100644 --- a/web/src/components/Message.tsx +++ b/web/src/components/Message.tsx @@ -6,6 +6,7 @@ import remarkGfm from 'remark-gfm'; import { SourceCitation } from './SourceCitation'; import { useSpeech } from '../hooks/useSpeech'; import type { SourceInfo } from '../types/source'; +import { getChatEvidenceStatus } from '../lib/chatEvidence'; /** * Clean markdown content to prevent unwanted code blocks. @@ -128,6 +129,10 @@ export const Message = memo(function Message({ role, content, imageUrl, file_url const [isEditing, setIsEditing] = useState(false); const [editContent, setEditContent] = useState(content); const { speak, stop, extractChamorroText, isSpeaking, isSupported } = useSpeech(); + const evidenceStatus = useMemo( + () => getChatEvidenceStatus(sources?.length ?? 0, Boolean(used_web_search)), + [sources?.length, used_web_search], + ); // Clean content to prevent unwanted code blocks from leading whitespace const cleanedContent = useMemo(() => cleanMarkdownContent(content), [content]); @@ -677,12 +682,28 @@ export const Message = memo(function Message({ role, content, imageUrl, file_url )} {!isUser && !isSystem && !isStreaming && ( -

- {sources && sources.length > 0 ? ( - <>

+ {evidenceStatus.level === 'source_supported' ? ( +

)} diff --git a/web/src/lib/chatEvidence.test.ts b/web/src/lib/chatEvidence.test.ts new file mode 100644 index 0000000..b67fb8f --- /dev/null +++ b/web/src/lib/chatEvidence.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest'; +import { getChatEvidenceStatus } from './chatEvidence'; + +describe('chat evidence status', () => { + it('prioritizes attached citations over retrieval flags', () => { + expect(getChatEvidenceStatus(2, true)).toEqual({ + level: 'source_supported', + label: 'Source-supported', + detail: 'Check the citations below.', + }); + }); + + it('distinguishes current web context from an unsupported answer', () => { + expect(getChatEvidenceStatus(0, true).level).toBe('web_informed'); + expect(getChatEvidenceStatus(0, false)).toEqual({ + level: 'best_effort', + label: 'Unverified best effort', + detail: 'No supporting source matched.', + }); + }); +}); diff --git a/web/src/lib/chatEvidence.ts b/web/src/lib/chatEvidence.ts new file mode 100644 index 0000000..4e31986 --- /dev/null +++ b/web/src/lib/chatEvidence.ts @@ -0,0 +1,33 @@ +export type ChatEvidenceLevel = 'source_supported' | 'web_informed' | 'best_effort'; + +export interface ChatEvidenceStatus { + level: ChatEvidenceLevel; + label: string; + detail: string; +} + +/** Classify the evidence actually attached to or used by a completed answer. */ +export function getChatEvidenceStatus( + sourceCount: number, + usedWebResults: boolean, +): ChatEvidenceStatus { + if (sourceCount > 0) { + return { + level: 'source_supported', + label: 'Source-supported', + detail: 'Check the citations below.', + }; + } + if (usedWebResults) { + return { + level: 'web_informed', + label: 'Web-informed', + detail: 'Current web results were used.', + }; + } + return { + level: 'best_effort', + label: 'Unverified best effort', + detail: 'No supporting source matched.', + }; +}