From 9439cf9f6793e55a25232df09637624479aa01fb Mon Sep 17 00:00:00 2001 From: Aditya30ag Date: Thu, 10 Jul 2025 19:49:34 +0530 Subject: [PATCH 1/4] feat: Implement Enhanced FAQ Handler with Web Search for Organizational Queries (#97) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✨ Features: - Enhanced FAQ Tool with intelligent organizational query detection - Dynamic web search for current organizational information - LLM-powered response synthesis from official sources - Source attribution with clear citations - Backward compatibility with existing FAQ responses 🔧 Architecture: - Created EnhancedFAQTool with pattern-based query classification - Added organizational FAQ handler with LLM synthesis - Enhanced ReAct supervisor for better query routing - Comprehensive prompts for query detection and synthesis 📝 Implementation: - Detects organizational queries using regex patterns and keywords - Generates targeted search queries for official sources - Uses Tavily Search API for web search capabilities - Synthesizes responses using Gemini LLM - Maintains fallback mechanisms for reliability 🎯 Addresses: - Dynamic organizational information retrieval - Reduces manual FAQ maintenance overhead - Provides up-to-date information from official sources - Maintains existing technical FAQ functionality Resolves #97 --- ENHANCED_FAQ_IMPLEMENTATION.md | 223 +++++++++++++++++ backend/app/agents/devrel/agent.py | 2 +- .../app/agents/devrel/nodes/handlers/faq.py | 92 ++++++- .../nodes/handlers/organizational_faq.py | 133 ++++++++++ .../prompts/organizational_faq_prompt.py | 101 ++++++++ .../app/agents/devrel/prompts/react_prompt.py | 25 +- .../agents/devrel/tools/enhanced_faq_tool.py | 231 ++++++++++++++++++ backend/app/agents/devrel/tools/faq_tool.py | 64 +++-- 8 files changed, 830 insertions(+), 41 deletions(-) create mode 100644 ENHANCED_FAQ_IMPLEMENTATION.md create mode 100644 backend/app/agents/devrel/nodes/handlers/organizational_faq.py create mode 100644 backend/app/agents/devrel/prompts/organizational_faq_prompt.py create mode 100644 backend/app/agents/devrel/tools/enhanced_faq_tool.py diff --git a/ENHANCED_FAQ_IMPLEMENTATION.md b/ENHANCED_FAQ_IMPLEMENTATION.md new file mode 100644 index 00000000..cc9b5457 --- /dev/null +++ b/ENHANCED_FAQ_IMPLEMENTATION.md @@ -0,0 +1,223 @@ +# Enhanced FAQ Handler with Web Search for Organizational Queries + +## Overview + +This implementation fulfills the feature request for an enhanced FAQ Handler that leverages web search to answer general, high-level questions about the Devr.AI organization. The system provides dynamic, up-to-date information by searching official sources and synthesizing comprehensive responses. + +## Features Implemented + +### 🌟 Core Capabilities + +1. **Intelligent Query Classification**: Automatically detects whether a question is about organizational information or technical support +2. **Dynamic Web Search**: Performs targeted web searches for organizational queries using official sources +3. **LLM-Enhanced Synthesis**: Uses AI to synthesize search results into comprehensive, accurate answers +4. **Source Attribution**: Provides clear citations and links to official sources +5. **Fallback Mechanisms**: Maintains backward compatibility with existing static FAQ responses + +### 🔍 Organizational Query Detection + +The system recognizes queries such as: +- "What is Devr.AI all about?" +- "What projects does this organization work on?" +- "What are the main goals of Devr.AI?" +- "What platforms does Devr.AI support?" +- "How does this organization work?" + +### 🎯 Technical Query Support + +Maintains support for existing technical FAQs: +- "How do I contribute?" +- "How do I report a bug?" +- "How to get started?" +- "What is LangGraph?" + +## Architecture + +### Components Created + +1. **Enhanced FAQ Tool** (`enhanced_faq_tool.py`) + - Core logic for query classification and web search + - Pattern-based organizational query detection + - Targeted search query generation + - Response synthesis and source management + +2. **Organizational FAQ Handler** (`organizational_faq.py`) + - Specialized handler for organizational queries + - LLM-powered response synthesis + - Advanced formatting and source attribution + +3. **Enhanced Prompts** (`organizational_faq_prompt.py`) + - Query classification prompts + - Search query generation prompts + - Response synthesis prompts + - Fallback response prompts + +4. **Updated Components** + - Modified existing `faq_tool.py` to use enhanced functionality + - Updated `faq.py` handler with enhanced response handling + - Enhanced ReAct supervisor prompt for better routing + +## Workflow + +### Step 1: Query Reception +User asks a question → FAQ Handler receives the query + +### Step 2: Query Classification +```python +# Example: "What projects does Devr.AI work on?" +is_organizational = _is_organizational_query(question) +# Returns: True +``` + +### Step 3: Search Query Generation +```python +search_queries = [ + "Devr.AI open source projects", + "Devr.AI GitHub repositories", + "Devr.AI projects developer relations" +] +``` + +### Step 4: Web Search Execution +- Performs targeted searches using Tavily Search API +- Focuses on official sources (website, GitHub, documentation) +- Deduplicates results by URL + +### Step 5: Response Synthesis +- Uses LLM to synthesize search results +- Maintains accuracy by only using information from search results +- Formats response with proper structure and citations + +### Step 6: Response Delivery +```json +{ + "status": "success", + "answer": "Devr.AI primarily focuses on creating tools for developer relations...", + "sources": [ + {"title": "Devr.AI - Official Website", "url": "https://devr.ai/projects"}, + {"title": "Devr.AI on GitHub", "url": "https://github.com/AOSSIE-Org/Devr.AI"} + ], + "type": "organizational_faq" +} +``` + +## Integration Points + +### ReAct Supervisor Integration +The ReAct supervisor now intelligently routes organizational queries to the FAQ handler: + +```python +# Enhanced decision logic: +# - Organizational questions → faq_handler +# - Technical questions → faq_handler +# - External research → web_search +# - GitHub operations → github_toolkit +``` + +### Backward Compatibility +- Existing technical FAQ responses preserved +- Legacy API methods maintained +- Graceful fallback to static responses if web search fails + +## Key Files Modified/Created + +### New Files +- `backend/app/agents/devrel/tools/enhanced_faq_tool.py` +- `backend/app/agents/devrel/nodes/handlers/organizational_faq.py` +- `backend/app/agents/devrel/prompts/organizational_faq_prompt.py` + +### Modified Files +- `backend/app/agents/devrel/tools/faq_tool.py` +- `backend/app/agents/devrel/nodes/handlers/faq.py` +- `backend/app/agents/devrel/prompts/react_prompt.py` +- `backend/app/agents/devrel/agent.py` + +## Usage Examples + +### Organizational Query +**Input**: "What kind of projects does Devr.AI work on?" + +**Process**: +1. Detects organizational query +2. Generates search queries: "Devr.AI open source projects", "Devr.AI GitHub repositories" +3. Searches web for current information +4. Synthesizes response from official sources +5. Returns comprehensive answer with citations + +**Output**: +``` +Devr.AI primarily focuses on creating tools for developer relations (DevRel), +including AI-powered assistants for community engagement, issue triage, and +onboarding. You can explore our main projects on our official website and GitHub page. + +**Sources:** +1. [Devr.AI - Official Website](https://devr.ai/projects) +2. [Devr.AI on GitHub](https://github.com/AOSSIE-Org/Devr.AI) +``` + +### Technical Query +**Input**: "How do I contribute to Devr.AI?" + +**Process**: +1. Matches against static technical FAQ +2. Returns immediate response + +**Output**: +``` +You can contribute by visiting our GitHub repository, checking open issues, +and submitting pull requests. We welcome all types of contributions including +code, documentation, and bug reports. +``` + +## Configuration + +### Environment Variables Required +- `TAVILY_API_KEY`: For web search functionality +- `GEMINI_API_KEY`: For LLM-powered synthesis + +### Dependencies +- `tavily-python`: Web search API client +- `langchain-google-genai`: LLM integration +- `langgraph`: Agent workflow framework + +## Benefits + +### 🚀 Dynamic Information +- Always up-to-date organizational information +- Reduces manual FAQ maintenance +- Leverages official sources automatically + +### 🎯 Accuracy +- Source attribution for transparency +- LLM synthesis prevents hallucination +- Fallback mechanisms ensure reliability + +### 📈 Scalability +- No need to manually update organizational FAQs +- Automatically discovers new information +- Handles diverse question phrasings + +### 🔧 Maintainability +- Modular architecture +- Clear separation of concerns +- Backward compatibility preserved + +## Testing + +The implementation includes: +- Error handling for API failures +- Fallback mechanisms for offline scenarios +- Comprehensive logging for debugging +- Type hints for better code maintainability + +## Future Enhancements + +Potential improvements could include: +- Caching of search results for performance +- User feedback integration for response quality +- Multi-language support for international users +- Integration with internal knowledge bases + +--- + +This implementation successfully transforms the FAQ Handler from a static knowledge base into a dynamic, intelligent system that can provide current, accurate information about the Devr.AI organization while maintaining all existing functionality. \ No newline at end of file diff --git a/backend/app/agents/devrel/agent.py b/backend/app/agents/devrel/agent.py index 53a5a93f..0dfae887 100644 --- a/backend/app/agents/devrel/agent.py +++ b/backend/app/agents/devrel/agent.py @@ -28,7 +28,7 @@ def __init__(self, config: Dict[str, Any] = None): google_api_key=settings.gemini_api_key ) self.search_tool = TavilySearchTool() - self.faq_tool = FAQTool() + self.faq_tool = FAQTool(search_tool=self.search_tool) self.github_toolkit = GitHubToolkit() self.checkpointer = InMemorySaver() super().__init__("DevRelAgent", self.config) diff --git a/backend/app/agents/devrel/nodes/handlers/faq.py b/backend/app/agents/devrel/nodes/handlers/faq.py index 8855c323..3288b88a 100644 --- a/backend/app/agents/devrel/nodes/handlers/faq.py +++ b/backend/app/agents/devrel/nodes/handlers/faq.py @@ -1,10 +1,11 @@ import logging +from typing import Dict, Any from app.agents.state import AgentState logger = logging.getLogger(__name__) -async def handle_faq_node(state: AgentState, faq_tool) -> dict: - """Handle FAQ requests""" +async def handle_faq_node(state: AgentState, faq_tool) -> Dict[str, Any]: + """Handle FAQ requests with enhanced organizational query support""" logger.info(f"Handling FAQ for session {state.session_id}") latest_message = "" @@ -13,14 +14,79 @@ async def handle_faq_node(state: AgentState, faq_tool) -> dict: elif state.context.get("original_message"): latest_message = state.context["original_message"] - # faq_tool will be passed from the agent, similar to llm for classify_intent - faq_response = await faq_tool.get_response(latest_message) - - return { - "task_result": { - "type": "faq", - "response": faq_response, - "source": "faq_database" - }, - "current_task": "faq_handled" - } + try: + # Get enhanced response with metadata + enhanced_response = await faq_tool.get_enhanced_response(latest_message) + + # Extract response details + response_text = enhanced_response.get("response") + response_type = enhanced_response.get("type", "unknown") + sources = enhanced_response.get("sources", []) + search_queries = enhanced_response.get("search_queries", []) + + # Log the type of response for monitoring + logger.info(f"FAQ response type: {response_type} for session {state.session_id}") + + if response_text: + # Successfully got a response + return { + "task_result": { + "type": response_type, + "response": response_text, + "source": enhanced_response.get("source", "enhanced_faq"), + "sources": sources, + "search_queries": search_queries, + "has_sources": len(sources) > 0 + }, + "current_task": "faq_handled", + "tools_used": ["enhanced_faq_tool"] + } + else: + # No response found + logger.info(f"No FAQ response found for: {latest_message[:100]}") + return { + "task_result": { + "type": "no_match", + "response": None, + "source": "enhanced_faq", + "sources": [], + "search_queries": [], + "has_sources": False + }, + "current_task": "faq_no_match" + } + + except Exception as e: + logger.error(f"Error in enhanced FAQ handler: {str(e)}") + + # Fallback to simple response + try: + simple_response = await faq_tool.get_response(latest_message) + if simple_response: + return { + "task_result": { + "type": "fallback_faq", + "response": simple_response, + "source": "faq_fallback", + "sources": [], + "search_queries": [], + "has_sources": False + }, + "current_task": "faq_handled" + } + except Exception as fallback_error: + logger.error(f"Fallback FAQ also failed: {str(fallback_error)}") + + # Return error state + return { + "task_result": { + "type": "error", + "response": None, + "source": "error", + "error": str(e), + "sources": [], + "search_queries": [], + "has_sources": False + }, + "current_task": "faq_error" + } diff --git a/backend/app/agents/devrel/nodes/handlers/organizational_faq.py b/backend/app/agents/devrel/nodes/handlers/organizational_faq.py new file mode 100644 index 00000000..db6a9a86 --- /dev/null +++ b/backend/app/agents/devrel/nodes/handlers/organizational_faq.py @@ -0,0 +1,133 @@ +import logging +from typing import Dict, Any, List +from app.agents.state import AgentState +from langchain_core.messages import HumanMessage + +logger = logging.getLogger(__name__) + +# Prompt for synthesizing organizational responses +ORGANIZATIONAL_SYNTHESIS_PROMPT = """You are a helpful AI assistant representing Devr.AI. \ +Based on the search results below, provide a comprehensive and accurate answer to the \ +user's question about our organization. + +User Question: "{question}" + +Search Results: +{search_results} + +Instructions: +1. Synthesize the information from the search results into a coherent, informative response +2. Focus on providing accurate information about Devr.AI +3. Be concise but comprehensive +4. If the search results don't contain enough information, acknowledge this and provide what you can +5. Maintain a professional and friendly tone +6. Do not make up information not present in the search results + +Response:""" + +async def handle_organizational_faq_node(state: AgentState, enhanced_faq_tool, llm) -> Dict[str, Any]: + """Handle organizational FAQ requests with web search and LLM synthesis""" + logger.info(f"Handling organizational FAQ for session {state.session_id}") + + latest_message = "" + if state.messages: + latest_message = state.messages[-1].get("content", "") + elif state.context.get("original_message"): + latest_message = state.context["original_message"] + + # Get response from enhanced FAQ tool + faq_response = await enhanced_faq_tool.get_response(latest_message) + + # If it's an organizational query, enhance with LLM synthesis + if faq_response.get("type") == "organizational_faq": + search_results = faq_response.get("sources", []) + + if search_results: + # Format search results for LLM + formatted_results = _format_search_results_for_llm(search_results) + + # Use LLM to synthesize a better response + synthesis_prompt = ORGANIZATIONAL_SYNTHESIS_PROMPT.format( + question=latest_message, + search_results=formatted_results + ) + + try: + llm_response = await llm.ainvoke([HumanMessage(content=synthesis_prompt)]) + synthesized_answer = llm_response.content.strip() + + # Update the response with the synthesized answer + faq_response["response"] = synthesized_answer + faq_response["synthesis_method"] = "llm_enhanced" + + logger.info( + f"Enhanced organizational response with LLM synthesis " + f"for session {state.session_id}" + ) + except Exception as e: + logger.error(f"Error in LLM synthesis: {str(e)}") + # Keep the original response if LLM synthesis fails + faq_response["synthesis_method"] = "basic" + + return { + "task_result": { + "type": "organizational_faq", + "response": faq_response.get("response"), + "source": faq_response.get("source", "enhanced_faq"), + "sources": faq_response.get("sources", []), + "search_queries": faq_response.get("search_queries", []), + "synthesis_method": faq_response.get("synthesis_method", "none"), + "query_type": faq_response.get("type", "unknown") + }, + "current_task": "organizational_faq_handled" + } + +def _format_search_results_for_llm(search_results: List[Dict[str, Any]]) -> str: + """Format search results for LLM synthesis""" + if not search_results: + return "No search results available." + + formatted_parts = [] + for i, result in enumerate(search_results, 1): + title = result.get('title', 'No title') + url = result.get('url', 'No URL') + content = result.get('content', 'No content available') + + formatted_part = f""" +Result {i}: +Title: {title} +URL: {url} +Content: {content[:500]}{"..." if len(content) > 500 else ""} +""" + formatted_parts.append(formatted_part) + + return "\n".join(formatted_parts) + +def create_organizational_response(task_result: Dict[str, Any]) -> str: + """Create a user-friendly response string from organizational FAQ results""" + response = task_result.get("response", "") + sources = task_result.get("sources", []) + query_type = task_result.get("query_type", "") + + if not response: + return ("I couldn't find specific information about that. Please try rephrasing your " + "question or check our official documentation.") + + # Start with the main response + response_parts = [response] + + # Add sources if available + if sources: + response_parts.append("\n\n**Sources:**") + for i, source in enumerate(sources[:3], 1): + title = source.get('title', 'Source') + url = source.get('url', '') + response_parts.append(f"{i}. [{title}]({url})") + + # Add helpful footer for organizational queries + if query_type == "organizational_faq": + response_parts.append( + "\n\nFor more information, you can also visit our official website or GitHub repository." + ) + + return "\n".join(response_parts) diff --git a/backend/app/agents/devrel/prompts/organizational_faq_prompt.py b/backend/app/agents/devrel/prompts/organizational_faq_prompt.py new file mode 100644 index 00000000..c2674660 --- /dev/null +++ b/backend/app/agents/devrel/prompts/organizational_faq_prompt.py @@ -0,0 +1,101 @@ +# Prompts for organizational FAQ handling and synthesis + +# Prompt for detecting organizational queries +ORGANIZATIONAL_QUERY_DETECTION_PROMPT = """You are an AI assistant that helps classify user questions. +Determine if the following question is asking about organizational information (about the company, +projects, mission, goals, etc.) or technical support. + +User Question: "{question}" + +Classification Guidelines: +- ORGANIZATIONAL: Questions about the company, its mission, projects, team, goals, platforms, + general information about what the organization does +- TECHNICAL: Questions about how to use the product, troubleshooting, implementation details, + contribution guidelines, specific feature requests + +Examples of ORGANIZATIONAL questions: +- "What is Devr.AI?" +- "What projects does this organization work on?" +- "What are the main goals of Devr.AI?" +- "What platforms does Devr.AI support?" +- "Tell me about this organization" + +Examples of TECHNICAL questions: +- "How do I contribute to the project?" +- "How do I report a bug?" +- "What is LangGraph?" +- "How do I get started with development?" + +Respond with only: ORGANIZATIONAL or TECHNICAL""" + +# Prompt for generating targeted search queries for organizational information +ORGANIZATIONAL_SEARCH_QUERY_GENERATION_PROMPT = """You are an AI assistant that helps generate effective +search queries. Based on the user's organizational question, generate 2-3 specific search queries that +would find relevant information about Devr.AI. + +User Question: "{question}" + +Guidelines for search queries: +1. Include "Devr.AI" in each query +2. Focus on official sources (website, GitHub, documentation) +3. Be specific to the type of information requested +4. Avoid overly broad or generic terms + +Generate search queries that would find information about: +- Official website content +- GitHub repositories and documentation +- Project descriptions and goals +- Platform integrations and capabilities + +Format your response as a JSON list of strings: +["query1", "query2", "query3"]""" + +# Enhanced synthesis prompt for organizational responses +ORGANIZATIONAL_SYNTHESIS_PROMPT = """You are the official AI representative for Devr.AI. +Your task is to provide a comprehensive, accurate, and helpful answer to the user's question about +our organization based on the search results provided. + +User Question: "{question}" + +Search Results: +{search_results} + +Instructions: +1. **Accuracy First**: Only use information directly found in the search results +2. **Comprehensive Coverage**: Address all aspects of the user's question if information is available +3. **Professional Tone**: Maintain a friendly but professional tone appropriate for developer relations +4. **Structured Response**: Organize information logically with clear sections if needed +5. **Source Attribution**: If specific claims are made, they should be traceable to the search results +6. **Acknowledge Limitations**: If search results don't contain enough information, be honest about it +7. **Call to Action**: When appropriate, guide users to official resources for more information + +Response Format: +- Start with a direct answer to the main question +- Provide supporting details from search results +- Include relevant examples or specifics when available +- End with helpful next steps or resources if appropriate + +Avoid: +- Making up information not present in search results +- Being overly promotional or sales-like +- Providing outdated information +- Generic or vague responses + +Response:""" + +# Prompt for fallback responses when search results are insufficient +ORGANIZATIONAL_FALLBACK_PROMPT = """You are the AI representative for Devr.AI. The user asked an +organizational question, but the search results didn't provide sufficient information to answer +comprehensively. + +User Question: "{question}" + +Provide a helpful fallback response that: +1. Acknowledges their question +2. Provides any basic information you know about Devr.AI (AI-powered DevRel assistant) +3. Directs them to official sources for complete information +4. Maintains a helpful and professional tone + +Keep the response concise but useful, and avoid making specific claims without evidence. + +Response:""" diff --git a/backend/app/agents/devrel/prompts/react_prompt.py b/backend/app/agents/devrel/prompts/react_prompt.py index b92e5f68..53e9e9e0 100644 --- a/backend/app/agents/devrel/prompts/react_prompt.py +++ b/backend/app/agents/devrel/prompts/react_prompt.py @@ -13,19 +13,28 @@ {tool_results} AVAILABLE ACTIONS: -1. web_search - Search the web for external information -2. faq_handler - Answer common questions from knowledge base -3. onboarding - Welcome new users and guide exploration +1. web_search - Search the web for external information not related to our organization +2. faq_handler - Answer questions using knowledge base AND web search for organizational queries +3. onboarding - Welcome new users and guide exploration 4. github_toolkit - Handle GitHub operations (issues, PRs, repos, docs) 5. complete - Task is finished, format final response +ENHANCED FAQ HANDLER CAPABILITIES: +The faq_handler now has advanced capabilities for organizational queries: +- Detects questions about Devr.AI, our projects, mission, goals, and platforms +- Automatically searches the web for current organizational information +- Synthesizes responses from official sources (website, GitHub, docs) +- Provides static answers for technical FAQ questions +- Returns structured responses with source citations + THINK: Analyze the user's request and current context. What needs to be done? -Then choose ONE action: -- If you need external information or recent updates → web_search -- If this is a common question with a known answer → faq_handler -- If this is a new user needing guidance → onboarding -- If this involves GitHub repositories, issues, PRs, or code → github_toolkit +Choose ONE action based on these guidelines: +- If asking about Devr.AI organization, projects, mission, goals, or "what is..." → faq_handler +- If asking technical questions like "how to contribute", "report bugs" → faq_handler +- If you need external information unrelated to our organization → web_search +- If this is a new user needing general guidance → onboarding +- If this involves GitHub repositories, issues, PRs, or code operations → github_toolkit - If you have enough information to fully answer → complete Respond in this exact format: diff --git a/backend/app/agents/devrel/tools/enhanced_faq_tool.py b/backend/app/agents/devrel/tools/enhanced_faq_tool.py new file mode 100644 index 00000000..43e65b4f --- /dev/null +++ b/backend/app/agents/devrel/tools/enhanced_faq_tool.py @@ -0,0 +1,231 @@ +import logging +import re +from typing import Dict, Any, List +from .search_tool import TavilySearchTool + +logger = logging.getLogger(__name__) + +class EnhancedFAQTool: + """Enhanced FAQ handling tool with web search for organizational queries""" + + def __init__(self, search_tool: TavilySearchTool | None = None): + self.search_tool = search_tool or TavilySearchTool() + + # Static FAQ responses for technical questions + self.technical_faq_responses = { + "how do i contribute": ( + "You can contribute by visiting our GitHub repository, checking open issues, " + "and submitting pull requests. We welcome all types of contributions including " + "code, documentation, and bug reports." + ), + "how do i report a bug": ( + "You can report a bug by opening an issue on our GitHub repository. " + "Please include detailed information about the bug, steps to reproduce it, " + "and your environment." + ), + "how to get started": ( + "To get started with Devr.AI: 1) Check our documentation, 2) Join our Discord " + "community, 3) Explore the GitHub repository, 4) Try contributing to open issues." + ), + "what is langgraph": ( + "LangGraph is a framework for building stateful, multi-actor applications " + "with large language models. We use it to create intelligent agent workflows " + "for our DevRel automation." + ) + } + + # Patterns that indicate organizational queries + self.organizational_patterns = [ + r"what.*is.*(devr\.ai|organization|company|this.*project)", + r"what.*does.*(devr\.ai|organization|company|this.*project).*do", + r"(about|tell me about).*(devr\.ai|organization|company|this.*project)", + r"what.*projects.*(do you have|does.*have|are there)", + r"what.*goals.*(devr\.ai|organization|company)", + r"what.*mission.*(devr\.ai|organization|company)", + r"how.*does.*(devr\.ai|organization|company).*work", + r"what.*platforms.*(devr\.ai|support|integrate)", + r"who.*maintains.*(devr\.ai|organization|company)", + r"what.*kind.*projects.*(devr\.ai|organization|company)" + ] + + def _is_organizational_query(self, question: str) -> bool: + """Check if the question is about the organization using pattern matching""" + question_lower = question.lower().strip() + + for pattern in self.organizational_patterns: + if re.search(pattern, question_lower): + return True + + # Additional keyword-based detection + org_keywords = ["devr.ai", "organization", "company", "projects", "mission", "goals", "about us"] + question_keywords = ["what", "how", "tell me", "explain", "describe"] + + has_org_keyword = any(keyword in question_lower for keyword in org_keywords) + has_question_keyword = any(keyword in question_lower for keyword in question_keywords) + + return has_org_keyword and has_question_keyword + + def _generate_search_queries(self, question: str) -> List[str]: + """Generate targeted search queries for organizational information""" + base_queries = [] + question_lower = question.lower() + + # Project-related queries + if any(word in question_lower for word in ["project", "projects", "work on", "building"]): + base_queries.extend([ + "Devr.AI open source projects", + "Devr.AI GitHub repositories", + "Devr.AI projects developer relations" + ]) + + # Mission/goals queries + if any(word in question_lower for word in ["mission", "goal", "purpose", "about"]): + base_queries.extend([ + "Devr.AI mission developer relations", + "About Devr.AI organization", + "Devr.AI goals AI automation" + ]) + + # Platform/integration queries + if any(word in question_lower for word in ["platform", "platforms", "integrate", "support"]): + base_queries.extend([ + "Devr.AI supported platforms integrations", + "Devr.AI Discord Slack GitHub integration", + "Devr.AI platform compatibility" + ]) + + # General organizational info + if any(word in question_lower for word in ["what is", "tell me about", "how does", "work"]): + base_queries.extend([ + "Devr.AI developer relations AI assistant", + "Devr.AI official website", + "Devr.AI community automation" + ]) + + # If no specific patterns matched, use generic queries + if not base_queries: + base_queries = [ + "Devr.AI developer relations", + "Devr.AI AI automation", + "Devr.AI open source" + ] + + return base_queries + + async def _search_organizational_info(self, question: str) -> Dict[str, Any]: + """Search for organizational information using web search""" + search_queries = self._generate_search_queries(question) + all_results = [] + + for query in search_queries[:3]: # Limit to 3 queries to avoid rate limits + try: + results = await self.search_tool.search(query, max_results=3) + all_results.extend(results) + except Exception as e: + logger.error(f"Error searching for '{query}': {str(e)}") + + # Remove duplicates based on URL + seen_urls = set() + unique_results = [] + for result in all_results: + url = result.get('url', '') + if url not in seen_urls: + seen_urls.add(url) + unique_results.append(result) + + return { + "type": "organizational_info", + "query": question, + "search_queries": search_queries, + "results": unique_results[:5], # Limit to top 5 results + "source": "web_search" + } + + def _synthesize_organizational_response(self, question: str, search_results: List[Dict[str, Any]]) -> str: + """Create a synthesized response from search results""" + if not search_results: + return self._get_fallback_response(question) + + # Extract relevant information from search results + response_parts = [] + sources = [] + + for result in search_results[:3]: # Use top 3 results + title = result.get('title', '') + content = result.get('content', '') + url = result.get('url', '') + + if content and len(content) > 50: # Only use substantial content + # Take first 200 characters of content + snippet = content[:200] + "..." if len(content) > 200 else content + response_parts.append(snippet) + sources.append({"title": title, "url": url}) + + if response_parts: + synthesized_answer = " ".join(response_parts) + return synthesized_answer + else: + return self._get_fallback_response(question) + + def _get_fallback_response(self, question: str) -> str: + """Provide a fallback response when search results are insufficient""" + return ("Devr.AI is an AI-powered Developer Relations assistant that helps open-source communities " + "by automating engagement, issue tracking, and providing intelligent support to developers. " + "For the most up-to-date information about our projects and organization, please visit our " + "official website and GitHub repository.") + + async def get_response(self, question: str) -> Dict[str, Any]: + """Get FAQ response - either static or dynamic based on query type""" + question_lower = question.lower().strip() + + # First, check static technical FAQ responses + for faq_key, response in self.technical_faq_responses.items(): + if self._is_similar_question(question_lower, faq_key): + return { + "type": "static_faq", + "response": response, + "source": "faq_database", + "sources": [] + } + + # Check if this is an organizational query + if self._is_organizational_query(question): + logger.info(f"Detected organizational query: {question}") + search_result = await self._search_organizational_info(question) + + synthesized_response = self._synthesize_organizational_response( + question, search_result.get("results", []) + ) + + # Format sources for response + sources = [] + for result in search_result.get("results", [])[:3]: + if result.get('title') and result.get('url'): + sources.append({ + "title": result.get('title', ''), + "url": result.get('url', '') + }) + + return { + "type": "organizational_faq", + "response": synthesized_response, + "source": "web_search", + "sources": sources, + "search_queries": search_result.get("search_queries", []) + } + + # If no match found, return None to indicate no FAQ response available + return { + "type": "no_match", + "response": None, + "source": "none", + "sources": [] + } + + def _is_similar_question(self, question: str, faq_key: str) -> bool: + """Check if question is similar to FAQ key using simple keyword matching""" + question_words = set(question.split()) + faq_words = set(faq_key.split()) + + common_words = question_words.intersection(faq_words) + return len(common_words) >= 2 # At least 2 common words diff --git a/backend/app/agents/devrel/tools/faq_tool.py b/backend/app/agents/devrel/tools/faq_tool.py index df331a5a..b1e7dcec 100644 --- a/backend/app/agents/devrel/tools/faq_tool.py +++ b/backend/app/agents/devrel/tools/faq_tool.py @@ -1,44 +1,70 @@ import logging -from typing import Optional +from typing import Optional, Dict, Any +from .enhanced_faq_tool import EnhancedFAQTool +from .search_tool import TavilySearchTool logger = logging.getLogger(__name__) class FAQTool: - """FAQ handling tool""" - - # TODO: Add FAQ responses from a database to refer organization's FAQ and Repo's FAQ - - def __init__(self): - self.faq_responses = { - "what is devr.ai": "Devr.AI is an AI-powered Developer Relations assistant that helps open-source communities by automating engagement, issue tracking, and providing intelligent support to developers.", - "how do i contribute": "You can contribute by visiting our GitHub repository, checking open issues, and submitting pull requests. We welcome all types of contributions including code, documentation, and bug reports.", - "what platforms does devr.ai support": "Devr.AI integrates with Discord, Slack, GitHub, and can be extended to other platforms. We use these integrations to provide seamless developer support across multiple channels.", - "who maintains devr.ai": "Devr.AI is maintained by an open-source community of developers passionate about improving developer relations and community engagement.", - "how do i report a bug": "You can report a bug by opening an issue on our GitHub repository. Please include detailed information about the bug, steps to reproduce it, and your environment.", - "how to get started": "To get started with Devr.AI: 1) Check our documentation, 2) Join our Discord community, 3) Explore the GitHub repository, 4) Try contributing to open issues.", - "what is langgraph": "LangGraph is a framework for building stateful, multi-actor applications with large language models. We use it to create intelligent agent workflows for our DevRel automation." + """FAQ handling tool with enhanced organizational query support""" + + def __init__(self, search_tool: TavilySearchTool | None = None): + """Initialize FAQ tool with enhanced functionality""" + self.enhanced_faq_tool = EnhancedFAQTool(search_tool) + + # Legacy FAQ responses for backward compatibility + self.legacy_faq_responses = { + "what is devr.ai": ( + "Devr.AI is an AI-powered Developer Relations assistant that helps open-source " + "communities by automating engagement, issue tracking, and providing intelligent " + "support to developers." + ), + "what platforms does devr.ai support": ( + "Devr.AI integrates with Discord, Slack, GitHub, and can be extended to other " + "platforms. We use these integrations to provide seamless developer support " + "across multiple channels." + ), + "who maintains devr.ai": ( + "Devr.AI is maintained by an open-source community of developers passionate " + "about improving developer relations and community engagement." + ) } async def get_response(self, question: str) -> Optional[str]: - """Get FAQ response for a question""" + """Get FAQ response for a question - enhanced with web search for organizational queries""" + try: + # Use the enhanced FAQ tool for better responses + enhanced_response = await self.enhanced_faq_tool.get_response(question) + + if enhanced_response and enhanced_response.get("response"): + return enhanced_response.get("response") + + # Fallback to legacy responses for backward compatibility question_lower = question.lower().strip() # Direct match - if question_lower in self.faq_responses: - return self.faq_responses[question_lower] + if question_lower in self.legacy_faq_responses: + return self.legacy_faq_responses[question_lower] # Fuzzy matching - for faq_key, response in self.faq_responses.items(): + for faq_key, response in self.legacy_faq_responses.items(): if self._is_similar_question(question_lower, faq_key): return response return None + except Exception as e: + logger.error(f"Error in FAQ tool: {str(e)}") + return None + def _is_similar_question(self, question: str, faq_key: str) -> bool: """Check if question is similar to FAQ key""" - # Simple keyword matching - in production, use better similarity question_words = set(question.split()) faq_words = set(faq_key.split()) common_words = question_words.intersection(faq_words) return len(common_words) >= 2 # At least 2 common words + + async def get_enhanced_response(self, question: str) -> Dict[str, Any]: + """Get enhanced FAQ response with metadata for advanced usage""" + return await self.enhanced_faq_tool.get_response(question) From 33e29249b17b6aca519f722298976b48e53d00d3 Mon Sep 17 00:00:00 2001 From: Aditya30ag Date: Wed, 16 Jul 2025 17:08:39 +0530 Subject: [PATCH 2/4] deleted the ENHANCED_FAQ_IMPLEMENTATION.md as not need --- ENHANCED_FAQ_IMPLEMENTATION.md | 223 --------------------------------- 1 file changed, 223 deletions(-) delete mode 100644 ENHANCED_FAQ_IMPLEMENTATION.md diff --git a/ENHANCED_FAQ_IMPLEMENTATION.md b/ENHANCED_FAQ_IMPLEMENTATION.md deleted file mode 100644 index cc9b5457..00000000 --- a/ENHANCED_FAQ_IMPLEMENTATION.md +++ /dev/null @@ -1,223 +0,0 @@ -# Enhanced FAQ Handler with Web Search for Organizational Queries - -## Overview - -This implementation fulfills the feature request for an enhanced FAQ Handler that leverages web search to answer general, high-level questions about the Devr.AI organization. The system provides dynamic, up-to-date information by searching official sources and synthesizing comprehensive responses. - -## Features Implemented - -### 🌟 Core Capabilities - -1. **Intelligent Query Classification**: Automatically detects whether a question is about organizational information or technical support -2. **Dynamic Web Search**: Performs targeted web searches for organizational queries using official sources -3. **LLM-Enhanced Synthesis**: Uses AI to synthesize search results into comprehensive, accurate answers -4. **Source Attribution**: Provides clear citations and links to official sources -5. **Fallback Mechanisms**: Maintains backward compatibility with existing static FAQ responses - -### 🔍 Organizational Query Detection - -The system recognizes queries such as: -- "What is Devr.AI all about?" -- "What projects does this organization work on?" -- "What are the main goals of Devr.AI?" -- "What platforms does Devr.AI support?" -- "How does this organization work?" - -### 🎯 Technical Query Support - -Maintains support for existing technical FAQs: -- "How do I contribute?" -- "How do I report a bug?" -- "How to get started?" -- "What is LangGraph?" - -## Architecture - -### Components Created - -1. **Enhanced FAQ Tool** (`enhanced_faq_tool.py`) - - Core logic for query classification and web search - - Pattern-based organizational query detection - - Targeted search query generation - - Response synthesis and source management - -2. **Organizational FAQ Handler** (`organizational_faq.py`) - - Specialized handler for organizational queries - - LLM-powered response synthesis - - Advanced formatting and source attribution - -3. **Enhanced Prompts** (`organizational_faq_prompt.py`) - - Query classification prompts - - Search query generation prompts - - Response synthesis prompts - - Fallback response prompts - -4. **Updated Components** - - Modified existing `faq_tool.py` to use enhanced functionality - - Updated `faq.py` handler with enhanced response handling - - Enhanced ReAct supervisor prompt for better routing - -## Workflow - -### Step 1: Query Reception -User asks a question → FAQ Handler receives the query - -### Step 2: Query Classification -```python -# Example: "What projects does Devr.AI work on?" -is_organizational = _is_organizational_query(question) -# Returns: True -``` - -### Step 3: Search Query Generation -```python -search_queries = [ - "Devr.AI open source projects", - "Devr.AI GitHub repositories", - "Devr.AI projects developer relations" -] -``` - -### Step 4: Web Search Execution -- Performs targeted searches using Tavily Search API -- Focuses on official sources (website, GitHub, documentation) -- Deduplicates results by URL - -### Step 5: Response Synthesis -- Uses LLM to synthesize search results -- Maintains accuracy by only using information from search results -- Formats response with proper structure and citations - -### Step 6: Response Delivery -```json -{ - "status": "success", - "answer": "Devr.AI primarily focuses on creating tools for developer relations...", - "sources": [ - {"title": "Devr.AI - Official Website", "url": "https://devr.ai/projects"}, - {"title": "Devr.AI on GitHub", "url": "https://github.com/AOSSIE-Org/Devr.AI"} - ], - "type": "organizational_faq" -} -``` - -## Integration Points - -### ReAct Supervisor Integration -The ReAct supervisor now intelligently routes organizational queries to the FAQ handler: - -```python -# Enhanced decision logic: -# - Organizational questions → faq_handler -# - Technical questions → faq_handler -# - External research → web_search -# - GitHub operations → github_toolkit -``` - -### Backward Compatibility -- Existing technical FAQ responses preserved -- Legacy API methods maintained -- Graceful fallback to static responses if web search fails - -## Key Files Modified/Created - -### New Files -- `backend/app/agents/devrel/tools/enhanced_faq_tool.py` -- `backend/app/agents/devrel/nodes/handlers/organizational_faq.py` -- `backend/app/agents/devrel/prompts/organizational_faq_prompt.py` - -### Modified Files -- `backend/app/agents/devrel/tools/faq_tool.py` -- `backend/app/agents/devrel/nodes/handlers/faq.py` -- `backend/app/agents/devrel/prompts/react_prompt.py` -- `backend/app/agents/devrel/agent.py` - -## Usage Examples - -### Organizational Query -**Input**: "What kind of projects does Devr.AI work on?" - -**Process**: -1. Detects organizational query -2. Generates search queries: "Devr.AI open source projects", "Devr.AI GitHub repositories" -3. Searches web for current information -4. Synthesizes response from official sources -5. Returns comprehensive answer with citations - -**Output**: -``` -Devr.AI primarily focuses on creating tools for developer relations (DevRel), -including AI-powered assistants for community engagement, issue triage, and -onboarding. You can explore our main projects on our official website and GitHub page. - -**Sources:** -1. [Devr.AI - Official Website](https://devr.ai/projects) -2. [Devr.AI on GitHub](https://github.com/AOSSIE-Org/Devr.AI) -``` - -### Technical Query -**Input**: "How do I contribute to Devr.AI?" - -**Process**: -1. Matches against static technical FAQ -2. Returns immediate response - -**Output**: -``` -You can contribute by visiting our GitHub repository, checking open issues, -and submitting pull requests. We welcome all types of contributions including -code, documentation, and bug reports. -``` - -## Configuration - -### Environment Variables Required -- `TAVILY_API_KEY`: For web search functionality -- `GEMINI_API_KEY`: For LLM-powered synthesis - -### Dependencies -- `tavily-python`: Web search API client -- `langchain-google-genai`: LLM integration -- `langgraph`: Agent workflow framework - -## Benefits - -### 🚀 Dynamic Information -- Always up-to-date organizational information -- Reduces manual FAQ maintenance -- Leverages official sources automatically - -### 🎯 Accuracy -- Source attribution for transparency -- LLM synthesis prevents hallucination -- Fallback mechanisms ensure reliability - -### 📈 Scalability -- No need to manually update organizational FAQs -- Automatically discovers new information -- Handles diverse question phrasings - -### 🔧 Maintainability -- Modular architecture -- Clear separation of concerns -- Backward compatibility preserved - -## Testing - -The implementation includes: -- Error handling for API failures -- Fallback mechanisms for offline scenarios -- Comprehensive logging for debugging -- Type hints for better code maintainability - -## Future Enhancements - -Potential improvements could include: -- Caching of search results for performance -- User feedback integration for response quality -- Multi-language support for international users -- Integration with internal knowledge bases - ---- - -This implementation successfully transforms the FAQ Handler from a static knowledge base into a dynamic, intelligent system that can provide current, accurate information about the Devr.AI organization while maintaining all existing functionality. \ No newline at end of file From fb9f2a92e53ca034e00354a592bcc4337a9b0703 Mon Sep 17 00:00:00 2001 From: Aditya30ag Date: Wed, 16 Jul 2025 17:24:07 +0530 Subject: [PATCH 3/4] incoprate the coderabbit issue resolve --- .../nodes/handlers/organizational_faq.py | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/backend/app/agents/devrel/nodes/handlers/organizational_faq.py b/backend/app/agents/devrel/nodes/handlers/organizational_faq.py index db6a9a86..1d6c7db5 100644 --- a/backend/app/agents/devrel/nodes/handlers/organizational_faq.py +++ b/backend/app/agents/devrel/nodes/handlers/organizational_faq.py @@ -2,30 +2,15 @@ from typing import Dict, Any, List from app.agents.state import AgentState from langchain_core.messages import HumanMessage +from app.agents.devrel.prompts.organizational_faq_prompt import ORGANIZATIONAL_SYNTHESIS_PROMPT logger = logging.getLogger(__name__) -# Prompt for synthesizing organizational responses -ORGANIZATIONAL_SYNTHESIS_PROMPT = """You are a helpful AI assistant representing Devr.AI. \ -Based on the search results below, provide a comprehensive and accurate answer to the \ -user's question about our organization. - -User Question: "{question}" - -Search Results: -{search_results} - -Instructions: -1. Synthesize the information from the search results into a coherent, informative response -2. Focus on providing accurate information about Devr.AI -3. Be concise but comprehensive -4. If the search results don't contain enough information, acknowledge this and provide what you can -5. Maintain a professional and friendly tone -6. Do not make up information not present in the search results - -Response:""" - -async def handle_organizational_faq_node(state: AgentState, enhanced_faq_tool, llm) -> Dict[str, Any]: +async def handle_organizational_faq_node( + state: AgentState, + enhanced_faq_tool: Any, + llm: Any +) -> Dict[str, Any]: """Handle organizational FAQ requests with web search and LLM synthesis""" logger.info(f"Handling organizational FAQ for session {state.session_id}") @@ -64,10 +49,13 @@ async def handle_organizational_faq_node(state: AgentState, enhanced_faq_tool, l f"Enhanced organizational response with LLM synthesis " f"for session {state.session_id}" ) - except Exception as e: + except (ValueError, AttributeError, TypeError) as e: logger.error(f"Error in LLM synthesis: {str(e)}") # Keep the original response if LLM synthesis fails faq_response["synthesis_method"] = "basic" + except Exception as e: + logger.error(f"Unexpected error in LLM synthesis: {str(e)}") + faq_response["synthesis_method"] = "basic" return { "task_result": { From 98a709da03d8e5b005faa19e92e8eac7b73aeb2b Mon Sep 17 00:00:00 2001 From: Aditya30ag Date: Wed, 16 Jul 2025 17:50:40 +0530 Subject: [PATCH 4/4] correct this file also acc to the coderabbit --- backend/app/agents/devrel/tools/faq_tool.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/app/agents/devrel/tools/faq_tool.py b/backend/app/agents/devrel/tools/faq_tool.py index b1e7dcec..b814cd5e 100644 --- a/backend/app/agents/devrel/tools/faq_tool.py +++ b/backend/app/agents/devrel/tools/faq_tool.py @@ -40,18 +40,18 @@ async def get_response(self, question: str) -> Optional[str]: return enhanced_response.get("response") # Fallback to legacy responses for backward compatibility - question_lower = question.lower().strip() + question_lower = question.lower().strip() - # Direct match - if question_lower in self.legacy_faq_responses: + # Direct match + if question_lower in self.legacy_faq_responses: return self.legacy_faq_responses[question_lower] - # Fuzzy matching + # Fuzzy matching for faq_key, response in self.legacy_faq_responses.items(): - if self._is_similar_question(question_lower, faq_key): - return response + if self._is_similar_question(question_lower, faq_key): + return response - return None + return None except Exception as e: logger.error(f"Error in FAQ tool: {str(e)}")