fix: replace synchronous open() with async file API and add keyboard accessibility

backend/delete_collection.py

@@ -0,0 +1,39 @@
+"""
+Script to delete the Qdrant collection and start fresh.
+Run this before re-ingesting to avoid duplicates.
+"""
+import os
+from dotenv import load_dotenv
+from src.services.qdrant_service import get_qdrant_client
+
+load_dotenv()
+
+def delete_collection():
+    """Delete the book_content_chunks collection."""
+    try:
+        client = get_qdrant_client()
+        collection_name = "book_content_chunks"
+
+        if client.collection_exists(collection_name=collection_name):
+            print(f"🗑️  Deleting collection '{collection_name}'...")
+            client.delete_collection(collection_name=collection_name)
+            print(f"✅ Collection '{collection_name}' deleted successfully!")
+            print(f"📝 You can now run ingestion to create fresh data with language tags.")
+        else:
+            print(f"ℹ️  Collection '{collection_name}' does not exist. Nothing to delete.")
+
+    except Exception as e:
+        print(f"❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+
+if __name__ == "__main__":
+    print("🔍 Qdrant Collection Cleanup")
+    print("=" * 80)
+
+    response = input("⚠️  This will delete ALL data in the collection. Continue? (yes/no): ")
+
+    if response.lower() in ['yes', 'y']:
+        delete_collection()
+    else:
+        print("❌ Operation cancelled.")

backend/pyproject.toml

@@ -5,18 +5,26 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
-    "aiosqlite>=0.21.0",
+    "aiohttp>=3.13.3",
+    "aiosqlite>=0.22.1",
     "asyncpg>=0.31.0",
-    "fastapi>=0.124.0",
+    "bcrypt>=5.0.0",
+    "fastapi>=0.128.6",
+    "fastapi-limiter>=0.2.0",
     "fastembed>=0.7.4",
+    "google-generativeai>=0.8.6",
+    "groq>=1.0.0",
     "httpx>=0.28.1",
-    "mangum>=0.19.0",
-    "openai>=2.9.0",
+    "openai>=2.18.0",
     "passlib[bcrypt]>=1.7.4",
+    "psycopg2>=2.9.11",
     "pydantic>=2.12.5",
     "pytest>=9.0.2",
     "python-dotenv>=1.2.1",
-    "qdrant-client>=1.16.1",
-    "sqlmodel>=0.0.27",
-    "uvicorn>=0.38.0",
+    "python-jose[cryptography]>=3.5.0",
+    "qdrant-client>=1.16.2",
+    "redis>=7.1.1",
+    "sqlalchemy>=2.0.46",
+    "sqlmodel>=0.0.32",
+    "uvicorn>=0.40.0",
 ]

backend/src/api/auth.py

@@ -13,7 +13,6 @@
     ErrorResponse
 )
 from ..services.auth_service import auth_service
-from fastapi_limiter.depends import RateLimiter # Import RateLimiter
 
 auth_router = APIRouter(prefix="/auth", tags=["auth"])
 

backend/src/api/chat.py

@@ -1,17 +1,26 @@
 import traceback
+import re
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 from typing import List, Optional
+from groq import Groq
+import os
+from dotenv import load_dotenv
 
-from ..services.embedding_service import get_embeddings, get_gemini_embedding_client
+from ..services.embedding_service import get_embeddings
 from ..services.qdrant_service import get_qdrant_client
-from .ingest import DocumentChunk # Reuse DocumentChunk model
-from dotenv import load_dotenv
+from .ingest import DocumentChunk
 
 load_dotenv()
 
 router = APIRouter()
 
+groq_client = Groq(api_key=os.getenv("GROQ_API_KEY"))
+
+# -------------------------
+# Models
+# -------------------------
+
 class Citation(BaseModel):
     doc_id: str
     chunk_id: str
@@ -26,28 +35,75 @@ class ChatResponse(BaseModel):
     answer: str
     citations: List[Citation] = []
 
-@router.post("/chat", response_model=ChatResponse)
-async def chat_with_bot(request: ChatRequest):
+
+# -------------------------
+# Utility: Clean Context
+# -------------------------
+
+def clean_context(text: str) -> str:
     """
-    Interacts with the RAG chatbot, retrieves context, and provides answers with citations.
+    Remove markdown, frontmatter, JSX components,
+    and Docusaurus formatting before sending to LLM.
     """
+    # Remove frontmatter (--- title ---)
+    text = re.sub(r"---.*?---", "", text, flags=re.DOTALL)
+
+    # Remove import lines
+    text = re.sub(r"import .*", "", text)
+
+    # Remove JSX components like <PersonalizeButton />
+    text = re.sub(r"<.*?>", "", text)
+
+    # Remove ::: blocks (tip, danger, etc.)
+    text = re.sub(r":::[\s\S]*?:::", "", text)
+
+    # Remove markdown symbols
+    text = re.sub(r"[#>*`]", "", text)
+
+    # Remove extra whitespace
+    text = re.sub(r"\n\s*\n", "\n\n", text)
+
+    return text.strip()
+
+
+# -------------------------
+# Chat Endpoint
+# -------------------------
+
+@router.post("/chat", response_model=ChatResponse)
+async def chat_with_bot(request: ChatRequest):
     try:
+        # -------------------------
+        # 1️⃣ Basic Query Guard
+        # -------------------------
+        if len(request.query.strip().split()) < 3:
+            return ChatResponse(
+                answer="Hi, there. Please ask a question related to the book's content.",
+                citations=[]
+            )
+
         qdrant_client = get_qdrant_client()
         collection_name = "book_content_chunks"
 
         if not qdrant_client.collection_exists(collection_name=collection_name):
-            raise HTTPException(status_code=404, detail=f"Qdrant collection '{collection_name}' not found. Please ingest documents first.")
+            raise HTTPException(
+                status_code=404,
+                detail=f"Qdrant collection '{collection_name}' not found. Please ingest documents first."
+            )
 
-        # Determine the text to use for retrieval
+        # -------------------------
+        # 2️⃣ Create Embedding
+        # -------------------------
         retrieval_query_text = request.selected_text if request.selected_text else request.query
-
         query_embedding = get_embeddings([retrieval_query_text])[0]
 
-        # Retrieve relevant chunks from Qdrant
+        # -------------------------
+        # 3️⃣ Retrieve Top Chunks
+        # -------------------------
         search_result = qdrant_client.query_points(
             collection_name=collection_name,
-            query=query_embedding, # Use query_embedding as the vector argument
-            limit=3, # Retrieve top N relevant documents for context
+            query=query_embedding,
+            limit=2,  # Reduced from 3 to prevent token overflow
             with_payload=True
         )
 
@@ -58,51 +114,67 @@ async def chat_with_bot(request: ChatRequest):
             for scored_point in search_result.points:
                 if scored_point.payload:
                     chunk = DocumentChunk(**scored_point.payload)
+
                     context_chunks.append(chunk)
+
                     citations.append(Citation(
-                        doc_id=chunk.chapter_title, # Using chapter_title as doc_id for now
-                        chunk_id=f"{chunk.url_slug}-{scored_point.id}", # Unique ID for each chunk
-                        url=f"/docs{chunk.url_slug}" # Assuming Docusaurus URL structure
+                        doc_id=chunk.chapter_title,
+                        chunk_id=f"{chunk.url_slug}-{scored_point.id}",
+                        url=f"/docs{chunk.url_slug}"
                     ))
 
-        # Construct prompt for the LLM
-        context_text = "\n\n".join([chunk.page_content for chunk in context_chunks])
-
-        # User-specified instruction: "in chatbot the agent must only get the query if its is related to book or its content otherwise aplogy reply"
-        # This will be handled by the prompt engineering
+        # -------------------------
+        # 4️⃣ Build Clean Context
+        # -------------------------
+        context_text = "\n\n".join([
+            clean_context(chunk.page_content)
+            for chunk in context_chunks
+        ])
+
         if not context_text:
-             return ChatResponse(answer="I apologize, but I couldn't find any relevant information in the book to answer your question. Please try rephrasing or ask a question directly related to the book's content.", citations=[])
+            return ChatResponse(
+                answer="I apologize, but I couldn't find relevant information in the book.",
+                citations=[]
+            )
 
+        # -------------------------
+        # 5️⃣ HARD TOKEN SAFETY LIMIT
+        # -------------------------
+        MAX_CONTEXT_CHARS = 12000  # ~3000 tokens approx
+        context_text = context_text[:MAX_CONTEXT_CHARS]
 
+        # -------------------------
+        # 6️⃣ Construct Prompt
+        # -------------------------
         prompt_messages = [
-            {"role": "system", "content": "You are a helpful book assistant. Answer the user's question ONLY based on the provided context from the book. If the question cannot be answered from the context, politely state that you cannot answer it and suggest asking a question related to the book's content. Do not use outside knowledge."},
-            {"role": "user", "content": f"Context from the book:\n{context_text}\n\nUser Question: {request.query}"}
+            {
+                "role": "system",
+                "content": "You are a helpful book assistant. Answer only from the provided context. If the answer is not in the context, say you cannot answer."
+            },
+            {
+                "role": "user",
+                "content": f"Context:\n{context_text}\n\nQuestion: {request.query}"
+            }
         ]
-
-        # Get Gemini LLM client configured via OpenAI client
-        gemini_llm_client = get_gemini_embedding_client() # Reusing the same client setup, but will call chat.completions
-
-        # For chat completion, Gemini uses models like "gemini-pro" or "gemini-1.5-pro"
-        # I'll use "gemini-1.5-pro" as it's a capable model.
-        # This might need adjustment based on available models via the OpenAI-compatible endpoint.
-        try:
-            chat_response = gemini_llm_client.chat.completions.create(
-                model="gemini-2.5-flash-lite", # Use an appropriate Gemini chat model
-                messages=prompt_messages,
-                temperature=0.7,
-                max_tokens=500
-            )
-            answer = chat_response.choices[0].message.content
-        except Exception as llm_error:
-            # Fallback for polite refusal if LLM call fails
-            print(f"LLM call failed: {llm_error}")
-            return ChatResponse(answer="I apologize, but I couldn't process your request at the moment. Please try again later.", citations=[])
+
+        # -------------------------
+        # 7️⃣ Call Groq
+        # -------------------------
+        chat_response = groq_client.chat.completions.create(
+            model="llama-3.1-8b-instant",
+            messages=prompt_messages,
+            temperature=0.5,
+            max_tokens=500
+        )
+
+        answer = chat_response.choices[0].message.content
 
         return ChatResponse(answer=answer, citations=citations)
 
     except HTTPException as e:
         raise e
+
     except Exception as e:
-        print(f"An unexpected error occurred: {e}")
-        traceback.print_exc() # Print the full traceback
-        raise HTTPException(status_code=500, detail=str(e))
+        print(f"Unexpected error: {e}")
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail="Internal server error.")