diff --git a/.env.example b/.env.example index 778b79d..969557b 100644 --- a/.env.example +++ b/.env.example @@ -14,8 +14,11 @@ GITHUB_CLIENT_SECRET="github_client_secret_placeholder" GOOGLE_CLIENT_ID="google_client_id_placeholder" GOOGLE_CLIENT_SECRET="google_client_secret_placeholder" -# AI services API keys -ANTHROPIC_API_KEY="sk-ant-api53-..." +# AI services — OpenRouter (unified access to 200+ models) +OPENROUTER_API_KEY="sk-or-v1-..." +# Model selection (optional, defaults to google/gemini-2.0-flash-001) +# Options: google/gemini-2.0-flash-001, openai/gpt-4o-mini, anthropic/claude-sonnet-4-20250514, deepseek/deepseek-chat +LLM_MODEL="google/gemini-2.0-flash-001" # Cloudflare R2 Storage credentials R2_ACCOUNT_ID="r2_account_id_placeholder" diff --git a/apps/ai-service/app/config.py b/apps/ai-service/app/config.py new file mode 100644 index 0000000..cb0dcb9 --- /dev/null +++ b/apps/ai-service/app/config.py @@ -0,0 +1,44 @@ +"""Application configuration loaded from environment variables.""" + +import os +from functools import lru_cache + + +class Settings: + """Settings loaded from environment / .env file.""" + + def __init__(self) -> None: + # OpenRouter provides unified access to 200+ models + self.openrouter_api_key: str = os.getenv("OPENROUTER_API_KEY", "") + self.openrouter_base_url: str = os.getenv( + "OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1" + ) + # Your site URL for OpenRouter rankings (set to your domain or leave blank) + self.openrouter_site_url: str = os.getenv("OPENROUTER_SITE_URL", "") + self.openrouter_app_name: str = os.getenv("OPENROUTER_APP_NAME", "UnVibe") + + # Model selection — change this to switch providers + # Options: google/gemini-2.0-flash-001, openai/gpt-4o-mini, + # anthropic/claude-sonnet-4-20250514, meta-llama/llama-3.3-70b-instruct, + # deepseek/deepseek-chat, mistralai/mistral-large-2407 + self.llm_model: str = os.getenv("LLM_MODEL", "google/gemini-2.0-flash-001") + self.max_tokens: int = int(os.getenv("LLM_MAX_TOKENS", "4096")) + self.ai_service_port: int = int(os.getenv("AI_SERVICE_PORT", "8000")) + + @property + def has_llm_key(self) -> bool: + """Check if a usable API key is configured.""" + key = self.openrouter_api_key + return bool(key) and key != "" and not key.startswith("sk-or-v1-placeholder") + + @property + def llm_key_preview(self) -> str: + """Return masked key for logging (first 8 chars).""" + k = self.openrouter_api_key + return f"{k[:8]}...{k[-4:]}" if len(k) > 12 else "(invalid)" + + +@lru_cache() +def get_settings() -> Settings: + """Return singleton settings instance.""" + return Settings() diff --git a/apps/ai-service/app/prompts/__init__.py b/apps/ai-service/app/prompts/__init__.py new file mode 100644 index 0000000..1e77a0c --- /dev/null +++ b/apps/ai-service/app/prompts/__init__.py @@ -0,0 +1 @@ +# Versioned Claude prompt templates for all AI features. diff --git a/apps/ai-service/app/prompts/v1/__init__.py b/apps/ai-service/app/prompts/v1/__init__.py new file mode 100644 index 0000000..f05c257 --- /dev/null +++ b/apps/ai-service/app/prompts/v1/__init__.py @@ -0,0 +1 @@ +# v1 prompt templates diff --git a/apps/ai-service/app/prompts/v1/code_generation.txt b/apps/ai-service/app/prompts/v1/code_generation.txt new file mode 100644 index 0000000..efdc5df --- /dev/null +++ b/apps/ai-service/app/prompts/v1/code_generation.txt @@ -0,0 +1,22 @@ +You are a senior software engineer. Generate a production-grade solution for the following problem. + +Problem: {problem_description} + +Language: {language} +Difficulty: {difficulty} + +Requirements: +- Write clean, well-commented code +- Handle edge cases (empty input, invalid values, boundary conditions) +- Follow {language} best practices and conventions +- Include type hints / type annotations where applicable +- Optimize for readability and maintainability +- DO NOT include any explanation or introductory text — return ONLY the code +- Return the code inside a single markdown code block with the language identifier + +Example output format: +```python +def solution(input_data): + # implementation + pass +``` \ No newline at end of file diff --git a/apps/ai-service/app/prompts/v1/defend_evaluation.txt b/apps/ai-service/app/prompts/v1/defend_evaluation.txt new file mode 100644 index 0000000..73e30ad --- /dev/null +++ b/apps/ai-service/app/prompts/v1/defend_evaluation.txt @@ -0,0 +1,24 @@ +You are evaluating a developer's answer in a Defend session. Determine whether their response demonstrates genuine understanding of their code. + +Question asked: {question} +Developer's answer: {answer} + +Full code context: +{code} + +Evaluate on these criteria: +1. Accuracy — Is the answer technically correct? +2. Depth — Does it show understanding beyond surface level? +3. Specificity — Does it reference specific lines, variables, or decisions in the code? +4. Confidence — Does the answer sound certain or guess-like? + +Return ONLY a JSON object with no additional text: +{{ + "passed": true, + "feedback": "Constructive feedback explaining what was right and what could be improved.", + "score": 85 +}} + +- passed: true if score >= 60, false otherwise +- feedback: 1-3 sentences with specific, actionable feedback referencing their answer +- score: 0-100 integer reflecting overall understanding \ No newline at end of file diff --git a/apps/ai-service/app/prompts/v1/defend_question.txt b/apps/ai-service/app/prompts/v1/defend_question.txt new file mode 100644 index 0000000..40c7e3f --- /dev/null +++ b/apps/ai-service/app/prompts/v1/defend_question.txt @@ -0,0 +1,22 @@ +You are conducting a Socratic interview about a developer's code submission. The developer has rebuilt a solution from memory during a Defend session. Ask them a probing question to test whether they truly understand the code's design decisions. + +Original problem: {problem_description} + +Developer's submitted code: +{code} + +Conversation so far: +{messages} + +Ask ONE probing question that: +- Challenges a specific design choice in their code (data structure, algorithm, pattern) +- Tests understanding of time or space complexity tradeoffs +- Probes error handling or edge cases they may have missed +- Asks WHY they chose one approach over an alternative +- Pushes them to think about scalability or real-world implications + +Rules: +- Do NOT ask yes/no questions — require a substantive answer +- Do NOT ask about obvious syntax — ask about design reasoning +- Do NOT repeat a question already asked (check conversation history) +- Return ONLY the question text, no explanations or prefixes \ No newline at end of file diff --git a/apps/ai-service/app/prompts/v1/quiz_generation.txt b/apps/ai-service/app/prompts/v1/quiz_generation.txt new file mode 100644 index 0000000..fe3a7f1 --- /dev/null +++ b/apps/ai-service/app/prompts/v1/quiz_generation.txt @@ -0,0 +1,31 @@ +You are a technical quiz generator. You are given a piece of code and optional annotations explaining parts of it. Generate {count} multiple-choice questions that test deep understanding of the code. + +Code: +{code} + +Annotations (student's explanations of code sections): +{annotations} + +Instructions for each question: +1. Focus on understanding WHY the code works, not just WHAT it does +2. Include exactly 4 options labeled A through D +3. Exactly one option must be correct +4. Include at least one distractor that reflects a common misconception +5. Vary difficulty — some surface-level, some requiring deep reasoning +6. Each question should have a brief explanation of why the correct answer is right + +Return ONLY a valid JSON object with no additional text. Use this exact format: +{{ + "title": "Comprehension Check: {topic}", + "questions": [ + {{ + "id": "q-1", + "question": "Why does the code do X on line Y?", + "options": ["Option A text", "Option B text", "Option C text", "Option D text"], + "correct_option": 0, + "explanation": "Brief explanation of why A is correct." + }} + ] +}} + +Generate exactly {count} questions. \ No newline at end of file diff --git a/apps/ai-service/app/routes/defend.py b/apps/ai-service/app/routes/defend.py index 2841561..b003f8a 100644 --- a/apps/ai-service/app/routes/defend.py +++ b/apps/ai-service/app/routes/defend.py @@ -1,35 +1,206 @@ +"""Defend session Q&A endpoint — Socratic questioning via Claude.""" + +import json +from typing import Optional + from fastapi import APIRouter, HTTPException from pydantic import BaseModel -from typing import List, Dict, Any from loguru import logger +from app.config import get_settings +from app.services.llm_client import llm, LLMClientError +from app.services.prompt_manager import render_prompt, strip_markdown_fence + router = APIRouter(prefix="/defend", tags=["defend"]) +MAX_QUESTIONS = 5 # Maximum questions before auto-evaluation + + class DefendMessage(BaseModel): - role: str # user or assistant + """A single message in the defend conversation.""" + role: str # "user" or "assistant" content: str + class DefendSessionRequest(BaseModel): + """Request to ask a question or evaluate in a defend session.""" session_id: str - messages: List[DefendMessage] code: str + problem_description: str + messages: list[DefendMessage] + class DefendResponse(BaseModel): - next_question: str - passed: bool - feedback: str | None = None + """Response from the defend endpoint.""" + next_question: Optional[str] = None + passed: bool = False + feedback: Optional[str] = None + score: Optional[int] = None + @router.post("/respond", response_model=DefendResponse) -async def respond_defend(req: DefendSessionRequest): - logger.info(f"Processing defend response for session: {req.session_id}") - # Simple defense evaluation mockup - if len(req.messages) >= 3: - return DefendResponse( - next_question="Defense completed.", - passed=True, - feedback="Great work defending your solution! You demonstrated strong conceptual understanding." +async def respond_defend(req: DefendSessionRequest) -> DefendResponse: + """ + Process a defend session interaction. + + Two modes based on conversation state: + 1. **Ask mode** (default) — Generates the next Socratic question using Claude, + based on the code, problem, and conversation history. + 2. **Evaluate mode** — Triggered when the conversation reaches MAX_QUESTIONS + exchanges. Claude evaluates the user's overall understanding and returns + a pass/fail verdict with feedback. + + The endpoint determines which mode to use based on the number of + assistant messages (questions asked so far) in the conversation. + """ + settings = get_settings() + + if not settings.has_llm_key: + raise HTTPException( + status_code=503, + detail="AI Service unavailable: OPENROUTER_API_KEY not configured.", ) - return DefendResponse( - next_question="Why did you choose this specific data structure here?", - passed=False + + # Count how many questions have been asked so far + questions_asked = sum(1 for m in req.messages if m.role == "assistant") + user_answers = sum(1 for m in req.messages if m.role == "user") + + logger.info( + "Defend session request", + session_id=req.session_id, + questions_asked=questions_asked, + user_answers=user_answers, + total_messages=len(req.messages), ) + + # Determine mode: if we've reached the limit, evaluate + if questions_asked >= MAX_QUESTIONS: + return await _evaluate_answer(req) + else: + return await _ask_question(req) + + +async def _ask_question(req: DefendSessionRequest) -> DefendResponse: + """Generate a Socratic question using Claude.""" + try: + # Format conversation history for the prompt + messages_text = _format_conversation(req.messages) + + prompt = render_prompt( + "defend_question", + problem_description=req.problem_description, + code=req.code, + messages=messages_text, + ) + + question = await llm.generate_async(prompt=prompt) + + return DefendResponse( + next_question=question.strip(), + passed=False, + feedback=None, + score=None, + ) + + except LLMClientError as exc: + logger.error(f"Defend question generation failed: {exc}") + raise HTTPException( + status_code=502, + detail=f"Failed to generate defend question: {exc}", + ) from exc + + +async def _evaluate_answer(req: DefendSessionRequest) -> DefendResponse: + """ + Evaluate the user's last answer and overall performance. + + Uses Claude to assess the last user response against the code context, + then returns pass/fail with detailed feedback and a score. + """ + try: + # Get the last user message + last_answer = "" + for msg in reversed(req.messages): + if msg.role == "user": + last_answer = msg.content + break + + # Get the question that was asked before that answer + last_question = "" + for msg in reversed(req.messages): + if msg.role == "assistant": + last_question = msg.content + break + + prompt = render_prompt( + "defend_evaluation", + question=last_question, + answer=last_answer, + code=req.code, + ) + + text = await llm.generate_async(prompt=prompt) + + # Parse JSON response + result = _parse_evaluation(text) + + passed = result.get("passed", False) + feedback = result.get("feedback", "No feedback provided.") + score = result.get("score", 0) + + logger.info( + "Defend evaluation complete", + session_id=req.session_id, + passed=passed, + score=score, + ) + + return DefendResponse( + next_question=None, + passed=passed, + feedback=feedback, + score=score, + ) + + except LLMClientError as exc: + logger.error(f"Defend evaluation failed: {exc}") + raise HTTPException( + status_code=502, + detail=f"Failed to evaluate defend answer: {exc}", + ) from exc + except (json.JSONDecodeError, KeyError, ValueError) as exc: + logger.error(f"Failed to parse evaluation from Claude: {exc}") + raise HTTPException( + status_code=502, + detail="Defend evaluation returned an invalid response format.", + ) from exc + + +def _format_conversation(messages: list[DefendMessage]) -> str: + """Format conversation history for the prompt template.""" + if not messages: + return "No previous conversation." + + lines = [] + for m in messages: + role_label = "Interviewer" if m.role == "assistant" else "Developer" + lines.append(f"{role_label}: {m.content}") + return "\n\n".join(lines) + + +def _parse_evaluation(text: str) -> dict: + """Parse Claude's JSON evaluation response.""" + # Strip markdown code fences if present + text = strip_markdown_fence(text) + + data = json.loads(text) + + # Validate expected fields + if "passed" not in data: + raise ValueError("Missing 'passed' field in evaluation response") + + return { + "passed": bool(data["passed"]), + "feedback": str(data.get("feedback", "")), + "score": max(0, min(100, int(data.get("score", 0)))), + } diff --git a/apps/ai-service/app/routes/diff.py b/apps/ai-service/app/routes/diff.py index 94a5671..d6ebdea 100644 --- a/apps/ai-service/app/routes/diff.py +++ b/apps/ai-service/app/routes/diff.py @@ -1,22 +1,65 @@ -from fastapi import APIRouter, HTTPException +"""Code diff endpoint — scores user rebuilds against original solutions.""" + +from fastapi import APIRouter from pydantic import BaseModel from loguru import logger +from app.services.ast_differ import differ as ast_differ + router = APIRouter(prefix="/diff", tags=["diff"]) + class DiffRequest(BaseModel): original_code: str updated_code: str + language: str = "python" -class DiffResponse(BaseModel): + +class DimensionScoreOut(BaseModel): + dimension: str + score: float explanation: str + + +class DiffResponse(BaseModel): + overall_score: float + dimensions: list[DimensionScoreOut] + summary: str clean_diff: str + @router.post("/", response_model=DiffResponse) -async def generate_diff_explanation(req: DiffRequest): - logger.info("Generating explanation for code differences") - # Simple mockup +async def generate_diff_explanation(req: DiffRequest) -> DiffResponse: + """ + Score a user's rebuild against the original solution. + + Uses the AST diff engine to compare code structurally and qualitatively. + For Python code, performs full AST-based analysis across four dimensions. + For other languages, falls back to text-based similarity. + """ + logger.info( + "Running diff", + language=req.language, + original_length=len(req.original_code), + updated_length=len(req.updated_code), + ) + + result = ast_differ.compare( + original=req.original_code, + updated=req.updated_code, + language=req.language, + ) + return DiffResponse( - explanation="Refactored the loop to be more memory efficient by utilizing generators instead of loading all items in-memory.", - clean_diff="@@ -1,3 +1,3 @@\n-items = [x for x in data]\n+items = (x for x in data)" + overall_score=result.overall_score, + dimensions=[ + DimensionScoreOut( + dimension=d.dimension, + score=d.score, + explanation=d.explanation, + ) + for d in result.dimensions + ], + summary=result.summary, + clean_diff=result.clean_diff, ) diff --git a/apps/ai-service/app/routes/generate.py b/apps/ai-service/app/routes/generate.py index cb6b345..4dd44d1 100644 --- a/apps/ai-service/app/routes/generate.py +++ b/apps/ai-service/app/routes/generate.py @@ -1,28 +1,79 @@ +"""Code generation endpoint — calls Claude to produce production-grade code.""" + from fastapi import APIRouter, HTTPException from pydantic import BaseModel from loguru import logger -import os + +from app.config import get_settings +from app.services.llm_client import llm, LLMClientError +from app.services.prompt_manager import render_prompt, strip_markdown_fence router = APIRouter(prefix="/generate", tags=["generate"]) + class GenerateRequest(BaseModel): - prompt: str - max_tokens: int = 1024 + problem_description: str + language: str = "python" + difficulty: str = "medium" + class GenerateResponse(BaseModel): - text: str + code: str + language: str + model_used: str + token_count: int + @router.post("/", response_model=GenerateResponse) -async def generate_text(req: GenerateRequest): - logger.info(f"Generating content for prompt: {req.prompt[:50]}...") - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - logger.warning("ANTHROPIC_API_KEY is not set. Returning mock response.") - return GenerateResponse(text=f"Mock response for prompt: {req.prompt}") - - # In a real implementation: - # client = anthropic.Anthropic(api_key=api_key) - # response = client.messages.create(...) - # return GenerateResponse(text=response.content[0].text) - - return GenerateResponse(text=f"Successfully processed prompt on mock backend: {req.prompt}") +async def generate_text(req: GenerateRequest) -> GenerateResponse: + """ + Generate production-grade code for a given problem using Claude. + + Uses the code_generation prompt template with the provided problem + description, language, and difficulty. Returns the generated code + along with metadata about the model used. + """ + settings = get_settings() + + if not settings.has_llm_key: + raise HTTPException( + status_code=503, + detail="AI Service unavailable: OPENROUTER_API_KEY not configured. " + "Set it in your .env file to enable code generation.", + ) + + try: + prompt = render_prompt( + "code_generation", + problem_description=req.problem_description, + language=req.language, + difficulty=req.difficulty, + ) + + logger.info( + "Generating code", + language=req.language, + difficulty=req.difficulty, + model=settings.llm_model, + prompt_length=len(prompt), + ) + + text = await llm.generate_async(prompt=prompt) + text = strip_markdown_fence(text) + + # Estimate token count from response (rough: ~4 chars per token) + estimated_tokens = max(1, len(text) // 4) + + return GenerateResponse( + code=text, + language=req.language, + model_used=settings.llm_model, + token_count=estimated_tokens, + ) + + except LLMClientError as exc: + logger.error(f"Code generation failed: {exc}") + raise HTTPException( + status_code=502, + detail=f"AI generation failed: {exc}", + ) from exc diff --git a/apps/ai-service/app/routes/quiz.py b/apps/ai-service/app/routes/quiz.py index 0b9b79d..997ec37 100644 --- a/apps/ai-service/app/routes/quiz.py +++ b/apps/ai-service/app/routes/quiz.py @@ -1,30 +1,138 @@ +"""Quiz generation endpoint — generates comprehension quizzes from code using Claude.""" + +import json +from typing import Optional + from fastapi import APIRouter, HTTPException from pydantic import BaseModel -from typing import List from loguru import logger +from app.config import get_settings +from app.services.llm_client import llm, LLMClientError +from app.services.prompt_manager import render_prompt, strip_markdown_fence + router = APIRouter(prefix="/quiz", tags=["quiz"]) + +class Annotation(BaseModel): + """A single annotation on a piece of code.""" + line_start: int + line_end: int + text: str + + class Question(BaseModel): + """A single multiple-choice question.""" id: str question: str - options: List[str] + options: list[str] correct_option: int + explanation: Optional[str] = None + + +class QuizRequest(BaseModel): + code: str + annotations: list[Annotation] = [] + topic: str + count: int = 5 + class QuizGenerateResponse(BaseModel): title: str - questions: List[Question] + questions: list[Question] + @router.post("/generate", response_model=QuizGenerateResponse) -async def generate_quiz(topic: str, count: int = 5): - logger.info(f"Generating quiz for topic: {topic} with {count} questions") - # Mock quiz generation - questions = [ - Question( - id=f"q-{i}", - question=f"Sample question {i} about {topic}", - options=["Option A", "Option B", "Option C", "Option D"], - correct_option=0 - ) for i in range(1, count + 1) - ] - return QuizGenerateResponse(title=f"{topic} Quiz", questions=questions) +async def generate_quiz(req: QuizRequest) -> QuizGenerateResponse: + """ + Generate a multiple-choice quiz from code and optional annotations using Claude. + + Accepts the submitted code and any user annotations, then uses Claude + to generate comprehension questions that test deep understanding. + """ + settings = get_settings() + + if not settings.has_llm_key: + raise HTTPException( + status_code=503, + detail="AI Service unavailable: OPENROUTER_API_KEY not configured.", + ) + + try: + annotations_text = "\n".join( + f"Lines {a.line_start}-{a.line_end}: {a.text}" for a in req.annotations + ) if req.annotations else "No annotations provided." + + prompt = render_prompt( + "quiz_generation", + code=req.code, + annotations=annotations_text, + count=str(req.count), + topic=req.topic, + ) + + logger.info( + "Generating quiz", + topic=req.topic, + question_count=req.count, + code_length=len(req.code), + annotation_count=len(req.annotations), + ) + + text = await llm.generate_async(prompt=prompt) + + # Parse Claude's JSON response + quiz_data = _parse_quiz_response(text, req.topic, req.count) + + return QuizGenerateResponse( + title=quiz_data["title"], + questions=quiz_data["questions"], + ) + + except LLMClientError as exc: + logger.error(f"Quiz generation failed: {exc}") + raise HTTPException( + status_code=502, + detail=f"Quiz generation failed: {exc}", + ) from exc + except (json.JSONDecodeError, KeyError, ValueError) as exc: + logger.error(f"Failed to parse quiz response from Claude: {exc}") + raise HTTPException( + status_code=502, + detail="Quiz generation returned an invalid response format.", + ) from exc + + +def _parse_quiz_response(text: str, topic: str, expected_count: int) -> dict: + """ + Parse Claude's JSON response into a validated quiz structure. + + Handles cases where Claude wraps JSON in markdown code blocks or + includes explanatory text before/after the JSON. + """ + # Strip markdown code fences if present + text = strip_markdown_fence(text) + + data = json.loads(text) + + title = data.get("title", f"Comprehension Check: {topic}") + questions_raw = data.get("questions", []) + + if not questions_raw: + raise ValueError("No questions returned from Claude") + + questions = [] + for i, q in enumerate(questions_raw): + options = q.get("options", []) + if len(options) != 4: + raise ValueError(f"Question {i} has {len(options)} options, expected 4") + + questions.append(Question( + id=q.get("id", f"q-{i + 1}"), + question=q.get("question", ""), + options=options, + correct_option=q.get("correct_option", 0), + explanation=q.get("explanation"), + )) + + return {"title": title, "questions": questions} diff --git a/apps/ai-service/app/services/__init__.py b/apps/ai-service/app/services/__init__.py new file mode 100644 index 0000000..a296e08 --- /dev/null +++ b/apps/ai-service/app/services/__init__.py @@ -0,0 +1 @@ +# AI Service business logic modules diff --git a/apps/ai-service/app/services/ast_differ.py b/apps/ai-service/app/services/ast_differ.py new file mode 100644 index 0000000..4cfb213 --- /dev/null +++ b/apps/ai-service/app/services/ast_differ.py @@ -0,0 +1,504 @@ +""" +AST-based code diff engine for scoring rebuild submissions. + +Compares a user's rebuilt code against the original AI-generated solution +across four dimensions: + - Structural similarity (40%) — control flow, signatures, data structures + - Correctness (30%) — does the code handle edge cases correctly + - Readability (15%) — naming, comments, organization + - Simplicity (15%) — no unnecessary complexity + +The engine uses Python's built-in `ast` module for structural comparison +and text-based heuristics for the other dimensions. It does NOT call any +external API — it runs entirely offline. +""" + +import ast +import difflib +import re +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class DimensionScore: + """Score for a single evaluation dimension.""" + dimension: str + score: float # 0.0 to 1.0 + explanation: str + + +@dataclass +class DiffResult: + """Complete diff scoring result.""" + overall_score: float + dimensions: list[DimensionScore] = field(default_factory=list) + summary: str = "" + clean_diff: str = "" + + +class AstDiffer: + """ + Compares two code snippets structurally and qualitatively. + + Usage: + differ = AstDiffer() + result = differ.compare(original_code, user_code) + """ + + # Node type categories for structural comparison + CONTROL_FLOW_NODES = { + ast.If, ast.While, ast.For, ast.Try, ast.With, + ast.AsyncFor, ast.AsyncWith, + ast.Match, # Python 3.10+ + } + FUNCTION_DEF_NODES = {ast.FunctionDef, ast.AsyncFunctionDef} + CLASS_DEF_NODES = {ast.ClassDef} + LOOP_NODES = {ast.For, ast.While, ast.AsyncFor} + RETURN_NODES = {ast.Return, ast.Yield, ast.Raise} + + def compare(self, original: str, updated: str, language: str = "python") -> DiffResult: + """ + Compare two code strings and return a scored diff result. + + Args: + original: The original (AI-generated) solution code. + updated: The user's rebuilt solution code. + language: Programming language (only "python" supported for AST analysis). + + Returns: + A DiffResult with overall score, dimension scores, summary, and clean diff. + """ + # Fast path: identical strings = perfect score + if original == updated: + dims = [ + DimensionScore("Structural similarity", 1.0, "Code is byte-identical to the original."), + DimensionScore("Correctness", 1.0, "Code is identical — same logic and edge cases."), + DimensionScore("Readability", 1.0, "Code is identical — same readability characteristics."), + DimensionScore("Simplicity", 1.0, "Code is identical — same complexity characteristics."), + ] + return DiffResult( + overall_score=1.0, + dimensions=dims, + summary="Perfect rebuild — code is identical to the original.", + clean_diff="", + ) + + if language != "python": + return self._fallback_text_diff(original, updated) + + original_ast = self._parse_safe(original) + updated_ast = self._parse_safe(updated) + + if original_ast is None or updated_ast is None: + # Fall back to text diff if either code can't be parsed + return self._fallback_text_diff(original, updated) + + dimensions = [ + self._score_structural(original_ast, updated_ast), + self._score_correctness(original, updated, original_ast, updated_ast), + self._score_readability_comparative(original, updated), + self._score_simplicity_comparative(original, updated, original_ast, updated_ast), + ] + + weights = {"Structural similarity": 0.40, "Correctness": 0.30, "Readability": 0.15, "Simplicity": 0.15} + overall_score = sum(d.score * weights[d.dimension] for d in dimensions) + + clean_diff = self._generate_unified_diff(original, updated) + summary = self._generate_summary(overall_score, dimensions, original, updated) + + return DiffResult( + overall_score=round(overall_score, 2), + dimensions=dimensions, + summary=summary, + clean_diff=clean_diff, + ) + + # ------------------------------------------------------------------ + # Structural scoring + # ------------------------------------------------------------------ + + def _score_structural(self, original: ast.AST, updated: ast.AST) -> DimensionScore: + """ + Compare AST structure: function/class definitions, control flow, + loops, returns, and overall node distribution. + """ + orig_stats = self._count_nodes(original) + upd_stats = self._count_nodes(updated) + + if orig_stats["total"] == 0 and upd_stats["total"] == 0: + return DimensionScore("Structural similarity", 1.0, "Both solutions are empty.") + + # Compare individual categories + scores = [] + for category in ("functions", "classes", "control_flow", "loops", "returns"): + o_count = orig_stats.get(category, 0) + u_count = upd_stats.get(category, 0) + if o_count == 0 and u_count == 0: + cat_score = 1.0 + elif o_count == 0 or u_count == 0: + cat_score = 0.0 + else: + cat_score = 1.0 - abs(o_count - u_count) / max(o_count, u_count) + scores.append(cat_score) + + # Compare total node count similarity + total_ratio = min(orig_stats["total"], upd_stats["total"]) / max(orig_stats["total"], upd_stats["total"], 1) + + avg_score = (sum(scores) / len(scores)) * 0.6 + total_ratio * 0.4 + final_score = min(max(avg_score, 0.0), 1.0) + + explanation = self._structural_explanation(orig_stats, upd_stats, final_score) + return DimensionScore("Structural similarity", round(final_score, 2), explanation) + + def _count_nodes(self, tree: ast.AST) -> dict: + """Count AST node categories in a parsed tree.""" + stats: dict = { + "total": 0, "functions": 0, "classes": 0, + "control_flow": 0, "loops": 0, "returns": 0, + } + for node in ast.walk(tree): + stats["total"] += 1 + if type(node) in self.FUNCTION_DEF_NODES: + stats["functions"] += 1 + elif type(node) in self.CLASS_DEF_NODES: + stats["classes"] += 1 + if type(node) in self.CONTROL_FLOW_NODES: + stats["control_flow"] += 1 + if type(node) in self.LOOP_NODES: + stats["loops"] += 1 + if type(node) in self.RETURN_NODES: + stats["returns"] += 1 + return stats + + def _structural_explanation(self, orig: dict, upd: dict, score: float) -> str: + """Generate human explanation for structural score.""" + parts = [] + if orig["functions"] == upd["functions"]: + parts.append(f"same number of functions ({orig['functions']})") + else: + parts.append(f"functions: {orig['functions']} vs {upd['functions']}") + + if orig["classes"] == upd["classes"]: + parts.append(f"same classes ({orig['classes']})") + else: + parts.append(f"classes: {orig['classes']} vs {upd['classes']}") + + parts.append(f"control flow nodes: {orig['control_flow']} vs {upd['control_flow']}") + parts.append(f"total nodes: {orig['total']} vs {upd['total']}") + + return f"Structural score {score:.0%}. " + ", ".join(parts) + "." + + # ------------------------------------------------------------------ + # Correctness scoring + # ------------------------------------------------------------------ + + def _score_correctness( + self, original: str, updated: str, + original_ast: ast.AST, updated_ast: ast.AST, + ) -> DimensionScore: + """ + Score correctness by comparing function signatures, return statements, + and key identifiers (function names, variable names) between versions. + + Uses fuzzy matching on function names and parameter counts so that + renamed functions (e.g. 'find_max' → 'find_maximum') don't penalize. + """ + # Compare function names with fuzzy matching + orig_funcs = { + node.name: node for node in ast.walk(original_ast) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + upd_funcs = { + node.name: node for node in ast.walk(updated_ast) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + if not orig_funcs and not upd_funcs: + func_score = 1.0 + details = ["No functions to compare."] + elif not orig_funcs or not upd_funcs: + func_score = 0.3 # Some functions exist in one but not the other + details = ["Function count mismatch."] + else: + # Use fuzzy matching on function names with proper bipartite matching + # Each updated function can match at most one original function + matched_pairs = [] + matched_orig_names = set() + available_upd = dict(upd_funcs) + + for o_name, o_node in orig_funcs.items(): + best_score = 0.0 + best_u_name = None + for u_name, u_node in available_upd.items(): + name_sim = difflib.SequenceMatcher(None, o_name, u_name).ratio() + # Compare parameter counts + o_params = len(o_node.args.args) if hasattr(o_node.args, 'args') else 0 + u_params = len(u_node.args.args) if hasattr(u_node.args, 'args') else 0 + param_sim = 1.0 if o_params == u_params else (0.5 if abs(o_params - u_params) <= 1 else 0.2) + combined = name_sim * 0.7 + param_sim * 0.3 + if combined > best_score: + best_score = combined + best_u_name = u_name + + if best_u_name: + matched_pairs.append(best_score) + matched_orig_names.add(o_name) + del available_upd[best_u_name] # Remove to prevent duplicate matching + # else: no match found for this original function — contributes 0 + + func_score = sum(matched_pairs) / max(len(orig_funcs), 1) + + matched_upd_names = set(upd_funcs.keys()) - set(available_upd.keys()) + only_orig_names = set(orig_funcs.keys()) - matched_orig_names + only_upd_names = set(upd_funcs.keys()) - matched_upd_names + details = [] + if matched_upd_names: + details.append(f"matched {len(matched_upd_names)} function(s)") + if only_orig_names: + details.append(f"original-only: {', '.join(sorted(only_orig_names))}") + if only_upd_names: + details.append(f"rebuild-only: {', '.join(sorted(only_upd_names))}") + + # Check for common correctness patterns + has_except_orig = "except" in original + has_except_upd = "except" in updated + has_none_check_orig = "is None" in original or "is not None" in original + has_none_check_upd = "is None" in updated or "is not None" in updated + has_type_check_orig = "isinstance" in original or "type(" in original + has_type_check_upd = "isinstance" in updated or "type(" in updated + + # Check if rebuild handles same conditionals (if statements) as original + orig_ifs = len([n for n in ast.walk(original_ast) if isinstance(n, ast.If)]) + upd_ifs = len([n for n in ast.walk(updated_ast) if isinstance(n, ast.If)]) + has_if_orig = orig_ifs > 0 + has_if_upd = upd_ifs > 0 + + edge_case_matches = 0 + edge_case_total = 0 + + if has_except_orig or has_except_upd: + edge_case_total += 1 + if has_except_orig == has_except_upd: + edge_case_matches += 1 + + if has_none_check_orig or has_none_check_upd: + edge_case_total += 1 + if has_none_check_orig == has_none_check_upd: + edge_case_matches += 1 + + if has_type_check_orig or has_type_check_upd: + edge_case_total += 1 + if has_type_check_orig == has_type_check_upd: + edge_case_matches += 1 + + # If original has conditionals but rebuild doesn't, penalize + if has_if_orig and not has_if_upd: + edge_case_total += 1 + # Not a match — rebuild skipped a conditional branch + + edge_score = edge_case_matches / max(edge_case_total, 1) + + # Check for completeness: rebuild with significantly fewer AST nodes + # than the original suggests incomplete implementation + orig_nodes = sum(1 for _ in ast.walk(original_ast)) + upd_nodes = sum(1 for _ in ast.walk(updated_ast)) + node_ratio = upd_nodes / max(orig_nodes, 1) + # Penalize if rebuild has less than 40% of original's node count + completeness_penalty = 0.0 + if node_ratio < 0.4: + completeness_penalty = 0.3 * (1 - node_ratio) + + # Final score: 55% function interface, 35% edge case handling, 10% completeness + final_score = func_score * 0.55 + edge_score * 0.35 + min(node_ratio, 1.0) * 0.10 + final_score = max(0.0, final_score - completeness_penalty) + + detail_str = "; ".join(details) if details else "Same interface detected." + return DimensionScore( + "Correctness", + round(final_score, 2), + f"Correctness score {final_score:.0%}. {detail_str}", + ) + + # ------------------------------------------------------------------ + # Readability scoring (comparative) + # ------------------------------------------------------------------ + + def _score_readability_comparative(self, original: str, updated: str) -> DimensionScore: + """ + Compare readability metrics between original and rebuilt code. + + Measures how similar the two versions are in terms of identifier + quality, commenting patterns, and indentation consistency. + """ + orig_metrics = self._readability_metrics(original) + upd_metrics = self._readability_metrics(updated) + + # Compare average identifier length similarity + id_sim = 1.0 - abs(orig_metrics["avg_id_len"] - upd_metrics["avg_id_len"]) / max(orig_metrics["avg_id_len"], upd_metrics["avg_id_len"], 1) + id_sim = max(0.0, min(1.0, id_sim)) + + # Compare comment ratio similarity + cr_sim = 1.0 - abs(orig_metrics["comment_ratio"] - upd_metrics["comment_ratio"]) / max(orig_metrics["comment_ratio"], upd_metrics["comment_ratio"], 0.01) + cr_sim = max(0.0, min(1.0, cr_sim)) + + # Compare indentation consistency + indent_sim = 1.0 if orig_metrics["indent_count"] == upd_metrics["indent_count"] else 0.5 + + final_score = id_sim * 0.4 + cr_sim * 0.4 + indent_sim * 0.2 + + details = [ + f"avg identifier length: {orig_metrics['avg_id_len']:.1f} → {upd_metrics['avg_id_len']:.1f}", + ] + if orig_metrics["comment_lines"] > 0 or upd_metrics["comment_lines"] > 0: + details.append(f"comment ratio: {orig_metrics['comment_ratio']:.0%} → {upd_metrics['comment_ratio']:.0%}") + + return DimensionScore("Readability", round(final_score, 2), "; ".join(details)) + + @staticmethod + def _readability_metrics(code: str) -> dict: + """Extract readability metrics from a code string.""" + lines = code.split("\n") + non_empty = [l for l in lines if l.strip()] + + identifiers = re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]{2,}\b', code) + avg_id_len = sum(len(i) for i in identifiers) / max(len(identifiers), 1) if identifiers else 0 + + comment_lines = len([l for l in lines if l.strip().startswith("#")]) + comment_ratio = comment_lines / max(len(non_empty), 1) + + indent_patterns = set() + for l in non_empty: + stripped = l.lstrip() + if stripped and l != stripped: + indent = l[: len(l) - len(stripped)] + indent_patterns.add(indent) + + return { + "avg_id_len": avg_id_len, + "comment_lines": comment_lines, + "comment_ratio": comment_ratio, + "indent_count": len(indent_patterns), + } + + # ------------------------------------------------------------------ + # Simplicity scoring (comparative) + # ------------------------------------------------------------------ + + def _score_simplicity_comparative( + self, original: str, updated: str, + original_ast: ast.AST, updated_ast: ast.AST, + ) -> DimensionScore: + """ + Compare complexity metrics between original and rebuilt code. + + Measures how similar the two versions are in terms of line length, + nesting depth, and verbosity patterns. + """ + orig_metrics = self._simplicity_metrics(original, original_ast) + upd_metrics = self._simplicity_metrics(updated, updated_ast) + + # Compare average line length similarity + ll_sim = 1.0 - abs(orig_metrics["avg_line_len"] - upd_metrics["avg_line_len"]) / max(orig_metrics["avg_line_len"], upd_metrics["avg_line_len"], 1) + ll_sim = max(0.0, min(1.0, ll_sim)) + + # Compare nesting depth similarity + nd_sim = 1.0 - abs(orig_metrics["max_depth"] - upd_metrics["max_depth"]) / max(orig_metrics["max_depth"], upd_metrics["max_depth"], 1) + nd_sim = max(0.0, min(1.0, nd_sim)) + + final_score = ll_sim * 0.5 + nd_sim * 0.5 + + details = [ + f"avg line length: {orig_metrics['avg_line_len']:.0f} → {upd_metrics['avg_line_len']:.0f}", + f"max nesting depth: {orig_metrics['max_depth']} → {upd_metrics['max_depth']}", + ] + + return DimensionScore("Simplicity", round(final_score, 2), "; ".join(details)) + + def _simplicity_metrics(self, code: str, tree: ast.AST) -> dict: + """Extract simplicity metrics from code string and AST.""" + lines = [l for l in code.split("\n") if l.strip()] + avg_line_len = sum(len(l) for l in lines) / max(len(lines), 1) + max_depth = self._max_nesting_depth(tree) + return {"avg_line_len": avg_line_len, "max_depth": max_depth} + + @staticmethod + def _max_nesting_depth(node: ast.AST, current_depth: int = 0) -> int: + """Recursively compute maximum nesting depth of an AST.""" + max_depth = current_depth + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.If, ast.While, ast.For, ast.Try, ast.With, ast.AsyncFor, ast.AsyncWith)): + child_depth = AstDiffer._max_nesting_depth(child, current_depth + 1) + else: + child_depth = AstDiffer._max_nesting_depth(child, current_depth) + max_depth = max(max_depth, child_depth) + return max_depth + + # ------------------------------------------------------------------ + # Utilities + # ------------------------------------------------------------------ + + @staticmethod + def _parse_safe(code: str) -> Optional[ast.AST]: + """Safely parse code into AST, returning None on failure.""" + try: + return ast.parse(code) + except SyntaxError: + return None + + def _generate_unified_diff(self, original: str, updated: str) -> str: + """Generate a text-based unified diff for display in the UI.""" + orig_lines = original.splitlines(keepends=True) + upd_lines = updated.splitlines(keepends=True) + diff = list(difflib.unified_diff( + orig_lines, upd_lines, + fromfile="original", tofile="rebuild", + lineterm="", + )) + return "".join(diff) + + def _generate_summary(self, overall: float, dimensions: list[DimensionScore], original: str, updated: str) -> str: + """Generate a human-readable summary of the diff results.""" + if overall >= 0.9: + verdict = "Excellent rebuild! Nearly identical in structure and quality." + elif overall >= 0.75: + verdict = "Great rebuild. Minor differences in style or approach." + elif overall >= 0.6: + verdict = "Good rebuild with some differences in approach or structure." + elif overall >= 0.4: + verdict = "Adequate rebuild. The solution works but differs significantly from the original." + else: + verdict = "Significant differences. Review the original solution and try again." + + weak_areas = [d for d in dimensions if d.score < 0.6] + if weak_areas: + areas = ", ".join(d.dimension for d in weak_areas) + verdict += f" Focus on improving: {areas}." + + return verdict + + def _fallback_text_diff(self, original: str, updated: str) -> DiffResult: + """ + Fallback for non-Python languages — use text-based similarity via + SequenceMatcher instead of AST comparison. + """ + ratio = difflib.SequenceMatcher(None, original, updated).ratio() + clean_diff = self._generate_unified_diff(original, updated) + + dims = [ + DimensionScore("Structural similarity", round(ratio, 2), "Text-based similarity (AST not available for this language)."), + DimensionScore("Correctness", 0.5, "Cannot assess correctness for non-Python code — manual review recommended."), + DimensionScore("Readability", 0.5, "Cannot assess readability for non-Python code."), + DimensionScore("Simplicity", 0.5, "Cannot assess simplicity for non-Python code."), + ] + + return DiffResult( + overall_score=round(ratio, 2), + dimensions=dims, + summary=f"Text-based similarity: {ratio:.0%}. Language is not Python — AST analysis unavailable.", + clean_diff=clean_diff, + ) + + +# Module-level singleton +differ = AstDiffer() diff --git a/apps/ai-service/app/services/llm_client.py b/apps/ai-service/app/services/llm_client.py new file mode 100644 index 0000000..feca358 --- /dev/null +++ b/apps/ai-service/app/services/llm_client.py @@ -0,0 +1,191 @@ +""" +Universal LLM client using OpenRouter's unified API. + +OpenRouter provides a single endpoint for 200+ models from OpenAI, Anthropic, +Google, Meta, Mistral, DeepSeek, and more. Switch models by changing ONE env var. + +Requirements: + OPENROUTER_API_KEY=sk-or-v1-... + LLM_MODEL=google/gemini-2.0-flash-001 (or any OpenRouter model ID) + +Uses the OpenAI SDK pointed at OpenRouter's base URL for maximum model compatibility. +""" + +import time +from typing import Optional + +from openai import OpenAI, APIError, RateLimitError, APITimeoutError, APIConnectionError, Timeout +from loguru import logger + +from app.config import get_settings + + +class LLMClientError(Exception): + """Raised when the LLM API call fails after all retries.""" + + +class LLMClient: + """ + Universal LLM client via OpenRouter with retry and logging. + + Works with ANY model OpenRouter supports — just change the LLM_MODEL env var. + """ + + def __init__(self) -> None: + settings = get_settings() + self.api_key = settings.openrouter_api_key + self.base_url = settings.openrouter_base_url + self.model = settings.llm_model + self.default_max_tokens = settings.max_tokens + self.site_url = settings.openrouter_site_url + self.app_name = settings.openrouter_app_name + self._client: Optional[OpenAI] = None + + def _ensure_client(self) -> OpenAI: + if self._client is None: + if not self.api_key: + raise LLMClientError( + "OPENROUTER_API_KEY is not set. " + "Add it to your .env file. Get one at https://openrouter.ai/keys" + ) + self._client = OpenAI( + base_url=self.base_url, + api_key=self.api_key, + timeout=Timeout(30.0, connect=10.0), + default_headers={ + "HTTP-Referer": self.site_url or "https://github.com/unvibe", + "X-Title": self.app_name, + }, + ) + return self._client + + def generate( + self, + prompt: str, + system: str = "", + max_tokens: Optional[int] = None, + retries: int = 2, + ) -> str: + """ + Send a prompt to the configured LLM via OpenRouter and return the text response. + + Args: + prompt: The user message content. + system: Optional system prompt. + max_tokens: Max tokens in response (defaults to settings). + retries: Number of retries on failure. + + Returns: + The response text from the LLM. + + Raises: + LLMClientError: If all retries are exhausted. + """ + client = self._ensure_client() + max_tokens = max_tokens or self.default_max_tokens + + messages = [] + if system: + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": prompt}) + + last_error: Optional[Exception] = None + for attempt in range(1 + retries): + try: + logger.info( + "LLM API call via OpenRouter", + model=self.model, + prompt_length=len(prompt), + system_length=len(system), + attempt=attempt + 1, + ) + + response = client.chat.completions.create( + model=self.model, + messages=messages, + max_tokens=max_tokens, + ) + + text = response.choices[0].message.content or "" + usage = response.usage + logger.info( + "LLM API success", + model=self.model, + input_tokens=usage.input_tokens if usage else "unknown", + output_tokens=usage.output_tokens if usage else "unknown", + ) + return text + + except RateLimitError as exc: + last_error = exc + wait = 2 ** (attempt + 1) + logger.warning( + f"Rate limited by {self.model}, retrying in {wait}s " + f"(attempt {attempt + 1}/{retries + 1})" + ) + time.sleep(wait) + + except (APIError, APITimeoutError, APIConnectionError) as exc: + last_error = exc + if attempt < retries: + wait = 2 ** attempt + logger.warning( + f"LLM API error: {exc}. Retrying in {wait}s " + f"(attempt {attempt + 1}/{retries + 1})" + ) + time.sleep(wait) + else: + logger.error( + f"LLM API failed after {retries + 1} attempts: {exc}" + ) + + except Exception as exc: + last_error = exc + logger.error(f"Unexpected LLM client error: {exc}") + break + + raise LLMClientError( + f"LLM API call to {self.model} failed after {retries + 1} attempts" + ) from last_error + + async def generate_async( + self, + prompt: str, + system: str = "", + max_tokens: Optional[int] = None, + retries: int = 2, + ) -> str: + """Async version — runs the sync generate in a thread pool.""" + from anyio import to_thread + + return await to_thread.run_sync( + self.generate, prompt, system, max_tokens, retries + ) + + # ------------------------------------------------------------------ + # Model management + # ------------------------------------------------------------------ + + @property + def model_name(self) -> str: + """The currently configured model identifier.""" + return self.model + + def list_available_models(self) -> list[str]: + """ + Fetch available models from OpenRouter. + + Returns a list of model IDs (e.g. 'google/gemini-2.0-flash-001'). + May be empty if the API call fails. + """ + try: + client = self._ensure_client() + models = client.models.list() + return sorted(m.id for m in models) + except Exception as exc: + logger.warning(f"Failed to fetch model list: {exc}") + return [] + + +# Module-level singleton — import and use directly +llm = LLMClient() diff --git a/apps/ai-service/app/services/prompt_manager.py b/apps/ai-service/app/services/prompt_manager.py new file mode 100644 index 0000000..c9c45b0 --- /dev/null +++ b/apps/ai-service/app/services/prompt_manager.py @@ -0,0 +1,88 @@ +"""Prompt template loader with versioning support and text utilities.""" + +import os +from functools import lru_cache +from pathlib import Path +from typing import Optional + +from loguru import logger + +# Directory where prompt templates live relative to this file +PROMPTS_DIR = Path(__file__).resolve().parent.parent / "prompts" + + +class PromptNotFoundError(FileNotFoundError): + """Raised when a prompt template file cannot be found.""" + + +@lru_cache(maxsize=32) +def load_prompt_template(name: str, version: str = "v1") -> str: + """ + Load a prompt template from the prompts directory. + + Args: + name: Template filename without extension (e.g. "code_generation"). + version: Version subdirectory (e.g. "v1", "v2"). + + Returns: + The raw text content of the template file. + + Raises: + PromptNotFoundError: If the template file does not exist. + """ + template_path = PROMPTS_DIR / version / f"{name}.txt" + if not template_path.exists(): + available = ", ".join( + str(p.relative_to(PROMPTS_DIR)) + for p in (PROMPTS_DIR / version).glob("*.txt") + ) + raise PromptNotFoundError( + f"Prompt template '{name}' not found at {template_path}. " + f"Available templates: {available}" + ) + return template_path.read_text(encoding="utf-8") + + +def render_prompt(name: str, version: str = "v1", **kwargs: object) -> str: + """ + Load a prompt template and render it with the given keyword arguments. + + Usage: + prompt = render_prompt("code_generation", problem_description="...", language="python") + + Args: + name: Template name (without .txt). + version: Prompt version directory. + **kwargs: Variables to substitute into the template via .format(). + + Returns: + Fully rendered prompt string. + """ + template = load_prompt_template(name, version) + return template.format(**kwargs) + + +def list_available_templates(version: str = "v1") -> list[str]: + """List all template names available for a given version.""" + dir_path = PROMPTS_DIR / version + if not dir_path.exists(): + return [] + return sorted(p.stem for p in dir_path.glob("*.txt")) + + +def strip_markdown_fence(text: str) -> str: + """Strip markdown code fences (```json ... ```) if present. + + Handles cases where the LLM wraps JSON or code output in markdown + code blocks with optional language identifiers. + """ + text = text.strip() + if text.startswith("```"): + first_newline = text.find("\n") + if first_newline != -1: + text = text[first_newline + 1:] + if text.endswith("```"): + text = text[:-3].strip() + elif "```" in text: + text = text[: text.rindex("```")].strip() + return text diff --git a/apps/ai-service/pytest.ini b/apps/ai-service/pytest.ini new file mode 100644 index 0000000..6dc8339 --- /dev/null +++ b/apps/ai-service/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_* +asyncio_mode = auto +markers = + asyncio: mark test as async diff --git a/apps/ai-service/requirements.txt b/apps/ai-service/requirements.txt index 4717ca5..f799c7a 100644 --- a/apps/ai-service/requirements.txt +++ b/apps/ai-service/requirements.txt @@ -1,7 +1,8 @@ fastapi>=0.110.0 uvicorn>=0.28.0 -anthropic>=0.21.0 +openai>=1.0.0 httpx>=0.27.0 pydantic>=2.6.4 python-dotenv>=1.0.1 loguru>=0.7.2 +anyio>=4.0.0 diff --git a/apps/ai-service/tests/__init__.py b/apps/ai-service/tests/__init__.py new file mode 100644 index 0000000..97b8c08 --- /dev/null +++ b/apps/ai-service/tests/__init__.py @@ -0,0 +1 @@ +# Tests for the UnVibe AI Service diff --git a/apps/ai-service/tests/conftest.py b/apps/ai-service/tests/conftest.py new file mode 100644 index 0000000..70b2dfe --- /dev/null +++ b/apps/ai-service/tests/conftest.py @@ -0,0 +1,88 @@ +"""Shared test fixtures for AI service tests.""" + +import os +import pytest + + +@pytest.fixture(autouse=True) +def mock_env(): + """Set environment variables for testing. Applied to every test.""" + old_key = os.environ.get("OPENROUTER_API_KEY") + os.environ["OPENROUTER_API_KEY"] = "sk-or-v1-placeholder-test-key-disabled" + yield + if old_key is None: + os.environ.pop("OPENROUTER_API_KEY", None) + else: + os.environ["OPENROUTER_API_KEY"] = old_key + + +# Sample code fixtures + +SAMPLE_FUNCTION_ORIGINAL = """\ +def calculate_average(numbers: list[float]) -> float: + \"\"\"Calculate the average of a list of numbers.\"\"\" + if not numbers: + raise ValueError("List cannot be empty") + total = sum(numbers) + return total / len(numbers) +""" + +SAMPLE_FUNCTION_REBUILD = """\ +def calculate_average(values): + if not values: + raise ValueError("List cannot be empty") + return sum(values) / len(values) +""" + +SAMPLE_FUNCTION_DIFFERENT = """\ +def compute_mean(arr): + return sum(arr) / max(len(arr), 1) if arr else 0 +""" + +SAMPLE_CLASS_ORIGINAL = """\ +class Stack: + def __init__(self): + self._items = [] + + def push(self, item): + self._items.append(item) + + def pop(self): + if not self._items: + raise IndexError("pop from empty stack") + return self._items.pop() + + def peek(self): + if not self._items: + raise IndexError("peek from empty stack") + return self._items[-1] + + def is_empty(self): + return len(self._items) == 0 +""" + +SAMPLE_QUIZ_CODE = """\ +def is_palindrome(s: str) -> bool: + s = s.lower().replace(" ", "") + return s == s[::-1] +""" + + +@pytest.fixture +def sample_code() -> str: + return SAMPLE_FUNCTION_ORIGINAL + + +@pytest.fixture +def sample_rebuild() -> str: + return SAMPLE_FUNCTION_REBUILD + + +@pytest.fixture +def sample_class_code() -> str: + return SAMPLE_CLASS_ORIGINAL + + +@pytest.fixture +def quiz_code() -> str: + return SAMPLE_QUIZ_CODE diff --git a/apps/ai-service/tests/test_defend.py b/apps/ai-service/tests/test_defend.py new file mode 100644 index 0000000..5190f0f --- /dev/null +++ b/apps/ai-service/tests/test_defend.py @@ -0,0 +1,117 @@ +"""Tests for the defend session endpoint.""" + +import pytest +from httpx import AsyncClient, ASGITransport +from app.main import app + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.mark.asyncio +async def test_defend_endpoint_exists(client): + """The defend endpoint should accept POST and return structured response.""" + response = await client.post("/defend/respond", json={ + "session_id": "test-session-1", + "code": "def hello(): return 'world'", + "problem_description": "Write a hello world function", + "messages": [], + }) + assert response.status_code in (200, 502, 503) + if response.status_code == 200: + data = response.json() + # Should return a next_question (first question) + assert "next_question" in data + assert data["next_question"] is not None + + +@pytest.mark.asyncio +async def test_defend_with_history(client): + """Should accept conversation history.""" + response = await client.post("/defend/respond", json={ + "session_id": "test-session-2", + "code": "def add(a, b): return a + b", + "problem_description": "Write an add function", + "messages": [ + {"role": "assistant", "content": "Why did you choose this approach?"}, + {"role": "user", "content": "I used simple addition because it's the most readable."}, + {"role": "assistant", "content": "What about edge cases?"}, + {"role": "user", "content": "I'd add type checking for non-number inputs."}, + ], + }) + assert response.status_code in (200, 502, 503) + + +@pytest.mark.asyncio +async def test_defend_missing_session(client): + """Missing session_id should return 422.""" + response = await client.post("/defend/respond", json={ + "code": "x = 1", + "messages": [], + }) + assert response.status_code == 422 + + +# --------------------------------------------------------------------------- +# Parse evaluation +# --------------------------------------------------------------------------- + + +def test_parse_evaluation(): + """Test evaluation JSON parser.""" + from app.routes.defend import _parse_evaluation + + sample = '''{ + "passed": true, + "feedback": "Good understanding of recursion.", + "score": 85 + }''' + + result = _parse_evaluation(sample) + assert result["passed"] is True + assert result["score"] == 85 + assert "recursion" in result["feedback"] + + +def test_parse_evaluation_failed(): + """Test parsing a failed evaluation.""" + from app.routes.defend import _parse_evaluation + + sample = '''{ + "passed": false, + "feedback": "You missed the key concept of time complexity.", + "score": 40 + }''' + + result = _parse_evaluation(sample) + assert result["passed"] is False + assert result["score"] == 40 + + +def test_parse_evaluation_missing_field(): + """Missing 'passed' field should raise ValueError.""" + from app.routes.defend import _parse_evaluation + + with pytest.raises(Exception): + _parse_evaluation('{"feedback": "OK"}') + + +def test_parse_evaluation_with_markdown(): + """Should handle Claude's markdown-wrapped JSON.""" + from app.routes.defend import _parse_evaluation + + sample = '''```json + { + "passed": true, + "feedback": "Nice work!", + "score": 92 + } + ```''' + + result = _parse_evaluation(sample) + assert result["passed"] is True + assert result["score"] == 92 diff --git a/apps/ai-service/tests/test_diff.py b/apps/ai-service/tests/test_diff.py new file mode 100644 index 0000000..3dade8e --- /dev/null +++ b/apps/ai-service/tests/test_diff.py @@ -0,0 +1,250 @@ +"""Tests for the AST diff engine — the most critical component.""" + +import pytest + +from app.services.ast_differ import AstDiffer, DimensionScore, DiffResult + +differ = AstDiffer() + + +# --------------------------------------------------------------------------- +# Identical code +# --------------------------------------------------------------------------- + + +def test_identical_code(): + """Identical code should score near 1.0.""" + code = """\ +def hello(name): + return f"Hello, {name}!" +""" + result = differ.compare(code, code) + assert result.overall_score > 0.90 + assert len(result.dimensions) == 4 + + +# --------------------------------------------------------------------------- +# Structural similarity (same logic, different variable names) +# --------------------------------------------------------------------------- + + +def test_same_structure_different_names(): + """Same algorithm with renamed variables should score high.""" + original = """\ +def find_max(items): + max_val = items[0] + for item in items: + if item > max_val: + max_val = item + return max_val +""" + rebuild = """\ +def find_maximum(elements): + current_max = elements[0] + for element in elements: + if element > current_max: + current_max = element + return current_max +""" + result = differ.compare(original, rebuild) + # Same structure, different names — should be high + assert result.overall_score >= 0.7, f"Expected >= 0.7, got {result.overall_score}" + + +# --------------------------------------------------------------------------- +# Different algorithm, same output +# --------------------------------------------------------------------------- + + +def test_different_algorithm(): + """Different algorithm solving same problem should score lower but not zero.""" + original = """\ +def sum_list(nums): + total = 0 + for n in nums: + total += n + return total +""" + rebuild = """\ +def sum_list(nums): + return sum(nums) +""" + result = differ.compare(original, rebuild) + # Different approach — moderate score + assert 0.4 <= result.overall_score <= 0.9, f"Unexpected score: {result.overall_score}" + + +# --------------------------------------------------------------------------- +# Wrong implementation +# --------------------------------------------------------------------------- + + +def test_completely_wrong(): + """Completely different code should score low.""" + original = """\ +def fibonacci(n): + if n <= 1: + return n + return fibonacci(n-1) + fibonacci(n-2) +""" + rebuild = """\ +def fibonacci(n): + return 42 +""" + result = differ.compare(original, rebuild) + assert result.overall_score < 0.5 + + +# --------------------------------------------------------------------------- +# Dimension breakdown +# --------------------------------------------------------------------------- + + +def test_returns_all_dimensions(): + """Diff result should contain all 4 dimension scores.""" + result = differ.compare("x = 1", "y = 2") + dimension_names = {d.dimension for d in result.dimensions} + expected = {"Structural similarity", "Correctness", "Readability", "Simplicity"} + assert dimension_names == expected, f"Missing dimensions: {expected - dimension_names}" + + +# --------------------------------------------------------------------------- +# Edge cases +# --------------------------------------------------------------------------- + + +def test_empty_code(): + """Empty code should not crash.""" + result = differ.compare("", "") + assert result.overall_score >= 0 + + +def test_syntax_error(): + """Code with syntax errors should fall back gracefully.""" + result = differ.compare("def foo(:", "def bar(): pass") + # Should not crash — should produce some score + assert result.overall_score >= 0 + assert len(result.dimensions) == 4 + + +def test_non_python_language(): + """Non-Python language should use text fallback.""" + result = differ.compare( + "function hello() { return 1; }", + "function hello() { return 2; }", + language="javascript", + ) + assert result.overall_score >= 0 + assert len(result.dimensions) == 4 + + +# --------------------------------------------------------------------------- +# No-op changes (whitespace, comments) +# --------------------------------------------------------------------------- + + +def test_whitespace_only_changes(): + """Whitespace-only changes should not reduce structural score.""" + original = """\ +def add(a, b): + return a + b +""" + rebuild = """\ + + +def add(a, b): + return a + b + + +""" + result = differ.compare(original, rebuild) + # Structural similarity should be near 1.0 since AST is identical + struct_dim = next(d for d in result.dimensions if d.dimension == "Structural similarity") + assert struct_dim.score >= 0.90, f"Structural score too low: {struct_dim.score}" + + +# --------------------------------------------------------------------------- +# Class comparison +# --------------------------------------------------------------------------- + + +def test_class_structure(): + """Classes with same interface should score higher.""" + original = """\ +class Counter: + def __init__(self): + self.count = 0 + def increment(self): + self.count += 1 + def get_count(self): + return self.count +""" + rebuild = """\ +class Counter: + def __init__(self): + self.value = 0 + def increment(self): + self.value += 1 + def get_count(self): + return self.value +""" + result = differ.compare(original, rebuild) + # Same structure (same class + 3 methods), different field name + assert result.overall_score >= 0.6, f"Class structure score too low: {result.overall_score}" + + +# --------------------------------------------------------------------------- +# Readability scoring +# --------------------------------------------------------------------------- + + +def test_readability_descriptive_vs_cryptic(): + """Descriptive identifiers should score higher on readability.""" + cryptic = """\ +def f(a, b): + c = [] + for i in a: + if i % 2 == 0: + c.append(i * b) + return c +""" + descriptive = """\ +def multiply_even_numbers(numbers: list[int], multiplier: int) -> list[int]: + result = [] + for num in numbers: + if num % 2 == 0: + result.append(num * multiplier) + return result +""" + # Both are valid structural matches against themselves + # Just verify readability score is reasonable + result = differ.compare(descriptive, descriptive) + read_dim = next(d for d in result.dimensions if d.dimension == "Readability") + assert read_dim.score > 0.5, f"Readability score too low for descriptive code: {read_dim.score}" + + +# --------------------------------------------------------------------------- +# Simplicity scoring +# --------------------------------------------------------------------------- + + +def test_simple_vs_overly_complex(): + """Simpler code should score higher on simplicity.""" + simple = """\ +def double(n): + return n * 2 +""" + complex_code = """\ +def double(n): + result = 0 + for _ in range(2): + result += n + return result +""" + result_simple = differ.compare(simple, simple) + result_complex = differ.compare(complex_code, complex_code) + + simp_dim_simple = next(d for d in result_simple.dimensions if d.dimension == "Simplicity") + simp_dim_complex = next(d for d in result_complex.dimensions if d.dimension == "Simplicity") + + assert simp_dim_simple.score >= simp_dim_complex.score, "Simpler code should have higher simplicity score" diff --git a/apps/ai-service/tests/test_generate.py b/apps/ai-service/tests/test_generate.py new file mode 100644 index 0000000..62c73ed --- /dev/null +++ b/apps/ai-service/tests/test_generate.py @@ -0,0 +1,50 @@ +"""Tests for the code generation endpoint.""" + +import pytest +from httpx import AsyncClient, ASGITransport +from app.main import app + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.mark.asyncio +async def test_generate_endpoint_exists(client): + """The generate endpoint should accept POST and return structured response.""" + response = await client.post("/generate/", json={ + "problem_description": "Write a function to add two numbers", + "language": "python", + "difficulty": "easy", + }) + # With real API key it returns 200; without it returns 503 or 502 (we can't control which) + assert response.status_code in (200, 502, 503) + if response.status_code == 200: + data = response.json() + assert "code" in data + assert "language" in data + assert "model_used" in data + + +@pytest.mark.asyncio +async def test_generate_missing_problem(client): + """Missing problem_description should return 422 validation error.""" + response = await client.post("/generate/", json={ + "language": "python", + }) + assert response.status_code == 422 + + +@pytest.mark.asyncio +async def test_generate_different_languages(client): + """Should accept different language values.""" + for lang in ["python", "javascript", "typescript", "go", "rust"]: + response = await client.post("/generate/", json={ + "problem_description": "Write a hello world function", + "language": lang, + "difficulty": "easy", + }) + assert response.status_code in (200, 502, 503) diff --git a/apps/ai-service/tests/test_quiz.py b/apps/ai-service/tests/test_quiz.py new file mode 100644 index 0000000..31f36b9 --- /dev/null +++ b/apps/ai-service/tests/test_quiz.py @@ -0,0 +1,117 @@ +"""Tests for the quiz generation endpoint.""" + +import pytest +from httpx import AsyncClient, ASGITransport +from app.main import app + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.mark.asyncio +async def test_quiz_endpoint_exists(client, quiz_code): + """The quiz endpoint should accept POST and return structured response.""" + response = await client.post("/quiz/generate", json={ + "code": quiz_code, + "annotations": [], + "topic": "Strings", + "count": 3, + }) + assert response.status_code in (200, 502, 503) + if response.status_code == 200: + data = response.json() + assert "title" in data + assert "questions" in data + assert len(data["questions"]) > 0 + # Verify question structure + q = data["questions"][0] + assert "id" in q + assert "question" in q + assert "options" in q + assert len(q["options"]) == 4 + assert "correct_option" in q + + +@pytest.mark.asyncio +async def test_quiz_with_annotations(client, quiz_code): + """Should accept and process annotations.""" + response = await client.post("/quiz/generate", json={ + "code": quiz_code, + "annotations": [ + {"line_start": 1, "line_end": 1, "text": "This converts to lowercase"}, + {"line_start": 2, "line_end": 2, "text": "This reverses the string"}, + ], + "topic": "Strings", + "count": 2, + }) + assert response.status_code in (200, 502, 503) + + +@pytest.mark.asyncio +async def test_quiz_missing_code(client): + """Missing code should return 422.""" + response = await client.post("/quiz/generate", json={ + "topic": "Strings", + }) + assert response.status_code == 422 + + +@pytest.mark.asyncio +async def test_quiz_parse_response(): + """Test the JSON parser directly with sample Claude output.""" + from app.routes.quiz import _parse_quiz_response + + sample = '''{ + "title": "Test Quiz", + "questions": [ + { + "id": "q-1", + "question": "What does line 2 do?", + "options": ["Reverses the string", "Sorts it", "Capitalizes it", "Splits it"], + "correct_option": 0, + "explanation": "s[::-1] reverses the string." + } + ] + }''' + + result = _parse_quiz_response(sample, "Test", 1) + assert result["title"] == "Test Quiz" + assert len(result["questions"]) == 1 + assert result["questions"][0].correct_option == 0 + + +@pytest.mark.asyncio +async def test_quiz_parse_with_markdown_fence(): + """Test parsing Claude output wrapped in markdown code blocks.""" + from app.routes.quiz import _parse_quiz_response + + sample = '''```json + { + "title": "Quiz", + "questions": [ + { + "id": "q-1", + "question": "What does this code do?", + "options": ["A", "B", "C", "D"], + "correct_option": 1 + } + ] + } + ```''' + + result = _parse_quiz_response(sample, "Test", 1) + assert len(result["questions"]) == 1 + assert result["questions"][0].correct_option == 1 + + +@pytest.mark.asyncio +async def test_quiz_parse_invalid_json(): + """Invalid JSON from Claude should raise ValueError.""" + from app.routes.quiz import _parse_quiz_response + + with pytest.raises(Exception): + _parse_quiz_response("not json at all", "Test", 1) diff --git a/apps/api/jest.config.ts b/apps/api/jest.config.ts new file mode 100644 index 0000000..3c7a61f --- /dev/null +++ b/apps/api/jest.config.ts @@ -0,0 +1,15 @@ +import type { Config } from 'jest'; + +const config: Config = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: ['/src'], + testMatch: ['**/__tests__/**/*.test.ts'], + clearMocks: true, + collectCoverageFrom: [ + 'src/services/**/*.ts', + '!src/__tests__/**', + ], +}; + +export default config; diff --git a/apps/api/package.json b/apps/api/package.json index 567f878..edeb809 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "dev": "tsx watch src/index.ts", + "prebuild": "prisma generate", "build": "tsc", "db:migrate": "prisma migrate dev", "db:seed": "prisma db seed", diff --git a/apps/api/src/__tests__/ai-client.test.ts b/apps/api/src/__tests__/ai-client.test.ts new file mode 100644 index 0000000..327aa4a --- /dev/null +++ b/apps/api/src/__tests__/ai-client.test.ts @@ -0,0 +1,250 @@ +/** + * Tests for the AIClient bridge. + * + * These tests mock fetch to avoid calling the real AI service. + */ + +import { AIClient, AIClientError } from '../services/ai-client'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function mockFetch(response: unknown, ok = true, status = 200): jest.SpyInstance { + return jest.spyOn(global, 'fetch').mockResolvedValue({ + ok, + status, + json: async () => response, + text: async () => JSON.stringify(response), + } as Response); +} + +function mockFetchError(error: Error): jest.SpyInstance { + return jest.spyOn(global, 'fetch').mockRejectedValue(error); +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('AIClient', () => { + let client: AIClient; + + beforeEach(() => { + client = new AIClient({ baseUrl: 'http://test-ai:8000', timeoutMs: 5000, maxRetries: 1 }); + jest.clearAllMocks(); + }); + + // ----------------------------------------------------------------------- + // generateCode + // ----------------------------------------------------------------------- + + describe('generateCode', () => { + it('should call POST /generate/ and return mapped result', async () => { + const mockResponse = { + code: 'def hello(): pass', + language: 'python', + model_used: 'claude-sonnet-4-20250514', + token_count: 42, + }; + mockFetch(mockResponse); + + const result = await client.generateCode({ + problemDescription: 'Write hello world', + language: 'python', + difficulty: 'easy', + }); + + expect(result.code).toBe('def hello(): pass'); + expect(result.modelUsed).toBe('claude-sonnet-4-20250514'); + expect(result.tokenCount).toBe(42); + }); + + it('should throw AIClientError on 500', async () => { + mockFetch({ error: 'Internal Server Error' }, false, 500); + + await expect( + client.generateCode({ problemDescription: 'test', language: 'python', difficulty: 'easy' }), + ).rejects.toThrow(AIClientError); + }); + + it('should throw AIClientError on 4xx without retry', async () => { + mockFetch({ error: 'Bad Request' }, false, 400); + + await expect( + client.generateCode({ problemDescription: 'test', language: 'python', difficulty: 'easy' }), + ).rejects.toThrow(AIClientError); + }); + }); + + // ----------------------------------------------------------------------- + // generateQuiz + // ----------------------------------------------------------------------- + + describe('generateQuiz', () => { + it('should map snake_case response to camelCase', async () => { + const mockResponse = { + title: 'Test Quiz', + questions: [ + { + id: 'q-1', + question: 'What does X do?', + options: ['A', 'B', 'C', 'D'], + correct_option: 0, + explanation: 'Because X does Y.', + }, + ], + }; + mockFetch(mockResponse); + + const result = await client.generateQuiz({ + code: 'x = 1', + annotations: [], + topic: 'Test', + count: 1, + }); + + expect(result.title).toBe('Test Quiz'); + expect(result.questions[0].correctOption).toBe(0); + expect(result.questions[0].explanation).toBe('Because X does Y.'); + }); + }); + + // ----------------------------------------------------------------------- + // diffCode + // ----------------------------------------------------------------------- + + describe('diffCode', () => { + it('should map snake_case diff response to camelCase', async () => { + const mockResponse = { + overall_score: 0.85, + dimensions: [ + { dimension: 'Structural similarity', score: 0.9, explanation: 'Good match' }, + ], + summary: 'Good rebuild', + clean_diff: '@@ -1 +1 @@\n-x\n+y', + }; + mockFetch(mockResponse); + + const result = await client.diffCode({ + originalCode: 'x = 1', + updatedCode: 'y = 1', + language: 'python', + }); + + expect(result.overallScore).toBe(0.85); + expect(result.dimensions[0].dimension).toBe('Structural similarity'); + expect(result.cleanDiff).toContain('-x'); + }); + }); + + // ----------------------------------------------------------------------- + // defend + // ----------------------------------------------------------------------- + + describe('defend', () => { + it('should return nextQuestion from ask mode', async () => { + const mockResponse = { + next_question: 'Why did you choose a list?', + passed: false, + feedback: null, + score: null, + }; + mockFetch(mockResponse); + + const result = await client.defendAsk({ + sessionId: 's1', + code: 'x = []', + problemDescription: 'Test', + messages: [], + }); + + expect(result.nextQuestion).toBe('Why did you choose a list?'); + expect(result.passed).toBe(false); + }); + + it('should return pass/fail from evaluate mode', async () => { + const mockResponse = { + next_question: null, + passed: true, + feedback: 'Great answer!', + score: 90, + }; + mockFetch(mockResponse); + + const result = await client.defendEvaluate({ + sessionId: 's1', + code: 'x = []', + problemDescription: 'Test', + messages: [{ role: 'assistant', content: 'Q?' }, { role: 'user', content: 'A!' }], + }); + + expect(result.passed).toBe(true); + expect(result.feedback).toBe('Great answer!'); + expect(result.score).toBe(90); + }); + }); + + // ----------------------------------------------------------------------- + // Retry logic + // ----------------------------------------------------------------------- + + describe('retry behavior', () => { + it('should retry on transient failure and succeed', async () => { + const mock = jest.spyOn(global, 'fetch'); + const mockResponse = { + code: 'success after retry', + language: 'python', + model_used: 'claude', + token_count: 10, + }; + + // First call fails, second succeeds + mock + .mockRejectedValueOnce(new Error('Network error')) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => mockResponse, + text: async () => JSON.stringify(mockResponse), + } as Response); + + const result = await client.generateCode({ + problemDescription: 'test', + language: 'python', + difficulty: 'easy', + }); + + expect(result.code).toBe('success after retry'); + expect(mock).toHaveBeenCalledTimes(2); + }); + + it('should throw after exhausting retries', async () => { + mockFetchError(new Error('Persistent error')); + + await expect( + client.generateCode({ problemDescription: 'test', language: 'python', difficulty: 'easy' }), + ).rejects.toThrow(AIClientError); + }); + }); + + // ----------------------------------------------------------------------- + // Health check + // ----------------------------------------------------------------------- + + describe('healthCheck', () => { + it('should return true when service responds', async () => { + mockFetch({ status: 'ok' }); + + const healthy = await client.healthCheck(); + expect(healthy).toBe(true); + }); + + it('should return false when service is down', async () => { + mockFetchError(new Error('Connection refused')); + + const healthy = await client.healthCheck(); + expect(healthy).toBe(false); + }); + }); +}); diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 99ee8ba..8edeaff 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -6,8 +6,10 @@ import { Server } from 'socket.io'; import pino from 'pino'; import * as Sentry from '@sentry/node'; import { PrismaClient } from '@prisma/client'; -import { Queue, Worker } from 'bullmq'; +import { Queue } from 'bullmq'; +import net from 'net'; import { router, publicProcedure } from './trpc'; +import { createSubmissionWorker } from './services/submission-worker'; import dotenv from 'dotenv'; dotenv.config({ path: '../../.env' }); @@ -32,28 +34,78 @@ if (process.env.SENTRY_DSN_API) { // Initialize Prisma const prisma = new PrismaClient(); -// Initialize BullMQ (Create a dummy queue for scaffolding check) +// --------------------------------------------------------------------------- +// Redis / BullMQ setup (resilient — works without Redis running) +// --------------------------------------------------------------------------- + const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379'; const connectionOpts = { host: redisUrl.split('://')[1]?.split(':')[0] || 'localhost', port: parseInt(redisUrl.split(':')[2]) || 6379, }; -const submissionQueue = new Queue('submissions', { - connection: connectionOpts, -}); +/** + * Quick TCP connectivity check — avoids BullMQ's infinite retry spam when + * Redis is not available (e.g. Docker not running). + */ +function checkRedisReachable(host: string, port: number, timeoutMs = 2000): Promise { + return new Promise((resolve) => { + const socket = new net.Socket(); + socket.setTimeout(timeoutMs); + socket.on('connect', () => { + socket.destroy(); + resolve(true); + }); + socket.on('error', () => { + socket.destroy(); + resolve(false); + }); + socket.on('timeout', () => { + socket.destroy(); + resolve(false); + }); + socket.connect(port, host); + }); +} -const submissionWorker = new Worker( - 'submissions', - async (job) => { - logger.info({ jobId: job.id }, 'Processing submission job'); - return { processed: true }; - }, - { connection: connectionOpts } -); +let submissionQueue: Queue | null = null; +// declared here for scope; assigned inside initRedisDeps +let submissionWorker: ReturnType | null = null; + +async function initRedisDeps(): Promise { + const available = await checkRedisReachable(connectionOpts.host, connectionOpts.port); + + if (!available) { + logger.warn( + 'Redis unavailable — job queue and submission worker disabled. ' + + 'Start Docker with: docker compose -f infra/docker-compose.yml up -d', + ); + return; + } + + try { + submissionQueue = new Queue('submissions', { + connection: connectionOpts, + }); + await submissionQueue.waitUntilReady(); + + submissionWorker = createSubmissionWorker(prisma, connectionOpts); + + logger.info('Redis connected — job queue and submission worker enabled'); + } catch (err) { + logger.warn( + { err }, + 'Failed to initialize BullMQ — job queue and submission worker disabled. ' + + 'Start Docker with: docker compose -f infra/docker-compose.yml up -d', + ); + submissionQueue = null; + submissionWorker = null; + } +} -submissionWorker.on('error', (err) => { - logger.error(err, 'Submission worker error'); +// Fire-and-forget: server starts immediately even if Redis init is pending +initRedisDeps().catch((err) => { + logger.error({ err }, 'Unexpected error during Redis initialization'); }); // tRPC router diff --git a/apps/api/src/services/ai-client.ts b/apps/api/src/services/ai-client.ts new file mode 100644 index 0000000..e7a9313 --- /dev/null +++ b/apps/api/src/services/ai-client.ts @@ -0,0 +1,290 @@ +/** + * HTTP client for the UnVibe AI Service (Python FastAPI). + * + * Provides typed methods for all AI endpoints: code generation, quiz + * generation, code diff scoring, and defend session Q&A. + * + * Includes retry logic, timeouts, and structured logging via pino. + */ + +import pino from 'pino'; + +const logger = pino({ name: 'ai-client' }); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface GenerateCodeParams { + problemDescription: string; + language: string; + difficulty: string; +} + +export interface GenerateCodeResult { + code: string; + language: string; + modelUsed: string; + tokenCount: number; +} + +export interface QuizParams { + code: string; + annotations: Array<{ lineStart: number; lineEnd: number; text: string }>; + topic: string; + count: number; +} + +export interface Question { + id: string; + question: string; + options: string[]; + correctOption: number; + explanation?: string; +} + +export interface QuizResult { + title: string; + questions: Question[]; +} + +export interface DiffParams { + originalCode: string; + updatedCode: string; + language: string; +} + +export interface DimensionScore { + dimension: string; + score: number; + explanation: string; +} + +export interface DiffResult { + overallScore: number; + dimensions: DimensionScore[]; + summary: string; + cleanDiff: string; +} + +export interface DefendMessage { + role: 'user' | 'assistant'; + content: string; +} + +export interface DefendParams { + sessionId: string; + code: string; + problemDescription: string; + messages: DefendMessage[]; +} + +export interface DefendResult { + nextQuestion: string | null; + passed: boolean; + feedback: string | null; + score: number | null; +} + +// --------------------------------------------------------------------------- +// Error types +// --------------------------------------------------------------------------- + +export class AIClientError extends Error { + constructor( + message: string, + public statusCode?: number, + public endpoint?: string, + ) { + super(message); + this.name = 'AIClientError'; + } +} + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +export class AIClient { + private readonly baseUrl: string; + private readonly timeoutMs: number; + private readonly maxRetries: number; + + constructor(options?: { baseUrl?: string; timeoutMs?: number; maxRetries?: number }) { + this.baseUrl = options?.baseUrl ?? process.env.AI_SERVICE_URL ?? 'http://localhost:8000'; + this.timeoutMs = options?.timeoutMs ?? 10_000; + this.maxRetries = options?.maxRetries ?? 2; + } + + // ----------------------------------------------------------------------- + // Public API methods + // ----------------------------------------------------------------------- + + async generateCode(params: GenerateCodeParams): Promise { + const body = { + problem_description: params.problemDescription, + language: params.language, + difficulty: params.difficulty, + }; + const data = await this.request<{ code: string; language: string; model_used: string; token_count: number }>( + 'POST', + '/generate/', + body, + ); + return { + code: data.code, + language: data.language, + modelUsed: data.model_used, + tokenCount: data.token_count, + }; + } + + async generateQuiz(params: QuizParams): Promise { + const body = { + code: params.code, + annotations: params.annotations.map((a) => ({ + line_start: a.lineStart, + line_end: a.lineEnd, + text: a.text, + })), + topic: params.topic, + count: params.count, + }; + const data = await this.request<{ title: string; questions: any[] }>('POST', '/quiz/generate', body); + return { + title: data.title, + questions: data.questions.map((q: any) => ({ + id: q.id, + question: q.question, + options: q.options, + correctOption: q.correct_option, + explanation: q.explanation, + })), + }; + } + + async diffCode(params: DiffParams): Promise { + const body = { + original_code: params.originalCode, + updated_code: params.updatedCode, + language: params.language, + }; + const data = await this.request<{ + overall_score: number; + dimensions: Array<{ dimension: string; score: number; explanation: string }>; + summary: string; + clean_diff: string; + }>('POST', '/diff/', body); + return { + overallScore: data.overall_score, + dimensions: data.dimensions, + summary: data.summary, + cleanDiff: data.clean_diff, + }; + } + + async defendAsk(params: DefendParams): Promise { + const body = this.buildDefendBody(params); + const data = await this.request<{ + next_question: string | null; + passed: boolean; + feedback: string | null; + score: number | null; + }>('POST', '/defend/respond', body); + return { + nextQuestion: data.next_question, + passed: data.passed, + feedback: data.feedback, + score: data.score, + }; + } + + async defendEvaluate(params: DefendParams): Promise { + // Same endpoint — the service determines mode based on conversation length + return this.defendAsk(params); + } + + // ----------------------------------------------------------------------- + // Health check + // ----------------------------------------------------------------------- + + async healthCheck(): Promise { + try { + const res = await fetch(`${this.baseUrl}/health`, { + method: 'GET', + signal: AbortSignal.timeout(5_000), + }); + return res.ok; + } catch { + return false; + } + } + + // ----------------------------------------------------------------------- + // Private helpers + // ----------------------------------------------------------------------- + + private buildDefendBody(params: DefendParams): Record { + return { + session_id: params.sessionId, + code: params.code, + problem_description: params.problemDescription, + messages: params.messages.map((m) => ({ + role: m.role, + content: m.content, + })), + }; + } + + private async request(method: string, path: string, body?: unknown): Promise { + const url = `${this.baseUrl}${path}`; + let lastError: Error | null = null; + + for (let attempt = 0; attempt <= this.maxRetries; attempt++) { + try { + const response = await fetch(url, { + method, + headers: { 'Content-Type': 'application/json' }, + body: body ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(this.timeoutMs), + }); + + if (!response.ok) { + const errorBody = await response.text().catch(() => ''); + throw new AIClientError( + `AI service returned ${response.status}: ${errorBody || response.statusText}`, + response.status, + path, + ); + } + + const data = (await response.json()) as T; + logger.info({ endpoint: path, attempt: attempt + 1 }, 'AI service call succeeded'); + return data; + } catch (err) { + lastError = err instanceof Error ? err : new Error(String(err)); + + if (lastError instanceof AIClientError && lastError.statusCode && lastError.statusCode < 500) { + // Client errors (4xx) should not be retried + logger.warn({ endpoint: path, status: lastError.statusCode }, 'Non-retryable AI client error'); + throw lastError; + } + + if (attempt < this.maxRetries) { + const wait = 2 ** attempt * 500; + logger.warn({ endpoint: path, attempt: attempt + 1, wait }, 'Retrying AI service call'); + await new Promise((resolve) => setTimeout(resolve, wait)); + } + } + } + + throw new AIClientError( + `AI service call failed after ${this.maxRetries + 1} attempts: ${lastError?.message}`, + undefined, + path, + ); + } +} + +// Singleton instance +export const aiClient = new AIClient(); diff --git a/apps/api/src/services/submission-worker.ts b/apps/api/src/services/submission-worker.ts new file mode 100644 index 0000000..f2a4016 --- /dev/null +++ b/apps/api/src/services/submission-worker.ts @@ -0,0 +1,208 @@ +/** + * BullMQ worker that processes code submissions asynchronously. + * + * Flow: + * 1. Receives a job with submissionId, userId, moduleId, code, originalCode + * 2. Calls the AI service diff endpoint to score the rebuild + * 3. Stores the score in the Submission record via Prisma + * 4. Triggers IRS recalculation + * 5. Schedules a Defend session + */ + +import { Job, Worker, Queue, ConnectionOptions } from 'bullmq'; +import { PrismaClient } from '@prisma/client'; +import pino from 'pino'; +import { aiClient, AIClientError } from './ai-client'; + +const logger = pino({ name: 'submission-worker' }); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface SubmissionJobData { + submissionId: string; + userId: string; + moduleId: string; + code: string; + originalCode: string; + language?: string; +} + +export interface SubmissionJobResult { + overallScore: number; + feedback?: string; + defendScheduled: boolean; +} + +// --------------------------------------------------------------------------- +// Worker factory +// --------------------------------------------------------------------------- + +export function createSubmissionWorker( + prisma: PrismaClient, + connection: ConnectionOptions, +): Worker { + const worker = new Worker( + 'submissions', + async (job: Job) => { + logger.info({ jobId: job.id, submissionId: job.data.submissionId }, 'Processing submission'); + + const { submissionId, code, originalCode, moduleId, userId, language } = job.data; + + try { + // 1. Call AI service for diff scoring + const diffResult = await aiClient.diffCode({ + originalCode, + updatedCode: code, + language: language ?? 'python', + }); + + logger.info( + { jobId: job.id, overallScore: diffResult.overallScore }, + 'Diff scoring complete', + ); + + // 2. Update Submission record with score and feedback + await prisma.submission.update({ + where: { id: submissionId }, + data: { + status: 'scored', + feedback: JSON.stringify({ + overallScore: diffResult.overallScore, + dimensions: diffResult.dimensions, + summary: diffResult.summary, + }), + }, + }); + + logger.info({ jobId: job.id, submissionId }, 'Submission score saved'); + + // 3. Trigger IRS recalculation via the IRS engine + // (called asynchronously — the IRS service handles the actual calc) + await triggerIRSRecalculation(prisma, userId); + + // 4. Schedule a Defend session + const defendScheduled = await scheduleDefendSession(prisma, submissionId, userId, moduleId); + + return { + overallScore: diffResult.overallScore, + feedback: diffResult.summary, + defendScheduled, + }; + } catch (err) { + logger.error({ jobId: job.id, err }, 'Submission processing failed'); + + // Mark submission as failed + await prisma.submission.update({ + where: { id: submissionId }, + data: { status: 'failed' }, + }).catch((e: unknown) => logger.error({ err: e }, 'Failed to update submission status')); + + if (err instanceof AIClientError) { + // Re-throw so BullMQ can retry according to its configured retry policy + throw err; + } + throw err; + } + }, + { + connection, + concurrency: 5, // Process up to 5 submissions in parallel + }, + ); + + worker.on('completed', (job: Job) => { + logger.info({ jobId: job.id, result: job.returnvalue }, 'Submission job completed'); + }); + + worker.on('failed', (job: Job | undefined, err: Error) => { + logger.error({ jobId: job?.id, err: err.message }, 'Submission job failed'); + }); + + return worker; +} + +// --------------------------------------------------------------------------- +// IRS recalculation +// --------------------------------------------------------------------------- + +async function triggerIRSRecalculation(prisma: PrismaClient, userId: string): Promise { + // Calculate aggregate score from all scored submissions + const submissions = await prisma.submission.findMany({ + where: { userId, status: 'scored' }, + select: { feedback: true }, + }); + + let totalScore = 0; + let scoredCount = 0; + + for (const sub of submissions) { + if (sub.feedback) { + try { + const parsed = JSON.parse(sub.feedback); + if (typeof parsed.overallScore === 'number') { + totalScore += parsed.overallScore; + scoredCount++; + } + } catch { + // Skip unparseable feedback + } + } + } + + const averageScore = scoredCount > 0 ? Math.round((totalScore / scoredCount) * 100) : 0; + + // Create or update the latest IRS score + await prisma.iRSScore.create({ + data: { + userId, + score: averageScore, + details: { + submissionsScored: scoredCount, + lastCalculated: new Date().toISOString(), + }, + }, + }); + + logger.info({ userId, averageScore, scoredCount }, 'IRS score recalculated'); +} + +// --------------------------------------------------------------------------- +// Defend session scheduling +// --------------------------------------------------------------------------- + +async function scheduleDefendSession( + prisma: PrismaClient, + submissionId: string, + userId: string, + moduleId: string, +): Promise { + try { + // Check if a defend session already exists for this (user, module) pair + const existing = await prisma.defendSession.findFirst({ + where: { userId, moduleId, status: { notIn: ['completed', 'expired'] } }, + }); + + if (existing) { + logger.info({ userId, moduleId }, 'Active defend session already exists — skipping'); + return false; + } + + // Create a new defend session record + await prisma.defendSession.create({ + data: { + userId, + moduleId, + status: 'pending', + conversation: [], + }, + }); + + logger.info({ userId, moduleId, submissionId }, 'Defend session scheduled'); + return true; + } catch (err) { + logger.error({ err, userId, moduleId }, 'Failed to schedule defend session'); + return false; + } +} diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json index bbd5970..12f4a4f 100644 --- a/apps/api/tsconfig.json +++ b/apps/api/tsconfig.json @@ -6,5 +6,6 @@ "outDir": "./dist", "noEmit": false }, - "include": ["src/**/*"] + "include": ["src/**/*"], + "exclude": ["src/__tests__"] }