From 1c6f362b0f8381ed4fbf52589b98d87ccc7dae94 Mon Sep 17 00:00:00 2001 From: ananya092302 Date: Sun, 2 Aug 2026 16:46:40 +0530 Subject: [PATCH] refactor to V2 weighted, scenario-aware evaluation engine --- README.md | 2 +- changeLog.md | 17 ++ client/package-lock.json | 1 + server/package-lock.json | 16 ++ server/src/data/db.json | 131 ++++++++++++++- server/src/routes/sessions.js | 6 +- server/src/services/learningEngine.js | 233 +++++++++++++++++++++----- 7 files changed, 356 insertions(+), 50 deletions(-) create mode 100644 changeLog.md diff --git a/README.md b/README.md index 5bc349d..830c0a0 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,6 @@ npm run dev ## Notes -The AI behavior in this prototype is deterministic and local. The abstraction mapper, prompt evaluator, and Python construct generator use rule-based logic so you can run everything without external AI keys. Later phases can replace those services with OpenAI, RAG, or TinyLLM components. +The AI behavior in this prototype is deterministic and local. The abstraction mapper, prompt evaluator, and Python construct generator use a robust V2 rule-based logic engine with weighted keyword matching, scenario-aware evaluation, and dynamic feedback generation, allowing you to run everything locally without external AI keys. Later phases can replace those services with OpenAI, RAG, or TinyLLM components. Learning data is stored in `server/src/data/db.json`. This keeps the prototype simple and fully local, without MongoDB, Docker, Atlas, or any external database. diff --git a/changeLog.md b/changeLog.md new file mode 100644 index 0000000..074cc9d --- /dev/null +++ b/changeLog.md @@ -0,0 +1,17 @@ +# Changelog + +## [1.1.0] - Smarter Learning Engine & Keyword Collision Fixes + +This release focuses on significantly upgrading the core evaluation logic in `learningEngine.js` to provide a more dynamic, accurate, and scenario-aware learning experience. + +### Enhanced +- **Two-Tier Weighted Keyword System**: Replaced flat substring matching with a primary/secondary weighted system. The engine now distinguishes between strong technical indicators (e.g., `variable`, `loop`) and conversational phrasing (e.g., `store`, `value`), requiring higher thresholds for the latter to prevent false positives. +- **Regex Pre-compilation**: Replaced `.includes()` with pre-compiled regular expressions using word boundaries (`\b`), significantly improving performance and matching accuracy. +- **Scenario-Aware Mapping**: The evaluation engine now cross-references detected keywords against the specific `scenario.concepts`. +- **Dynamic Over-engineering Feedback**: If a learner suggests an overly complex concept (e.g., a loop) for a scenario that only requires a simpler one (e.g., a variable), the engine dynamically guides them back to the expected concept. +- **Composable Code Generation**: Replaced static code templates with a dynamic AST-like string builder. It now composes variable assignments, loops, conditionals, and functions dynamically based on the exact combination of detected concepts, while injecting scenario context directly into the code. +- **Granular Prompt Evaluation**: `evaluatePrompt` calculates continuous scores based on length and intelligently verifies whether the learner mentioned the specific Python concepts required for the active scenario. +- **Context-Specific Misconceptions**: Introduced a `misconceptionsDictionary` that flags concept-specific anti-patterns (e.g., warning against manual repetition only when a loop is expected). + +### Fixed +- **Keyword Collision Bug**: Removed broad conversational words (`store`, `value`) from lists and dictionaries patterns, reassigning them to a new dedicated `Data storage` rule for variables. This prevents the engine from incorrectly triggering lists and dictionaries on normal conversational sentences. diff --git a/client/package-lock.json b/client/package-lock.json index 15e5231..b7f5809 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -17,6 +17,7 @@ } }, "..": { + "name": "pybe-mern-app", "version": "1.0.0", "dependencies": { "concurrently": "^9.1.2" diff --git a/server/package-lock.json b/server/package-lock.json index 3fcc765..0f5b259 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -19,6 +19,7 @@ } }, "..": { + "name": "pybe-mern-app", "version": "1.0.0", "dependencies": { "concurrently": "^9.1.2" @@ -402,6 +403,21 @@ "node": ">= 0.6" } }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "license": "MIT", diff --git a/server/src/data/db.json b/server/src/data/db.json index 6547d2e..e5ef733 100644 --- a/server/src/data/db.json +++ b/server/src/data/db.json @@ -596,5 +596,134 @@ "updatedAt": "2026-06-19T11:54:44.733Z" } ], - "sessions": [] + "sessions": [ + { + "_id": "b3e4ae1c-13c0-441b-bb9f-555948716f6f", + "learnerName": "Guest learner", + "scenario": "8cf644f0-1402-4763-9542-02bd2ca0d1dc", + "reasoning": "a float variable .", + "promptText": "float x = 5.18", + "abstractionMap": [ + { + "pattern": "Sequential thinking", + "pythonConcept": "statements and variables", + "explanation": "You described a step-by-step solution. Python starts by representing those steps as statements." + } + ], + "generatedCode": "scenario = \"Bag Weight Label\"\nreasoning = \"Break the situation into clear steps\"\nprint(scenario)\nprint(reasoning)", + "codeExplanation": "The code starts from your natural reasoning and turns it into Python structure: Sequential thinking becomes statements and variables.", + "promptScore": 35, + "promptFeedback": [ + "Add more context about the situation and expected output.", + "Ask the AI to explain its reasoning, not just produce code.", + "Include an example input or output to make the prompt testable.", + "Name the Python concept you think may apply." + ], + "reflection": "i'm good in c and python", + "misconceptions": [ + "Reasoning is brief. Try naming the inputs, decision rule, and expected result." + ], + "masterySignals": [ + "Recognized sequential thinking" + ], + "createdAt": "2026-07-31T14:15:29.260Z", + "updatedAt": "2026-07-31T14:15:29.260Z" + }, + { + "_id": "c624af90-3a46-4baa-ba8b-0b99fc53fd26", + "learnerName": "Guest learner", + "scenario": "8cf644f0-1402-4763-9542-02bd2ca0d1dc", + "reasoning": "i will need to store a decimal or float type of value for weights in a variable .", + "promptText": "i will make a variable named x and assign it a value of school bag weight. eg: float x = 10.05 kg", + "abstractionMap": [ + { + "pattern": "Collection handling", + "pythonConcept": "lists", + "explanation": "You grouped multiple values, so Python lists help store and process them. (Note: This might be over-engineering for this specific scenario)", + "confidence": "60% confidence" + }, + { + "pattern": "Data mapping", + "pythonConcept": "dictionaries", + "explanation": "You are associating pieces of data together, which maps perfectly to a Python dictionary. (Note: This might be over-engineering for this specific scenario)", + "confidence": "60% confidence" + } + ], + "generatedCode": "# Scenario: Bag Weight Label\n\nitems = [target_value, \"other_data\", \"more_data\"]\ndata_map = {\"key1\": target_value, \"key2\": \"other_data\"}\n", + "codeExplanation": "The code starts from your natural reasoning and turns it into Python structure: Collection handling becomes lists; Data mapping becomes dictionaries.", + "promptScore": 79, + "promptFeedback": [ + "Ask the AI to explain its reasoning, not just produce code." + ], + "reflection": "i think i understand the flow of the problem ", + "misconceptions": [ + "Try to store values in names (variables) instead of hardcoding numbers directly." + ], + "masterySignals": [ + "Recognized collection handling", + "Recognized data mapping", + "Prompt maturity is developing" + ], + "createdAt": "2026-08-02T10:01:55.517Z", + "updatedAt": "2026-08-02T10:01:55.517Z" + }, + { + "_id": "df3eab14-f99a-4d81-9515-98c15cc553c8", + "learnerName": "Guest learner", + "scenario": "8cf644f0-1402-4763-9542-02bd2ca0d1dc", + "reasoning": "i will store a decimal(float) point number for the weight and store the value in a variable.", + "promptText": "i will first declare a variable and then assign it the float value of the weight. eg float x = 10.5 kg.", + "abstractionMap": [ + { + "pattern": "Data storage", + "pythonConcept": "variables", + "explanation": "You identified data that needs to be kept in memory, which maps to declaring variables.", + "confidence": "90% confidence" + } + ], + "generatedCode": "# Scenario: Bag Weight Label\n\ntarget_value = \"sample_data\"\nprint(f\"Processed {target_value}\")\n", + "codeExplanation": "The code starts from your natural reasoning and turns it into Python structure: Data storage becomes variables.", + "promptScore": 80, + "promptFeedback": [ + "Ask the AI to explain its reasoning, not just produce code." + ], + "reflection": "i just know things", + "misconceptions": [], + "masterySignals": [ + "Recognized data storage", + "Prompt maturity is developing" + ], + "createdAt": "2026-08-02T10:11:29.243Z", + "updatedAt": "2026-08-02T10:11:29.243Z" + }, + { + "_id": "d39ca5cd-e2ea-4d40-8c05-1eb805e6600c", + "learnerName": "Guest learner", + "scenario": "2a3496ef-bb71-4dde-83f5-37844063678d", + "reasoning": "for every break i will use a loop that will iterate through all the breaks once and reminds.", + "promptText": "i will first see how many breaks are there and then use a loop from 1st break to end break and in each break print the reminder .", + "abstractionMap": [ + { + "pattern": "Repetition", + "pythonConcept": "loops", + "explanation": "Your reasoning repeats an action, which maps naturally to loop constructs.", + "confidence": "90% confidence" + } + ], + "generatedCode": "# Scenario: Water Bottle Reminder\n\nfor item in items:\n # Process item\n print(item)\n", + "codeExplanation": "The code starts from your natural reasoning and turns it into Python structure: Repetition becomes loops.", + "promptScore": 100, + "promptFeedback": [ + "Strong prompt: it includes context, reasoning, examples, and the right Python direction." + ], + "reflection": "i just know it", + "misconceptions": [], + "masterySignals": [ + "Recognized repetition", + "Prompt maturity is developing" + ], + "createdAt": "2026-08-02T10:13:47.165Z", + "updatedAt": "2026-08-02T10:13:47.165Z" + } + ] } diff --git a/server/src/routes/sessions.js b/server/src/routes/sessions.js index 86d942a..89665e7 100644 --- a/server/src/routes/sessions.js +++ b/server/src/routes/sessions.js @@ -18,9 +18,9 @@ router.post('/', async (req, res, next) => { const scenario = await store.getScenario(req.body.scenarioId); if (!scenario) return res.status(404).json({ message: 'Scenario not found' }); - const abstractionMap = engine.mapReasoning(req.body.reasoning); + const abstractionMap = engine.mapReasoning(scenario, req.body.reasoning); const generatedCode = engine.generateCode(scenario, abstractionMap); - const prompt = engine.evaluatePrompt(req.body.promptText); + const prompt = engine.evaluatePrompt(scenario, req.body.promptText); const session = await store.addSession({ learnerName: req.body.learnerName || 'Guest learner', scenario: scenario._id, @@ -32,7 +32,7 @@ router.post('/', async (req, res, next) => { promptScore: prompt.score, promptFeedback: prompt.feedback, reflection: req.body.reflection || '', - misconceptions: engine.detectMisconceptions(req.body.reasoning), + misconceptions: engine.detectMisconceptions(scenario, req.body.reasoning), masterySignals: engine.masterySignals(abstractionMap, prompt.score) }); res.status(201).json(session); diff --git a/server/src/services/learningEngine.js b/server/src/services/learningEngine.js index 9e6b82b..921a66c 100644 --- a/server/src/services/learningEngine.js +++ b/server/src/services/learningEngine.js @@ -1,102 +1,245 @@ const conceptRules = [ { - keywords: ['repeat', 'again', 'each', 'every', 'loop'], + primary: ['variable', 'assign'], + secondary: ['store', 'value', 'remember', 'save', 'name', 'keep'], + pattern: 'Data storage', + pythonConcept: 'variables', + explanation: 'You identified data that needs to be kept in memory, which maps to declaring variables.' + }, + { + primary: ['loop', 'iterate', 'for loop', 'while loop'], + secondary: ['repeat', 'again', 'each', 'every', 'all'], pattern: 'Repetition', - pythonConcept: 'for / while loops', + pythonConcept: 'loops', explanation: 'Your reasoning repeats an action, which maps naturally to loop constructs.' }, { - keywords: ['if', 'when', 'unless', 'decide', 'choose', 'condition'], + primary: ['if', 'elif', 'else', 'condition', 'conditional'], + secondary: ['when', 'unless', 'decide', 'choose', 'check', 'otherwise', 'depends'], pattern: 'Decision making', - pythonConcept: 'if / elif / else', + pythonConcept: 'conditionals', explanation: 'You are branching based on a condition, which is exactly what conditional statements express.' }, { - keywords: ['list', 'items', 'collection', 'group', 'many'], + primary: ['list', 'array', 'sequence'], + secondary: ['items', 'collection', 'group', 'many', 'multiple'], pattern: 'Collection handling', - pythonConcept: 'lists and dictionaries', - explanation: 'You grouped multiple values, so Python collections help store and process them.' + pythonConcept: 'lists', + explanation: 'You grouped multiple pieces of data, so Python lists help store and process them.' + }, + { + primary: ['dictionary', 'dict', 'key-value'], + secondary: ['map', 'key', 'lookup', 'pair', 'associate'], + pattern: 'Data mapping', + pythonConcept: 'dictionaries', + explanation: 'You are associating pieces of data together, which maps perfectly to a Python dictionary.' }, { - keywords: ['calculate', 'total', 'average', 'sum', 'score', 'cost'], + primary: ['arithmetic', 'math', 'calculate', 'multiply', 'divide'], + secondary: ['total', 'average', 'sum', 'score', 'cost', 'add'], pattern: 'Computation', - pythonConcept: 'variables and arithmetic expressions', + pythonConcept: 'arithmetic', explanation: 'You are transforming values into a result, so variables and expressions become useful.' }, { - keywords: ['step', 'process', 'recipe', 'function', 'reuse'], + primary: ['function', 'def', 'routine'], + secondary: ['step', 'process', 'recipe', 'reuse', 'helper', 'task'], pattern: 'Reusable procedure', pythonConcept: 'functions', explanation: 'You described a repeatable process, which maps to a Python function.' }, { - keywords: ['compare', 'match', 'filter', 'find', 'search'], + primary: ['compare', 'filter', 'search'], + secondary: ['match', 'find', 'only', 'specific'], pattern: 'Selection and filtering', - pythonConcept: 'comparisons and list comprehensions', + pythonConcept: 'comparisons', explanation: 'You are narrowing options using rules, which Python can express with comparisons and filters.' + }, + { + primary: ['string', 'text format'], + secondary: ['text', 'word', 'character', 'letter', 'sentence', 'format', 'message'], + pattern: 'Text processing', + pythonConcept: 'strings', + explanation: 'You are dealing with text data, which requires string operations in Python.' + }, + { + primary: ['try', 'except', 'catch', 'error handling'], + secondary: ['error', 'fail', 'invalid', 'wrong', 'safe'], + pattern: 'Error handling', + pythonConcept: 'try/except', + explanation: 'You are anticipating things going wrong, which is handled gracefully with try/except.' + } +].map(rule => ({ + ...rule, + primaryRegex: new RegExp(`\\b(${rule.primary.join('|')})\\b`, 'i'), + secondaryRegexes: rule.secondary.map(sec => new RegExp(`\\b${sec}\\b`, 'i')) +})); + +const misconceptionsDictionary = { + 'loops': { + regex: /\b(copy|paste|manually|repeat it|write it out)\b/i, + feedback: 'Watch out for manual repetition. Let the loop do the heavy lifting rather than copying steps.' + }, + 'variables': { + regex: /\b(write the number|type|hardcode)\b/i, + feedback: 'Try to store values in names (variables) instead of hardcoding numbers directly.' + } +}; + +function mapReasoning(scenario, reasoning = '') { + const matches = []; + const expectedConcepts = scenario.concepts || []; + + for (const rule of conceptRules) { + const hasPrimary = rule.primaryRegex.test(reasoning); + + let secondaryCount = 0; + for (const secRegex of rule.secondaryRegexes) { + if (secRegex.test(reasoning)) { + secondaryCount++; + } + } + + if (hasPrimary || secondaryCount >= 2) { + let confidenceScore = hasPrimary ? 70 : 40; + confidenceScore += secondaryCount * 10; + + if (expectedConcepts.includes(rule.pythonConcept)) { + confidenceScore += 20; + } + + confidenceScore = Math.min(100, confidenceScore); + + const expectedText = expectedConcepts.length > 0 ? expectedConcepts.join(" or ") : "simpler concepts"; + + matches.push({ + pattern: rule.pattern, + pythonConcept: rule.pythonConcept, + explanation: expectedConcepts.includes(rule.pythonConcept) + ? rule.explanation + : `${rule.explanation} (Note: This might be over-engineering for this specific scenario. You can also try to use ${expectedText}.)`, + confidence: `${confidenceScore}% confidence` + }); + } } -]; -function mapReasoning(reasoning = '') { - const lower = reasoning.toLowerCase(); - const matches = conceptRules.filter((rule) => rule.keywords.some((keyword) => lower.includes(keyword))); + matches.sort((a, b) => parseInt(b.confidence) - parseInt(a.confidence)); + return matches.length ? matches : [{ pattern: 'Sequential thinking', - pythonConcept: 'statements and variables', - explanation: 'You described a step-by-step solution. Python starts by representing those steps as statements.' + pythonConcept: 'variables', + explanation: 'You described a step-by-step solution. Python starts by representing those steps as statements.', + confidence: 'Default' }]; } function generateCode(scenario, maps) { - const concepts = maps.map((item) => item.pythonConcept).join(', '); - const hasLoop = concepts.includes('loop'); - const hasCondition = concepts.includes('if'); - const hasFunction = concepts.includes('function'); + const concepts = maps.map((item) => item.pythonConcept); + const cleanTitle = scenario.title ? scenario.title.replace(/"/g, '\\"') : 'Scenario task'; + let code = `# Scenario: ${cleanTitle}\n\n`; - if (hasLoop && hasCondition) { - return 'items = [12, 7, 19, 4]\nthreshold = 10\n\nfor item in items:\n if item >= threshold:\n print(f"{item} needs attention")\n else:\n print(f"{item} is okay")'; + if (concepts.includes('variables') || concepts.includes('strings') || concepts.includes('arithmetic')) { + code += `target_value = "sample_data"\n`; } - - if (hasFunction) { - return 'def solve_scenario(inputs):\n result = []\n for value in inputs:\n result.append(value * 2)\n return result\n\nprint(solve_scenario([1, 2, 3]))'; + + if (concepts.includes('lists')) { + code += `items = [target_value, "other_data", "more_data"]\n`; + } + + if (concepts.includes('dictionaries')) { + code += `data_map = {"key1": target_value, "key2": "other_data"}\n`; } - if (hasLoop) { - return 'steps = ["notice the situation", "look for a pattern", "apply the rule"]\n\nfor step in steps:\n print(step)'; + if (concepts.includes('loops')) { + code += `for item in items:\n # Process item\n`; + if (concepts.includes('conditionals')) { + code += ` if item == target_value:\n print(f"Found {item}")\n else:\n print("Not found")\n`; + } else { + code += ` print(item)\n`; + } + } else if (concepts.includes('conditionals')) { + code += `if target_value == "sample_data":\n print("Condition met")\nelse:\n print("Condition failed")\n`; + } else if (!concepts.includes('lists') && !concepts.includes('dictionaries')) { + code += `print(f"Processed {target_value}")\n`; } - if (hasCondition) { - return 'temperature = 32\n\nif temperature > 30:\n print("Take action now")\nelse:\n print("Keep observing")'; + if (concepts.includes('functions')) { + let lines = code.split('\n'); + lines.pop(); + code = `def handle_task():\n ${lines.join('\n ')}\n\nhandle_task()\n`; + } + + if (concepts.includes('try/except')) { + let lines = code.split('\n'); + lines.pop(); + code = `try:\n ${lines.join('\n ')}\nexcept Exception as e:\n print(f"Error occurred: {e}")\n`; } - return `scenario = "${scenario.title.replace(/"/g, '\\"')}"\nreasoning = "Break the situation into clear steps"\nprint(scenario)\nprint(reasoning)`; + return code; } function explainCode(maps) { return `The code starts from your natural reasoning and turns it into Python structure: ${maps.map((map) => `${map.pattern} becomes ${map.pythonConcept}`).join('; ')}.`; } -function evaluatePrompt(promptText = '') { +function evaluatePrompt(scenario, promptText = '') { const feedback = []; - let score = 35; - if (promptText.length > 40) score += 15; - else feedback.push('Add more context about the situation and expected output.'); - if (/step|explain|why|reason/i.test(promptText)) score += 20; - else feedback.push('Ask the AI to explain its reasoning, not just produce code.'); - if (/example|input|output|data/i.test(promptText)) score += 15; - else feedback.push('Include an example input or output to make the prompt testable.'); - if (/python|loop|if|list|function/i.test(promptText)) score += 15; - else feedback.push('Name the Python concept you think may apply.'); + let score = 20; + + const lengthScore = Math.min(25, Math.floor(promptText.length / 4)); + score += lengthScore; + if (promptText.length < 40) { + feedback.push('Add more context about the situation and expected output.'); + } + + if (/\b(step|explain|why|reason|how|approach)\b/i.test(promptText)) { + score += 20; + } else { + feedback.push('Ask the AI to explain its reasoning, not just produce code.'); + } + + const expectedConcepts = scenario.concepts || []; + let mentionedExpected = false; + + for (const concept of expectedConcepts) { + let searchTerms = []; + if (concept === 'conditionals') searchTerms = ['if', 'condition']; + if (concept === 'loops') searchTerms = ['loop', 'for', 'while']; + if (concept === 'variables') searchTerms = ['variable', 'store']; + if (concept === 'lists') searchTerms = ['list', 'array']; + if (concept === 'functions') searchTerms = ['function', 'def']; + + if (searchTerms.some(term => new RegExp(`\\b${term}\\b`, 'i').test(promptText))) { + mentionedExpected = true; + break; + } + } + + if (mentionedExpected) { + score += 35; + } else if (expectedConcepts.length > 0) { + feedback.push(`Try to name the specific Python concept this scenario needs (e.g. something related to ${expectedConcepts[0]}).`); + } + return { score: Math.min(score, 100), - feedback: feedback.length ? feedback : ['Strong prompt: it includes context, reasoning, examples, and a Python direction.'] + feedback: feedback.length ? feedback : ['Strong prompt: it includes context, reasoning, examples, and the right Python direction.'] }; } -function detectMisconceptions(reasoning = '') { +function detectMisconceptions(scenario, reasoning = '') { const misses = []; - if (/always|never/i.test(reasoning)) misses.push('Watch for absolute rules. Programming logic often needs explicit edge cases.'); + if (/\b(always|never)\b/i.test(reasoning)) misses.push('Watch for absolute rules. Programming logic often needs explicit edge cases.'); if (reasoning.length < 60) misses.push('Reasoning is brief. Try naming the inputs, decision rule, and expected result.'); + + const expectedConcepts = scenario.concepts || []; + for (const concept of expectedConcepts) { + const rule = misconceptionsDictionary[concept]; + if (rule && rule.regex.test(reasoning)) { + misses.push(rule.feedback); + } + } + return misses; }