Golden set creator

.gitignore

@@ -4,7 +4,6 @@
 __pycache__/
 *.pyc
 .mypy_cache
-.deepeval
 .pytest_cache
 .claude
 .ruff_cache
@@ -25,3 +24,9 @@ htmlcov/
 
 /**/vm_session*.json
 vm_config.json
+
+# evals
+wandb/
+debug_*
+.deepeval
+eval/screenshots

.vscode/launch.json

@@ -2,7 +2,7 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "VM Automation - RDP",
+            "name": "VM Automation",
             "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/src/main.py",
@@ -25,8 +25,8 @@
             "justMyCode": false
         },
         {
-            "name": "VM Automation - RDP with Config File",
-            "type": "python",
+            "name": "VM Automation - with Config File",
+            "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/src/main.py",
             "args": [
@@ -44,36 +44,21 @@
             "justMyCode": false
         },
         {
-            "name": "VM Automation - Validate Environment",
-            "type": "python",
+            "name": "Debug Golden Set Creator",
+            "type": "debugpy", 
             "request": "launch",
-            "program": "${workspaceFolder}/src/main.py",
+            "program": "${workspaceFolder}/eval/golden_set_creator.py",
             "args": [
-                "--validate-env"
+                "--images", "${workspaceFolder}/eval/screenshots",
+                "--output", "${workspaceFolder}/eval/debug_golden_set"
             ],
             "console": "integratedTerminal",
             "cwd": "${workspaceFolder}",
             "env": {
                 "PYTHONPATH": "${workspaceFolder}/src"
             },
-            "stopOnEntry": false,
-            "justMyCode": false
-        },
-        {
-            "name": "VM Automation - Create Samples",
-            "type": "python",
-            "request": "launch",
-            "program": "${workspaceFolder}/src/main.py",
-            "args": [
-                "--create-samples"
-            ],
-            "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}",
-            "env": {
-                "PYTHONPATH": "${workspaceFolder}/src"
-            },
-            "stopOnEntry": false,
-            "justMyCode": false
+            "justMyCode": false,
+            "stopOnEntry": false
         }
     ]
 }