diff --git a/infrastructure/docker/.dockerignore b/.dockerignore similarity index 100% rename from infrastructure/docker/.dockerignore rename to .dockerignore diff --git a/.gitignore b/.gitignore index 3dec87e..3ca650c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ __pycache__/ *.pyc .mypy_cache -.deepeval .pytest_cache .claude .ruff_cache @@ -25,3 +24,9 @@ htmlcov/ /**/vm_session*.json vm_config.json + +# evals +wandb/ +debug_* +.deepeval +eval/screenshots diff --git a/.vscode/launch.json b/.vscode/launch.json index 42fe2cd..dde88d8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,7 +2,7 @@ "version": "0.2.0", "configurations": [ { - "name": "VM Automation - RDP", + "name": "VM Automation", "type": "debugpy", "request": "launch", "program": "${workspaceFolder}/src/main.py", @@ -25,8 +25,8 @@ "justMyCode": false }, { - "name": "VM Automation - RDP with Config File", - "type": "python", + "name": "VM Automation - with Config File", + "type": "debugpy", "request": "launch", "program": "${workspaceFolder}/src/main.py", "args": [ @@ -44,36 +44,21 @@ "justMyCode": false }, { - "name": "VM Automation - Validate Environment", - "type": "python", + "name": "Debug Golden Set Creator", + "type": "debugpy", "request": "launch", - "program": "${workspaceFolder}/src/main.py", + "program": "${workspaceFolder}/eval/golden_set_creator.py", "args": [ - "--validate-env" + "--images", "${workspaceFolder}/eval/screenshots", + "--output", "${workspaceFolder}/eval/debug_golden_set" ], "console": "integratedTerminal", "cwd": "${workspaceFolder}", "env": { "PYTHONPATH": "${workspaceFolder}/src" }, - "stopOnEntry": false, - "justMyCode": false - }, - { - "name": "VM Automation - Create Samples", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/src/main.py", - "args": [ - "--create-samples" - ], - "console": "integratedTerminal", - "cwd": "${workspaceFolder}", - "env": { - "PYTHONPATH": "${workspaceFolder}/src" - }, - "stopOnEntry": false, - "justMyCode": false + "justMyCode": false, + "stopOnEntry": false } ] } diff --git a/infrastructure/docker/Dockerfile.vnc b/Dockerfile similarity index 100% rename from infrastructure/docker/Dockerfile.vnc rename to Dockerfile diff --git a/README.md b/README.md deleted file mode 100644 index a7fbe55..0000000 --- a/README.md +++ /dev/null @@ -1,1267 +0,0 @@ -# VM Automation - Production Ready GUI Automation System - -๐Ÿค– **Multi-Architecture Computer Vision Automation Platform** - -A production-ready AI system that automates GUI interactions across VMs and local desktops with advanced computer vision, featuring unified OCR architecture, MCP server integration, and healthcare-grade patient safety verification. - -**Version**: 2.0.0 -**Architecture**: OCR System -**AI Integration**: OpenAI, Claude, and Custom LLM Support -**Protocols**: VNC, RDP, Local Desktop (macOS/AppleScript) - -## ๐ŸŽฏ System Overview - -**Multi-Agent Architecture with Flexible Backends:** - -- **VM Navigator Agent**: Connects to remote VMs (VNC/RDP), launches applications, verifies patient safety -- **App Controller Agent**: Performs precise GUI interactions using shared computer vision components -- **Local Desktop Agent**: Native macOS automation using AppleScript for local screen control -- **MCP Server**: Model Context Protocol server exposing vision tools to LLMs -- **Unified OCR System**: Pure function-based computer vision core serving all agents - -**Key Technologies:** - -- **YOLOv8s-ONNX**: Real-time UI element detection with CPU-optimized inference -- **PaddleOCR**: Professional multilingual text recognition and extraction -- **Unified OCR Architecture**: Pure function-based computer vision with spatial reasoning -- **Action Verification**: Screenshot-based verification with confidence scoring -- **MCP Protocol**: Model Context Protocol for LLM tool integration -- **Multi-Backend Support**: VNC, RDP, and native macOS automation -- **Healthcare Safety**: HIPAA-compliant patient identity verification -- **Production Features**: Error handling, audit logging, session management - -## ๐Ÿš€ Quick Start - -### 1. Setup - -```bash -# Clone repository -git clone https://github.com/your-org/computer-use-agent.git -cd computer-use-agent - -# Install dependencies (requires Python 3.10+) -uv sync - -# Download and setup AI models (YOLO + PaddleOCR) -uv run src/vision/setup_models.py - -# Verify installation -uv run python -c "from vision import detect_ui_elements; print('โœ… Installation complete')" -``` - -**System Requirements:** -- Python 3.10 or later -- 4GB RAM minimum (8GB recommended for optimal performance) -- macOS: For local automation features -- Linux/Windows: For VM connections only - -### 2. Basic Usage - -**Quick Test (Local Desktop):** - -```bash -# Test local desktop automation (macOS only) -uv run python -c " -from automation.local.form_interface import FormFiller -from automation.local.desktop_control import DesktopControl -print('Testing local automation...') -desktop = DesktopControl() -print('โœ… Local automation ready') -" - -# Run comprehensive test suite -./tests/run_tests.py all -``` - -**VM Automation:** - -```bash -# Quick start with environment variables -export VM_HOST="192.168.1.100" -export VM_PASSWORD="your_password" -export CONNECTION_TYPE="vnc" -uv run src/vm/main.py - -# Or use configuration file -uv run vm-automation --create-samples -# Edit vm_config.json, then: -uv run vm-automation -``` - -**MCP Server (for LLM Integration):** - -```bash -# Start MCP server for Claude/OpenAI integration -uv run python -c " -from agent.mcp import start_server -start_server(port=8000) -" -``` - -### 3. Configuration Options - -**Via JSON file (vm_config.json):** - -*VNC Configuration:* - -```json -{ - "vm_host": "192.168.1.100", - "vm_port": 5900, - "vm_password": "vnc_password", - "connection_type": "vnc", - "target_app_name": "Greenway Dev", - "target_button_text": "Submit", - "ocr_confidence_threshold": 0.6, - "yolo_confidence_threshold": 0.5, - "screenshot_interval": 1.0, - "max_retries": 3, - "enable_audit_logging": true, - "log_level": "INFO" -} -``` - -*RDP Configuration:* - -```json -{ - "vm_host": "192.168.1.100", - "vm_port": 3389, - "vm_username": "user", - "vm_password": "password", - "connection_type": "rdp", - "rdp_domain": "COMPANY", - "rdp_width": 1920, - "rdp_height": 1080, - "rdp_color_depth": 24, - "target_app_name": "Greenway Dev", - "target_button_text": "Submit", - "ocr_confidence_threshold": 0.7, - "yolo_confidence_threshold": 0.6, - "screenshot_interval": 0.8, - "enable_gpu_acceleration": false, - "timeout_seconds": 30 -} -``` - -**Via Environment Variables:** - -```bash -# VNC Connection -export CONNECTION_TYPE="vnc" -export VM_HOST="192.168.1.100" -export VM_PORT="5900" -export VM_PASSWORD="vnc_password" -export TARGET_APP="Greenway Dev" -export TARGET_BUTTON="Submit" -vm-automation - -# RDP Connection -export CONNECTION_TYPE="rdp" -export VM_HOST="192.168.1.100" -export VM_PORT="3389" -export VM_USERNAME="user" -export VM_PASSWORD="password" -export RDP_DOMAIN="COMPANY" -export RDP_WIDTH="1920" -export RDP_HEIGHT="1080" -export TARGET_APP="Greenway Dev" -export TARGET_BUTTON="Submit" -vm-automation -``` - -## ๐Ÿค– AI Agent Computer Vision Tools - -The system includes professional computer vision tools designed for AI agent automation. Any AI agent can use these tools with simple prompts - no complex setup required. - -### ๐Ÿ”ง Core Vision Components - -**YOLOv8s-ONNX Detector:** -- Real-time UI element detection -- UI-focused class filtering (removes irrelevant objects like animals, food) -- Optimized for screen/desktop environments -- CPU-optimized ONNX inference - -**PaddleOCR Integration:** -- Professional text recognition and extraction -- Multi-language support -- Region-aware text detection -- High-confidence text filtering - -**Unified OCR System:** -- Pure function-based design with YOLO + OCR integration -- Intelligent element detection and text extraction -- Spatial reasoning for UI element relationships -- Unified element representation with confidence scoring - -**Integrated Verification:** -- Function-based action verification -- Screenshot-based change detection -- Text input verification -- Element presence confirmation - -### ๐ŸŽฏ AI Agent Function Tools - -**Simple Function Interface:** - -```python -from agent.vision_tools import * - -# Analyze current screen -analysis = analyze_screen("What buttons and forms are visible?") - -# Find specific elements -button = find_element("Submit button") -field = find_element("Username input field") - -# Perform actions -click_element(button) -type_text_in_field("john.doe", field) - -# Verify outcomes -verify_action("Form should be submitted successfully") -``` - -**Available Functions:** -- `analyze_screen(prompt)` - Analyze screen contents with natural language -- `find_element(description)` - Find UI elements by description -- `click_element(element)` - Click on elements or coordinates -- `type_text_in_field(text, field)` - Type text in input fields -- `verify_action(expected)` - Verify action outcomes -- `wait_for_element(description)` - Wait for elements to appear -- `scroll_screen(direction, pixels)` - Scroll screen content -- `take_screenshot(path)` - Capture screen images - -### ๐Ÿ”Œ AI Framework Integration - -**OpenAI Functions:** - -```python -from openai import OpenAI -from agent.function_definitions import get_vision_function_tools -from agent.vision_tools import * - -client = OpenAI() -tools = get_vision_function_tools() - -response = client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "Click the Submit button"}], - tools=tools, - tool_choice="auto" -) -``` - -**Claude Tools:** - -```python -import anthropic -from agent.function_definitions import get_vision_function_tools - -client = anthropic.Anthropic() -tools = get_vision_function_tools() - -response = client.messages.create( - model="claude-3-opus-20240229", - tools=tools, - messages=[{"role": "user", "content": "Fill out this form"}] -) -``` - -**Other Frameworks:** -- Compatible with any AI framework that supports function calling -- JSON schema definitions provided for easy integration -- Standardized prompt-to-action interface - -### ๐ŸŽฌ Usage Examples - -**Healthcare Workflow Automation:** -```python -# Healthcare EMR automation with patient safety -from agent.vision_tools import * -from vm.automation.vm_navigator import VMNavigatorTools - -# Initialize with patient safety verification -navigator = VMNavigatorTools(session, vm_target) - -# Verify patient banner before proceeding -patient_info = { - "name": "John Doe", - "mrn": "123456", - "dob": "01/01/1980" -} -safety_check = navigator.verify_patient_banner(patient_info) -if not safety_check["success"]: - raise Exception("Patient safety verification failed") - -# Proceed with form automation -username_field = find_element("username field") -type_text_in_field("doctor@hospital.com", username_field) -submit_btn = find_element("login button") -click_element(submit_btn) -verify_action("Successfully logged into EMR system") -``` - -**Multi-Backend Automation:** -```python -# Example using different backends for different tasks -from automation.desktop_control import DesktopControl # Local macOS -from vm.connections.vnc_connection import VNCConnection # Remote VM - -# Local screenshot analysis -local_control = DesktopControl() -local_screenshot = local_control.capture_screen() -local_elements = detect_ui_elements(local_screenshot) - -# Remote VM interaction -vm_connection = VNCConnection("192.168.1.100", 5900, "password") -vm_screenshot = vm_connection.capture_screen() -vm_elements = find_elements_by_text(vm_screenshot, "Submit") -``` - -**Form Automation:** -```python -# AI agent can automate forms with simple prompts -analysis = analyze_screen("Analyze this login form") -username_field = find_element("username field") -type_text_in_field("user@example.com", username_field) - -password_field = find_element("password field") -type_text_in_field("secure_pass", password_field) - -submit_btn = find_element("login button") -click_element(submit_btn) - -verify_action("Login should be successful") -``` - -**Web Navigation:** -```python -# Navigate websites with computer vision -search_box = find_element("search box") -type_text_in_field("computer vision AI", search_box) - -search_btn = find_element("search button") -click_element(search_btn) - -results = wait_for_element("search results", max_attempts=10) -verify_action("Search results should be displayed") -``` - -**Settings Management:** -```python -# Navigate application settings -settings = find_element("settings menu") -click_element(settings) - -privacy_tab = find_element("privacy tab") -click_element(privacy_tab) - -toggle = find_element("data collection toggle") -click_element(toggle) - -verify_action("Privacy settings should be updated") -``` - -### โš™๏ธ Advanced Configuration Options - -**Vision System Configuration:** -```python -from agent.vision_tools import configure_vision_tools - -# Healthcare/Financial (High Precision) -configure_vision_tools( - confidence_threshold=0.8, - ocr_language="en", - enable_gpu=False, # CPU-only for consistency - max_screenshot_size=(1920, 1080), - element_timeout=10.0 -) - -# General Automation (Balanced) -configure_vision_tools( - confidence_threshold=0.6, - ocr_language="en", - enable_preprocessing=True, - screenshot_quality=85 -) - -# Development/Testing (Fast) -configure_vision_tools( - confidence_threshold=0.4, - ocr_language="en", - enable_caching=True, - debug_mode=True -) -``` - -**MCP Server Configuration:** -```python -from agent.mcp import create_mcp_server - -# Production MCP server -server = create_mcp_server( - host="0.0.0.0", - port=8000, - max_concurrent_requests=10, - enable_cors=True, - log_level="INFO" -) - -# Development MCP server -server = create_mcp_server( - host="localhost", - port=8001, - debug=True, - enable_detailed_logging=True -) -``` - -**VM Connection Tuning:** -```python -from vm.connections.vnc_connection import VNCConnection - -# High-performance VNC -vnc = VNCConnection( - host="192.168.1.100", - port=5900, - password="password", - pixel_format="bgr233", # Faster but lower quality - enable_compression=True, - compression_level=6 -) - -# High-quality VNC -vnc = VNCConnection( - host="192.168.1.100", - port=5900, - password="password", - pixel_format="truecolor", - enable_compression=False, - capture_rate=30 # FPS -) -``` - -### ๐Ÿ“ Complete System Architecture - -``` -src/ -โ”œโ”€โ”€ vision/ # Unified Computer Vision System -โ”‚ โ”œโ”€โ”€ detector.py # YOLOv8s-ONNX UI element detection -โ”‚ โ”œโ”€โ”€ reader.py # PaddleOCR text extraction -โ”‚ โ”œโ”€โ”€ finder.py # Pure function UI element search and analysis -โ”‚ โ”œโ”€โ”€ verification.py # Pure function action verification -โ”‚ โ”œโ”€โ”€ setup_models.py # Model download and management -โ”‚ โ””โ”€โ”€ models/ # Pre-trained models -โ”‚ โ”œโ”€โ”€ yolov8s.onnx # YOLO ONNX model for inference -โ”‚ โ””โ”€โ”€ yolov8s.pt # YOLO PyTorch model -โ”œโ”€โ”€ agent/ # AI Agent Interface -โ”‚ โ”œโ”€โ”€ vision_tools.py # Function interface for AI agents -โ”‚ โ”œโ”€โ”€ function_definitions.py # JSON schemas for AI frameworks -โ”‚ โ”œโ”€โ”€ examples.py # Usage examples and demos -โ”‚ โ””โ”€โ”€ mcp/ # Model Context Protocol Server -โ”‚ โ”œโ”€โ”€ __init__.py # MCP server interface exports -โ”‚ โ”œโ”€โ”€ handlers.py # MCP tool handlers -โ”‚ โ”œโ”€โ”€ mcp_server.py # Model Context Protocol server -โ”‚ โ””โ”€โ”€ tools.py # Tool definitions for LLMs -โ”œโ”€โ”€ vm/ # VM Connection and Automation -โ”‚ โ”œโ”€โ”€ connections/ # VNC/RDP/Desktop connection backends -โ”‚ โ”œโ”€โ”€ tools/ # Screen capture and input actions -โ”‚ โ””โ”€โ”€ automation/ # VM navigator and app controller agents -โ”œโ”€โ”€ automation/ # Local Desktop Automation -โ”‚ โ”œโ”€โ”€ desktop_control.py # Native macOS automation (AppleScript) -โ”‚ โ””โ”€โ”€ form_interface.py # High-level form filling interface -``` - -**๐Ÿ—๏ธ Multi-Layered Architecture:** - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ AI Agent Functions โ”‚ โ† Prompt-based interface -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ MCP Server Interface โ”‚ โ† Model Context Protocol tools -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Unified OCR Functions โ”‚ โ† Pure function computer vision -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ VM Connection Backends โ”‚ โ† VNC/RDP/Desktop abstraction -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Vision Models (YOLO + PaddleOCR) โ”‚ โ† Computer vision core -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -**๐ŸŽฏ Agent Complexity Levels:** - -- **SIMPLE**: `analyze_screen()`, `find_element()`, `click_element()` -- **STRUCTURED**: `execute_structured_workflow()`, `create_form_filling_workflow()` -- **ADVANCED**: `AgentSession`, spatial reasoning, custom runtime patterns -- **EXPERT**: Full infrastructure access with `StepRunner`, protocol adapters - -### ๐Ÿงช Testing Multi-Agent Infrastructure - -**Basic Vision Tools:** -```bash -# Test basic agent functions -uv run python -c "from agent import analyze_screen, find_element; print('โœ… Basic tools ready')" - -# Test advanced workflow capabilities -uv run python -c "from agent import execute_structured_workflow, create_form_filling_workflow; print('โœ… Advanced tools ready')" - -# Test multi-agent service -uv run python -c "from agent.service_architecture import get_service_status; print('โœ… Service ready')" -``` - -**Infrastructure Testing:** -```bash -# Test unified OCR functions -uv run python -c "from vision import detect_ui_elements, extract_text, find_elements_by_text; print('โœ… Unified OCR functions')" - -# Test verification functions -uv run python -c "from vision import verify_click_success, verify_element_present; print('โœ… Verification functions')" - -# Test MCP server interface -uv run python -c "from agent.mcp import create_mcp_server, get_function_tools; print('โœ… MCP server interface')" - -# Run comprehensive examples -uv run python src/agent/examples.py - -# Setup models if not downloaded -uv run python src/vision/setup_models.py -``` - -**Multi-Agent Service Demo:** -```python -from agent.service_architecture import ( - OCRService, ConnectionBackend, get_service_status -) - -# Create service -service = OCRService() - -# Create desktop agent session -session_id = service.create_session( - ConnectionBackend.DESKTOP, - {"models_dir": "src/vision/models"} -) - -# Execute actions in session -result = service.execute_in_session( - session_id, - "analyze_screen", - {"prompt": "What's on the screen?"} -) - -# Get service status -status = get_service_status() -print(f"Active sessions: {status['active_sessions']}") -``` - -## ๐Ÿ”— Connection Types - -The system supports both VNC and RDP connections through a flexible abstraction layer: - -### VNC Connection - -- **Use case**: Any VM with VNC server installed -- **Requirements**: VNC server running on target VM (port 5900) -- **Advantages**: Lightweight, cross-platform -- **Setup**: Install VNC server on VM (TightVNC, RealVNC, UltraVNC) - -### RDP Connection - -- **Use case**: Windows VMs with Remote Desktop enabled -- **Requirements**: FreeRDP, Xvfb, xdotool, ImageMagick on automation host -- **Advantages**: Native Windows protocol, no additional VM setup needed -- **Note**: Works best on Linux. For macOS, consider VNC for simpler setup. -- **Setup**: - - **Linux:** - ```bash - sudo apt install freerdp2-x11 xvfb xdotool imagemagick scrot - ``` - - **macOS:** - ```bash - # Core RDP dependencies - brew install freerdp imagemagick xorg-server xdotool - - # Optional: Install scrot for better screenshot performance (if available) - brew install scrot || echo "scrot not available, will use xwd fallback" - ``` - -The same computer vision and UI automation logic works seamlessly with both connection types. - -## ๐Ÿ”ง File Structure - -``` -โ”œโ”€โ”€ src/ -โ”‚ โ”œโ”€โ”€ main.py # Main orchestrator (consolidated entry point) -โ”‚ โ”œโ”€โ”€ vision/ # Unified Computer Vision System -โ”‚ โ”‚ โ”œโ”€โ”€ detector.py # YOLOv8s-ONNX UI element detection -โ”‚ โ”‚ โ”œโ”€โ”€ reader.py # PaddleOCR text extraction -โ”‚ โ”‚ โ”œโ”€โ”€ finder.py # Pure function UI element search -โ”‚ โ”‚ โ”œโ”€โ”€ verification.py # Pure function action verification -โ”‚ โ”‚ โ”œโ”€โ”€ setup_models.py # Model download and management -โ”‚ โ”‚ โ””โ”€โ”€ models/ # Pre-trained models -โ”‚ โ”‚ โ”œโ”€โ”€ yolov8s.onnx # YOLO ONNX model -โ”‚ โ”‚ โ””โ”€โ”€ yolov8s.pt # YOLO PyTorch model -โ”‚ โ”œโ”€โ”€ agent/ # AI Agent Interface -โ”‚ โ”‚ โ”œโ”€โ”€ vision_tools.py # Function interface for AI agents -โ”‚ โ”‚ โ”œโ”€โ”€ function_definitions.py # JSON schemas for AI frameworks -โ”‚ โ”‚ โ”œโ”€โ”€ examples.py # Usage examples and demos -โ”‚ โ”‚ โ””โ”€โ”€ mcp/ # Model Context Protocol Server -โ”‚ โ”‚ โ”œโ”€โ”€ __init__.py # MCP server interface exports -โ”‚ โ”‚ โ”œโ”€โ”€ handlers.py # MCP tool handlers -โ”‚ โ”‚ โ”œโ”€โ”€ mcp_server.py # Model Context Protocol server -โ”‚ โ”‚ โ””โ”€โ”€ tools.py # Tool definitions for LLMs -โ”‚ โ”œโ”€โ”€ vm/ # VM connection management -โ”‚ โ”‚ โ”œโ”€โ”€ connections/ # Connection implementations -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ base.py # Abstract base classes -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ vnc_connection.py # VNC implementation -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ rdp_connection.py # RDP implementation -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ desktop_connection.py # Local desktop connection -โ”‚ โ”‚ โ”œโ”€โ”€ tools/ # Screen capture and input actions -โ”‚ โ”‚ โ””โ”€โ”€ automation/ # VM-specific agent implementations -โ”‚ โ”‚ โ”œโ”€โ”€ vm_navigator.py # Agent 1: VM connection & navigation -โ”‚ โ”‚ โ””โ”€โ”€ app_controller.py # Agent 2: GUI interactions -โ”œโ”€โ”€ automation/ # Local Desktop Automation -โ”‚ โ”œโ”€โ”€ desktop_control.py # Native macOS automation (AppleScript) -โ”‚ โ””โ”€โ”€ form_interface.py # High-level form filling interface -โ”œโ”€โ”€ tests/ # Comprehensive Test Suite (136 tests) -โ”‚ โ”œโ”€โ”€ conftest.py # Test configuration and fixtures -โ”‚ โ”œโ”€โ”€ run_tests.py # Test execution script -โ”‚ โ”œโ”€โ”€ unit/ # Unit tests (100% core coverage) -โ”‚ โ”‚ โ”œโ”€โ”€ automation/ # Automation framework tests -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ core/ # Core types and base classes -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ local/ # macOS desktop automation -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ remote/ # VM connection protocols -โ”‚ โ”‚ โ”œโ”€โ”€ agent/ # AI agent interface tests -โ”‚ โ”‚ โ””โ”€โ”€ vision/ # Computer vision tests -โ”‚ โ”œโ”€โ”€ integration/ # Integration test framework -โ”‚ โ”‚ โ”œโ”€โ”€ automation/ # End-to-end automation tests -โ”‚ โ”‚ โ”œโ”€โ”€ workflows/ # Multi-agent workflows -โ”‚ โ”‚ โ””โ”€โ”€ performance/ # Performance benchmarks -โ”‚ โ”œโ”€โ”€ .coveragerc # Coverage configuration -โ”‚ โ””โ”€โ”€ pytest.ini # Test runner configuration -โ”œโ”€โ”€ Dockerfile # Container deployment -โ””โ”€โ”€ vm_config.sample.json # Sample configuration -``` - -## ๐Ÿ›ก๏ธ Healthcare Features (Optional) - -For healthcare applications, the system includes patient safety verification: - -**Configuration with patient safety:** - -```json -{ - "vm_host": "192.168.1.100", - "connection_type": "rdp", - "vm_username": "user", - "vm_password": "password", - "target_app_name": "EMR System", - "target_button_text": "Submit", - "patient_name": "John Doe", - "patient_mrn": "123456", - "patient_dob": "01/01/1980", - "log_phi": false -} -``` - -**Safety Features:** - -- Automatic patient identity verification using OCR -- Requires 2+ matching patient identifiers before proceeding -- Immediately stops if patient verification fails -- PHI-compliant logging (disabled by default) - -## ๐Ÿงช Testing - -The system includes a comprehensive test strategy with **136 unit tests** and full CI/CD integration: - -### ๐Ÿ“Š Test Coverage Summary - -| Module | Coverage | Tests | Status | -|--------|----------|-------|---------| -| **automation.core** | 100% | 31 | โœ… Complete | -| **automation.local** | 75-93% | 47 | โœ… Complete | -| **automation.remote.connections** | 77-100% | 58 | โœ… Complete | -| **Overall Coverage** | **42%** | **136** | ๐ŸŽฏ **Production Ready** | - -### ๐Ÿš€ Quick Test Commands - -```bash -# Run all tests with coverage (recommended) -./tests/run_tests.py all - -# Run specific test suites -./tests/run_tests.py unit # Unit tests only -./tests/run_tests.py coverage # Generate detailed coverage report -./tests/run_tests.py lint # Code quality checks - -# Direct pytest usage -python -m pytest tests/unit/ --cov=src --cov-report=html -v - -# Run specific modules -python -m pytest tests/unit/automation/core/ -v -python -m pytest tests/unit/automation/local/ -v -python -m pytest tests/unit/automation/remote/connections/ -v -``` - -### ๐Ÿ“‹ Test Architecture Overview - -| Test Type | Description | VM Required | Coverage | -|-----------|-------------|-------------|----------| -| **Unit Tests** | Mock-based tests for all core functionality | โŒ No | 100% core modules | -| **Integration Tests** | End-to-end automation workflows | โœ… Yes | Pending | -| **CI/CD Tests** | Automated testing on every commit | โŒ No | Full pipeline | - -### ๐Ÿ”ง Comprehensive Unit Test Suite - -**Production-Ready Testing Features:** -- **Mock-based testing** - No external dependencies required -- **Cross-platform compatibility** - Proper path and process mocking -- **Error handling coverage** - Exception scenarios thoroughly tested -- **Automated CI/CD** - GitHub Actions integration with coverage reporting - -**Test Categories:** - -```bash -# Core automation framework -python -m pytest tests/unit/automation/core/ -v # Types, base classes (100% coverage) - -# Local macOS automation -python -m pytest tests/unit/automation/local/ -v # Desktop control, form interface (75-93% coverage) - -# Remote VM connections -python -m pytest tests/unit/automation/remote/ -v # VNC, RDP connections (77-100% coverage) -``` - -**Unit tests cover:** -- Connection lifecycle management (connect, disconnect, error handling) -- Desktop automation actions (click, type, scroll, screenshot) -- Form filling and OCR integration -- VM protocol implementations (VNC, RDP) -- Patient safety verification workflows -- Action result validation and timestamping - -### ๐ŸŒ Integration Tests & CI/CD Pipeline - -**GitHub Actions CI/CD Pipeline:** -- **Automated testing** on every push and pull request -- **Code quality checks** with ruff linting and pyright type checking -- **Security scanning** with bandit -- **Coverage reporting** with automatic Codecov integration -- **Cross-platform support** (Ubuntu with system dependencies) - -**CI/CD Features:** -```yaml -# .github/workflows/test.yml includes: -- Unit tests with 70% coverage threshold -- Integration tests (mock-only in CI) -- Lint and format checking -- Type checking with pyright -- Security vulnerability scanning -- Test result artifact collection -``` - -**Local Integration Testing:** -```bash -# Integration tests are available but require VM setup -python -m pytest tests/integration/ -m "mock and not real_vm" -v - -# For real VM testing (manual setup required): -# 1. Configure VM connection parameters -# 2. Set up test environment variables -# 3. Run integration tests with real VM connections -``` - -**Integration test coverage includes:** -- End-to-end automation workflows -- Multi-agent coordination testing -- Connection error recovery -- Performance benchmarking - -### ๐Ÿฅ Patient Workflow Tests (Healthcare VM Required) - -For testing patient-specific application workflows using computer vision: - -```bash -# Set patient application parameters -export PATIENT_TEST_VM_HOST="192.168.1.100" -export PATIENT_TEST_VM_PORT="5900" -export PATIENT_TEST_VM_PASSWORD="vnc_password" -export PATIENT_APP_NAME="EMR_System.exe" -export PATIENT_APP_TITLE="Electronic Medical Records" -export TEST_PATIENT_ID="PAT001234" -export TEST_PATIENT_NAME="John Smith" -export TEST_PATIENT_MRN="12345678" -export TEST_PATIENT_DOB="01/15/1975" -export SEARCH_FIELD_LABEL="Patient ID" -export SEARCH_BUTTON_TEXT="Search" - -# Remove skip markers and run patient workflow tests -uv run pytest tests/test_patient_workflow_integration.py::TestPatientWorkflowIntegration -v -``` - -**Patient workflow tests cover:** - -- PaddleOCR text extraction from patient applications -- YOLO-based UI element detection -- Patient search and verification workflows -- Computer vision tool integration -- Healthcare safety verification - -### ๐Ÿ“Š Test Configuration & Environment Variables - -#### Required Environment Variables by Test Type: - -**VNC Integration Tests:** - -```bash -TEST_VNC_HOST=192.168.1.100 # VM IP address -TEST_VNC_PORT=5900 # VNC port (default 5900) -TEST_VNC_PASSWORD=vnc_password # VNC password -``` - -**RDP Integration Tests:** - -```bash -TEST_RDP_HOST=192.168.1.101 # VM IP address -TEST_RDP_PORT=3389 # RDP port (default 3389) -TEST_RDP_USERNAME=your_username # Windows username -TEST_RDP_PASSWORD=your_password # Windows password -TEST_RDP_DOMAIN=COMPANY # Domain (optional) -``` - -**Patient Application Tests:** - -```bash -PATIENT_TEST_VM_HOST=192.168.1.100 # Patient app VM IP -PATIENT_TEST_VM_PORT=5900 # Connection port -PATIENT_TEST_VM_PASSWORD=password # Connection password -PATIENT_APP_NAME=EMR_System.exe # Patient application name -TEST_PATIENT_ID=PAT001234 # Test patient ID -TEST_PATIENT_NAME=John_Smith # Test patient name -TEST_PATIENT_MRN=12345678 # Test patient MRN -``` - -**Application Test Parameters:** - -```bash -TEST_APP_NAME=Calculator.exe # Target application to launch -TEST_BUTTON_TEXT=1 # Target button to click -SEARCH_FIELD_LABEL=Patient_ID # Input field label -SEARCH_BUTTON_TEXT=Search # Search button text -``` - -### โš ๏ธ Important Testing Notes - -1. **VM Setup Required**: Integration tests assume VMs are running and accessible -2. **Tests Will Fail**: Without proper VM configuration, integration tests will fail - this is expected -3. **Skip Markers**: Tests are marked with `@pytest.mark.skip` by default to prevent accidental VM connections -4. **Mock Tests First**: Always run unit tests with mocks first to verify system logic -5. **Environment Isolation**: Each test type uses different environment variable prefixes - -### ๐Ÿ” Test Development & Debugging - -```bash -# Run tests with detailed output -uv run pytest tests/test_clicking_unit_mocks.py -v -s - -# Run single test method -uv run pytest tests/test_clicking_unit_mocks.py::TestInputActionsMocking::test_mock_click_success -v - -# Run tests with coverage (if available) -uv run pytest tests/ --cov=src --cov-report=term-missing - -# Debug test collection issues -uv run pytest tests/ --collect-only -v -``` - -### ๐ŸŽฏ Recommended Test Workflow - -1. **Start with Unit Tests**: `uv run pytest tests/test_clicking_unit_mocks.py -v` -2. **Verify System Logic**: Ensure all mock-based tests pass -3. **Setup VM Environment**: Configure VM connection environment variables -4. **Enable Integration Tests**: Remove `@pytest.mark.skip` decorators as needed -5. **Run Integration Tests**: Test real VM connections and workflows -6. **Validate Patient Workflows**: Test healthcare application scenarios - -This testing approach ensures system reliability while providing flexibility for different deployment environments. - -## ๐Ÿ” Troubleshooting - -**Connection fails:** - -*VNC Issues:* - -- Verify VM IP and VNC port (default 5900) -- Check VNC server is running on target VM -- Test manual VNC connection first - -*RDP Issues:* - -- Verify VM IP and RDP port (default 3389) -- Check RDP is enabled on target VM -- Install dependencies: - - **Linux**: `sudo apt install freerdp2-x11 xvfb xdotool imagemagick scrot` - - **Mac**: `brew install freerdp imagemagick xorg-server xdotool xinit` (for isolated X11 display) - - Note: Requires Xvfb from xorg-server for proper VM display isolation - - May need additional setup if ImageMagick lacks XWD format support -- Test manual RDP connection: `xfreerdp /v:VM_IP /u:username` - -**Element not found:** - -- Check exact text matching (case-sensitive) -- Ensure application is fully loaded -- Try scrolling to make elements visible - -**Models not loading:** - -```bash -# Re-download AI models -uv run src/vision/setup_models.py -``` - -## ๐Ÿณ Docker Support - -The system includes Docker support with connection-specific optimized containers: - -### **Container Options:** - -- **`vm-automation:vnc`** - Lightweight VNC-only container -- **`vm-automation:rdp`** - RDP-optimized with all dependencies -- **`vm-automation:latest`** - General-purpose (supports both) - -### **Quick Run:** - -```bash -# Run containers directly -docker run --rm -it vm-automation:vnc -docker run --rm -it vm-automation:rdp -``` - -### **Docker Compose Usage:** - -**VNC Connection:** - -```bash -# Set environment variables -export VM_HOST=192.168.1.100 -export VM_PASSWORD=vnc_password -export TARGET_APP="Greenway Dev" - -# Run VNC-optimized container -docker compose --profile vnc up -``` - -**RDP Connection:** - -```bash -# Set environment variables -export VM_HOST=192.168.1.100 -export VM_USERNAME=your_username -export VM_PASSWORD=your_password -export RDP_DOMAIN=COMPANY -export TARGET_APP="Greenway Dev" - -# Run RDP-optimized container -docker compose --profile rdp up -``` - -**General-purpose container:** - -```bash -# Supports both connection types via environment -export CONNECTION_TYPE=rdp # or vnc -docker compose --profile general up -``` - -### **Production Docker Usage:** - -```bash -# With config file mount -docker run --rm -it \ - -v ./vm_config.json:/app/vm_config.json:ro \ - -v ./logs:/app/logs \ - -v ./screenshots:/app/screenshots \ - vm-automation:rdp - -# With environment variables -docker run --rm -it \ - -e VM_HOST=192.168.1.100 \ - -e VM_USERNAME=user \ - -e VM_PASSWORD=pass \ - -e CONNECTION_TYPE=rdp \ - -e TARGET_APP="Greenway Dev" \ - vm-automation:rdp -``` - -## ๐Ÿ“„ Available Commands - -| Command | Purpose | Use Case | -|---------|---------|----------| -| `uv run src/main.py` | Direct script execution | Development & testing | -| `uv run src/vm/main.py` | VM-specific entry point | VM automation only | -| `uv run src/vision/setup_models.py` | Download AI models | Initial setup | -| `uv run vm-automation` | Production CLI | Production deployment | -| `uv run vm-automation --connection rdp` | Use RDP connection | Windows VMs with RDP | -| `uv run vm-automation --connection vnc` | Use VNC connection | Any VM with VNC server | -| `uv run vm-automation --create-samples` | Generate config files | First-time setup | -| `uv run vm-automation --validate-env` | Environment check | Troubleshooting | -| `uv run vm-automation --health-check` | System health check | Monitoring | -| `uv run vm-automation --benchmark` | Performance benchmark | Optimization | - -## ๐Ÿ”ง Production Deployment - -### Environment Configuration - -**Production Environment Variables:** -```bash -# Core Configuration -export COMPUTER_USE_AGENT_ENV="production" -export LOG_LEVEL="INFO" -export MAX_CONCURRENT_SESSIONS=5 -export SESSION_TIMEOUT=3600 - -# Security -export ENABLE_AUDIT_LOGGING="true" -export AUDIT_LOG_PATH="/var/log/computer-use-agent/audit.log" -export ENCRYPT_SCREENSHOTS="true" - -# Performance -export OCR_CONFIDENCE_THRESHOLD="0.7" -export YOLO_CONFIDENCE_THRESHOLD="0.6" -export SCREENSHOT_CACHE_SIZE=100 -export ENABLE_GPU_ACCELERATION="false" # CPU-only for consistency - -# Healthcare/HIPAA -export HIPAA_COMPLIANCE_MODE="true" -export PHI_LOGGING_ENABLED="false" -export PATIENT_VERIFICATION_REQUIRED="true" -``` - -### Docker Production Setup - -**Multi-stage Production Dockerfile:** -```dockerfile -FROM python:3.11-slim as production - -# System dependencies -RUN apt-get update && apt-get install -y \ - libgl1-mesa-glx \ - libglib2.0-0 \ - libsm6 \ - libxext6 \ - libxrender-dev \ - libgomp1 \ - && rm -rf /var/lib/apt/lists/* - -# Install UV package manager -COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv - -# Copy application -WORKDIR /app -COPY . . - -# Install dependencies and download models -RUN uv sync --frozen -RUN uv run src/vision/setup_models.py - -# Create non-root user -RUN useradd -m -u 1000 agent -RUN chown -R agent:agent /app -USER agent - -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD uv run vm-automation --health-check - -EXPOSE 8000 -CMD ["uv", "run", "vm-automation", "--production"] -``` - -### Monitoring and Observability - -**Health Check Endpoint:** -```python -from agent.mcp import create_mcp_server - -# Add health monitoring -server = create_mcp_server( - enable_metrics=True, - metrics_port=9090, - health_check_endpoint="/health" -) - -# Custom health checks -def custom_health_check(): - """Custom health validation""" - try: - # Test OCR functionality - from vision import detect_ui_elements - import numpy as np - test_image = np.zeros((100, 100, 3), dtype=np.uint8) - detect_ui_elements(test_image) - - # Test model availability - from vision.setup_models import verify_models - models_status = verify_models() - - return { - "status": "healthy", - "ocr_functional": True, - "models_loaded": all(models_status.values()), - "timestamp": time.time() - } - except Exception as e: - return { - "status": "unhealthy", - "error": str(e), - "timestamp": time.time() - } -``` - -**Logging Configuration:** -```python -import logging -from logging.handlers import RotatingFileHandler - -# Production logging setup -def setup_production_logging(): - """Configure production-grade logging""" - - # Main application logger - logger = logging.getLogger("computer_use_agent") - logger.setLevel(logging.INFO) - - # Rotating file handler (100MB max, keep 5 files) - file_handler = RotatingFileHandler( - "logs/app.log", - maxBytes=100*1024*1024, - backupCount=5 - ) - file_handler.setFormatter(logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - )) - logger.addHandler(file_handler) - - # Audit logger (for compliance) - audit_logger = logging.getLogger("audit") - audit_handler = RotatingFileHandler( - "logs/audit.log", - maxBytes=50*1024*1024, - backupCount=10 - ) - audit_handler.setFormatter(logging.Formatter( - '%(asctime)s - AUDIT - %(message)s' - )) - audit_logger.addHandler(audit_handler) - - return logger, audit_logger -``` - -## ๐Ÿ“Š Performance Optimization - -### Benchmarking - -```bash -# Run performance benchmark -uv run vm-automation --benchmark - -# OCR-specific benchmarking -uv run python -c " -from vision.setup_models import verify_models -from vision import detect_ui_elements, extract_text -import time -import numpy as np - -# Generate test image -test_image = np.random.randint(0, 255, (1080, 1920, 3), dtype=np.uint8) - -# Benchmark detection -start_time = time.time() -elements = detect_ui_elements(test_image) -detection_time = time.time() - start_time - -# Benchmark OCR -start_time = time.time() -text_results = extract_text(test_image) -ocr_time = time.time() - start_time - -print(f'YOLO Detection: {detection_time:.2f}s') -print(f'OCR Processing: {ocr_time:.2f}s') -print(f'Total Elements Found: {len(elements)}') -print(f'Text Regions Found: {len(text_results)}') -" -``` - -### Resource Usage Monitoring - -```python -import psutil -import time -from dataclasses import dataclass - -@dataclass -class PerformanceMetrics: - cpu_percent: float - memory_mb: float - screenshot_fps: float - ocr_latency_ms: float - detection_latency_ms: float - -def monitor_performance(): - """Monitor system performance""" - process = psutil.Process() - - # CPU and Memory - cpu_percent = process.cpu_percent(interval=1) - memory_mb = process.memory_info().rss / 1024 / 1024 - - # Vision performance (simplified) - start_time = time.time() - # ... perform OCR operations ... - ocr_latency = (time.time() - start_time) * 1000 - - return PerformanceMetrics( - cpu_percent=cpu_percent, - memory_mb=memory_mb, - screenshot_fps=30.0, # Example - ocr_latency_ms=ocr_latency, - detection_latency_ms=150.0 # Example - ) -``` - ---- - -## โœ… Production Ready - -This system is ready for real-world deployment with comprehensive testing and quality assurance: - -### ๐Ÿงช **Test Coverage & Quality** -- **136 comprehensive unit tests** with mock-based isolation -- **100% coverage** on core automation modules (types, base classes) -- **75-100% coverage** on connection protocols (VNC, RDP) -- **GitHub Actions CI/CD** with automated testing on every commit -- **Code quality enforcement** (ruff linting, pyright type checking) -- **Security scanning** with vulnerability detection - -### ๐Ÿค– **AI-Ready Architecture** -- **Computer Vision**: YOLO + OCR for precise GUI automation -- **Multi-Agent Architecture**: Efficient component sharing between agents -- **MCP Protocol**: Model Context Protocol server for LLM integration -- **Function-based Interface**: Simple prompt-to-action for AI agents - -### ๐Ÿฅ **Enterprise Features** -- **Healthcare Safety**: Optional patient verification with HIPAA compliance -- **Production Logging**: Audit trails and comprehensive error handling -- **Configuration Management**: JSON and environment-based configuration -- **Performance Monitoring**: Built-in benchmarking and health checks -- **Container Support**: Docker deployment with optimized images - -### ๐Ÿ“Š **Quality Metrics** -- **42% overall code coverage** with 100% on critical paths -- **Cross-platform testing** (macOS, Linux, Windows VM support) -- **Error resilience** with comprehensive exception handling -- **Performance optimized** CPU-only inference for consistency diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..01f3b06 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +# Docker Compose for VM Automation with different connection types +version: '3.8' + +services: + # VNC-optimized container (lightweight) + vm-automation-vnc: + build: + context: . + dockerfile: Dockerfile.vnc + container_name: vm-automation-vnc + environment: + - CONNECTION_TYPE=vnc + - VM_HOST=${VM_HOST:-192.168.1.100} + - VM_PORT=${VM_PORT:-5900} + - VM_PASSWORD=${VM_PASSWORD:-} + - TARGET_APP=${TARGET_APP:-Greenway Dev} + - TARGET_BUTTON=${TARGET_BUTTON:-Submit} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - SAVE_SCREENSHOTS=${SAVE_SCREENSHOTS:-true} + volumes: + - ./vm_config.json:/app/vm_config.json:ro + - ./logs:/app/logs + - ./screenshots:/app/screenshots + profiles: + - vnc + +volumes: + logs: + driver: local + screenshots: + driver: local diff --git a/eval/.keep b/eval_legacy/.keep similarity index 100% rename from eval/.keep rename to eval_legacy/.keep diff --git a/eval_legacy/__init__.py b/eval_legacy/__init__.py new file mode 100644 index 0000000..e0dcda6 --- /dev/null +++ b/eval_legacy/__init__.py @@ -0,0 +1,48 @@ +"""Vision Evaluation Framework + +Enterprise-grade evaluation system for computer vision with: +- Industry-standard COCO detection metrics (pycocotools) +- Professional OCR evaluation (jiwer, torchmetrics) +- Robustness testing with Albumentations +- Interactive error analysis with FiftyOne +- Experiment tracking with Weights & Biases +- CI/CD integration with automated thresholds +- Performance profiling and optimization + +Main entry points: + eval.py - CLI evaluation harness + evaluator.py - Core evaluation engine + performance_profiler.py - Performance analysis + +Usage: + # Command line interface + python eval.py --test-data eval/test_data --output eval/results + python eval.py --ci --golden-set eval/golden_100 + + # Programmatic interface + from eval import VisionEvaluator, EvaluationConfig + config = EvaluationConfig(test_data_dir="data", output_dir="results") + evaluator = VisionEvaluator(config) + results = evaluator.run_comprehensive_evaluation() +""" + +# Main components +from .evaluator import ( + EvaluationConfig, + EvaluationResults, + VisionEvaluator, +) +from .performance_profiler import BenchmarkResults, PerformanceMetrics, PerformanceProfiler + +__version__ = "2.0.0" + +__all__ = [ + # Core evaluation + "VisionEvaluator", + "EvaluationConfig", + "EvaluationResults", + # Performance analysis + "PerformanceProfiler", + "PerformanceMetrics", + "BenchmarkResults", +] diff --git a/eval_legacy/coordinate_finder.py b/eval_legacy/coordinate_finder.py new file mode 100644 index 0000000..20aa3d7 --- /dev/null +++ b/eval_legacy/coordinate_finder.py @@ -0,0 +1,200 @@ +"""Interactive Coordinate Finder + +Click on images to get exact pixel coordinates for golden set creation. + +Usage: + python eval/coordinate_finder.py path/to/image.png +""" + +import argparse +import json +from pathlib import Path +from typing import Any + +import cv2 + + +class CoordinateFinder: + """Interactive tool to find coordinates by clicking on images""" + + def __init__(self): + self.coordinates = [] + self.current_image = None + self.image_path = None + + def mouse_callback(self, event, x, y, flags, param): + """Handle mouse clicks to record coordinates""" + if event == cv2.EVENT_LBUTTONDOWN: + print(f"\n๐Ÿ“ Clicked at: [{x}, {y}]") + + # Ask for details about this coordinate + target = input("Enter target name (e.g., 'Submit', 'save_icon'): ").strip() + if not target: + print("โŒ Skipping - no target name provided") + return + + method = input("Method (text/visual) [text]: ").strip().lower() + if not method: + method = "text" + + coord_data = {"method": method, "target": target, "expected_coord": [x, y]} + + if method == "visual": + visual_class = input("Visual class (button/icon/textbox) [button]: ").strip() + if not visual_class: + visual_class = "button" + coord_data["visual_class"] = visual_class + + self.coordinates.append(coord_data) + print(f"โœ… Added: {coord_data}") + + # Draw a marker on the image + cv2.circle(self.current_image, (x, y), 5, (0, 255, 0), -1) # Green circle + cv2.putText( + self.current_image, + target, + (x + 10, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (0, 255, 0), + 1, + ) + cv2.imshow("Coordinate Finder", self.current_image) + + def find_coordinates(self, image_path: Path) -> list[dict[str, Any]]: + """Interactive coordinate finding for an image""" + self.image_path = image_path + self.coordinates = [] + + # Load image + original_image = cv2.imread(str(image_path)) + if original_image is None: + raise ValueError(f"Could not load image: {image_path}") + + self.current_image = original_image.copy() + + print(f"\n๐Ÿ–ผ๏ธ Finding coordinates for: {image_path.name}") + print("๐Ÿ“ Instructions:") + print(" - Click on UI elements to record coordinates") + print(" - Enter target name and method for each click") + print(" - Press 's' to save and continue") + print(" - Press 'q' to quit without saving") + print(" - Press 'r' to reset all coordinates") + + # Setup window and mouse callback + cv2.namedWindow("Coordinate Finder", cv2.WINDOW_NORMAL) + cv2.resizeWindow("Coordinate Finder", 1200, 800) + cv2.setMouseCallback("Coordinate Finder", self.mouse_callback) + cv2.imshow("Coordinate Finder", self.current_image) + + while True: + key = cv2.waitKey(1) & 0xFF + + if key == ord("q"): + print("โŒ Quitting without saving") + break + elif key == ord("s"): + print("๐Ÿ’พ Saving coordinates...") + break + elif key == ord("r"): + print("๐Ÿ”„ Resetting coordinates...") + self.coordinates = [] + self.current_image = original_image.copy() + cv2.imshow("Coordinate Finder", self.current_image) + + cv2.destroyAllWindows() + return self.coordinates + + def save_coordinates(self, coordinates: list[dict[str, Any]], output_file: Path): + """Save coordinates to JSON file""" + coordinate_data = [] + for coord in coordinates: + coord_with_image = {"image_path": self.image_path.name, **coord} + coordinate_data.append(coord_with_image) + + with open(output_file, "w") as f: + json.dump(coordinate_data, f, indent=2) + + print(f"โœ… Coordinates saved to: {output_file}") + + def process_multiple_images(self, image_dir: Path, output_dir: Path): + """Process multiple images and create coordinate test data""" + image_dir = Path(image_dir) + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + image_files = list(image_dir.glob("*.png")) + list(image_dir.glob("*.jpg")) + if not image_files: + print(f"โŒ No images found in {image_dir}") + return + + all_coordinates = [] + + for i, image_path in enumerate(image_files): + print(f"\n[{i + 1}/{len(image_files)}] Processing {image_path.name}") + + coordinates = self.find_coordinates(image_path) + if coordinates: + all_coordinates.extend( + [{"image_path": image_path.name, **coord} for coord in coordinates] + ) + print(f" โœ… Added {len(coordinates)} coordinates") + else: + print(" โš ๏ธ No coordinates recorded") + + if all_coordinates: + output_file = output_dir / "coordinate_test_data.json" + with open(output_file, "w") as f: + json.dump(all_coordinates, f, indent=2) + + print(f"\n๐ŸŽ‰ Complete! Total coordinates: {len(all_coordinates)}") + print(f"๐Ÿ“ Saved to: {output_file}") + + # Display summary + print("\n๐Ÿ“‹ Summary:") + for coord in all_coordinates: + method = coord.get("method", "text") + target = coord.get("target", "unknown") + coords = coord.get("expected_coord", [0, 0]) + image = coord.get("image_path", "unknown") + print(f" {image}: {method} '{target}' -> [{coords[0]}, {coords[1]}]") + else: + print("โŒ No coordinates collected") + + +def main(): + parser = argparse.ArgumentParser(description="Interactive coordinate finder for golden sets") + parser.add_argument("input", help="Image file or directory containing images") + parser.add_argument( + "--output", + "-o", + default="./coordinates_output", + help="Output directory for coordinate data", + ) + + args = parser.parse_args() + + finder = CoordinateFinder() + input_path = Path(args.input) + + if input_path.is_file(): + # Single image + print(f"๐ŸŽฏ Finding coordinates for single image: {input_path}") + coordinates = finder.find_coordinates(input_path) + + if coordinates: + output_dir = Path(args.output) + output_dir.mkdir(parents=True, exist_ok=True) + finder.save_coordinates(coordinates, output_dir / "coordinate_test_data.json") + + elif input_path.is_dir(): + # Multiple images + print(f"๐ŸŽฏ Finding coordinates for images in: {input_path}") + finder.process_multiple_images(input_path, Path(args.output)) + + else: + print(f"โŒ Input path not found: {input_path}") + + +if __name__ == "__main__": + main() diff --git a/eval_legacy/eval.py b/eval_legacy/eval.py new file mode 100755 index 0000000..d210faa --- /dev/null +++ b/eval_legacy/eval.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +"""Evaluation CLI + +Single entry point for running comprehensive vision system evaluation. +Designed for both development and CI/CD integration. + +Usage: + python eval.py --test-data eval/test_data --output eval/results + python eval.py --ci --golden-set eval/golden_100 --output eval/ci_results + +This is the main evaluation harness that orchestrates: +- COCO detection evaluation (mAP metrics) +- OCR evaluation (CER/WER with jiwer/torchmetrics) +- Coordinate accuracy testing +- Robustness testing (Albumentations corruptions) +- Performance profiling +- FiftyOne dataset creation +- W&B experiment tracking +- CI threshold enforcement +""" + +import argparse +import json +import logging +import sys +import time +from pathlib import Path + +from eval.evaluator import ( + PRODUCTION_DEPS_AVAILABLE, + EvaluationConfig, + VisionEvaluator, +) + + +def setup_logging(output_dir: Path, verbose: bool = False) -> logging.Logger: + """Setup logging for evaluation""" + log_level = logging.DEBUG if verbose else logging.INFO + + # Create output directory + output_dir.mkdir(parents=True, exist_ok=True) + + # Setup logger + logger = logging.getLogger("eval") + logger.setLevel(log_level) + + # Console handler with colored output + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + + # File handler + file_handler = logging.FileHandler(output_dir / "evaluation.log") + file_handler.setLevel(logging.DEBUG) + + # Formatters + console_formatter = logging.Formatter("%(levelname)s: %(message)s") + file_formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + + console_handler.setFormatter(console_formatter) + file_handler.setFormatter(file_formatter) + + logger.addHandler(console_handler) + logger.addHandler(file_handler) + + return logger + + +def check_dependencies(): + """Check if all dependencies are available""" + if not PRODUCTION_DEPS_AVAILABLE: + print("โŒ Dependencies not available!") + print("\n๐Ÿ“ฆ Install required packages:") + print( + "pip install pycocotools torchmetrics jiwer fiftyone wandb albumentations torch torchvision" + ) + print("\n๐Ÿณ Or use Docker:") + print("docker run -it python:3.9 bash") + print("pip install -r requirements.txt") + return False + return True + + +def validate_test_data(test_data_dir: Path, logger: logging.Logger) -> bool: + """Validate test data structure""" + required_files = [ + "images/", + "annotations/ocr_ground_truth.json", + "annotations/coordinate_test_data.json", + ] + + missing_files = [] + for file_path in required_files: + full_path = test_data_dir / file_path + if not full_path.exists(): + missing_files.append(file_path) + + if missing_files: + logger.error(f"โŒ Missing required test data files: {missing_files}") + return False + + # Count images + image_dir = test_data_dir / "images" + image_count = len(list(image_dir.glob("*.png")) + list(image_dir.glob("*.jpg"))) + + if image_count == 0: + logger.error("โŒ No images found in test data") + return False + + logger.info(f"โœ… Test data validated: {image_count} images") + return True + + +def run_full_evaluation(config: EvaluationConfig, logger: logging.Logger): + """Run complete evaluation suite""" + logger.info("๐Ÿš€ Running full evaluation suite") + + evaluator = VisionEvaluator(config) + results = evaluator.run_comprehensive_evaluation() + + return results + + +def run_ci_evaluation(config: EvaluationConfig, logger: logging.Logger): + """Run CI evaluation with strict thresholds""" + logger.info("๐Ÿš€ Running CI evaluation with threshold enforcement") + + # Enable CI mode + config.enforce_thresholds = True + config.use_wandb = True # Log CI runs + + try: + evaluator = VisionEvaluator(config) + results = evaluator.run_comprehensive_evaluation() + + if results.passes_ci_thresholds: + logger.info("โœ… CI evaluation PASSED") + return 0 + else: + logger.error("โŒ CI evaluation FAILED") + for failure in results.threshold_failures: + logger.error(f" - {failure}") + return 1 + + except Exception as e: + logger.error(f"โŒ CI evaluation ERROR: {e}") + return 1 + + +def create_github_actions_workflow(output_dir: Path): + """Create GitHub Actions workflow for CI evaluation""" + workflow = { + "name": "Vision System Evaluation", + "on": {"push": {"branches": ["main", "develop"]}, "pull_request": {"branches": ["main"]}}, + "jobs": { + "vision-eval": { + "runs-on": "ubuntu-latest", + "timeout-minutes": 30, + "steps": [ + {"name": "Checkout code", "uses": "actions/checkout@v4"}, + { + "name": "Setup Python", + "uses": "actions/setup-python@v4", + "with": {"python-version": "3.9"}, + }, + {"name": "Install dependencies", "run": "pip install -r eval/requirements.txt"}, + { + "name": "Run evaluation", + "run": "python eval/eval.py --ci --golden-set eval/golden_100 --output eval/ci_results", + "env": {"WANDB_API_KEY": "${{ secrets.WANDB_API_KEY }}"}, + }, + { + "name": "Upload results", + "uses": "actions/upload-artifact@v4", + "if": "always()", + "with": {"name": "evaluation-results", "path": "eval/ci_results/"}, + }, + { + "name": "Comment PR", + "if": "github.event_name == 'pull_request'", + "uses": "actions/github-script@v7", + "with": { + "script": """ + const fs = require('fs'); + const path = 'eval/ci_results/eval_results.json'; + if (fs.existsSync(path)) { + const results = JSON.parse(fs.readFileSync(path)); + const summary = ` + ## ๐Ÿ” Vision System Evaluation Results + + | Metric | Value | Threshold | Status | + |--------|-------|-----------|---------| + | Detection mAP | ${results.detection_metrics.mAP.toFixed(3)} | 0.500 | ${results.detection_metrics.mAP >= 0.5 ? 'โœ…' : 'โŒ'} | + | OCR CER | ${results.ocr_metrics.CER.toFixed(3)} | 0.050 | ${results.ocr_metrics.CER <= 0.05 ? 'โœ…' : 'โŒ'} | + | Coordinate Accuracy | ${results.coordinate_metrics.accuracy.toFixed(3)} | 0.800 | ${results.coordinate_metrics.accuracy >= 0.8 ? 'โœ…' : 'โŒ'} | + + **Overall Status**: ${results.ci_status.passes_thresholds ? 'โœ… PASS' : 'โŒ FAIL'} + `; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: summary + }); + } + """ + }, + }, + ], + } + }, + } + + workflow_dir = output_dir / ".github" / "workflows" + workflow_dir.mkdir(parents=True, exist_ok=True) + + with open(workflow_dir / "vision-evaluation.yml", "w") as f: + import yaml + + yaml.dump(workflow, f, default_flow_style=False, sort_keys=False) + + print(f"๐Ÿ”ง GitHub Actions workflow created: {workflow_dir / 'vision-evaluation.yml'}") + + +def create_docker_setup(output_dir: Path): + """Create Docker setup for reproducible evaluation""" + dockerfile = """ +FROM python:3.9-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y \\ + libgl1-mesa-glx \\ + libglib2.0-0 \\ + libsm6 \\ + libxext6 \\ + libxrender-dev \\ + libgomp1 \\ + wget \\ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy requirements +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy evaluation code +COPY . . + +# Default command +CMD ["python", "eval/eval.py", "--help"] +""" + + docker_compose = """ +version: '3.8' + +services: + vision-eval: + build: . + volumes: + - ./eval:/app/eval + - ./src:/app/src + environment: + - WANDB_API_KEY=${WANDB_API_KEY} + command: python eval/eval.py --test-data eval/test_data --output eval/results + + vision-eval-ci: + build: . + volumes: + - ./eval:/app/eval + - ./src:/app/src + environment: + - WANDB_API_KEY=${WANDB_API_KEY} + command: python eval/eval.py --ci --golden-set eval/golden_100 --output eval/ci_results +""" + + # Save Docker files + with open(output_dir / "Dockerfile", "w") as f: + f.write(dockerfile) + + with open(output_dir / "docker-compose.yml", "w") as f: + f.write(docker_compose) + + print(f"๐Ÿณ Docker setup created in {output_dir}") + print("Build with: docker-compose build") + print("Run with: docker-compose up vision-eval") + + +def print_evaluation_summary(results, logger: logging.Logger): + """Print comprehensive evaluation summary""" + print(f"\n{'=' * 80}") + print("VISION SYSTEM EVALUATION SUMMARY") + print(f"{'=' * 80}") + + # Detection Results + print("\n๐ŸŽฏ DETECTION METRICS (COCO)") + print(f" mAP@[0.5:0.95]: {results.detection_map:.3f}") + print(f" mAP@0.5: {results.detection_map50:.3f}") + print(f" mAP@0.75: {results.detection_map75:.3f}") + print(f" mAR: {results.detection_mar:.3f}") + + if results.per_class_ap: + print(" Per-class AP:") + for class_name, ap in results.per_class_ap.items(): + print(f" {class_name}: {ap:.3f}") + + # OCR Results + print("\n๐Ÿ“ OCR METRICS (jiwer/torchmetrics)") + print(f" Character Error Rate (CER): {results.ocr_cer:.3f}") + print(f" Word Error Rate (WER): {results.ocr_wer:.3f}") + print(f" Word Accuracy: {results.ocr_word_accuracy:.3f}") + + # Coordinate Results + print("\n๐Ÿ“ COORDINATE METRICS") + print(f" Detection Rate: {results.coordinate_detection_rate:.3f}") + print(f" Accuracy (20px): {results.coordinate_accuracy:.3f}") + print(f" Avg Pixel Distance: {results.avg_pixel_distance:.1f}px") + + # Robustness Results + if results.robustness_scores: + print("\n๐Ÿ›ก๏ธ ROBUSTNESS METRICS") + for corruption, score in results.robustness_scores.items(): + print(f" {corruption}: {score:.3f}") + + # Performance Results + print("\nโšก PERFORMANCE METRICS") + print(f" Avg Inference Time: {results.avg_inference_time:.3f}s") + print(f" Peak Memory Usage: {results.peak_memory_usage:.1f}MB") + + # CI Status + print("\n๐Ÿ” CI STATUS") + if results.passes_ci_thresholds: + print(" โœ… PASSES all thresholds") + else: + print(" โŒ FAILS thresholds:") + for failure in results.threshold_failures: + print(f" - {failure}") + + # Summary Stats + print("\n๐Ÿ“Š SUMMARY") + print(f" Total Images: {results.total_images}") + print(f" Total Detections: {results.total_detections}") + print(f" Failed Images: {len(results.failed_images)}") + + print(f"\n{'=' * 80}") + + +def main(): + """Main CLI entry point""" + parser = argparse.ArgumentParser( + description="Vision System Evaluation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Full evaluation + python eval.py --test-data eval/test_data --output eval/results + + # CI evaluation with thresholds + python eval.py --ci --golden-set eval/golden_100 --output eval/ci + + # Generate Docker/CI setup + python eval.py --setup-ci --output . + + # Check dependencies only + python eval.py --check-deps + """, + ) + + # Data paths + parser.add_argument( + "--test-data", + type=Path, + default="eval/test_data", + help="Test data directory (default: eval/test_data)", + ) + parser.add_argument( + "--golden-set", type=Path, help="Golden dataset for CI (overrides --test-data)" + ) + parser.add_argument( + "--output", + type=Path, + default="eval/results", + help="Output directory (default: eval/results)", + ) + + # Evaluation modes + eval_group = parser.add_mutually_exclusive_group() + eval_group.add_argument( + "--ci", action="store_true", help="CI evaluation with threshold enforcement" + ) + + # Configuration + parser.add_argument( + "--confidence", type=float, default=0.5, help="Confidence threshold (default: 0.5)" + ) + parser.add_argument( + "--coordinate-tolerance", + type=int, + default=20, + help="Coordinate tolerance in pixels (default: 20)", + ) + + # CI thresholds + parser.add_argument( + "--min-map", type=float, default=0.5, help="Minimum detection mAP (default: 0.5)" + ) + parser.add_argument( + "--max-cer", type=float, default=0.05, help="Maximum OCR CER (default: 0.05)" + ) + parser.add_argument( + "--min-coord-acc", + type=float, + default=0.8, + help="Minimum coordinate accuracy (default: 0.8)", + ) + + # Experiment tracking + parser.add_argument("--wandb-project", default="vision-evaluation", help="W&B project name") + parser.add_argument("--wandb-entity", help="W&B entity/team name") + parser.add_argument("--no-wandb", action="store_true", help="Disable W&B logging") + + # Setup utilities + parser.add_argument( + "--setup-ci", action="store_true", help="Generate CI setup files (GitHub Actions, Docker)" + ) + parser.add_argument("--check-deps", action="store_true", help="Check dependencies only") + parser.add_argument( + "--create-requirements", action="store_true", help="Create requirements.txt file" + ) + + # Misc + parser.add_argument("--verbose", "-v", action="store_true", help="Verbose logging") + parser.add_argument("--no-robustness", action="store_true", help="Skip robustness testing") + + args = parser.parse_args() + + # Handle utility commands + if args.check_deps: + success = check_dependencies() + sys.exit(0 if success else 1) + + if args.create_requirements: + print("๐Ÿ“„ Dependencies are now managed in pyproject.toml") + print("Install evaluation dependencies with:") + print(" uv sync --group eval") + print(" # or") + print(" pip install -e .[eval]") + sys.exit(0) + + if args.setup_ci: + print("๐Ÿ”ง Generating CI setup files...") + create_github_actions_workflow(args.output) + create_docker_setup(args.output) + print("โœ… CI setup complete!") + sys.exit(0) + + # Setup logging + logger = setup_logging(args.output, args.verbose) + + # Check dependencies + if not check_dependencies(): + sys.exit(1) + + # Determine test data directory + test_data_dir = args.golden_set if args.golden_set else args.test_data + + # Validate test data + if not validate_test_data(test_data_dir, logger): + logger.error("Test data missing") + sys.exit(1) + + # Create evaluation configuration + config = EvaluationConfig( + test_data_dir=test_data_dir, + output_dir=args.output, + confidence_threshold=args.confidence, + coordinate_tolerance=args.coordinate_tolerance, + enable_robustness_testing=not args.no_robustness, + use_wandb=not args.no_wandb, + wandb_project=args.wandb_project, + wandb_entity=args.wandb_entity, + enforce_thresholds=args.ci, + min_detection_map=args.min_map, + max_ocr_cer=args.max_cer, + min_coordinate_accuracy=args.min_coord_acc, + ) + + # Run evaluation + start_time = time.time() + + try: + if args.ci: + exit_code = run_ci_evaluation(config, logger) + if exit_code == 0: + # Load results for display + results_file = args.output / "eval_results.json" + if results_file.exists(): + with open(results_file) as f: + results_dict = json.load(f) + + # Create minimal results object for display + class SimpleResults: + pass + + results = SimpleResults() + results.detection_map = results_dict["detection_metrics"]["mAP"] + results.ocr_cer = results_dict["ocr_metrics"]["CER"] + results.coordinate_accuracy = results_dict["coordinate_metrics"]["accuracy"] + results.passes_ci_thresholds = results_dict["ci_status"]["passes_thresholds"] + results.threshold_failures = results_dict["ci_status"]["failures"] + else: + results = None + else: + results = run_full_evaluation(config, logger) + exit_code = 0 + + # Print summary + if results and hasattr(results, "detection_map"): + print_evaluation_summary(results, logger) + + total_time = time.time() - start_time + logger.info(f"๐ŸŽ‰ Evaluation completed in {total_time:.1f}s") + + except KeyboardInterrupt: + logger.info("โŒ Evaluation interrupted by user") + exit_code = 130 + except Exception as e: + logger.error(f"โŒ Evaluation failed: {e}") + if args.verbose: + import traceback + + logger.error(traceback.format_exc()) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/eval_legacy/evaluator.py b/eval_legacy/evaluator.py new file mode 100644 index 0000000..4c3158d --- /dev/null +++ b/eval_legacy/evaluator.py @@ -0,0 +1,1061 @@ +"""Vision Evaluation System + +Based on industry best practices for computer vision evaluation: +- pycocotools for gold-standard detection metrics +- torchmetrics for robust OCR evaluation +- FiftyOne for error analysis +- Weights & Biases for experiment tracking +- Albumentations for robustness testing +- CI-ready with automated thresholds + +This replaces the basic evaluation framework with tooling. +""" + +import json +import logging +import time +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pycocotools.coco import COCO + +import cv2 +import numpy as np + +try: + # Core dependencies + import albumentations as A + import jiwer + import torchmetrics + from pycocotools import mask as maskUtils + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + from torchmetrics.detection import MeanAveragePrecision + from torchmetrics.text import CharErrorRate, WordErrorRate + + import wandb + + PRODUCTION_DEPS_AVAILABLE = True +except ImportError as e: + logging.warning(f"Core dependencies not available: {e}") + logging.warning("Install with: uv sync --group eval") + PRODUCTION_DEPS_AVAILABLE = False + +# FiftyOne is optional due to sse-starlette conflicts +try: + import fiftyone as fo + + FIFTYONE_AVAILABLE = True +except ImportError: + logging.info("FiftyOne not available (install separately: pip install fiftyone==1.8.0)") + FIFTYONE_AVAILABLE = False + +try: + # Import vision system + from src.vision import detect_ui_elements, extract_text, find_elements_by_text + + VISION_AVAILABLE = True +except ImportError: + logging.error("Vision system not available - check src/vision/ module") + VISION_AVAILABLE = False + + +@dataclass +class EvaluationConfig: + """Configuration for evaluation""" + + # Data paths + test_data_dir: Path + output_dir: Path + + # Evaluation settings + confidence_threshold: float = 0.5 + iou_threshold: float = 0.5 + coordinate_tolerance: int = 20 + + # Robustness testing + enable_robustness_testing: bool = True + corruption_severity: list[int] = None + + # Tracking + use_wandb: bool = True + wandb_project: str = "vision-evaluation" + wandb_entity: str | None = None + + # CI settings + enforce_thresholds: bool = False + min_detection_map: float = 0.5 + max_ocr_cer: float = 0.05 + min_coordinate_accuracy: float = 0.8 + + def __post_init__(self): + if self.corruption_severity is None: + self.corruption_severity = [1, 2, 3, 4, 5] + + self.test_data_dir = Path(self.test_data_dir) + self.output_dir = Path(self.output_dir) + + +@dataclass +class EvaluationResults: + """Comprehensive evaluation results""" + + # COCO Detection Results + detection_map: float + detection_map50: float + detection_map75: float + detection_mar: float + per_class_ap: dict[str, float] + + # OCR Results (jiwer/torchmetrics) + ocr_cer: float + ocr_wer: float + ocr_word_accuracy: float + + # Coordinate Results + coordinate_accuracy: float + coordinate_detection_rate: float + avg_pixel_distance: float + + # Robustness Results + robustness_scores: dict[str, float] + corruption_analysis: dict[str, dict[str, float]] + + # Performance Metrics + avg_inference_time: float + peak_memory_usage: float + + # Quality Metrics + total_images: int + total_detections: int + failed_images: list[str] + + # Threshold Status + passes_ci_thresholds: bool + threshold_failures: list[str] + + +class VisionEvaluator: + """Vision evaluation system""" + + def __init__(self, config: EvaluationConfig): + self.config = config + self.logger = self._setup_logging() + + if not PRODUCTION_DEPS_AVAILABLE: + raise RuntimeError( + "Dependencies not available. Install with: pip install pycocotools torchmetrics jiwer fiftyone wandb albumentations" + ) + + if not VISION_AVAILABLE: + raise RuntimeError("Vision system not available. Check src/vision/ module") + + # Initialize metrics + self.detection_metric = MeanAveragePrecision() + self.cer_metric = CharErrorRate() + self.wer_metric = WordErrorRate() + + # Initialize tracking + if config.use_wandb: + wandb.init( + project=config.wandb_project, entity=config.wandb_entity, config=config.__dict__ + ) + + # Setup output directory + config.output_dir.mkdir(parents=True, exist_ok=True) + + def _setup_logging(self) -> logging.Logger: + """Setup logging""" + logger = logging.getLogger("vision_eval") + logger.setLevel(logging.INFO) + + # Console handler + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + + # File handler + fh = logging.FileHandler(self.config.output_dir / "evaluation.log") + fh.setLevel(logging.DEBUG) + + # Formatter + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ch.setFormatter(formatter) + fh.setFormatter(formatter) + + logger.addHandler(ch) + logger.addHandler(fh) + + return logger + + def run_comprehensive_evaluation(self) -> EvaluationResults: + """Run complete evaluation pipeline""" + self.logger.info("๐Ÿš€ Starting vision evaluation") + start_time = time.time() + + try: + # Load and validate data + coco_gt, image_paths = self._load_coco_data() + self.logger.info(f"๐Ÿ“Š Loaded {len(image_paths)} images for evaluation") + + # Run inference on all images + all_detections, all_ocr_results, inference_times = self._run_inference_batch( + image_paths + ) + + # COCO Detection Evaluation + self.logger.info("๐ŸŽฏ Running COCO detection evaluation") + detection_results = self._evaluate_detection_coco(coco_gt, all_detections) + + # OCR Evaluation with professional metrics + self.logger.info("๐Ÿ“ Running OCR evaluation") + ocr_results = self._evaluate_ocr_professional(all_ocr_results) + + # Coordinate Accuracy Evaluation + self.logger.info("๐Ÿ“ Running coordinate accuracy evaluation") + coordinate_results = self._evaluate_coordinate_accuracy(image_paths) + + # Robustness Testing + robustness_results = {} + if self.config.enable_robustness_testing: + self.logger.info("๐Ÿ›ก๏ธ Running robustness evaluation") + robustness_results = self._evaluate_robustness(image_paths[:10]) # Sample for speed + + # Performance Analysis + performance_results = self._analyze_performance(inference_times) + + # Compile comprehensive results + results = EvaluationResults( + # Detection + detection_map=detection_results.get("mAP", 0.0), + detection_map50=detection_results.get("mAP@0.5", 0.0), + detection_map75=detection_results.get("mAP@0.75", 0.0), + detection_mar=detection_results.get("mAR", 0.0), + per_class_ap=detection_results.get("per_class_ap", {}), + # OCR + ocr_cer=ocr_results.get("cer", 1.0), + ocr_wer=ocr_results.get("wer", 1.0), + ocr_word_accuracy=ocr_results.get("word_accuracy", 0.0), + # Coordinates + coordinate_accuracy=coordinate_results.get("accuracy", 0.0), + coordinate_detection_rate=coordinate_results.get("detection_rate", 0.0), + avg_pixel_distance=coordinate_results.get("avg_distance", float("inf")), + # Robustness + robustness_scores=robustness_results.get("scores", {}), + corruption_analysis=robustness_results.get("corruption_analysis", {}), + # Performance + avg_inference_time=performance_results["avg_time"], + peak_memory_usage=performance_results["peak_memory"], + # Quality + total_images=len(image_paths), + total_detections=sum(len(dets) for dets in all_detections.values()), + failed_images=performance_results.get("failed_images", []), + # CI Status + passes_ci_thresholds=False, # Will be set below + threshold_failures=[], + ) + + # Check CI thresholds + results.passes_ci_thresholds, results.threshold_failures = self._check_ci_thresholds( + results + ) + + # Save results and create FiftyOne dataset + self._save_results(results, all_detections) + if FIFTYONE_AVAILABLE: + self._create_fiftyone_dataset(image_paths, all_detections, results) + else: + self.logger.info("Skipping FiftyOne dataset creation (not available)") + + # Log to W&B + if self.config.use_wandb: + self._log_to_wandb(results) + + total_time = time.time() - start_time + self.logger.info(f"โœ… Evaluation complete in {total_time:.1f}s") + + if self.config.enforce_thresholds and not results.passes_ci_thresholds: + self.logger.error(f"โŒ CI thresholds failed: {results.threshold_failures}") + raise ValueError("Evaluation failed CI thresholds") + + return results + + except Exception as e: + self.logger.error(f"โŒ Evaluation failed: {e}") + self.logger.error(traceback.format_exc()) + raise + + def _load_coco_data(self) -> tuple["COCO", list[Path]]: + """Load COCO ground truth data""" + # Check for COCO annotation file + coco_file = self.config.test_data_dir / "annotations" / "coco_ground_truth.json" + + if not coco_file.exists(): + self.logger.warning("COCO file not found, converting existing annotations") + self._convert_to_coco_format() + + # Load COCO ground truth + coco_gt = COCO(str(coco_file)) + + # Get image paths + image_dir = self.config.test_data_dir / "images" + image_paths = list(image_dir.glob("*.png")) + list(image_dir.glob("*.jpg")) + + return coco_gt, image_paths + + def _convert_to_coco_format(self): + """Convert existing annotations to COCO format""" + # Load existing UI detection ground truth + ui_gt_file = self.config.test_data_dir / "annotations" / "ui_detection_ground_truth.json" + + if not ui_gt_file.exists(): + raise FileNotFoundError(f"No ground truth found: {ui_gt_file}") + + with open(ui_gt_file) as f: + ui_annotations = json.load(f) + + # Convert to COCO format + coco_data = { + "images": [], + "annotations": [], + "categories": [ + {"id": 1, "name": "button"}, + {"id": 2, "name": "textbox"}, + {"id": 3, "name": "checkbox"}, + {"id": 4, "name": "label"}, + {"id": 5, "name": "dialog"}, + {"id": 6, "name": "menu"}, + {"id": 7, "name": "icon"}, + ], + } + + category_map = {cat["name"]: cat["id"] for cat in coco_data["categories"]} + + img_id = 1 + ann_id = 1 + + for filename, annotations in ui_annotations.items(): + # Add image + image_path = self.config.test_data_dir / "images" / filename + if image_path.exists(): + img = cv2.imread(str(image_path)) + height, width = img.shape[:2] + + coco_data["images"].append( + {"id": img_id, "file_name": filename, "width": width, "height": height} + ) + + # Add annotations + for ann in annotations: + x1, y1, x2, y2 = ann["x1"], ann["y1"], ann["x2"], ann["y2"] + width_box = x2 - x1 + height_box = y2 - y1 + + cat_id = category_map.get(ann.get("label", "button"), 1) + + coco_data["annotations"].append( + { + "id": ann_id, + "image_id": img_id, + "category_id": cat_id, + "bbox": [x1, y1, width_box, height_box], + "area": width_box * height_box, + "iscrowd": 0, + } + ) + ann_id += 1 + + img_id += 1 + + # Save COCO format + coco_output = self.config.test_data_dir / "annotations" / "coco_ground_truth.json" + coco_output.parent.mkdir(exist_ok=True) + + with open(coco_output, "w") as f: + json.dump(coco_data, f, indent=2) + + self.logger.info( + f"โœ… Converted to COCO format: {len(coco_data['images'])} images, {len(coco_data['annotations'])} annotations" + ) + + def _run_inference_batch(self, image_paths: list[Path]) -> tuple[dict, dict, list[float]]: + """Run inference on all images and collect results""" + all_detections = {} + all_ocr_results = {} + inference_times = [] + + for image_path in image_paths: + try: + start_time = time.time() + self.logger.debug(f"๐Ÿ” Processing: {image_path.name}") + + # Load image + image = cv2.imread(str(image_path)) + if image is None: + self.logger.warning(f"โŒ Failed to load image: {image_path}") + continue + + self.logger.debug(f" Image loaded: {image.shape} (Hร—Wร—C)") + + # Run detection + detections = detect_ui_elements( + image, confidence_threshold=self.config.confidence_threshold + ) + self.logger.debug( + f" Found {len(detections)} UI elements (threshold={self.config.confidence_threshold})" + ) + + # Run OCR + ocr_results = extract_text( + image, confidence_threshold=self.config.confidence_threshold + ) + self.logger.debug(f" Found {len(ocr_results)} text regions") + + inference_time = time.time() - start_time + inference_times.append(inference_time) + + # Store results + all_detections[image_path.name] = detections + all_ocr_results[image_path.name] = ocr_results + + self.logger.debug(f" Processed in {inference_time:.3f}s") + + except Exception as e: + self.logger.error(f"Failed to process {image_path}: {e}") + continue + + return all_detections, all_ocr_results, inference_times + + def _evaluate_detection_coco(self, coco_gt: "COCO", all_detections: dict) -> dict[str, float]: + """Evaluate detection using COCO metrics""" + # Log COCO ground truth info + ground_truth_files = [img["file_name"] for img in coco_gt.dataset["images"]] + categories = [cat["name"] for cat in coco_gt.dataset["categories"]] + + self.logger.info( + f"๐Ÿ“Š COCO Ground Truth: {len(ground_truth_files)} images, {len(categories)} categories" + ) + self.logger.debug(f" Categories: {categories}") + self.logger.debug(f" Expected files: {ground_truth_files}") + + # Log detection results + self.logger.info(f"๐Ÿ” Detection Results: {len(all_detections)} images processed") + for filename, detections in all_detections.items(): + self.logger.debug(f" {filename}: {len(detections)} detections") + for i, det in enumerate(detections): + self.logger.debug( + f" {i + 1}. {det.label} (conf={det.confidence:.3f}) bbox={det.bbox}" + ) + + # Convert predictions to COCO format + coco_predictions = [] + unmatched_files = [] + + for filename, detections in all_detections.items(): + # Get image info + img_info = None + for img in coco_gt.dataset["images"]: + if img["file_name"] == filename: + img_info = img + break + + if img_info is None: + unmatched_files.append(filename) + self.logger.debug(f" โŒ {filename}: No matching ground truth image found") + continue + else: + self.logger.debug( + f" โœ… {filename}: Matched to ground truth image ID {img_info['id']}" + ) + + # Convert detections + for det in detections: + x1, y1, x2, y2 = det.bbox + width = x2 - x1 + height = y2 - y1 + + # Map label to category ID + category_id = 1 # Default to button + original_category_id = category_id + for cat in coco_gt.dataset["categories"]: + if cat["name"].lower() in det.label.lower(): + category_id = cat["id"] + break + + if category_id != original_category_id: + self.logger.debug(f" Mapped '{det.label}' -> category {category_id}") + else: + self.logger.debug( + f" Using default category {category_id} for '{det.label}'" + ) + + coco_predictions.append( + { + "image_id": img_info["id"], + "category_id": category_id, + "bbox": [x1, y1, width, height], + "score": det.confidence, + } + ) + + if not coco_predictions: + total_images = len(coco_gt.dataset["images"]) + total_detections = sum(len(dets) for dets in all_detections.values()) + matched_images = len(all_detections) + + self.logger.warning( + f"No valid predictions for COCO evaluation. " + f"Found {total_detections} detections across {matched_images} images, " + f"but none matched the {total_images} ground truth images." + ) + + if unmatched_files: + self.logger.warning(f" Unmatched detection files: {unmatched_files}") + + if total_detections == 0: + self.logger.warning( + " No detections found - check confidence threshold or model performance" + ) + elif matched_images == 0: + self.logger.warning( + " Filename mismatch - detection files don't match ground truth filenames" + ) + + self.logger.info(" ๐Ÿ’ก Debug tips:") + self.logger.info(" - Run with --verbose to see detailed detection logs") + self.logger.info(" - Check image files exist in images/ directory") + self.logger.info(" - Verify ground truth annotations match image names") + + return {} + + # Save predictions temporarily + pred_file = self.config.output_dir / "coco_predictions.json" + with open(pred_file, "w") as f: + json.dump(coco_predictions, f) + + # Load predictions in COCO format + coco_dt = coco_gt.loadRes(str(pred_file)) + + # Run evaluation + coco_eval = COCOeval(coco_gt, coco_dt, "bbox") + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + # Extract metrics + results = { + "mAP": coco_eval.stats[0], # mAP@[0.5:0.95] + "mAP@0.5": coco_eval.stats[1], # mAP@0.5 + "mAP@0.75": coco_eval.stats[2], # mAP@0.75 + "mAR": coco_eval.stats[8], # mAR@[0.5:0.95] + } + + # Per-class AP + per_class_ap = {} + for cat_id, cat_info in enumerate(coco_gt.cats.values()): + if cat_id < len(coco_eval.eval["precision"]): + # Average over IoU thresholds and area ranges + precision = coco_eval.eval["precision"][cat_id, :, :, 0, 2] + ap = np.mean(precision[precision > -1]) + per_class_ap[cat_info["name"]] = ap if not np.isnan(ap) else 0.0 + + results["per_class_ap"] = per_class_ap + + return results + + def _evaluate_ocr_professional(self, all_ocr_results: dict) -> dict[str, float]: + """Evaluate OCR using jiwer and torchmetrics""" + # Load OCR ground truth + ocr_gt_file = self.config.test_data_dir / "annotations" / "ocr_ground_truth.json" + + if not ocr_gt_file.exists(): + self.logger.warning("OCR ground truth not found") + return {} + + with open(ocr_gt_file) as f: + ocr_ground_truth = json.load(f) + + # Collect predictions and ground truth + predictions = [] + references = [] + + for filename, ocr_results in all_ocr_results.items(): + if filename in ocr_ground_truth: + # Combine all detected text + predicted_text = " ".join([r.text for r in ocr_results]) + ground_truth_text = ocr_ground_truth[filename] + + predictions.append(predicted_text) + references.append(ground_truth_text) + + if not predictions: + self.logger.warning("No OCR predictions to evaluate") + return {} + + # Calculate metrics using jiwer (more robust than simple Levenshtein) + cer = jiwer.cer(references, predictions) + wer = jiwer.wer(references, predictions) + + # Word accuracy using jiwer + word_accuracy = 1.0 - wer + + # Additional metrics using torchmetrics + try: + # Convert to tensors for torchmetrics + cer_metric = CharErrorRate() + wer_metric = WordErrorRate() + + cer_torchmetrics = cer_metric(predictions, references).item() + wer_torchmetrics = wer_metric(predictions, references).item() + + # Use torchmetrics results (usually more accurate) + cer = cer_torchmetrics + wer = wer_torchmetrics + + except Exception as e: + self.logger.warning(f"TorchMetrics calculation failed: {e}") + + return { + "cer": cer, + "wer": wer, + "word_accuracy": word_accuracy, + "num_samples": len(predictions), + } + + def _evaluate_coordinate_accuracy(self, image_paths: list[Path]) -> dict[str, float]: + """Evaluate coordinate accuracy for automation""" + # Load coordinate test data + coord_file = self.config.test_data_dir / "annotations" / "coordinate_test_data.json" + + if not coord_file.exists(): + self.logger.warning("Coordinate test data not found") + return {} + + with open(coord_file) as f: + coord_tests = json.load(f) + + results = [] + + for test in coord_tests: + image_path = self.config.test_data_dir / "images" / test["image_path"] + + if not image_path.exists(): + continue + + image = cv2.imread(str(image_path)) + if image is None: + continue + + expected_coord = tuple(test["expected_coord"]) + target = test["target"] + method = test.get("method", "text") + + # Find element + predicted_coord = None + + if method == "text": + elements = find_elements_by_text( + image, target, confidence_threshold=self.config.confidence_threshold + ) + if elements: + predicted_coord = elements[0].center + + if predicted_coord: + # Calculate pixel distance + distance = np.sqrt( + (predicted_coord[0] - expected_coord[0]) ** 2 + + (predicted_coord[1] - expected_coord[1]) ** 2 + ) + + within_tolerance = distance <= self.config.coordinate_tolerance + + results.append( + {"found": True, "distance": distance, "within_tolerance": within_tolerance} + ) + else: + results.append( + {"found": False, "distance": float("inf"), "within_tolerance": False} + ) + + if not results: + return {} + + # Calculate metrics + detection_rate = sum(1 for r in results if r["found"]) / len(results) + accuracy = sum(1 for r in results if r["within_tolerance"]) / len(results) + + valid_distances = [r["distance"] for r in results if r["distance"] != float("inf")] + avg_distance = np.mean(valid_distances) if valid_distances else float("inf") + + return { + "detection_rate": detection_rate, + "accuracy": accuracy, + "avg_distance": avg_distance, + "num_tests": len(results), + } + + def _evaluate_robustness(self, image_paths: list[Path]) -> dict[str, Any]: + """Evaluate robustness using Albumentations corruptions""" + if not image_paths: + return {} + + # Define corruption transforms + corruptions = { + "blur": A.Compose([A.Blur(blur_limit=(3, 7), p=1.0)]), + "noise": A.Compose([A.GaussNoise(noise_scale_factor=0.3, p=1.0)]), + "brightness": A.Compose( + [A.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0, p=1.0)] + ), + "contrast": A.Compose( + [A.RandomBrightnessContrast(brightness_limit=0, contrast_limit=0.3, p=1.0)] + ), + "jpeg": A.Compose([A.ImageCompression(quality_range=(30, 80), p=1.0)]), + } + + baseline_scores = {} + corruption_scores = {} + + # Sample images for robustness testing + sample_images = image_paths[:5] # Test on subset for speed + + for image_path in sample_images: + image = cv2.imread(str(image_path)) + if image is None: + continue + + # Baseline performance + baseline_detections = detect_ui_elements( + image, confidence_threshold=self.config.confidence_threshold + ) + baseline_ocr = extract_text( + image, confidence_threshold=self.config.confidence_threshold + ) + + baseline_scores[image_path.name] = { + "detections": len(baseline_detections), + "ocr_results": len(baseline_ocr), + } + + # Test each corruption + for corruption_name, transform in corruptions.items(): + if corruption_name not in corruption_scores: + corruption_scores[corruption_name] = {} + + try: + # Apply corruption + corrupted = transform(image=image)["image"] + + # Run inference on corrupted image + corrupted_detections = detect_ui_elements( + corrupted, confidence_threshold=self.config.confidence_threshold + ) + corrupted_ocr = extract_text( + corrupted, confidence_threshold=self.config.confidence_threshold + ) + + # Calculate degradation + detection_ratio = len(corrupted_detections) / max(len(baseline_detections), 1) + ocr_ratio = len(corrupted_ocr) / max(len(baseline_ocr), 1) + + corruption_scores[corruption_name][image_path.name] = { + "detection_ratio": detection_ratio, + "ocr_ratio": ocr_ratio, + "avg_ratio": (detection_ratio + ocr_ratio) / 2, + } + + except Exception as e: + self.logger.warning( + f"Corruption {corruption_name} failed on {image_path.name}: {e}" + ) + continue + + # Aggregate results + robustness_summary = {} + for corruption_name, results in corruption_scores.items(): + if results: + avg_detection_ratio = np.mean([r["detection_ratio"] for r in results.values()]) + avg_ocr_ratio = np.mean([r["ocr_ratio"] for r in results.values()]) + avg_overall = np.mean([r["avg_ratio"] for r in results.values()]) + + robustness_summary[corruption_name] = { + "detection_robustness": avg_detection_ratio, + "ocr_robustness": avg_ocr_ratio, + "overall_robustness": avg_overall, + } + + return { + "scores": { + name: results["overall_robustness"] for name, results in robustness_summary.items() + }, + "corruption_analysis": robustness_summary, + "baseline_performance": baseline_scores, + } + + def _analyze_performance(self, inference_times: list[float]) -> dict[str, Any]: + """Analyze performance metrics""" + import psutil + + return { + "avg_time": np.mean(inference_times) if inference_times else 0.0, + "p50_time": np.percentile(inference_times, 50) if inference_times else 0.0, + "p95_time": np.percentile(inference_times, 95) if inference_times else 0.0, + "peak_memory": psutil.Process().memory_info().rss / 1024 / 1024, # MB + "num_processed": len(inference_times), + "failed_images": [], + } + + def _check_ci_thresholds(self, results: EvaluationResults) -> tuple[bool, list[str]]: + """Check if results pass CI thresholds""" + if not self.config.enforce_thresholds: + return True, [] + + failures = [] + + if results.detection_map < self.config.min_detection_map: + failures.append( + f"Detection mAP {results.detection_map:.3f} < {self.config.min_detection_map}" + ) + + if results.ocr_cer > self.config.max_ocr_cer: + failures.append(f"OCR CER {results.ocr_cer:.3f} > {self.config.max_ocr_cer}") + + if results.coordinate_accuracy < self.config.min_coordinate_accuracy: + failures.append( + f"Coordinate accuracy {results.coordinate_accuracy:.3f} < {self.config.min_coordinate_accuracy}" + ) + + return len(failures) == 0, failures + + def _save_results(self, results: EvaluationResults, all_detections: dict): + """Save comprehensive results""" + # Save JSON results + results_dict = { + "detection_metrics": { + "mAP": results.detection_map, + "mAP@0.5": results.detection_map50, + "mAP@0.75": results.detection_map75, + "mAR": results.detection_mar, + "per_class_ap": results.per_class_ap, + }, + "ocr_metrics": { + "CER": results.ocr_cer, + "WER": results.ocr_wer, + "word_accuracy": results.ocr_word_accuracy, + }, + "coordinate_metrics": { + "accuracy": results.coordinate_accuracy, + "detection_rate": results.coordinate_detection_rate, + "avg_pixel_distance": results.avg_pixel_distance, + }, + "robustness_metrics": results.robustness_scores, + "performance_metrics": { + "avg_inference_time": results.avg_inference_time, + "peak_memory_usage": results.peak_memory_usage, + }, + "ci_status": { + "passes_thresholds": results.passes_ci_thresholds, + "failures": results.threshold_failures, + }, + "summary": { + "total_images": results.total_images, + "total_detections": results.total_detections, + "failed_images": results.failed_images, + }, + } + + with open(self.config.output_dir / "results.json", "w") as f: + json.dump(results_dict, f, indent=2, default=str) + + # Save detailed predictions + with open(self.config.output_dir / "detailed_predictions.json", "w") as f: + # Convert detection objects to serializable format + serializable_detections = {} + for filename, detections in all_detections.items(): + serializable_detections[filename] = [ + { + "bbox": det.bbox, + "label": det.label, + "confidence": det.confidence, + "center": det.center, + } + for det in detections + ] + json.dump(serializable_detections, f, indent=2) + + self.logger.info(f"๐Ÿ“Š Results saved to {self.config.output_dir}") + + def _create_fiftyone_dataset( + self, image_paths: list[Path], all_detections: dict, results: EvaluationResults + ): + """Create FiftyOne dataset for interactive error analysis""" + if not FIFTYONE_AVAILABLE: + self.logger.warning("FiftyOne not available - skipping dataset creation") + return + + try: + # Create dataset + dataset = fo.Dataset(f"vision_eval_{int(time.time())}") + + samples = [] + for image_path in image_paths: + sample = fo.Sample(filepath=str(image_path)) + + # Add predictions + detections = all_detections.get(image_path.name, []) + fo_detections = [] + + for det in detections: + x1, y1, x2, y2 = det.bbox + + # Convert to FiftyOne format (normalized coordinates) + img = cv2.imread(str(image_path)) + if img is not None: + h, w = img.shape[:2] + + fo_detection = fo.Detection( + label=det.label, + bounding_box=[x1 / w, y1 / h, (x2 - x1) / w, (y2 - y1) / h], + confidence=det.confidence, + ) + fo_detections.append(fo_detection) + + sample["predictions"] = fo.Detections(detections=fo_detections) + + # Add metadata + sample["inference_time"] = ( + results.avg_inference_time + ) # Would be per-image in real implementation + sample["num_detections"] = len(detections) + + samples.append(sample) + + dataset.add_samples(samples) + + # Save dataset info + dataset_info = { + "name": dataset.name, + "num_samples": len(dataset), + "launch_command": f"fiftyone app launch --dataset-name {dataset.name}", + } + + with open(self.config.output_dir / "fiftyone_dataset.json", "w") as f: + json.dump(dataset_info, f, indent=2) + + self.logger.info(f"๐Ÿ“ฑ FiftyOne dataset created: {dataset.name}") + self.logger.info(f"Launch with: fiftyone app launch --dataset-name {dataset.name}") + + except Exception as e: + self.logger.warning(f"FiftyOne dataset creation failed: {e}") + + def _log_to_wandb(self, results: EvaluationResults): + """Log results to Weights & Biases""" + if not self.config.use_wandb: + return + + try: + # Log metrics + wandb.log( + { + "detection/mAP": results.detection_map, + "detection/mAP@0.5": results.detection_map50, + "detection/mAP@0.75": results.detection_map75, + "detection/mAR": results.detection_mar, + "ocr/CER": results.ocr_cer, + "ocr/WER": results.ocr_wer, + "ocr/word_accuracy": results.ocr_word_accuracy, + "coordinate/accuracy": results.coordinate_accuracy, + "coordinate/detection_rate": results.coordinate_detection_rate, + "coordinate/avg_pixel_distance": results.avg_pixel_distance, + "performance/avg_inference_time": results.avg_inference_time, + "performance/peak_memory_mb": results.peak_memory_usage, + "ci/passes_thresholds": results.passes_ci_thresholds, + } + ) + + # Log per-class AP + for class_name, ap in results.per_class_ap.items(): + wandb.log({f"detection/ap_{class_name}": ap}) + + # Log robustness scores + for corruption, score in results.robustness_scores.items(): + wandb.log({f"robustness/{corruption}": score}) + + # Log summary table + summary_data = [ + ["Metric", "Value", "Threshold", "Pass"], + [ + "Detection mAP", + f"{results.detection_map:.3f}", + f"{self.config.min_detection_map:.3f}", + results.detection_map >= self.config.min_detection_map, + ], + [ + "OCR CER", + f"{results.ocr_cer:.3f}", + f"{self.config.max_ocr_cer:.3f}", + results.ocr_cer <= self.config.max_ocr_cer, + ], + [ + "Coordinate Accuracy", + f"{results.coordinate_accuracy:.3f}", + f"{self.config.min_coordinate_accuracy:.3f}", + results.coordinate_accuracy >= self.config.min_coordinate_accuracy, + ], + ] + + wandb.log( + { + "evaluation_summary": wandb.Table( + columns=["Metric", "Value", "Threshold", "Pass"], + data=summary_data[1:], # Skip header + ) + } + ) + + self.logger.info("๐Ÿ“Š Results logged to W&B") + + except Exception as e: + self.logger.warning(f"W&B logging failed: {e}") + + +def main(): + """Main entry point for evaluation""" + import argparse + + parser = argparse.ArgumentParser(description="Vision Evaluation") + parser.add_argument("--test-data", default="eval/test_data", help="Test data directory") + parser.add_argument("--output", default="eval/results", help="Output directory") + parser.add_argument("--wandb-project", default="vision-evaluation", help="W&B project name") + parser.add_argument("--enforce-ci", action="store_true", help="Enforce CI thresholds") + parser.add_argument("--no-robustness", action="store_true", help="Skip robustness testing") + + args = parser.parse_args() + + config = EvaluationConfig( + test_data_dir=args.test_data, + output_dir=args.output, + wandb_project=args.wandb_project, + enforce_thresholds=args.enforce_ci, + enable_robustness_testing=not args.no_robustness, + ) + + evaluator = VisionEvaluator(config) + results = evaluator.run_comprehensive_evaluation() + + print(f"\n{'=' * 60}") + print("EVALUATION RESULTS") + print(f"{'=' * 60}") + print(f"Detection mAP: {results.detection_map:.3f}") + print(f"OCR CER: {results.ocr_cer:.3f}") + print(f"Coordinate Accuracy: {results.coordinate_accuracy:.3f}") + print(f"CI Status: {'โœ… PASS' if results.passes_ci_thresholds else 'โŒ FAIL'}") + if results.threshold_failures: + print("Failures:") + for failure in results.threshold_failures: + print(f" - {failure}") + + +if __name__ == "__main__": + main() diff --git a/eval_legacy/golden_set_creator.py b/eval_legacy/golden_set_creator.py new file mode 100644 index 0000000..922d003 --- /dev/null +++ b/eval_legacy/golden_set_creator.py @@ -0,0 +1,393 @@ +"""Golden Set Creation Tool + +This tool helps create golden sets for vision system evaluation. +It supports both text-based and visual element detection methods. + +Usage: + python eval/golden_set_creator.py --images path/to/screenshots --output eval/my_golden_set +""" + +import argparse +from pathlib import Path +from typing import Any + +import cv2 + +from eval.ground_truth_manager import GroundTruthManager + + +class GoldenSetCreator: + """Creates golden sets with both automated assistance and manual annotation""" + + def __init__(self, output_dir: Path): + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + (self.output_dir / "images").mkdir(exist_ok=True) + + self.gt_manager = GroundTruthManager(self.output_dir) + + # Try to import vision system for automated assistance + try: + from vision import ( + detect_ui_elements, + extract_text, + find_clickable_elements, + find_elements_by_text, + ) + + self.vision_available = True + self.detect_ui_elements = detect_ui_elements + self.extract_text = extract_text + self.find_elements_by_text = find_elements_by_text + self.find_clickable_elements = find_clickable_elements + print("โœ… Vision system available - will auto-generate initial annotations") + except ImportError as e: + self.vision_available = False + print(f"โš ๏ธ Vision system not available ({e}) - manual annotation mode") + + def create_from_screenshots(self, image_dir: Path, interactive: bool = True) -> dict[str, Any]: + """Create golden set from a directory of screenshots""" + + image_dir = Path(image_dir) + if not image_dir.exists(): + raise ValueError(f"Image directory {image_dir} does not exist") + + image_files = list(image_dir.glob("*.png")) + list(image_dir.glob("*.jpg")) + if not image_files: + raise ValueError(f"No PNG/JPG images found in {image_dir}") + + print(f"๐Ÿ“ธ Found {len(image_files)} images") + + ground_truth_data = {} + + for i, image_path in enumerate(image_files): + print(f"\n[{i + 1}/{len(image_files)}] Processing {image_path.name}") + + # Copy image to golden set + dest_path = self.output_dir / "images" / image_path.name + if not dest_path.exists(): + import shutil + + shutil.copy2(image_path, dest_path) + + # Create annotations + annotations = self._annotate_image(image_path, interactive) + ground_truth_data[image_path.name] = annotations + + # Save all ground truth data + self.gt_manager.save_all_formats(ground_truth_data) + + print(f"\nโœ… Golden set created in {self.output_dir}") + return ground_truth_data + + def _annotate_image(self, image_path: Path, interactive: bool) -> dict[str, Any]: + """Annotate a single image""" + + print(f" ๐Ÿ“‚ Loading image: {image_path}") + image = cv2.imread(str(image_path)) + if image is None: + raise ValueError(f"Could not load image {image_path}") + print(f" โœ… Image loaded: {image.shape}") + + annotations = {"ocr_text": "", "ui_elements": [], "coordinate_tests": []} + + if self.vision_available: + print(" ๐Ÿค– Starting auto-annotation...") + # Auto-generate initial annotations + annotations = self._auto_annotate(image, image_path.name) + print(" โœ… Auto-annotation complete") + + if interactive: + # Allow manual review/editing + annotations = self._manual_review(image, image_path.name, annotations) + + return annotations + + def _auto_annotate(self, image, image_name: str) -> dict[str, Any]: + """Use vision system to auto-generate annotations""" + + print(" ๐Ÿค– Auto-generating annotations...") + + # Extract OCR text ONCE and reuse + try: + print(" ๐Ÿ” Running OCR extraction...") + text_results = self.extract_text(image) + ocr_text = " ".join([result.text for result in text_results]) + print(f" โœ… OCR extracted: {len(text_results)} text regions") + except Exception as e: + print(f" โš ๏ธ OCR failed: {e}") + ocr_text = "" + text_results = [] + + # Detect UI elements + try: + print(" ๐ŸŽฏ Running UI element detection...") + ui_detections = self.detect_ui_elements(image) + ui_elements = [] + for detection in ui_detections: + ui_elements.append( + { + "x1": int(detection.bbox[0]), + "y1": int(detection.bbox[1]), + "x2": int(detection.bbox[2]), + "y2": int(detection.bbox[3]), + "label": detection.class_name, + "text": "", + "confidence": float(detection.confidence), + } + ) + print(f" โœ… UI detection found: {len(ui_detections)} elements") + except Exception as e: + print(f" โš ๏ธ UI detection failed: {e}") + ui_elements = [] + + # Generate coordinate tests for common targets + print(" ๐Ÿ“ Searching for coordinate test targets...") + coordinate_tests = [] + common_targets = [ + "Login", + "Submit", + "Cancel", + "OK", + "Yes", + "No", + "Save", + "Delete", + "Edit", + "Close", + "Back", + "Next", + "Settings", + "Menu", + ] + + # Search in already-extracted OCR results instead of re-running OCR + for target in common_targets: + try: + # Search in pre-extracted text results (no additional OCR calls) + target_lower = target.lower() + found_elements = [] + + for text_result in text_results: + if target_lower in text_result.text.lower(): + found_elements.append(text_result) + + if found_elements: + # Use the first match + center = found_elements[0].center + coordinate_tests.append( + { + "method": "text", + "target": target, + "expected_coord": [int(center[0]), int(center[1])], + } + ) + print(f" โœ… Found '{target}' at [{center[0]}, {center[1]}]") + except Exception as e: + print(f" โš ๏ธ Failed to find '{target}': {e}") + continue + + # Try to find clickable elements (visual detection) + try: + print(" ๐Ÿ” Searching for clickable elements...") + clickable = self.find_clickable_elements(image) + for i, element in enumerate(clickable[:5]): # Limit to first 5 + if hasattr(element, "visual_detection") and element.visual_detection: + detection = element.visual_detection + center_x = (detection.bbox[0] + detection.bbox[2]) // 2 + center_y = (detection.bbox[1] + detection.bbox[3]) // 2 + + coordinate_tests.append( + { + "method": "visual", + "target": f"{detection.class_name}_{i}", + "expected_coord": [int(center_x), int(center_y)], + "visual_class": detection.class_name, + } + ) + except Exception as e: + print(f" โš ๏ธ Clickable element detection failed: {e}") + + print(f" ๐Ÿ“ OCR: {len(ocr_text.split())} words") + print(f" ๐ŸŽฏ UI Elements: {len(ui_elements)}") + print(f" ๐Ÿ“ Coordinate Tests: {len(coordinate_tests)}") + + return { + "ocr_text": ocr_text, + "ui_elements": ui_elements, + "coordinate_tests": coordinate_tests, + } + + def _manual_review(self, image, image_name: str, annotations: dict[str, Any]) -> dict[str, Any]: + """Manual review and editing of annotations""" + + print(f" ๐Ÿ‘ค Manual review for {image_name}") + print(" Current annotations:") + print(f" OCR: {annotations['ocr_text'][:100]}...") + print(f" UI Elements: {len(annotations['ui_elements'])}") + print(f" Coordinate Tests: {len(annotations['coordinate_tests'])}") + + while True: + print("\n Options:") + print(" 1. Edit OCR text") + print(" 2. Add coordinate test (text-based)") + print(" 3. Add coordinate test (visual element)") + print(" 4. Remove coordinate test") + print(" 5. View current coordinate tests") + print(" 6. Done with this image") + + choice = input(" Choose option (1-6): ").strip() + + if choice == "1": + print(f" Current OCR: {annotations['ocr_text']}") + new_text = input(" Enter corrected OCR text (or press Enter to keep): ") + if new_text.strip(): + annotations["ocr_text"] = new_text + + elif choice == "2": + target = input(" Enter text to find (e.g., 'Submit', 'Login'): ") + x = input(" Enter X coordinate: ") + y = input(" Enter Y coordinate: ") + try: + annotations["coordinate_tests"].append( + {"method": "text", "target": target, "expected_coord": [int(x), int(y)]} + ) + print(f" โœ… Added text-based test for '{target}'") + except ValueError: + print(" โŒ Invalid coordinates") + + elif choice == "3": + visual_class = input(" Enter visual element class (e.g., 'button', 'textbox'): ") + description = input(" Enter description (e.g., 'submit_button_1'): ") + x = input(" Enter X coordinate: ") + y = input(" Enter Y coordinate: ") + try: + annotations["coordinate_tests"].append( + { + "method": "visual", + "target": description, + "expected_coord": [int(x), int(y)], + "visual_class": visual_class, + } + ) + print(f" โœ… Added visual test for '{description}'") + except ValueError: + print(" โŒ Invalid coordinates") + + elif choice == "4": + self._remove_coordinate_test(annotations) + + elif choice == "5": + self._show_coordinate_tests(annotations) + + elif choice == "6": + break + + else: + print(" โŒ Invalid choice") + + return annotations + + def _show_coordinate_tests(self, annotations: dict[str, Any]): + """Show current coordinate tests""" + tests = annotations["coordinate_tests"] + if not tests: + print(" No coordinate tests defined") + return + + print(" Current coordinate tests:") + for i, test in enumerate(tests): + method = test.get("method", "text") + target = test.get("target", "unknown") + coord = test.get("expected_coord", [0, 0]) + print(f" {i + 1}. {method}: '{target}' -> [{coord[0]}, {coord[1]}]") + + def _remove_coordinate_test(self, annotations: dict[str, Any]): + """Remove a coordinate test""" + tests = annotations["coordinate_tests"] + if not tests: + print(" No coordinate tests to remove") + return + + self._show_coordinate_tests(annotations) + try: + idx = int(input(" Enter test number to remove: ")) - 1 + if 0 <= idx < len(tests): + removed = tests.pop(idx) + print(f" โœ… Removed test: {removed['target']}") + else: + print(" โŒ Invalid test number") + except ValueError: + print(" โŒ Invalid input") + + def create_template(self, image_paths: list[str]) -> dict[str, Any]: + """Create a template golden set for manual completion""" + + ground_truth_data = {} + + for image_path in image_paths: + image_name = Path(image_path).name + + # Copy image + dest_path = self.output_dir / "images" / image_name + if not dest_path.exists(): + import shutil + + shutil.copy2(image_path, dest_path) + + # Create template + ground_truth_data[image_name] = { + "ocr_text": "TODO: Add expected OCR text from this image", + "ui_elements": [], # Skip detailed bounding boxes for manual creation + "coordinate_tests": [ + { + "method": "text", + "target": "TODO: button_text", + "expected_coord": [0, 0], # TODO: Add actual coordinates + } + ], + } + + # Save template + self.gt_manager.save_all_formats(ground_truth_data) + + print(f"โœ… Template created in {self.output_dir}") + print("๐Ÿ“ Edit the JSON files in annotations/ to complete the golden set") + + return ground_truth_data + + +def main(): + parser = argparse.ArgumentParser(description="Create golden sets for vision evaluation") + parser.add_argument( + "--images", type=Path, required=True, help="Directory containing screenshot images" + ) + parser.add_argument( + "--output", type=Path, required=True, help="Output directory for golden set" + ) + parser.add_argument( + "--interactive", action="store_true", help="Enable interactive annotation mode" + ) + parser.add_argument( + "--template-only", action="store_true", help="Create template only (no auto-annotation)" + ) + + args = parser.parse_args() + + creator = GoldenSetCreator(args.output) + + if args.template_only: + # Create template for manual completion + image_files = list(args.images.glob("*.png")) + list(args.images.glob("*.jpg")) + creator.create_template([str(f) for f in image_files]) + else: + # Create with auto-annotation + creator.create_from_screenshots(args.images, interactive=args.interactive) + + print("\n๐ŸŽฏ Next steps:") + print(f"1. Review annotations in {args.output}/annotations/") + print(f"2. Test golden set: uv run eval-vision --golden-set {args.output}") + + +if __name__ == "__main__": + main() diff --git a/eval_legacy/ground_truth_manager.py b/eval_legacy/ground_truth_manager.py new file mode 100644 index 0000000..38c4373 --- /dev/null +++ b/eval_legacy/ground_truth_manager.py @@ -0,0 +1,75 @@ +"""Ground Truth Management + +Handles saving and loading ground truth annotations in different formats +for various evaluation components. +""" + +import json +from pathlib import Path +from typing import Any + + +class GroundTruthManager: + """Manages ground truth annotations in multiple formats""" + + def __init__(self, output_dir: Path): + self.output_dir = Path(output_dir) + self.annotations_dir = self.output_dir / "annotations" + self.annotations_dir.mkdir(parents=True, exist_ok=True) + + def save_all_formats(self, ground_truth_data: dict[str, dict[str, Any]]): + """Save ground truth in all required formats""" + self._save_ocr_ground_truth(ground_truth_data) + self._save_ui_detection_ground_truth(ground_truth_data) + self._save_coordinate_test_data(ground_truth_data) + self._save_complete_ground_truth(ground_truth_data) + + def _save_ocr_ground_truth(self, data: dict[str, dict[str, Any]]): + """Save OCR ground truth (filename -> text)""" + ocr_gt = {filename: gt_data["ocr_text"] for filename, gt_data in data.items()} + + with open(self.annotations_dir / "ocr_ground_truth.json", "w") as f: + json.dump(ocr_gt, f, indent=2) + + def _save_ui_detection_ground_truth(self, data: dict[str, dict[str, Any]]): + """Save UI detection ground truth (filename -> bounding boxes)""" + ui_gt = {filename: gt_data["ui_elements"] for filename, gt_data in data.items()} + + with open(self.annotations_dir / "ui_detection_ground_truth.json", "w") as f: + json.dump(ui_gt, f, indent=2) + + def _save_coordinate_test_data(self, data: dict[str, dict[str, Any]]): + """Save coordinate test data (list of test cases)""" + coord_tests = [] + + for filename, gt_data in data.items(): + for test in gt_data["coordinate_tests"]: + coord_tests.append({"image_path": filename, **test}) + + with open(self.annotations_dir / "coordinate_test_data.json", "w") as f: + json.dump(coord_tests, f, indent=2) + + def _save_complete_ground_truth(self, data: dict[str, dict[str, Any]]): + """Save complete ground truth (for reference)""" + with open(self.annotations_dir / "complete_ground_truth.json", "w") as f: + json.dump(data, f, indent=2) + + def load_ocr_ground_truth(self) -> dict[str, str]: + """Load OCR ground truth""" + with open(self.annotations_dir / "ocr_ground_truth.json") as f: + return json.load(f) + + def load_ui_detection_ground_truth(self) -> dict[str, list[dict]]: + """Load UI detection ground truth""" + with open(self.annotations_dir / "ui_detection_ground_truth.json") as f: + return json.load(f) + + def load_coordinate_test_data(self) -> list[dict[str, Any]]: + """Load coordinate test data""" + with open(self.annotations_dir / "coordinate_test_data.json") as f: + return json.load(f) + + def create_dataset_manifest(self, generated_files: dict[str, list[str]]): + """Create a manifest of generated files""" + with open(self.output_dir / "dataset_manifest.json", "w") as f: + json.dump(generated_files, f, indent=2) diff --git a/eval_legacy/performance_profiler.py b/eval_legacy/performance_profiler.py new file mode 100644 index 0000000..814d827 --- /dev/null +++ b/eval_legacy/performance_profiler.py @@ -0,0 +1,720 @@ +"""Performance Profiling and Benchmarking + +Advanced performance analysis for the vision system including: +- Detailed timing breakdowns (preprocessing, inference, postprocessing) +- Memory usage profiling with peak detection +- GPU utilization monitoring (if available) +- Throughput analysis under different loads +- Bottleneck identification +- Performance regression detection + +This complements the main evaluation with detailed performance metrics. +""" + +import gc +import json +import sys +import time +import tracemalloc +from dataclasses import dataclass +from pathlib import Path + +import cv2 +import numpy as np +import psutil + +try: + import torch + + TORCH_AVAILABLE = True +except ImportError: + TORCH_AVAILABLE = False + +try: + import GPUtil + + GPU_MONITORING_AVAILABLE = True +except ImportError: + GPU_MONITORING_AVAILABLE = False + + +try: + # Import vision system + from src.vision import detect_ui_elements, extract_text, find_elements_by_text + + VISION_AVAILABLE = True +except ImportError: + VISION_AVAILABLE = False + + +@dataclass +class PerformanceMetrics: + """Comprehensive performance metrics""" + + # Timing (seconds) + total_time: float + preprocessing_time: float + inference_time: float + postprocessing_time: float + + # Memory (MB) + peak_memory_usage: float + memory_delta: float + + # GPU metrics (if available) + gpu_memory_used: float | None = None + gpu_utilization: float | None = None + + # Throughput + images_per_second: float = 0.0 + + # Quality metrics + num_detections: int = 0 + num_text_elements: int = 0 + + # Per-stage breakdown + stage_timings: dict[str, float] = None + + def __post_init__(self): + if self.stage_timings is None: + self.stage_timings = {} + + +@dataclass +class BenchmarkResults: + """Benchmark results across different conditions""" + + single_image_metrics: PerformanceMetrics + batch_metrics: list[PerformanceMetrics] + load_test_metrics: dict[str, PerformanceMetrics] + memory_profile: dict[str, float] + bottleneck_analysis: dict[str, str] + recommendations: list[str] + + +class PerformanceProfiler: + """Advanced performance profiling for vision system""" + + def __init__(self, enable_gpu_monitoring: bool = True): + self.enable_gpu_monitoring = enable_gpu_monitoring and GPU_MONITORING_AVAILABLE + self.process = psutil.Process() + + if not VISION_AVAILABLE: + raise RuntimeError("Vision system not available") + + # Warmup models + self._warmup_models() + + def _warmup_models(self): + """Warmup vision models to get accurate timing""" + dummy_image = np.ones((480, 640, 3), dtype=np.uint8) * 128 + + # Warmup detection + try: + detect_ui_elements(dummy_image, confidence_threshold=0.5) + except Exception: + pass + + # Warmup OCR + try: + extract_text(dummy_image, confidence_threshold=0.5) + except Exception: + pass + + def profile_single_image( + self, image: np.ndarray, detailed_breakdown: bool = True + ) -> PerformanceMetrics: + """Profile performance on a single image with detailed breakdown""" + + # Start memory tracking + tracemalloc.start() + initial_memory = self.process.memory_info().rss / 1024 / 1024 # MB + + # GPU baseline (if available) + initial_gpu_memory = None + if self.enable_gpu_monitoring and torch.cuda.is_available(): + initial_gpu_memory = torch.cuda.memory_allocated() / 1024 / 1024 + + total_start = time.time() + stage_timings = {} + + # Preprocessing + prep_start = time.time() + # Image is already loaded, but we could add preprocessing steps here + preprocessed_image = image.copy() + stage_timings["preprocessing"] = time.time() - prep_start + + # Detection inference + det_start = time.time() + detections = detect_ui_elements(preprocessed_image, confidence_threshold=0.5) + stage_timings["detection"] = time.time() - det_start + + # OCR inference + ocr_start = time.time() + text_results = extract_text(preprocessed_image, confidence_threshold=0.5) + stage_timings["ocr"] = time.time() - ocr_start + + # Postprocessing (combining results, filtering, etc.) + post_start = time.time() + # Simulate some postprocessing + combined_results = { + "detections": detections, + "text_results": text_results, + "metadata": { + "image_shape": image.shape, + "num_detections": len(detections), + "num_text_elements": len(text_results), + }, + } + stage_timings["postprocessing"] = time.time() - post_start + + total_time = time.time() - total_start + + # Memory analysis + current_memory, peak_memory = tracemalloc.get_traced_memory() + tracemalloc.stop() + + final_memory = self.process.memory_info().rss / 1024 / 1024 + memory_delta = final_memory - initial_memory + peak_memory_mb = peak_memory / 1024 / 1024 + + # GPU metrics + gpu_memory_used = None + gpu_utilization = None + + if self.enable_gpu_monitoring: + if torch.cuda.is_available(): + final_gpu_memory = torch.cuda.memory_allocated() / 1024 / 1024 + gpu_memory_used = final_gpu_memory - (initial_gpu_memory or 0) + + try: + gpus = GPUtil.getGPUs() + if gpus: + gpu_utilization = gpus[0].load * 100 + except Exception: + pass + + # Additional detailed breakdown if requested + if detailed_breakdown: + # Text search timing + search_start = time.time() + find_elements_by_text(image, "test", confidence_threshold=0.5) + stage_timings["text_search"] = time.time() - search_start + + return PerformanceMetrics( + total_time=total_time, + preprocessing_time=stage_timings.get("preprocessing", 0), + inference_time=stage_timings.get("detection", 0) + stage_timings.get("ocr", 0), + postprocessing_time=stage_timings.get("postprocessing", 0), + peak_memory_usage=peak_memory_mb, + memory_delta=memory_delta, + gpu_memory_used=gpu_memory_used, + gpu_utilization=gpu_utilization, + images_per_second=1.0 / total_time, + num_detections=len(detections), + num_text_elements=len(text_results), + stage_timings=stage_timings, + ) + + def profile_batch_processing( + self, images: list[np.ndarray], batch_sizes: list[int] = None + ) -> list[PerformanceMetrics]: + """Profile batch processing with different batch sizes""" + + if batch_sizes is None: + batch_sizes = [1, 5, 10, 20] + + results = [] + + for batch_size in batch_sizes: + if batch_size > len(images): + continue + + batch_images = images[:batch_size] + + # Batch processing timing + start_time = time.time() + initial_memory = self.process.memory_info().rss / 1024 / 1024 + + batch_detections = [] + batch_text_results = [] + + for img in batch_images: + detections = detect_ui_elements(img, confidence_threshold=0.5) + text_results = extract_text(img, confidence_threshold=0.5) + batch_detections.append(detections) + batch_text_results.append(text_results) + + total_time = time.time() - start_time + final_memory = self.process.memory_info().rss / 1024 / 1024 + + # Calculate metrics + total_detections = sum(len(dets) for dets in batch_detections) + total_text_elements = sum(len(texts) for texts in batch_text_results) + + metrics = PerformanceMetrics( + total_time=total_time, + preprocessing_time=0, # Not tracked in batch mode + inference_time=total_time, # Simplified for batch + postprocessing_time=0, + peak_memory_usage=final_memory, + memory_delta=final_memory - initial_memory, + images_per_second=batch_size / total_time, + num_detections=total_detections, + num_text_elements=total_text_elements, + stage_timings={"batch_size": batch_size}, + ) + + results.append(metrics) + + return results + + def run_load_test( + self, image: np.ndarray, duration_seconds: int = 60, concurrent_requests: list[int] = None + ) -> dict[str, PerformanceMetrics]: + """Run load test to find performance under stress""" + + if concurrent_requests is None: + concurrent_requests = [1, 2, 5, 10] + + results = {} + + for concurrency in concurrent_requests: + print(f"Running load test with {concurrency} concurrent requests...") + + start_time = time.time() + initial_memory = self.process.memory_info().rss / 1024 / 1024 + + total_requests = 0 + total_detections = 0 + total_text_elements = 0 + + # Simulate concurrent load by rapid sequential processing + # (True concurrency would require threading/multiprocessing) + while (time.time() - start_time) < duration_seconds: + for _ in range(concurrency): + detections = detect_ui_elements(image, confidence_threshold=0.5) + text_results = extract_text(image, confidence_threshold=0.5) + + total_requests += 1 + total_detections += len(detections) + total_text_elements += len(text_results) + + # Small delay to prevent overwhelming + time.sleep(0.01) + + total_time = time.time() - start_time + final_memory = self.process.memory_info().rss / 1024 / 1024 + + metrics = PerformanceMetrics( + total_time=total_time, + preprocessing_time=0, + inference_time=total_time, + postprocessing_time=0, + peak_memory_usage=final_memory, + memory_delta=final_memory - initial_memory, + images_per_second=total_requests / total_time, + num_detections=total_detections, + num_text_elements=total_text_elements, + stage_timings={ + "concurrency": concurrency, + "total_requests": total_requests, + "requests_per_second": total_requests / total_time, + }, + ) + + results[f"concurrency_{concurrency}"] = metrics + + return results + + def analyze_bottlenecks(self, metrics: PerformanceMetrics) -> dict[str, str]: + """Analyze performance bottlenecks and provide insights""" + analysis = {} + + # Time distribution analysis + total_inference = metrics.inference_time + if total_inference > 0: + preprocessing_pct = (metrics.preprocessing_time / metrics.total_time) * 100 + inference_pct = (total_inference / metrics.total_time) * 100 + postprocessing_pct = (metrics.postprocessing_time / metrics.total_time) * 100 + + if preprocessing_pct > 20: + analysis["preprocessing"] = ( + f"High preprocessing overhead ({preprocessing_pct:.1f}%)" + ) + + if inference_pct > 70: + analysis["inference"] = f"Inference dominates ({inference_pct:.1f}%)" + + if postprocessing_pct > 15: + analysis["postprocessing"] = ( + f"High postprocessing overhead ({postprocessing_pct:.1f}%)" + ) + + # Memory analysis + if metrics.memory_delta > 100: # > 100MB + analysis["memory"] = f"High memory usage ({metrics.memory_delta:.1f}MB delta)" + + if metrics.peak_memory_usage > 500: # > 500MB + analysis["peak_memory"] = f"High peak memory ({metrics.peak_memory_usage:.1f}MB)" + + # Throughput analysis + if metrics.images_per_second < 1.0: + analysis["throughput"] = f"Low throughput ({metrics.images_per_second:.2f} img/s)" + + # GPU analysis + if metrics.gpu_memory_used and metrics.gpu_memory_used > 1000: # > 1GB + analysis["gpu_memory"] = f"High GPU memory usage ({metrics.gpu_memory_used:.1f}MB)" + + return analysis + + def generate_recommendations( + self, + single_metrics: PerformanceMetrics, + batch_metrics: list[PerformanceMetrics], + bottlenecks: dict[str, str], + ) -> list[str]: + """Generate performance optimization recommendations""" + recommendations = [] + + # Batch processing recommendations + if len(batch_metrics) > 1: + best_batch = max(batch_metrics, key=lambda m: m.images_per_second) + best_batch_size = best_batch.stage_timings.get("batch_size", 1) + + if best_batch_size > 1: + efficiency_gain = best_batch.images_per_second / single_metrics.images_per_second + recommendations.append( + f"Use batch processing with size {best_batch_size} for {efficiency_gain:.1f}x throughput improvement" + ) + + # Memory optimization + if "memory" in bottlenecks or "peak_memory" in bottlenecks: + recommendations.extend( + [ + "Consider image preprocessing to reduce memory usage", + "Implement batch size limiting based on available memory", + "Use memory-efficient data types (e.g., uint8 vs float32)", + ] + ) + + # Inference optimization + if "inference" in bottlenecks: + recommendations.extend( + [ + "Consider model quantization for faster inference", + "Optimize confidence thresholds to reduce computation", + "Explore GPU acceleration if not already using", + "Consider model pruning or distillation", + ] + ) + + # Throughput optimization + if single_metrics.images_per_second < 5.0: + recommendations.extend( + [ + "Profile individual model components", + "Consider asynchronous processing pipeline", + "Implement result caching for repeated images", + ] + ) + + # GPU optimization + if TORCH_AVAILABLE and torch.cuda.is_available(): + if not single_metrics.gpu_utilization or single_metrics.gpu_utilization < 50: + recommendations.append( + "GPU underutilized - consider batch processing or mixed precision" + ) + + return recommendations + + def run_comprehensive_benchmark( + self, test_images: list[np.ndarray], output_dir: Path + ) -> BenchmarkResults: + """Run comprehensive performance benchmark""" + + print("๐Ÿš€ Starting comprehensive performance benchmark...") + + # Single image profiling + print("๐Ÿ“Š Profiling single image performance...") + single_metrics = self.profile_single_image(test_images[0], detailed_breakdown=True) + + # Batch processing profiling + print("๐Ÿ“ฆ Profiling batch processing...") + batch_metrics = self.profile_batch_processing(test_images[:20]) # Use subset + + # Load testing + print("๐Ÿ”ฅ Running load tests...") + load_metrics = self.run_load_test(test_images[0], duration_seconds=30) + + # Memory profiling + print("๐Ÿ’พ Analyzing memory usage...") + memory_profile = self._profile_memory_usage(test_images[0]) + + # Bottleneck analysis + bottlenecks = self.analyze_bottlenecks(single_metrics) + + # Generate recommendations + recommendations = self.generate_recommendations(single_metrics, batch_metrics, bottlenecks) + + results = BenchmarkResults( + single_image_metrics=single_metrics, + batch_metrics=batch_metrics, + load_test_metrics=load_metrics, + memory_profile=memory_profile, + bottleneck_analysis=bottlenecks, + recommendations=recommendations, + ) + + # Save results + self._save_benchmark_results(results, output_dir) + + return results + + def _profile_memory_usage(self, image: np.ndarray) -> dict[str, float]: + """Detailed memory usage profiling""" + gc.collect() # Clean up before measurement + + baseline_memory = self.process.memory_info().rss / 1024 / 1024 + memory_profile = {"baseline": baseline_memory} + + # Memory after loading image + image_copy = image.copy() + memory_profile["after_image_load"] = self.process.memory_info().rss / 1024 / 1024 + + # Memory after detection + detections = detect_ui_elements(image_copy, confidence_threshold=0.5) + memory_profile["after_detection"] = self.process.memory_info().rss / 1024 / 1024 + + # Memory after OCR + text_results = extract_text(image_copy, confidence_threshold=0.5) + memory_profile["after_ocr"] = self.process.memory_info().rss / 1024 / 1024 + + # Calculate deltas + memory_profile["detection_delta"] = ( + memory_profile["after_detection"] - memory_profile["after_image_load"] + ) + memory_profile["ocr_delta"] = ( + memory_profile["after_ocr"] - memory_profile["after_detection"] + ) + memory_profile["total_delta"] = memory_profile["after_ocr"] - memory_profile["baseline"] + + return memory_profile + + def _save_benchmark_results(self, results: BenchmarkResults, output_dir: Path): + """Save benchmark results to files""" + output_dir.mkdir(parents=True, exist_ok=True) + + # Convert to serializable format + results_dict = { + "single_image_metrics": { + "total_time": results.single_image_metrics.total_time, + "preprocessing_time": results.single_image_metrics.preprocessing_time, + "inference_time": results.single_image_metrics.inference_time, + "postprocessing_time": results.single_image_metrics.postprocessing_time, + "peak_memory_usage": results.single_image_metrics.peak_memory_usage, + "memory_delta": results.single_image_metrics.memory_delta, + "images_per_second": results.single_image_metrics.images_per_second, + "num_detections": results.single_image_metrics.num_detections, + "num_text_elements": results.single_image_metrics.num_text_elements, + "stage_timings": results.single_image_metrics.stage_timings, + "gpu_memory_used": results.single_image_metrics.gpu_memory_used, + "gpu_utilization": results.single_image_metrics.gpu_utilization, + }, + "batch_metrics": [ + { + "batch_size": m.stage_timings.get("batch_size", 1), + "total_time": m.total_time, + "images_per_second": m.images_per_second, + "memory_delta": m.memory_delta, + "num_detections": m.num_detections, + "num_text_elements": m.num_text_elements, + } + for m in results.batch_metrics + ], + "load_test_metrics": { + k: { + "concurrency": v.stage_timings.get("concurrency", 1), + "total_requests": v.stage_timings.get("total_requests", 0), + "requests_per_second": v.stage_timings.get("requests_per_second", 0), + "images_per_second": v.images_per_second, + "memory_delta": v.memory_delta, + } + for k, v in results.load_test_metrics.items() + }, + "memory_profile": results.memory_profile, + "bottleneck_analysis": results.bottleneck_analysis, + "recommendations": results.recommendations, + } + + # Save JSON results + with open(output_dir / "performance_benchmark.json", "w") as f: + json.dump(results_dict, f, indent=2) + + # Save human-readable report + self._create_performance_report(results, output_dir) + + def _create_performance_report(self, results: BenchmarkResults, output_dir: Path): + """Create human-readable performance report""" + report_lines = [ + "# Vision System Performance Benchmark Report", + f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}", + "", + "## Single Image Performance", + f"- **Total Time**: {results.single_image_metrics.total_time:.3f}s", + f"- **Preprocessing**: {results.single_image_metrics.preprocessing_time:.3f}s", + f"- **Inference**: {results.single_image_metrics.inference_time:.3f}s", + f"- **Postprocessing**: {results.single_image_metrics.postprocessing_time:.3f}s", + f"- **Throughput**: {results.single_image_metrics.images_per_second:.2f} images/sec", + f"- **Peak Memory**: {results.single_image_metrics.peak_memory_usage:.1f} MB", + f"- **Memory Delta**: {results.single_image_metrics.memory_delta:.1f} MB", + "", + "### Stage Breakdown", + ] + + for stage, timing in results.single_image_metrics.stage_timings.items(): + if isinstance(timing, (int, float)): + report_lines.append(f"- **{stage}**: {timing:.3f}s") + + # Batch performance + if results.batch_metrics: + report_lines.extend( + [ + "", + "## Batch Processing Performance", + "| Batch Size | Time (s) | Throughput (img/s) | Memory (MB) |", + "|------------|----------|-------------------|-------------|", + ] + ) + + for metrics in results.batch_metrics: + batch_size = metrics.stage_timings.get("batch_size", 1) + report_lines.append( + f"| {batch_size} | {metrics.total_time:.3f} | " + f"{metrics.images_per_second:.2f} | {metrics.memory_delta:.1f} |" + ) + + # Load test results + if results.load_test_metrics: + report_lines.extend( + [ + "", + "## Load Test Results", + "| Concurrency | Requests/s | Throughput | Memory (MB) |", + "|-------------|------------|------------|-------------|", + ] + ) + + for test_name, metrics in results.load_test_metrics.items(): + concurrency = metrics.stage_timings.get("concurrency", 1) + rps = metrics.stage_timings.get("requests_per_second", 0) + report_lines.append( + f"| {concurrency} | {rps:.1f} | " + f"{metrics.images_per_second:.2f} | {metrics.memory_delta:.1f} |" + ) + + # Bottleneck analysis + if results.bottleneck_analysis: + report_lines.extend(["", "## Bottleneck Analysis"]) + for component, issue in results.bottleneck_analysis.items(): + report_lines.append(f"- **{component.title()}**: {issue}") + + # Recommendations + if results.recommendations: + report_lines.extend(["", "## Performance Recommendations"]) + for rec in results.recommendations: + report_lines.append(f"- {rec}") + + # Save report + with open(output_dir / "performance_report.md", "w") as f: + f.write("\n".join(report_lines)) + + +def main(): + """Command line interface for performance profiling""" + import argparse + + parser = argparse.ArgumentParser(description="Vision System Performance Profiler") + parser.add_argument( + "--test-data", type=Path, default="eval/test_data", help="Test data directory" + ) + parser.add_argument( + "--output", type=Path, default="eval/performance_results", help="Output directory" + ) + parser.add_argument("--quick", action="store_true", help="Quick profile (single image only)") + parser.add_argument( + "--load-test-duration", type=int, default=60, help="Load test duration in seconds" + ) + + args = parser.parse_args() + + # Load test images + image_dir = args.test_data / "images" + image_files = list(image_dir.glob("*.png")) + list(image_dir.glob("*.jpg")) + + if not image_files: + print(f"โŒ No images found in {image_dir}") + return 1 + + # Load images + test_images = [] + for img_file in image_files[:10]: # Limit to 10 for performance + img = cv2.imread(str(img_file)) + if img is not None: + test_images.append(img) + + if not test_images: + print("โŒ No valid images could be loaded") + return 1 + + print(f"๐Ÿ“Š Loaded {len(test_images)} test images") + + # Create profiler + profiler = PerformanceProfiler() + + if args.quick: + # Quick single image profile + print("๐Ÿš€ Running quick performance profile...") + metrics = profiler.profile_single_image(test_images[0]) + + print(f"\n{'=' * 50}") + print("QUICK PERFORMANCE PROFILE") + print(f"{'=' * 50}") + print(f"Total Time: {metrics.total_time:.3f}s") + print(f"Throughput: {metrics.images_per_second:.2f} images/s") + print(f"Memory Usage: {metrics.peak_memory_usage:.1f} MB") + print(f"Detections: {metrics.num_detections}") + print(f"Text Elements: {metrics.num_text_elements}") + + else: + # Full benchmark + results = profiler.run_comprehensive_benchmark(test_images, args.output) + + print(f"\n{'=' * 60}") + print("COMPREHENSIVE PERFORMANCE BENCHMARK") + print(f"{'=' * 60}") + print( + f"Single Image Throughput: {results.single_image_metrics.images_per_second:.2f} img/s" + ) + print(f"Peak Memory Usage: {results.single_image_metrics.peak_memory_usage:.1f} MB") + + if results.batch_metrics: + best_batch = max(results.batch_metrics, key=lambda m: m.images_per_second) + best_size = best_batch.stage_timings.get("batch_size", 1) + print(f"Best Batch Size: {best_size} ({best_batch.images_per_second:.2f} img/s)") + + print(f"\nBottlenecks: {len(results.bottleneck_analysis)}") + for component, issue in results.bottleneck_analysis.items(): + print(f" - {component}: {issue}") + + print(f"\nRecommendations: {len(results.recommendations)}") + for rec in results.recommendations[:3]: # Show top 3 + print(f" - {rec}") + + print(f"\n๐Ÿ“Š Detailed results saved to: {args.output}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval_legacy/screenshots/screenshot_1.png b/eval_legacy/screenshots/screenshot_1.png new file mode 100644 index 0000000..e37e584 Binary files /dev/null and b/eval_legacy/screenshots/screenshot_1.png differ diff --git a/eval_legacy/ui_generators.py b/eval_legacy/ui_generators.py new file mode 100644 index 0000000..b2f29fd --- /dev/null +++ b/eval_legacy/ui_generators.py @@ -0,0 +1,388 @@ +"""UI Image Generators + +Focused generators for creating different types of synthetic UI screenshots. +Each generator creates a specific type of UI with proper ground truth annotations. +""" + +from typing import Any + +import cv2 +import numpy as np + + +class UIImageGenerator: + """Base class for UI image generators""" + + def __init__(self): + self.default_font = cv2.FONT_HERSHEY_SIMPLEX + self.default_font_scale = 0.6 + self.default_thickness = 1 + self.text_color = (0, 0, 0) # Black + + def _draw_button( + self, + image: np.ndarray, + x: int, + y: int, + w: int, + h: int, + text: str, + color: tuple[int, int, int] = (200, 200, 200), + ): + """Draw a button with text""" + border_color = (100, 100, 100) + + # Fill rectangle + cv2.rectangle(image, (x, y), (x + w, y + h), color, -1) + # Draw border + cv2.rectangle(image, (x, y), (x + w, y + h), border_color, 2) + + # Center text + text_size = cv2.getTextSize( + text, self.default_font, self.default_font_scale, self.default_thickness + )[0] + text_x = x + (w - text_size[0]) // 2 + text_y = y + (h + text_size[1]) // 2 + + cv2.putText( + image, + text, + (text_x, text_y), + self.default_font, + self.default_font_scale, + self.text_color, + self.default_thickness, + ) + + def _draw_textbox( + self, image: np.ndarray, x: int, y: int, w: int, h: int, placeholder: str = "" + ): + """Draw a text input box""" + color = (255, 255, 255) # White + border_color = (150, 150, 150) # Light gray + + cv2.rectangle(image, (x, y), (x + w, y + h), color, -1) + cv2.rectangle(image, (x, y), (x + w, y + h), border_color, 1) + + if placeholder: + cv2.putText( + image, + placeholder, + (x + 5, y + h // 2 + 5), + self.default_font, + 0.5, + (128, 128, 128), + 1, + ) + + +class LoginFormGenerator(UIImageGenerator): + """Generates login form screenshots""" + + def create_login_form( + self, width: int = 800, height: int = 600 + ) -> tuple[np.ndarray, dict[str, Any]]: + """Create a simple login form""" + # White background + image = np.ones((height, width, 3), dtype=np.uint8) * 255 + + # Title + cv2.putText( + image, "Login", (width // 2 - 50, 80), self.default_font, 1.2, self.text_color, 2 + ) + + elements = [] + all_text = ["Login"] + + # Username field + username_y = 150 + cv2.putText( + image, + "Username:", + (100, username_y - 10), + self.default_font, + self.default_font_scale, + self.text_color, + self.default_thickness, + ) + self._draw_textbox(image, 100, username_y, 300, 35) + all_text.append("Username:") + + elements.append( + { + "x1": 100, + "y1": username_y, + "x2": 400, + "y2": username_y + 35, + "label": "textbox", + "text": "", + } + ) + + # Password field + password_y = 220 + cv2.putText( + image, + "Password:", + (100, password_y - 10), + self.default_font, + self.default_font_scale, + self.text_color, + self.default_thickness, + ) + self._draw_textbox(image, 100, password_y, 300, 35) + all_text.append("Password:") + + elements.append( + { + "x1": 100, + "y1": password_y, + "x2": 400, + "y2": password_y + 35, + "label": "textbox", + "text": "", + } + ) + + # Buttons + buttons = [ + {"text": "Login", "pos": (150, 300), "size": (100, 40), "color": (100, 200, 100)}, + {"text": "Cancel", "pos": (270, 300), "size": (100, 40), "color": (200, 100, 100)}, + ] + + coordinate_tests = [] + + for button in buttons: + x, y = button["pos"] + w, h = button["size"] + text = button["text"] + color = button.get("color", (200, 200, 200)) + + self._draw_button(image, x, y, w, h, text, color) + all_text.append(text) + + elements.append( + {"x1": x, "y1": y, "x2": x + w, "y2": y + h, "label": "button", "text": text} + ) + + coordinate_tests.append( + {"method": "text", "target": text, "expected_coord": [x + w // 2, y + h // 2]} + ) + + return image, { + "ocr_text": " ".join(all_text), + "ui_elements": elements, + "coordinate_tests": coordinate_tests, + } + + +class DialogGenerator(UIImageGenerator): + """Generates dialog box screenshots""" + + def create_confirmation_dialog( + self, width: int = 600, height: int = 400 + ) -> tuple[np.ndarray, dict[str, Any]]: + """Create a confirmation dialog""" + # Light gray background + image = np.ones((height, width, 3), dtype=np.uint8) * 240 + + # Dialog box + dialog_x, dialog_y = 50, 50 + dialog_w, dialog_h = 500, 300 + + # Dialog background + cv2.rectangle( + image, + (dialog_x, dialog_y), + (dialog_x + dialog_w, dialog_y + dialog_h), + (255, 255, 255), + -1, + ) + cv2.rectangle( + image, + (dialog_x, dialog_y), + (dialog_x + dialog_w, dialog_y + dialog_h), + (100, 100, 100), + 2, + ) + + # Title bar + cv2.rectangle( + image, (dialog_x, dialog_y), (dialog_x + dialog_w, dialog_y + 40), (200, 200, 200), -1 + ) + cv2.putText( + image, + "Confirm Action", + (dialog_x + 10, dialog_y + 25), + self.default_font, + self.default_font_scale, + self.text_color, + self.default_thickness, + ) + + # Message + message = "Are you sure you want to delete this file?" + cv2.putText( + image, + message, + (dialog_x + 20, dialog_y + 120), + self.default_font, + 0.7, + self.text_color, + 1, + ) + + # Buttons + buttons = [ + {"text": "Yes", "pos": (dialog_x + 150, dialog_y + 200), "size": (80, 35)}, + {"text": "No", "pos": (dialog_x + 250, dialog_y + 200), "size": (80, 35)}, + {"text": "Cancel", "pos": (dialog_x + 350, dialog_y + 200), "size": (80, 35)}, + ] + + elements = [] + coordinate_tests = [] + + for button in buttons: + x, y = button["pos"] + w, h = button["size"] + text = button["text"] + + self._draw_button(image, x, y, w, h, text) + + elements.append( + {"x1": x, "y1": y, "x2": x + w, "y2": y + h, "label": "button", "text": text} + ) + + coordinate_tests.append( + {"method": "text", "target": text, "expected_coord": [x + w // 2, y + h // 2]} + ) + + return image, { + "ocr_text": "Confirm Action " + message + " Yes No Cancel", + "ui_elements": elements, + "coordinate_tests": coordinate_tests, + } + + +class FormGenerator(UIImageGenerator): + """Generates complex form screenshots""" + + def create_registration_form( + self, width: int = 900, height: int = 700 + ) -> tuple[np.ndarray, dict[str, Any]]: + """Create a registration form""" + # Light background + image = np.ones((height, width, 3), dtype=np.uint8) * 250 + + # Form container + form_x, form_y = 50, 50 + form_w, form_h = 800, 600 + + cv2.rectangle( + image, (form_x, form_y), (form_x + form_w, form_y + form_h), (255, 255, 255), -1 + ) + cv2.rectangle( + image, (form_x, form_y), (form_x + form_w, form_y + form_h), (150, 150, 150), 2 + ) + + # Title + cv2.putText( + image, + "Registration Form", + (form_x + 300, form_y + 50), + self.default_font, + 1.0, + self.text_color, + 2, + ) + + elements = [] + coordinate_tests = [] + all_text = ["Registration Form"] + + # Form fields + fields = [ + {"label": "First Name:", "pos": (form_x + 50, form_y + 120)}, + {"label": "Last Name:", "pos": (form_x + 50, form_y + 180)}, + {"label": "Email:", "pos": (form_x + 50, form_y + 240)}, + {"label": "Phone:", "pos": (form_x + 50, form_y + 300)}, + ] + + for field in fields: + label = field["label"] + x, y = field["pos"] + + # Draw label + cv2.putText( + image, + label, + (x, y), + self.default_font, + self.default_font_scale, + self.text_color, + self.default_thickness, + ) + all_text.append(label) + + # Draw input field + field_x = x + 150 + field_y = y - 20 + field_w, field_h = 300, 30 + + self._draw_textbox(image, field_x, field_y, field_w, field_h) + + elements.append( + { + "x1": field_x, + "y1": field_y, + "x2": field_x + field_w, + "y2": field_y + field_h, + "label": "textbox", + "text": "", + } + ) + + # Submit buttons + buttons = [ + { + "text": "Submit", + "pos": (form_x + 200, form_y + 500), + "size": (120, 40), + "color": (100, 200, 100), + }, + { + "text": "Reset", + "pos": (form_x + 350, form_y + 500), + "size": (120, 40), + "color": (200, 200, 200), + }, + { + "text": "Cancel", + "pos": (form_x + 500, form_y + 500), + "size": (120, 40), + "color": (200, 100, 100), + }, + ] + + for button in buttons: + x, y = button["pos"] + w, h = button["size"] + text = button["text"] + color = button.get("color", (200, 200, 200)) + + self._draw_button(image, x, y, w, h, text, color) + all_text.append(text) + + elements.append( + {"x1": x, "y1": y, "x2": x + w, "y2": y + h, "label": "button", "text": text} + ) + + coordinate_tests.append( + {"method": "text", "target": text, "expected_coord": [x + w // 2, y + h // 2]} + ) + + return image, { + "ocr_text": " ".join(all_text), + "ui_elements": elements, + "coordinate_tests": coordinate_tests, + } diff --git a/infrastructure/docker/Dockerfile.arm b/infrastructure/docker/Dockerfile.arm deleted file mode 100644 index d9a05ad..0000000 --- a/infrastructure/docker/Dockerfile.arm +++ /dev/null @@ -1,78 +0,0 @@ -# ARM64/Apple Silicon compatible Dockerfile -FROM --platform=linux/arm64 python:3.12-slim AS base - -# For ARM compatibility, use explicit platform -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PYTHONOPTIMIZE=2 - -# Update packages and install core dependencies -RUN apt-get update -qq && apt-get install -y --no-install-recommends \ - ca-certificates \ - wget \ - curl \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - -# Install Python computer vision dependencies -RUN apt-get update -qq && apt-get install -y --no-install-recommends \ - libglib2.0-0 \ - libsm6 \ - libxext6 \ - libxrender-dev \ - libgomp1 \ - libgthread-2.0-0 \ - && rm -rf /var/lib/apt/lists/* - -# Install X11 and basic tools (minimal set for ARM compatibility) -RUN apt-get update -qq && apt-get install -y --no-install-recommends \ - xvfb \ - x11-utils \ - scrot \ - imagemagick \ - && rm -rf /var/lib/apt/lists/* - -# Try to install xdotool (may not be available on all ARM repos) -RUN apt-get update -qq && apt-get install -y --no-install-recommends \ - xdotool || echo "xdotool not available on this ARM platform" - -# Try to install FreeRDP (may have different package name on ARM) -RUN apt-get update -qq && apt-get install -y --no-install-recommends \ - freerdp2-x11 || \ - apt-get install -y --no-install-recommends freerdp-x11 || \ - echo "FreeRDP not available, OCR testing will still work" - -RUN rm -rf /var/lib/apt/lists/* - -# ---------- Builder stage ---------- -FROM base AS builder -COPY --from=ghcr.io/astral-sh/uv:0.8.13 /uv /usr/local/bin/uv -WORKDIR /app - -COPY pyproject.toml /app/ -RUN --mount=type=cache,target=/root/.cache/uv uv lock -RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-install-project --no-dev - -COPY . /app -RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev - -# Install debugpy for remote debugging -RUN --mount=type=cache,target=/root/.cache/uv uv add debugpy - -# ---------- Runtime stage ---------- -FROM base AS runtime -WORKDIR /app - -# Copy uv binary from builder stage -COPY --from=builder /usr/local/bin/uv /usr/local/bin/uv - -RUN useradd -m -u 10001 appuser -COPY --from=builder /app /app -ENV PATH="/app/.venv/bin:${PATH}" - -ENV DISPLAY=:99 -RUN mkdir -p /tmp/screenshots && chown appuser:appuser /tmp/screenshots - -USER appuser -CMD ["bash"] \ No newline at end of file diff --git a/infrastructure/docker/Dockerfile.rdp b/infrastructure/docker/Dockerfile.rdp deleted file mode 100644 index 0669d47..0000000 --- a/infrastructure/docker/Dockerfile.rdp +++ /dev/null @@ -1,103 +0,0 @@ -# RDP-specific Dockerfile for VM Automation -# Includes all dependencies needed for RDP connections: FreeRDP, Xvfb, xdotool, ImageMagick - -# ---------- Base image (RDP-specific) ---------- -FROM python:3.12-slim AS base -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PYTHONOPTIMIZE=2 - -# Update package lists first -RUN apt-get update -qq - -# Install system deps for computer vision and RDP in separate steps -RUN apt-get install -y --no-install-recommends \ - ca-certificates \ - libglib2.0-0 \ - libsm6 \ - libxext6 \ - libxrender-dev \ - libgomp1 \ - libgl1-mesa-glx \ - libgthread-2.0-0 - -# Install X11 and display tools -RUN apt-get install -y --no-install-recommends \ - xvfb \ - x11-apps \ - x11-utils \ - xdotool - -# Install RDP and screenshot tools -RUN apt-get install -y --no-install-recommends \ - freerdp2-x11 \ - scrot \ - imagemagick - -# Install network tools (try different package name if needed) -RUN apt-get install -y --no-install-recommends \ - netbase || apt-get install -y --no-install-recommends inetutils-ping - -# Install image processing tools -RUN apt-get install -y --no-install-recommends \ - netpbm - -# Clean up -RUN rm -rf /var/lib/apt/lists/* - -# Configure ImageMagick security policy for screenshot conversion -RUN sed -i 's/rights="none" pattern="PDF"/rights="read|write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml || true && \ - sed -i 's/rights="none" pattern="XWD"/rights="read|write" pattern="XWD"/g' /etc/ImageMagick-6/policy.xml || true - -# ---------- Builder: uv + lock + install (no dev) ---------- -FROM base AS builder -# Bring in uv binary -COPY --from=ghcr.io/astral-sh/uv:0.8.13 /uv /usr/local/bin/uv -WORKDIR /app - -# Copy project metadata first for layer caching -COPY pyproject.toml /app/ - -# Create or refresh lock deterministically for Python 3.12 -RUN --mount=type=cache,target=/root/.cache/uv \ - uv lock - -# Install ONLY runtime deps into a local venv (no project yet, and no dev deps) -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-dev - -# Now copy the source and install the project into the same venv -COPY . /app -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev - -# ---------- Final runtime image ---------- -FROM base AS runtime -WORKDIR /app - -# Create a non-root user -RUN useradd -m -u 10001 appuser - -# Copy the entire app (including .venv from builder) -COPY --from=builder /app /app - -# Put venv on PATH -ENV PATH="/app/.venv/bin:${PATH}" - -# Set default connection type to RDP -ENV CONNECTION_TYPE=rdp -ENV DISPLAY=:99 - -# Create temp directory for RDP screenshots (with proper permissions) -RUN mkdir -p /tmp/rdp_capture && chown appuser:appuser /tmp/rdp_capture - -USER appuser -EXPOSE 8000 - -# Health check to verify RDP dependencies -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD which xfreerdp > /dev/null 2>&1 || exit 1 - -# Default command: run the VM automation CLI with RDP -CMD ["uv", "run", "vm-automation", "--connection", "rdp"] diff --git a/infrastructure/docker/docker-compose.yml b/infrastructure/docker/docker-compose.yml deleted file mode 100644 index 38c2f69..0000000 --- a/infrastructure/docker/docker-compose.yml +++ /dev/null @@ -1,94 +0,0 @@ -# Docker Compose for VM Automation with different connection types -version: '3.8' - -services: - # VNC-optimized container (lightweight) - vm-automation-vnc: - build: - context: . - dockerfile: Dockerfile.vnc - container_name: vm-automation-vnc - environment: - - CONNECTION_TYPE=vnc - - VM_HOST=${VM_HOST:-192.168.1.100} - - VM_PORT=${VM_PORT:-5900} - - VM_PASSWORD=${VM_PASSWORD:-} - - TARGET_APP=${TARGET_APP:-Greenway Dev} - - TARGET_BUTTON=${TARGET_BUTTON:-Submit} - - LOG_LEVEL=${LOG_LEVEL:-INFO} - - SAVE_SCREENSHOTS=${SAVE_SCREENSHOTS:-true} - volumes: - - ./vm_config.json:/app/vm_config.json:ro - - ./logs:/app/logs - - ./screenshots:/app/screenshots - profiles: - - vnc - - # RDP-optimized container (with all RDP dependencies) - vm-automation-rdp: - build: - context: . - dockerfile: Dockerfile.rdp - container_name: vm-automation-rdp - environment: - - CONNECTION_TYPE=rdp - - VM_HOST=${VM_HOST:-192.168.1.100} - - VM_PORT=${VM_PORT:-3389} - - VM_USERNAME=${VM_USERNAME:-} - - VM_PASSWORD=${VM_PASSWORD:-} - - RDP_DOMAIN=${RDP_DOMAIN:-} - - RDP_WIDTH=${RDP_WIDTH:-1920} - - RDP_HEIGHT=${RDP_HEIGHT:-1080} - - TARGET_APP=${TARGET_APP:-Greenway Dev} - - TARGET_BUTTON=${TARGET_BUTTON:-Submit} - - LOG_LEVEL=${LOG_LEVEL:-INFO} - - SAVE_SCREENSHOTS=${SAVE_SCREENSHOTS:-true} - - DISPLAY=:99 - volumes: - - ./vm_config.json:/app/vm_config.json:ro - - ./logs:/app/logs - - ./screenshots:/app/screenshots - - /tmp/.X11-unix:/tmp/.X11-unix:rw - tmpfs: - - /tmp/rdp_capture:noexec,nosuid,size=100m - profiles: - - rdp - privileged: false - # Add capabilities for X11 forwarding - cap_add: - - SYS_ADMIN - - # General-purpose container (supports both VNC and RDP) - vm-automation: - build: - context: . - dockerfile: Dockerfile - container_name: vm-automation-general - environment: - - CONNECTION_TYPE=${CONNECTION_TYPE:-vnc} - - VM_HOST=${VM_HOST:-192.168.1.100} - - VM_PORT=${VM_PORT:-5900} - - VM_USERNAME=${VM_USERNAME:-} - - VM_PASSWORD=${VM_PASSWORD:-} - - RDP_DOMAIN=${RDP_DOMAIN:-} - - RDP_WIDTH=${RDP_WIDTH:-1920} - - RDP_HEIGHT=${RDP_HEIGHT:-1080} - - TARGET_APP=${TARGET_APP:-Greenway Dev} - - TARGET_BUTTON=${TARGET_BUTTON:-Submit} - - LOG_LEVEL=${LOG_LEVEL:-INFO} - - SAVE_SCREENSHOTS=${SAVE_SCREENSHOTS:-true} - volumes: - - ./vm_config.json:/app/vm_config.json:ro - - ./logs:/app/logs - - ./screenshots:/app/screenshots - - /tmp/.X11-unix:/tmp/.X11-unix:rw - tmpfs: - - /tmp/rdp_capture:noexec,nosuid,size=100m - profiles: - - general - -volumes: - logs: - driver: local - screenshots: - driver: local \ No newline at end of file diff --git a/infrastructure/docs/External Windows VM + Microservices.mermaid b/infrastructure/docs/External Windows VM + Microservices.mermaid deleted file mode 100644 index c2e4c33..0000000 --- a/infrastructure/docs/External Windows VM + Microservices.mermaid +++ /dev/null @@ -1,38 +0,0 @@ -flowchart LR - subgraph K8s[Container: Kubernetes Cluster] - Orchestrator[ - Container: - Agent/Orchestrator Pod - LLM client - VNC client - Gate - ] - Vision[ - Container: - Vision Service Pod - PaddleOCR - YOLO API - HTTP/gRPC - ] - Queue[[Container: Queue]] - Artifacts[ - Container: - Artifacts PVC/Bucket - ] - end - - WinVM[[ - System Ext: - Windows VM - Greenway - VNC - ]] - LLM[[ - System Ext: - LLM API - ]] - - Queue --> Orchestrator - Orchestrator --> WinVM - Orchestrator -- "HTTP/gRPC (image/ROI)" --> Vision - Orchestrator --> LLM - Orchestrator --> Artifacts diff --git a/infrastructure/docs/External Windows VM + Monolith Pod (Preferred).mermaid b/infrastructure/docs/External Windows VM + Monolith Pod (Preferred).mermaid deleted file mode 100644 index 5293f47..0000000 --- a/infrastructure/docs/External Windows VM + Monolith Pod (Preferred).mermaid +++ /dev/null @@ -1,33 +0,0 @@ -flowchart TB - subgraph K8s[Container: Kubernetes Cluster] - Pod[ - Container: - Agent Pod - Monolith - VNC client - PaddleOCR - YOLO - LLM client - Gate - ] - Queue[[Container: Queue]] - Artifacts[ - Container: - Artifacts PVC/Bucket - ] - end - - WinVM[[ - System Ext: - Windows VM - Greenway - VNC - ]] - LLM[[ - System Ext: - LLM API - ]] - - Queue --> Pod - Pod --> WinVM - Pod --> LLM - Pod --> Artifacts diff --git a/infrastructure/docs/KubeVirt Windows VM + Microservices.mermaid b/infrastructure/docs/KubeVirt Windows VM + Microservices.mermaid deleted file mode 100644 index 6c7264e..0000000 --- a/infrastructure/docs/KubeVirt Windows VM + Microservices.mermaid +++ /dev/null @@ -1,37 +0,0 @@ -flowchart LR - subgraph K8s[Container: Kubernetes Cluster] - Orchestrator[ - Container: - Orchestrator Pod - LLM client - VNC client - Gate - ] - Vision[ - Container: - Vision Service Pod - PaddleOCR - YOLO API - ] - WinVMI[ - Container: - KubeVirt Windows VMI - Greenway + VNC - ] - Queue[[Container: Queue]] - Artifacts[ - Container: - Artifacts PVC/Bucket - ] - end - - LLM[[ - System Ext: - LLM API - ]] - - Queue --> Orchestrator - Orchestrator --> WinVMI - Orchestrator -- "HTTP/gRPC (image/ROI)" --> Vision - Orchestrator --> LLM - Orchestrator --> Artifacts diff --git a/infrastructure/docs/KubeVirt Windows VM + Monolith Pod.mermaid b/infrastructure/docs/KubeVirt Windows VM + Monolith Pod.mermaid deleted file mode 100644 index dbb91c7..0000000 --- a/infrastructure/docs/KubeVirt Windows VM + Monolith Pod.mermaid +++ /dev/null @@ -1,30 +0,0 @@ -flowchart TB - subgraph K8s[Container: Kubernetes ] - Pod[ - Container: - Agent Pod - Monolith - VNC client - PaddleOCR - YOLO - LLM client - Gate - ] - WinVMI[ - Container: - Windows VM - Greenway - VNC - ] - Queue[[Container: Queue]] - Artifacts[ - Container: - Artifacts PVC/Bucket - ] - end - - LLM[[System Ext: LLM API]] - - Queue --> Pod - Pod --> WinVMI - Pod --> LLM - Pod --> Artifacts diff --git "a/infrastructure/docs/Patient Data Automation with LLM Advisor \342\200\223 Architecture & Roadmap.pages" "b/infrastructure/docs/Patient Data Automation with LLM Advisor \342\200\223 Architecture & Roadmap.pages" deleted file mode 100755 index d92ac8c..0000000 Binary files "a/infrastructure/docs/Patient Data Automation with LLM Advisor \342\200\223 Architecture & Roadmap.pages" and /dev/null differ diff --git a/infrastructure/docs/System Context.mermaid b/infrastructure/docs/System Context.mermaid deleted file mode 100644 index ba2d53f..0000000 --- a/infrastructure/docs/System Context.mermaid +++ /dev/null @@ -1,19 +0,0 @@ -flowchart LR - User([User]) - Ops([Ops / Platform]) - - subgraph K8s[System: Kubernetes Cluster] - Agent["System: Computer-Use Agent\n(pod)"] - end - - WinVM[[System Ext: Windows EHR Workstation / VM]] - LLM[["System Ext: LLM API (GPT-4o/GPT-5, text-only)"]] - Bus[["System Ext: Queue / Service Bus (optional)"]] - Art["(System Ext: Artifacts Store\n(PVC/S3))"] - - User --> Bus - Bus --> Agent - Agent --> WinVM - Agent --> LLM - Agent --> Art - Ops --> K8s diff --git a/infrastructure/docs/sequence.mermaid b/infrastructure/docs/sequence.mermaid deleted file mode 100644 index 886b69d..0000000 --- a/infrastructure/docs/sequence.mermaid +++ /dev/null @@ -1,15 +0,0 @@ -sequenceDiagram - participant Queue as Queue (optional) - participant Pod as Agent Pod (Monolith) - participant VM as Windows VM (EHR) - participant LLM as LLM API - participant Store as Artifacts - - Queue->>Pod: Start job (row.json) - Pod->>VM: Connect RDP/VNC, capture screenshot - Pod->>Pod: OCR+YOLO detect UI (Gate: verify patient banner) - Pod->>LLM: Send text+coords for plan/tool calls - LLM-->>Pod: Steps (click/type/select) - Pod->>VM: Execute actions (only if gate passed) - Pod->>Store: Upload screenshots + run.json - Pod-->>Queue: Status/Result diff --git a/pyproject.toml b/pyproject.toml index 45912b1..b3a0b36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,3 @@ -[build-system] -requires = ["hatchling>=1.25.0"] -build-backend = "hatchling.build" - [project] name = "computer-use-agent" version = "0.1.0" @@ -25,17 +21,14 @@ dependencies = [ "openai-agents>=0.2.11", "opencv-python>=4.11.0.86", "paddleocr[all]>=3.2.0", - "paddlepaddle>=3.1.1", + "paddlepaddle>=3.2.0", "pillow>=11.3.0", "ultralytics>=8.3.193", "uvicorn>=0.35.0", "vncdotool>=1.2.0", ] -# Dev / tooling -[project.scripts] -vm-automation = "vm.main:cli_main" [tool.ruff] line-length = 100 @@ -51,6 +44,7 @@ line-ending = "auto" skip-magic-trailing-comma = false [tool.pytest.ini_options] +pythonpath = "." addopts = "-q" testpaths = ["tests"] markers = [ @@ -59,12 +53,15 @@ markers = [ "vnc: marks tests as VNC-specific integration tests", "rdp: marks tests as RDP-specific integration tests", "patient_workflow: marks tests as patient workflow tests (healthcare applications)", - "ui_detection: marks tests as UI detection tests (YOLO/OCR)", + "ui_detection: marks tests as UI detection tests", "real_vm: marks tests as requiring real VM connections", + "eval: marks tests as evaluation system tests (require eval dependencies)", + "eval_results: marks tests as evaluation tests (require full eval stack)", + "eval_performance: marks tests as performance profiling tests", + "eval_robustness: marks tests as robustness evaluation tests", ] -[tool.hatch.build.targets.wheel] -packages = [ "src/agent", "src/automation","src/vision"] +# Dev / tooling [tool.uv] default-groups = ["dev"] @@ -80,3 +77,42 @@ dev = [ "pytest-cov>=6.2.1", "ruff>=0.12.12", ] + +# Vision Evaluation Dependencies +eval = [ + # Core ML libraries + "torch>=1.12.0", + "torchvision>=0.13.0", + "torchmetrics>=0.11.0", + + # Computer Vision (opencv-python already in main deps) + "albumentations>=1.3.0", + + # COCO Evaluation + "pycocotools>=2.0.6", + + # OCR Evaluation + "jiwer>=2.5.0", + "python-Levenshtein>=0.12.0", + "editdistance>=0.6.0", + + # Dataset Analysis - optional due to sse-starlette conflict + # Install separately if needed: pip install fiftyone==1.8.0 + + # Experiment Tracking + "wandb>=0.15.0", + + # Performance Monitoring + "psutil>=5.9.0", + "GPUtil>=1.4.0", + + # Utilities + "tqdm>=4.64.0", + "matplotlib>=3.5.0", + "seaborn>=0.11.0", + "pyyaml>=6.0", + + # Type stubs + "types-PyYAML>=6.0.0", +] + diff --git a/src/agent/examples.py b/src/agents/examples.py similarity index 99% rename from src/agent/examples.py rename to src/agents/examples.py index 0b8516d..09deb30 100644 --- a/src/agent/examples.py +++ b/src/agents/examples.py @@ -4,7 +4,7 @@ in various scenarios and AI agent frameworks. """ -from agent.vision_tools import ( +from src.agents.vision_tools import ( analyze_screen, click_element, configure_vision_tools, diff --git a/src/agent/function_definitions.py b/src/agents/function_definitions.py similarity index 100% rename from src/agent/function_definitions.py rename to src/agents/function_definitions.py diff --git a/src/agent/mcp/__init__.py b/src/agents/mcp/__init__.py similarity index 83% rename from src/agent/mcp/__init__.py rename to src/agents/mcp/__init__.py index 0ac622f..9f96424 100644 --- a/src/agent/mcp/__init__.py +++ b/src/agents/mcp/__init__.py @@ -21,18 +21,18 @@ server.run() """ -from .handlers import ( +from src.agents.mcp.handlers import ( handle_analyze_screen, handle_detect_elements, handle_extract_text, handle_find_elements, ) -from .mcp_server import create_mcp_server, start_server -from .tools import get_function_tools, get_tool_schemas +from src.agents.mcp.mcp_server import create_mcp_server, start_server +from src.agents.mcp.tools import get_function_tools, get_tool_schemas __version__ = "1.0.0" -__all__ = [ +__all__ = [ # noqa: RUF022 # MCP server functions "create_mcp_server", "start_server", diff --git a/src/agent/mcp/handlers.py b/src/agents/mcp/handlers.py similarity index 72% rename from src/agent/mcp/handlers.py rename to src/agents/mcp/handlers.py index 9f13f19..cf41c53 100644 --- a/src/agent/mcp/handlers.py +++ b/src/agents/mcp/handlers.py @@ -8,17 +8,14 @@ import io # Import pure OCR functions -import sys -from pathlib import Path from typing import Any import cv2 import numpy as np from PIL import Image -sys.path.append(str(Path(__file__).parent.parent)) - -from vision import ( +from src.agents.mcp.tools import validate_tool_parameters +from src.vision import ( analyze_screen_content, detect_ui_elements, extract_text, @@ -26,8 +23,12 @@ find_clickable_elements, find_elements_by_text, ) - -from .tools import validate_tool_parameters +from src.vision.template_manager import ( + TemplateManager, + TemplateRequest, + TemplateStrategy, + get_template_manager, +) def decode_base64_image(base64_data: str) -> np.ndarray: @@ -72,7 +73,7 @@ def decode_base64_image(base64_data: str) -> np.ndarray: def handle_detect_elements(parameters: dict[str, Any]) -> dict[str, Any]: """ - Handle detect_ui_elements tool execution + Handle detect_ui_elements tool execution with template strategies Args: parameters: Tool parameters from MCP call @@ -87,31 +88,68 @@ def handle_detect_elements(parameters: dict[str, Any]) -> dict[str, Any]: # Decode image image = decode_base64_image(validated_params["image_base64"]) - # Call OCR function + # Get template manager + template_manager = get_template_manager() + + # Resolve template strategy + template_strategy = validated_params.get("template_strategy", "base64") + template_sources = _resolve_template_strategy( + validated_params, template_strategy, template_manager + ) + + if not template_sources: + return { + "success": False, + "error": "No valid templates could be resolved", + "detections": [], + "total_found": 0, + } + + # Call template detection function detections = detect_ui_elements( image=image, - confidence_threshold=validated_params.get("confidence_threshold", 0.6), - ui_focused=validated_params.get("ui_focused", True), + template_sources=template_sources, + template_name=validated_params.get("template_name", "template"), + confidence_threshold=validated_params.get("confidence_threshold", 0.8), + method=validated_params.get("method", "auto"), max_detections=validated_params.get("max_detections", 50), + enable_multiscale=validated_params.get("enable_multiscale", True), + scale_range=validated_params.get("scale_range", [0.5, 2.0]), + enable_rotation=validated_params.get("enable_rotation", False), + template_manager=template_manager, ) # Format response formatted_detections = [] for detection in detections: - formatted_detections.append( - { - "class_name": detection.class_name, - "confidence": float(detection.confidence), - "bbox": list(detection.bbox), - "center": list(detection.center), - "area": detection.area, + detection_data = { + "template_name": detection.template_name, + "confidence": float(detection.confidence), + "bbox": list(detection.bbox), + "center": list(detection.center), + "area": detection.area, + "scale": detection.scale, + "angle": detection.angle, + "method": detection.method, + "template_match_method": detection.template_match_method, + "template_confidence": detection.template_confidence, + } + + # Add template source info if available + if detection.template_source: + detection_data["template_source"] = { + "strategy": detection.template_source.strategy.value, + "metadata": detection.template_source.metadata, } - ) + + formatted_detections.append(detection_data) return { "success": True, "detections": formatted_detections, "total_found": len(formatted_detections), + "strategy_used": template_strategy, + "templates_resolved": len(template_sources), "image_dimensions": {"height": image.shape[0], "width": image.shape[1]}, } @@ -119,6 +157,64 @@ def handle_detect_elements(parameters: dict[str, Any]) -> dict[str, Any]: return {"success": False, "error": str(e), "detections": [], "total_found": 0} +def _resolve_template_strategy( + parameters: dict[str, Any], strategy: str, template_manager: TemplateManager +) -> list: + """Resolve template strategy to template sources""" + try: + if strategy == "base64": + if "template_base64" not in parameters: + raise ValueError("template_base64 required for base64 strategy") + + request = TemplateRequest( + strategy=TemplateStrategy.BASE64, + data=parameters["template_base64"], + weight=1.0, + name=parameters.get("template_name", "base64_template"), + ) + return template_manager.resolve_templates(request) + + elif strategy == "library": + if "template_library_id" not in parameters: + raise ValueError("template_library_id required for library strategy") + + request = TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={ + "id": parameters["template_library_id"], + "category": parameters.get("template_category", "common"), + }, + weight=1.0, + name=parameters.get("template_name", parameters["template_library_id"]), + ) + return template_manager.resolve_templates(request) + + elif strategy == "multi": + if "templates" not in parameters: + raise ValueError("templates array required for multi strategy") + + requests = [] + for template_spec in parameters["templates"]: + template_strategy = TemplateStrategy(template_spec["strategy"]) + + request = TemplateRequest( + strategy=template_strategy, + data=template_spec["data"], + weight=template_spec.get("weight", 1.0), + name=template_spec.get("name", "multi_template"), + required=template_spec.get("required", True), + ) + requests.append(request) + + return template_manager.resolve_templates(requests) + + else: + raise ValueError(f"Unknown template strategy: {strategy}") + + except Exception as e: + raise ValueError(f"Failed to resolve template strategy '{strategy}': {e}") + + def handle_extract_text(parameters: dict[str, Any]) -> dict[str, Any]: """ Handle extract_text tool execution diff --git a/src/agent/mcp/mcp_server.py b/src/agents/mcp/mcp_server.py similarity index 89% rename from src/agent/mcp/mcp_server.py rename to src/agents/mcp/mcp_server.py index 3e2d148..40a5d26 100644 --- a/src/agent/mcp/mcp_server.py +++ b/src/agents/mcp/mcp_server.py @@ -10,8 +10,8 @@ import sys from typing import Any -from .handlers import execute_tool -from .tools import get_function_tools +from src.agents.mcp.handlers import execute_tool +from src.agents.mcp.tools import get_function_tools class MCPServer: @@ -187,25 +187,6 @@ def main(): """Command line entry point""" print("Starting Computer Vision MCP Server...", file=sys.stderr) - # Check if models are available - try: - import sys - from pathlib import Path - - # Add parent directory to path to import ocr_clean - parent_dir = Path(__file__).parent.parent - sys.path.append(str(parent_dir)) - - from vision.setup_models import verify_models - - verification = verify_models() - if not all(verification.values()): - print("โš ๏ธ Warning: Some models are not available.", file=sys.stderr) - print("Run: python -m ocr_clean.setup_models", file=sys.stderr) - - except Exception as e: - print(f"โš ๏ธ Could not verify models: {e}", file=sys.stderr) - # Run server try: asyncio.run(start_server()) diff --git a/src/agent/mcp/tools.py b/src/agents/mcp/tools.py similarity index 66% rename from src/agent/mcp/tools.py rename to src/agents/mcp/tools.py index 2462587..0b842a7 100644 --- a/src/agent/mcp/tools.py +++ b/src/agents/mcp/tools.py @@ -24,7 +24,7 @@ def get_function_tools() -> list[dict[str, Any]]: "type": "function", "function": { "name": "detect_ui_elements", - "description": "Detect UI elements in an image using YOLO computer vision", + "description": "Detect custom UI elements using template matching strategies", "parameters": { "type": "object", "properties": { @@ -32,17 +32,88 @@ def get_function_tools() -> list[dict[str, Any]]: "type": "string", "description": "Base64-encoded image data (PNG, JPG, etc.)", }, + "template_strategy": { + "type": "string", + "default": "base64", + "enum": ["base64", "library", "multi"], + "description": "Template resolution strategy", + }, + "template_base64": { + "type": "string", + "description": "Base64-encoded template image (for base64 strategy)", + }, + "template_library_id": { + "type": "string", + "description": "Template ID from library (for library strategy)", + }, + "template_category": { + "type": "string", + "default": "common", + "description": "Template category in library (buttons, forms, navigation, etc.)", + }, + "templates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "strategy": {"type": "string", "enum": ["base64", "library"]}, + "data": {"oneOf": [{"type": "string"}, {"type": "object"}]}, + "weight": { + "type": "number", + "default": 1.0, + "minimum": 0.1, + "maximum": 2.0, + }, + "name": {"type": "string"}, + "required": {"type": "boolean", "default": true}, + }, + "required": ["strategy", "data"], + }, + "description": "Multiple templates for multi strategy", + }, + "template_name": { + "type": "string", + "default": "template", + "description": "Name identifier for the template", + }, "confidence_threshold": { "type": "number", - "default": 0.6, + "default": 0.8, "minimum": 0.0, "maximum": 1.0, "description": "Minimum confidence threshold for detections (0.0-1.0)", }, - "ui_focused": { + "method": { + "type": "string", + "default": "auto", + "enum": [ + "auto", + "TM_CCOEFF", + "TM_CCOEFF_NORMED", + "TM_CCORR", + "TM_CCORR_NORMED", + "TM_SQDIFF", + "TM_SQDIFF_NORMED", + ], + "description": "Template matching method ('auto' for automatic selection)", + }, + "enable_multiscale": { "type": "boolean", "default": True, - "description": "If true, filter to UI-relevant object classes only", + "description": "Enable multi-scale template matching", + }, + "scale_range": { + "type": "array", + "items": {"type": "number"}, + "minItems": 2, + "maxItems": 2, + "default": [0.5, 2.0], + "description": "Scale range for multi-scale matching [min_scale, max_scale]", + }, + "enable_rotation": { + "type": "boolean", + "default": False, + "description": "Enable rotation-invariant template matching", }, "max_detections": { "type": "integer", @@ -103,7 +174,7 @@ def get_function_tools() -> list[dict[str, Any]]: "type": "function", "function": { "name": "find_elements_by_text", - "description": "Find UI elements that contain specific text using combined YOLO+OCR", + "description": "Find UI elements that contain specific text using combined OCR", "parameters": { "type": "object", "properties": { @@ -283,24 +354,64 @@ def validate_tool_parameters(tool_name: str, parameters: dict[str, Any]) -> dict # Tool usage examples for documentation TOOL_EXAMPLES = { "detect_ui_elements": { - "description": "Detect UI elements like buttons, input fields, and interactive components", - "example": { - "image_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", - "confidence_threshold": 0.7, - "ui_focused": True, - "max_detections": 20, + "description": "Detect custom UI elements using template matching strategies", + "examples": { + "base64_strategy": { + "image_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", + "template_strategy": "base64", + "template_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", + "template_name": "login_button", + "confidence_threshold": 0.8, + "method": "auto", + "enable_multiscale": True, + "max_detections": 20, + }, + "library_strategy": { + "image_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", + "template_strategy": "library", + "template_library_id": "submit_button", + "template_category": "buttons", + "confidence_threshold": 0.8, + "method": "auto", + }, + "multi_strategy": { + "image_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", + "template_strategy": "multi", + "templates": [ + { + "strategy": "library", + "data": {"id": "submit_button", "category": "buttons"}, + "weight": 1.0, + "name": "submit_library", + }, + { + "strategy": "base64", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAY...", + "weight": 0.8, + "name": "custom_submit", + "required": false, + }, + ], + "confidence_threshold": 0.7, + }, }, "response": { "detections": [ { - "class_name": "laptop", - "confidence": 0.85, - "bbox": [100, 200, 300, 400], - "center": [200, 300], - "area": 40000, + "template_name": "login_button", + "confidence": 0.92, + "bbox": [100, 200, 200, 240], + "center": [150, 220], + "area": 4000, + "scale": 1.0, + "angle": 0.0, + "method": "TM_CCOEFF_NORMED", + "template_match_method": "single", + "template_confidence": 1.0, } ], "total_found": 1, + "strategy_used": "base64", }, }, "extract_text": { diff --git a/src/agent/vision_tools.py b/src/agents/vision_tools.py similarity index 99% rename from src/agent/vision_tools.py rename to src/agents/vision_tools.py index d1c9541..ad1ccba 100644 --- a/src/agent/vision_tools.py +++ b/src/agents/vision_tools.py @@ -23,7 +23,7 @@ import cv2 import numpy as np -from vision import ( +from src.vision import ( find_elements_by_text, verify_click_success, verify_element_present, diff --git a/src/automation/__init__.py b/src/automation/__init__.py index f645307..202e270 100644 --- a/src/automation/__init__.py +++ b/src/automation/__init__.py @@ -3,12 +3,12 @@ Provides both local and remote automation capabilities with a unified API. Structure: -- core/: Shared types and base classes (ActionResult, ConnectionResult, VMConnection) -- local/: Local desktop automation (DesktopControl, FormFiller) -- remote/: VM/remote automation - - connections/: VM connection implementations (VNC, RDP, Desktop) - - agents/: VM automation agents (VMNavigator, AppController) - - tools/: VM interaction tools (ScreenCapture, InputActions) + - core/: Shared types and base classes (ActionResult, ConnectionResult, VMConnection) + - local/: Local desktop automation (DesktopControl, FormFiller) + - remote/: VM/remote automation + - connections/: VM connection implementations (VNC, RDP, Desktop) + - agents/: VM automation agents (VMNavigator, AppController) + - tools/: VM interaction tools (ScreenCapture, InputActions) - orchestrator.py: Main VM automation orchestrator Usage: @@ -24,16 +24,16 @@ """ # Core types -from .core import ActionResult, ConnectionResult +from src.automation.core import ActionResult, ConnectionResult # Local automation -from .local import DesktopControl, FormFiller +from src.automation.local import DesktopControl, FormFiller # Orchestrator -from .orchestrator import VMAutomation, VMConfig +from src.automation.orchestrator import VMAutomation, VMConfig # Remote automation -from .remote import ( +from src.automation.remote import ( AppControllerAgent, DesktopConnection, InputActions, @@ -48,7 +48,7 @@ __version__ = "2.0.0" -__all__ = [ +__all__ = [ # noqa: RUF022 # Core types "ActionResult", "ConnectionResult", diff --git a/src/automation/core/__init__.py b/src/automation/core/__init__.py index 22e34df..893cfb7 100644 --- a/src/automation/core/__init__.py +++ b/src/automation/core/__init__.py @@ -1,5 +1,5 @@ """Core automation types and base classes""" -from .types import ActionResult, ConnectionResult +from src.automation.core.types import ActionResult, ConnectionResult __all__ = ["ActionResult", "ConnectionResult"] diff --git a/src/automation/core/base.py b/src/automation/core/base.py index 18417e7..358242d 100644 --- a/src/automation/core/base.py +++ b/src/automation/core/base.py @@ -5,7 +5,7 @@ import numpy as np -from automation.core.types import ActionResult, ConnectionResult +from src.automation.core.types import ActionResult, ConnectionResult class VMConnection(ABC): diff --git a/src/automation/local/__init__.py b/src/automation/local/__init__.py index 4f060ce..3b50446 100644 --- a/src/automation/local/__init__.py +++ b/src/automation/local/__init__.py @@ -3,7 +3,7 @@ Provides local mouse/keyboard control capabilities for desktop automation. """ -from .desktop_control import DesktopControl -from .form_interface import FormFiller +from src.automation.local.desktop_control import DesktopControl +from src.automation.local.form_interface import FormFiller __all__ = ["DesktopControl", "FormFiller"] diff --git a/src/automation/local/desktop_control.py b/src/automation/local/desktop_control.py index a0e1f63..2d6ef7d 100644 --- a/src/automation/local/desktop_control.py +++ b/src/automation/local/desktop_control.py @@ -14,7 +14,7 @@ import cv2 import numpy as np -from automation.core import ActionResult +from src.automation.core import ActionResult class DesktopControl: diff --git a/src/automation/local/form_interface.py b/src/automation/local/form_interface.py index eacbeff..b0e0757 100644 --- a/src/automation/local/form_interface.py +++ b/src/automation/local/form_interface.py @@ -11,18 +11,13 @@ form_filler.click_button("Submit") """ -import sys -from pathlib import Path from typing import Any -# Add src to path for imports -sys.path.append(str(Path(__file__).parent.parent)) - import numpy as np -from automation.core import ActionResult -from automation.local.desktop_control import DesktopControl -from vision import find_elements_by_text +from src.automation.core import ActionResult +from src.automation.local.desktop_control import DesktopControl +from src.vision import find_elements_by_text class FormFiller: diff --git a/src/automation/orchestrator.py b/src/automation/orchestrator.py index 94ae802..1afa8d8 100644 --- a/src/automation/orchestrator.py +++ b/src/automation/orchestrator.py @@ -10,10 +10,9 @@ import time import uuid from dataclasses import dataclass -from pathlib import Path from typing import Any -from automation.remote.agents import AppControllerAgent, VMNavigatorAgent, VMSession, VMTarget +from src.automation.remote.agents import AppControllerAgent, VMNavigatorAgent, VMSession, VMTarget @dataclass @@ -369,8 +368,8 @@ def register_connection(self, connection): def cli_main(): - """CLI entry point for production deployment""" - parser = argparse.ArgumentParser(description="VM Automation - Production GUI Automation System") + """CLI entry point for deployment""" + parser = argparse.ArgumentParser(description="VM Automation - GUI Automation System") parser.add_argument("--config", "-c", help="Configuration file path (JSON)") parser.add_argument( "--connection", @@ -448,14 +447,6 @@ def validate_environment() -> bool: """Validate that environment is ready for automation""" issues = [] - # Check for AI models - models_dir = Path(__file__).parent / "models" - yolo_path = models_dir / "yolov8s.onnx" - - if not yolo_path.exists(): - issues.append(f"โŒ YOLO model missing: {yolo_path}") - issues.append(" Run: uv run src/setup_models.py") - # Check configuration if not os.path.exists("vm_config.json") and not os.getenv("VM_HOST"): issues.append("โŒ No configuration found") diff --git a/src/automation/remote/__init__.py b/src/automation/remote/__init__.py index 303c25d..8f94290 100644 --- a/src/automation/remote/__init__.py +++ b/src/automation/remote/__init__.py @@ -1,10 +1,15 @@ """Remote VM automation module""" -from .agents import AppControllerAgent, VMNavigatorAgent, VMSession, VMTarget -from .connections import DesktopConnection, RDPConnection, VNCConnection, create_connection -from .tools import InputActions, ScreenCapture +from src.automation.remote.agents import AppControllerAgent, VMNavigatorAgent, VMSession, VMTarget +from src.automation.remote.connections import ( + DesktopConnection, + RDPConnection, + VNCConnection, + create_connection, +) +from src.automation.remote.tools import InputActions, ScreenCapture -__all__ = [ +__all__ = [ # noqa: RUF022 # Connections "VNCConnection", "RDPConnection", diff --git a/src/automation/remote/agents/__init__.py b/src/automation/remote/agents/__init__.py index e203856..2866e53 100644 --- a/src/automation/remote/agents/__init__.py +++ b/src/automation/remote/agents/__init__.py @@ -1,8 +1,8 @@ """VM automation agents""" -from .app_controller import AppControllerAgent -from .shared_context import VMConnectionInfo, VMSession, VMTarget -from .vm_navigator import VMNavigatorAgent +from src.automation.remote.agents.app_controller import AppControllerAgent +from src.automation.remote.agents.shared_context import VMConnectionInfo, VMSession, VMTarget +from src.automation.remote.agents.vm_navigator import VMNavigatorAgent __all__ = [ "AppControllerAgent", diff --git a/src/automation/remote/agents/app_controller.py b/src/automation/remote/agents/app_controller.py index 19fdb8f..b6723a3 100644 --- a/src/automation/remote/agents/app_controller.py +++ b/src/automation/remote/agents/app_controller.py @@ -1,14 +1,14 @@ -"""App Controller Agent - Production version without mocks""" +"""App Controller Agent - Version without mocks""" import asyncio import time from typing import Any -from automation.remote.agents.shared_context import VMSession, VMTarget +from src.automation.remote.agents.shared_context import VMSession, VMTarget class AppControllerTools: - """Production tools for App Controller Agent""" + """Tools for App Controller Agent""" def __init__(self, session: VMSession, vm_target: VMTarget, shared_components: dict[str, Any]): """ @@ -307,7 +307,7 @@ def scroll_and_search(self, element_description: str, max_scrolls: int = 3) -> d class AppControllerAgent: - """Production App Controller Agent""" + """App Controller Agent""" def __init__(self, session: VMSession, vm_target: VMTarget, shared_components: dict[str, Any]): """ diff --git a/src/automation/remote/agents/vm_navigator.py b/src/automation/remote/agents/vm_navigator.py index 2757d27..07e5fcf 100644 --- a/src/automation/remote/agents/vm_navigator.py +++ b/src/automation/remote/agents/vm_navigator.py @@ -1,17 +1,12 @@ -"""VM Navigator Agent - Production version without mocks""" +"""VM Navigator Agent - Version without mocks""" import asyncio -import sys import time -from pathlib import Path from typing import Any -# Add src to path for clean OCR imports -sys.path.append(str(Path(__file__).parent.parent.parent)) - # ActionVerifier functionality now available via vision.verification module -from automation.remote.agents.shared_context import VMSession, VMTarget -from vision import ( +from src.automation.remote.agents.shared_context import VMSession, VMTarget +from src.vision import ( detect_ui_elements, extract_text, find_elements_by_text, @@ -21,10 +16,10 @@ class VMNavigatorTools: - """Production tools for VM Navigator Agent""" + """Tools for VM Navigator Agent""" def __init__(self, session: VMSession, vm_target: VMTarget): - """Initialize production tools for VM navigation""" + """Initialize tools for VM navigation""" self.session = session self.vm_target = vm_target @@ -504,7 +499,7 @@ def verify_application_loaded_enhanced(self) -> dict[str, Any]: class VMNavigatorAgent: - """Production VM Navigator Agent""" + """VM Navigator Agent""" def __init__( self, diff --git a/src/automation/remote/connections/__init__.py b/src/automation/remote/connections/__init__.py index c9943d9..2917711 100644 --- a/src/automation/remote/connections/__init__.py +++ b/src/automation/remote/connections/__init__.py @@ -1,8 +1,8 @@ """Remote VM connection implementations""" -from .desktop import DesktopConnection -from .rdp import RDPConnection -from .vnc import VNCConnection +from src.automation.remote.connections.desktop import DesktopConnection +from src.automation.remote.connections.rdp import RDPConnection +from src.automation.remote.connections.vnc import VNCConnection __all__ = ["DesktopConnection", "RDPConnection", "VNCConnection"] diff --git a/src/automation/remote/connections/desktop.py b/src/automation/remote/connections/desktop.py index 54bd505..dd02b6d 100644 --- a/src/automation/remote/connections/desktop.py +++ b/src/automation/remote/connections/desktop.py @@ -6,8 +6,8 @@ import cv2 import numpy as np -from automation.core import ActionResult, ConnectionResult -from automation.core.base import VMConnection +from src.automation.core import ActionResult, ConnectionResult +from src.automation.core.base import VMConnection class DesktopConnection(VMConnection): diff --git a/src/automation/remote/connections/rdp.py b/src/automation/remote/connections/rdp.py index bf1d230..301bfe9 100644 --- a/src/automation/remote/connections/rdp.py +++ b/src/automation/remote/connections/rdp.py @@ -11,8 +11,8 @@ import cv2 import numpy as np -from automation.core import ActionResult, ConnectionResult -from automation.core.base import VMConnection +from src.automation.core import ActionResult, ConnectionResult +from src.automation.core.base import VMConnection class RDPConnection(VMConnection): diff --git a/src/automation/remote/connections/vnc.py b/src/automation/remote/connections/vnc.py index 0fc61e0..84d5f08 100644 --- a/src/automation/remote/connections/vnc.py +++ b/src/automation/remote/connections/vnc.py @@ -6,8 +6,8 @@ import numpy as np import vncdotool.api as vnc -from automation.core import ActionResult, ConnectionResult -from automation.core.base import VMConnection +from src.automation.core import ActionResult, ConnectionResult +from src.automation.core.base import VMConnection class VNCConnection(VMConnection): diff --git a/src/automation/remote/tools/__init__.py b/src/automation/remote/tools/__init__.py index 1d53f92..5b39087 100644 --- a/src/automation/remote/tools/__init__.py +++ b/src/automation/remote/tools/__init__.py @@ -1,6 +1,6 @@ """Remote VM interaction tools""" -from .input_actions import InputActions -from .screen_capture import ScreenCapture +from src.automation.remote.tools.input_actions import InputActions +from src.automation.remote.tools.screen_capture import ScreenCapture __all__ = ["InputActions", "ScreenCapture"] diff --git a/src/automation/remote/tools/input_actions.py b/src/automation/remote/tools/input_actions.py index 0820088..6af5525 100644 --- a/src/automation/remote/tools/input_actions.py +++ b/src/automation/remote/tools/input_actions.py @@ -2,8 +2,8 @@ import time -from automation.core import ActionResult -from automation.core.base import VMConnection +from src.automation.core import ActionResult +from src.automation.core.base import VMConnection class InputActions: diff --git a/src/automation/remote/tools/screen_capture.py b/src/automation/remote/tools/screen_capture.py index bbf356c..b643b85 100644 --- a/src/automation/remote/tools/screen_capture.py +++ b/src/automation/remote/tools/screen_capture.py @@ -5,8 +5,8 @@ import cv2 import numpy as np -from automation.core.base import VMConnection -from automation.remote import create_connection +from src.automation.core.base import VMConnection +from src.automation.remote import create_connection class ScreenCapture: diff --git a/src/vision/__init__.py b/src/vision/__init__.py index 782ca08..954b9f7 100644 --- a/src/vision/__init__.py +++ b/src/vision/__init__.py @@ -1,9 +1,10 @@ """Pure Computer Vision Functions for Generic Use This module provides clean, standalone computer vision functions using: -- YOLO for UI element detection +- Template matching for UI element detection with multi-template support - PaddleOCR for text recognition - Combined search and analysis capabilities +- Template library management No dependencies on connections, sessions, workflows, or adapters. Perfect for MCP servers and LLM function calling. @@ -11,24 +12,41 @@ Example usage: import cv2 from vision import detect_ui_elements, extract_text, find_elements_by_text + from vision.template_manager import get_template_manager, TemplateRequest, TemplateStrategy image = cv2.imread("screenshot.png") - elements = detect_ui_elements(image) + + # Multi-template detection + template_manager = get_template_manager() + template_sources = template_manager.resolve_templates([ + TemplateRequest(strategy=TemplateStrategy.LIBRARY, data={"id": "submit_button"}), + TemplateRequest(strategy=TemplateStrategy.BASE64, data=base64_template) + ]) + elements = detect_ui_elements(image, template_sources) + + # Text extraction and combined search text_results = extract_text(image) buttons = find_elements_by_text(image, "Submit") """ -from .detector import Detection, detect_ui_elements -from .finder import ( +from src.vision.detector_template import Detection, detect_ui_elements +from src.vision.finder import ( ScreenAnalysis, UIElement, analyze_screen_content, find_clickable_elements, find_elements_by_text, ) -from .reader import TextResult, extract_text, extract_text_from_region -from .setup_models import download_models, get_model_paths, setup_models -from .verification import ( +from src.vision.reader import TextResult, extract_text, extract_text_from_region +from src.vision.template_manager import ( + TemplateManager, + TemplateRequest, + TemplateSource, + TemplateStrategy, + TemplateValidationError, + get_template_manager, +) +from src.vision.verification import ( VerificationResult, compare_screenshots, create_diff_visualization, @@ -41,7 +59,7 @@ __version__ = "1.0.0" -__all__ = [ +__all__ = [ # noqa: RUF022 # Detection functions "detect_ui_elements", "Detection", @@ -50,22 +68,25 @@ "extract_text_from_region", "TextResult", # Combined search functions - "find_elements_by_text", - "find_clickable_elements", "analyze_screen_content", + "find_clickable_elements", + "find_elements_by_text", "UIElement", "ScreenAnalysis", + # Template management + "TemplateManager", + "TemplateRequest", + "TemplateSource", + "TemplateStrategy", + "TemplateValidationError", + "get_template_manager", # Verification functions + "create_diff_visualization", + "compare_screenshots", + "wait_for_element", + "VerificationResult", "verify_click_success", - "verify_text_input", "verify_element_present", "verify_page_loaded", - "wait_for_element", - "compare_screenshots", - "create_diff_visualization", - "VerificationResult", - # Model management - "setup_models", - "download_models", - "get_model_paths", + "verify_text_input", ] diff --git a/src/vision/detector.py b/src/vision/detector.py deleted file mode 100644 index ffe509e..0000000 --- a/src/vision/detector.py +++ /dev/null @@ -1,369 +0,0 @@ -"""Pure YOLO Detection Functions - -Standalone YOLO-based UI element detection with no external dependencies. - -UI Element Detection. Uses YOLO to detect buttons, forms, and UI components -""" - -import os -from dataclasses import dataclass -from pathlib import Path - -import cv2 -import numpy as np -import onnxruntime as ort - - -@dataclass -class Detection: - """UI element detection result""" - - class_name: str - confidence: float - bbox: tuple[int, int, int, int] # x1, y1, x2, y2 - center: tuple[int, int] - area: int - - -# UI-focused COCO classes (desktop/screen relevant objects) -UI_FOCUSED_CLASSES = { - 0: "person", # User avatars, profile pictures - 62: "tv", # Monitors, displays - 63: "laptop", # Device detection - 64: "mouse", # Peripheral detection - 65: "remote", # Control devices - 66: "keyboard", # Input device detection - 67: "cell phone", # Mobile device detection - 73: "book", # Document/reading interfaces - 74: "clock", # Time displays, widgets - 76: "scissors", # Cut/edit tools in UI -} - -# Full COCO class set for comprehensive detection -FULL_COCO_CLASSES = { - 0: "person", - 1: "bicycle", - 2: "car", - 3: "motorcycle", - 4: "airplane", - 5: "bus", - 6: "train", - 7: "truck", - 8: "boat", - 9: "traffic light", - 10: "fire hydrant", - 11: "stop sign", - 12: "parking meter", - 13: "bench", - 14: "bird", - 15: "cat", - 16: "dog", - 17: "horse", - 18: "sheep", - 19: "cow", - 20: "elephant", - 21: "bear", - 22: "zebra", - 23: "giraffe", - 24: "backpack", - 25: "umbrella", - 26: "handbag", - 27: "tie", - 28: "suitcase", - 29: "frisbee", - 30: "skis", - 31: "snowboard", - 32: "sports ball", - 33: "kite", - 34: "baseball bat", - 35: "baseball glove", - 36: "skateboard", - 37: "surfboard", - 38: "tennis racket", - 39: "bottle", - 40: "wine glass", - 41: "cup", - 42: "fork", - 43: "knife", - 44: "spoon", - 45: "bowl", - 46: "banana", - 47: "apple", - 48: "sandwich", - 49: "orange", - 50: "broccoli", - 51: "carrot", - 52: "hot dog", - 53: "pizza", - 54: "donut", - 55: "cake", - 56: "chair", - 57: "couch", - 58: "potted plant", - 59: "bed", - 60: "dining table", - 61: "toilet", - 62: "tv", - 63: "laptop", - 64: "mouse", - 65: "remote", - 66: "keyboard", - 67: "cell phone", - 68: "microwave", - 69: "oven", - 70: "toaster", - 71: "sink", - 72: "refrigerator", - 73: "book", - 74: "clock", - 75: "vase", - 76: "scissors", - 77: "teddy bear", - 78: "hair drier", - 79: "toothbrush", -} - - -def detect_ui_elements( - image: np.ndarray, - confidence_threshold: float = 0.6, - model_path: str | None = None, - ui_focused: bool = True, - max_detections: int = 100, -) -> list[Detection]: - """ - Detect UI elements in image using YOLO - - Args: - image: Input image as numpy array (BGR format from cv2) - confidence_threshold: Minimum confidence for detections (0.0-1.0) - model_path: Path to YOLO ONNX model (uses default if None) - ui_focused: If True, filter to UI-relevant classes only - max_detections: Maximum number of detections to return - - Returns: - List of Detection objects sorted by confidence - - Example: - image = cv2.imread("screenshot.png") - detections = detect_ui_elements(image, confidence_threshold=0.7) - for det in detections: - print(f"Found {det.class_name} at {det.center} with confidence {det.confidence}") - """ - # Get model path - if model_path is None: - model_path = _get_default_model_path() - - if not os.path.exists(model_path): - raise FileNotFoundError( - f"YOLO model not found at {model_path}. Run setup_models() to download the model." - ) - - # Choose class set - classes = UI_FOCUSED_CLASSES if ui_focused else FULL_COCO_CLASSES - - # Initialize ONNX session - session = ort.InferenceSession(model_path) - input_name = session.get_inputs()[0].name - input_shape = session.get_inputs()[0].shape - input_height, input_width = input_shape[2], input_shape[3] - - # Preprocess image - processed_image = _preprocess_image(image, input_width, input_height) - - # Run inference - outputs = session.run(None, {input_name: processed_image}) - - # Postprocess results - detections = _postprocess_outputs( - outputs, - image.shape[:2], # original height, width - input_width, - input_height, - classes, - confidence_threshold, - ) - - # Sort by confidence and limit results - detections.sort(key=lambda x: x.confidence, reverse=True) - return detections[:max_detections] - - -def _get_default_model_path() -> str: - """Get default YOLO model path""" - current_dir = Path(__file__).parent - models_dir = current_dir / "models" - return str(models_dir / "yolov8s.onnx") - - -def _preprocess_image(image: np.ndarray, target_width: int, target_height: int) -> np.ndarray: - """Preprocess image for YOLO inference""" - # Resize image - resized = cv2.resize(image, (target_width, target_height)) - - # Convert BGR to RGB - rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) - - # Normalize to [0, 1] and convert to float32 - normalized = rgb.astype(np.float32) / 255.0 - - # Transpose to CHW format and add batch dimension - input_tensor = normalized.transpose(2, 0, 1)[np.newaxis, ...] - - return input_tensor - - -def _postprocess_outputs( - outputs: list[np.ndarray], - original_shape: tuple[int, int], - input_width: int, - input_height: int, - classes: dict, - confidence_threshold: float, -) -> list[Detection]: - """Postprocess YOLO outputs to detections""" - predictions = outputs[0] # Shape: [1, 84, 8400] for YOLOv8s - - # Transpose to [8400, 84] - predictions = predictions.transpose(0, 2, 1)[0] - - detections = [] - orig_h, orig_w = original_shape - - for pred in predictions: - # Extract box coordinates and confidence - x_center, y_center, width, height = pred[0:4] - confidence = pred[4] - class_scores = pred[5:] - - if confidence < confidence_threshold: - continue - - # Get class with highest score - class_id = np.argmax(class_scores) - class_confidence = class_scores[class_id] * confidence - - if class_confidence < confidence_threshold: - continue - - # Skip classes not in our active set - if class_id not in classes: - continue - - # Convert to original image coordinates - x_center *= orig_w / input_width - y_center *= orig_h / input_height - width *= orig_w / input_width - height *= orig_h / input_height - - # Convert to bbox format - x1 = int(x_center - width / 2) - y1 = int(y_center - height / 2) - x2 = int(x_center + width / 2) - y2 = int(y_center + height / 2) - - # Ensure bbox is within image bounds - x1 = max(0, min(x1, orig_w)) - y1 = max(0, min(y1, orig_h)) - x2 = max(0, min(x2, orig_w)) - y2 = max(0, min(y2, orig_h)) - - # Calculate area - area = (x2 - x1) * (y2 - y1) - - detection = Detection( - class_name=classes.get(class_id, f"class_{class_id}"), - confidence=float(class_confidence), - bbox=(x1, y1, x2, y2), - center=(int(x_center), int(y_center)), - area=area, - ) - - detections.append(detection) - - # Apply Non-Maximum Suppression - detections = _apply_nms(detections, iou_threshold=0.5) - - return detections - - -def _apply_nms(detections: list[Detection], iou_threshold: float) -> list[Detection]: - """Apply Non-Maximum Suppression""" - if not detections: - return detections - - # Sort by confidence - detections.sort(key=lambda x: x.confidence, reverse=True) - - filtered_detections = [] - - for detection in detections: - # Check if this detection overlaps significantly with any kept detection - keep = True - for kept_detection in filtered_detections: - if _calculate_iou(detection.bbox, kept_detection.bbox) > iou_threshold: - keep = False - break - - if keep: - filtered_detections.append(detection) - - return filtered_detections - - -def _calculate_iou(box1: tuple[int, int, int, int], box2: tuple[int, int, int, int]) -> float: - """Calculate Intersection over Union""" - x1_1, y1_1, x2_1, y2_1 = box1 - x1_2, y1_2, x2_2, y2_2 = box2 - - # Calculate intersection area - x1_i = max(x1_1, x1_2) - y1_i = max(y1_1, y1_2) - x2_i = min(x2_1, x2_2) - y2_i = min(y2_1, y2_2) - - if x2_i <= x1_i or y2_i <= y1_i: - return 0.0 - - intersection_area = (x2_i - x1_i) * (y2_i - y1_i) - - # Calculate union area - area1 = (x2_1 - x1_1) * (y2_1 - y1_1) - area2 = (x2_2 - x1_2) * (y2_2 - y1_2) - union_area = area1 + area2 - intersection_area - - return intersection_area / union_area if union_area > 0 else 0.0 - - -def draw_detections(image: np.ndarray, detections: list[Detection]) -> np.ndarray: - """ - Draw detection bounding boxes on image - - Args: - image: Input image - detections: List of detections to draw - - Returns: - Image with drawn bounding boxes - """ - result_image = image.copy() - - for detection in detections: - x1, y1, x2, y2 = detection.bbox - - # Draw bounding box - cv2.rectangle(result_image, (x1, y1), (x2, y2), (0, 255, 0), 2) - - # Draw label - label = f"{detection.class_name}: {detection.confidence:.2f}" - label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] - cv2.rectangle( - result_image, - (x1, y1 - label_size[1] - 10), - (x1 + label_size[0], y1), - (0, 255, 0), - -1, - ) - cv2.putText(result_image, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2) - - return result_image diff --git a/src/vision/detector_template.py b/src/vision/detector_template.py new file mode 100644 index 0000000..46f412c --- /dev/null +++ b/src/vision/detector_template.py @@ -0,0 +1,582 @@ +"""Template Matching UI Element Detection + +Comprehensive template matching implementation for finding custom UI elements. +Uses OpenCV's template matching with multi-scale and rotation support. +Supports single and multi-template detection strategies. +""" + +from dataclasses import dataclass + +import cv2 +import numpy as np + +from src.vision.template_manager import ( + TemplateManager, + TemplateSource, + TemplateStrategy, + get_template_manager, +) + + +@dataclass +class Detection: + """Template matching detection result""" + + template_name: str + confidence: float + bbox: tuple[int, int, int, int] # x1, y1, x2, y2 + center: tuple[int, int] + area: int + scale: float = 1.0 + angle: float = 0.0 + method: str = "TM_CCOEFF_NORMED" + + # Enhanced multi-template fields + template_source: TemplateSource | None = None + template_match_method: str = "single" + template_confidence: float = 1.0 + + +class TemplateMatchingMethods: + """Available OpenCV template matching methods""" + + TM_CCOEFF = cv2.TM_CCOEFF + TM_CCOEFF_NORMED = cv2.TM_CCOEFF_NORMED + TM_CCORR = cv2.TM_CCORR + TM_CCORR_NORMED = cv2.TM_CCORR_NORMED + TM_SQDIFF = cv2.TM_SQDIFF + TM_SQDIFF_NORMED = cv2.TM_SQDIFF_NORMED + + @classmethod + def get_method_by_name(cls, method_name: str) -> int: + """Get OpenCV method constant by name""" + method_map = { + "TM_CCOEFF": cls.TM_CCOEFF, + "TM_CCOEFF_NORMED": cls.TM_CCOEFF_NORMED, + "TM_CCORR": cls.TM_CCORR, + "TM_CCORR_NORMED": cls.TM_CCORR_NORMED, + "TM_SQDIFF": cls.TM_SQDIFF, + "TM_SQDIFF_NORMED": cls.TM_SQDIFF_NORMED, + } + return method_map.get(method_name, cls.TM_CCOEFF_NORMED) + + +def detect_ui_elements( + image: np.ndarray, + template_sources: TemplateSource | list[TemplateSource] | np.ndarray, + template_name: str = "template", + confidence_threshold: float = 0.8, + method: str = "auto", + max_detections: int = 100, + enable_multiscale: bool = True, + scale_range: tuple[float, float] = (0.5, 2.0), + scale_steps: int = 20, + enable_rotation: bool = False, + rotation_range: tuple[float, float] = (-30, 30), + rotation_steps: int = 13, + nms_threshold: float = 0.3, + template_manager: TemplateManager | None = None, +) -> list[Detection]: + """ + Detect UI elements using template matching with advanced features + + Args: + image: Input image as numpy array (BGR format from cv2) + template_sources: Template source(s) - TemplateSource, list of TemplateSource, or np.ndarray for backward compatibility + template_name: Name identifier for the template (used if template_sources is np.ndarray) + confidence_threshold: Minimum confidence for detections (0.0-1.0) + method: Template matching method ("auto", "TM_CCOEFF_NORMED", "TM_SQDIFF_NORMED", etc.) + max_detections: Maximum number of detections to return + enable_multiscale: Enable multi-scale template matching + scale_range: Range of scales to test (min_scale, max_scale) + scale_steps: Number of scale steps to test + enable_rotation: Enable rotation-invariant template matching + rotation_range: Range of rotations in degrees (min_angle, max_angle) + rotation_steps: Number of rotation steps to test + nms_threshold: Non-maximum suppression threshold + template_manager: Optional template manager instance + + Returns: + List of Detection objects sorted by confidence + + Example: + # Multi-template detection + template_manager = get_template_manager() + template_sources = template_manager.resolve_templates([ + TemplateRequest(strategy=TemplateStrategy.LIBRARY, data={"id": "submit_button"}), + TemplateRequest(strategy=TemplateStrategy.BASE64, data=base64_template) + ]) + detections = detect_ui_elements(image, template_sources) + + # Single template (backward compatibility) + template = cv2.imread("button_template.png") + detections = detect_ui_elements(image, template, "login_button") + """ + if image is None: + return [] + + # Normalize template_sources to list of TemplateSource objects + if template_manager is None: + template_manager = get_template_manager() + + template_sources_list = _normalize_template_sources( + template_sources, template_name, template_manager + ) + + if not template_sources_list: + return [] + + # Convert image to grayscale for better template matching + image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) if len(image.shape) == 3 else image + + # Detect with multiple templates if needed + if len(template_sources_list) == 1: + detections = _detect_single_template( + image_gray, + template_sources_list[0], + method, + confidence_threshold, + enable_multiscale, + scale_range, + scale_steps, + enable_rotation, + rotation_range, + rotation_steps, + ) + else: + detections = _detect_multi_template( + image_gray, + template_sources_list, + method, + confidence_threshold, + enable_multiscale, + scale_range, + scale_steps, + enable_rotation, + rotation_range, + rotation_steps, + ) + + # Apply Cross-Template Non-Maximum Suppression + detections = _apply_cross_template_nms(detections, nms_threshold) + + # Sort by confidence and limit results + detections.sort(key=lambda x: x.confidence, reverse=True) + return detections[:max_detections] + + +def _select_best_method(image: np.ndarray, template: np.ndarray) -> str: + """ + Automatically select the best template matching method based on image characteristics + + Returns: + Best method name for the given image and template + """ + # For UI element detection, TM_CCOEFF_NORMED is generally the best choice + # as it's most robust to lighting variations and provides reliable results + + # Calculate some basic image statistics to inform method selection + template_std = np.std(template.astype(np.float32)) + image_std = np.std(image.astype(np.float32)) + + # If template has very low variance (e.g., solid color), use SQDIFF + if template_std < 10: + return "TM_SQDIFF_NORMED" + + # If there's a big difference in image vs template variance, use CCOEFF + # which handles lighting differences better + if abs(image_std - template_std) > 20: + return "TM_CCOEFF_NORMED" + + # Default to the most robust method based on research + return "TM_CCOEFF_NORMED" + + +def _basic_template_matching( + image: np.ndarray, + template: np.ndarray, + template_name: str, + method: int, + confidence_threshold: float, +) -> list[Detection]: + """Basic template matching without scaling or rotation""" + result = cv2.matchTemplate(image, template, method) + return _process_match_result( + result, template, template_name, method, confidence_threshold, scale=1.0, angle=0.0 + ) + + +def _advanced_template_matching( + image: np.ndarray, + template: np.ndarray, + template_name: str, + method: int, + confidence_threshold: float, + enable_multiscale: bool, + scale_range: tuple[float, float], + scale_steps: int, + enable_rotation: bool, + rotation_range: tuple[float, float], + rotation_steps: int, +) -> list[Detection]: + """Advanced template matching with scaling and rotation""" + detections = [] + + # Generate scale factors + scales = [1.0] # Always include original scale + if enable_multiscale: + min_scale, max_scale = scale_range + scales = np.linspace(min_scale, max_scale, scale_steps) + + # Generate rotation angles + angles = [0.0] # Always include no rotation + if enable_rotation: + min_angle, max_angle = rotation_range + angles = np.linspace(min_angle, max_angle, rotation_steps) + + for scale in scales: + for angle in angles: + # Transform template + transformed_template = _transform_template(template, scale, angle) + if transformed_template is None: + continue + + # Perform template matching + result = cv2.matchTemplate(image, transformed_template, method) + + # Process results and add to detections + scale_detections = _process_match_result( + result, + transformed_template, + template_name, + method, + confidence_threshold, + scale, + angle, + ) + detections.extend(scale_detections) + + return detections + + +def _process_match_result( + result: np.ndarray, + template: np.ndarray, + template_name: str, + method: int, + confidence_threshold: float, + scale: float, + angle: float, +) -> list[Detection]: + """Process template matching result and create Detection objects""" + # Handle different method types for thresholding + if method in [cv2.TM_SQDIFF, cv2.TM_SQDIFF_NORMED]: + # For SQDIFF methods, lower values are better matches + locations = np.where(result <= (1.0 - confidence_threshold)) + confidences = 1.0 - result[locations] + else: + # For other methods, higher values are better matches + locations = np.where(result >= confidence_threshold) + confidences = result[locations] + + detections = [] + template_h, template_w = template.shape[:2] + + for pt, conf in zip(zip(*locations[::-1], strict=False), confidences, strict=False): + x1, y1 = pt + x2, y2 = x1 + template_w, y1 + template_h + center = (x1 + template_w // 2, y1 + template_h // 2) + area = template_w * template_h + + detection = Detection( + template_name=template_name, + confidence=float(conf), + bbox=(x1, y1, x2, y2), + center=center, + area=area, + scale=scale, + angle=angle, + method=_get_method_name(method), + ) + detections.append(detection) + + return detections + + +def _transform_template(template: np.ndarray, scale: float, angle: float) -> np.ndarray | None: + """Transform template with scaling and rotation""" + h, w = template.shape[:2] + + # Scale template + if scale != 1.0: + new_w, new_h = int(w * scale), int(h * scale) + if new_w < 1 or new_h < 1: + return None + template = cv2.resize(template, (new_w, new_h), interpolation=cv2.INTER_CUBIC) + h, w = new_h, new_w + + # Rotate template + if angle != 0.0: + # Calculate rotation matrix + center = (w // 2, h // 2) + rotation_matrix = cv2.getRotationMatrix2D(center, angle, 1.0) + + # Calculate new dimensions to contain rotated image + cos_a, sin_a = abs(rotation_matrix[0, 0]), abs(rotation_matrix[0, 1]) + new_w = int((h * sin_a) + (w * cos_a)) + new_h = int((h * cos_a) + (w * sin_a)) + + # Adjust the rotation matrix to account for translation + rotation_matrix[0, 2] += (new_w / 2) - center[0] + rotation_matrix[1, 2] += (new_h / 2) - center[1] + + # Apply rotation + template = cv2.warpAffine( + template, + rotation_matrix, + (new_w, new_h), + flags=cv2.INTER_CUBIC, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0, 0), + ) + + return template + + +def _calculate_iou(box1: tuple[int, int, int, int], box2: tuple[int, int, int, int]) -> float: + """Calculate Intersection over Union""" + x1_1, y1_1, x2_1, y2_1 = box1 + x1_2, y1_2, x2_2, y2_2 = box2 + + # Calculate intersection area + x1_i = max(x1_1, x1_2) + y1_i = max(y1_1, y1_2) + x2_i = min(x2_1, x2_2) + y2_i = min(y2_1, y2_2) + + if x2_i <= x1_i or y2_i <= y1_i: + return 0.0 + + intersection_area = (x2_i - x1_i) * (y2_i - y1_i) + + # Calculate union area + area1 = (x2_1 - x1_1) * (y2_1 - y1_1) + area2 = (x2_2 - x1_2) * (y2_2 - y1_2) + union_area = area1 + area2 - intersection_area + + return intersection_area / union_area if union_area > 0 else 0.0 + + +def _get_method_name(method: int) -> str: + """Get method name from OpenCV constant""" + method_names = { + cv2.TM_CCOEFF: "TM_CCOEFF", + cv2.TM_CCOEFF_NORMED: "TM_CCOEFF_NORMED", + cv2.TM_CCORR: "TM_CCORR", + cv2.TM_CCORR_NORMED: "TM_CCORR_NORMED", + cv2.TM_SQDIFF: "TM_SQDIFF", + cv2.TM_SQDIFF_NORMED: "TM_SQDIFF_NORMED", + } + return method_names.get(method, "UNKNOWN") + + +def _normalize_template_sources( + template_sources: TemplateSource | list[TemplateSource] | np.ndarray, + template_name: str, + template_manager: TemplateManager, +) -> list[TemplateSource]: + """Normalize template_sources parameter to list of TemplateSource objects""" + if isinstance(template_sources, TemplateSource): + return [template_sources] + + elif isinstance(template_sources, list): + # Check if it's a list of TemplateSource objects + if template_sources and all(isinstance(ts, TemplateSource) for ts in template_sources): + return template_sources + elif not template_sources: + return [] + else: + raise ValueError( + f"List contains non-TemplateSource objects: {[type(ts) for ts in template_sources]}" + ) + + elif isinstance(template_sources, np.ndarray): + # Backward compatibility: create TemplateSource from numpy array + template_source = TemplateSource( + strategy=TemplateStrategy.BASE64, + template=template_sources, + metadata={ + "name": template_name, + "source": "numpy_array", + "size": template_sources.shape, + }, + ) + return [template_source] + + else: + raise ValueError(f"Unsupported template_sources type: {type(template_sources)}") + + +def _detect_single_template( + image_gray: np.ndarray, + template_source: TemplateSource, + method: str, + confidence_threshold: float, + enable_multiscale: bool, + scale_range: tuple[float, float], + scale_steps: int, + enable_rotation: bool, + rotation_range: tuple[float, float], + rotation_steps: int, +) -> list[Detection]: + """Detect using single template""" + template = template_source.template + template_name = template_source.metadata.get("name", "template") + + # Convert template to grayscale + template_gray = ( + cv2.cvtColor(template, cv2.COLOR_BGR2GRAY) if len(template.shape) == 3 else template + ) + + # Auto-select best method or use specified method + if method == "auto": + method = _select_best_method(image_gray, template_gray) + + # Get OpenCV method constant + cv_method = TemplateMatchingMethods.get_method_by_name(method) + + if enable_multiscale or enable_rotation: + detections = _advanced_template_matching( + image_gray, + template_gray, + template_name, + cv_method, + confidence_threshold, + enable_multiscale, + scale_range, + scale_steps, + enable_rotation, + rotation_range, + rotation_steps, + ) + else: + detections = _basic_template_matching( + image_gray, template_gray, template_name, cv_method, confidence_threshold + ) + + # Add template source info to detections + for detection in detections: + detection.template_source = template_source + detection.template_match_method = "single" + detection.template_confidence = template_source.confidence_weight + + return detections + + +def _detect_multi_template( + image_gray: np.ndarray, + template_sources: list[TemplateSource], + method: str, + confidence_threshold: float, + enable_multiscale: bool, + scale_range: tuple[float, float], + scale_steps: int, + enable_rotation: bool, + rotation_range: tuple[float, float], + rotation_steps: int, +) -> list[Detection]: + """Detect using multiple templates""" + all_detections = [] + + for template_source in template_sources: + # Get detections for this template + template_detections = _detect_single_template( + image_gray, + template_source, + method, + confidence_threshold * template_source.confidence_weight, # Weight the threshold + enable_multiscale, + scale_range, + scale_steps, + enable_rotation, + rotation_range, + rotation_steps, + ) + + # Apply template confidence weighting + for detection in template_detections: + detection.confidence *= template_source.confidence_weight + detection.template_match_method = "multi" + + all_detections.extend(template_detections) + + return all_detections + + +def _apply_cross_template_nms(detections: list[Detection], iou_threshold: float) -> list[Detection]: + """Apply Non-Maximum Suppression across multiple templates""" + if not detections: + return detections + + # Sort by confidence + detections.sort(key=lambda x: x.confidence, reverse=True) + + filtered_detections = [] + + for detection in detections: + # Check if this detection overlaps significantly with any kept detection + keep = True + for kept_detection in filtered_detections: + if _calculate_iou(detection.bbox, kept_detection.bbox) > iou_threshold: + keep = False + break + + if keep: + filtered_detections.append(detection) + + return filtered_detections + + +def draw_detections(image: np.ndarray, detections: list[Detection]) -> np.ndarray: + """ + Draw detection bounding boxes on image + + Args: + image: Input image + detections: List of detections to draw + + Returns: + Image with drawn bounding boxes + """ + result_image = image.copy() + + for detection in detections: + x1, y1, x2, y2 = detection.bbox + + # Draw bounding box + cv2.rectangle(result_image, (x1, y1), (x2, y2), (0, 255, 0), 2) + + # Draw center point + cv2.circle(result_image, detection.center, 3, (0, 0, 255), -1) + + # Create label with template name, confidence, scale, and angle + label = f"{detection.template_name}: {detection.confidence:.2f}" + if detection.scale != 1.0: + label += f" s:{detection.scale:.2f}" + if detection.angle != 0.0: + label += f" a:{detection.angle:.1f}ยฐ" + + # Draw label background + label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] + cv2.rectangle( + result_image, + (x1, y1 - label_size[1] - 10), + (x1 + label_size[0], y1), + (0, 255, 0), + -1, + ) + + # Draw label text + cv2.putText(result_image, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2) + + return result_image diff --git a/src/vision/finder.py b/src/vision/finder.py index 30c7549..0956b2a 100644 --- a/src/vision/finder.py +++ b/src/vision/finder.py @@ -1,9 +1,9 @@ -"""Combined YOLO+OCR Search and Analysis Functions +"""Combined Template+OCR Search and Analysis Functions -Combines YOLO detection and PaddleOCR to provide intelligent UI element finding +Combines template matching detection and PaddleOCR to provide intelligent UI element finding and screen analysis capabilities. -Element Finding: Combines detection and OCR to locate UI elements by text +Element Finding: Combines template detection and OCR to locate UI elements by text """ import math @@ -11,8 +11,9 @@ import numpy as np -from .detector import Detection, detect_ui_elements -from .reader import TextResult, extract_text +from src.vision.detector_template import Detection, detect_ui_elements +from src.vision.reader import TextResult, extract_text +from src.vision.template_manager import TemplateRequest, get_template_manager @dataclass @@ -52,31 +53,29 @@ class ScreenAnalysis: def find_elements_by_text( image: np.ndarray, text_query: str, - confidence_threshold: float = 0.6, - search_radius: int = 100, + confidence_threshold: float = 0.3, case_sensitive: bool = False, ) -> list[UIElement]: """ - Find UI elements that contain or are near specific text + Find UI elements by text content using pure OCR (no template matching) Args: image: Input image text_query: Text to search for - confidence_threshold: Minimum confidence for detections - search_radius: Radius to search for nearby visual elements + confidence_threshold: Minimum confidence for OCR detections case_sensitive: Whether text search is case sensitive Returns: - List of UIElement objects that match the text query + List of UIElement objects that match the text query (text-only) Example: - # Find all Submit buttons + # Find all text containing "Submit" submit_elements = find_elements_by_text(image, "Submit") # Find username input fields username_fields = find_elements_by_text(image, "username", case_sensitive=False) """ - # Get text detections + # Get text detections using pure OCR text_results = extract_text(image, confidence_threshold=confidence_threshold) # Find matching text @@ -84,52 +83,190 @@ def find_elements_by_text( if not case_sensitive: query = query.lower() - matching_text = [] + elements = [] for text_result in text_results: detected_text = text_result.text.strip() if not case_sensitive: detected_text = detected_text.lower() if query in detected_text: - matching_text.append(text_result) + # Create pure text element (no template matching) + text_element = UIElement( + element_type="text", + bbox=text_result.rect_bbox, + center=text_result.center, + confidence=text_result.confidence, + area=text_result.area, + text_detection=text_result, + text=text_result.text, + description=f"Text: '{text_result.text}'", + ) + elements.append(text_element) + + return elements - if not matching_text: + +def find_elements_by_template( + image: np.ndarray, + template_requests: list[TemplateRequest], + confidence_threshold: float = 0.8, +) -> list[UIElement]: + """ + Find UI elements by template matching (no OCR) + + Args: + image: Input image + template_requests: List of template requests to search for + confidence_threshold: Minimum confidence for template detections + + Returns: + List of UIElement objects from template matching (visual-only) + + Example: + # Find specific buttons using templates + template_requests = [ + TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": "submit_blue", "category": "buttons"}, + name="submit_button" + ) + ] + button_elements = find_elements_by_template(image, template_requests) + """ + if not template_requests: return [] - # Get visual detections - visual_detections = detect_ui_elements(image, confidence_threshold=confidence_threshold) + # Get visual detections only + template_manager = get_template_manager() + + try: + template_sources = template_manager.resolve_templates(template_requests) + if not template_sources: + return [] + + visual_detections = detect_ui_elements( + image, template_sources, confidence_threshold=confidence_threshold + ) + except Exception: + return [] + # Create UIElement objects from visual detections elements = [] + for detection in visual_detections: + element = UIElement( + element_type="visual", + bbox=detection.bbox, + center=detection.center, + confidence=detection.confidence, + area=detection.area, + visual_detection=detection, + description=f"{detection.template_name} ({detection.confidence:.2f})", + ) + elements.append(element) - for text_result in matching_text: - # Create text element - text_element = UIElement( - element_type="text", - bbox=text_result.rect_bbox, - center=text_result.center, - confidence=text_result.confidence, - area=text_result.area, - text_detection=text_result, - text=text_result.text, - description=f"Text: '{text_result.text}'", + return elements + + +def find_elements_combined( + image: np.ndarray, + text_query: str | None = None, + template_requests: list[TemplateRequest] | None = None, + combine_nearby: bool = True, + search_radius: int = 100, + confidence_threshold: float = 0.6, + case_sensitive: bool = False, +) -> list[UIElement]: + """ + Smart combined search: explicitly combine OCR and template matching when both are needed + + Args: + image: Input image + text_query: Optional text to search for using OCR + template_requests: Optional template requests for visual detection + combine_nearby: Whether to combine nearby text and visual elements + search_radius: Radius for combining nearby elements + confidence_threshold: Minimum confidence for both OCR and template detections + case_sensitive: Whether text search is case sensitive + + Returns: + List of UIElement objects combining OCR and template matching results + + Example: + # Find "Submit" text near submit button templates + template_requests = [ + TemplateRequest(strategy=TemplateStrategy.LIBRARY, + data={"id": "submit_blue"}, name="submit_button") + ] + combined = find_elements_combined( + image, + text_query="Submit", + template_requests=template_requests, + combine_nearby=True ) + """ + elements = [] - # Look for nearby visual elements - nearby_visual = None + # Get text elements if text_query provided + text_elements = [] + if text_query: + text_elements = find_elements_by_text( + image, text_query, confidence_threshold, case_sensitive + ) + elements.extend(text_elements) + + # Get visual elements if template_requests provided + visual_elements = [] + if template_requests: + visual_elements = find_elements_by_template(image, template_requests, confidence_threshold) + elements.extend(visual_elements) + + # Combine nearby elements if requested + if combine_nearby and text_elements and visual_elements: + combined_elements = _combine_text_visual_elements( + text_elements, visual_elements, search_radius + ) + # Replace individual elements with combined ones + elements = combined_elements + [ + e for e in elements if e.element_type not in ["text", "visual"] + ] + + return elements + + +def _combine_text_visual_elements( + text_elements: list[UIElement], visual_elements: list[UIElement], search_radius: int +) -> list[UIElement]: + """ + Combine nearby text and visual elements into combined elements + """ + combined_elements = [] + used_text_indices = set() + used_visual_indices = set() + + # For each visual element, find the closest text element within radius + for i, visual_elem in enumerate(visual_elements): + closest_text_idx = None min_distance = float("inf") - for visual_det in visual_detections: - distance = _calculate_distance(text_result.center, visual_det.center) + for j, text_elem in enumerate(text_elements): + if j in used_text_indices: + continue + + distance = _calculate_distance(visual_elem.center, text_elem.center) if distance <= search_radius and distance < min_distance: min_distance = distance - nearby_visual = visual_det + closest_text_idx = j + + if closest_text_idx is not None: + text_elem = text_elements[closest_text_idx] + used_text_indices.add(closest_text_idx) + used_visual_indices.add(i) - if nearby_visual: # Create combined element - combined_bbox = _merge_bboxes(text_result.rect_bbox, nearby_visual.bbox) + combined_bbox = _merge_bboxes(visual_elem.bbox, text_elem.bbox) combined_center = ( - (text_result.center[0] + nearby_visual.center[0]) // 2, - (text_result.center[1] + nearby_visual.center[1]) // 2, + (visual_elem.center[0] + text_elem.center[0]) // 2, + (visual_elem.center[1] + text_elem.center[1]) // 2, ) combined_area = (combined_bbox[2] - combined_bbox[0]) * ( combined_bbox[3] - combined_bbox[1] @@ -139,46 +276,64 @@ def find_elements_by_text( element_type="combined", bbox=combined_bbox, center=combined_center, - confidence=(text_result.confidence + nearby_visual.confidence) / 2, + confidence=(visual_elem.confidence + text_elem.confidence) / 2, area=combined_area, - visual_detection=nearby_visual, - text_detection=text_result, - text=text_result.text, - description=f"{nearby_visual.class_name}: '{text_result.text}'", + visual_detection=visual_elem.visual_detection, + text_detection=text_elem.text_detection, + text=text_elem.text, + description=f"{visual_elem.visual_detection.template_name if visual_elem.visual_detection else 'Visual'}: '{text_elem.text}'", ) - elements.append(combined_element) - else: - elements.append(text_element) + combined_elements.append(combined_element) - return elements + # Add remaining uncombined elements + for i, visual_elem in enumerate(visual_elements): + if i not in used_visual_indices: + combined_elements.append(visual_elem) + + for j, text_elem in enumerate(text_elements): + if j not in used_text_indices: + combined_elements.append(text_elem) + + return combined_elements def find_clickable_elements( - image: np.ndarray, confidence_threshold: float = 0.6 + image: np.ndarray, confidence_threshold: float = 0.6, ui_strategy=None ) -> list[UIElement]: """ - Find elements that are likely to be clickable + Find elements that are likely to be clickable using smart UI pattern detection Args: image: Input image confidence_threshold: Minimum confidence for detections + ui_strategy: UI pattern strategy to use (defaults to CLICKABLE_PATTERNS) Returns: List of potentially clickable UIElement objects Example: + # Use default clickable patterns clickable = find_clickable_elements(image) - for element in clickable: - print(f"Clickable: {element.description} at {element.center}") + + # Use specific button patterns only + from src.vision.ui_patterns import BUTTON_PATTERNS + buttons = find_clickable_elements(image, ui_strategy=BUTTON_PATTERNS) """ - # Get all elements - all_elements = _get_all_ui_elements(image, confidence_threshold) + # Import here to avoid circular imports + from src.vision.ui_patterns import CLICKABLE_PATTERNS - clickable_elements = [] + if ui_strategy is None: + ui_strategy = CLICKABLE_PATTERNS - # Define clickable indicators - clickable_visual_classes = {"laptop", "mouse", "remote", "keyboard", "cell phone", "book"} + # Get template requests from strategy + template_requests = ui_strategy.create_template_requests() + # Get all elements using the strategy's templates + all_elements = _get_all_ui_elements(image, confidence_threshold, template_requests) + + clickable_elements = [] + + # Define clickable text indicators clickable_text_terms = { "button", "click", @@ -194,24 +349,25 @@ def find_clickable_elements( "back", "menu", "settings", + "search", + "go", } for element in all_elements: is_clickable = False - # Check visual indicators - if element.visual_detection: - if element.visual_detection.class_name.lower() in clickable_visual_classes: - is_clickable = True + # Visual elements from UI patterns are inherently clickable + if element.element_type in ["visual", "combined"]: + is_clickable = True - # Check text indicators - if element.text: + # Check text indicators for text-only elements + elif element.element_type == "text" and element.text: text_lower = element.text.lower() - if any(term in text_lower for term in clickable_text_terms): - is_clickable = True - # Short text often indicates buttons/links - if len(element.text.split()) <= 3 and len(element.text) <= 20: + # Check for clickable terms + if any(term in text_lower for term in clickable_text_terms) or ( + len(element.text.split()) <= 3 and len(element.text) <= 20 + ): is_clickable = True if is_clickable: @@ -239,8 +395,8 @@ def analyze_screen_content( print(f"Found {len(analysis.clickable_elements)} clickable elements") print(f"Total UI elements: {analysis.summary['total_elements']}") """ - # Get all UI elements - all_elements = _get_all_ui_elements(image, confidence_threshold) + # Get all UI elements (OCR-only by default for screen analysis) + all_elements = _get_all_ui_elements(image, confidence_threshold, template_requests=None) # Separate by type visual_elements = [e for e in all_elements if e.element_type in ["visual", "combined"]] @@ -254,9 +410,9 @@ def analyze_screen_content( "text_elements": len(text_elements), "clickable_elements": len(clickable_elements), "unique_text_content": len(set(e.text for e in text_elements if e.text)), - "visual_classes": list( - set(e.visual_detection.class_name for e in visual_elements if e.visual_detection) - ), + "visual_classes": list({ + e.visual_detection.template_name for e in visual_elements if e.visual_detection + }), } return ScreenAnalysis( @@ -269,10 +425,33 @@ def analyze_screen_content( ) -def _get_all_ui_elements(image: np.ndarray, confidence_threshold: float) -> list[UIElement]: - """Get all UI elements combining YOLO and OCR results""" - # Get detections - visual_detections = detect_ui_elements(image, confidence_threshold=confidence_threshold) +def _get_all_ui_elements( + image: np.ndarray, + confidence_threshold: float, + template_requests: list[TemplateRequest] | None = None, +) -> list[UIElement]: + """Get all UI elements combining template detection and OCR results + + Args: + image: Input image + confidence_threshold: Minimum confidence threshold + template_requests: Optional template requests for visual detection + """ + # Get visual detections if template requests provided + visual_detections = [] + if template_requests: + template_manager = get_template_manager() + try: + template_sources = template_manager.resolve_templates(template_requests) + if template_sources: + visual_detections = detect_ui_elements( + image, template_sources, confidence_threshold=confidence_threshold + ) + except Exception: + # Continue with OCR-only if template resolution fails + visual_detections = [] + + # Get text detections text_results = extract_text(image, confidence_threshold=confidence_threshold) elements = [] @@ -286,7 +465,7 @@ def _get_all_ui_elements(image: np.ndarray, confidence_threshold: float) -> list confidence=detection.confidence, area=detection.area, visual_detection=detection, - description=f"{detection.class_name} ({detection.confidence:.2f})", + description=f"{detection.template_name} ({detection.confidence:.2f})", ) elements.append(element) @@ -356,7 +535,7 @@ def _combine_nearby_elements( visual_detection=visual_elem.visual_detection, text_detection=closest_text.text_detection, text=closest_text.text, - description=f"{visual_elem.visual_detection.class_name}: '{closest_text.text}'", + description=f"{visual_elem.visual_detection.template_name}: '{closest_text.text}'", ) combined_elements.append(combined_element) else: @@ -406,7 +585,7 @@ def find_elements_near_point( Returns: List of nearby UIElement objects sorted by distance """ - all_elements = _get_all_ui_elements(image, confidence_threshold) + all_elements = _get_all_ui_elements(image, confidence_threshold, template_requests=None) nearby_elements = [] px, py = point diff --git a/src/vision/library/metadata/buttons.json b/src/vision/library/metadata/buttons.json new file mode 100644 index 0000000..2c18734 --- /dev/null +++ b/src/vision/library/metadata/buttons.json @@ -0,0 +1,34 @@ +{ + "category": "buttons", + "description": "Button templates for UI automation", + "templates": { + "submit_blue": { + "description": "Blue submit button with white text", + "use_cases": ["forms", "dialogs", "confirmations"], + "frameworks": ["generic", "bootstrap", "material"], + "confidence_weight": 1.0, + "tags": ["submit", "primary", "blue"] + }, + "submit_green": { + "description": "Green submit button with white text", + "use_cases": ["forms", "success_actions"], + "frameworks": ["generic", "bootstrap"], + "confidence_weight": 1.0, + "tags": ["submit", "success", "green"] + }, + "cancel_gray": { + "description": "Gray cancel button", + "use_cases": ["dialogs", "forms", "cancellation"], + "frameworks": ["generic", "bootstrap"], + "confidence_weight": 1.0, + "tags": ["cancel", "secondary", "gray"] + }, + "ok_small": { + "description": "Small OK button for dialogs", + "use_cases": ["dialog_confirmation", "alerts"], + "frameworks": ["generic", "windows"], + "confidence_weight": 1.0, + "tags": ["ok", "confirm", "small"] + } + } +} diff --git a/src/vision/library/metadata/forms.json b/src/vision/library/metadata/forms.json new file mode 100644 index 0000000..60d0a2f --- /dev/null +++ b/src/vision/library/metadata/forms.json @@ -0,0 +1,34 @@ +{ + "category": "forms", + "description": "Form element templates for UI automation", + "templates": { + "text_input_rect": { + "description": "Rectangular text input field with border", + "use_cases": ["login_forms", "registration", "data_entry"], + "frameworks": ["generic", "bootstrap", "material"], + "confidence_weight": 1.0, + "tags": ["input", "text", "field"] + }, + "dropdown_arrow": { + "description": "Dropdown arrow indicator", + "use_cases": ["dropdowns", "selects", "comboboxes"], + "frameworks": ["generic", "bootstrap"], + "confidence_weight": 1.0, + "tags": ["dropdown", "arrow", "select"] + }, + "checkbox_empty": { + "description": "Empty checkbox (unchecked state)", + "use_cases": ["forms", "preferences", "selections"], + "frameworks": ["generic", "bootstrap", "material"], + "confidence_weight": 1.0, + "tags": ["checkbox", "empty", "unchecked"] + }, + "checkbox_checked": { + "description": "Checked checkbox with checkmark", + "use_cases": ["forms", "preferences", "selections"], + "frameworks": ["generic", "bootstrap", "material"], + "confidence_weight": 1.0, + "tags": ["checkbox", "checked", "selected"] + } + } +} diff --git a/src/vision/library/template_loader.py b/src/vision/library/template_loader.py new file mode 100644 index 0000000..7bb1c01 --- /dev/null +++ b/src/vision/library/template_loader.py @@ -0,0 +1,234 @@ +"""Template Library Loader + +Handles loading and management of template library resources. +Provides efficient access to template metadata and images. +""" + +import json +from pathlib import Path + +import cv2 +import numpy as np + + +class TemplateLibraryLoader: + """Loads and manages template library resources""" + + def __init__(self, library_path: Path | None = None): + """Initialize template library loader + + Args: + library_path: Path to template library directory + """ + self.library_path = library_path or Path(__file__).parent + self._metadata_cache: dict[str, dict] = {} + + + def get_category_metadata(self, category: str) -> dict: + """Get metadata for a specific category""" + if category not in self._metadata_cache: + metadata_path = self.library_path / "metadata" / f"{category}.json" + if metadata_path.exists(): + with open(metadata_path) as f: + self._metadata_cache[category] = json.load(f) + else: + self._metadata_cache[category] = {"templates": {}} + return self._metadata_cache[category] + + def list_categories(self) -> list[str]: + """List available template categories by scanning metadata directory""" + metadata_dir = self.library_path / "metadata" + if not metadata_dir.exists(): + return [] + + categories = [] + for metadata_file in metadata_dir.glob("*.json"): + categories.append(metadata_file.stem) + return sorted(categories) + + def list_templates(self, category: str | None = None) -> dict[str, list[str]]: + """List available templates by category + + Args: + category: Specific category to list, or None for all categories + + Returns: + Dictionary mapping category names to template lists + """ + if category: + metadata = self.get_category_metadata(category) + templates = list(metadata.get("templates", {}).keys()) + return {category: templates} + + # Get all categories and their templates + result = {} + for cat in self.list_categories(): + metadata = self.get_category_metadata(cat) + templates = list(metadata.get("templates", {}).keys()) + result[cat] = templates + + return result + + def get_template_info(self, template_id: str, category: str = "common") -> dict | None: + """Get detailed information about a specific template""" + metadata = self.get_category_metadata(category) + return metadata.get("templates", {}).get(template_id) + + def load_template_image(self, template_id: str, category: str = "common") -> np.ndarray | None: + """Load template image as numpy array + + Args: + template_id: Template identifier + category: Template category + + Returns: + Template image as OpenCV numpy array, or None if not found + """ + # Try category subdirectory first + template_path = self.library_path / "templates" / category / f"{template_id}.png" + + if not template_path.exists(): + # Try root templates directory + template_path = self.library_path / "templates" / f"{template_id}.png" + + if not template_path.exists(): + return None + + # Load with OpenCV + template = cv2.imread(str(template_path)) + return template + + def template_exists(self, template_id: str, category: str = "common") -> bool: + """Check if a template exists in the library""" + return self.load_template_image(template_id, category) is not None + + def search_templates( + self, query: str, category: str | None = None, tags: list[str] | None = None + ) -> list[dict]: + """Search templates by name, description, or tags + + Args: + query: Search query string + category: Limit search to specific category + tags: Filter by specific tags + + Returns: + List of matching template info dictionaries + """ + results = [] + query_lower = query.lower() + + categories_to_search = [category] if category else self.list_categories() + + for cat in categories_to_search: + metadata = self.get_category_metadata(cat) + templates = metadata.get("templates", {}) + + for template_id, template_info in templates.items(): + # Check template ID + if query_lower in template_id.lower(): + match_info = template_info.copy() + match_info["template_id"] = template_id + match_info["category"] = cat + results.append(match_info) + continue + + # Check description + description = template_info.get("description", "").lower() + if query_lower in description: + match_info = template_info.copy() + match_info["template_id"] = template_id + match_info["category"] = cat + results.append(match_info) + continue + + # Check tags + template_tags = template_info.get("tags", []) + if any(query_lower in tag.lower() for tag in template_tags): + match_info = template_info.copy() + match_info["template_id"] = template_id + match_info["category"] = cat + results.append(match_info) + continue + + # Filter by specific tags if provided + if tags and not any(tag in template_tags for tag in tags): + continue + + return results + + def get_healthcare_templates(self) -> list[dict]: + """Get all healthcare-specific templates""" + return self.search_templates("", category="healthcare") + + def get_safety_critical_templates(self) -> list[dict]: + """Get templates marked as safety-critical""" + results = [] + + for category in self.list_categories(): + metadata = self.get_category_metadata(category) + templates = metadata.get("templates", {}) + + for template_id, template_info in templates.items(): + if template_info.get("safety_critical", False): + match_info = template_info.copy() + match_info["template_id"] = template_id + match_info["category"] = category + results.append(match_info) + + return results + + def validate_library(self) -> dict[str, list[str]]: + """Validate template library consistency + + Returns: + Dictionary with validation results and any errors found + """ + errors = [] + warnings = [] + + # Check if metadata directory exists + metadata_dir = self.library_path / "metadata" + if not metadata_dir.exists(): + errors.append("Metadata directory not found") + + # Check each category + for category in self.list_categories(): + metadata = self.get_category_metadata(category) + templates = metadata.get("templates", {}) + + # Check metadata file exists + metadata_path = self.library_path / "metadata" / f"{category}.json" + if not metadata_path.exists(): + warnings.append(f"No metadata file found for category: {category}") + + # Check each template + for template_id in templates.keys(): + if not self.template_exists(template_id, category): + errors.append(f"Template image not found: {category}/{template_id}") + + return { + "valid": len(errors) == 0, + "errors": errors, + "warnings": warnings, + "total_categories": len(self.list_categories()), + "total_templates": sum(len(templates) for templates in self.list_templates().values()), + } + + +# Global loader instance +_global_loader: TemplateLibraryLoader | None = None + + +def get_template_library_loader() -> TemplateLibraryLoader: + """Get global template library loader instance""" + global _global_loader + if _global_loader is None: + _global_loader = TemplateLibraryLoader() + return _global_loader + + +def set_template_library_loader(loader: TemplateLibraryLoader) -> None: + """Set global template library loader instance""" + global _global_loader + _global_loader = loader diff --git a/src/vision/library/templates/buttons/cancel_gray.png b/src/vision/library/templates/buttons/cancel_gray.png new file mode 100644 index 0000000..dc4e542 Binary files /dev/null and b/src/vision/library/templates/buttons/cancel_gray.png differ diff --git a/src/vision/library/templates/buttons/ok_small.png b/src/vision/library/templates/buttons/ok_small.png new file mode 100644 index 0000000..81fd3a3 Binary files /dev/null and b/src/vision/library/templates/buttons/ok_small.png differ diff --git a/src/vision/library/templates/buttons/submit_blue.png b/src/vision/library/templates/buttons/submit_blue.png new file mode 100644 index 0000000..2181394 Binary files /dev/null and b/src/vision/library/templates/buttons/submit_blue.png differ diff --git a/src/vision/library/templates/buttons/submit_green.png b/src/vision/library/templates/buttons/submit_green.png new file mode 100644 index 0000000..038e9e2 Binary files /dev/null and b/src/vision/library/templates/buttons/submit_green.png differ diff --git a/src/vision/library/templates/forms/checkbox_checked.png b/src/vision/library/templates/forms/checkbox_checked.png new file mode 100644 index 0000000..359d1b7 Binary files /dev/null and b/src/vision/library/templates/forms/checkbox_checked.png differ diff --git a/src/vision/library/templates/forms/checkbox_empty.png b/src/vision/library/templates/forms/checkbox_empty.png new file mode 100644 index 0000000..aaec080 Binary files /dev/null and b/src/vision/library/templates/forms/checkbox_empty.png differ diff --git a/src/vision/library/templates/forms/dropdown_arrow.png b/src/vision/library/templates/forms/dropdown_arrow.png new file mode 100644 index 0000000..5a82d9c Binary files /dev/null and b/src/vision/library/templates/forms/dropdown_arrow.png differ diff --git a/src/vision/library/templates/forms/text_input_rect.png b/src/vision/library/templates/forms/text_input_rect.png new file mode 100644 index 0000000..a25599e Binary files /dev/null and b/src/vision/library/templates/forms/text_input_rect.png differ diff --git a/src/vision/models/yolov8s.onnx b/src/vision/models/yolov8s.onnx deleted file mode 100644 index dc26b2a..0000000 Binary files a/src/vision/models/yolov8s.onnx and /dev/null differ diff --git a/src/vision/models/yolov8s.pt b/src/vision/models/yolov8s.pt deleted file mode 100644 index 90b4a2c..0000000 Binary files a/src/vision/models/yolov8s.pt and /dev/null differ diff --git a/src/vision/reader.py b/src/vision/reader.py index f1192d8..b797b78 100644 --- a/src/vision/reader.py +++ b/src/vision/reader.py @@ -5,6 +5,7 @@ Text Recognition: PaddleOCR for text extraction (true OCR) """ +import os from dataclasses import dataclass import cv2 @@ -12,6 +13,35 @@ import paddleocr +class OCRManager: + """Manages OCR instances to avoid reloading models""" + + def __init__(self): + self._instances = {} + # Set PaddleOCR cache directory to avoid repeated downloads + os.environ.setdefault("PADDLE_HOME", os.path.expanduser("~/.paddleocr_cache")) + + def get_ocr(self, language: str = "en") -> paddleocr.PaddleOCR: + """Get or create OCR instance for specified language""" + if language not in self._instances: + self._instances[language] = paddleocr.PaddleOCR( + use_textline_orientation=True, lang=language + ) + return self._instances[language] + + def clear_cache(self): + """Clear all cached OCR instances to free memory""" + self._instances.clear() + + def get_cached_languages(self) -> list[str]: + """Get list of currently cached languages""" + return list(self._instances.keys()) + + +# Global OCR manager instance +_ocr_manager = OCRManager() + + @dataclass class TextResult: """Text recognition result""" @@ -24,10 +54,18 @@ class TextResult: area: int +def resize_max_side(img, max_side=1600): + h, w = img.shape[:2] + s = min(1.0, max_side / max(h, w)) + if s < 1.0: + img = cv2.resize(img, (int(w * s), int(h * s)), interpolation=cv2.INTER_AREA) + return img + + def extract_text( image: np.ndarray, language: str = "en", - confidence_threshold: float = 0.5, + confidence_threshold: float = 0.3, max_results: int = 100, ) -> list[TextResult]: """ @@ -48,12 +86,12 @@ def extract_text( for result in text_results: print(f"Found text: '{result.text}' at {result.center}") """ - # Initialize PaddleOCR with new API - ocr = paddleocr.PaddleOCR(use_textline_orientation=True, lang=language) + # Get OCR instance from manager (cached to avoid reloading models) + ocr = _ocr_manager.get_ocr(language) # Run OCR using new predict method try: - results = ocr.predict(image) + results = ocr.predict(resize_max_side(image)) if not results: return [] @@ -289,3 +327,13 @@ def draw_text_results(image: np.ndarray, text_results: list[TextResult]) -> np.n ) return result_image + + +def get_ocr_manager() -> OCRManager: + """Get the global OCR manager for advanced use cases""" + return _ocr_manager + + +def clear_ocr_cache(): + """Clear OCR cache to free memory""" + _ocr_manager.clear_cache() diff --git a/src/vision/setup_models.py b/src/vision/setup_models.py deleted file mode 100644 index 7ba5b4e..0000000 --- a/src/vision/setup_models.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Model Setup and Management Functions - -Downloads and manages YOLO and PaddleOCR models for computer vision tasks. -""" - -import os -from pathlib import Path - -import paddleocr -from ultralytics import YOLO - - -def get_model_paths() -> dict[str, str]: - """ - Get paths to model files - - Returns: - Dictionary with model paths - - Example: - paths = get_model_paths() - yolo_path = paths['yolo_onnx'] - """ - models_dir = Path(__file__).parent / "models" - - return { - "models_dir": str(models_dir), - "yolo_onnx": str(models_dir / "yolov8s.onnx"), - "yolo_pt": str(models_dir / "yolov8s.pt"), - } - - -def download_models() -> dict[str, str]: - """ - Download and setup all required models - - Returns: - Dictionary with downloaded model paths - - Example: - paths = download_models() - print(f"Models ready at: {paths}") - """ - print("๐Ÿ” Setting up computer vision models...") - - # Setup YOLO - yolo_path = setup_yolo_model() - - # Setup PaddleOCR - setup_paddle_ocr() - - paths = get_model_paths() - - print("โœ… All models ready!") - return paths - - -def setup_models() -> dict[str, str]: - """Alias for download_models() for backward compatibility""" - return download_models() - - -def setup_yolo_model() -> str: - """ - Download and convert YOLOv8s to ONNX format - - Returns: - Path to ONNX model file - """ - models_dir = Path(__file__).parent / "models" - models_dir.mkdir(exist_ok=True) - - onnx_path = models_dir / "yolov8s.onnx" - pt_path = models_dir / "yolov8s.pt" - - if onnx_path.exists(): - print(f"โœ… YOLOv8s ONNX already exists: {onnx_path}") - return str(onnx_path) - - print("๐Ÿ“ฅ Downloading YOLOv8s and converting to ONNX...") - - try: - # Load pretrained YOLOv8s model (will download if not present) - model = YOLO("yolov8s.pt") - - # Export to ONNX format optimized for CPU inference - model.export( - format="onnx", - imgsz=640, - optimize=True, - half=False, # Keep FP32 for CPU compatibility - dynamic=False, - simplify=True, - ) - - # Move to models directory - if os.path.exists("yolov8s.onnx"): - os.rename("yolov8s.onnx", str(onnx_path)) - if os.path.exists("yolov8s.pt"): - os.rename("yolov8s.pt", str(pt_path)) - - print(f"โœ… YOLOv8s ONNX saved to: {onnx_path}") - - except Exception as e: - print(f"โŒ YOLOv8s setup failed: {e}") - raise - - return str(onnx_path) - - -def setup_paddle_ocr() -> bool: - """ - Initialize PaddleOCR (downloads models on first use) - - Returns: - True if successful - """ - print("๐Ÿ“ฅ Setting up PaddleOCR...") - - try: - # Initialize PaddleOCR - this will download models on first use - ocr = paddleocr.PaddleOCR(use_textline_orientation=True, lang="en") - - print("โœ… PaddleOCR initialized successfully") - return True - - except Exception as e: - print(f"โŒ PaddleOCR setup failed: {e}") - raise - - -def verify_models() -> dict[str, bool]: - """ - Verify that all models are properly installed - - Returns: - Dictionary with verification status for each model - """ - print("๐Ÿ” Verifying model installations...") - - results = {"yolo_onnx": False, "paddle_ocr": False} - - # Check YOLO ONNX - try: - import onnxruntime as ort - - paths = get_model_paths() - - if os.path.exists(paths["yolo_onnx"]): - session = ort.InferenceSession(paths["yolo_onnx"]) - results["yolo_onnx"] = True - print("โœ… YOLOv8s ONNX model verified") - else: - print("โŒ YOLOv8s ONNX model not found") - - except Exception as e: - print(f"โŒ YOLOv8s ONNX verification failed: {e}") - - # Check PaddleOCR - try: - ocr = paddleocr.PaddleOCR(use_textline_orientation=True, lang="en") - results["paddle_ocr"] = True - print("โœ… PaddleOCR verified") - - except Exception as e: - print(f"โŒ PaddleOCR verification failed: {e}") - - return results - - -def cleanup_temp_files(): - """Clean up temporary files from model downloads""" - temp_files = ["yolov8s.pt", "yolov8s.onnx"] - - for temp_file in temp_files: - if os.path.exists(temp_file): - os.remove(temp_file) - print(f"๐Ÿงน Cleaned up {temp_file}") - - -if __name__ == "__main__": - # Run setup when called directly - print("Computer Vision Model Setup") - print("=" * 40) - - try: - paths = download_models() - print("\n๐Ÿ“‹ Model Summary:") - for name, path in paths.items(): - print(f" {name}: {path}") - - print("\n๐Ÿ” Verification:") - verification = verify_models() - for model, status in verification.items(): - status_icon = "โœ…" if status else "โŒ" - print(f" {model}: {status_icon}") - - cleanup_temp_files() - - if all(verification.values()): - print("\n๐ŸŽ‰ All models ready for computer vision tasks!") - else: - print("\nโš ๏ธ Some models failed verification. Check errors above.") - - except Exception as e: - print(f"\nโŒ Setup failed: {e}") - cleanup_temp_files() diff --git a/src/vision/template_manager.py b/src/vision/template_manager.py new file mode 100644 index 0000000..127d6ac --- /dev/null +++ b/src/vision/template_manager.py @@ -0,0 +1,316 @@ +"""Template Management System + +Core template resolution and management for UI element detection. +Handles multiple template sources: base64, library, and multi-template strategies. +""" + +import base64 +import hashlib +import io +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + + +class TemplateStrategy(Enum): + """Template resolution strategies""" + + BASE64 = "base64" + LIBRARY = "library" + MULTI = "multi" + + +@dataclass +class TemplateSource: + """Template source with metadata and caching info""" + + strategy: TemplateStrategy + template: np.ndarray + metadata: dict[str, any] = field(default_factory=dict) + confidence_weight: float = 1.0 + cache_key: str | None = None + + def __post_init__(self): + """Generate cache key if not provided""" + if self.cache_key is None: + self.cache_key = self._generate_cache_key() + + def _generate_cache_key(self) -> str: + """Generate unique cache key for this template""" + # Create hash from template array and metadata + template_hash = hashlib.sha256(self.template.tobytes()).hexdigest()[:16] + strategy_key = self.strategy.value + metadata_key = str(sorted(self.metadata.items()))[:32] + return f"{strategy_key}_{template_hash}_{hashlib.sha256(metadata_key.encode()).hexdigest()[:8]}" + + +@dataclass +class TemplateRequest: + """Template request specification""" + + strategy: TemplateStrategy + data: str | dict[str, any] # base64 string or library specification + weight: float = 1.0 + required: bool = True + name: str | None = None + + +class TemplateValidationError(Exception): + """Raised when template validation fails""" + + +class TemplateManager: + """Central template management system""" + + def __init__(self, cache_enabled: bool = True, library_path: Path | None = None): + """Initialize template manager + + Args: + cache_enabled: Enable template caching + library_path: Path to template library directory + """ + self.cache_enabled = cache_enabled + self.library_path = library_path or Path(__file__).parent / "library" + self._template_cache: dict[str, TemplateSource] = {} + self._library_index: dict[str, any] | None = None + + def resolve_templates( + self, request: TemplateRequest | list[TemplateRequest] + ) -> list[TemplateSource]: + """Resolve template request(s) to actual template sources + + Args: + request: Single template request or list of requests + + Returns: + List of resolved template sources + + Raises: + TemplateValidationError: If template resolution fails + """ + # Normalize to list + if isinstance(request, TemplateRequest): + requests = [request] + else: + requests = request + + resolved_templates = [] + + for req in requests: + try: + template_source = self._resolve_single_request(req) + resolved_templates.append(template_source) + except Exception as e: + if req.required: + raise TemplateValidationError( + f"Failed to resolve required template '{req.name}': {e}" + ) + # Skip optional templates that fail + continue + + if not resolved_templates: + raise TemplateValidationError("No templates could be resolved") + + return resolved_templates + + def _resolve_single_request(self, request: TemplateRequest) -> TemplateSource: + """Resolve a single template request""" + if request.strategy == TemplateStrategy.BASE64: + return self._resolve_base64_template(request) + elif request.strategy == TemplateStrategy.LIBRARY: + return self._resolve_library_template(request) + else: + raise TemplateValidationError(f"Unsupported template strategy: {request.strategy}") + + def _resolve_base64_template(self, request: TemplateRequest) -> TemplateSource: + """Resolve base64 template""" + if not isinstance(request.data, str): + raise TemplateValidationError("Base64 template data must be string") + + # Check cache first + cache_key = f"base64_{hashlib.sha256(request.data.encode()).hexdigest()[:16]}" + if self.cache_enabled and cache_key in self._template_cache: + cached = self._template_cache[cache_key] + cached.confidence_weight = request.weight + return cached + + # Decode base64 to template + template_array = self._decode_base64_image(request.data) + self._validate_template(template_array) + + template_source = TemplateSource( + strategy=TemplateStrategy.BASE64, + template=template_array, + confidence_weight=request.weight, + cache_key=cache_key, + metadata={ + "name": request.name or "base64_template", + "source": "base64", + "size": template_array.shape, + }, + ) + + # Cache template + if self.cache_enabled: + self._template_cache[cache_key] = template_source + + return template_source + + def _resolve_library_template(self, request: TemplateRequest) -> TemplateSource: + """Resolve library template""" + if not isinstance(request.data, dict) or "id" not in request.data: + raise TemplateValidationError("Library template data must contain 'id' key") + + template_id = request.data["id"] + category = request.data.get("category", "common") + + # Check cache first + cache_key = f"library_{category}_{template_id}" + if self.cache_enabled and cache_key in self._template_cache: + cached = self._template_cache[cache_key] + cached.confidence_weight = request.weight + return cached + + # Load from library + template_array = self._load_library_template(template_id, category) + self._validate_template(template_array) + + template_source = TemplateSource( + strategy=TemplateStrategy.LIBRARY, + template=template_array, + confidence_weight=request.weight, + cache_key=cache_key, + metadata={ + "name": request.name or template_id, + "source": "library", + "id": template_id, + "category": category, + "size": template_array.shape, + }, + ) + + # Cache template + if self.cache_enabled: + self._template_cache[cache_key] = template_source + + return template_source + + def _decode_base64_image(self, base64_data: str) -> np.ndarray: + """Decode base64 string to OpenCV image array""" + try: + # Remove data URL prefix if present + if base64_data.startswith("data:image"): + base64_data = base64_data.split(",", 1)[1] + + # Decode base64 + image_bytes = base64.b64decode(base64_data) + + # Convert to PIL Image + pil_image = Image.open(io.BytesIO(image_bytes)) + + # Convert to RGB if needed + if pil_image.mode != "RGB": + pil_image = pil_image.convert("RGB") + + # Convert to numpy array + image_array = np.array(pil_image) + + # Convert RGB to BGR for OpenCV + image_bgr = cv2.cvtColor(image_array, cv2.COLOR_RGB2BGR) + + return image_bgr + + except Exception as e: + raise TemplateValidationError(f"Failed to decode base64 image: {e}") + + def _load_library_template(self, template_id: str, category: str = "common") -> np.ndarray: + """Load template from library""" + # Build template path + template_path = self.library_path / "templates" / category / f"{template_id}.png" + + if not template_path.exists(): + # Try without category subdirectory + template_path = self.library_path / "templates" / f"{template_id}.png" + + if not template_path.exists(): + raise TemplateValidationError(f"Template '{template_id}' not found in library") + + # Load image + template = cv2.imread(str(template_path)) + if template is None: + raise TemplateValidationError(f"Failed to load template image: {template_path}") + + return template + + def _validate_template(self, template: np.ndarray) -> None: + """Validate template image""" + if template is None: + raise TemplateValidationError("Template is None") + + if template.size == 0: + raise TemplateValidationError("Template is empty") + + height, width = template.shape[:2] + + if height < 5 or width < 5: + raise TemplateValidationError(f"Template too small: {width}x{height} (minimum 5x5)") + + if height > 1000 or width > 1000: + raise TemplateValidationError( + f"Template too large: {width}x{height} (maximum 1000x1000)" + ) + + def create_template_from_base64( + self, base64_data: str, name: str, weight: float = 1.0 + ) -> TemplateSource: + """Convenience method to create template from base64""" + request = TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, weight=weight, name=name + ) + return self._resolve_single_request(request) + + def create_template_from_library( + self, template_id: str, category: str = "common", weight: float = 1.0 + ) -> TemplateSource: + """Convenience method to create template from library""" + request = TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": template_id, "category": category}, + weight=weight, + name=template_id, + ) + return self._resolve_single_request(request) + + def clear_cache(self) -> None: + """Clear template cache""" + self._template_cache.clear() + + def get_cache_stats(self) -> dict[str, any]: + """Get cache statistics""" + return { + "cached_templates": len(self._template_cache), + "cache_enabled": self.cache_enabled, + "library_path": str(self.library_path), + } + + +# Global template manager instance +_global_template_manager: TemplateManager | None = None + + +def get_template_manager() -> TemplateManager: + """Get global template manager instance""" + global _global_template_manager + if _global_template_manager is None: + _global_template_manager = TemplateManager() + return _global_template_manager + + +def set_template_manager(manager: TemplateManager) -> None: + """Set global template manager instance""" + global _global_template_manager + _global_template_manager = manager diff --git a/src/vision/ui_patterns.py b/src/vision/ui_patterns.py new file mode 100644 index 0000000..203c836 --- /dev/null +++ b/src/vision/ui_patterns.py @@ -0,0 +1,206 @@ +"""UI Pattern Strategies for Common Element Detection + +Provides reusable strategies for detecting common UI elements without hardcoded dependencies. +Each strategy dynamically checks what templates are available and builds appropriate requests. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path + +from src.vision.template_manager import TemplateRequest, TemplateStrategy, get_template_manager + + +@dataclass +class UIPattern: + """Definition of a UI pattern with metadata""" + + id: str + category: str + name: str + description: str + weight: float = 1.0 + required: bool = False + + +class UIPatternStrategy(ABC): + """Abstract base class for UI pattern strategies""" + + @abstractmethod + def get_patterns(self) -> list[UIPattern]: + """Get list of UI patterns this strategy provides""" + pass + + @abstractmethod + def create_template_requests(self, template_manager=None) -> list[TemplateRequest]: + """Create template requests for patterns that exist in the library""" + pass + + +class ButtonPatternStrategy(UIPatternStrategy): + """Strategy for detecting common button patterns""" + + def get_patterns(self) -> list[UIPattern]: + """Get button patterns""" + return [ + UIPattern("submit_blue", "buttons", "submit_button", "Blue submit button"), + UIPattern("ok_button", "buttons", "ok_button", "OK confirmation button"), + UIPattern("cancel_gray", "buttons", "cancel_button", "Gray cancel button"), + UIPattern("close_red", "buttons", "close_button", "Red close button"), + UIPattern("save_green", "buttons", "save_button", "Green save button"), + ] + + def create_template_requests(self, template_manager=None) -> list[TemplateRequest]: + """Create template requests for button patterns that exist""" + if template_manager is None: + template_manager = get_template_manager() + + requests = [] + for pattern in self.get_patterns(): + # Check if template exists in library before creating request + if self._template_exists(template_manager, pattern.id, pattern.category): + request = TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": pattern.id, "category": pattern.category}, + weight=pattern.weight, + required=pattern.required, + name=pattern.name + ) + requests.append(request) + + return requests + + def _template_exists(self, template_manager, template_id: str, category: str) -> bool: + """Check if template exists in library""" + # Build template path + template_path = template_manager.library_path / "templates" / category / f"{template_id}.png" + if template_path.exists(): + return True + + # Try without category subdirectory + template_path = template_manager.library_path / "templates" / f"{template_id}.png" + return template_path.exists() + + +class ClickablePatternStrategy(UIPatternStrategy): + """Strategy for detecting clickable UI elements (buttons, links, inputs)""" + + def get_patterns(self) -> list[UIPattern]: + """Get clickable element patterns""" + patterns = [] + + # Include button patterns + button_strategy = ButtonPatternStrategy() + patterns.extend(button_strategy.get_patterns()) + + # Add other clickable patterns + patterns.extend([ + UIPattern("link_blue", "links", "hyperlink", "Blue hyperlink"), + UIPattern("input_text", "inputs", "text_input", "Text input field"), + UIPattern("search_box", "inputs", "search_input", "Search input box"), + UIPattern("checkbox_empty", "controls", "checkbox", "Empty checkbox"), + UIPattern("checkbox_checked", "controls", "checkbox_checked", "Checked checkbox"), + UIPattern("radio_empty", "controls", "radio_button", "Empty radio button"), + UIPattern("radio_selected", "controls", "radio_selected", "Selected radio button"), + ]) + + return patterns + + def create_template_requests(self, template_manager=None) -> list[TemplateRequest]: + """Create template requests for clickable patterns that exist""" + if template_manager is None: + template_manager = get_template_manager() + + requests = [] + for pattern in self.get_patterns(): + if self._template_exists(template_manager, pattern.id, pattern.category): + request = TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": pattern.id, "category": pattern.category}, + weight=pattern.weight, + required=False, # All clickable patterns are optional + name=pattern.name + ) + requests.append(request) + + return requests + + def _template_exists(self, template_manager, template_id: str, category: str) -> bool: + """Check if template exists in library""" + template_path = template_manager.library_path / "templates" / category / f"{template_id}.png" + if template_path.exists(): + return True + + template_path = template_manager.library_path / "templates" / f"{template_id}.png" + return template_path.exists() + + +class FormPatternStrategy(UIPatternStrategy): + """Strategy for detecting form-related UI elements""" + + def get_patterns(self) -> list[UIPattern]: + """Get form element patterns""" + return [ + UIPattern("input_text", "inputs", "text_input", "Text input field"), + UIPattern("input_password", "inputs", "password_input", "Password input field"), + UIPattern("search_box", "inputs", "search_input", "Search input box"), + UIPattern("textarea_large", "inputs", "textarea", "Large text area"), + UIPattern("dropdown_closed", "controls", "dropdown", "Closed dropdown menu"), + UIPattern("dropdown_open", "controls", "dropdown_open", "Open dropdown menu"), + UIPattern("submit_blue", "buttons", "submit_button", "Submit button"), + UIPattern("reset_gray", "buttons", "reset_button", "Reset button"), + ] + + def create_template_requests(self, template_manager=None) -> list[TemplateRequest]: + """Create template requests for form patterns that exist""" + if template_manager is None: + template_manager = get_template_manager() + + requests = [] + for pattern in self.get_patterns(): + if self._template_exists(template_manager, pattern.id, pattern.category): + request = TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": pattern.id, "category": pattern.category}, + weight=pattern.weight, + required=False, + name=pattern.name + ) + requests.append(request) + + return requests + + def _template_exists(self, template_manager, template_id: str, category: str) -> bool: + """Check if template exists in library""" + template_path = template_manager.library_path / "templates" / category / f"{template_id}.png" + if template_path.exists(): + return True + + template_path = template_manager.library_path / "templates" / f"{template_id}.png" + return template_path.exists() + + +# Convenience instances +BUTTON_PATTERNS = ButtonPatternStrategy() +CLICKABLE_PATTERNS = ClickablePatternStrategy() +FORM_PATTERNS = FormPatternStrategy() + + +def get_available_templates_summary() -> dict[str, list[str]]: + """Get summary of available templates for each strategy""" + template_manager = get_template_manager() + + summary = {} + strategies = { + "buttons": BUTTON_PATTERNS, + "clickable": CLICKABLE_PATTERNS, + "forms": FORM_PATTERNS, + } + + for strategy_name, strategy in strategies.items(): + available_templates = [] + for request in strategy.create_template_requests(template_manager): + available_templates.append(request.name or request.data.get("id", "unknown")) + summary[strategy_name] = available_templates + + return summary \ No newline at end of file diff --git a/src/vision/verification.py b/src/vision/verification.py index a7cde43..01f58be 100644 --- a/src/vision/verification.py +++ b/src/vision/verification.py @@ -13,7 +13,9 @@ import cv2 import numpy as np -from . import detect_ui_elements, extract_text_from_region, find_elements_by_text +from src.vision.detector_template import detect_ui_elements +from src.vision.finder import find_elements_by_text +from src.vision.reader import extract_text_from_region @dataclass diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e3aa1e6..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Test suite for the computer-use-agent automation framework.""" diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index aa5eadb..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Test configuration and fixtures for the automation test suite.""" - -import sys -from pathlib import Path -from unittest.mock import MagicMock, Mock - -import pytest - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - - -@pytest.fixture -def mock_subprocess(): - """Mock subprocess calls for testing local automation.""" - mock = Mock() - mock.run.return_value = Mock(returncode=0, stdout="", stderr="") - return mock - - -@pytest.fixture -def mock_vnc_connection(): - """Mock VNC connection for testing.""" - mock = MagicMock() - mock.connect.return_value = True - mock.disconnect.return_value = True - mock.is_connected = True - return mock - - -@pytest.fixture -def mock_rdp_connection(): - """Mock RDP connection for testing.""" - mock = MagicMock() - mock.connect.return_value = True - mock.disconnect.return_value = True - mock.is_connected = True - return mock - - -@pytest.fixture -def mock_screenshot(): - """Mock screenshot data for testing.""" - import numpy as np - from PIL import Image - - # Create a simple 100x100 RGB image - img_array = np.zeros((100, 100, 3), dtype=np.uint8) - img_array[:, :] = [255, 255, 255] # White background - return Image.fromarray(img_array) - - -@pytest.fixture -def mock_ocr_results(): - """Mock OCR results for testing.""" - return [ - {"text": "Username", "bbox": [10, 10, 100, 30], "confidence": 0.95}, - {"text": "Password", "bbox": [10, 50, 100, 70], "confidence": 0.92}, - {"text": "Login", "bbox": [10, 90, 60, 110], "confidence": 0.98}, - ] - - -@pytest.fixture -def mock_yolo_results(): - """Mock YOLO detection results for testing.""" - return [ - {"class": "textbox", "bbox": [10, 10, 200, 40], "confidence": 0.89}, - {"class": "button", "bbox": [10, 90, 80, 120], "confidence": 0.94}, - ] - - -@pytest.fixture -def sample_vm_config(): - """Sample VM configuration for testing.""" - return { - "host": "192.168.1.100", - "port": 5900, - "password": "test_password", - "connection_type": "vnc", - } - - -@pytest.fixture -def temp_test_dir(tmp_path): - """Create a temporary directory for test files.""" - test_dir = tmp_path / "test_automation" - test_dir.mkdir() - return test_dir diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py deleted file mode 100644 index 5a0941b..0000000 --- a/tests/integration/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Integration tests for the automation framework.""" diff --git a/tests/integration/vision/conftest.py b/tests/integration/vision/conftest.py new file mode 100644 index 0000000..fedf46f --- /dev/null +++ b/tests/integration/vision/conftest.py @@ -0,0 +1,258 @@ +"""Test configuration and fixtures for vision integration tests + +Provides fast synthetic fixtures for testing vision system integration. +Focuses on functional correctness, not performance evaluation. +""" + +import base64 +import tempfile +from collections.abc import Generator +from pathlib import Path + +import cv2 +import numpy as np +import pytest + +from src.vision.template_manager import TemplateManager + + +class SyntheticImageGenerator: + """Generates fast synthetic images for testing""" + + @staticmethod + def create_simple_button( + width: int = 40, + height: int = 20, + bg_color: tuple[int, int, int] = (0, 123, 255), # Blue + text_color: tuple[int, int, int] = (255, 255, 255), # White + text: str = "OK", + ) -> np.ndarray: + """Create synthetic button image""" + img = np.full((height, width, 3), bg_color, dtype=np.uint8) + + # Add border + border_color = tuple(max(0, c - 40) for c in bg_color) + cv2.rectangle(img, (0, 0), (width - 1, height - 1), border_color, 1) + + # Add text if space allows + if width > 20 and height > 10: + font_scale = min(0.4, (width - 4) / (len(text) * 8)) + (text_w, text_h), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 1) + text_x = (width - text_w) // 2 + text_y = (height + text_h) // 2 + cv2.putText( + img, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1 + ) + + return img + + @staticmethod + def create_text_image( + text: str = "Sample Text", + width: int = 100, + height: int = 30, + bg_color: tuple[int, int, int] = (255, 255, 255), # White + text_color: tuple[int, int, int] = (0, 0, 0), # Black + ) -> np.ndarray: + """Create synthetic text image for OCR testing""" + img = np.full((height, width, 3), bg_color, dtype=np.uint8) + + # Add text + font_scale = min(0.7, (width - 10) / (len(text) * 10)) + (text_w, text_h), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 1) + text_x = (width - text_w) // 2 + text_y = (height + text_h) // 2 + cv2.putText( + img, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, text_color, 1 + ) + + return img + + @staticmethod + def create_ui_mockup( + elements: list[str] | None = None, canvas_width: int = 200, canvas_height: int = 150 + ) -> np.ndarray: + """Create synthetic UI mockup with multiple elements""" + if elements is None: + elements = ["button", "text", "input"] + + # Light gray background + img = np.full((canvas_height, canvas_width, 3), (240, 240, 240), dtype=np.uint8) + + element_positions = [ + (20, 20), # Top-left + (20, 60), # Middle-left + (20, 100), # Bottom-left + (120, 20), # Top-right + (120, 60), # Middle-right + ] + + for i, element_type in enumerate(elements[: len(element_positions)]): + x, y = element_positions[i] + + if element_type == "button": + button = SyntheticImageGenerator.create_simple_button(60, 25, text="Submit") + h, w = button.shape[:2] + if y + h <= canvas_height and x + w <= canvas_width: + img[y : y + h, x : x + w] = button + + elif element_type == "text": + text_img = SyntheticImageGenerator.create_text_image("Label:", 50, 20) + h, w = text_img.shape[:2] + if y + h <= canvas_height and x + w <= canvas_width: + img[y : y + h, x : x + w] = text_img + + elif element_type == "input": + # Create input field (white rectangle with border) + input_w, input_h = 80, 25 + if y + input_h <= canvas_height and x + input_w <= canvas_width: + cv2.rectangle( + img, (x, y), (x + input_w - 1, y + input_h - 1), (255, 255, 255), -1 + ) + cv2.rectangle( + img, (x, y), (x + input_w - 1, y + input_h - 1), (128, 128, 128), 1 + ) + + return img + + +@pytest.fixture +def synthetic_generator() -> SyntheticImageGenerator: + """Provide synthetic image generator""" + return SyntheticImageGenerator() + + +@pytest.fixture +def simple_button_image(synthetic_generator) -> np.ndarray: + """Provide simple button image for testing""" + return synthetic_generator.create_simple_button() + + +@pytest.fixture +def simple_text_image(synthetic_generator) -> np.ndarray: + """Provide simple text image for OCR testing""" + return synthetic_generator.create_text_image("Test Text") + + +@pytest.fixture +def ui_mockup_image(synthetic_generator) -> np.ndarray: + """Provide complete UI mockup for integration testing""" + return synthetic_generator.create_ui_mockup(["button", "text", "input"]) + + +@pytest.fixture +def template_base64(simple_button_image) -> str: + """Provide base64 encoded template for testing""" + _, buffer = cv2.imencode(".png", simple_button_image) + return base64.b64encode(buffer).decode("utf-8") + + +@pytest.fixture +def mock_template_requests() -> list[dict]: + """Provide mock template request data""" + return [ + {"strategy": "base64", "data": "mock_base64_data", "name": "test_button"}, + {"strategy": "library", "data": {"id": "submit_blue", "category": "buttons"}}, + { + "strategy": "multi", + "data": [ + {"strategy": "library", "data": {"id": "submit_blue", "category": "buttons"}}, + {"strategy": "library", "data": {"id": "cancel_gray", "category": "buttons"}}, + ], + }, + ] + + +@pytest.fixture +def template_manager() -> Generator[TemplateManager, None, None]: + """Provide clean template manager instance""" + # Import here to avoid circular imports + from src.vision.template_manager import get_template_manager + + manager = get_template_manager() + yield manager + + +@pytest.fixture +def temp_template_dir() -> Generator[Path, None, None]: + """Provide temporary directory for template testing""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create mock template structure + buttons_dir = temp_path / "templates" / "buttons" + buttons_dir.mkdir(parents=True) + + metadata_dir = temp_path / "metadata" + metadata_dir.mkdir(parents=True) + + yield temp_path + + +@pytest.fixture +def expected_detection_structure() -> dict: + """Expected structure of Detection objects""" + return { + "required_fields": [ + "template_name", + "confidence", + "bbox", + "center", + "area", + "scale", + "angle", + "method", + ], + "optional_fields": ["template_source", "template_match_method", "template_confidence"], + "bbox_format": "tuple[int, int, int, int]", # x1, y1, x2, y2 + "center_format": "tuple[int, int]", # x, y + "confidence_range": (0.0, 1.0), + } + + +@pytest.fixture +def expected_text_result_structure() -> dict: + """Expected structure of TextResult objects""" + return { + "required_fields": ["text", "confidence", "bbox"], + "optional_fields": ["language", "direction"], + "bbox_format": "tuple[int, int, int, int]", # x1, y1, x2, y2 + "confidence_range": (0.0, 1.0), + } + + +@pytest.fixture +def test_image_with_template( + synthetic_generator, simple_button_image +) -> tuple[np.ndarray, np.ndarray]: + """Provide test image containing the template for guaranteed detection""" + # Create larger background image + canvas_width, canvas_height = 150, 100 + template_h, template_w = simple_button_image.shape[:2] + + # Create background + test_image = np.full((canvas_height, canvas_width, 3), (240, 240, 240), dtype=np.uint8) + + # Place template at center + start_y = (canvas_height - template_h) // 2 + start_x = (canvas_width - template_w) // 2 + test_image[start_y : start_y + template_h, start_x : start_x + template_w] = simple_button_image + + return test_image, simple_button_image + + +@pytest.fixture +def mock_mcp_parameters(template_base64) -> dict: + """Mock MCP server parameters for integration testing""" + # Create test image + test_image = SyntheticImageGenerator.create_ui_mockup() + _, img_buffer = cv2.imencode(".png", test_image) + image_b64 = base64.b64encode(img_buffer).decode("utf-8") + + return { + "image_base64": image_b64, + "template_strategy": "base64", + "template_base64": template_base64, + "template_name": "test_button", + "confidence_threshold": 0.7, + } diff --git a/tests/integration/vision/test_combined_search_integration.py b/tests/integration/vision/test_combined_search_integration.py new file mode 100644 index 0000000..b7627f4 --- /dev/null +++ b/tests/integration/vision/test_combined_search_integration.py @@ -0,0 +1,550 @@ +"""Combined Search Integration Tests + +Tests the integration of template detection + OCR combined search system: +- find_elements_by_text() combining template detection and OCR +- find_clickable_elements() intelligent UI element identification +- analyze_screen_content() comprehensive screen analysis +- UIElement and ScreenAnalysis object API contracts +- Integration with downstream MCP server requirements + +Big Picture: Combined search is the heart of intelligent UI automation. +LLM agents use this to understand and interact with complex healthcare interfaces. +""" + +# Add src to path for imports + +import numpy as np + +from src.vision.finder import ( + ScreenAnalysis, + UIElement, + analyze_screen_content, + find_clickable_elements, + find_elements_by_text, + find_elements_near_point, +) + + +class TestCombinedSearchIntegration: + """Test combined template detection + OCR search integration""" + + def test_find_elements_by_text_returns_valid_uielement_objects(self, ui_mockup_image): + """ + INTEGRATION: Text-based element finding with template detection + + Big Picture: Healthcare workflows need to find UI elements by their + text labels. This combines OCR and template detection for precision. + """ + # Test finding elements by text (pure OCR) + elements = find_elements_by_text( + image=ui_mockup_image, + text_query="Submit", + confidence_threshold=0.3, + case_sensitive=False, + ) + + # Verify API contract: returns list of UIElement objects + assert isinstance(elements, list), "find_elements_by_text must return list" + + for element in elements: + # Verify UIElement structure + assert isinstance(element, UIElement), "Must return UIElement objects" + + # Verify required fields + required_fields = ["element_type", "bbox", "center", "confidence", "area"] + for field in required_fields: + assert hasattr(element, field), f"UIElement missing required field: {field}" + assert getattr(element, field) is not None, f"Field {field} cannot be None" + + # Verify data types for MCP integration + assert element.element_type in ["visual", "text", "combined"], ( + "element_type must be valid" + ) + assert isinstance(element.bbox, tuple), "bbox must be tuple" + assert len(element.bbox) == 4, "bbox must be (x1, y1, x2, y2)" + assert isinstance(element.center, tuple), "center must be tuple" + assert len(element.center) == 2, "center must be (x, y)" + assert isinstance(element.confidence, (int, float)), "confidence must be numeric" + assert 0.0 <= element.confidence <= 1.0, "confidence must be in range [0,1]" + assert isinstance(element.area, int), "area must be integer" + + # Verify optional fields are properly typed + if element.text_detection is not None: + assert hasattr(element.text_detection, "text"), ( + "text_detection should have text field" + ) + if element.visual_detection is not None: + assert hasattr(element.visual_detection, "bbox"), ( + "visual_detection should have bbox field" + ) + + def test_combined_element_integration(self, synthetic_generator): + """ + INTEGRATION: Combined visual + text element creation + + Big Picture: Healthcare UIs have labeled buttons and form fields. + System must combine visual detection with text recognition. + """ + # Create UI with labeled elements + test_image = synthetic_generator.create_ui_mockup( + elements=["button", "text"], canvas_width=200, canvas_height=120 + ) + + # Import the new combined function + from src.vision.finder import find_elements_combined + from src.vision.ui_patterns import BUTTON_PATTERNS + + # Test pure OCR (text-only) + text_elements = find_elements_by_text( + image=test_image, text_query="Submit", confidence_threshold=0.2 + ) + + # Test combined search (OCR + templates) + combined_elements = find_elements_combined( + image=test_image, + text_query="Submit", + template_requests=BUTTON_PATTERNS.create_template_requests(), + confidence_threshold=0.2, + combine_nearby=True + ) + + # Verify text-only elements + for element in text_elements: + assert isinstance(element, UIElement), "Should create UIElement objects" + assert element.element_type == "text", "Pure OCR should create text-only elements" + assert element.text is not None, "Text elements should have text content" + + # Verify combined elements when available + for element in combined_elements: + assert isinstance(element, UIElement), "Should create UIElement objects" + assert element.element_type in ["visual", "text", "combined"], ( + "Should classify element type correctly" + ) + + # If combined, should have both visual and text components + if element.element_type == "combined": + assert element.visual_detection is not None, ( + "Combined element should have visual component" + ) + assert element.text_detection is not None, ( + "Combined element should have text component" + ) + assert element.text is not None, "Combined element should have text content" + + # Verify combined bbox encompasses both components + visual_bbox = element.visual_detection.bbox + text_bbox = element.text_detection.rect_bbox + combined_bbox = element.bbox + + # Combined bbox should contain both individual bboxes + assert combined_bbox[0] <= min(visual_bbox[0], text_bbox[0]), ( + "Combined bbox should contain visual bbox" + ) + assert combined_bbox[1] <= min(visual_bbox[1], text_bbox[1]), ( + "Combined bbox should contain text bbox" + ) + + def test_find_clickable_elements_integration(self, ui_mockup_image): + """ + INTEGRATION: Clickable element identification + + Big Picture: Healthcare systems have many interactive elements. + Intelligent identification of clickable elements is essential. + """ + # Find potentially clickable elements + clickable_elements = find_clickable_elements( + image=ui_mockup_image, confidence_threshold=0.3 + ) + + # Verify clickable element detection + assert isinstance(clickable_elements, list), "Should return list of clickable elements" + + for element in clickable_elements: + assert isinstance(element, UIElement), "Should return UIElement objects" + + # Verify clickable indicators are present + has_clickable_text = False + has_clickable_visual = False + + # Check for clickable text indicators + if element.text: + clickable_terms = ["button", "click", "submit", "ok", "cancel", "save", "open"] + text_lower = element.text.lower() + has_clickable_text = any(term in text_lower for term in clickable_terms) + + # Short text often indicates clickable elements + if len(element.text.split()) <= 3 and len(element.text) <= 20: + has_clickable_text = True + + # Check for clickable visual indicators (UI templates) + if element.visual_detection: + clickable_ui_patterns = {"button", "submit", "ok", "cancel", "close", "link", "input"} + if hasattr(element.visual_detection, "template_name"): + template_name = element.visual_detection.template_name.lower() + has_clickable_visual = any(pattern in template_name for pattern in clickable_ui_patterns) + + # Should have some indication of clickability + # (May be relaxed for synthetic test data) + if element.text or element.visual_detection: + # Element should have some identifying information + assert element.description is not None, "Clickable element should have description" + + def test_analyze_screen_content_integration(self, ui_mockup_image): + """ + INTEGRATION: Comprehensive screen analysis + + Big Picture: Healthcare systems need complete UI understanding. + Screen analysis provides comprehensive element inventory for LLM agents. + """ + # Perform comprehensive screen analysis + analysis = analyze_screen_content( + image=ui_mockup_image, + query="Analyze all UI elements and interactions", + confidence_threshold=0.3, + ) + + # Verify ScreenAnalysis API contract + assert isinstance(analysis, ScreenAnalysis), "Should return ScreenAnalysis object" + + # Verify required fields + required_fields = [ + "ui_elements", + "text_elements", + "visual_elements", + "clickable_elements", + "summary", + "query", + ] + + for field in required_fields: + assert hasattr(analysis, field), f"ScreenAnalysis missing required field: {field}" + assert getattr(analysis, field) is not None, f"Field {field} cannot be None" + + # Verify element lists + assert isinstance(analysis.ui_elements, list), "ui_elements must be list" + assert isinstance(analysis.text_elements, list), "text_elements must be list" + assert isinstance(analysis.visual_elements, list), "visual_elements must be list" + assert isinstance(analysis.clickable_elements, list), "clickable_elements must be list" + + # Verify all elements are UIElement objects + for element_list in [ + analysis.ui_elements, + analysis.text_elements, + analysis.visual_elements, + analysis.clickable_elements, + ]: + for element in element_list: + assert isinstance(element, UIElement), "All elements should be UIElement objects" + + # Verify summary structure + assert isinstance(analysis.summary, dict), "summary must be dictionary" + + expected_summary_keys = [ + "total_elements", + "visual_elements", + "text_elements", + "clickable_elements", + "unique_text_content", + "visual_classes", + ] + + for key in expected_summary_keys: + assert key in analysis.summary, f"Summary missing key: {key}" + + # Verify summary data types + assert isinstance(analysis.summary["total_elements"], int), "total_elements should be int" + assert isinstance(analysis.summary["visual_elements"], int), "visual_elements should be int" + assert isinstance(analysis.summary["text_elements"], int), "text_elements should be int" + assert isinstance(analysis.summary["clickable_elements"], int), ( + "clickable_elements should be int" + ) + assert isinstance(analysis.summary["unique_text_content"], int), ( + "unique_text_content should be int" + ) + assert isinstance(analysis.summary["visual_classes"], list), "visual_classes should be list" + + # Verify query preservation + assert analysis.query == "Analyze all UI elements and interactions", "Should preserve query" + + def test_find_elements_near_point_integration(self, ui_mockup_image): + """ + INTEGRATION: Spatial element search + + Big Picture: Healthcare UIs need context-aware interaction. + Finding elements near specific points enables precise targeting. + """ + height, width = ui_mockup_image.shape[:2] + + # Test center point search + center_point = (width // 2, height // 2) + center_elements = find_elements_near_point( + image=ui_mockup_image, point=center_point, radius=100, confidence_threshold=0.3 + ) + + # Test corner point search (should find fewer elements) + corner_point = (10, 10) + corner_elements = find_elements_near_point( + image=ui_mockup_image, point=corner_point, radius=50, confidence_threshold=0.3 + ) + + # Verify spatial search integration + assert isinstance(center_elements, list), "Center search should return list" + assert isinstance(corner_elements, list), "Corner search should return list" + + # Verify distance-based results + for element in center_elements: + assert isinstance(element, UIElement), "Should return UIElement objects" + + # Should have distance information for sorting + assert hasattr(element, "distance"), "Should add distance attribute" + + # Distance should be reasonable + if hasattr(element, "distance"): + assert element.distance >= 0, "Distance should be non-negative" + assert element.distance <= 100, "Distance should be within search radius" + + # If multiple elements, should be sorted by distance + if len(center_elements) > 1: + distances = [getattr(elem, "distance", float("inf")) for elem in center_elements] + assert distances == sorted(distances), "Elements should be sorted by distance" + + def test_case_sensitivity_and_text_matching_integration(self, synthetic_generator): + """ + INTEGRATION: Text matching with case sensitivity options + + Big Picture: Healthcare systems may have standardized terminology + that requires precise or flexible text matching. + """ + # Create test image with specific text + test_text = "Patient Record" + test_image = synthetic_generator.create_text_image(text=test_text, width=150, height=40) + + # Test case-sensitive search + case_sensitive_matches = find_elements_by_text( + image=test_image, text_query="Patient", case_sensitive=True, confidence_threshold=0.2 + ) + + # Test case-insensitive search + case_insensitive_matches = find_elements_by_text( + image=test_image, + text_query="patient", # Different case + case_sensitive=False, + confidence_threshold=0.2, + ) + + # Test partial matching + partial_matches = find_elements_by_text( + image=test_image, text_query="Record", case_sensitive=False, confidence_threshold=0.2 + ) + + # Verify text matching integration + all_searches = [case_sensitive_matches, case_insensitive_matches, partial_matches] + + for matches in all_searches: + assert isinstance(matches, list), "Text search should return list" + + for element in matches: + assert isinstance(element, UIElement), "Should return UIElement objects" + + # Should have text information + if element.text_detection: + assert isinstance(element.text, str), "Should have text content" + assert len(element.text.strip()) > 0, "Text should not be empty" + + +class TestCombinedSearchErrorHandling: + """Test error handling in combined search system""" + + def test_invalid_image_data_handled_gracefully(self): + """ + ERROR HANDLING: Invalid image input + + Big Picture: MCP server may receive corrupted image data. + Combined search must handle errors gracefully. + """ + # Test None image + elements = find_elements_by_text(image=None, text_query="Submit", confidence_threshold=0.5) + assert isinstance(elements, list), "Should handle None image gracefully" + assert len(elements) == 0, "Should return empty list for None image" + + # Test invalid image format + invalid_image = np.array([1, 2, 3]) # Wrong shape + elements = find_elements_by_text( + image=invalid_image, text_query="Submit", confidence_threshold=0.5 + ) + assert isinstance(elements, list), "Should handle invalid image format" + + def test_invalid_parameters_handled_gracefully(self, ui_mockup_image): + """ + ERROR HANDLING: Invalid parameter handling + + Big Picture: MCP server parameters may be malformed. + System should validate inputs and handle errors gracefully. + """ + # Test invalid confidence threshold + elements = find_elements_by_text( + image=ui_mockup_image, + text_query="Submit", + confidence_threshold=-0.5, # Invalid range + ) + assert isinstance(elements, list), "Should handle invalid confidence" + + # Test empty text query + elements = find_elements_by_text( + image=ui_mockup_image, + text_query="", # Empty query + confidence_threshold=0.5, + ) + assert isinstance(elements, list), "Should handle empty text query" + + # Test invalid search radius + elements = find_elements_near_point( + image=ui_mockup_image, + point=(100, 100), + radius=-50, # Negative radius + confidence_threshold=0.5, + ) + assert isinstance(elements, list), "Should handle invalid search radius" + + def test_empty_results_handled_gracefully(self, synthetic_generator): + """ + ERROR HANDLING: Empty search results + + Big Picture: Not all searches will find matching elements. + System must handle empty results gracefully for robust operation. + """ + # Create simple image with no text matching the query + simple_image = synthetic_generator.create_simple_button( + text="OK" # Different from search query + ) + + # Search for non-existent text + elements = find_elements_by_text( + image=simple_image, text_query="NonExistentText", confidence_threshold=0.5 + ) + + # Should handle gracefully + assert isinstance(elements, list), "Should return list for no matches" + assert len(elements) == 0, "Should return empty list when no matches found" + + # Test screen analysis with no elements + analysis = analyze_screen_content( + image=simple_image, + query="Find complex elements", + confidence_threshold=0.9, # High threshold to minimize false positives + ) + + # Should still return valid ScreenAnalysis + assert isinstance(analysis, ScreenAnalysis), ( + "Should return ScreenAnalysis for empty results" + ) + assert isinstance(analysis.ui_elements, list), "Should have empty element lists" + assert isinstance(analysis.summary, dict), "Should have summary even with no elements" + + def test_concurrent_search_safety(self, ui_mockup_image): + """ + ERROR HANDLING: Concurrent search thread safety + + Big Picture: Multiple LLM agents may search simultaneously. + Combined search must handle concurrent access safely. + """ + import threading + + # Test concurrent element searches + results = [] + errors = [] + + def search_elements(): + try: + elements = find_elements_by_text( + image=ui_mockup_image, text_query="Submit", confidence_threshold=0.3 + ) + results.append(elements) + except Exception as e: + errors.append(e) + + # Create multiple threads + threads = [threading.Thread(target=search_elements) for _ in range(3)] + + # Start all threads + for thread in threads: + thread.start() + + # Wait for completion + for thread in threads: + thread.join(timeout=30) + + # Verify thread safety + assert len(errors) == 0, f"Should handle concurrent access without errors: {errors}" + assert len(results) > 0, "Should produce results from concurrent access" + + # Results should be consistent (same input should give same output) + if len(results) > 1: + first_result_count = len(results[0]) + for result in results[1:]: + # Allow for minor variations in OCR/detection, but should be roughly consistent + assert isinstance(result, list), "All results should be lists" + + +class TestCombinedSearchPerformanceIntegration: + """Test performance aspects of combined search integration""" + + def test_large_image_handling(self, synthetic_generator): + """ + INTEGRATION: Large image processing + + Big Picture: Healthcare systems may have high-resolution screens. + Combined search should handle large images efficiently. + """ + # Create larger test image + large_image = synthetic_generator.create_ui_mockup( + elements=["button", "text", "input"], canvas_width=800, canvas_height=600 + ) + + # Test large image processing + elements = find_elements_by_text( + image=large_image, text_query="Submit", confidence_threshold=0.4 + ) + + # Should handle large images without crashing + assert isinstance(elements, list), "Should process large images" + + # Test comprehensive analysis on large image + analysis = analyze_screen_content( + image=large_image, query="Complete analysis", confidence_threshold=0.4 + ) + + # Should produce valid analysis + assert isinstance(analysis, ScreenAnalysis), "Should analyze large images" + assert isinstance(analysis.summary, dict), "Should generate summary for large images" + + def test_multiple_element_handling(self, synthetic_generator): + """ + INTEGRATION: Multiple element processing + + Big Picture: Healthcare UIs have many interactive elements. + System must efficiently process screens with numerous elements. + """ + # Create image with multiple elements + complex_image = synthetic_generator.create_ui_mockup( + elements=["button", "text", "input", "button", "text"], + canvas_width=300, + canvas_height=200, + ) + + # Find all elements + all_elements = analyze_screen_content( + image=complex_image, query="Find all elements", confidence_threshold=0.3 + ) + + # Verify multiple element handling + assert isinstance(all_elements.ui_elements, list), "Should handle multiple elements" + assert len(all_elements.ui_elements) >= 0, "Should process all elements" + + # Summary should reflect multiple elements + if all_elements.summary["total_elements"] > 0: + assert all_elements.summary["total_elements"] >= 0, "Should count elements correctly" + + # Test clickable element identification with multiple elements + clickable = find_clickable_elements(complex_image, confidence_threshold=0.3) + assert isinstance(clickable, list), "Should identify multiple clickable elements" diff --git a/tests/integration/vision/test_ocr_integration.py b/tests/integration/vision/test_ocr_integration.py new file mode 100644 index 0000000..c82435a --- /dev/null +++ b/tests/integration/vision/test_ocr_integration.py @@ -0,0 +1,369 @@ +"""OCR Integration Tests + +Tests PaddleOCR integration and text extraction functionality: +- extract_text() core functionality and API contracts +- TextResult object structure for downstream processing +- extract_text_from_region() regional text extraction +- find_text_by_content() content-based search +- OCR manager caching and performance optimization + +Big Picture: OCR is critical for healthcare UI navigation where text-based +element identification is essential for precision workflows. +""" + +import numpy as np + +# Add src to path for imports +from src.vision.reader import ( + TextResult, + clear_ocr_cache, + extract_text, + extract_text_from_region, + find_text_by_content, + get_ocr_manager, + get_text_near_point, +) + + +class TestOCRIntegration: + """Test OCR integration with PaddleOCR backend""" + + def test_extract_text_returns_valid_textresult_objects( + self, simple_text_image, expected_text_result_structure + ): + """ + INTEGRATION: Core OCR pipeline produces correct API contracts + + Big Picture: MCP server and LLM agents depend on consistent TextResult + object structure. This test ensures the API contract is maintained. + """ + # Call core OCR function + text_results = extract_text( + image=simple_text_image, + language="en", + confidence_threshold=0.1, # Low threshold for synthetic text + ) + + # Verify API contract: returns list + assert isinstance(text_results, list), "extract_text must return list" + + # May not detect synthetic text reliably, but should not error + if text_results: + # Verify API contract: TextResult object structure + text_result = text_results[0] + assert isinstance(text_result, TextResult), "Must return TextResult objects" + + required_fields = expected_text_result_structure["required_fields"] + for field in required_fields: + assert hasattr(text_result, field), f"TextResult missing required field: {field}" + assert getattr(text_result, field) is not None, f"Field {field} cannot be None" + + # Verify data types and ranges for MCP server integration + assert isinstance(text_result.text, str), "text must be string" + assert isinstance(text_result.confidence, (int, float)), "confidence must be numeric" + assert 0.0 <= text_result.confidence <= 1.0, "confidence must be in range [0,1]" + assert isinstance(text_result.bbox, list), "bbox must be list of points" + assert len(text_result.bbox) == 4, "bbox must have 4 corner points" + assert isinstance(text_result.rect_bbox, tuple), "rect_bbox must be tuple" + assert len(text_result.rect_bbox) == 4, "rect_bbox must be (x1, y1, x2, y2)" + + def test_extract_text_from_region_integration(self, ui_mockup_image): + """ + INTEGRATION: Regional text extraction + + Big Picture: Healthcare UIs may require focused text extraction + from specific regions to avoid interference from other text. + """ + # Define test region (top half of image) + height, width = ui_mockup_image.shape[:2] + test_region = (0, 0, width, height // 2) + + # Test regional extraction + region_text = extract_text_from_region( + image=ui_mockup_image, region=test_region, language="en", confidence_threshold=0.1 + ) + + # Verify integration + assert isinstance(region_text, list), "extract_text_from_region must return list" + + # Verify coordinate adjustment to full image + for text_result in region_text: + x1, y1, x2, y2 = text_result.rect_bbox + + # Coordinates should be within full image bounds + assert 0 <= x1 < width, "Adjusted x1 coordinate should be valid" + assert 0 <= y1 < height, "Adjusted y1 coordinate should be valid" + assert x1 < x2 <= width, "Adjusted x2 coordinate should be valid" + assert y1 < y2 <= height, "Adjusted y2 coordinate should be valid" + + # Center should be within region bounds (adjusted to full image) + center_x, center_y = text_result.center + assert 0 <= center_x < width, "Adjusted center_x should be valid" + assert 0 <= center_y < height, "Adjusted center_y should be valid" + + def test_find_text_by_content_integration(self, synthetic_generator): + """ + INTEGRATION: Content-based text search + + Big Picture: Healthcare workflows often require finding specific + text content like medication names or procedure codes. + """ + # Create image with known text content + test_text = "Submit Form" + test_image = synthetic_generator.create_text_image(text=test_text, width=120, height=40) + + # Test content search - exact match + exact_matches = find_text_by_content( + image=test_image, + target_text=test_text, + language="en", + similarity_threshold=0.5, + case_sensitive=True, + ) + + # Test content search - case insensitive + case_insensitive_matches = find_text_by_content( + image=test_image, + target_text="submit form", # Different case + language="en", + similarity_threshold=0.5, + case_sensitive=False, + ) + + # Test content search - partial match + partial_matches = find_text_by_content( + image=test_image, + target_text="Submit", # Partial text + language="en", + similarity_threshold=0.5, + case_sensitive=False, + ) + + # Verify content search integration + assert isinstance(exact_matches, list), "Content search should return list" + assert isinstance(case_insensitive_matches, list), ( + "Case insensitive search should return list" + ) + assert isinstance(partial_matches, list), "Partial search should return list" + + # May not detect synthetic text reliably, but should handle all cases + for matches in [exact_matches, case_insensitive_matches, partial_matches]: + for text_result in matches: + assert isinstance(text_result, TextResult), "Should return TextResult objects" + assert isinstance(text_result.text, str), "Found text should be string" + + def test_get_text_near_point_integration(self, synthetic_generator): + """ + INTEGRATION: Spatial text search + + Big Picture: Healthcare UIs may need to find text labels + near specific UI elements for context-aware interaction. + """ + # Create UI with text at known positions + test_image = synthetic_generator.create_ui_mockup( + elements=["button", "text"], canvas_width=200, canvas_height=150 + ) + + # Test near point search - center of image + center_point = (100, 75) + nearby_text = get_text_near_point( + image=test_image, point=center_point, radius=100, language="en" + ) + + # Test near point search - corner (should find less/no text) + corner_point = (10, 10) + corner_text = get_text_near_point( + image=test_image, point=corner_point, radius=30, language="en" + ) + + # Verify spatial search integration + assert isinstance(nearby_text, list), "Spatial search should return list" + assert isinstance(corner_text, list), "Corner search should return list" + + # Verify distance-based sorting (may be empty for synthetic) + for text_result in nearby_text: + assert isinstance(text_result, TextResult), "Should return TextResult objects" + assert hasattr(text_result, "distance"), "Should add distance attribute for sorting" + + # If multiple results, should be sorted by distance + if len(nearby_text) > 1: + distances = [getattr(result, "distance", float("inf")) for result in nearby_text] + assert distances == sorted(distances), "Results should be sorted by distance" + + def test_ocr_manager_integration(self): + """ + INTEGRATION: OCR manager caching and language handling + + Big Picture: Healthcare systems may need multiple languages + and efficient caching to avoid reloading models. + """ + # Get OCR manager + ocr_manager = get_ocr_manager() + + # Test language caching + cached_languages_before = ocr_manager.get_cached_languages() + + # Get OCR instance (should cache) + ocr_en = ocr_manager.get_ocr("en") + assert ocr_en is not None, "Should return OCR instance" + + # Check caching worked + cached_languages_after = ocr_manager.get_cached_languages() + assert "en" in cached_languages_after, "Should cache English OCR instance" + assert len(cached_languages_after) >= len(cached_languages_before), "Cache should grow" + + # Get same instance should return cached + ocr_en_2 = ocr_manager.get_ocr("en") + assert ocr_en is ocr_en_2, "Should return same cached instance" + + # Test cache clearing + clear_ocr_cache() + cached_after_clear = ocr_manager.get_cached_languages() + assert len(cached_after_clear) == 0, "Cache should be cleared" + + # Verify cache functionality for performance optimization + assert hasattr(ocr_manager, "_instances"), "Manager should have instances cache" + assert callable(ocr_manager.clear_cache), "Manager should have clear_cache method" + + +class TestOCRErrorHandling: + """Test OCR error handling and robustness""" + + def test_invalid_image_data_handled_gracefully(self): + """ + ERROR HANDLING: Invalid image input + + Big Picture: MCP server may receive corrupted image data. + OCR system must handle errors gracefully without crashing. + """ + # Test None image + text_results = extract_text(image=None, language="en", confidence_threshold=0.5) + assert isinstance(text_results, list), "Should handle None image gracefully" + assert len(text_results) == 0, "Should return empty list for None image" + + # Test invalid image format + invalid_image = np.array([1, 2, 3]) # Wrong shape + text_results = extract_text(image=invalid_image, language="en", confidence_threshold=0.5) + assert isinstance(text_results, list), "Should handle invalid image format" + + def test_invalid_parameters_handled_gracefully(self, simple_text_image): + """ + ERROR HANDLING: Invalid parameter handling + + Big Picture: MCP server parameters may be malformed. + OCR system should validate inputs and handle errors gracefully. + """ + # Test invalid confidence threshold + text_results = extract_text( + image=simple_text_image, + confidence_threshold=-0.5, # Invalid range + ) + assert isinstance(text_results, list), "Should handle invalid confidence" + + # Test confidence threshold > 1.0 + text_results = extract_text( + image=simple_text_image, + confidence_threshold=1.5, # Invalid range + ) + assert isinstance(text_results, list), "Should handle confidence > 1.0" + + # Test invalid language (should not crash) + try: + text_results = extract_text(image=simple_text_image, language="invalid_lang") + assert isinstance(text_results, list), "Should handle invalid language" + except Exception: + # Acceptable to raise exception for invalid language + pass + + def test_invalid_region_parameters_handled_gracefully(self, simple_text_image): + """ + ERROR HANDLING: Invalid region parameter handling + + Big Picture: Region coordinates may be out of bounds or malformed. + System should handle gracefully to prevent crashes. + """ + height, width = simple_text_image.shape[:2] + + # Test region outside image bounds + invalid_region = (width, height, width + 100, height + 100) + text_results = extract_text_from_region(image=simple_text_image, region=invalid_region) + assert isinstance(text_results, list), "Should handle out-of-bounds region" + + # Test negative coordinates + negative_region = (-10, -10, 50, 50) + text_results = extract_text_from_region(image=simple_text_image, region=negative_region) + assert isinstance(text_results, list), "Should handle negative coordinates" + + # Test inverted region (x2 < x1, y2 < y1) + inverted_region = (50, 50, 10, 10) + text_results = extract_text_from_region(image=simple_text_image, region=inverted_region) + assert isinstance(text_results, list), "Should handle inverted region coordinates" + + +class TestOCRPerformanceIntegration: + """Test OCR performance and optimization integration""" + + def test_ocr_caching_performance(self, synthetic_generator): + """ + INTEGRATION: OCR caching for performance optimization + + Big Picture: Healthcare systems process many images. + OCR model caching prevents redundant loading for better performance. + """ + # Create test image + test_image = synthetic_generator.create_text_image("Test Performance", 150, 50) + + # Clear cache first + clear_ocr_cache() + ocr_manager = get_ocr_manager() + + # First extraction should load model + cached_before = len(ocr_manager.get_cached_languages()) + + text_results_1 = extract_text(test_image, language="en") + + cached_after_first = len(ocr_manager.get_cached_languages()) + assert cached_after_first > cached_before, "Should cache OCR model after first use" + + # Second extraction should use cached model + text_results_2 = extract_text(test_image, language="en") + + cached_after_second = len(ocr_manager.get_cached_languages()) + assert cached_after_second == cached_after_first, "Should reuse cached model" + + # Results should be consistent + assert isinstance(text_results_1, list), "First extraction should work" + assert isinstance(text_results_2, list), "Second extraction should work" + + def test_multiple_language_handling(self): + """ + INTEGRATION: Multiple language support + + Big Picture: Healthcare may require multilingual OCR + for diverse patient populations and international systems. + """ + ocr_manager = get_ocr_manager() + + # Test English language support + ocr_en = ocr_manager.get_ocr("en") + assert ocr_en is not None, "Should support English OCR" + + # Test that different languages create different instances + languages = ["en"] # Start with just English for testing + + for lang in languages: + ocr_instance = ocr_manager.get_ocr(lang) + assert ocr_instance is not None, f"Should support {lang} language" + + # Verify language caching + cached_langs = ocr_manager.get_cached_languages() + for lang in languages: + assert lang in cached_langs, f"Should cache {lang} language instance" + + # Test cache management + initial_count = len(cached_langs) + ocr_manager.clear_cache() + + cleared_langs = ocr_manager.get_cached_languages() + assert len(cleared_langs) == 0, "Cache should be cleared" + assert len(cleared_langs) < initial_count, "Should reduce cached instances" diff --git a/tests/integration/vision/test_template_detection_integration.py b/tests/integration/vision/test_template_detection_integration.py new file mode 100644 index 0000000..4d885d1 --- /dev/null +++ b/tests/integration/vision/test_template_detection_integration.py @@ -0,0 +1,384 @@ +"""Template Detection Integration Tests + +Tests the core template matching pipeline integration: +- detect_ui_elements() functionality and API contracts +- Template source normalization and processing +- Multi-template detection capabilities +- Integration with downstream MCP server requirements + +Big Picture: This is the heart of the vision system +for healthcare precision. LLM agents depend on reliable, consistent results. +""" + +# Add src to path for imports + +import numpy as np + +from src.vision.detector_template import Detection, detect_ui_elements +from src.vision.template_manager import TemplateSource, TemplateStrategy + + +class TestTemplateDetectionPipeline: + """Test core template detection pipeline integration""" + + def test_detect_ui_elements_returns_valid_detection_objects( + self, test_image_with_template, expected_detection_structure + ): + """ + INTEGRATION: Core detection pipeline produces correct API contracts + + Big Picture: MCP server and LLM agents depend on consistent Detection + object structure. This test ensures the API contract is maintained. + """ + test_image, template = test_image_with_template + + # Call core detection function + detections = detect_ui_elements( + image=test_image, + template_sources=template, + template_name="test_button", + confidence_threshold=0.5, + ) + + # Verify API contract: returns list + assert isinstance(detections, list), "detect_ui_elements must return list" + assert len(detections) > 0, "Should detect template in test image" + + # Verify API contract: Detection object structure + detection = detections[0] + assert isinstance(detection, Detection), "Must return Detection objects" + + required_fields = expected_detection_structure["required_fields"] + for field in required_fields: + assert hasattr(detection, field), f"Detection missing required field: {field}" + assert getattr(detection, field) is not None, f"Field {field} cannot be None" + + # Verify data types and ranges for MCP server integration + assert isinstance(detection.template_name, str), "template_name must be string" + assert isinstance(detection.confidence, (int, float)), "confidence must be numeric" + assert 0.0 <= detection.confidence <= 1.0, "confidence must be in range [0,1]" + assert isinstance(detection.bbox, tuple), "bbox must be tuple" + assert len(detection.bbox) == 4, "bbox must be (x1, y1, x2, y2)" + assert isinstance(detection.center, tuple), "center must be tuple" + assert len(detection.center) == 2, "center must be (x, y)" + + def test_template_source_normalization_numpy_array(self, simple_button_image): + """ + INTEGRATION: Backward compatibility with numpy array input + + Big Picture: Some parts of the system may still pass raw numpy arrays. + Template source normalization ensures consistent internal processing. + """ + test_image = np.full((80, 120, 3), (240, 240, 240), dtype=np.uint8) + test_image[30:50, 40:80] = simple_button_image + + # Test numpy array input (backward compatibility) + detections = detect_ui_elements( + image=test_image, + template_sources=simple_button_image, # Raw numpy array + template_name="numpy_button", + confidence_threshold=0.4, + ) + + # Verify normalization worked correctly + assert isinstance(detections, list), "Should handle numpy array input" + if detections: # May not find exact match, but should not error + assert detections[0].template_name == "numpy_button" + assert hasattr(detections[0], "template_source"), "Should create TemplateSource" + + def test_template_source_normalization_template_source_object(self, simple_button_image): + """ + INTEGRATION: TemplateSource object handling + + Big Picture: Template manager creates TemplateSource objects. + Detection pipeline must handle them correctly. + """ + test_image = np.full((80, 120, 3), (240, 240, 240), dtype=np.uint8) + test_image[30:50, 40:80] = simple_button_image + + # Create TemplateSource object + template_source = TemplateSource( + strategy=TemplateStrategy.BASE64, + template=simple_button_image, + metadata={"name": "source_button", "test": True}, + confidence_weight=1.0, + ) + + # Test TemplateSource input + detections = detect_ui_elements( + image=test_image, template_sources=template_source, confidence_threshold=0.4 + ) + + # Verify TemplateSource integration + assert isinstance(detections, list), "Should handle TemplateSource input" + if detections: + detection = detections[0] + assert detection.template_source is not None, "Should preserve TemplateSource" + assert detection.template_source.strategy == TemplateStrategy.BASE64 + assert detection.template_source.metadata["test"] == True + + def test_multi_template_detection_integration(self, synthetic_generator): + """ + INTEGRATION: Multi-template detection pipeline + + Big Picture: Healthcare UIs may have multiple elements to detect. + Multi-template detection is critical for comprehensive UI analysis. + """ + # Create test image with multiple button-like elements + test_image = np.full((120, 200, 3), (240, 240, 240), dtype=np.uint8) + + # Add different colored buttons + blue_button = synthetic_generator.create_simple_button(40, 20, (255, 123, 0), text="Submit") + green_button = synthetic_generator.create_simple_button(40, 20, (76, 175, 80), text="Save") + + test_image[20:40, 20:60] = blue_button + test_image[20:40, 80:120] = green_button + + # Create multiple template sources + template_sources = [ + TemplateSource( + strategy=TemplateStrategy.BASE64, + template=blue_button, + metadata={"name": "submit_button"}, + confidence_weight=1.0, + ), + TemplateSource( + strategy=TemplateStrategy.BASE64, + template=green_button, + metadata={"name": "save_button"}, + confidence_weight=1.0, + ), + ] + + # Test multi-template detection + detections = detect_ui_elements( + image=test_image, template_sources=template_sources, confidence_threshold=0.3 + ) + + # Verify multi-template integration + assert isinstance(detections, list), "Multi-template should return list" + + # Check that we can identify different templates + template_names = [d.template_name for d in detections] + unique_names = set(template_names) + + # Should detect multiple elements (may not be exact matches, but should find some) + assert len(detections) >= 1, "Should detect at least one template" + + # Verify multi-template metadata preservation + for detection in detections: + assert detection.template_source is not None, "Should preserve template source" + assert detection.template_match_method in ["single", "multi"], "Should set match method" + + def test_confidence_threshold_filtering_integration(self, test_image_with_template): + """ + INTEGRATION: Confidence threshold filtering + + Big Picture: Healthcare requires precise confidence control. + Different thresholds should produce consistent filtering behavior. + """ + test_image, template = test_image_with_template + + # Test different confidence thresholds + thresholds = [0.1, 0.5, 0.8, 0.95] + detection_counts = [] + + for threshold in thresholds: + detections = detect_ui_elements( + image=test_image, + template_sources=template, + template_name="threshold_test", + confidence_threshold=threshold, + ) + + detection_counts.append(len(detections)) + + # Verify confidence filtering + for detection in detections: + assert detection.confidence >= threshold, ( + f"Detection confidence {detection.confidence} below threshold {threshold}" + ) + + # Verify threshold behavior: higher thresholds should generally produce fewer detections + # (Though in synthetic tests this may not always hold) + assert all(isinstance(count, int) for count in detection_counts), ( + "All threshold tests should return integer counts" + ) + + def test_scale_and_rotation_options_integration(self, simple_button_image): + """ + INTEGRATION: Multi-scale and rotation options + + Big Picture: Healthcare UIs may appear at different scales/orientations. + These options must integrate correctly with the detection pipeline. + """ + # Create scaled test image + template_h, template_w = simple_button_image.shape[:2] + scaled_template = np.full( + (int(template_h * 1.5), int(template_w * 1.5), 3), (255, 123, 0), dtype=np.uint8 + ) + + test_image = np.full((120, 150, 3), (240, 240, 240), dtype=np.uint8) + test_image[20 : 20 + scaled_template.shape[0], 20 : 20 + scaled_template.shape[1]] = ( + scaled_template + ) + + # Test multi-scale detection + detections_multiscale = detect_ui_elements( + image=test_image, + template_sources=simple_button_image, + template_name="scale_test", + confidence_threshold=0.3, + enable_multiscale=True, + scale_range=(0.8, 2.0), + ) + + # Test without multi-scale for comparison + detections_single_scale = detect_ui_elements( + image=test_image, + template_sources=simple_button_image, + template_name="scale_test", + confidence_threshold=0.3, + enable_multiscale=False, + ) + + # Verify options integration + assert isinstance(detections_multiscale, list), "Multi-scale should return list" + assert isinstance(detections_single_scale, list), "Single-scale should return list" + + # Multi-scale should potentially find more matches + # (In synthetic tests this may not always hold, but should not error) + assert len(detections_multiscale) >= 0, "Multi-scale detection should work" + assert len(detections_single_scale) >= 0, "Single-scale detection should work" + + def test_detection_metadata_preservation(self, simple_button_image): + """ + INTEGRATION: Detection metadata preservation + + Big Picture: LLM agents may need metadata for decision making. + All detection metadata must be preserved through the pipeline. + """ + test_image = np.full((80, 120, 3), (240, 240, 240), dtype=np.uint8) + test_image[30:50, 40:80] = simple_button_image + + # Create template with rich metadata + template_source = TemplateSource( + strategy=TemplateStrategy.BASE64, + template=simple_button_image, + metadata={ + "name": "metadata_button", + "category": "healthcare", + "safety_critical": True, + "description": "Patient medication button", + }, + confidence_weight=1.2, + ) + + detections = detect_ui_elements( + image=test_image, template_sources=template_source, confidence_threshold=0.3 + ) + + # Verify metadata preservation + if detections: + detection = detections[0] + + # Core metadata preserved + assert detection.template_source is not None, "Should preserve TemplateSource" + assert detection.template_confidence == 1.2, "Should preserve confidence weight" + + # Rich metadata accessible + metadata = detection.template_source.metadata + assert metadata["category"] == "healthcare", "Should preserve category" + assert metadata["safety_critical"] == True, "Should preserve safety flags" + assert "description" in metadata, "Should preserve description" + + +class TestTemplateDetectionErrorHandling: + """Test error handling in template detection pipeline""" + + def test_invalid_image_data_handled_gracefully(self): + """ + ERROR HANDLING: Invalid image input + + Big Picture: MCP server may receive corrupted data. + System must handle errors gracefully without crashing. + """ + template = np.full((20, 40, 3), (255, 0, 0), dtype=np.uint8) + + # Test None image + detections = detect_ui_elements( + image=None, template_sources=template, template_name="error_test" + ) + assert isinstance(detections, list), "Should return empty list for None image" + assert len(detections) == 0, "Should return empty list for None image" + + # Test invalid image format (should be handled gracefully) + invalid_image = np.array([1, 2, 3]) # Wrong shape + detections = detect_ui_elements( + image=invalid_image, template_sources=template, template_name="error_test" + ) + assert isinstance(detections, list), "Should handle invalid image format" + + def test_invalid_template_data_handled_gracefully(self, simple_button_image): + """ + ERROR HANDLING: Invalid template input + + Big Picture: Template resolution may fail. + Detection must handle template errors gracefully. + """ + test_image = np.full((80, 120, 3), (240, 240, 240), dtype=np.uint8) + + # Test None template + detections = detect_ui_elements( + image=test_image, template_sources=None, template_name="error_test" + ) + assert isinstance(detections, list), "Should handle None template" + assert len(detections) == 0, "Should return empty for None template" + + # Test invalid template format + try: + invalid_template = "not_an_array" + detections = detect_ui_elements( + image=test_image, template_sources=invalid_template, template_name="error_test" + ) + # Should either handle gracefully or raise ValueError + assert isinstance(detections, list), "Should handle invalid template" + except ValueError: + # Acceptable to raise ValueError for invalid input + pass + + def test_invalid_parameters_handled_gracefully(self, test_image_with_template): + """ + ERROR HANDLING: Invalid parameter handling + + Big Picture: MCP server parameters may be malformed. + System should validate inputs and handle errors gracefully. + """ + test_image, template = test_image_with_template + + # Test invalid confidence threshold + detections = detect_ui_elements( + image=test_image, + template_sources=template, + template_name="param_test", + confidence_threshold=-0.5, # Invalid range + ) + assert isinstance(detections, list), "Should handle invalid confidence" + + # Test confidence threshold > 1.0 + detections = detect_ui_elements( + image=test_image, + template_sources=template, + template_name="param_test", + confidence_threshold=1.5, # Invalid range + ) + assert isinstance(detections, list), "Should handle confidence > 1.0" + + # Test invalid scale range + detections = detect_ui_elements( + image=test_image, + template_sources=template, + template_name="param_test", + scale_range=(2.0, 0.5), # Inverted range + ) + assert isinstance(detections, list), "Should handle invalid scale range" diff --git a/tests/integration/vision/test_template_manager_integration.py b/tests/integration/vision/test_template_manager_integration.py new file mode 100644 index 0000000..0622376 --- /dev/null +++ b/tests/integration/vision/test_template_manager_integration.py @@ -0,0 +1,446 @@ +"""Template Manager Integration Tests + +Tests template resolution and caching integration: +- Base64 template resolution from MCP server data +- Library template resolution from template library +- Multi-template resolution for complex scenarios +- Template caching mechanism for performance +- Integration with downstream detection pipeline + +Big Picture: Template Manager is the bridge between MCP server parameters +and the detection pipeline. LLM agents depend on reliable template resolution. +""" + +import base64 + +# Add src to path for imports +import cv2 +import numpy as np +import pytest + +from src.vision.template_manager import ( + TemplateRequest, + TemplateSource, + TemplateStrategy, + TemplateValidationError, +) + + +class TestTemplateManagerIntegration: + """Test template manager integration with detection pipeline""" + + def test_base64_template_resolution_integration(self, template_manager, simple_button_image): + """ + INTEGRATION: Base64 template resolution from MCP server + + Big Picture: MCP server receives base64 templates from LLM agents. + Template manager must convert these reliably for detection pipeline. + """ + # Create base64 template (simulating MCP server input) + _, buffer = cv2.imencode(".png", simple_button_image) + base64_data = base64.b64encode(buffer).decode("utf-8") + + # Create template request (simulating MCP handler) + template_request = TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="mcp_button" + ) + + # Test template resolution + resolved_templates = template_manager.resolve_templates([template_request]) + + # Verify resolution integration + assert isinstance(resolved_templates, list), "Should return list of TemplateSource" + assert len(resolved_templates) == 1, "Should resolve single template" + + template_source = resolved_templates[0] + assert isinstance(template_source, TemplateSource), "Should create TemplateSource object" + assert template_source.strategy == TemplateStrategy.BASE64, "Should preserve strategy" + assert isinstance(template_source.template, np.ndarray), "Should create numpy array" + assert template_source.template.shape == simple_button_image.shape, ( + "Should preserve image dimensions" + ) + + # Verify metadata for downstream integration + metadata = template_source.metadata + assert "name" in metadata, "Should include name in metadata" + assert metadata["source"] == "base64", "Should mark source type" + assert "size" in metadata, "Should include size metadata" + + def test_library_template_resolution_integration(self, template_manager): + """ + INTEGRATION: Library template resolution + + Big Picture: Healthcare teams will use library templates for consistency. + Template manager must resolve these from the template library. + """ + # Create library template request (simulating healthcare workflow) + template_request = TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": "submit_blue", "category": "buttons"}, + name="library_button", + ) + + try: + # Test library resolution + resolved_templates = template_manager.resolve_templates([template_request]) + + # Verify library integration + assert isinstance(resolved_templates, list), "Should return list for library resolution" + + if resolved_templates: # Library may not be available in test environment + template_source = resolved_templates[0] + assert isinstance(template_source, TemplateSource), "Should create TemplateSource" + assert template_source.strategy == TemplateStrategy.LIBRARY, ( + "Should preserve strategy" + ) + assert isinstance(template_source.template, np.ndarray), ( + "Should load template image" + ) + + # Verify library metadata preservation + metadata = template_source.metadata + assert metadata.get("id") == "submit_blue", "Should preserve template ID" + assert metadata.get("category") == "buttons", "Should preserve category" + assert metadata["source"] == "library", "Should mark library source" + + except (FileNotFoundError, TemplateValidationError): + # Template library may not be available in test environment + pytest.skip("Template library not available in test environment") + + def test_multi_template_resolution_integration(self, template_manager, simple_button_image): + """ + INTEGRATION: Multi-template resolution for complex scenarios + + Big Picture: Healthcare UIs may require multiple templates for complete analysis. + Template manager must handle mixed template sources efficiently. + """ + # Create base64 template for mixing with library + _, buffer = cv2.imencode(".png", simple_button_image) + base64_data = base64.b64encode(buffer).decode("utf-8") + + # Create mixed template requests (realistic healthcare scenario) + template_requests = [ + TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="custom_button", weight=1.0 + ), + TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": "submit_blue", "category": "buttons"}, + name="standard_button", + weight=0.8, + ), + ] + + # Test multi-template resolution + resolved_templates = template_manager.resolve_templates(template_requests) + + # Verify multi-template integration + assert isinstance(resolved_templates, list), "Should return list for multi-template" + assert len(resolved_templates) >= 1, "Should resolve at least base64 template" + + # Verify mixed strategy handling + strategies = [t.strategy for t in resolved_templates] + assert TemplateStrategy.BASE64 in strategies, "Should resolve base64 template" + + # Verify weight preservation for detection pipeline + base64_template = next( + t for t in resolved_templates if t.strategy == TemplateStrategy.BASE64 + ) + assert base64_template.confidence_weight == 1.0, "Should preserve confidence weight" + + # Verify unique cache keys for performance + cache_keys = [t.cache_key for t in resolved_templates] + assert len(set(cache_keys)) == len(cache_keys), "Should generate unique cache keys" + + def test_template_validation_integration(self, template_manager): + """ + INTEGRATION: Template validation for robustness + + Big Picture: MCP server may receive invalid data from LLM agents. + Template manager must validate inputs and handle errors gracefully. + """ + # Test invalid base64 data + invalid_base64_request = TemplateRequest( + strategy=TemplateStrategy.BASE64, data="invalid_base64_data", name="invalid_template" + ) + + # Test validation handling + try: + resolved_templates = template_manager.resolve_templates([invalid_base64_request]) + # If no exception, should return empty list or handle gracefully + assert isinstance(resolved_templates, list), "Should handle invalid base64 gracefully" + except TemplateValidationError: + # Acceptable to raise validation error + pass + + # Test empty template requests + empty_resolved = template_manager.resolve_templates([]) + assert isinstance(empty_resolved, list), "Should handle empty requests" + assert len(empty_resolved) == 0, "Should return empty list for empty requests" + + # Test None template request (error recovery) + try: + none_resolved = template_manager.resolve_templates(None) + assert isinstance(none_resolved, list), "Should handle None requests" + except (TypeError, ValueError): + # Acceptable to raise error for None input + pass + + def test_template_cache_key_generation_integration(self, template_manager, simple_button_image): + """ + INTEGRATION: Cache key generation for performance optimization + + Big Picture: Template resolution is expensive. Caching prevents + redundant processing when LLM agents request same templates. + """ + # Create identical template requests + _, buffer = cv2.imencode(".png", simple_button_image) + base64_data = base64.b64encode(buffer).decode("utf-8") + + template_request1 = TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="cache_test" + ) + + template_request2 = TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="cache_test" + ) + + # Resolve templates multiple times + resolved1 = template_manager.resolve_templates([template_request1]) + resolved2 = template_manager.resolve_templates([template_request2]) + + # Verify cache key consistency + assert len(resolved1) > 0, "Should resolve template" + assert len(resolved2) > 0, "Should resolve template" + + cache_key1 = resolved1[0].cache_key + cache_key2 = resolved2[0].cache_key + + assert cache_key1 == cache_key2, "Identical templates should have same cache key" + assert cache_key1 is not None, "Cache key should not be None" + assert len(cache_key1) > 0, "Cache key should not be empty" + + # Test cache key uniqueness for different templates + different_request = TemplateRequest( + strategy=TemplateStrategy.BASE64, + data=base64_data, + name="different_cache_test", # Different name + ) + + resolved3 = template_manager.resolve_templates([different_request]) + cache_key3 = resolved3[0].cache_key + + assert cache_key3 != cache_key1, "Different templates should have different cache keys" + + def test_template_metadata_enrichment_integration(self, template_manager, simple_button_image): + """ + INTEGRATION: Metadata enrichment for downstream processing + + Big Picture: Detection pipeline and LLM agents need rich metadata + for decision making. Template manager must enrich metadata appropriately. + """ + # Create template request with minimal data + _, buffer = cv2.imencode(".png", simple_button_image) + base64_data = base64.b64encode(buffer).decode("utf-8") + + template_request = TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="metadata_test", weight=1.2 + ) + + # Test metadata enrichment + resolved_templates = template_manager.resolve_templates([template_request]) + template_source = resolved_templates[0] + + # Verify automatic metadata enrichment + metadata = template_source.metadata + + # Core metadata from request + assert metadata["name"] == "metadata_test", "Should preserve request name" + assert template_source.confidence_weight == 1.2, "Should preserve confidence weight" + + # Automatic enrichment + assert "source" in metadata, "Should add source type" + assert "size" in metadata, "Should add image size" + assert metadata["source"] == "base64", "Should identify source correctly" + + # Size metadata format for downstream integration + size_data = metadata["size"] + assert isinstance(size_data, (tuple, list)), "Size should be tuple/list" + assert len(size_data) >= 2, "Size should include width/height" + + def test_template_source_immutability_integration(self, template_manager, simple_button_image): + """ + INTEGRATION: Template source immutability for thread safety + + Big Picture: Multiple LLM agents may access templates concurrently. + Template sources must be immutable to prevent threading issues. + """ + _, buffer = cv2.imencode(".png", simple_button_image) + base64_data = base64.b64encode(buffer).decode("utf-8") + + template_request = TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="immutable_test" + ) + + resolved_templates = template_manager.resolve_templates([template_request]) + template_source = resolved_templates[0] + + # Verify immutable data structures + original_metadata = template_source.metadata.copy() + original_template_shape = template_source.template.shape + + # Attempt to modify template (should not affect original) + template_copy = template_source.template.copy() + template_copy[0, 0, 0] = 255 # Modify copy + + # Verify original unchanged + assert template_source.template.shape == original_template_shape, ( + "Original template unchanged" + ) + assert template_source.metadata == original_metadata, "Original metadata unchanged" + + # Verify immutable properties + assert hasattr(template_source, "strategy"), "Should have strategy property" + assert hasattr(template_source, "cache_key"), "Should have cache_key property" + assert hasattr(template_source, "confidence_weight"), ( + "Should have confidence_weight property" + ) + + +class TestTemplateManagerErrorRecovery: + """Test template manager error recovery and robustness""" + + def test_partial_resolution_recovery(self, template_manager, simple_button_image): + """ + ERROR RECOVERY: Partial template resolution handling + + Big Picture: In healthcare workflows, some templates may fail to load. + System should continue with available templates, not fail completely. + """ + # Create mixed requests with one invalid + _, buffer = cv2.imencode(".png", simple_button_image) + base64_data = base64.b64encode(buffer).decode("utf-8") + + mixed_requests = [ + TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="valid_template" + ), + TemplateRequest( + strategy=TemplateStrategy.BASE64, data="invalid_base64", name="invalid_template" + ), + TemplateRequest( + strategy=TemplateStrategy.LIBRARY, + data={"id": "nonexistent_template", "category": "invalid"}, + name="missing_template", + ), + ] + + # Test partial resolution recovery + try: + resolved_templates = template_manager.resolve_templates(mixed_requests) + + # Should recover and return valid templates + assert isinstance(resolved_templates, list), "Should return list despite errors" + + # Should have at least the valid base64 template + valid_templates = [ + t for t in resolved_templates if t.metadata.get("name") == "valid_template" + ] + assert len(valid_templates) > 0, "Should resolve at least valid templates" + + except Exception as e: + # If complete failure, should be a clear error message + assert isinstance(e, (TemplateValidationError, ValueError)), ( + f"Should raise specific error, got {type(e)}" + ) + + def test_memory_pressure_handling(self, template_manager): + """ + ERROR RECOVERY: Memory pressure handling + + Big Picture: Healthcare systems may process many large templates. + Template manager should handle memory constraints gracefully. + """ + # Create large template requests (simulating memory pressure) + large_requests = [] + + for i in range(10): # Multiple large templates + # Create large synthetic template + large_template = np.full((200, 200, 3), (i * 25, 100, 150), dtype=np.uint8) + _, buffer = cv2.imencode(".png", large_template) + base64_data = base64.b64encode(buffer).decode("utf-8") + + large_requests.append( + TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name=f"large_template_{i}" + ) + ) + + # Test memory handling + try: + resolved_templates = template_manager.resolve_templates(large_requests) + + # Should handle large requests without crashing + assert isinstance(resolved_templates, list), "Should handle large requests" + + # Verify all resolved templates are valid + for template_source in resolved_templates: + assert isinstance(template_source.template, np.ndarray), ( + "Should create valid arrays" + ) + assert template_source.template.size > 0, "Templates should not be empty" + + except MemoryError: + # Acceptable to fail with memory error if system constraints hit + pytest.skip("System memory constraints exceeded") + + def test_concurrent_resolution_safety(self, template_manager, simple_button_image): + """ + ERROR RECOVERY: Concurrent resolution thread safety + + Big Picture: Multiple LLM agents may request templates simultaneously. + Template manager must handle concurrent access safely. + """ + import threading + + # Create template request + _, buffer = cv2.imencode(".png", simple_button_image) + base64_data = base64.b64encode(buffer).decode("utf-8") + + template_request = TemplateRequest( + strategy=TemplateStrategy.BASE64, data=base64_data, name="concurrent_test" + ) + + # Thread-safe resolution test + results = [] + errors = [] + + def resolve_template(): + try: + resolved = template_manager.resolve_templates([template_request]) + results.append(resolved) + except Exception as e: + errors.append(e) + + # Create multiple threads + threads = [threading.Thread(target=resolve_template) for _ in range(5)] + + # Start all threads + for thread in threads: + thread.start() + + # Wait for completion + for thread in threads: + thread.join(timeout=10) + + # Verify thread safety + assert len(errors) == 0, f"Should handle concurrent access without errors: {errors}" + assert len(results) > 0, "Should produce results from concurrent access" + + # Verify consistency across threads + if len(results) > 1: + cache_keys = [r[0].cache_key for r in results if r] + unique_keys = set(cache_keys) + assert len(unique_keys) == 1, ( + "Concurrent resolutions should produce consistent cache keys" + ) diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index 3b67e8b..0000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,33 +0,0 @@ -[pytest] -# Configuration for pytest -minversion = 6.0 -addopts = - -ra - --strict-markers - --strict-config - --cov=src - --cov-branch - --cov-report=term-missing:skip-covered - --cov-report=html:htmlcov - --cov-report=xml - --cov-fail-under=80 - -testpaths = tests -python_files = test_*.py *_test.py -python_classes = Test* -python_functions = test_* - -markers = - slow: marks tests as slow (deselect with '-m "not slow"') - integration: marks tests as integration tests - mock: marks tests as using mock components - vnc: marks tests as VNC-specific - rdp: marks tests as RDP-specific - patient_workflow: marks tests as patient workflow tests - ui_detection: marks tests as UI detection tests - real_vm: marks tests as requiring real VM connections - -filterwarnings = - ignore::UserWarning - ignore::DeprecationWarning - ignore::PendingDeprecationWarning \ No newline at end of file diff --git a/tests/run_tests.py b/tests/run_tests.py deleted file mode 100755 index 6f15604..0000000 --- a/tests/run_tests.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python3 -""" -Test execution script for the computer-use-agent automation framework. - -This script provides different test execution modes: -- Unit tests only -- Integration tests only -- All tests -- Coverage reporting -- Performance testing -""" - -import argparse -import subprocess -import sys -from pathlib import Path - - -def run_command(cmd: list[str], description: str) -> bool: - """Run a command and return success status.""" - print(f"\n๐Ÿ”„ {description}") - print(f"Command: {' '.join(cmd)}") - - try: - result = subprocess.run(cmd, check=True, capture_output=False) - print(f"โœ… {description} completed successfully") - return True - except subprocess.CalledProcessError as e: - print(f"โŒ {description} failed with exit code {e.returncode}") - return False - - -def run_unit_tests(coverage: bool = True, verbose: bool = False) -> bool: - """Run unit tests.""" - cmd = ["python", "-m", "pytest", "tests/unit/"] - - if coverage: - cmd.extend(["--cov=src", "--cov-report=html", "--cov-report=term-missing"]) - - if verbose: - cmd.append("-v") - else: - cmd.append("-q") - - return run_command(cmd, "Running unit tests") - - -def run_integration_tests(mock_only: bool = True, verbose: bool = False) -> bool: - """Run integration tests.""" - cmd = ["python", "-m", "pytest", "tests/integration/"] - - if mock_only: - cmd.extend(["-m", "mock and not real_vm"]) - - if verbose: - cmd.append("-v") - else: - cmd.append("-q") - - return run_command(cmd, "Running integration tests") - - -def run_linting() -> bool: - """Run code linting.""" - success = True - - # Ruff check - success &= run_command(["ruff", "check", "src/", "tests/"], "Running ruff linter") - - # Ruff format check - success &= run_command( - ["ruff", "format", "--check", "src/", "tests/"], "Checking code formatting" - ) - - return success - - -def run_type_checking() -> bool: - """Run type checking.""" - return run_command(["pyright", "src/"], "Running type checking") - - -def run_all_tests(coverage: bool = True, verbose: bool = False) -> bool: - """Run all tests and checks.""" - success = True - - print("๐Ÿงช Running complete test suite for computer-use-agent") - - # Linting and formatting - success &= run_linting() - - # Type checking - success &= run_type_checking() - - # Unit tests - success &= run_unit_tests(coverage=coverage, verbose=verbose) - - # Integration tests (mock only by default) - success &= run_integration_tests(mock_only=True, verbose=verbose) - - if success: - print("\n๐ŸŽ‰ All tests passed!") - if coverage: - print("๐Ÿ“Š Coverage report generated in htmlcov/") - else: - print("\n๐Ÿ’ฅ Some tests failed!") - - return success - - -def generate_coverage_report() -> bool: - """Generate a detailed coverage report.""" - cmd = [ - "python", - "-m", - "pytest", - "tests/unit/", - "--cov=src", - "--cov-report=html", - "--cov-report=xml", - "--cov-report=term", - "--cov-fail-under=70", - "-v", - ] - - success = run_command(cmd, "Generating coverage report") - - if success: - print("๐Ÿ“Š Coverage report generated:") - print(f" - HTML: {Path.cwd() / 'htmlcov' / 'index.html'}") - print(f" - XML: {Path.cwd() / 'coverage.xml'}") - - return success - - -def main(): - """Main entry point.""" - parser = argparse.ArgumentParser( - description="Test runner for computer-use-agent automation framework" - ) - - parser.add_argument( - "mode", - choices=["unit", "integration", "all", "coverage", "lint", "types"], - help="Test mode to run", - ) - - parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") - - parser.add_argument("--no-coverage", action="store_true", help="Skip coverage reporting") - - parser.add_argument( - "--real-vm", - action="store_true", - help="Include real VM tests (requires actual VM connections)", - ) - - args = parser.parse_args() - - # Set working directory to script location - script_dir = Path(__file__).parent - if Path.cwd() != script_dir: - print(f"Changing directory to {script_dir}") - import os - - os.chdir(script_dir) - - coverage_enabled = not args.no_coverage - success = False - - if args.mode == "unit": - success = run_unit_tests(coverage=coverage_enabled, verbose=args.verbose) - elif args.mode == "integration": - success = run_integration_tests(mock_only=not args.real_vm, verbose=args.verbose) - elif args.mode == "all": - success = run_all_tests(coverage=coverage_enabled, verbose=args.verbose) - elif args.mode == "coverage": - success = generate_coverage_report() - elif args.mode == "lint": - success = run_linting() - elif args.mode == "types": - success = run_type_checking() - - sys.exit(0 if success else 1) - - -if __name__ == "__main__": - main() diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index 523c0ea..0000000 --- a/tests/unit/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for the automation framework.""" diff --git a/tests/unit/automation/core/__init__.py b/tests/unit/automation/core/__init__.py deleted file mode 100644 index 934fd6e..0000000 --- a/tests/unit/automation/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for automation.core module.""" diff --git a/tests/unit/automation/core/test_base.py b/tests/unit/automation/core/test_base.py deleted file mode 100644 index 2c8e3c3..0000000 --- a/tests/unit/automation/core/test_base.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Unit tests for automation.core.base module.""" - - -import numpy as np -import pytest - -from automation.core.base import VMConnection -from automation.core.types import ActionResult, ConnectionResult - - -class ConcreteVMConnection(VMConnection): - """Concrete implementation of VMConnection for testing.""" - - def connect( - self, - host: str, - port: int, - username: str | None = None, - password: str | None = None, - **kwargs, - ) -> ConnectionResult: - self.is_connected = True - self.connection_info = {"host": host, "port": port, "username": username, **kwargs} - return ConnectionResult(success=True, message="Connected") - - def disconnect(self) -> ConnectionResult: - self.is_connected = False - self.connection_info = {} - return ConnectionResult(success=True, message="Disconnected") - - def capture_screen(self) -> tuple[bool, np.ndarray | None]: - if self.is_connected: - # Return a mock 100x100 RGB image - image = np.zeros((100, 100, 3), dtype=np.uint8) - return True, image - return False, None - - def click(self, x: int, y: int, button: str = "left") -> ActionResult: - if self.is_connected: - return ActionResult(success=True, message=f"Clicked at ({x}, {y}) with {button}") - return ActionResult(success=False, message="Not connected") - - def type_text(self, text: str) -> ActionResult: - if self.is_connected: - return ActionResult(success=True, message=f"Typed: {text}") - return ActionResult(success=False, message="Not connected") - - def key_press(self, key: str) -> ActionResult: - if self.is_connected: - return ActionResult(success=True, message=f"Pressed key: {key}") - return ActionResult(success=False, message="Not connected") - - -class TestVMConnection: - """Test cases for VMConnection abstract base class.""" - - def test_vm_connection_initialization(self): - """Test VMConnection initialization.""" - connection = ConcreteVMConnection() - - assert connection.is_connected is False - assert connection.connection_info == {} - - def test_cannot_instantiate_abstract_class(self): - """Test that VMConnection cannot be instantiated directly.""" - with pytest.raises(TypeError): - VMConnection() - - def test_connect_method(self): - """Test connect method implementation.""" - connection = ConcreteVMConnection() - - result = connection.connect( - host="192.168.1.100", port=5900, username="test", password="secret" - ) - - assert isinstance(result, ConnectionResult) - assert result.success is True - assert result.message == "Connected" - assert connection.is_connected is True - assert connection.connection_info["host"] == "192.168.1.100" - assert connection.connection_info["port"] == 5900 - assert connection.connection_info["username"] == "test" - - def test_connect_with_kwargs(self): - """Test connect method with additional kwargs.""" - connection = ConcreteVMConnection() - - result = connection.connect(host="test.host", port=8080, timeout=30, ssl=True) - - assert result.success is True - assert connection.connection_info["timeout"] == 30 - assert connection.connection_info["ssl"] is True - - def test_disconnect_method(self): - """Test disconnect method implementation.""" - connection = ConcreteVMConnection() - connection.connect("192.168.1.100", 5900) - - result = connection.disconnect() - - assert isinstance(result, ConnectionResult) - assert result.success is True - assert result.message == "Disconnected" - assert connection.is_connected is False - assert connection.connection_info == {} - - def test_capture_screen_when_connected(self): - """Test screen capture when connected.""" - connection = ConcreteVMConnection() - connection.connect("192.168.1.100", 5900) - - success, image = connection.capture_screen() - - assert success is True - assert image is not None - assert isinstance(image, np.ndarray) - assert image.shape == (100, 100, 3) - assert image.dtype == np.uint8 - - def test_capture_screen_when_disconnected(self): - """Test screen capture when not connected.""" - connection = ConcreteVMConnection() - - success, image = connection.capture_screen() - - assert success is False - assert image is None - - def test_click_when_connected(self): - """Test click action when connected.""" - connection = ConcreteVMConnection() - connection.connect("192.168.1.100", 5900) - - result = connection.click(100, 200) - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Clicked at (100, 200) with left" in result.message - - def test_click_with_different_button(self): - """Test click action with different mouse button.""" - connection = ConcreteVMConnection() - connection.connect("192.168.1.100", 5900) - - result = connection.click(50, 75, button="right") - - assert result.success is True - assert "Clicked at (50, 75) with right" in result.message - - def test_click_when_disconnected(self): - """Test click action when not connected.""" - connection = ConcreteVMConnection() - - result = connection.click(100, 200) - - assert result.success is False - assert result.message == "Not connected" - - def test_type_text_when_connected(self): - """Test type text action when connected.""" - connection = ConcreteVMConnection() - connection.connect("192.168.1.100", 5900) - - result = connection.type_text("Hello, World!") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Typed: Hello, World!" in result.message - - def test_type_text_when_disconnected(self): - """Test type text action when not connected.""" - connection = ConcreteVMConnection() - - result = connection.type_text("Hello") - - assert result.success is False - assert result.message == "Not connected" - - def test_key_press_when_connected(self): - """Test key press action when connected.""" - connection = ConcreteVMConnection() - connection.connect("192.168.1.100", 5900) - - result = connection.key_press("Enter") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Pressed key: Enter" in result.message - - def test_key_press_when_disconnected(self): - """Test key press action when not connected.""" - connection = ConcreteVMConnection() - - result = connection.key_press("Escape") - - assert result.success is False - assert result.message == "Not connected" - - def test_get_connection_info(self): - """Test get_connection_info method.""" - connection = ConcreteVMConnection() - connection.connect(host="test.example.com", port=3389, username="admin", timeout=60) - - info = connection.get_connection_info() - - assert isinstance(info, dict) - assert info["host"] == "test.example.com" - assert info["port"] == 3389 - assert info["username"] == "admin" - assert info["timeout"] == 60 - - def test_get_connection_info_returns_copy(self): - """Test that get_connection_info returns a copy, not reference.""" - connection = ConcreteVMConnection() - connection.connect(host="test.host", port=1234) - - info1 = connection.get_connection_info() - info2 = connection.get_connection_info() - - # Modify one copy - info1["modified"] = True - - # Original and second copy should not be affected - assert "modified" not in connection.connection_info - assert "modified" not in info2 - - def test_get_connection_info_when_disconnected(self): - """Test get_connection_info when not connected.""" - connection = ConcreteVMConnection() - - info = connection.get_connection_info() - - assert isinstance(info, dict) - assert len(info) == 0 - - def test_connection_state_consistency(self): - """Test that connection state remains consistent through operations.""" - connection = ConcreteVMConnection() - - # Initially disconnected - assert connection.is_connected is False - - # Connect - connection.connect("192.168.1.100", 5900) - assert connection.is_connected is True - - # Actions work when connected - assert connection.click(10, 10).success is True - assert connection.type_text("test").success is True - assert connection.key_press("Tab").success is True - - # Disconnect - connection.disconnect() - assert connection.is_connected is False - - # Actions fail when disconnected - assert connection.click(10, 10).success is False - assert connection.type_text("test").success is False - assert connection.key_press("Tab").success is False diff --git a/tests/unit/automation/core/test_types.py b/tests/unit/automation/core/test_types.py deleted file mode 100644 index 95c990f..0000000 --- a/tests/unit/automation/core/test_types.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Unit tests for automation.core.types module.""" - -import time -from unittest.mock import patch - -from automation.core.types import ActionResult, ConnectionResult - - -class TestActionResult: - """Test cases for ActionResult dataclass.""" - - def test_action_result_creation_with_timestamp(self): - """Test ActionResult creation with explicit timestamp.""" - timestamp = time.time() - result = ActionResult(success=True, message="Test action completed", timestamp=timestamp) - - assert result.success is True - assert result.message == "Test action completed" - assert result.timestamp == timestamp - - def test_action_result_creation_without_timestamp(self): - """Test ActionResult creation with auto-generated timestamp.""" - with patch("automation.core.types.time.time", return_value=1234567890.0): - result = ActionResult(success=False, message="Test action failed") - - assert result.success is False - assert result.message == "Test action failed" - assert result.timestamp == 1234567890.0 - - def test_action_result_timestamp_auto_generation(self): - """Test that timestamp is automatically generated when None.""" - before_time = time.time() - result = ActionResult(success=True, message="Test message", timestamp=None) - after_time = time.time() - - assert before_time <= result.timestamp <= after_time - - def test_action_result_success_types(self): - """Test ActionResult with different success values.""" - success_result = ActionResult(success=True, message="Success") - failure_result = ActionResult(success=False, message="Failure") - - assert success_result.success is True - assert failure_result.success is False - - def test_action_result_message_types(self): - """Test ActionResult with different message types.""" - result = ActionResult(success=True, message="Test message") - empty_result = ActionResult(success=True, message="") - - assert result.message == "Test message" - assert empty_result.message == "" - - -class TestConnectionResult: - """Test cases for ConnectionResult dataclass.""" - - def test_connection_result_creation_with_timestamp(self): - """Test ConnectionResult creation with explicit timestamp.""" - timestamp = time.time() - result = ConnectionResult( - success=True, message="Connected successfully", timestamp=timestamp - ) - - assert result.success is True - assert result.message == "Connected successfully" - assert result.timestamp == timestamp - - def test_connection_result_creation_without_timestamp(self): - """Test ConnectionResult creation with auto-generated timestamp.""" - with patch("automation.core.types.time.time", return_value=9876543210.0): - result = ConnectionResult(success=False, message="Connection failed") - - assert result.success is False - assert result.message == "Connection failed" - assert result.timestamp == 9876543210.0 - - def test_connection_result_timestamp_auto_generation(self): - """Test that timestamp is automatically generated when None.""" - before_time = time.time() - result = ConnectionResult(success=True, message="Test connection", timestamp=None) - after_time = time.time() - - assert before_time <= result.timestamp <= after_time - - def test_connection_result_success_types(self): - """Test ConnectionResult with different success values.""" - success_result = ConnectionResult(success=True, message="Connected") - failure_result = ConnectionResult(success=False, message="Failed") - - assert success_result.success is True - assert failure_result.success is False - - def test_connection_result_message_variations(self): - """Test ConnectionResult with various message formats.""" - messages = [ - "Connection established", - "Failed to connect: timeout", - "", - "192.168.1.100:5900 connected", - ] - - for msg in messages: - result = ConnectionResult(success=True, message=msg) - assert result.message == msg - - -class TestTimestampBehavior: - """Test timestamp behavior across both result types.""" - - def test_timestamp_consistency_across_types(self): - """Test that both result types handle timestamps consistently.""" - timestamp = 1234567890.123 - - action_result = ActionResult(success=True, message="Action", timestamp=timestamp) - connection_result = ConnectionResult( - success=True, message="Connection", timestamp=timestamp - ) - - assert action_result.timestamp == connection_result.timestamp - assert action_result.timestamp == timestamp - - def test_none_timestamp_handling(self): - """Test that None timestamp triggers auto-generation for both types.""" - action_result = ActionResult(success=True, message="Action", timestamp=None) - connection_result = ConnectionResult(success=True, message="Connection", timestamp=None) - - assert action_result.timestamp is not None - assert connection_result.timestamp is not None - assert isinstance(action_result.timestamp, float) - assert isinstance(connection_result.timestamp, float) - - @patch("automation.core.types.time.time") - def test_multiple_instances_same_time(self, mock_time): - """Test multiple instances created at the same mocked time.""" - mock_time.return_value = 1500000000.0 - - result1 = ActionResult(success=True, message="First") - result2 = ConnectionResult(success=True, message="Second") - - assert result1.timestamp == 1500000000.0 - assert result2.timestamp == 1500000000.0 - assert result1.timestamp == result2.timestamp diff --git a/tests/unit/automation/local/__init__.py b/tests/unit/automation/local/__init__.py deleted file mode 100644 index 2706473..0000000 --- a/tests/unit/automation/local/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for automation.local module.""" diff --git a/tests/unit/automation/local/test_desktop_control.py b/tests/unit/automation/local/test_desktop_control.py deleted file mode 100644 index 110bde4..0000000 --- a/tests/unit/automation/local/test_desktop_control.py +++ /dev/null @@ -1,452 +0,0 @@ -"""Unit tests for automation.local.desktop_control module.""" - -import subprocess -import sys -from pathlib import Path -from unittest.mock import Mock, call, patch - -import numpy as np - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src")) - -from automation.core.types import ActionResult -from automation.local.desktop_control import DesktopControl - - -class TestDesktopControl: - """Test cases for DesktopControl class.""" - - def test_init(self): - """Test DesktopControl initialization.""" - desktop = DesktopControl() - - assert desktop.screenshot_path == Path("/tmp/desktop_screenshot.png") - assert desktop.is_active is True - - @patch("subprocess.run") - @patch("cv2.imread") - def test_capture_screen_success(self, mock_imread, mock_subprocess): - """Test successful screen capture.""" - desktop = DesktopControl() - - # Mock successful subprocess call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - # Mock OpenCV image loading - mock_image = np.zeros((1080, 1920, 3), dtype=np.uint8) - mock_imread.return_value = mock_image - - # Mock file existence - with patch("pathlib.Path.exists", return_value=True): - success, image = desktop.capture_screen() - - assert success is True - assert np.array_equal(image, mock_image) - mock_subprocess.assert_called_once_with( - ["screencapture", "-x", str(desktop.screenshot_path)], check=True, capture_output=True - ) - - @patch("subprocess.run") - def test_capture_screen_subprocess_error(self, mock_subprocess): - """Test screen capture with subprocess error.""" - desktop = DesktopControl() - - # Mock subprocess error - mock_subprocess.side_effect = subprocess.CalledProcessError(1, "screencapture") - - success, image = desktop.capture_screen() - - assert success is False - assert image is None - - @patch("subprocess.run") - @patch("cv2.imread") - def test_capture_screen_file_not_exists(self, mock_imread, mock_subprocess): - """Test screen capture when file doesn't exist.""" - desktop = DesktopControl() - - # Mock successful subprocess call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - # Mock file not existing - with patch("pathlib.Path.exists", return_value=False): - success, image = desktop.capture_screen() - - assert success is False - assert image is None - - @patch("subprocess.run") - def test_capture_screen_nonzero_returncode(self, mock_subprocess): - """Test screen capture with non-zero return code.""" - desktop = DesktopControl() - - # Mock subprocess with non-zero return code - mock_result = Mock() - mock_result.returncode = 1 - mock_result.stderr.decode.return_value = "Permission denied" - mock_subprocess.return_value = mock_result - - success, image = desktop.capture_screen() - - assert success is False - assert image is None - - @patch("subprocess.run") - @patch("cv2.imread") - def test_capture_window_interactive(self, mock_imread, mock_subprocess): - """Test interactive window capture.""" - desktop = DesktopControl() - - # Mock successful subprocess call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - # Mock OpenCV image loading - mock_image = np.zeros((600, 800, 3), dtype=np.uint8) - mock_imread.return_value = mock_image - - with patch("pathlib.Path.exists", return_value=True): - success, image = desktop.capture_window(interactive=True) - - assert success is True - assert np.array_equal(image, mock_image) - mock_subprocess.assert_called_once_with( - ["screencapture", "-w", "-x", str(desktop.screenshot_path)], check=True - ) - - @patch.object(DesktopControl, "capture_screen") - def test_capture_window_non_interactive(self, mock_capture_screen): - """Test non-interactive window capture falls back to screen capture.""" - desktop = DesktopControl() - mock_capture_screen.return_value = (True, np.zeros((100, 100, 3), dtype=np.uint8)) - - success, image = desktop.capture_window(interactive=False) - - assert success is True - mock_capture_screen.assert_called_once() - - @patch("subprocess.run") - def test_click_left_button_success(self, mock_subprocess): - """Test successful left click.""" - desktop = DesktopControl() - - # Mock successful subprocess call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = desktop.click(100, 200, "left") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Clicked left at (100, 200)" in result.message - mock_subprocess.assert_called_once_with( - ["osascript", "-e", 'tell application "System Events" to click at {100, 200}'], - check=True, - capture_output=True, - text=True, - ) - - @patch("subprocess.run") - def test_click_right_button(self, mock_subprocess): - """Test right click with control modifier.""" - desktop = DesktopControl() - - # Mock successful subprocess call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = desktop.click(50, 75, "right") - - assert result.success is True - assert "Clicked right at (50, 75)" in result.message - # Should use control-click for right click - expected_script = ( - 'tell application "System Events" to tell (click at {50, 75}) to key down control' - ) - mock_subprocess.assert_called_once_with( - ["osascript", "-e", expected_script], check=True, capture_output=True, text=True - ) - - @patch("subprocess.run") - def test_click_subprocess_error(self, mock_subprocess): - """Test click with subprocess error.""" - desktop = DesktopControl() - - mock_subprocess.side_effect = subprocess.CalledProcessError(1, "osascript") - - result = desktop.click(100, 200) - - assert result.success is False - assert "Desktop click error" in result.message - - @patch("shutil.which") - @patch("subprocess.run") - def test_click_with_cliclick_available(self, mock_subprocess, mock_which): - """Test click with cliclick when available.""" - desktop = DesktopControl() - - # Mock cliclick being available - mock_which.return_value = "/opt/homebrew/bin/cliclick" - - # Mock successful subprocess call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = desktop.click_with_cliclick(100, 200, "left") - - assert result.success is True - assert "Clicked left at (100, 200) via cliclick" in result.message - mock_subprocess.assert_called_once_with( - ["cliclick", "c:100,200"], check=True, capture_output=True, text=True - ) - - @patch("shutil.which") - @patch.object(DesktopControl, "click") - def test_click_with_cliclick_fallback(self, mock_click, mock_which): - """Test cliclick fallback to AppleScript when not available.""" - desktop = DesktopControl() - - # Mock cliclick not being available - mock_which.return_value = None - mock_click.return_value = ActionResult(True, "Clicked via AppleScript") - - result = desktop.click_with_cliclick(100, 200) - - assert result.success is True - mock_click.assert_called_once_with(100, 200, "left") - - @patch.object(DesktopControl, "click") - @patch("time.sleep") - def test_double_click_success(self, mock_sleep, mock_click): - """Test successful double click.""" - desktop = DesktopControl() - - # Mock successful single clicks - mock_click.return_value = ActionResult(True, "Clicked") - - result = desktop.double_click(150, 250) - - assert result.success is True - assert "Double-clicked at (150, 250)" in result.message - assert mock_click.call_count == 2 - mock_click.assert_has_calls([call(150, 250, "left"), call(150, 250, "left")]) - mock_sleep.assert_called_once_with(0.1) - - @patch.object(DesktopControl, "click") - def test_double_click_first_fail(self, mock_click): - """Test double click when first click fails.""" - desktop = DesktopControl() - - # Mock first click failure - mock_click.return_value = ActionResult(False, "Click failed") - - result = desktop.double_click(150, 250) - - assert result.success is False - assert "Click failed" in result.message - assert mock_click.call_count == 1 - - @patch("subprocess.run") - def test_type_text_success(self, mock_subprocess): - """Test successful text typing.""" - desktop = DesktopControl() - - # Mock successful subprocess call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = desktop.type_text("Hello World") - - assert result.success is True - assert "Typed: Hello World" in result.message - expected_script = 'tell application "System Events" to keystroke "Hello World"' - mock_subprocess.assert_called_once_with( - ["osascript", "-e", expected_script], check=True, capture_output=True, text=True - ) - - @patch("subprocess.run") - def test_type_text_with_quotes(self, mock_subprocess): - """Test typing text with quotes (should be escaped).""" - desktop = DesktopControl() - - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = desktop.type_text("Hello \"World\" and 'test'") - - assert result.success is True - # Should escape both double and single quotes - expected_script = ( - r'tell application "System Events" to keystroke "Hello \"World\" and \'test\'"' - ) - mock_subprocess.assert_called_once_with( - ["osascript", "-e", expected_script], check=True, capture_output=True, text=True - ) - - @patch("subprocess.run") - def test_key_press_simple_key(self, mock_subprocess): - """Test pressing simple key.""" - desktop = DesktopControl() - - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = desktop.key_press("enter") - - assert result.success is True - assert "Pressed key: enter" in result.message - expected_script = 'tell application "System Events" to keystroke "return"' - mock_subprocess.assert_called_once_with( - ["osascript", "-e", expected_script], check=True, capture_output=True, text=True - ) - - @patch("subprocess.run") - def test_key_press_combination(self, mock_subprocess): - """Test pressing key combination.""" - desktop = DesktopControl() - - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = desktop.key_press("cmd+c") - - assert result.success is True - expected_script = 'tell application "System Events" to keystroke "c" using command down' - mock_subprocess.assert_called_once_with( - ["osascript", "-e", expected_script], check=True, capture_output=True, text=True - ) - - @patch.object(DesktopControl, "click") - @patch.object(DesktopControl, "key_press") - @patch("time.sleep") - def test_scroll_up_success(self, mock_sleep, mock_key_press, mock_click): - """Test successful upward scrolling.""" - desktop = DesktopControl() - - mock_click.return_value = ActionResult(True, "Clicked") - mock_key_press.return_value = ActionResult(True, "Key pressed") - - result = desktop.scroll(100, 200, "up", 3) - - assert result.success is True - assert "Scrolled up 3 times at (100, 200)" in result.message - mock_click.assert_called_once_with(100, 200) - assert mock_key_press.call_count == 3 - mock_key_press.assert_has_calls([call("up")] * 3) - assert mock_sleep.call_count == 3 - - @patch("subprocess.run") - def test_get_desktop_info(self, mock_subprocess): - """Test getting desktop information.""" - desktop = DesktopControl() - - # Mock successful system_profiler call - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - with patch("shutil.which", return_value="/usr/bin/cliclick"): - info = desktop.get_desktop_info() - - assert info["platform"] == "macOS" - assert info["type"] == "local_desktop" - assert info["screenshot_capability"] is True - assert info["click_capability"] is True - assert info["keyboard_capability"] is True - assert info["cliclick_available"] is True - assert info["system_profiler_available"] is True - - def test_cleanup(self): - """Test cleanup method.""" - desktop = DesktopControl() - - with patch("pathlib.Path.exists", return_value=True): - with patch("pathlib.Path.unlink") as mock_unlink: - desktop.cleanup() - mock_unlink.assert_called_once() - - def test_cleanup_file_not_exists(self): - """Test cleanup when file doesn't exist.""" - desktop = DesktopControl() - - with patch("pathlib.Path.exists", return_value=False): - # Should not raise exception - desktop.cleanup() - - def test_cleanup_exception(self): - """Test cleanup handles exceptions gracefully.""" - desktop = DesktopControl() - - with patch("pathlib.Path.exists", return_value=True): - with patch("pathlib.Path.unlink", side_effect=OSError("Permission denied")): - # Should not raise exception - desktop.cleanup() - - -class TestDesktopControlIntegration: - """Integration-style tests for DesktopControl with multiple components.""" - - @patch("subprocess.run") - @patch("cv2.imread") - @patch("time.sleep") - def test_screenshot_and_click_workflow(self, mock_sleep, mock_imread, mock_subprocess): - """Test combined screenshot and click workflow.""" - desktop = DesktopControl() - - # Mock screenshot capture - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - mock_image = np.zeros((1080, 1920, 3), dtype=np.uint8) - mock_imread.return_value = mock_image - - # Test workflow - with patch("pathlib.Path.exists", return_value=True): - # Capture screenshot - success, image = desktop.capture_screen() - assert success is True - - # Click on screen - click_result = desktop.click(960, 540) # Center of screen - assert click_result.success is True - - # Type some text - type_result = desktop.type_text("test input") - assert type_result.success is True - - @patch("subprocess.run") - def test_key_combinations(self, mock_subprocess): - """Test various key combinations.""" - desktop = DesktopControl() - - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - test_keys = [ - ("cmd+c", 'tell application "System Events" to keystroke "c" using command down'), - ("ctrl+v", 'tell application "System Events" to keystroke "v" using control down'), - ("tab", 'tell application "System Events" to keystroke "tab"'), - ("escape", 'tell application "System Events" to keystroke "escape"'), - ] - - for key, expected_script in test_keys: - result = desktop.key_press(key) - assert result.success is True - - # Verify all calls were made - assert mock_subprocess.call_count == len(test_keys) diff --git a/tests/unit/automation/local/test_form_interface.py b/tests/unit/automation/local/test_form_interface.py deleted file mode 100644 index 2e40be0..0000000 --- a/tests/unit/automation/local/test_form_interface.py +++ /dev/null @@ -1,507 +0,0 @@ -"""Unit tests for automation.local.form_interface module.""" - -import sys -from dataclasses import dataclass -from pathlib import Path -from unittest.mock import Mock, patch - -import numpy as np - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src")) - -from automation.core.types import ActionResult -from automation.local.form_interface import FormFiller - - -@dataclass -class MockOCRElement: - """Mock OCR element for testing.""" - - text: str - center: tuple[int, int] - bbox: tuple[int, int, int, int] - confidence: float - description: str = "" - - -class TestFormFiller: - """Test cases for FormFiller class.""" - - def test_init(self): - """Test FormFiller initialization.""" - form_filler = FormFiller() - - assert form_filler.desktop is not None - assert form_filler.last_screenshot is None - assert form_filler.screenshot_cache_time == 0 - - @patch("time.time") - def test_get_current_screenshot_cache_hit(self, mock_time): - """Test screenshot caching when cache is fresh.""" - form_filler = FormFiller() - - # Set up cache - mock_screenshot = np.zeros((100, 100, 3), dtype=np.uint8) - form_filler.last_screenshot = mock_screenshot - form_filler.screenshot_cache_time = 1000.0 - - # Mock current time to be within cache window - mock_time.return_value = 1000.5 # 0.5 seconds later, within 1.0s max_age - - # This should return cached screenshot without calling desktop.capture_screen - result = form_filler._get_current_screenshot(max_age=1.0) - - assert np.array_equal(result, mock_screenshot) - - @patch("time.time") - def test_get_current_screenshot_cache_miss(self, mock_time): - """Test screenshot capture when cache is stale.""" - form_filler = FormFiller() - - # Set up stale cache - old_screenshot = np.zeros((50, 50, 3), dtype=np.uint8) - form_filler.last_screenshot = old_screenshot - form_filler.screenshot_cache_time = 1000.0 - - # Mock current time to be outside cache window - mock_time.return_value = 1002.0 # 2 seconds later, outside 1.0s max_age - - # Mock desktop capture - new_screenshot = np.zeros((100, 100, 3), dtype=np.uint8) - form_filler.desktop.capture_screen = Mock(return_value=(True, new_screenshot)) - - result = form_filler._get_current_screenshot(max_age=1.0) - - assert np.array_equal(result, new_screenshot) - assert np.array_equal(form_filler.last_screenshot, new_screenshot) - assert form_filler.screenshot_cache_time == 1002.0 - - def test_get_current_screenshot_capture_fails(self): - """Test screenshot capture failure.""" - form_filler = FormFiller() - - # Mock desktop capture failure - form_filler.desktop.capture_screen = Mock(return_value=(False, None)) - - result = form_filler._get_current_screenshot() - - assert result is None - - def test_find_field_by_label_success(self): - """Test successful field finding by label.""" - form_filler = FormFiller() - - # Mock screenshot - mock_screenshot = np.zeros((100, 100, 3), dtype=np.uint8) - form_filler._get_current_screenshot = Mock(return_value=mock_screenshot) - - # Mock OCR results - mock_element = MockOCRElement( - text="Username", - center=(100, 50), - bbox=(50, 40, 150, 60), - confidence=0.95, - description="text field label", - ) - - with patch("automation.local.form_interface.find_elements_by_text", return_value=[mock_element]): - result = form_filler.find_field_by_label("Username") - - assert result is not None - assert result["text"] == "Username" - assert result["center"] == (100, 50) - assert result["bbox"] == (50, 40, 150, 60) - assert result["confidence"] == 0.95 - assert result["description"] == "text field label" - - def test_find_field_by_label_not_found(self): - """Test field not found by label.""" - form_filler = FormFiller() - - # Mock screenshot - mock_screenshot = np.zeros((100, 100, 3), dtype=np.uint8) - form_filler._get_current_screenshot = Mock(return_value=mock_screenshot) - - # Mock empty OCR results - with patch("automation.local.form_interface.find_elements_by_text", return_value=[]): - result = form_filler.find_field_by_label("NonexistentField") - - assert result is None - - def test_find_field_by_label_multiple_matches(self): - """Test field finding with multiple matches (should return best).""" - form_filler = FormFiller() - - # Mock screenshot - mock_screenshot = np.zeros((100, 100, 3), dtype=np.uint8) - form_filler._get_current_screenshot = Mock(return_value=mock_screenshot) - - # Mock multiple OCR results with different confidence - mock_elements = [ - MockOCRElement("Username", (100, 50), (50, 40, 150, 60), 0.85), - MockOCRElement("Username", (200, 50), (150, 40, 250, 60), 0.95), # Best match - MockOCRElement("Username", (300, 50), (250, 40, 350, 60), 0.75), - ] - - with patch("automation.local.form_interface.find_elements_by_text", return_value=mock_elements): - result = form_filler.find_field_by_label("Username") - - assert result is not None - assert result["center"] == (200, 50) # Should return the highest confidence match - assert result["confidence"] == 0.95 - - def test_find_input_field_near(self): - """Test finding input field near a label.""" - form_filler = FormFiller() - - # Mock screenshot - mock_screenshot = np.zeros((200, 200, 3), dtype=np.uint8) - form_filler._get_current_screenshot = Mock(return_value=mock_screenshot) - - # Mock input field element - mock_element = MockOCRElement( - text="textfield", - center=(150, 50), # 50 pixels right of label - bbox=(130, 40, 170, 60), - confidence=0.8, - ) - - with patch("automation.local.form_interface.find_elements_by_text", return_value=[mock_element]): - result = form_filler.find_input_field_near((100, 50), search_radius=100) - - assert result is not None - assert result["center"] == (150, 50) - assert result["distance_from_label"] == 50.0 - - def test_find_input_field_near_outside_radius(self): - """Test finding input field outside search radius.""" - form_filler = FormFiller() - - # Mock screenshot - mock_screenshot = np.zeros((300, 300, 3), dtype=np.uint8) - form_filler._get_current_screenshot = Mock(return_value=mock_screenshot) - - # Mock input field element far from label - mock_element = MockOCRElement( - text="textfield", - center=(250, 50), # 150 pixels away - bbox=(230, 40, 270, 60), - confidence=0.8, - ) - with patch( - "automation.local.form_interface.find_elements_by_text", return_value=[mock_element] - ): - result = form_filler.find_input_field_near((100, 50), search_radius=100) - - assert result is None # Should be None because it's outside the radius - - def test_fill_field_success_with_input_field(self): - """Test successful field filling with detected input field.""" - form_filler = FormFiller() - - # Mock find_field_by_label - form_filler.find_field_by_label = Mock( - return_value={"text": "Username", "center": (100, 50), "confidence": 0.95} - ) - - # Mock find_input_field_near - form_filler.find_input_field_near = Mock( - return_value={"center": (150, 50), "confidence": 0.8} - ) - - # Mock desktop actions - form_filler.desktop.click = Mock(return_value=ActionResult(True, "Clicked")) - form_filler.desktop.key_press = Mock(return_value=ActionResult(True, "Key pressed")) - form_filler.desktop.type_text = Mock(return_value=ActionResult(True, "Text typed")) - - result = form_filler.fill_field("Username", "john.doe") - - assert result.success is True - assert "Successfully filled 'Username' with 'john.doe'" in result.message - - # Verify interactions - form_filler.desktop.click.assert_called_once_with(150, 50) # Click input field - form_filler.desktop.key_press.assert_called_once_with("cmd+a") # Select all - form_filler.desktop.type_text.assert_called_once_with("john.doe") - - def test_fill_field_success_with_offset(self): - """Test successful field filling using label offset (no input field found).""" - form_filler = FormFiller() - - # Mock find_field_by_label - form_filler.find_field_by_label = Mock( - return_value={"text": "Password", "center": (100, 50), "confidence": 0.95} - ) - - # Mock find_input_field_near returns None - form_filler.find_input_field_near = Mock(return_value=None) - - # Mock desktop actions - form_filler.desktop.click = Mock(return_value=ActionResult(True, "Clicked")) - form_filler.desktop.key_press = Mock(return_value=ActionResult(True, "Key pressed")) - form_filler.desktop.type_text = Mock(return_value=ActionResult(True, "Text typed")) - - result = form_filler.fill_field("Password", "secret123", click_offset=(10, 30)) - - assert result.success is True - - # Verify click at label + offset - form_filler.desktop.click.assert_called_once_with(110, 80) # (100+10, 50+30) - - def test_fill_field_label_not_found(self): - """Test field filling when label is not found.""" - form_filler = FormFiller() - - # Mock find_field_by_label returns None - form_filler.find_field_by_label = Mock(return_value=None) - - result = form_filler.fill_field("NonexistentField", "value") - - assert result.success is False - assert "Could not find field with label: NonexistentField" in result.message - - def test_fill_field_click_fails(self): - """Test field filling when click fails.""" - form_filler = FormFiller() - - # Mock find_field_by_label - form_filler.find_field_by_label = Mock( - return_value={"text": "Username", "center": (100, 50), "confidence": 0.95} - ) - - form_filler.find_input_field_near = Mock(return_value=None) - - # Mock click failure - form_filler.desktop.click = Mock(return_value=ActionResult(False, "Click failed")) - - result = form_filler.fill_field("Username", "value") - - assert result.success is False - assert "Failed to click field: Click failed" in result.message - - def test_fill_field_type_fails(self): - """Test field filling when typing fails.""" - form_filler = FormFiller() - - # Mock successful label finding and clicking - form_filler.find_field_by_label = Mock( - return_value={"text": "Username", "center": (100, 50), "confidence": 0.95} - ) - form_filler.find_input_field_near = Mock(return_value=None) - form_filler.desktop.click = Mock(return_value=ActionResult(True, "Clicked")) - form_filler.desktop.key_press = Mock(return_value=ActionResult(True, "Key pressed")) - - # Mock typing failure - form_filler.desktop.type_text = Mock(return_value=ActionResult(False, "Type failed")) - - result = form_filler.fill_field("Username", "value") - - assert result.success is False - assert "Failed to type text: Type failed" in result.message - - def test_click_button_success(self): - """Test successful button clicking.""" - form_filler = FormFiller() - - # Mock find_field_by_label for button - form_filler.find_field_by_label = Mock( - return_value={"text": "Submit", "center": (200, 100), "confidence": 0.95} - ) - - # Mock desktop click - form_filler.desktop.click = Mock(return_value=ActionResult(True, "Clicked")) - - result = form_filler.click_button("Submit") - - assert result.success is True - assert "Successfully clicked 'Submit' button" in result.message - form_filler.desktop.click.assert_called_once_with(200, 100) - - def test_click_button_not_found(self): - """Test button clicking when button is not found.""" - form_filler = FormFiller() - - # Mock find_field_by_label returns None - form_filler.find_field_by_label = Mock(return_value=None) - - result = form_filler.click_button("NonexistentButton") - - assert result.success is False - assert "Could not find button: NonexistentButton" in result.message - - def test_fill_form_multiple_fields(self): - """Test filling multiple form fields.""" - form_filler = FormFiller() - - # Mock individual field filling - def mock_fill_field(label, value): - if label == "Username" or label == "Password": - return ActionResult(True, f"Filled {label}") - else: - return ActionResult(False, f"Failed to fill {label}") - - form_filler.fill_field = Mock(side_effect=mock_fill_field) - - form_data = {"Username": "john.doe", "Password": "secret123", "Email": "john@example.com"} - - results = form_filler.fill_form(form_data) - - assert len(results) == 3 - assert results["Username"].success is True - assert results["Password"].success is True - assert results["Email"].success is False - - def test_get_form_fields_default_labels(self): - """Test getting form fields with default labels.""" - form_filler = FormFiller() - - # Mock find_field_by_label to return different results for different labels - def mock_find_field(label, confidence_threshold=0.5): - if label == "Username": - return {"text": "Username", "center": (100, 50)} - elif label == "Password": - return {"text": "Password", "center": (100, 80)} - return None - - form_filler.find_field_by_label = Mock(side_effect=mock_find_field) - - fields = form_filler.get_form_fields() - - # Should find Username and Password - assert len(fields) == 2 - assert fields[0]["text"] == "Username" - assert fields[1]["text"] == "Password" - - def test_get_form_fields_custom_labels(self): - """Test getting form fields with custom labels.""" - form_filler = FormFiller() - - # Mock find_field_by_label - def mock_find_field(label, confidence_threshold=0.5): - if label == "CustomField": - return {"text": "CustomField", "center": (100, 50)} - return None - - form_filler.find_field_by_label = Mock(side_effect=mock_find_field) - - fields = form_filler.get_form_fields(["CustomField", "AnotherField"]) - - assert len(fields) == 1 - assert fields[0]["text"] == "CustomField" - - @patch("time.time") - @patch("time.sleep") - def test_wait_for_element_found(self, mock_sleep, mock_time): - """Test waiting for element that is found.""" - form_filler = FormFiller() - - # Mock time progression - mock_time.side_effect = [0, 0.5, 1.0] # Start, first check, element found - - # Mock find_field_by_label to return None first, then element - def mock_find_field(text): - if mock_time.call_count > 2: # After second time call - return {"text": "Submit", "center": (100, 50)} - return None - - form_filler.find_field_by_label = Mock(side_effect=mock_find_field) - - result = form_filler.wait_for_element("Submit", timeout=5.0, check_interval=0.5) - - assert result is not None - assert result["text"] == "Submit" - mock_sleep.assert_called_with(0.5) - - @patch("time.time") - @patch("time.sleep") - def test_wait_for_element_timeout(self, mock_sleep, mock_time): - """Test waiting for element that times out.""" - form_filler = FormFiller() - - # Mock time progression to exceed timeout - mock_time.side_effect = [0, 1, 2, 3, 4, 5, 6] # Exceeds 5 second timeout - - # Mock find_field_by_label to always return None - form_filler.find_field_by_label = Mock(return_value=None) - - result = form_filler.wait_for_element("Submit", timeout=5.0, check_interval=1.0) - - assert result is None - - def test_screenshot_cache_invalidation(self): - """Test that screenshot cache is properly invalidated.""" - form_filler = FormFiller() - - # Set initial cache - form_filler.last_screenshot = np.zeros((100, 100, 3), dtype=np.uint8) - form_filler.screenshot_cache_time = 1000.0 - - # fill_form should invalidate cache for each field - form_filler.fill_field = Mock(return_value=ActionResult(True, "Success")) - - form_data = {"Field1": "Value1", "Field2": "Value2"} - form_filler.fill_form(form_data) - - # Cache should have been cleared twice (once per field) - assert form_filler.last_screenshot is None - - -class TestFormFillerErrorHandling: - """Test error handling in FormFiller.""" - - def test_find_field_by_label_exception(self): - """Test exception handling in find_field_by_label.""" - form_filler = FormFiller() - - # Mock screenshot - form_filler._get_current_screenshot = Mock(return_value=np.zeros((100, 100, 3))) - - # Mock find_elements_by_text to raise exception - with patch( - "automation.local.form_interface.find_elements_by_text", - side_effect=Exception("OCR error"), - ): - result = form_filler.find_field_by_label("Username") - - assert result is None - - def test_find_input_field_near_exception(self): - """Test exception handling in find_input_field_near.""" - form_filler = FormFiller() - - # Mock screenshot - form_filler._get_current_screenshot = Mock(return_value=np.zeros((100, 100, 3))) - - # Mock find_elements_by_text to raise exception - with patch( - "automation.local.form_interface.find_elements_by_text", - side_effect=Exception("OCR error"), - ): - result = form_filler.find_input_field_near((100, 50)) - - assert result is None - - def test_fill_field_exception(self): - """Test exception handling in fill_field.""" - form_filler = FormFiller() - - # Mock find_field_by_label to raise exception - form_filler.find_field_by_label = Mock(side_effect=Exception("Test error")) - - result = form_filler.fill_field("Username", "value") - - assert result.success is False - assert "Error filling field 'Username': Test error" in result.message - - def test_click_button_exception(self): - """Test exception handling in click_button.""" - form_filler = FormFiller() - - # Mock find_field_by_label to raise exception - form_filler.find_field_by_label = Mock(side_effect=Exception("Test error")) - - result = form_filler.click_button("Submit") - - assert result.success is False - assert "Error clicking button 'Submit': Test error" in result.message diff --git a/tests/unit/automation/remote/connections/__init__.py b/tests/unit/automation/remote/connections/__init__.py deleted file mode 100644 index 7f13f7e..0000000 --- a/tests/unit/automation/remote/connections/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for automation.remote.connections module.""" diff --git a/tests/unit/automation/remote/connections/test_rdp.py b/tests/unit/automation/remote/connections/test_rdp.py deleted file mode 100644 index b00bd8f..0000000 --- a/tests/unit/automation/remote/connections/test_rdp.py +++ /dev/null @@ -1,625 +0,0 @@ -"""Unit tests for automation.remote.connections.rdp module.""" - -import sys -from pathlib import Path -from unittest.mock import Mock, patch - -import numpy as np - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent / "src")) - -from automation.core.types import ActionResult, ConnectionResult -from automation.remote.connections.rdp import RDPConnection - - -class TestRDPConnection: - """Test cases for RDPConnection class.""" - - def test_init(self): - """Test RDPConnection initialization.""" - rdp = RDPConnection() - - assert rdp.rdp_process is None - assert rdp.xvfb_process is None - assert rdp.display is None - assert rdp.temp_dir is None - assert rdp.screenshot_path is None - assert rdp.is_connected is False - assert rdp.connection_info == {} - - @patch("automation.remote.connections.rdp.shutil.which") - def test_connect_no_freerdp(self, mock_which): - """Test connection when FreeRDP is not available.""" - rdp = RDPConnection() - - mock_which.return_value = None - - result = rdp.connect("test.host") - - assert isinstance(result, ConnectionResult) - assert result.success is False - assert "FreeRDP (xfreerdp) not found" in result.message - - @patch("automation.remote.connections.rdp.tempfile.mkdtemp") - @patch("automation.remote.connections.rdp.subprocess.Popen") - @patch("automation.remote.connections.rdp.shutil.which") - @patch("automation.remote.connections.rdp.os.path.exists") - @patch("automation.remote.connections.rdp.os.makedirs") - @patch("automation.remote.connections.rdp.time.sleep") - def test_connect_success_with_xvfb( - self, mock_sleep, mock_makedirs, mock_exists, mock_which, mock_popen, mock_mkdtemp - ): - """Test successful RDP connection with Xvfb.""" - rdp = RDPConnection() - - # Mock dependencies are available - mock_which.side_effect = lambda cmd: { - "xfreerdp": "/usr/bin/xfreerdp", - "Xvfb": "/usr/bin/Xvfb", - }.get(cmd) - - # Mock X11 directory exists - mock_exists.return_value = True - - # Mock temporary directory - mock_mkdtemp.return_value = "/tmp/rdp_test" - - # Mock processes - mock_xvfb_process = Mock() - mock_xvfb_process.poll.return_value = None # Still running - mock_rdp_process = Mock() - mock_rdp_process.poll.return_value = None # Still running - - mock_popen.side_effect = [mock_xvfb_process, mock_rdp_process] - - # Mock _find_free_display - with patch.object(rdp, "_find_free_display", return_value=10): - result = rdp.connect("test.host", 3389, "user", "pass", domain="DOMAIN") - - assert isinstance(result, ConnectionResult) - assert result.success is True - assert "Connected to RDP server at test.host:3389" in result.message - assert rdp.is_connected is True - assert rdp.display == ":10" - assert rdp.temp_dir == "/tmp/rdp_test" - assert rdp.screenshot_path == "/tmp/rdp_test/screenshot.png" - - # Verify connection info - assert rdp.connection_info["type"] == "rdp" - assert rdp.connection_info["host"] == "test.host" - assert rdp.connection_info["port"] == 3389 - assert rdp.connection_info["username"] == "user" - assert rdp.connection_info["domain"] == "DOMAIN" - assert rdp.connection_info["display"] == ":10" - assert rdp.connection_info["resolution"] == "1920x1080" - - @patch("automation.remote.connections.rdp.shutil.which") - def test_connect_no_xvfb_macos(self, mock_which): - """Test connection failure on macOS without Xvfb.""" - rdp = RDPConnection() - - # FreeRDP available but no Xvfb - mock_which.side_effect = lambda cmd: {"xfreerdp": "/usr/bin/xfreerdp", "Xvfb": None}.get( - cmd - ) - - result = rdp.connect("test.host") - - assert result.success is False - assert "RDP on macOS requires isolated X11 display" in result.message - assert "brew install freerdp imagemagick xorg-server xdotool" in result.message - - @patch("automation.remote.connections.rdp.subprocess.Popen") - @patch("automation.remote.connections.rdp.shutil.which") - @patch("automation.remote.connections.rdp.os.path.exists") - @patch("automation.remote.connections.rdp.time.sleep") - def test_connect_xvfb_fails(self, mock_sleep, mock_exists, mock_which, mock_popen): - """Test connection when Xvfb process fails.""" - rdp = RDPConnection() - - mock_which.side_effect = lambda cmd: { - "xfreerdp": "/usr/bin/xfreerdp", - "Xvfb": "/usr/bin/Xvfb", - }.get(cmd) - - mock_exists.return_value = True - - # Mock Xvfb process that dies immediately - mock_xvfb_process = Mock() - mock_xvfb_process.poll.return_value = 1 # Process died - mock_popen.return_value = mock_xvfb_process - - with patch.object(rdp, "_find_free_display", return_value=10): - result = rdp.connect("test.host") - - assert result.success is False - assert "Xvfb process died immediately after start" in result.message - - @patch("automation.remote.connections.rdp.tempfile.mkdtemp") - @patch("automation.remote.connections.rdp.subprocess.Popen") - @patch("automation.remote.connections.rdp.shutil.which") - @patch("automation.remote.connections.rdp.os.path.exists") - @patch("automation.remote.connections.rdp.time.sleep") - def test_connect_freerdp_fails( - self, mock_sleep, mock_exists, mock_which, mock_popen, mock_mkdtemp - ): - """Test connection when FreeRDP process fails.""" - rdp = RDPConnection() - - mock_which.side_effect = lambda cmd: { - "xfreerdp": "/usr/bin/xfreerdp", - "Xvfb": "/usr/bin/Xvfb", - }.get(cmd) - - mock_exists.return_value = True - mock_mkdtemp.return_value = "/tmp/rdp_test" - - # Mock Xvfb succeeds, FreeRDP fails - mock_xvfb_process = Mock() - mock_xvfb_process.poll.return_value = None - - mock_rdp_process = Mock() - mock_rdp_process.poll.return_value = 1 # Process died - mock_rdp_process.communicate.return_value = (b"", b"Authentication failed") - - mock_popen.side_effect = [mock_xvfb_process, mock_rdp_process] - - with patch.object(rdp, "_find_free_display", return_value=10): - result = rdp.connect("test.host") - - assert result.success is False - assert "FreeRDP failed: Authentication failed" in result.message - - @patch("automation.remote.connections.rdp.shutil.rmtree") - @patch("automation.remote.connections.rdp.os.path.exists") - @patch("automation.remote.connections.rdp.os.unlink") - @patch("automation.remote.connections.rdp.subprocess.run") - def test_disconnect_success(self, mock_subprocess_run, mock_unlink, mock_exists, mock_rmtree): - """Test successful disconnection.""" - rdp = RDPConnection() - - # Set up connected state - mock_rdp_process = Mock() - mock_xvfb_process = Mock() - rdp.rdp_process = mock_rdp_process - rdp.xvfb_process = mock_xvfb_process - rdp.display = ":10" - rdp.temp_dir = "/tmp/rdp_test" - rdp.is_connected = True - rdp.connection_info = {"test": "data"} - - # Mock file operations - mock_exists.return_value = True - - result = rdp.disconnect() - - assert isinstance(result, ConnectionResult) - assert result.success is True - assert result.message == "RDP disconnected" - assert rdp.is_connected is False - assert rdp.connection_info == {} - assert rdp.rdp_process is None - assert rdp.xvfb_process is None - assert rdp.display is None - assert rdp.temp_dir is None - - # Verify cleanup calls - mock_rdp_process.terminate.assert_called_once() - mock_xvfb_process.terminate.assert_called_once() - mock_unlink.assert_called_once_with("/tmp/.X10-lock") - mock_rmtree.assert_called_once_with("/tmp/rdp_test") - - def test_disconnect_no_processes(self): - """Test disconnection when no processes exist.""" - rdp = RDPConnection() - - result = rdp.disconnect() - - assert result.success is True - assert result.message == "RDP disconnected" - - def test_disconnect_exception(self): - """Test disconnection with exception.""" - rdp = RDPConnection() - - mock_rdp_process = Mock() - mock_rdp_process.terminate.side_effect = Exception("Terminate failed") - rdp.rdp_process = mock_rdp_process - - result = rdp.disconnect() - - assert result.success is False - assert "RDP disconnect error: Terminate failed" in result.message - - @patch("automation.remote.connections.rdp.cv2.imread") - @patch("automation.remote.connections.rdp.os.path.exists") - @patch("automation.remote.connections.rdp.os.unlink") - @patch("automation.remote.connections.rdp.subprocess.run") - @patch("automation.remote.connections.rdp.shutil.which") - def test_capture_screen_scrot_success( - self, mock_which, mock_subprocess, mock_unlink, mock_exists, mock_imread - ): - """Test successful screen capture using scrot.""" - rdp = RDPConnection() - - # Set up connected state - rdp.is_connected = True - rdp.display = ":10" - rdp.screenshot_path = "/tmp/test_screenshot.png" - - # Mock scrot available and working - mock_which.return_value = "/usr/bin/scrot" - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - # Mock image loading - mock_exists.return_value = True - mock_image = np.zeros((100, 100, 3), dtype=np.uint8) - mock_imread.return_value = mock_image - - success, image = rdp.capture_screen() - - assert success is True - assert np.array_equal(image, mock_image) - # Verify subprocess was called with scrot command and DISPLAY env var - mock_subprocess.assert_called_once() - call_args = mock_subprocess.call_args - assert call_args[0] == (["scrot", rdp.screenshot_path],) - assert call_args[1]["capture_output"] is True - assert call_args[1]["env"]["DISPLAY"] == ":10" - mock_imread.assert_called_once_with(rdp.screenshot_path) - mock_unlink.assert_called_once_with(rdp.screenshot_path) - - def test_capture_screen_not_connected(self): - """Test screen capture when not connected.""" - rdp = RDPConnection() - - success, image = rdp.capture_screen() - - assert success is False - assert image is None - - def test_capture_screen_no_display(self): - """Test screen capture when no display is set.""" - rdp = RDPConnection() - rdp.is_connected = True # Connected but no display - - success, image = rdp.capture_screen() - - assert success is False - assert image is None - - @patch("automation.remote.connections.rdp.subprocess.run") - def test_click_success(self, mock_subprocess): - """Test successful click.""" - rdp = RDPConnection() - - # Set up connected state - rdp.is_connected = True - rdp.display = ":10" - - # Mock successful subprocess - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = rdp.click(100, 200, "left") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Clicked left at (100, 200)" in result.message - - expected_cmd = ["xdotool", "mousemove", "100", "200", "click", "1"] - # Verify subprocess was called with xdotool command and DISPLAY env var - mock_subprocess.assert_called_once() - call_args = mock_subprocess.call_args - assert call_args[0] == (expected_cmd,) - assert call_args[1]["capture_output"] is True - assert call_args[1]["env"]["DISPLAY"] == ":10" - - def test_click_button_mapping(self): - """Test click button mapping.""" - rdp = RDPConnection() - rdp.is_connected = True - rdp.display = ":10" - - test_cases = [ - ("left", "1"), - ("middle", "2"), - ("right", "3"), - ("unknown", "1"), # Should default to left - ] - - with patch("automation.remote.connections.rdp.subprocess.run") as mock_subprocess: - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - for button, expected_num in test_cases: - result = rdp.click(50, 75, button) - assert result.success is True - - # Check the button number in the call - call_args = mock_subprocess.call_args[0][0] - assert call_args[-1] == expected_num # Last argument should be button number - - def test_click_not_connected(self): - """Test click when not connected.""" - rdp = RDPConnection() - - result = rdp.click(100, 200) - - assert result.success is False - assert result.message == "No RDP connection" - - @patch("automation.remote.connections.rdp.subprocess.run") - def test_click_failure(self, mock_subprocess): - """Test click failure.""" - rdp = RDPConnection() - rdp.is_connected = True - rdp.display = ":10" - - # Mock failed subprocess - mock_result = Mock() - mock_result.returncode = 1 - mock_result.stderr.decode.return_value = "xdotool error" - mock_subprocess.return_value = mock_result - - result = rdp.click(100, 200) - - assert result.success is False - assert "Click failed: xdotool error" in result.message - - @patch("automation.remote.connections.rdp.subprocess.run") - def test_type_text_success(self, mock_subprocess): - """Test successful text typing.""" - rdp = RDPConnection() - - rdp.is_connected = True - rdp.display = ":10" - - # Mock successful subprocess - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = rdp.type_text("Hello World") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Typed: Hello World" in result.message - - expected_cmd = ["xdotool", "type", "--delay", "50", "Hello World"] - # Verify subprocess was called with xdotool command and DISPLAY env var - mock_subprocess.assert_called_once() - call_args = mock_subprocess.call_args - assert call_args[0] == (expected_cmd,) - assert call_args[1]["capture_output"] is True - assert call_args[1]["env"]["DISPLAY"] == ":10" - - def test_type_text_not_connected(self): - """Test text typing when not connected.""" - rdp = RDPConnection() - - result = rdp.type_text("Hello") - - assert result.success is False - assert result.message == "No RDP connection" - - @patch("automation.remote.connections.rdp.subprocess.run") - def test_key_press_success(self, mock_subprocess): - """Test successful key press.""" - rdp = RDPConnection() - - rdp.is_connected = True - rdp.display = ":10" - - # Mock successful subprocess - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - result = rdp.key_press("enter") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Pressed key: enter" in result.message - - expected_cmd = ["xdotool", "key", "Return"] - # Verify subprocess was called with xdotool command and DISPLAY env var - mock_subprocess.assert_called_once() - call_args = mock_subprocess.call_args - assert call_args[0] == (expected_cmd,) - assert call_args[1]["capture_output"] is True - assert call_args[1]["env"]["DISPLAY"] == ":10" - - def test_key_press_mapping(self): - """Test key press mapping.""" - rdp = RDPConnection() - rdp.is_connected = True - rdp.display = ":10" - - test_cases = [ - ("enter", "Return"), - ("escape", "Escape"), - ("tab", "Tab"), - ("space", "space"), - ("backspace", "BackSpace"), - ("delete", "Delete"), - ("ctrl", "ctrl"), - ("alt", "alt"), - ("shift", "shift"), - ("up", "Up"), - ("down", "Down"), - ("left", "Left"), - ("right", "Right"), - ("F1", "F1"), # Unmapped key should pass through - ] - - with patch("automation.remote.connections.rdp.subprocess.run") as mock_subprocess: - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - for input_key, expected_xdo_key in test_cases: - result = rdp.key_press(input_key) - assert result.success is True - - # Check the key in the call - call_args = mock_subprocess.call_args[0][0] - assert call_args[-1] == expected_xdo_key - - @patch("automation.remote.connections.rdp.os.path.exists") - def test_find_free_display(self, mock_exists): - """Test finding free display number.""" - rdp = RDPConnection() - - # Mock first few displays as taken, 13 as free - def mock_exists_side_effect(path): - if path in ["/tmp/.X10-lock", "/tmp/.X11-lock", "/tmp/.X12-lock"]: - return True # Taken - return False # Free - - mock_exists.side_effect = mock_exists_side_effect - - free_display = rdp._find_free_display() - - assert free_display == 13 - - def test_find_free_display_fallback(self): - """Test find_free_display fallback when all displays taken.""" - rdp = RDPConnection() - - with patch("automation.remote.connections.rdp.os.path.exists", return_value=True): - free_display = rdp._find_free_display() - assert free_display == 99 # Fallback value - - @patch("automation.remote.connections.rdp.os.unlink") - @patch("automation.remote.connections.rdp.os.path.exists") - @patch("automation.remote.connections.rdp.subprocess.run") - @patch("automation.remote.connections.rdp.shutil.which") - def test_capture_with_xwd_convert_success( - self, mock_which, mock_subprocess, mock_exists, mock_unlink - ): - """Test successful xwd+convert capture method.""" - rdp = RDPConnection() - rdp.screenshot_path = "/tmp/test.png" - - # Mock tools available - mock_which.side_effect = lambda cmd: { - "xwd": "/usr/bin/xwd", - "magick": "/usr/bin/magick", - }.get(cmd) - - # Mock successful subprocess calls - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess.return_value = mock_result - - # Mock file exists after conversion - mock_exists.return_value = True - - env = {"DISPLAY": ":10"} - success = rdp._capture_with_xwd_convert(env) - - assert success is True - - # Should call xwd first, then magick - assert mock_subprocess.call_count == 2 - - # First call should be xwd - first_call = mock_subprocess.call_args_list[0] - assert first_call[0][0][:3] == ["xwd", "-root", "-out"] - - # Second call should be magick - second_call = mock_subprocess.call_args_list[1] - assert second_call[0][0][0] == "magick" - - @patch("automation.remote.connections.rdp.shutil.which") - def test_capture_with_xwd_convert_no_tools(self, mock_which): - """Test xwd+convert method when tools unavailable.""" - rdp = RDPConnection() - rdp.screenshot_path = "/tmp/test.png" - - # Mock no tools available - mock_which.return_value = None - - env = {"DISPLAY": ":10"} - success = rdp._capture_with_xwd_convert(env) - - assert success is False - - -class TestRDPConnectionIntegration: - """Integration-style tests for RDPConnection.""" - - @patch("automation.remote.connections.rdp.tempfile.mkdtemp") - @patch("automation.remote.connections.rdp.subprocess.Popen") - @patch("automation.remote.connections.rdp.subprocess.run") - @patch("automation.remote.connections.rdp.shutil.which") - @patch("automation.remote.connections.rdp.os.path.exists") - @patch("automation.remote.connections.rdp.time.sleep") - def test_full_connection_workflow( - self, mock_sleep, mock_exists, mock_which, mock_subprocess_run, mock_popen, mock_mkdtemp - ): - """Test complete RDP workflow.""" - rdp = RDPConnection() - - # Setup mocks for successful connection - mock_which.side_effect = lambda cmd: { - "xfreerdp": "/usr/bin/xfreerdp", - "Xvfb": "/usr/bin/Xvfb", - }.get(cmd) - - mock_exists.return_value = True - mock_mkdtemp.return_value = "/tmp/rdp_test" - - # Mock processes - mock_xvfb_process = Mock() - mock_xvfb_process.poll.return_value = None - mock_rdp_process = Mock() - mock_rdp_process.poll.return_value = None - mock_popen.side_effect = [mock_xvfb_process, mock_rdp_process] - - # Mock operations - mock_result = Mock() - mock_result.returncode = 0 - mock_subprocess_run.return_value = mock_result - - with patch.object(rdp, "_find_free_display", return_value=10): - # Connect - connect_result = rdp.connect("test.host") - assert connect_result.success is True - assert rdp.is_connected is True - - # Use connection - click_result = rdp.click(100, 100) - assert click_result.success is True - - type_result = rdp.type_text("test") - assert type_result.success is True - - key_result = rdp.key_press("enter") - assert key_result.success is True - - # Disconnect - with patch("automation.remote.connections.rdp.shutil.rmtree"): - with patch("automation.remote.connections.rdp.os.unlink"): - disconnect_result = rdp.disconnect() - assert disconnect_result.success is True - assert rdp.is_connected is False - - def test_operations_without_connection(self): - """Test that operations fail gracefully without connection.""" - rdp = RDPConnection() - - # All operations should fail when not connected - assert rdp.capture_screen()[0] is False - assert rdp.click(100, 100).success is False - assert rdp.type_text("test").success is False - assert rdp.key_press("enter").success is False - - # Connection info should be empty - assert rdp.get_connection_info() == {} diff --git a/tests/unit/automation/remote/connections/test_vnc.py b/tests/unit/automation/remote/connections/test_vnc.py deleted file mode 100644 index 47c5492..0000000 --- a/tests/unit/automation/remote/connections/test_vnc.py +++ /dev/null @@ -1,520 +0,0 @@ -"""Unit tests for automation.remote.connections.vnc module.""" - -import sys -from pathlib import Path -from unittest.mock import Mock, patch - -import cv2 -import numpy as np - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent / "src")) - -from automation.core.types import ActionResult, ConnectionResult -from automation.remote.connections.vnc import VNCConnection - - -class TestVNCConnection: - """Test cases for VNCConnection class.""" - - def test_init(self): - """Test VNCConnection initialization.""" - vnc = VNCConnection() - - assert vnc.vnc_client is None - assert vnc.is_connected is False - assert vnc.connection_info == {} - - @patch("automation.remote.connections.vnc.vnc.connect") - def test_connect_success(self, mock_vnc_connect): - """Test successful VNC connection.""" - vnc = VNCConnection() - - # Mock vncdotool client - mock_client = Mock() - mock_vnc_connect.return_value = mock_client - - result = vnc.connect("192.168.1.100", 5900, password="secret") - - assert isinstance(result, ConnectionResult) - assert result.success is True - assert "Connected to VNC server at 192.168.1.100:5900" in result.message - assert vnc.is_connected is True - assert vnc.vnc_client is mock_client - assert vnc.connection_info["type"] == "vnc" - assert vnc.connection_info["host"] == "192.168.1.100" - assert vnc.connection_info["port"] == 5900 - assert vnc.connection_info["has_password"] is True - - mock_vnc_connect.assert_called_once_with("192.168.1.100", password="secret") - - @patch("automation.remote.connections.vnc.vnc.connect") - def test_connect_default_port(self, mock_vnc_connect): - """Test VNC connection with default port.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_vnc_connect.return_value = mock_client - - result = vnc.connect("test.host") - - assert result.success is True - assert vnc.connection_info["port"] == 5900 - mock_vnc_connect.assert_called_once_with("test.host", password=None) - - @patch("automation.remote.connections.vnc.vnc.connect") - def test_connect_custom_port(self, mock_vnc_connect): - """Test VNC connection with custom port.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_vnc_connect.return_value = mock_client - - result = vnc.connect("test.host", 5901) - - assert result.success is True - assert vnc.connection_info["port"] == 5901 - mock_vnc_connect.assert_called_once_with("test.host::5901", password=None) - - @patch("automation.remote.connections.vnc.vnc.connect") - def test_connect_no_password(self, mock_vnc_connect): - """Test VNC connection without password.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_vnc_connect.return_value = mock_client - - result = vnc.connect("test.host") - - assert result.success is True - assert vnc.connection_info["has_password"] is False - mock_vnc_connect.assert_called_once_with("test.host", password=None) - - @patch("automation.remote.connections.vnc.vnc.connect") - def test_connect_failure(self, mock_vnc_connect): - """Test VNC connection failure.""" - vnc = VNCConnection() - - mock_vnc_connect.side_effect = Exception("Connection refused") - - result = vnc.connect("invalid.host") - - assert isinstance(result, ConnectionResult) - assert result.success is False - assert "VNC connection failed: Connection refused" in result.message - assert vnc.is_connected is False - - def test_disconnect_success(self): - """Test successful VNC disconnection.""" - vnc = VNCConnection() - - # Set up connected state - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - vnc.connection_info = {"type": "vnc", "host": "test"} - - result = vnc.disconnect() - - assert isinstance(result, ConnectionResult) - assert result.success is True - assert result.message == "VNC disconnected" - assert vnc.is_connected is False - assert vnc.vnc_client is None - assert vnc.connection_info == {} - mock_client.disconnect.assert_called_once() - - def test_disconnect_no_client(self): - """Test disconnection when no client exists.""" - vnc = VNCConnection() - - result = vnc.disconnect() - - assert result.success is True - assert result.message == "VNC disconnected" - - def test_disconnect_exception(self): - """Test disconnection with exception.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_client.disconnect.side_effect = Exception("Disconnect error") - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.disconnect() - - assert result.success is False - assert "VNC disconnect error: Disconnect error" in result.message - - @patch("automation.remote.connections.vnc.cv2.cvtColor") - @patch("automation.remote.connections.vnc.np.array") - def test_capture_screen_success(self, mock_np_array, mock_cvtColor): - """Test successful screen capture.""" - vnc = VNCConnection() - - # Set up connected state - mock_client = Mock() - mock_pil_image = Mock() - mock_client.screen = mock_pil_image - vnc.vnc_client = mock_client - vnc.is_connected = True - - # Mock numpy array conversion - mock_rgb_array = np.zeros((100, 100, 3), dtype=np.uint8) - mock_bgr_array = np.zeros((100, 100, 3), dtype=np.uint8) - mock_np_array.return_value = mock_rgb_array - mock_cvtColor.return_value = mock_bgr_array - - success, image = vnc.capture_screen() - - assert success is True - assert np.array_equal(image, mock_bgr_array) - mock_np_array.assert_called_once_with(mock_pil_image) - mock_cvtColor.assert_called_once_with(mock_rgb_array, cv2.COLOR_RGB2BGR) - - def test_capture_screen_not_connected(self): - """Test screen capture when not connected.""" - vnc = VNCConnection() - - success, image = vnc.capture_screen() - - assert success is False - assert image is None - - def test_capture_screen_no_client(self): - """Test screen capture when no client exists.""" - vnc = VNCConnection() - vnc.is_connected = True # Connected but no client - - success, image = vnc.capture_screen() - - assert success is False - assert image is None - - def test_capture_screen_no_image(self): - """Test screen capture when client returns no image.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_client.screen = None - vnc.vnc_client = mock_client - vnc.is_connected = True - - success, image = vnc.capture_screen() - - assert success is False - assert image is None - - @patch("automation.remote.connections.vnc.cv2.cvtColor") - @patch("automation.remote.connections.vnc.np.array") - def test_capture_screen_exception(self, mock_np_array, mock_cvtColor): - """Test screen capture with exception.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_client.screen = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - mock_np_array.side_effect = Exception("Conversion error") - - success, image = vnc.capture_screen() - - assert success is False - assert image is None - - @patch("automation.remote.connections.vnc.time.sleep") - def test_click_left_success(self, mock_sleep): - """Test successful left click.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.click(100, 200, "left") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Clicked left at (100, 200)" in result.message - - mock_client.mousemove.assert_called_once_with(100, 200) - mock_client.mousedown.assert_called_once_with(1) - mock_client.mouseup.assert_called_once_with(1) - mock_sleep.assert_called_once_with(0.05) - - @patch("automation.remote.connections.vnc.time.sleep") - def test_click_right_success(self, mock_sleep): - """Test successful right click.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.click(50, 75, "right") - - assert result.success is True - assert "Clicked right at (50, 75)" in result.message - - mock_client.mousemove.assert_called_once_with(50, 75) - mock_client.mousedown.assert_called_once_with(3) - mock_client.mouseup.assert_called_once_with(3) - - @patch("automation.remote.connections.vnc.time.sleep") - def test_click_middle_success(self, mock_sleep): - """Test successful middle click.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.click(25, 30, "middle") - - assert result.success is True - assert "Clicked middle at (25, 30)" in result.message - - mock_client.mousemove.assert_called_once_with(25, 30) - mock_client.mousedown.assert_called_once_with(2) - mock_client.mouseup.assert_called_once_with(2) - - def test_click_unknown_button(self): - """Test click with unknown button.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.click(10, 20, "unknown") - - assert result.success is False - assert "Unknown button: unknown" in result.message - - def test_click_not_connected(self): - """Test click when not connected.""" - vnc = VNCConnection() - - result = vnc.click(100, 200) - - assert result.success is False - assert result.message == "No VNC connection" - - def test_click_exception(self): - """Test click with exception.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_client.mousemove.side_effect = Exception("Mouse error") - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.click(100, 200) - - assert result.success is False - assert "VNC click error: Mouse error" in result.message - - @patch("automation.remote.connections.vnc.time.sleep") - def test_type_text_success(self, mock_sleep): - """Test successful text typing.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.type_text("Hello") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Typed: Hello" in result.message - - # Should call keypress for each character - assert mock_client.keypress.call_count == 5 - mock_client.keypress.assert_any_call("H") - mock_client.keypress.assert_any_call("e") - mock_client.keypress.assert_any_call("l") - mock_client.keypress.assert_any_call("l") - mock_client.keypress.assert_any_call("o") - - # Should sleep between keystrokes - assert mock_sleep.call_count == 5 - - def test_type_text_not_connected(self): - """Test text typing when not connected.""" - vnc = VNCConnection() - - result = vnc.type_text("Hello") - - assert result.success is False - assert result.message == "No VNC connection" - - def test_type_text_exception(self): - """Test text typing with exception.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_client.keypress.side_effect = Exception("Keypress error") - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.type_text("H") - - assert result.success is False - assert "VNC type error: Keypress error" in result.message - - def test_key_press_success(self): - """Test successful key press.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.key_press("enter") - - assert isinstance(result, ActionResult) - assert result.success is True - assert "Pressed key: enter" in result.message - mock_client.keypress.assert_called_once_with("Return") - - def test_key_press_unmapped_key(self): - """Test key press with unmapped key.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.key_press("F1") - - assert result.success is True - mock_client.keypress.assert_called_once_with("F1") # Should pass through unchanged - - def test_key_press_mapping(self): - """Test key press mapping for common keys.""" - vnc = VNCConnection() - - mock_client = Mock() - vnc.vnc_client = mock_client - vnc.is_connected = True - - test_cases = [ - ("enter", "Return"), - ("escape", "Escape"), - ("tab", "Tab"), - ("space", "space"), - ("backspace", "BackSpace"), - ("delete", "Delete"), - ("ctrl", "Control_L"), - ("alt", "Alt_L"), - ("shift", "Shift_L"), - ("up", "Up"), - ("down", "Down"), - ("left", "Left"), - ("right", "Right"), - ] - - for input_key, expected_vnc_key in test_cases: - mock_client.reset_mock() - result = vnc.key_press(input_key) - assert result.success is True - mock_client.keypress.assert_called_once_with(expected_vnc_key) - - def test_key_press_not_connected(self): - """Test key press when not connected.""" - vnc = VNCConnection() - - result = vnc.key_press("enter") - - assert result.success is False - assert result.message == "No VNC connection" - - def test_key_press_exception(self): - """Test key press with exception.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_client.keypress.side_effect = Exception("Key error") - vnc.vnc_client = mock_client - vnc.is_connected = True - - result = vnc.key_press("enter") - - assert result.success is False - assert "VNC key press error: Key error" in result.message - - -class TestVNCConnectionIntegration: - """Integration-style tests for VNCConnection.""" - - @patch("automation.remote.connections.vnc.vnc.connect") - def test_full_connection_workflow(self, mock_vnc_connect): - """Test complete connection workflow.""" - vnc = VNCConnection() - - # Mock vncdotool client - mock_client = Mock() - mock_client.screen = Mock() - mock_vnc_connect.return_value = mock_client - - # Connect - connect_result = vnc.connect("test.host", 5900, password="secret") - assert connect_result.success is True - assert vnc.is_connected is True - - # Use connection - click_result = vnc.click(100, 100) - assert click_result.success is True - - type_result = vnc.type_text("test") - assert type_result.success is True - - key_result = vnc.key_press("enter") - assert key_result.success is True - - # Disconnect - disconnect_result = vnc.disconnect() - assert disconnect_result.success is True - assert vnc.is_connected is False - - def test_operations_without_connection(self): - """Test that operations fail gracefully without connection.""" - vnc = VNCConnection() - - # All operations should fail when not connected - assert vnc.capture_screen()[0] is False - assert vnc.click(100, 100).success is False - assert vnc.type_text("test").success is False - assert vnc.key_press("enter").success is False - - # Connection info should be empty - assert vnc.get_connection_info() == {} - - @patch("automation.remote.connections.vnc.vnc.connect") - def test_connection_info_accuracy(self, mock_vnc_connect): - """Test that connection info is accurate.""" - vnc = VNCConnection() - - mock_client = Mock() - mock_vnc_connect.return_value = mock_client - - # Test with password - vnc.connect("192.168.1.100", 5901, password="secret123") - - info = vnc.get_connection_info() - assert info["type"] == "vnc" - assert info["host"] == "192.168.1.100" - assert info["port"] == 5901 - assert info["has_password"] is True - - # Test without password - vnc.disconnect() - vnc.connect("test.host") - - info = vnc.get_connection_info() - assert info["host"] == "test.host" - assert info["port"] == 5900 # Default port - assert info["has_password"] is False diff --git a/uv.lock b/uv.lock index 3417667..130f1cb 100644 --- a/uv.lock +++ b/uv.lock @@ -5,7 +5,8 @@ resolution-markers = [ "sys_platform == 'darwin'", "platform_machine == 'aarch64' and sys_platform == 'linux'", "sys_platform == 'win32'", - "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", ] [[package]] @@ -101,6 +102,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/d8/5674f7718f110c1545a982f9c689cc2ad3e5058248a45217df038c244ac5/aistudio_sdk-0.3.6-py3-none-any.whl", hash = "sha256:0aee688da1ea34f06898b29c03cee432dc4a6cca02118d8c2182c5ccf0c1d447", size = 63980 }, ] +[[package]] +name = "albucore" +version = "0.0.24" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "simsimd" }, + { name = "stringzilla" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/69/d4cbcf2a5768bf91cd14ffef783520458431e5d2b22fbc08418d3ba09a88/albucore-0.0.24.tar.gz", hash = "sha256:f2cab5431fadf94abf87fd0c89d9f59046e49fe5de34afea8f89bc8390253746", size = 16981 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/e2/91f145e1f32428e9e1f21f46a7022ffe63d11f549ee55c3b9265ff5207fc/albucore-0.0.24-py3-none-any.whl", hash = "sha256:adef6e434e50e22c2ee127b7a3e71f2e35fa088bcf54431e18970b62d97d0005", size = 15372 }, +] + +[[package]] +name = "albumentations" +version = "2.0.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "albucore" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/f4/85eb56c3217b53bcfc2d12e840a0b18ca60902086321cafa5a730f9c0470/albumentations-2.0.8.tar.gz", hash = "sha256:4da95e658e490de3c34af8fcdffed09e36aa8a4edd06ca9f9e7e3ea0b0b16856", size = 354460 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/64/013409c451a44b61310fb757af4527f3de57fc98a00f40448de28b864290/albumentations-2.0.8-py3-none-any.whl", hash = "sha256:c4c4259aaf04a7386ad85c7fdcb73c6c7146ca3057446b745cc035805acb1017", size = 369423 }, +] + [[package]] name = "alembic" version = "1.16.5" @@ -455,7 +488,7 @@ wheels = [ [[package]] name = "computer-use-agent" version = "0.1.0" -source = { editable = "." } +source = { virtual = "." } dependencies = [ { name = "fastapi" }, { name = "numpy" }, @@ -483,6 +516,24 @@ dev = [ { name = "pytest-cov" }, { name = "ruff" }, ] +eval = [ + { name = "albumentations" }, + { name = "editdistance" }, + { name = "gputil" }, + { name = "jiwer" }, + { name = "matplotlib" }, + { name = "psutil" }, + { name = "pycocotools" }, + { name = "python-levenshtein" }, + { name = "pyyaml" }, + { name = "seaborn" }, + { name = "torch" }, + { name = "torchmetrics" }, + { name = "torchvision" }, + { name = "tqdm" }, + { name = "types-pyyaml" }, + { name = "wandb" }, +] [package.metadata] requires-dist = [ @@ -494,7 +545,7 @@ requires-dist = [ { name = "openai-agents", specifier = ">=0.2.11" }, { name = "opencv-python", specifier = ">=4.11.0.86" }, { name = "paddleocr", extras = ["all"], specifier = ">=3.2.0" }, - { name = "paddlepaddle", specifier = ">=3.1.1" }, + { name = "paddlepaddle", specifier = ">=3.2.0" }, { name = "pillow", specifier = ">=11.3.0" }, { name = "ultralytics", specifier = ">=8.3.193" }, { name = "uvicorn", specifier = ">=0.35.0" }, @@ -512,6 +563,24 @@ dev = [ { name = "pytest-cov", specifier = ">=6.2.1" }, { name = "ruff", specifier = ">=0.12.12" }, ] +eval = [ + { name = "albumentations", specifier = ">=1.3.0" }, + { name = "editdistance", specifier = ">=0.6.0" }, + { name = "gputil", specifier = ">=1.4.0" }, + { name = "jiwer", specifier = ">=2.5.0" }, + { name = "matplotlib", specifier = ">=3.5.0" }, + { name = "psutil", specifier = ">=5.9.0" }, + { name = "pycocotools", specifier = ">=2.0.6" }, + { name = "python-levenshtein", specifier = ">=0.12.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "seaborn", specifier = ">=0.11.0" }, + { name = "torch", specifier = ">=1.12.0" }, + { name = "torchmetrics", specifier = ">=0.11.0" }, + { name = "torchvision", specifier = ">=0.13.0" }, + { name = "tqdm", specifier = ">=4.64.0" }, + { name = "types-pyyaml", specifier = ">=6.0.0" }, + { name = "wandb", specifier = ">=0.15.0" }, +] [[package]] name = "constantly" @@ -709,6 +778,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632 }, ] +[[package]] +name = "editdistance" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz", hash = "sha256:d1cdf80a5d5014b0c9126a69a42ce55a457b457f6986ff69ca98e4fe4d2d8fed", size = 50006 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/4c/7f195588949b4e72436dc7fc902632381f96e586af829685b56daebb38b8/editdistance-0.8.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04af61b3fcdd287a07c15b6ae3b02af01c5e3e9c3aca76b8c1d13bd266b6f57", size = 106723 }, + { url = "https://files.pythonhosted.org/packages/8d/82/31dc1640d830cd7d36865098329f34e4dad3b77f31cfb9404b347e700196/editdistance-0.8.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:18fc8b6eaae01bfd9cf999af726c1e8dcf667d120e81aa7dbd515bea7427f62f", size = 80998 }, + { url = "https://files.pythonhosted.org/packages/ea/2a/6b823e71cef694d6f070a1d82be2842706fa193541aab8856a8f42044cd0/editdistance-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a87839450a5987028738d061ffa5ef6a68bac2ddc68c9147a8aae9806629c7f", size = 79248 }, + { url = "https://files.pythonhosted.org/packages/e1/31/bfb8e590f922089dc3471ed7828a6da2fc9453eba38c332efa9ee8749fd7/editdistance-0.8.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24b5f9c9673c823d91b5973d0af8b39f883f414a55ade2b9d097138acd10f31e", size = 415262 }, + { url = "https://files.pythonhosted.org/packages/a9/c7/57423942b2f847cdbbb46494568d00cd8a45500904ea026f0aad6ca01bc7/editdistance-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c59248eabfad603f0fba47b0c263d5dc728fb01c2b6b50fb6ca187cec547fdb3", size = 418905 }, + { url = "https://files.pythonhosted.org/packages/1b/05/dfa4cdcce063596cbf0d7a32c46cd0f4fa70980311b7da64d35f33ad02a0/editdistance-0.8.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e239d88ff52821cf64023fabd06a1d9a07654f364b64bf1284577fd3a79d0e", size = 412511 }, + { url = "https://files.pythonhosted.org/packages/0e/14/39608ff724a9523f187c4e28926d78bc68f2798f74777ac6757981108345/editdistance-0.8.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2f7f71698f83e8c83839ac0d876a0f4ef996c86c5460aebd26d85568d4afd0db", size = 917293 }, + { url = "https://files.pythonhosted.org/packages/df/92/4a1c61d72da40dedfd0ff950fdc71ae83f478330c58a8bccfd776518bd67/editdistance-0.8.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:04e229d6f4ce0c12abc9f4cd4023a5b5fa9620226e0207b119c3c2778b036250", size = 975580 }, + { url = "https://files.pythonhosted.org/packages/47/3d/9877566e724c8a37f2228a84ec5cbf66dbfd0673515baf68a0fe07caff40/editdistance-0.8.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e16721636da6d6b68a2c09eaced35a94f4a4a704ec09f45756d4fd5e128ed18d", size = 929121 }, + { url = "https://files.pythonhosted.org/packages/d2/f5/8c50757d198b8ca30ddb91e8b8f0247a8dca04ff2ec30755245f0ab1ff0c/editdistance-0.8.1-cp312-cp312-win32.whl", hash = "sha256:87533cf2ebc3777088d991947274cd7e1014b9c861a8aa65257bcdc0ee492526", size = 81039 }, + { url = "https://files.pythonhosted.org/packages/28/f0/65101e51dc7c850e7b7581a5d8fa8721a1d7479a0dca6c08386328e19882/editdistance-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:09f01ed51746d90178af7dd7ea4ebb41497ef19f53c7f327e864421743dffb0a", size = 79853 }, +] + [[package]] name = "einops" version = "0.8.1" @@ -854,6 +942,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326 }, ] +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794 }, +] + +[[package]] +name = "gitpython" +version = "3.1.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168 }, +] + [[package]] name = "google-auth" version = "2.40.3" @@ -899,6 +1011,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530 }, ] +[[package]] +name = "gputil" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz", hash = "sha256:099e52c65e512cdfa8c8763fca67f5a5c2afb63469602d5dcb4d296b3661efb9", size = 5545 } + [[package]] name = "graphql-core" version = "3.2.6" @@ -1163,6 +1281,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/52/7ec47455e26f2d6e5f2ea4951a0652c06e5b995c291f723973ae9e724a65/jiter-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:a7c7d785ae9dda68c2678532a5a1581347e9c15362ae9f6e68f3fdbfb64f2e49", size = 206176 }, ] +[[package]] +name = "jiwer" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rapidfuzz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/1e/963dfc249d5bbe88079b57a22556e981ddd9208e4b6116e48bdbfc01f26b/jiwer-4.0.0.tar.gz", hash = "sha256:ae9c051469102a61ef0927100baeeb4546f78d180c9b0948281d08eaf44c191e", size = 28074 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/c9/172c525330c739a068c01050759a6f855ce16212db10a0359e690a03ac48/jiwer-4.0.0-py3-none-any.whl", hash = "sha256:7efaf0bd336b095d99ddef9dd67e1ee829d75d58aa2a81d9639870b01d6d95ea", size = 23034 }, +] + [[package]] name = "joblib" version = "1.5.2" @@ -1340,7 +1471,7 @@ wheels = [ [[package]] name = "langsmith" -version = "0.4.25" +version = "0.4.27" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -1351,9 +1482,49 @@ dependencies = [ { name = "requests-toolbelt" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0c/84/2f01d51a557d14a9a5590c32de5dde3b466b00a55521450c2c9e77ffa438/langsmith-0.4.25.tar.gz", hash = "sha256:56f0c45810384fba37582ca17fdbcf6ead51934d26d72672e5a810452c0d4ae3", size = 940274 } +sdist = { url = "https://files.pythonhosted.org/packages/62/6f/7d88228b7614fa0204e58b8b8c46e6f564659ee07a525c8aeae77a05598a/langsmith-0.4.27.tar.gz", hash = "sha256:6e8bbc425797202952d4e849431e6276e7985b44536ec0582eb96eaf9129c393", size = 956062 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/85/8a5ca8f6044bd74acd0d364878b459d84ec460cf40aec17ed9cd5716e908/langsmith-0.4.25-py3-none-any.whl", hash = "sha256:adb61784ff58e65f0290ba45770626219fb06a776e69fbcf98aec580478b4686", size = 379416 }, + { url = "https://files.pythonhosted.org/packages/2d/26/99bc52e1c47fb4b995aece85a5313349a5e2559e4143ee2345d8bd1446ff/langsmith-0.4.27-py3-none-any.whl", hash = "sha256:23708e6478d1c74ac0e428bbc92df6704993e34305fb62a0c64d2fefc35bd67f", size = 384752 }, +] + +[[package]] +name = "levenshtein" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rapidfuzz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/b3/b5f8011483ba9083a0bc74c4d58705e9cf465fbe55c948a1b1357d0a2aa8/levenshtein-0.27.1.tar.gz", hash = "sha256:3e18b73564cfc846eec94dd13fab6cb006b5d2e0cc56bad1fd7d5585881302e3", size = 382571 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/73/84a7126b9e6441c2547f1fbfd65f3c15c387d1fc04e0dd1d025a12107771/levenshtein-0.27.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25fb540d8c55d1dc7bdc59b7de518ea5ed9df92eb2077e74bcb9bb6de7b06f69", size = 173953 }, + { url = "https://files.pythonhosted.org/packages/8f/5c/06c01870c0cf336f9f29397bbfbfbbfd3a59918868716e7bb15828e89367/levenshtein-0.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f09cfab6387e9c908c7b37961c045e8e10eb9b7ec4a700367f8e080ee803a562", size = 156399 }, + { url = "https://files.pythonhosted.org/packages/c7/4a/c1d3f27ec8b3fff5a96617251bf3f61c67972869ac0a0419558fc3e2cbe6/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dafa29c0e616f322b574e0b2aeb5b1ff2f8d9a1a6550f22321f3bd9bb81036e3", size = 151061 }, + { url = "https://files.pythonhosted.org/packages/4d/8f/2521081e9a265891edf46aa30e1b59c1f347a452aed4c33baafbec5216fa/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be7a7642ea64392fa1e6ef7968c2e50ef2152c60948f95d0793361ed97cf8a6f", size = 183119 }, + { url = "https://files.pythonhosted.org/packages/1f/a0/a63e3bce6376127596d04be7f57e672d2f3d5f540265b1e30b9dd9b3c5a9/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:060b48c45ed54bcea9582ce79c6365b20a1a7473767e0b3d6be712fa3a22929c", size = 185352 }, + { url = "https://files.pythonhosted.org/packages/17/8c/8352e992063952b38fb61d49bad8d193a4a713e7eeceb3ae74b719d7863d/levenshtein-0.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:712f562c5e64dd0398d3570fe99f8fbb88acec7cc431f101cb66c9d22d74c542", size = 159879 }, + { url = "https://files.pythonhosted.org/packages/69/b4/564866e2038acf47c3de3e9292fc7fc7cc18d2593fedb04f001c22ac6e15/levenshtein-0.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6141ad65cab49aa4527a3342d76c30c48adb2393b6cdfeca65caae8d25cb4b8", size = 245005 }, + { url = "https://files.pythonhosted.org/packages/ba/f9/7367f87e3a6eed282f3654ec61a174b4d1b78a7a73f2cecb91f0ab675153/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:799b8d73cda3265331116f62932f553804eae16c706ceb35aaf16fc2a704791b", size = 1116865 }, + { url = "https://files.pythonhosted.org/packages/f5/02/b5b3bfb4b4cd430e9d110bad2466200d51c6061dae7c5a64e36047c8c831/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ec99871d98e517e1cc4a15659c62d6ea63ee5a2d72c5ddbebd7bae8b9e2670c8", size = 1401723 }, + { url = "https://files.pythonhosted.org/packages/ef/69/b93bccd093b3f06a99e67e11ebd6e100324735dc2834958ba5852a1b9fed/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8799164e1f83588dbdde07f728ea80796ea72196ea23484d78d891470241b222", size = 1226276 }, + { url = "https://files.pythonhosted.org/packages/ab/32/37dd1bc5ce866c136716619e6f7081d7078d7dd1c1da7025603dcfd9cf5f/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:583943813898326516ab451a83f734c6f07488cda5c361676150d3e3e8b47927", size = 1420132 }, + { url = "https://files.pythonhosted.org/packages/4b/08/f3bc828dd9f0f8433b26f37c4fceab303186ad7b9b70819f2ccb493d99fc/levenshtein-0.27.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5bb22956af44bb4eade93546bf95be610c8939b9a9d4d28b2dfa94abf454fed7", size = 1189144 }, + { url = "https://files.pythonhosted.org/packages/2d/54/5ecd89066cf579223d504abe3ac37ba11f63b01a19fd12591083acc00eb6/levenshtein-0.27.1-cp312-cp312-win32.whl", hash = "sha256:d9099ed1bcfa7ccc5540e8ad27b5dc6f23d16addcbe21fdd82af6440f4ed2b6d", size = 88279 }, + { url = "https://files.pythonhosted.org/packages/53/79/4f8fabcc5aca9305b494d1d6c7a98482e90a855e0050ae9ff5d7bf4ab2c6/levenshtein-0.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:7f071ecdb50aa6c15fd8ae5bcb67e9da46ba1df7bba7c6bf6803a54c7a41fd96", size = 100659 }, + { url = "https://files.pythonhosted.org/packages/cb/81/f8e4c0f571c2aac2e0c56a6e0e41b679937a2b7013e79415e4aef555cff0/levenshtein-0.27.1-cp312-cp312-win_arm64.whl", hash = "sha256:83b9033a984ccace7703f35b688f3907d55490182fd39b33a8e434d7b2e249e6", size = 88168 }, +] + +[[package]] +name = "lightning-utilities" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "setuptools" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/39/6fc58ca81492db047149b4b8fd385aa1bfb8c28cd7cacb0c7eb0c44d842f/lightning_utilities-0.15.2.tar.gz", hash = "sha256:cdf12f530214a63dacefd713f180d1ecf5d165338101617b4742e8f22c032e24", size = 31090 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/73/3d757cb3fc16f0f9794dd289bcd0c4a031d9cf54d8137d6b984b2d02edf3/lightning_utilities-0.15.2-py3-none-any.whl", hash = "sha256:ad3ab1703775044bbf880dbf7ddaaac899396c96315f3aa1779cec9d618a9841", size = 29431 }, ] [[package]] @@ -1662,7 +1833,7 @@ name = "nvidia-cudnn-cu12" version = "9.10.2.21" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467 }, @@ -1673,7 +1844,7 @@ name = "nvidia-cufft-cu12" version = "11.3.3.83" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695 }, @@ -1700,9 +1871,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.3.90" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, - { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905 }, @@ -1713,7 +1884,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.8.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466 }, @@ -1874,6 +2045,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/7d/f1c30a92854540bf789e9cd5dde7ef49bbe63f855b85a2e6b3db8135c591/opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec", size = 39488044 }, ] +[[package]] +name = "opencv-python-headless" +version = "4.11.0.86" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/53/2c50afa0b1e05ecdb4603818e85f7d174e683d874ef63a6abe3ac92220c8/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:48128188ade4a7e517237c8e1e11a9cdf5c282761473383e77beb875bb1e61ca", size = 37326460 }, + { url = "https://files.pythonhosted.org/packages/3b/43/68555327df94bb9b59a1fd645f63fafb0762515344d2046698762fc19d58/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81", size = 56723330 }, + { url = "https://files.pythonhosted.org/packages/45/be/1438ce43ebe65317344a87e4b150865c5585f4c0db880a34cdae5ac46881/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6efabcaa9df731f29e5ea9051776715b1bdd1845d7c9530065c7951d2a2899eb", size = 29487060 }, + { url = "https://files.pythonhosted.org/packages/dd/5c/c139a7876099916879609372bfa513b7f1257f7f1a908b0bdc1c2328241b/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b", size = 49969856 }, + { url = "https://files.pythonhosted.org/packages/95/dd/ed1191c9dc91abcc9f752b499b7928aacabf10567bb2c2535944d848af18/opencv_python_headless-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b", size = 29324425 }, + { url = "https://files.pythonhosted.org/packages/86/8a/69176a64335aed183529207ba8bc3d329c2999d852b4f3818027203f50e6/opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca", size = 39402386 }, +] + [[package]] name = "openinference-instrumentation" version = "0.1.38" @@ -2088,7 +2276,7 @@ all = [ [[package]] name = "paddlepaddle" -version = "3.1.1" +version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -2097,13 +2285,14 @@ dependencies = [ { name = "opt-einsum" }, { name = "pillow" }, { name = "protobuf" }, + { name = "safetensors" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/b1/c9d8cfe9d91c70b24e257fd681307b9b74e7487ecabc37a261d7a1c7c4a1/paddlepaddle-3.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fcc6994a2c6c637a458118732eb68ab75975a81dca4927cb92953bf2e8f907da", size = 98010656 }, - { url = "https://files.pythonhosted.org/packages/7b/30/83e52fd59a3c6296db0ec04ec4371700773a236f7615bf170a7741b76f4f/paddlepaddle-3.1.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:f2354a9bdaa4ae87291310ee47e05d35ad233529ff16cbcf0d86f69128a22d84", size = 187478421 }, - { url = "https://files.pythonhosted.org/packages/1d/39/713a48d5f7cdbe43c9a27e61cc13d767e88bdd70408fd9c3dd7c825e83e2/paddlepaddle-3.1.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:5ed977319eb1c439d384994ffc9132460781357be4cf03cb683f8ffc2769ae78", size = 86747223 }, - { url = "https://files.pythonhosted.org/packages/83/11/85c8d6de625f7d3b7d56055b4e9dd8ae0f428049ef3cd3a4fe35a3107c37/paddlepaddle-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:992d31c9e5298f524d1340c7a85db97def72f900ce3030ae6cd3bd4396a4622f", size = 100398619 }, + { url = "https://files.pythonhosted.org/packages/29/5d/41ed9bd2d6a33a191bb3e289d3ad78566356cd9c981b80db3c5007a379d2/paddlepaddle-3.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ac334af72d1b0d49d4c933cfd4eb42d73ec2dc7a6aaf2248c628643d419b6da7", size = 99428886 }, + { url = "https://files.pythonhosted.org/packages/3d/82/47a8001a9897f608c12218e2cc63e6b841b710e3359d752973d5e46c43ef/paddlepaddle-3.2.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:2c17f675a25ebd0675f703f7b82777ea92db0c360bd55a5c6b2dfaad7bc3f260", size = 189034109 }, + { url = "https://files.pythonhosted.org/packages/63/b8/5cfb96745d79917b3c26d72f2e68433cab7c092cd58b2085d37cee947117/paddlepaddle-3.2.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:4ce514114af7671875fc4927ea005afb9d5d0fd777aeb56a1078d22f7f4eb3fb", size = 87938157 }, + { url = "https://files.pythonhosted.org/packages/61/f9/eadca6e6499269f8de35b3f51675240d476023919b381acca5bf526d7fb2/paddlepaddle-3.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:918e69c98a1310db08610f28a2d0812ea779ec819053c44cc55ce7dc72ab6ee6", size = 101654665 }, ] [[package]] @@ -2470,6 +2659,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/3a/7d6292e3c94fb6b872d8d7e80d909dc527ee6b0af73b753c63fdde65a7da/pyclipper-1.3.0.post6-cp312-cp312-win_amd64.whl", hash = "sha256:d3f9da96f83b8892504923beb21a481cd4516c19be1d39eb57a92ef1c9a29548", size = 110278 }, ] +[[package]] +name = "pycocotools" +version = "2.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/a6/694fd661f0feb5e91f7049a202ea12de312ca9010c33bd9d9f0c63046c01/pycocotools-2.0.10.tar.gz", hash = "sha256:7a47609cdefc95e5e151313c7d93a61cf06e15d42c7ba99b601e3bc0f9ece2e1", size = 25389 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/b4/3b87dce90fc81b8283b2b0e32b22642939e25f3a949581cb6777f5eebb12/pycocotools-2.0.10-cp312-abi3-macosx_10_13_universal2.whl", hash = "sha256:e1359f556986c8c4ac996bf8e473ff891d87630491357aaabd12601687af5edb", size = 142896 }, + { url = "https://files.pythonhosted.org/packages/29/d5/b17bb67722432a191cb86121cda33cd8edb4d5b15beda43bc97a7d5ae404/pycocotools-2.0.10-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:075788c90bfa6a8989d628932854f3e32c25dac3c1bf7c1183cefad29aee16c8", size = 390111 }, + { url = "https://files.pythonhosted.org/packages/49/80/912b4c60f94e747dd2c3adbda5d4a4edc1d735fbfa0d91ab2eb231decb5d/pycocotools-2.0.10-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4539d8b29230de042f574012edd0b5227528da083c4f12bbd6488567aabd3920", size = 397099 }, + { url = "https://files.pythonhosted.org/packages/df/d7/b3c2f731252a096bbae1a47cb1bbeab4560620a82585d40cce67eca5f043/pycocotools-2.0.10-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:da7b339624d0f78aa5bdc1c86a53f2dcb36ae7e10ab5fe45ba69878bb7837c7a", size = 396111 }, + { url = "https://files.pythonhosted.org/packages/2c/6f/2eceba57245bfc86174263e12716cbe91b329a3677fbeff246148ce6a664/pycocotools-2.0.10-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffdbf8810f27b32c5c5c85d9cd65e8e066852fef9775e58a7b23abdffeaf8252", size = 416393 }, + { url = "https://files.pythonhosted.org/packages/e1/31/d87f781759b2ad177dd6d41c5fe0ce154f14fc8b384e9b80cd21a157395b/pycocotools-2.0.10-cp312-abi3-win_amd64.whl", hash = "sha256:998a88f90bb663548e767470181175343d406b6673b8b9ef5bdbb3a6d3eb3b11", size = 76824 }, + { url = "https://files.pythonhosted.org/packages/27/13/7674d61658b58b8310e3de1270bce18f92a6ee8136e54a7e5696d6f72fd4/pycocotools-2.0.10-cp312-abi3-win_arm64.whl", hash = "sha256:76cd86a80171f8f7da3250be0e40d75084f1f1505d376ae0d08ed0be1ba8a90d", size = 64753 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -2750,6 +2957,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/ed/539768cf28c661b5b068d66d96a2f155c4971a5d55684a514c1a0e0dec2f/python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc", size = 20556 }, ] +[[package]] +name = "python-levenshtein" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "levenshtein" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/f6/d865a565b7eeef4b5f9a18accafb03d5730c712420fc84a3a40555f7ea6b/python_levenshtein-0.27.1.tar.gz", hash = "sha256:3a5314a011016d373d309a68e875fd029caaa692ad3f32e78319299648045f11", size = 12326 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/95/8c8fd923b0a702388da4f9e0368f490d123cc5224279e6a083984304a15e/python_levenshtein-0.27.1-py3-none-any.whl", hash = "sha256:e1a4bc2a70284b2ebc4c505646142fecd0f831e49aa04ed972995895aec57396", size = 9426 }, +] + [[package]] name = "python-multipart" version = "0.0.20" @@ -2795,6 +3014,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, ] +[[package]] +name = "rapidfuzz" +version = "3.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/fc/a98b616db9a42dcdda7c78c76bdfdf6fe290ac4c5ffbb186f73ec981ad5b/rapidfuzz-3.14.1.tar.gz", hash = "sha256:b02850e7f7152bd1edff27e9d584505b84968cacedee7a734ec4050c655a803c", size = 57869570 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/77/2f4887c9b786f203e50b816c1cde71f96642f194e6fa752acfa042cf53fd/rapidfuzz-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:809515194f628004aac1b1b280c3734c5ea0ccbd45938c9c9656a23ae8b8f553", size = 1932216 }, + { url = "https://files.pythonhosted.org/packages/de/bd/b5e445d156cb1c2a87d36d8da53daf4d2a1d1729b4851660017898b49aa0/rapidfuzz-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0afcf2d6cb633d0d4260d8df6a40de2d9c93e9546e2c6b317ab03f89aa120ad7", size = 1393414 }, + { url = "https://files.pythonhosted.org/packages/de/bd/98d065dd0a4479a635df855616980eaae1a1a07a876db9400d421b5b6371/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1c3d07d53dcafee10599da8988d2b1f39df236aee501ecbd617bd883454fcd", size = 1377194 }, + { url = "https://files.pythonhosted.org/packages/d3/8a/1265547b771128b686f3c431377ff1db2fa073397ed082a25998a7b06d4e/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e9ee3e1eb0a027717ee72fe34dc9ac5b3e58119f1bd8dd15bc19ed54ae3e62b", size = 1669573 }, + { url = "https://files.pythonhosted.org/packages/a8/57/e73755c52fb451f2054196404ccc468577f8da023b3a48c80bce29ee5d4a/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:70c845b64a033a20c44ed26bc890eeb851215148cc3e696499f5f65529afb6cb", size = 2217833 }, + { url = "https://files.pythonhosted.org/packages/20/14/7399c18c460e72d1b754e80dafc9f65cb42a46cc8f29cd57d11c0c4acc94/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26db0e815213d04234298dea0d884d92b9cb8d4ba954cab7cf67a35853128a33", size = 3159012 }, + { url = "https://files.pythonhosted.org/packages/f8/5e/24f0226ddb5440cabd88605d2491f99ae3748a6b27b0bc9703772892ced7/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:6ad3395a416f8b126ff11c788531f157c7debeb626f9d897c153ff8980da10fb", size = 1227032 }, + { url = "https://files.pythonhosted.org/packages/40/43/1d54a4ad1a5fac2394d5f28a3108e2bf73c26f4f23663535e3139cfede9b/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:61c5b9ab6f730e6478aa2def566223712d121c6f69a94c7cc002044799442afd", size = 2395054 }, + { url = "https://files.pythonhosted.org/packages/0c/71/e9864cd5b0f086c4a03791f5dfe0155a1b132f789fe19b0c76fbabd20513/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:13e0ea3d0c533969158727d1bb7a08c2cc9a816ab83f8f0dcfde7e38938ce3e6", size = 2524741 }, + { url = "https://files.pythonhosted.org/packages/b2/0c/53f88286b912faf4a3b2619a60df4f4a67bd0edcf5970d7b0c1143501f0c/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6325ca435b99f4001aac919ab8922ac464999b100173317defb83eae34e82139", size = 2785311 }, + { url = "https://files.pythonhosted.org/packages/53/9a/229c26dc4f91bad323f07304ee5ccbc28f0d21c76047a1e4f813187d0bad/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:07a9fad3247e68798424bdc116c1094e88ecfabc17b29edf42a777520347648e", size = 3303630 }, + { url = "https://files.pythonhosted.org/packages/05/de/20e330d6d58cbf83da914accd9e303048b7abae2f198886f65a344b69695/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8ff5dbe78db0a10c1f916368e21d328935896240f71f721e073cf6c4c8cdedd", size = 4262364 }, + { url = "https://files.pythonhosted.org/packages/1f/10/2327f83fad3534a8d69fe9cd718f645ec1fe828b60c0e0e97efc03bf12f8/rapidfuzz-3.14.1-cp312-cp312-win32.whl", hash = "sha256:9c83270e44a6ae7a39fc1d7e72a27486bccc1fa5f34e01572b1b90b019e6b566", size = 1711927 }, + { url = "https://files.pythonhosted.org/packages/78/8d/199df0370133fe9f35bc72f3c037b53c93c5c1fc1e8d915cf7c1f6bb8557/rapidfuzz-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:e06664c7fdb51c708e082df08a6888fce4c5c416d7e3cc2fa66dd80eb76a149d", size = 1542045 }, + { url = "https://files.pythonhosted.org/packages/b3/c6/cc5d4bd1b16ea2657c80b745d8b1c788041a31fad52e7681496197b41562/rapidfuzz-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:6c7c26025f7934a169a23dafea6807cfc3fb556f1dd49229faf2171e5d8101cc", size = 813170 }, +] + [[package]] name = "referencing" version = "0.36.2" @@ -2961,6 +3203,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762 }, ] +[[package]] +name = "safetensors" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/cc/738f3011628920e027a11754d9cae9abec1aed00f7ae860abbf843755233/safetensors-0.6.2.tar.gz", hash = "sha256:43ff2aa0e6fa2dc3ea5524ac7ad93a9839256b8703761e76e2d0b2a3fa4f15d9", size = 197968 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/b1/3f5fd73c039fc87dba3ff8b5d528bfc5a32b597fea8e7a6a4800343a17c7/safetensors-0.6.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c85ede8ec58f120bad982ec47746981e210492a6db876882aa021446af8ffba", size = 454797 }, + { url = "https://files.pythonhosted.org/packages/8c/c9/bb114c158540ee17907ec470d01980957fdaf87b4aa07914c24eba87b9c6/safetensors-0.6.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6675cf4b39c98dbd7d940598028f3742e0375a6b4d4277e76beb0c35f4b843b", size = 432206 }, + { url = "https://files.pythonhosted.org/packages/d3/8e/f70c34e47df3110e8e0bb268d90db8d4be8958a54ab0336c9be4fe86dac8/safetensors-0.6.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d2d2b3ce1e2509c68932ca03ab8f20570920cd9754b05063d4368ee52833ecd", size = 473261 }, + { url = "https://files.pythonhosted.org/packages/2a/f5/be9c6a7c7ef773e1996dc214e73485286df1836dbd063e8085ee1976f9cb/safetensors-0.6.2-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:93de35a18f46b0f5a6a1f9e26d91b442094f2df02e9fd7acf224cfec4238821a", size = 485117 }, + { url = "https://files.pythonhosted.org/packages/c9/55/23f2d0a2c96ed8665bf17a30ab4ce5270413f4d74b6d87dd663258b9af31/safetensors-0.6.2-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89a89b505f335640f9120fac65ddeb83e40f1fd081cb8ed88b505bdccec8d0a1", size = 616154 }, + { url = "https://files.pythonhosted.org/packages/98/c6/affb0bd9ce02aa46e7acddbe087912a04d953d7a4d74b708c91b5806ef3f/safetensors-0.6.2-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc4d0d0b937e04bdf2ae6f70cd3ad51328635fe0e6214aa1fc811f3b576b3bda", size = 520713 }, + { url = "https://files.pythonhosted.org/packages/fe/5d/5a514d7b88e310c8b146e2404e0dc161282e78634d9358975fd56dfd14be/safetensors-0.6.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8045db2c872db8f4cbe3faa0495932d89c38c899c603f21e9b6486951a5ecb8f", size = 485835 }, + { url = "https://files.pythonhosted.org/packages/7a/7b/4fc3b2ba62c352b2071bea9cfbad330fadda70579f617506ae1a2f129cab/safetensors-0.6.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:81e67e8bab9878bb568cffbc5f5e655adb38d2418351dc0859ccac158f753e19", size = 521503 }, + { url = "https://files.pythonhosted.org/packages/5a/50/0057e11fe1f3cead9254315a6c106a16dd4b1a19cd247f7cc6414f6b7866/safetensors-0.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0e4d029ab0a0e0e4fdf142b194514695b1d7d3735503ba700cf36d0fc7136ce", size = 652256 }, + { url = "https://files.pythonhosted.org/packages/e9/29/473f789e4ac242593ac1656fbece6e1ecd860bb289e635e963667807afe3/safetensors-0.6.2-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:fa48268185c52bfe8771e46325a1e21d317207bcabcb72e65c6e28e9ffeb29c7", size = 747281 }, + { url = "https://files.pythonhosted.org/packages/68/52/f7324aad7f2df99e05525c84d352dc217e0fa637a4f603e9f2eedfbe2c67/safetensors-0.6.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:d83c20c12c2d2f465997c51b7ecb00e407e5f94d7dec3ea0cc11d86f60d3fde5", size = 692286 }, + { url = "https://files.pythonhosted.org/packages/ad/fe/cad1d9762868c7c5dc70c8620074df28ebb1a8e4c17d4c0cb031889c457e/safetensors-0.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d944cea65fad0ead848b6ec2c37cc0b197194bec228f8020054742190e9312ac", size = 655957 }, + { url = "https://files.pythonhosted.org/packages/59/a7/e2158e17bbe57d104f0abbd95dff60dda916cf277c9f9663b4bf9bad8b6e/safetensors-0.6.2-cp38-abi3-win32.whl", hash = "sha256:cab75ca7c064d3911411461151cb69380c9225798a20e712b102edda2542ddb1", size = 308926 }, + { url = "https://files.pythonhosted.org/packages/2c/c3/c0be1135726618dc1e28d181b8c442403d8dbb9e273fd791de2d4384bcdd/safetensors-0.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:c7b214870df923cbc1593c3faee16bec59ea462758699bd3fee399d00aac072c", size = 320192 }, +] + [[package]] name = "scikit-learn" version = "1.7.1" @@ -3000,6 +3264,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/c3/61f273ae550fbf1667675701112e380881905e28448c080b23b5a181df7c/scipy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7b8013c6c066609577d910d1a2a077021727af07b6fab0ee22c2f901f22352a", size = 38508060 }, ] +[[package]] +name = "seaborn" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914 }, +] + [[package]] name = "sentry-sdk" version = "2.37.0" @@ -3050,6 +3328,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, ] +[[package]] +name = "simsimd" +version = "6.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/2f/5a9ccc385f4d6e30aac2b843ef57ba3668ea86756f77f6a9312a3c94f43d/simsimd-6.5.3.tar.gz", hash = "sha256:5ff341e84fe1c46e7268ee9e31f885936b29c38ce59f423433aef5f4bb5bfd18", size = 184865 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/d8/ecb94ab75a0fbab1f9a36eac4cd7734836d7234788c2d3267d1f612e1cbd/simsimd-6.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:52495c13e8547c259a6da1ab5cbc95cb0ac4d2ca4ae33434b9514b64f39a122c", size = 177692 }, + { url = "https://files.pythonhosted.org/packages/90/79/5bab3fd20625b5cb83435f2a0c307af7077394cb963ce9ae92d4b486f8a3/simsimd-6.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:11358046752d72059e425946ac00001704a47869cc0d05b9f750a64720a2a6a9", size = 134107 }, + { url = "https://files.pythonhosted.org/packages/d5/ac/99db6d29819250ca86bd403a5869901e10b8abfa85843a5c33b28dbfe194/simsimd-6.5.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be0f4921c370f715995789eb780315b0456d0b9937209caab0343b98bda5b668", size = 563233 }, + { url = "https://files.pythonhosted.org/packages/5c/79/b3c00bdd2422de46a20add6e77dc34f66de5e157c28487a5e654fbf25965/simsimd-6.5.3-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:26c9920fe1bd3a1d15a24167e2d8777bed32b21b48868d0c785c1a821575bc56", size = 355529 }, + { url = "https://files.pythonhosted.org/packages/6a/85/c65cbeb2fd33ffca41e76c79e73585da20e5d5ce4b0216681e61b643e657/simsimd-6.5.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd0267b61c3128282b52388ce1390d95c8beab219da1b95d7aaadab9a18bf42b", size = 411360 }, + { url = "https://files.pythonhosted.org/packages/89/25/ba0dbdc1614bb35ac5756eb50fc86e322c1701b723e86691dbec45fec765/simsimd-6.5.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cab8670c7ed2754a6a5f3d2d568a43141c6494092fcc1693efecd20cefb51f61", size = 367963 }, + { url = "https://files.pythonhosted.org/packages/7c/f2/34bd80d5f9a1297f2cccab56d0b46fa017f6824ad162e2ea0646529dc539/simsimd-6.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:051c6493f07c4ec5938648accd351b16221a5d07633649b6f392e387811900a1", size = 1068417 }, + { url = "https://files.pythonhosted.org/packages/e6/22/dea38422695e9637ae82d05e28e59b319664ae3f118a9bb1d1a9a7df53fa/simsimd-6.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b1c26dd73960c9789e8e0f90750a2ede4e64120ad96b5f9ec46ef9e1f2039ac", size = 598297 }, + { url = "https://files.pythonhosted.org/packages/3f/df/dc02eeac0eda0eb199039a4569bfcce3a98a78aab6af76dd1915b08433b3/simsimd-6.5.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c827f13caf47cc255dea3455e4f68da9930c396e77ac6f116ab82ecab5d9b1e4", size = 402229 }, + { url = "https://files.pythonhosted.org/packages/a7/2a/e2c6c410bd29320c2666c03ffbba3314a07b2ffb338beabf9f98186c41d6/simsimd-6.5.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1cdcc253fdb9179b9273e4771c333b5d9adf99f911de0d8197a6ee5962bd9f86", size = 460979 }, + { url = "https://files.pythonhosted.org/packages/d7/18/c91afb131ee2bd58ef4f05646c7c0c8d0b3a39a2f45386cd84c019030e3c/simsimd-6.5.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9d0bc9132bf2bb887246c784bf6a6c0b37a96af0d4aec7cc728e9b1274868bdb", size = 372616 }, + { url = "https://files.pythonhosted.org/packages/c4/67/151b8855a0060cba592ef045f3655c144b19f98d896e1ad204c8e1dc6aeb/simsimd-6.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94a989ec638e4ebe33c6aacd31fec8586480017909e7c5016c91005d52512cad", size = 1001276 }, + { url = "https://files.pythonhosted.org/packages/5c/db/e458ae93987726f5b255148b259274c39e6f15b9d1158a0f0fa467539aa3/simsimd-6.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:98af777ea1b227d42efdcb42fa5a667aa30c324665ec35425fcaa31152e4ccad", size = 94877 }, + { url = "https://files.pythonhosted.org/packages/05/fa/a5c8533daf52021beece3666fe09d2d2f41bc807f4863ad582e7ee141649/simsimd-6.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:6e6a0bd069e02bb1f2f88f53a0abfbcf8040d2764668569e519a3360b9303858", size = 59508 }, +] + [[package]] name = "six" version = "1.17.0" @@ -3059,6 +3359,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 }, ] +[[package]] +name = "smmap" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303 }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -3123,7 +3432,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "sys_platform == 'darwin'", "platform_machine == 'aarch64' and sys_platform == 'linux'", - "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/7c/52/30b9bf4ab5810a4e46ef3cbccde8f118e4f59cbea9d913f1c9f4a7ef216c/sqlean_py-3.50.4.3.tar.gz", hash = "sha256:0c01d810191ec7931fca98a46806d6946f240e0275b678d093ce5a89a39035ff", size = 3481771 } wheels = [ @@ -3173,6 +3483,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/e0/c45d74578e7b8cb7e082697d998cebd8ef97afa3d7aedc22e4acd8ae7163/strawberry_graphql-0.270.1-py3-none-any.whl", hash = "sha256:3593086dc08614ae241cb88f7691e90f90b01cab6ee6351cb3838fc5ba8bfab0", size = 301232 }, ] +[[package]] +name = "stringzilla" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/84/e4708f06c9af2f3742942efa855255ccb7421271c911c37b8e8853828602/stringzilla-4.0.0.tar.gz", hash = "sha256:ee18a849150c7c95aba75213048a9a63ea77142b3b29876ea05dfa2d4a2a2b9c", size = 465190 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/bb/a5ebc920475e9a595d48b57ec08fbe3e1d901ab6cd2feeeddf3d4b36a84f/stringzilla-4.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bca15c02d351e4a25fac7730949a6da67e6aba116a7f57d7a03980320fb3d672", size = 125714 }, + { url = "https://files.pythonhosted.org/packages/33/2e/2ec74a4f78d2616b41657cae7d51ed942614a93aaba33bd7e57397adaafc/stringzilla-4.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5fb5e2d3aaaec9dc48f2b409f2d92c2d6c2669d97acf366be45e468b2da0868f", size = 123292 }, + { url = "https://files.pythonhosted.org/packages/96/02/07eb7f73fea5a72291ca40f4111d4ab6ea259d6165afbc76849b152c0016/stringzilla-4.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9b4e7132bacf7f3945cac933d5a16b53bb219f8ed3078fa2441396262caea8a", size = 313208 }, + { url = "https://files.pythonhosted.org/packages/38/88/0d2658876dfb69ca3ea6c88e9326cb3dd9cae0b2b526024fdd8330a1b130/stringzilla-4.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7199af397482e7ac772d91e0563e2db8845431ae196137955cc04814dfcc2d01", size = 361230 }, + { url = "https://files.pythonhosted.org/packages/76/98/1d39c567b896972067b2cb5156a8f965bf3bd2a47150e88885c02f3176ae/stringzilla-4.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:27a5afa30ac81242086fea6c541b87441a1b3036eb50dd27c2388586c3fd07c4", size = 328135 }, + { url = "https://files.pythonhosted.org/packages/53/30/533b97f1ac873ab603856124de19c3db7a1a775e26668bc49a3a696291b2/stringzilla-4.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f8ba4b73a2b7580f8e5bb7f237fd4a95956639a93642b0641e3a4482cb15e3f", size = 343532 }, + { url = "https://files.pythonhosted.org/packages/2a/84/1d2ed606b448ef3265ab89ff8ed0794b4f964af3ff1bfb67f4977e51781b/stringzilla-4.0.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb882a912f77d2a3ceb5eb902d81b64df607f88f4608485ef2abf35e038057d2", size = 318036 }, + { url = "https://files.pythonhosted.org/packages/19/0e/e0996fac66138b9db6e0e2bf63387741790a05ecd9fc3f800a90eccbbbac/stringzilla-4.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c123739d215002d6035d2edb4a6d10d3f8594f886b234c3d82a5adcc8fb36ff6", size = 299925 }, + { url = "https://files.pythonhosted.org/packages/74/4f/7ffeb67d2f7c12519351bdaa06125f17fc4187e5594073ad41bcfde6a017/stringzilla-4.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5e3607d6e9d325ee21a3d7669c3d4235bf5ef3df6a2bdbb67c04b80c82ef3e97", size = 324113 }, + { url = "https://files.pythonhosted.org/packages/00/34/bdfcbce01a89e0157fb13e3923567c3ea3125d48449972c944a76c35c837/stringzilla-4.0.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e914c9247e6709c4c623017306bf1389afb56ae651199e5643199e8f24df1a62", size = 305509 }, + { url = "https://files.pythonhosted.org/packages/be/74/bcb29bf134299e5ebcd6dea2540ef692bbc5308bba3865d37eb6b0e75506/stringzilla-4.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e7305f841288200bf2f64f821ce17c0aaa60cda62e38ad8d53459c3ea894fc3a", size = 330828 }, + { url = "https://files.pythonhosted.org/packages/b6/c5/541548836563e290b1f82e6f8e4070ac668bce733e44e075bba0686354e0/stringzilla-4.0.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:028dd186dccd4f6dd968037a13df40d4cd49494915ca893399ffef9123792fd2", size = 324929 }, + { url = "https://files.pythonhosted.org/packages/9a/4d/98ce42c5ea71bfc7666c56c2183b457a95185bbe9109ed59c473fc387c87/stringzilla-4.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dc3dd54fb8fc070d08b75d0845adcdbfbd5bd6e1e061db52191edf7e53585085", size = 337553 }, + { url = "https://files.pythonhosted.org/packages/45/97/ae91a6ee9727274aa8535ce8bb9b00b15e4c7ebcb90dddfbfe2dccd5e035/stringzilla-4.0.0-cp312-cp312-win32.whl", hash = "sha256:63520953e292c316e6eace31fb7cd8a1a5d2f6e90c7aba5981d90d29ac93d2a8", size = 83956 }, + { url = "https://files.pythonhosted.org/packages/47/10/661e97b45af554a70d173b412aa7702387c0132bebca939333e33a27ce44/stringzilla-4.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:d90806a5ec5e88d153c77cbcc9fcfdac4970403b9f04234043cba2c22a9be4fe", size = 102891 }, + { url = "https://files.pythonhosted.org/packages/88/31/dcba1028844cced15befc3b570f9880951dcbdb7e5a20275eb586ff5cfc5/stringzilla-4.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:1f7bfafb1b2cae818b2da9fada52d3f79815738c50a52228e108a86b980eab41", size = 90128 }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -3290,6 +3624,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/66/5c9a321b325aaecb92d4d1855421e3a055abd77903b7dab6575ca07796db/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:619c2869db3ada2c0105487ba21b5008defcc472d23f8b80ed91ac4a380283b0", size = 73630478 }, ] +[[package]] +name = "torchmetrics" +version = "1.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lightning-utilities" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/2e/48a887a59ecc4a10ce9e8b35b3e3c5cef29d902c4eac143378526e7485cb/torchmetrics-1.8.2.tar.gz", hash = "sha256:cf64a901036bf107f17a524009eea7781c9c5315d130713aeca5747a686fe7a5", size = 580679 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/21/aa0f434434c48490f91b65962b1ce863fdcce63febc166ca9fe9d706c2b6/torchmetrics-1.8.2-py3-none-any.whl", hash = "sha256:08382fd96b923e39e904c4d570f3d49e2cc71ccabd2a94e0f895d1f0dac86242", size = 983161 }, +] + [[package]] name = "torchvision" version = "0.23.0" @@ -3323,7 +3672,7 @@ name = "triton" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "setuptools", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d0/66/b1eb52839f563623d185f0927eb3530ee4d5ffe9d377cdaf5346b306689e/triton-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c1d84a5c0ec2c0f8e8a072d7fd150cab84a9c239eaddc6706c081bfae4eb04", size = 155560068 }, @@ -3362,6 +3711,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/e8/b3d537470e8404659a6335e7af868e90657efb73916ef31ddf3d8b9cb237/typer-0.17.3-py3-none-any.whl", hash = "sha256:643919a79182ab7ac7581056d93c6a2b865b026adf2872c4d02c72758e6f095b", size = 46494 }, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.20250822" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/85/90a442e538359ab5c9e30de415006fb22567aa4301c908c09f19e42975c2/types_pyyaml-6.0.12.20250822.tar.gz", hash = "sha256:259f1d93079d335730a9db7cff2bcaf65d7e04b4a56b5927d49a612199b59413", size = 17481 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/8e/8f0aca667c97c0d76024b37cffa39e76e2ce39ca54a38f285a64e6ae33ba/types_pyyaml-6.0.12.20250822-py3-none-any.whl", hash = "sha256:1fe1a5e146aa315483592d292b72a172b65b946a6d98aa6ddd8e4aa838ab7098", size = 20314 }, +] + [[package]] name = "types-requests" version = "2.32.4.20250809" @@ -3520,6 +3878,35 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/64/68/d43c145f110fbc7da6cbd4082d940fdbc8aaa7d9e8b52f3f574485d49208/vncdotool-1.2.0.tar.gz", hash = "sha256:53408d18ca7f9f21c525fc88189b01ca6594153ec1a9be09f6198306d166ea0d", size = 54809 } +[[package]] +name = "wandb" +version = "0.21.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/84/af6ccdf95e56f15aceb360e437fbfcca3dc91ad8ca335fe482083e29f7a5/wandb-0.21.3.tar.gz", hash = "sha256:031e24e2aad0ce735dfdcc74baf2f2c12c106f500ed24798de6ef9b9e63bb432", size = 40146972 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/e8/b5bfbbc7f76c11fd0665b92be8a38c6a83b27f353552233b9959b21be488/wandb-0.21.3-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:f85bac45b4482742ec9ff190af38eb00a877ddeb4875475e7e487dc19300ff03", size = 18820209 }, + { url = "https://files.pythonhosted.org/packages/59/a3/03f0fcde49609df1cb3a382fb5053f601b88da448bcd415ed7f75272eee7/wandb-0.21.3-py3-none-macosx_12_0_arm64.whl", hash = "sha256:8a2b3ba419b91d47edead2755f04cef54f9e3c4496ee0c9854c3cfeff4216dd3", size = 18310636 }, + { url = "https://files.pythonhosted.org/packages/1d/c3/d6048db30ff2e3c67089ba0e94878572fd26137b146f8e3b27bbdf428b31/wandb-0.21.3-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:35a1972881f3b85755befab004118234593792a9f05e07fd6345780172f4420e", size = 19053277 }, + { url = "https://files.pythonhosted.org/packages/ea/7f/805c3d2fa9e3b8b6bf2bc534887c9ed97bdf22007ca8ba59424a1c8bb360/wandb-0.21.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d9cf8588cb090a2a41f589037fda72c57c9e23edfbd2ad829e575f1305d942c", size = 18130850 }, + { url = "https://files.pythonhosted.org/packages/5b/af/a3252e5afac98a036f83c65ec92cadf6677ccdaacbbb2151da29f694d136/wandb-0.21.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff24b6b8e0f9da840b6bd5c7f60b0a5507bd998db40c9c2d476f9a340bec8ed", size = 19570305 }, + { url = "https://files.pythonhosted.org/packages/4d/f9/4404b5a24bfd4ba027c19d30152b0fc7ebca8c49b202dee6ecb7f316082c/wandb-0.21.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4975dec19e2b343e23ed6e60f7e1290120553719f82e87a22205bede758416ad", size = 18135806 }, + { url = "https://files.pythonhosted.org/packages/ff/32/9580f42899e54f3d0b4ea619b6f6a54980a4e36fd0675d58c09f0a08d3f6/wandb-0.21.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:514a0aad40ecc0bdb757b1dc86e4ac98f61d2d760445b6e1f555291562320f2d", size = 19646760 }, + { url = "https://files.pythonhosted.org/packages/75/d3/faa6ddb792a158c154fb704b25c96d0478e71eabf96e3f17529fb23b6894/wandb-0.21.3-py3-none-win32.whl", hash = "sha256:45aa3d8ad53c6ee06f37490d7a329ed7d0f5ca4dbd5d05bb0c01d5da22f14691", size = 18709408 }, + { url = "https://files.pythonhosted.org/packages/d8/2d/7ef56e25f78786e59fefd9b19867c325f9686317d9f7b93b5cb340360a3e/wandb-0.21.3-py3-none-win_amd64.whl", hash = "sha256:56d5a5697766f552a9933d8c6a564202194768eb0389bd5f9fe9a99cd4cee41e", size = 18709411 }, +] + [[package]] name = "wcwidth" version = "0.2.13"