diff --git a/Dockerfile.arm b/Dockerfile.arm new file mode 100644 index 0000000..d9a05ad --- /dev/null +++ b/Dockerfile.arm @@ -0,0 +1,78 @@ +# ARM64/Apple Silicon compatible Dockerfile +FROM --platform=linux/arm64 python:3.12-slim AS base + +# For ARM compatibility, use explicit platform +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONOPTIMIZE=2 + +# Update packages and install core dependencies +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + ca-certificates \ + wget \ + curl \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install Python computer vision dependencies +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev \ + libgomp1 \ + libgthread-2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +# Install X11 and basic tools (minimal set for ARM compatibility) +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + xvfb \ + x11-utils \ + scrot \ + imagemagick \ + && rm -rf /var/lib/apt/lists/* + +# Try to install xdotool (may not be available on all ARM repos) +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + xdotool || echo "xdotool not available on this ARM platform" + +# Try to install FreeRDP (may have different package name on ARM) +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + freerdp2-x11 || \ + apt-get install -y --no-install-recommends freerdp-x11 || \ + echo "FreeRDP not available, OCR testing will still work" + +RUN rm -rf /var/lib/apt/lists/* + +# ---------- Builder stage ---------- +FROM base AS builder +COPY --from=ghcr.io/astral-sh/uv:0.8.13 /uv /usr/local/bin/uv +WORKDIR /app + +COPY pyproject.toml /app/ +RUN --mount=type=cache,target=/root/.cache/uv uv lock +RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-install-project --no-dev + +COPY . /app +RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev + +# Install debugpy for remote debugging +RUN --mount=type=cache,target=/root/.cache/uv uv add debugpy + +# ---------- Runtime stage ---------- +FROM base AS runtime +WORKDIR /app + +# Copy uv binary from builder stage +COPY --from=builder /usr/local/bin/uv /usr/local/bin/uv + +RUN useradd -m -u 10001 appuser +COPY --from=builder /app /app +ENV PATH="/app/.venv/bin:${PATH}" + +ENV DISPLAY=:99 +RUN mkdir -p /tmp/screenshots && chown appuser:appuser /tmp/screenshots + +USER appuser +CMD ["bash"] \ No newline at end of file diff --git a/Dockerfile.rdp b/Dockerfile.rdp index 5e182db..0669d47 100644 --- a/Dockerfile.rdp +++ b/Dockerfile.rdp @@ -8,8 +8,11 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ PIP_DISABLE_PIP_VERSION_CHECK=1 \ PYTHONOPTIMIZE=2 -# Install system deps for computer vision and RDP -RUN apt-get update -qq && apt-get install -y --no-install-recommends \ +# Update package lists first +RUN apt-get update -qq + +# Install system deps for computer vision and RDP in separate steps +RUN apt-get install -y --no-install-recommends \ ca-certificates \ libglib2.0-0 \ libsm6 \ @@ -17,23 +20,34 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ libxrender-dev \ libgomp1 \ libgl1-mesa-glx \ - libgthread-2.0-0 \ - # RDP-specific dependencies - freerdp2-x11 \ + libgthread-2.0-0 + +# Install X11 and display tools +RUN apt-get install -y --no-install-recommends \ xvfb \ - xdotool \ - imagemagick \ - netbase-client \ - # X11 utilities for RDP screen capture x11-apps \ x11-utils \ + xdotool + +# Install RDP and screenshot tools +RUN apt-get install -y --no-install-recommends \ + freerdp2-x11 \ scrot \ - # Additional tools for image processing - netpbm \ - && rm -rf /var/lib/apt/lists/* + imagemagick + +# Install network tools (try different package name if needed) +RUN apt-get install -y --no-install-recommends \ + netbase || apt-get install -y --no-install-recommends inetutils-ping + +# Install image processing tools +RUN apt-get install -y --no-install-recommends \ + netpbm + +# Clean up +RUN rm -rf /var/lib/apt/lists/* # Configure ImageMagick security policy for screenshot conversion -RUN sed -i 's/rights="none" pattern="PDF"/rights="read|write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml && \ +RUN sed -i 's/rights="none" pattern="PDF"/rights="read|write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml || true && \ sed -i 's/rights="none" pattern="XWD"/rights="read|write" pattern="XWD"/g' /etc/ImageMagick-6/policy.xml || true # ---------- Builder: uv + lock + install (no dev) ---------- @@ -83,7 +97,7 @@ EXPOSE 8000 # Health check to verify RDP dependencies HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD xfreerdp --version > /dev/null 2>&1 || exit 1 + CMD which xfreerdp > /dev/null 2>&1 || exit 1 # Default command: run the VM automation CLI with RDP CMD ["uv", "run", "vm-automation", "--connection", "rdp"] diff --git a/README.md b/README.md index 820044c..df2eb77 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,98 @@ # VM Automation - Production Ready GUI Automation System -๐Ÿค– **Two-Agent Architecture for Remote Windows VM GUI Automation** +๐Ÿค– **Multi-Architecture Computer Vision Automation Platform** -A production-ready AI system that automates GUI interactions in Windows VMs with computer vision, featuring patient safety verification for healthcare applications. +A production-ready AI system that automates GUI interactions across VMs and local desktops with advanced computer vision, featuring unified OCR architecture, MCP server integration, and healthcare-grade patient safety verification. + +**Version**: 2.0.0 +**Architecture**: OCR System +**AI Integration**: OpenAI, Claude, and Custom LLM Support +**Protocols**: VNC, RDP, Local Desktop (macOS/AppleScript) ## ๐ŸŽฏ System Overview -**Two specialized AI agents working together:** +**Multi-Agent Architecture with Flexible Backends:** -- **Agent 1 (VM Navigator)**: Connects to VMs, launches applications, verifies patient safety -- **Agent 2 (App Controller)**: Performs precise GUI interactions using shared computer vision components +- **VM Navigator Agent**: Connects to remote VMs (VNC/RDP), launches applications, verifies patient safety +- **App Controller Agent**: Performs precise GUI interactions using shared computer vision components +- **Local Desktop Agent**: Native macOS automation using AppleScript for local screen control +- **MCP Server**: Model Context Protocol server exposing vision tools to LLMs +- **Unified OCR System**: Pure function-based computer vision core serving all agents **Key Technologies:** -- **YOLOv8s-ONNX**: Real-time UI element detection -- **PaddleOCR**: Text recognition and verification -- **VNC/RDP Protocol**: Flexible VM connection options -- **Patient Safety**: Healthcare-grade identity verification +- **YOLOv8s-ONNX**: Real-time UI element detection with CPU-optimized inference +- **PaddleOCR**: Professional multilingual text recognition and extraction +- **Unified OCR Architecture**: Pure function-based computer vision with spatial reasoning +- **Action Verification**: Screenshot-based verification with confidence scoring +- **MCP Protocol**: Model Context Protocol for LLM tool integration +- **Multi-Backend Support**: VNC, RDP, and native macOS automation +- **Healthcare Safety**: HIPAA-compliant patient identity verification +- **Production Features**: Error handling, audit logging, session management ## ๐Ÿš€ Quick Start ### 1. Setup ```bash -# Install dependencies +# Clone repository +git clone https://github.com/your-org/computer-use-agent.git +cd computer-use-agent + +# Install dependencies (requires Python 3.10+) uv sync -# Download AI models -uv run src/setup_models.py +# Download and setup AI models (YOLO + PaddleOCR) +uv run src/ocr/setup_models.py + +# Verify installation +uv run python -c "from ocr import detect_ui_elements; print('โœ… Installation complete')" ``` +**System Requirements:** +- Python 3.10 or later +- 4GB RAM minimum (8GB recommended for optimal performance) +- macOS: For local automation features +- Linux/Windows: For VM connections only + ### 2. Basic Usage -**Simple VM automation:** +**Quick Test (Local Desktop):** ```bash -# Run with basic configuration -uv run src/main.py +# Test local desktop automation (macOS only) +uv run python -c " +from automation.form_interface import FormFiller +from agent.vision_tools import analyze_screen +print('Testing local automation...') +analysis = analyze_screen('What applications are visible?') +print('โœ… Local automation ready') +" ``` -**Production usage with configuration:** +**VM Automation:** ```bash -# Create sample configuration -uv run vm-automation --create-samples +# Quick start with environment variables +export VM_HOST="192.168.1.100" +export VM_PASSWORD="your_password" +export CONNECTION_TYPE="vnc" +uv run src/vm/main.py -# Edit vm_config.json with your VM details -# Then run: +# Or use configuration file +uv run vm-automation --create-samples +# Edit vm_config.json, then: uv run vm-automation +``` + +**MCP Server (for LLM Integration):** -# Or specify connection type: -uv run vm-automation --connection rdp -uv run vm-automation --connection vnc +```bash +# Start MCP server for Claude/OpenAI integration +uv run python -c " +from agent.mcp import start_server +start_server(port=8000) +" ``` ### 3. Configuration Options @@ -67,7 +108,13 @@ uv run vm-automation --connection vnc "vm_password": "vnc_password", "connection_type": "vnc", "target_app_name": "Greenway Dev", - "target_button_text": "Submit" + "target_button_text": "Submit", + "ocr_confidence_threshold": 0.6, + "yolo_confidence_threshold": 0.5, + "screenshot_interval": 1.0, + "max_retries": 3, + "enable_audit_logging": true, + "log_level": "INFO" } ``` @@ -83,8 +130,14 @@ uv run vm-automation --connection vnc "rdp_domain": "COMPANY", "rdp_width": 1920, "rdp_height": 1080, + "rdp_color_depth": 24, "target_app_name": "Greenway Dev", - "target_button_text": "Submit" + "target_button_text": "Submit", + "ocr_confidence_threshold": 0.7, + "yolo_confidence_threshold": 0.6, + "screenshot_interval": 0.8, + "enable_gpu_acceleration": false, + "timeout_seconds": 30 } ``` @@ -114,6 +167,391 @@ export TARGET_BUTTON="Submit" vm-automation ``` +## ๐Ÿค– AI Agent Computer Vision Tools + +The system includes professional computer vision tools designed for AI agent automation. Any AI agent can use these tools with simple prompts - no complex setup required. + +### ๐Ÿ”ง Core Vision Components + +**YOLOv8s-ONNX Detector:** +- Real-time UI element detection +- UI-focused class filtering (removes irrelevant objects like animals, food) +- Optimized for screen/desktop environments +- CPU-optimized ONNX inference + +**PaddleOCR Integration:** +- Professional text recognition and extraction +- Multi-language support +- Region-aware text detection +- High-confidence text filtering + +**Unified OCR System:** +- Pure function-based design with YOLO + OCR integration +- Intelligent element detection and text extraction +- Spatial reasoning for UI element relationships +- Unified element representation with confidence scoring + +**Integrated Verification:** +- Function-based action verification +- Screenshot-based change detection +- Text input verification +- Element presence confirmation + +### ๐ŸŽฏ AI Agent Function Tools + +**Simple Function Interface:** + +```python +from agent.vision_tools import * + +# Analyze current screen +analysis = analyze_screen("What buttons and forms are visible?") + +# Find specific elements +button = find_element("Submit button") +field = find_element("Username input field") + +# Perform actions +click_element(button) +type_text_in_field("john.doe", field) + +# Verify outcomes +verify_action("Form should be submitted successfully") +``` + +**Available Functions:** +- `analyze_screen(prompt)` - Analyze screen contents with natural language +- `find_element(description)` - Find UI elements by description +- `click_element(element)` - Click on elements or coordinates +- `type_text_in_field(text, field)` - Type text in input fields +- `verify_action(expected)` - Verify action outcomes +- `wait_for_element(description)` - Wait for elements to appear +- `scroll_screen(direction, pixels)` - Scroll screen content +- `take_screenshot(path)` - Capture screen images + +### ๐Ÿ”Œ AI Framework Integration + +**OpenAI Functions:** + +```python +from openai import OpenAI +from agent.function_definitions import get_vision_function_tools +from agent.vision_tools import * + +client = OpenAI() +tools = get_vision_function_tools() + +response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "Click the Submit button"}], + tools=tools, + tool_choice="auto" +) +``` + +**Claude Tools:** + +```python +import anthropic +from agent.function_definitions import get_vision_function_tools + +client = anthropic.Anthropic() +tools = get_vision_function_tools() + +response = client.messages.create( + model="claude-3-opus-20240229", + tools=tools, + messages=[{"role": "user", "content": "Fill out this form"}] +) +``` + +**Other Frameworks:** +- Compatible with any AI framework that supports function calling +- JSON schema definitions provided for easy integration +- Standardized prompt-to-action interface + +### ๐ŸŽฌ Usage Examples + +**Healthcare Workflow Automation:** +```python +# Healthcare EMR automation with patient safety +from agent.vision_tools import * +from vm.automation.vm_navigator import VMNavigatorTools + +# Initialize with patient safety verification +navigator = VMNavigatorTools(session, vm_target) + +# Verify patient banner before proceeding +patient_info = { + "name": "John Doe", + "mrn": "123456", + "dob": "01/01/1980" +} +safety_check = navigator.verify_patient_banner(patient_info) +if not safety_check["success"]: + raise Exception("Patient safety verification failed") + +# Proceed with form automation +username_field = find_element("username field") +type_text_in_field("doctor@hospital.com", username_field) +submit_btn = find_element("login button") +click_element(submit_btn) +verify_action("Successfully logged into EMR system") +``` + +**Multi-Backend Automation:** +```python +# Example using different backends for different tasks +from automation.desktop_control import DesktopControl # Local macOS +from vm.connections.vnc_connection import VNCConnection # Remote VM + +# Local screenshot analysis +local_control = DesktopControl() +local_screenshot = local_control.capture_screen() +local_elements = detect_ui_elements(local_screenshot) + +# Remote VM interaction +vm_connection = VNCConnection("192.168.1.100", 5900, "password") +vm_screenshot = vm_connection.capture_screen() +vm_elements = find_elements_by_text(vm_screenshot, "Submit") +``` + +**Form Automation:** +```python +# AI agent can automate forms with simple prompts +analysis = analyze_screen("Analyze this login form") +username_field = find_element("username field") +type_text_in_field("user@example.com", username_field) + +password_field = find_element("password field") +type_text_in_field("secure_pass", password_field) + +submit_btn = find_element("login button") +click_element(submit_btn) + +verify_action("Login should be successful") +``` + +**Web Navigation:** +```python +# Navigate websites with computer vision +search_box = find_element("search box") +type_text_in_field("computer vision AI", search_box) + +search_btn = find_element("search button") +click_element(search_btn) + +results = wait_for_element("search results", max_attempts=10) +verify_action("Search results should be displayed") +``` + +**Settings Management:** +```python +# Navigate application settings +settings = find_element("settings menu") +click_element(settings) + +privacy_tab = find_element("privacy tab") +click_element(privacy_tab) + +toggle = find_element("data collection toggle") +click_element(toggle) + +verify_action("Privacy settings should be updated") +``` + +### โš™๏ธ Advanced Configuration Options + +**Vision System Configuration:** +```python +from agent.vision_tools import configure_vision_tools + +# Healthcare/Financial (High Precision) +configure_vision_tools( + confidence_threshold=0.8, + ocr_language="en", + enable_gpu=False, # CPU-only for consistency + max_screenshot_size=(1920, 1080), + element_timeout=10.0 +) + +# General Automation (Balanced) +configure_vision_tools( + confidence_threshold=0.6, + ocr_language="en", + enable_preprocessing=True, + screenshot_quality=85 +) + +# Development/Testing (Fast) +configure_vision_tools( + confidence_threshold=0.4, + ocr_language="en", + enable_caching=True, + debug_mode=True +) +``` + +**MCP Server Configuration:** +```python +from agent.mcp import create_mcp_server + +# Production MCP server +server = create_mcp_server( + host="0.0.0.0", + port=8000, + max_concurrent_requests=10, + enable_cors=True, + log_level="INFO" +) + +# Development MCP server +server = create_mcp_server( + host="localhost", + port=8001, + debug=True, + enable_detailed_logging=True +) +``` + +**VM Connection Tuning:** +```python +from vm.connections.vnc_connection import VNCConnection + +# High-performance VNC +vnc = VNCConnection( + host="192.168.1.100", + port=5900, + password="password", + pixel_format="bgr233", # Faster but lower quality + enable_compression=True, + compression_level=6 +) + +# High-quality VNC +vnc = VNCConnection( + host="192.168.1.100", + port=5900, + password="password", + pixel_format="truecolor", + enable_compression=False, + capture_rate=30 # FPS +) +``` + +### ๐Ÿ“ Complete System Architecture + +``` +src/ +โ”œโ”€โ”€ ocr/ # Unified Computer Vision System +โ”‚ โ”œโ”€โ”€ detector.py # YOLOv8s-ONNX UI element detection +โ”‚ โ”œโ”€โ”€ reader.py # PaddleOCR text extraction +โ”‚ โ”œโ”€โ”€ finder.py # Pure function UI element search and analysis +โ”‚ โ”œโ”€โ”€ verification.py # Pure function action verification +โ”‚ โ”œโ”€โ”€ setup_models.py # Model download and management +โ”‚ โ””โ”€โ”€ models/ # Pre-trained models +โ”‚ โ”œโ”€โ”€ yolov8s.onnx # YOLO ONNX model for inference +โ”‚ โ””โ”€โ”€ yolov8s.pt # YOLO PyTorch model +โ”œโ”€โ”€ agent/ # AI Agent Interface +โ”‚ โ”œโ”€โ”€ vision_tools.py # Function interface for AI agents +โ”‚ โ”œโ”€โ”€ function_definitions.py # JSON schemas for AI frameworks +โ”‚ โ”œโ”€โ”€ examples.py # Usage examples and demos +โ”‚ โ””โ”€โ”€ mcp/ # Model Context Protocol Server +โ”‚ โ”œโ”€โ”€ __init__.py # MCP server interface exports +โ”‚ โ”œโ”€โ”€ handlers.py # MCP tool handlers +โ”‚ โ”œโ”€โ”€ mcp_server.py # Model Context Protocol server +โ”‚ โ””โ”€โ”€ tools.py # Tool definitions for LLMs +โ”œโ”€โ”€ vm/ # VM Connection and Automation +โ”‚ โ”œโ”€โ”€ connections/ # VNC/RDP/Desktop connection backends +โ”‚ โ”œโ”€โ”€ tools/ # Screen capture and input actions +โ”‚ โ””โ”€โ”€ automation/ # VM navigator and app controller agents +โ”œโ”€โ”€ automation/ # Local Desktop Automation +โ”‚ โ”œโ”€โ”€ desktop_control.py # Native macOS automation (AppleScript) +โ”‚ โ””โ”€โ”€ form_interface.py # High-level form filling interface +``` + +**๐Ÿ—๏ธ Multi-Layered Architecture:** + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ AI Agent Functions โ”‚ โ† Prompt-based interface +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ MCP Server Interface โ”‚ โ† Model Context Protocol tools +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ Unified OCR Functions โ”‚ โ† Pure function computer vision +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ VM Connection Backends โ”‚ โ† VNC/RDP/Desktop abstraction +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ Vision Models (YOLO + PaddleOCR) โ”‚ โ† Computer vision core +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +**๐ŸŽฏ Agent Complexity Levels:** + +- **SIMPLE**: `analyze_screen()`, `find_element()`, `click_element()` +- **STRUCTURED**: `execute_structured_workflow()`, `create_form_filling_workflow()` +- **ADVANCED**: `AgentSession`, spatial reasoning, custom runtime patterns +- **EXPERT**: Full infrastructure access with `StepRunner`, protocol adapters + +### ๐Ÿงช Testing Multi-Agent Infrastructure + +**Basic Vision Tools:** +```bash +# Test basic agent functions +uv run python -c "from agent import analyze_screen, find_element; print('โœ… Basic tools ready')" + +# Test advanced workflow capabilities +uv run python -c "from agent import execute_structured_workflow, create_form_filling_workflow; print('โœ… Advanced tools ready')" + +# Test multi-agent service +uv run python -c "from agent.service_architecture import get_service_status; print('โœ… Service ready')" +``` + +**Infrastructure Testing:** +```bash +# Test unified OCR functions +uv run python -c "from ocr import detect_ui_elements, extract_text, find_elements_by_text; print('โœ… Unified OCR functions')" + +# Test verification functions +uv run python -c "from ocr import verify_click_success, verify_element_present; print('โœ… Verification functions')" + +# Test MCP server interface +uv run python -c "from agent.mcp import create_mcp_server, get_function_tools; print('โœ… MCP server interface')" + +# Run comprehensive examples +uv run python src/agent/examples.py + +# Setup models if not downloaded +uv run python src/ocr/setup_models.py +``` + +**Multi-Agent Service Demo:** +```python +from agent.service_architecture import ( + OCRService, ConnectionBackend, get_service_status +) + +# Create service +service = OCRService() + +# Create desktop agent session +session_id = service.create_session( + ConnectionBackend.DESKTOP, + {"models_dir": "src/ocr/models"} +) + +# Execute actions in session +result = service.execute_in_session( + session_id, + "analyze_screen", + {"prompt": "What's on the screen?"} +) + +# Get service status +status = get_service_status() +print(f"Active sessions: {status['active_sessions']}") +``` + ## ๐Ÿ”— Connection Types The system supports both VNC and RDP connections through a flexible abstraction layer: @@ -154,23 +592,37 @@ The same computer vision and UI automation logic works seamlessly with both conn ``` โ”œโ”€โ”€ src/ โ”‚ โ”œโ”€โ”€ main.py # Main orchestrator (consolidated entry point) -โ”‚ โ”œโ”€โ”€ setup_models.py # Download YOLO & PaddleOCR models -โ”‚ โ”œโ”€โ”€ agents/ # AI agent implementations -โ”‚ โ”‚ โ”œโ”€โ”€ vm_navigator.py # Agent 1: VM connection & navigation -โ”‚ โ”‚ โ”œโ”€โ”€ app_controller.py # Agent 2: GUI interactions -โ”‚ โ”‚ โ””โ”€โ”€ shared_context.py # Shared data structures -โ”‚ โ”œโ”€โ”€ connections/ # Connection abstraction layer -โ”‚ โ”‚ โ”œโ”€โ”€ base.py # Abstract base classes -โ”‚ โ”‚ โ”œโ”€โ”€ vnc_connection.py # VNC implementation -โ”‚ โ”‚ โ””โ”€โ”€ rdp_connection.py # RDP implementation -โ”‚ โ”œโ”€โ”€ tools/ # Low-level automation tools -โ”‚ โ”‚ โ”œโ”€โ”€ screen_capture.py # Connection-agnostic screen capture -โ”‚ โ”‚ โ”œโ”€โ”€ input_actions.py # Connection-agnostic input actions -โ”‚ โ”‚ โ””โ”€โ”€ verification.py # Action verification -โ”‚ โ””โ”€โ”€ vision/ # Computer vision components -โ”‚ โ”œโ”€โ”€ ui_finder.py # YOLO-based UI detection -โ”‚ โ”œโ”€โ”€ ocr_reader.py # PaddleOCR wrapper -โ”‚ โ””โ”€โ”€ yolo_detector.py # YOLO model interface +โ”‚ โ”œโ”€โ”€ ocr/ # Unified Computer Vision System +โ”‚ โ”‚ โ”œโ”€โ”€ detector.py # YOLOv8s-ONNX UI element detection +โ”‚ โ”‚ โ”œโ”€โ”€ reader.py # PaddleOCR text extraction +โ”‚ โ”‚ โ”œโ”€โ”€ finder.py # Pure function UI element search +โ”‚ โ”‚ โ”œโ”€โ”€ verification.py # Pure function action verification +โ”‚ โ”‚ โ”œโ”€โ”€ setup_models.py # Model download and management +โ”‚ โ”‚ โ””โ”€โ”€ models/ # Pre-trained models +โ”‚ โ”‚ โ”œโ”€โ”€ yolov8s.onnx # YOLO ONNX model +โ”‚ โ”‚ โ””โ”€โ”€ yolov8s.pt # YOLO PyTorch model +โ”‚ โ”œโ”€โ”€ agent/ # AI Agent Interface +โ”‚ โ”‚ โ”œโ”€โ”€ vision_tools.py # Function interface for AI agents +โ”‚ โ”‚ โ”œโ”€โ”€ function_definitions.py # JSON schemas for AI frameworks +โ”‚ โ”‚ โ”œโ”€โ”€ examples.py # Usage examples and demos +โ”‚ โ”‚ โ””โ”€โ”€ mcp/ # Model Context Protocol Server +โ”‚ โ”‚ โ”œโ”€โ”€ __init__.py # MCP server interface exports +โ”‚ โ”‚ โ”œโ”€โ”€ handlers.py # MCP tool handlers +โ”‚ โ”‚ โ”œโ”€โ”€ mcp_server.py # Model Context Protocol server +โ”‚ โ”‚ โ””โ”€โ”€ tools.py # Tool definitions for LLMs +โ”‚ โ”œโ”€โ”€ vm/ # VM connection management +โ”‚ โ”‚ โ”œโ”€โ”€ connections/ # Connection implementations +โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ base.py # Abstract base classes +โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ vnc_connection.py # VNC implementation +โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ rdp_connection.py # RDP implementation +โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ desktop_connection.py # Local desktop connection +โ”‚ โ”‚ โ”œโ”€โ”€ tools/ # Screen capture and input actions +โ”‚ โ”‚ โ””โ”€โ”€ automation/ # VM-specific agent implementations +โ”‚ โ”‚ โ”œโ”€โ”€ vm_navigator.py # Agent 1: VM connection & navigation +โ”‚ โ”‚ โ””โ”€โ”€ app_controller.py # Agent 2: GUI interactions +โ”œโ”€โ”€ automation/ # Local Desktop Automation +โ”‚ โ”œโ”€โ”€ desktop_control.py # Native macOS automation (AppleScript) +โ”‚ โ””โ”€โ”€ form_interface.py # High-level form filling interface โ”œโ”€โ”€ tests/ # Test suite โ”‚ โ”œโ”€โ”€ mock_components.py # Mock implementations for testing โ”‚ โ”œโ”€โ”€ test_integration.py # Integration tests @@ -441,7 +893,7 @@ This testing approach ensures system reliability while providing flexibility for ```bash # Re-download AI models -uv run src/setup_models.py +uv run src/ocr/setup_models.py ``` ## ๐Ÿณ Docker Support @@ -523,11 +975,236 @@ docker run --rm -it \ | Command | Purpose | Use Case | |---------|---------|----------| | `uv run src/main.py` | Direct script execution | Development & testing | +| `uv run src/vm/main.py` | VM-specific entry point | VM automation only | +| `uv run src/ocr/setup_models.py` | Download AI models | Initial setup | | `uv run vm-automation` | Production CLI | Production deployment | | `uv run vm-automation --connection rdp` | Use RDP connection | Windows VMs with RDP | | `uv run vm-automation --connection vnc` | Use VNC connection | Any VM with VNC server | | `uv run vm-automation --create-samples` | Generate config files | First-time setup | | `uv run vm-automation --validate-env` | Environment check | Troubleshooting | +| `uv run vm-automation --health-check` | System health check | Monitoring | +| `uv run vm-automation --benchmark` | Performance benchmark | Optimization | + +## ๐Ÿ”ง Production Deployment + +### Environment Configuration + +**Production Environment Variables:** +```bash +# Core Configuration +export COMPUTER_USE_AGENT_ENV="production" +export LOG_LEVEL="INFO" +export MAX_CONCURRENT_SESSIONS=5 +export SESSION_TIMEOUT=3600 + +# Security +export ENABLE_AUDIT_LOGGING="true" +export AUDIT_LOG_PATH="/var/log/computer-use-agent/audit.log" +export ENCRYPT_SCREENSHOTS="true" + +# Performance +export OCR_CONFIDENCE_THRESHOLD="0.7" +export YOLO_CONFIDENCE_THRESHOLD="0.6" +export SCREENSHOT_CACHE_SIZE=100 +export ENABLE_GPU_ACCELERATION="false" # CPU-only for consistency + +# Healthcare/HIPAA +export HIPAA_COMPLIANCE_MODE="true" +export PHI_LOGGING_ENABLED="false" +export PATIENT_VERIFICATION_REQUIRED="true" +``` + +### Docker Production Setup + +**Multi-stage Production Dockerfile:** +```dockerfile +FROM python:3.11-slim as production + +# System dependencies +RUN apt-get update && apt-get install -y \ + libgl1-mesa-glx \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +# Install UV package manager +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv + +# Copy application +WORKDIR /app +COPY . . + +# Install dependencies and download models +RUN uv sync --frozen +RUN uv run src/ocr/setup_models.py + +# Create non-root user +RUN useradd -m -u 1000 agent +RUN chown -R agent:agent /app +USER agent + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD uv run vm-automation --health-check + +EXPOSE 8000 +CMD ["uv", "run", "vm-automation", "--production"] +``` + +### Monitoring and Observability + +**Health Check Endpoint:** +```python +from agent.mcp import create_mcp_server + +# Add health monitoring +server = create_mcp_server( + enable_metrics=True, + metrics_port=9090, + health_check_endpoint="/health" +) + +# Custom health checks +def custom_health_check(): + """Custom health validation""" + try: + # Test OCR functionality + from ocr import detect_ui_elements + import numpy as np + test_image = np.zeros((100, 100, 3), dtype=np.uint8) + detect_ui_elements(test_image) + + # Test model availability + from ocr.setup_models import verify_models + models_status = verify_models() + + return { + "status": "healthy", + "ocr_functional": True, + "models_loaded": all(models_status.values()), + "timestamp": time.time() + } + except Exception as e: + return { + "status": "unhealthy", + "error": str(e), + "timestamp": time.time() + } +``` + +**Logging Configuration:** +```python +import logging +from logging.handlers import RotatingFileHandler + +# Production logging setup +def setup_production_logging(): + """Configure production-grade logging""" + + # Main application logger + logger = logging.getLogger("computer_use_agent") + logger.setLevel(logging.INFO) + + # Rotating file handler (100MB max, keep 5 files) + file_handler = RotatingFileHandler( + "logs/app.log", + maxBytes=100*1024*1024, + backupCount=5 + ) + file_handler.setFormatter(logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + )) + logger.addHandler(file_handler) + + # Audit logger (for compliance) + audit_logger = logging.getLogger("audit") + audit_handler = RotatingFileHandler( + "logs/audit.log", + maxBytes=50*1024*1024, + backupCount=10 + ) + audit_handler.setFormatter(logging.Formatter( + '%(asctime)s - AUDIT - %(message)s' + )) + audit_logger.addHandler(audit_handler) + + return logger, audit_logger +``` + +## ๐Ÿ“Š Performance Optimization + +### Benchmarking + +```bash +# Run performance benchmark +uv run vm-automation --benchmark + +# OCR-specific benchmarking +uv run python -c " +from ocr.setup_models import verify_models +from ocr import detect_ui_elements, extract_text +import time +import numpy as np + +# Generate test image +test_image = np.random.randint(0, 255, (1080, 1920, 3), dtype=np.uint8) + +# Benchmark detection +start_time = time.time() +elements = detect_ui_elements(test_image) +detection_time = time.time() - start_time + +# Benchmark OCR +start_time = time.time() +text_results = extract_text(test_image) +ocr_time = time.time() - start_time + +print(f'YOLO Detection: {detection_time:.2f}s') +print(f'OCR Processing: {ocr_time:.2f}s') +print(f'Total Elements Found: {len(elements)}') +print(f'Text Regions Found: {len(text_results)}') +" +``` + +### Resource Usage Monitoring + +```python +import psutil +import time +from dataclasses import dataclass + +@dataclass +class PerformanceMetrics: + cpu_percent: float + memory_mb: float + screenshot_fps: float + ocr_latency_ms: float + detection_latency_ms: float + +def monitor_performance(): + """Monitor system performance""" + process = psutil.Process() + + # CPU and Memory + cpu_percent = process.cpu_percent(interval=1) + memory_mb = process.memory_info().rss / 1024 / 1024 + + # Vision performance (simplified) + start_time = time.time() + # ... perform OCR operations ... + ocr_latency = (time.time() - start_time) * 1000 + + return PerformanceMetrics( + cpu_percent=cpu_percent, + memory_mb=memory_mb, + screenshot_fps=30.0, # Example + ocr_latency_ms=ocr_latency, + detection_latency_ms=150.0 # Example + ) +``` --- @@ -539,4 +1216,3 @@ This system is ready for real-world deployment with: - **Two-Agent Architecture**: Efficient component sharing - **Healthcare Safety**: Optional patient verification - **Production Features**: Configuration management, error handling, audit logging -- **Clean Architecture**: No mock code in production paths diff --git a/demo_form_automation.py b/demo_form_automation.py new file mode 100644 index 0000000..79b7c4c --- /dev/null +++ b/demo_form_automation.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +""" +Demo: Form Automation with OCR + Local Desktop Control + +This demonstrates the complete integration: +1. OCR (ocr) - Finds form elements by text +2. Automation (automation) - Clicks and types on local desktop +3. VM (vm) - For remote desktop control (when needed) + +The clean separation allows form entry to work both locally and remotely. +""" + +import sys +from pathlib import Path + +# Add src to path +sys.path.append(str(Path(__file__).parent / "src")) + +from automation import DesktopControl, FormFiller +from ocr import find_elements_by_text + + +def demo_low_level_automation(): + """Demo low-level desktop control""" + print("\n๐Ÿ–ฑ๏ธ Low-Level Desktop Automation Demo") + print("=" * 40) + + desktop = DesktopControl() + + print("1. Capturing desktop screenshot...") + success, screenshot = desktop.capture_screen() + if success: + print(f"โœ… Screenshot captured: {screenshot.shape}") + else: + print("โŒ Failed to capture screenshot") + return + + print("\n2. Testing desktop info...") + info = desktop.get_desktop_info() + print(f"โœ… Platform: {info['platform']}") + print(f"โœ… Click capability: {info['click_capability']}") + print(f"โœ… Cliclick available: {info['cliclick_available']}") + + +def demo_ocr_element_finding(): + """Demo OCR-based element detection""" + print("\n๐Ÿ” OCR Element Detection Demo") + print("=" * 35) + + desktop = DesktopControl() + + print("1. Capturing current screen...") + success, screenshot = desktop.capture_screen() + if not success: + print("โŒ Could not capture screen") + return + + print("2. Searching for common UI elements...") + common_elements = [ + "File", + "Edit", + "View", + "Window", + "Help", # Menu items + "Desktop", + "Finder", + "Applications", # macOS elements + "Search", + "Login", + "Username", + "Password", # Common form elements + ] + + found_elements = [] + + for element_text in common_elements: + try: + elements = find_elements_by_text( + screenshot, element_text, confidence_threshold=0.5, case_sensitive=False + ) + + if elements: + best_match = max(elements, key=lambda x: x.confidence) + found_elements.append( + { + "text": element_text, + "detected": best_match.text, + "center": best_match.center, + "confidence": best_match.confidence, + } + ) + except Exception as e: + print(f" Error searching for '{element_text}': {e}") + + print(f"โœ… Found {len(found_elements)} UI elements:") + for elem in found_elements[:5]: # Show first 5 + print( + f" - '{elem['detected']}' at {elem['center']} (confidence: {elem['confidence']:.2f})" + ) + + +def demo_high_level_form_filling(): + """Demo high-level form filling interface""" + print("\n๐Ÿ“ High-Level Form Filling Demo") + print("=" * 35) + + form_filler = FormFiller() + + print("1. Scanning for form fields on current screen...") + detected_fields = form_filler.get_form_fields() + + if detected_fields: + print(f"โœ… Found {len(detected_fields)} potential form fields:") + for field in detected_fields: + print( + f" - '{field['text']}' at {field['center']} (confidence: {field['confidence']:.2f})" + ) + else: + print("โ„น๏ธ No form fields detected on current screen") + print(" (This is normal if no forms are visible)") + + print("\n2. Testing element search capabilities...") + test_elements = ["Finder", "Desktop", "Applications"] + + for element in test_elements: + field_info = form_filler.find_field_by_label(element, confidence_threshold=0.4) + if field_info: + print(f"โœ… Found '{element}' at {field_info['center']}") + else: + print(f" '{element}' not found") + + +def demo_vm_vs_automation_comparison(): + """Show the difference between VM control and local automation""" + print("\n๐Ÿ–ฅ๏ธ VM vs Local Automation Comparison") + print("=" * 42) + + print("LOCAL AUTOMATION (src/automation/):") + print(" โ”œโ”€โ”€ Uses AppleScript & cliclick") + print(" โ”œโ”€โ”€ Direct macOS desktop control") + print(" โ”œโ”€โ”€ Perfect for local form filling") + print(" โ””โ”€โ”€ Example: FormFiller, DesktopControl") + + print("\nREMOTE VM CONTROL (src/vm/):") + print(" โ”œโ”€โ”€ Uses VNC/RDP protocols") + print(" โ”œโ”€โ”€ Controls remote machines") + print(" โ”œโ”€โ”€ Network-based automation") + print(" โ””โ”€โ”€ Example: vm.click(), vnc_connection") + + print("\nFORM ENTRY WORKFLOW:") + print(" 1. ๐Ÿ” OCR finds field locations") + print(" 2. ๐Ÿค– Choose automation method:") + print(" - Local: automation.click(x, y)") + print(" - Remote: vm.click(x, y)") + print(" 3. โŒจ๏ธ Type text into fields") + print(" 4. โœ… Submit form") + + +def main(): + """Run complete automation demo""" + print("๐Ÿš€ Complete Form Automation Architecture Demo") + print("๐Ÿ“ Clean Separation: OCR + Automation + VM") + print("=" * 60) + + try: + # Demo low-level automation + demo_low_level_automation() + + # Demo OCR capabilities + demo_ocr_element_finding() + + # Demo high-level form interface + demo_high_level_form_filling() + + # Show architectural comparison + demo_vm_vs_automation_comparison() + + print("\n๐ŸŽ‰ Demo completed successfully!") + + print("\n๐Ÿ“‹ Clean Architecture Summary:") + print(" โ”œโ”€โ”€ ๐Ÿ” src/ocr/ # Pure computer vision") + print(" โ”œโ”€โ”€ ๐Ÿ–ฑ๏ธ src/automation/ # Local desktop control") + print(" โ”œโ”€โ”€ ๐Ÿ–ฅ๏ธ src/vm/ # Remote desktop control") + print(" โ””โ”€โ”€ ๐Ÿค– src/agent/mcp/ # MCP server interface") + + print("\n๐Ÿ’ก Ready for both local and remote form automation!") + + except KeyboardInterrupt: + print("\n๐Ÿ‘‹ Demo interrupted") + except Exception as e: + print(f"โŒ Demo error: {e}") + import traceback + + traceback.print_exc() + + +if __name__ == "__main__": + main() diff --git a/demo_mcp_integration.py b/demo_mcp_integration.py new file mode 100644 index 0000000..afdaa80 --- /dev/null +++ b/demo_mcp_integration.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +""" +Demo: Complete MCP Server Integration + +This script demonstrates the clean architecture with: +1. Pure OCR functions (ocr package) +2. MCP server interface (agent.mcp package) +3. End-to-end computer vision tool integration + +Run this demo to test the complete system. +""" + +import asyncio +import base64 +import json +import sys +from pathlib import Path + +# Add src to path +sys.path.append(str(Path(__file__).parent / "src")) + +import cv2 +import numpy as np + +from agent.mcp import create_mcp_server +from ocr import extract_text + + +def create_demo_image() -> np.ndarray: + """Create a demo image with text and shapes for testing""" + img = np.ones((200, 600, 3), dtype=np.uint8) * 255 + + # Add some text + cv2.putText(img, "Computer Vision Demo", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2) + cv2.putText( + img, "MCP Server Integration", (50, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2 + ) + cv2.putText(img, "Button", (250, 150), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2) + + # Add some shapes (simulating UI elements) + cv2.rectangle(img, (200, 120), (350, 170), (100, 100, 200), -1) # Button + cv2.rectangle(img, (400, 50), (550, 90), (200, 200, 200), 2) # Input field outline + + return img + + +def image_to_base64(image: np.ndarray) -> str: + """Convert image to base64 string""" + _, buffer = cv2.imencode(".png", image) + return base64.b64encode(buffer).decode("utf-8") + + +async def test_mcp_tools(): + """Test all MCP tools with demo image""" + print("๐Ÿ” Computer Vision MCP Server Demo") + print("=" * 50) + + # Create demo image + demo_image = create_demo_image() + base64_data = image_to_base64(demo_image) + + # Create MCP server + server = create_mcp_server() + mcp = server.server + + print("\n1. Testing MCP Server Initialization...") + init_response = await mcp.handle_request( + {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}} + ) + print(f"โœ… Server initialized (protocol: {init_response['result']['protocolVersion']})") + + print("\n2. Listing Available Tools...") + tools_response = await mcp.handle_request( + {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}} + ) + tools = tools_response["result"]["tools"] + print(f"โœ… Found {len(tools)} tools:") + for tool in tools: + print(f" - {tool['name']}: {tool['description']}") + + print("\n3. Testing Text Extraction...") + text_response = await mcp.handle_request( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "extract_text", + "arguments": {"image_base64": base64_data, "confidence_threshold": 0.5}, + }, + } + ) + + text_result = json.loads(text_response["result"]["content"][0]["text"]) + print(f"โœ… Text extraction: {text_result['success']}") + print(f" Found {text_result['total_found']} text elements:") + for text_item in text_result["text_results"][:3]: # Show first 3 + print(f' "{text_item["text"]}" (confidence: {text_item["confidence"]:.3f})') + + print("\n4. Testing Element Search...") + search_response = await mcp.handle_request( + { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": { + "name": "find_elements_by_text", + "arguments": { + "image_base64": base64_data, + "text_query": "Button", + "confidence_threshold": 0.5, + }, + }, + } + ) + + search_result = json.loads(search_response["result"]["content"][0]["text"]) + print(f"โœ… Element search: {search_result['success']}") + print(f" Found {search_result['total_found']} elements matching 'Button'") + + print("\n5. Testing Screen Analysis...") + analysis_response = await mcp.handle_request( + { + "jsonrpc": "2.0", + "id": 5, + "method": "tools/call", + "params": { + "name": "analyze_screen_content", + "arguments": { + "image_base64": base64_data, + "query": "Find all interactive elements and text on this demo screen", + }, + }, + } + ) + + analysis_result = json.loads(analysis_response["result"]["content"][0]["text"]) + print(f"โœ… Screen analysis: {analysis_result['success']}") + if analysis_result["success"]: + analysis = analysis_result["analysis"] + print(f" Summary: {analysis['summary'][:100]}...") + print(f" UI Elements: {len(analysis['ui_elements'])}") + print(f" Text Elements: {len(analysis['text_elements'])}") + print(f" Clickable Elements: {len(analysis['clickable_elements'])}") + + +def test_pure_ocr_functions(): + """Test pure OCR functions directly (no MCP)""" + print("\n๐Ÿ”ง Testing Pure OCR Functions") + print("=" * 30) + + demo_image = create_demo_image() + + print("1. Direct text extraction...") + text_results = extract_text(demo_image, confidence_threshold=0.5) + print(f"โœ… Found {len(text_results)} text elements") + for result in text_results[:2]: + print(f' "{result.text}" at {result.center}') + + +async def main(): + """Run complete demo""" + print("๐Ÿš€ Clean Computer Vision Architecture Demo") + print("๐Ÿ” OCR Functions + ๐Ÿค– MCP Server Interface") + print("=" * 60) + + # Test pure OCR functions + test_pure_ocr_functions() + + # Test MCP integration + await test_mcp_tools() + + print("\n๐ŸŽ‰ Demo completed successfully!") + print("\n๐Ÿ“‹ Architecture Summary:") + print(" โ””โ”€โ”€ src/") + print(" โ”œโ”€โ”€ ocr/ # Pure computer vision functions") + print(" โ””โ”€โ”€ agent/mcp/ # MCP server interface") + print("\n๐Ÿ’ก The system is ready for LLM integration via MCP protocol!") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n๐Ÿ‘‹ Demo interrupted") + except Exception as e: + print(f"โŒ Demo failed: {e}") + import traceback + + traceback.print_exc() diff --git a/pyproject.toml b/pyproject.toml index 6d07d0c..72d00fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,26 +17,25 @@ classifiers = [ # Runtime deps dependencies = [ - "openai>=1.52.0", - "openai-agents>=0.2.9", # Agents SDK - # VM Automation POC dependencies - "ultralytics>=8.0.0", - "onnxruntime>=1.16.0", - "paddlepaddle>=2.5.0", - "paddleocr>=2.7.0", - "opencv-python>=4.8.0", - "vncdotool>=1.2.0", - "fastapi>=0.104.0", - "uvicorn>=0.24.0", - "pillow>=10.0.0", - "numpy>=1.24.0", + "fastapi>=0.116.1", + "numpy>=2.3.2", "onnx>=1.19.0", + "onnxruntime>=1.22.1", + "openai>=1.105.0", + "openai-agents>=0.2.11", + "opencv-python>=4.11.0.86", + "paddleocr[all]>=3.2.0", + "paddlepaddle>=3.1.1", + "pillow>=11.3.0", + "ultralytics>=8.3.193", + "uvicorn>=0.35.0", + "vncdotool>=1.2.0", ] # Dev / tooling [project.scripts] -vm-automation = "src.main:cli_main" +vm-automation = "vm.main:cli_main" [tool.ruff] line-length = 100 @@ -57,7 +56,6 @@ testpaths = ["tests"] markers = [ "integration: marks tests as integration tests (require VM connection)", "mock: marks tests as using mock components (no VM required)", - "slow: marks tests as slow running", "vnc: marks tests as VNC-specific integration tests", "rdp: marks tests as RDP-specific integration tests", "patient_workflow: marks tests as patient workflow tests (healthcare applications)", @@ -66,21 +64,19 @@ markers = [ ] [tool.hatch.build.targets.wheel] -packages = ["src"] -include = ["src/**"] +packages = ["src/automation", "src/agent", "src/ocr", "src/vm"] [tool.uv] default-groups = ["dev"] [dependency-groups] dev = [ - "pytest>=8.3.0", - "pytest-cov>=5.0.0", - "pytest-asyncio>=0.23.0", - "ruff>=0.5.7", - "pyright>=1.1.379", - "pre-commit>=3.7.1", - # POC evaluation dependencies - "deepeval>=0.21.0", - "arize-phoenix>=4.0.0", + "arize-phoenix>=11.32.1", + "deepeval>=3.4.7", + "pre-commit>=4.3.0", + "pyright>=1.1.405", + "pytest>=8.4.2", + "pytest-asyncio>=1.1.0", + "pytest-cov>=6.2.1", + "ruff>=0.12.12", ] diff --git a/src/agent/examples.py b/src/agent/examples.py new file mode 100644 index 0000000..0b8516d --- /dev/null +++ b/src/agent/examples.py @@ -0,0 +1,214 @@ +"""Example Usage of AI Agent Vision Tools + +Demonstrates how to use the computer vision automation tools +in various scenarios and AI agent frameworks. +""" + +from agent.vision_tools import ( + analyze_screen, + click_element, + configure_vision_tools, + find_element, + take_screenshot, + type_text_in_field, + verify_action, + wait_for_element, +) + + +def simple_form_automation_example(): + """Example: Automate filling out a simple form""" + print("=== Simple Form Automation Example ===") + + # 1. Analyze the current screen + analysis = analyze_screen("What form fields and buttons are visible?") + print(f"Found {analysis['total_elements']} UI elements") + print(f"Text regions: {analysis['total_text_regions']}") + if "summary" in analysis: + print(f"Summary: {analysis['summary']}") + + # 2. Find and fill username field + username_field = find_element("username field") + if username_field: + print("Found username field") + result = type_text_in_field("john.doe", username_field) + print(f"Username entry: {result['success']}") + + # 3. Find and fill password field + password_field = find_element("password field") + if password_field: + print("Found password field") + result = type_text_in_field("secure_password", password_field) + print(f"Password entry: {result['success']}") + + # 4. Find and click submit button + submit_button = find_element("submit button") + if submit_button: + print("Found submit button") + result = click_element(submit_button) + print(f"Submit click: {result['success']}") + + # 5. Verify the form was submitted + verification = verify_action("form should be submitted successfully") + print(f"Verification: {verification['success']} - {verification['message']}") + + +def web_automation_example(): + """Example: Automate web browsing tasks""" + print("=== Web Automation Example ===") + + # Take initial screenshot + screenshot_path = "/tmp/initial_screen.png" + screenshot = take_screenshot(screenshot_path) + print(f"Screenshot saved to {screenshot_path}") + + # Search for a search box + search_box = find_element("search box") + if search_box: + # Type search query + type_text_in_field("computer vision automation", search_box) + + # Find and click search button + search_button = find_element("search button") + if search_button: + click_element(search_button) + + # Wait for results to load + results = wait_for_element("search results", max_attempts=10) + if results.get("success", False): + print("Search completed successfully") + + # Verify results appeared + verification = verify_action("search results should be displayed") + print(f"Results verification: {verification['success']}") + + +def settings_navigation_example(): + """Example: Navigate through settings menus""" + print("=== Settings Navigation Example ===") + + # 1. Find and click settings menu + settings = find_element("settings menu") + if settings: + click_element(settings) + + # Wait for settings dialog to open + settings_dialog = wait_for_element("settings dialog") + if settings_dialog: + print("Settings dialog opened") + + # 2. Find a specific setting + privacy_setting = find_element("privacy settings") + if privacy_setting: + click_element(privacy_setting) + + # 3. Verify we're in privacy settings + verification = verify_action("privacy settings should be visible") + print(f"Privacy settings: {verification['success']}") + + +def error_handling_example(): + """Example: Demonstrate error handling and recovery""" + print("=== Error Handling Example ===") + + try: + # Try to find an element that doesn't exist + missing_element = find_element("nonexistent button") + if missing_element is None: + print("Element not found - handling gracefully") + + # Alternative approach: analyze screen for available options + analysis = analyze_screen("What buttons and options are available?") + if "error" not in analysis: + available_elements = [elem for elem in analysis["ui_elements"] if elem.get("text")] + + print("Available text elements:") + for elem in available_elements[:5]: # Show first 5 + print(f" - {elem['text']} ({elem['type']})") + else: + print(f"Analysis failed: {analysis['error']}") + + except Exception as e: + print(f"Error occurred: {e}") + print("Taking screenshot for debugging...") + debug_screenshot = take_screenshot("/tmp/debug_screen.png") + + +def ai_agent_integration_example(): + """Example: How to integrate with AI agent frameworks""" + print("=== AI Agent Integration Example ===") + + # This shows how an AI agent would use the tools + + def simulate_ai_agent_task(user_request: str): + """Simulate how an AI agent would handle a user request""" + print(f"AI Agent received request: {user_request}") + + if "fill form" in user_request.lower(): + # AI agent would call these functions based on the request + analysis = analyze_screen("Analyze form fields and buttons") + + # AI would interpret the analysis and decide next steps + if analysis.get("total_text_regions", 0) > 0 or analysis.get("total_elements", 0) > 0: + print("AI: I see there are UI elements. Let me interact with them.") + + # Find first field and fill it + first_field = find_element("first input field") + if first_field: + type_text_in_field("AI-generated content", first_field) + + elif "take screenshot" in user_request.lower(): + screenshot = take_screenshot("/tmp/ai_screenshot.png") + analysis = analyze_screen("Describe what's visible on screen") + if "error" not in analysis: + print(f"AI: I took a screenshot and found {analysis['total_elements']} elements") + else: + print(f"AI: Screenshot analysis failed: {analysis['error']}") + + return "Task completed by AI agent" + + # Simulate different requests + simulate_ai_agent_task("Please take a screenshot and tell me what you see") + simulate_ai_agent_task("Help me fill out this form") + + +def configuration_example(): + """Example: Configure vision tools for different scenarios""" + print("=== Configuration Example ===") + + # Standard configuration for general use + configure_vision_tools(confidence_threshold=0.6, ocr_language="en") + print("Configured for general UI automation") + + # High-precision configuration for important tasks + configure_vision_tools(confidence_threshold=0.8, ocr_language="en") + print("Configured for high-precision tasks") + + # Configuration for diverse content detection + configure_vision_tools( + confidence_threshold=0.5, + ocr_language="en", + ) + print("Configured for diverse content detection") + + +if __name__ == "__main__": + print("Computer Vision AI Agent Tools - Examples") + print("=" * 50) + + try: + # Run examples (commented out to avoid actual automation) + print("Running examples in simulation mode...") + print("(Actual automation would require proper screen setup)") + + # Uncomment these to run actual automation: + # simple_form_automation_example() + # web_automation_example() + # settings_navigation_example() + error_handling_example() + ai_agent_integration_example() + configuration_example() + + except Exception as e: + print(f"Example execution error: {e}") + print("This is expected when running without proper screen setup") diff --git a/src/agent/function_definitions.py b/src/agent/function_definitions.py new file mode 100644 index 0000000..5845043 --- /dev/null +++ b/src/agent/function_definitions.py @@ -0,0 +1,238 @@ +"""Function Tool Definitions for AI Agents + +This module provides JSON schema definitions for AI agent function tools, +compatible with OpenAI Functions, Claude Tools, and other AI frameworks. + +These can be imported and used directly in AI agent configurations. +""" + +from typing import Any + + +def get_vision_function_tools() -> list[dict[str, Any]]: + """ + Get function tool definitions for computer vision automation + + Returns: + List of function definitions compatible with AI frameworks + """ + return [ + { + "type": "function", + "function": { + "name": "analyze_screen", + "description": "Analyze the current screen and identify UI elements, text, and interactive components", + "parameters": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "Natural language description of what to analyze on the screen", + } + }, + "required": ["prompt"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "find_element", + "description": "Find a specific UI element on the screen by description", + "parameters": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "Natural language description of the element to find (e.g., 'Submit button', 'Username field', 'Settings menu')", + } + }, + "required": ["description"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "click_element", + "description": "Click on a UI element or specific coordinates", + "parameters": { + "type": "object", + "properties": { + "element": { + "type": ["object", "array"], + "description": "Element object from find_element() or [x, y] coordinates", + }, + "button": { + "type": "string", + "enum": ["left", "right", "middle"], + "default": "left", + "description": "Mouse button to click", + }, + }, + "required": ["element"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "type_text_in_field", + "description": "Type text in an input field, optionally clicking the field first", + "parameters": { + "type": "object", + "properties": { + "text": {"type": "string", "description": "Text to type"}, + "field_element": { + "type": "object", + "description": "Optional field element to click first (from find_element)", + }, + }, + "required": ["text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "verify_action", + "description": "Verify that an action had the expected outcome", + "parameters": { + "type": "object", + "properties": { + "expected_outcome": { + "type": "string", + "description": "Natural language description of what should have happened", + } + }, + "required": ["expected_outcome"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "wait_for_element", + "description": "Wait for a specific element to appear on screen", + "parameters": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "Description of element to wait for", + }, + "max_attempts": { + "type": "integer", + "default": 10, + "description": "Maximum number of attempts", + }, + "delay": { + "type": "number", + "default": 1.0, + "description": "Delay between attempts in seconds", + }, + }, + "required": ["description"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "scroll_screen", + "description": "Scroll the screen in a specified direction", + "parameters": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": ["up", "down", "left", "right"], + "default": "down", + "description": "Direction to scroll", + }, + "pixels": { + "type": "integer", + "default": 400, + "description": "Number of pixels to scroll", + }, + }, + }, + }, + }, + { + "type": "function", + "function": { + "name": "take_screenshot", + "description": "Take a screenshot of the current desktop", + "parameters": { + "type": "object", + "properties": { + "save_path": { + "type": "string", + "description": "Optional path to save the screenshot", + } + }, + }, + }, + }, + ] + + +# Example usage for different AI frameworks +OPENAI_FUNCTIONS_EXAMPLE = """ +# OpenAI Functions usage example: + +from openai import OpenAI +from agent.function_definitions import get_vision_function_tools +from agent.vision_tools import * + +client = OpenAI() + +tools = get_vision_function_tools() + +response = client.chat.completions.create( + model="gpt-4", + messages=[ + {"role": "user", "content": "Please click on the Submit button on my screen"} + ], + tools=tools, + tool_choice="auto" +) + +# Handle function calls +if response.choices[0].message.tool_calls: + for tool_call in response.choices[0].message.tool_calls: + if tool_call.function.name == "find_element": + element = find_element("Submit button") + elif tool_call.function.name == "click_element": + result = click_element(element) +""" + +CLAUDE_TOOLS_EXAMPLE = """ +# Claude Tools usage example: + +import anthropic +from agent.function_definitions import get_vision_function_tools +from agent.vision_tools import * + +client = anthropic.Anthropic() + +tools = get_vision_function_tools() + +response = client.messages.create( + model="claude-3-opus-20240229", + max_tokens=1000, + tools=tools, + messages=[ + {"role": "user", "content": "Please help me fill out this form on my screen"} + ] +) + +# Handle tool usage +for content in response.content: + if content.type == "tool_use": + if content.name == "analyze_screen": + result = analyze_screen(content.input["prompt"]) + elif content.name == "find_element": + element = find_element(content.input["description"]) +""" diff --git a/src/agent/mcp/__init__.py b/src/agent/mcp/__init__.py new file mode 100644 index 0000000..0ac622f --- /dev/null +++ b/src/agent/mcp/__init__.py @@ -0,0 +1,47 @@ +"""MCP Server Interface for Computer Vision + +Provides a clean Model Context Protocol (MCP) server that exposes +computer vision functions as tools for LLMs. + +This module acts as the interface layer between LLMs and the pure OCR functions, +handling: +- MCP protocol implementation +- Image data encoding/decoding (base64) +- Function tool definitions for LLMs +- Error handling and response formatting + +Example usage: + from agent.mcp import create_mcp_server, get_function_tools + + # Get function tools for LLM integration + tools = get_function_tools() + + # Create and run MCP server + server = create_mcp_server() + server.run() +""" + +from .handlers import ( + handle_analyze_screen, + handle_detect_elements, + handle_extract_text, + handle_find_elements, +) +from .mcp_server import create_mcp_server, start_server +from .tools import get_function_tools, get_tool_schemas + +__version__ = "1.0.0" + +__all__ = [ + # MCP server functions + "create_mcp_server", + "start_server", + # Tool definitions + "get_function_tools", + "get_tool_schemas", + # Tool handlers + "handle_detect_elements", + "handle_extract_text", + "handle_find_elements", + "handle_analyze_screen", +] diff --git a/src/agent/mcp/handlers.py b/src/agent/mcp/handlers.py new file mode 100644 index 0000000..0ce2ca5 --- /dev/null +++ b/src/agent/mcp/handlers.py @@ -0,0 +1,394 @@ +"""Tool Execution Handlers for MCP Server + +Handles execution of computer vision function tools, converting between +MCP protocol (base64 images) and pure OCR functions (numpy arrays). +""" + +import base64 +import io + +# Import pure OCR functions +import sys +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np +from PIL import Image + +sys.path.append(str(Path(__file__).parent.parent)) + +from ocr import ( + analyze_screen_content, + detect_ui_elements, + extract_text, + extract_text_from_region, + find_clickable_elements, + find_elements_by_text, +) + +from .tools import validate_tool_parameters + + +def decode_base64_image(base64_data: str) -> np.ndarray: + """ + Convert base64 image data to OpenCV numpy array + + Args: + base64_data: Base64 encoded image string + + Returns: + Image as numpy array in BGR format (OpenCV format) + + Raises: + ValueError: If image data is invalid + """ + try: + # Remove data URL prefix if present + if base64_data.startswith("data:image"): + base64_data = base64_data.split(",", 1)[1] + + # Decode base64 + image_bytes = base64.b64decode(base64_data) + + # Convert to PIL Image + pil_image = Image.open(io.BytesIO(image_bytes)) + + # Convert to RGB if needed + if pil_image.mode != "RGB": + pil_image = pil_image.convert("RGB") + + # Convert to numpy array + image_array = np.array(pil_image) + + # Convert RGB to BGR for OpenCV + image_bgr = cv2.cvtColor(image_array, cv2.COLOR_RGB2BGR) + + return image_bgr + + except Exception as e: + raise ValueError(f"Failed to decode image: {e!s}") + + +def handle_detect_elements(parameters: dict[str, Any]) -> dict[str, Any]: + """ + Handle detect_ui_elements tool execution + + Args: + parameters: Tool parameters from MCP call + + Returns: + Detection results formatted for MCP response + """ + try: + # Validate parameters + validated_params = validate_tool_parameters("detect_ui_elements", parameters) + + # Decode image + image = decode_base64_image(validated_params["image_base64"]) + + # Call OCR function + detections = detect_ui_elements( + image=image, + confidence_threshold=validated_params.get("confidence_threshold", 0.6), + ui_focused=validated_params.get("ui_focused", True), + max_detections=validated_params.get("max_detections", 50), + ) + + # Format response + formatted_detections = [] + for detection in detections: + formatted_detections.append( + { + "class_name": detection.class_name, + "confidence": float(detection.confidence), + "bbox": list(detection.bbox), + "center": list(detection.center), + "area": detection.area, + } + ) + + return { + "success": True, + "detections": formatted_detections, + "total_found": len(formatted_detections), + "image_dimensions": {"height": image.shape[0], "width": image.shape[1]}, + } + + except Exception as e: + return {"success": False, "error": str(e), "detections": [], "total_found": 0} + + +def handle_extract_text(parameters: dict[str, Any]) -> dict[str, Any]: + """ + Handle extract_text tool execution + + Args: + parameters: Tool parameters from MCP call + + Returns: + Text extraction results formatted for MCP response + """ + try: + # Validate parameters + validated_params = validate_tool_parameters("extract_text", parameters) + + # Decode image + image = decode_base64_image(validated_params["image_base64"]) + + # Check if region is specified + region = validated_params.get("region") + if region: + # Extract text from specific region + text_results = extract_text_from_region( + image=image, + region=tuple(region), + language=validated_params.get("language", "en"), + confidence_threshold=validated_params.get("confidence_threshold", 0.5), + ) + else: + # Extract text from entire image + text_results = extract_text( + image=image, + language=validated_params.get("language", "en"), + confidence_threshold=validated_params.get("confidence_threshold", 0.5), + max_results=validated_params.get("max_results", 100), + ) + + # Format response + formatted_results = [] + for text_result in text_results: + formatted_results.append( + { + "text": text_result.text, + "confidence": float(text_result.confidence), + "bbox": [list(point) for point in text_result.bbox], + "rect_bbox": list(text_result.rect_bbox), + "center": list(text_result.center), + "area": text_result.area, + } + ) + + return { + "success": True, + "text_results": formatted_results, + "total_found": len(formatted_results), + "image_dimensions": {"height": image.shape[0], "width": image.shape[1]}, + "language": validated_params.get("language", "en"), + } + + except Exception as e: + return {"success": False, "error": str(e), "text_results": [], "total_found": 0} + + +def handle_find_elements(parameters: dict[str, Any]) -> dict[str, Any]: + """ + Handle find_elements_by_text tool execution + + Args: + parameters: Tool parameters from MCP call + + Returns: + Element search results formatted for MCP response + """ + try: + # Validate parameters + validated_params = validate_tool_parameters("find_elements_by_text", parameters) + + # Decode image + image = decode_base64_image(validated_params["image_base64"]) + + # Call OCR function + elements = find_elements_by_text( + image=image, + text_query=validated_params["text_query"], + confidence_threshold=validated_params.get("confidence_threshold", 0.6), + search_radius=validated_params.get("search_radius", 100), + case_sensitive=validated_params.get("case_sensitive", False), + ) + + # Format response + formatted_elements = [] + for element in elements: + element_data = { + "element_type": element.element_type, + "bbox": list(element.bbox), + "center": list(element.center), + "confidence": float(element.confidence), + "area": element.area, + "text": element.text, + "description": element.description, + } + + # Add visual detection info if available + if element.visual_detection: + element_data["visual_detection"] = { + "class_name": element.visual_detection.class_name, + "confidence": float(element.visual_detection.confidence), + } + + formatted_elements.append(element_data) + + return { + "success": True, + "elements": formatted_elements, + "total_found": len(formatted_elements), + "query": validated_params["text_query"], + "image_dimensions": {"height": image.shape[0], "width": image.shape[1]}, + } + + except Exception as e: + return {"success": False, "error": str(e), "elements": [], "total_found": 0} + + +def handle_analyze_screen(parameters: dict[str, Any]) -> dict[str, Any]: + """ + Handle analyze_screen_content tool execution + + Args: + parameters: Tool parameters from MCP call + + Returns: + Screen analysis results formatted for MCP response + """ + try: + # Validate parameters + validated_params = validate_tool_parameters("analyze_screen_content", parameters) + + # Decode image + image = decode_base64_image(validated_params["image_base64"]) + + # Call OCR function + analysis = analyze_screen_content( + image=image, + query=validated_params["query"], + confidence_threshold=validated_params.get("confidence_threshold", 0.6), + ) + + # Format UI elements + def format_element_list(elements): + formatted = [] + for element in elements: + element_data = { + "element_type": element.element_type, + "bbox": list(element.bbox), + "center": list(element.center), + "confidence": float(element.confidence), + "area": element.area, + "text": element.text, + "description": element.description, + } + + if element.visual_detection: + element_data["visual_detection"] = { + "class_name": element.visual_detection.class_name, + "confidence": float(element.visual_detection.confidence), + } + + formatted.append(element_data) + return formatted + + return { + "success": True, + "analysis": { + "ui_elements": format_element_list(analysis.ui_elements), + "text_elements": format_element_list(analysis.text_elements), + "visual_elements": format_element_list(analysis.visual_elements), + "clickable_elements": format_element_list(analysis.clickable_elements), + "summary": analysis.summary, + "query": analysis.query, + }, + "image_dimensions": {"height": image.shape[0], "width": image.shape[1]}, + } + + except Exception as e: + return {"success": False, "error": str(e), "analysis": None} + + +def handle_find_clickable(parameters: dict[str, Any]) -> dict[str, Any]: + """ + Handle find_clickable_elements tool execution + + Args: + parameters: Tool parameters from MCP call + + Returns: + Clickable elements results formatted for MCP response + """ + try: + # Validate parameters + validated_params = validate_tool_parameters("find_clickable_elements", parameters) + + # Decode image + image = decode_base64_image(validated_params["image_base64"]) + + # Call OCR function + elements = find_clickable_elements( + image=image, confidence_threshold=validated_params.get("confidence_threshold", 0.6) + ) + + # Format response + formatted_elements = [] + for element in elements: + element_data = { + "element_type": element.element_type, + "bbox": list(element.bbox), + "center": list(element.center), + "confidence": float(element.confidence), + "area": element.area, + "text": element.text, + "description": element.description, + } + + if element.visual_detection: + element_data["visual_detection"] = { + "class_name": element.visual_detection.class_name, + "confidence": float(element.visual_detection.confidence), + } + + formatted_elements.append(element_data) + + return { + "success": True, + "clickable_elements": formatted_elements, + "total_found": len(formatted_elements), + "image_dimensions": {"height": image.shape[0], "width": image.shape[1]}, + } + + except Exception as e: + return {"success": False, "error": str(e), "clickable_elements": [], "total_found": 0} + + +# Map tool names to handler functions +TOOL_HANDLERS = { + "detect_ui_elements": handle_detect_elements, + "extract_text": handle_extract_text, + "find_elements_by_text": handle_find_elements, + "analyze_screen_content": handle_analyze_screen, + "find_clickable_elements": handle_find_clickable, +} + + +def execute_tool(tool_name: str, parameters: dict[str, Any]) -> dict[str, Any]: + """ + Execute a specific tool with given parameters + + Args: + tool_name: Name of the tool to execute + parameters: Tool parameters + + Returns: + Tool execution results + + Raises: + ValueError: If tool is not found + """ + if tool_name not in TOOL_HANDLERS: + return { + "success": False, + "error": f"Unknown tool: {tool_name}", + "available_tools": list(TOOL_HANDLERS.keys()), + } + + handler = TOOL_HANDLERS[tool_name] + return handler(parameters) diff --git a/src/agent/mcp/mcp_server.py b/src/agent/mcp/mcp_server.py new file mode 100644 index 0000000..d3d2ec6 --- /dev/null +++ b/src/agent/mcp/mcp_server.py @@ -0,0 +1,220 @@ +"""MCP Server Implementation for Computer Vision + +Implements the Model Context Protocol (MCP) server that exposes computer vision +functions as tools for LLMs. This server can be used with any MCP-compatible +LLM client (Claude, GPT-4, etc.). +""" + +import asyncio +import json +import sys +from typing import Any + +from .handlers import execute_tool +from .tools import get_function_tools + + +class MCPServer: + """Simple MCP server for computer vision tools""" + + def __init__(self): + self.tools = get_function_tools() + self.server_info = { + "name": "Computer Vision MCP Server", + "version": "1.0.0", + "protocol_version": "2024-11-05", + } + + async def handle_request(self, request: dict[str, Any]) -> dict[str, Any]: + """ + Handle incoming MCP requests + + Args: + request: MCP request object + + Returns: + MCP response object + """ + try: + method = request.get("method", "") + params = request.get("params", {}) + request_id = request.get("id") + + if method == "initialize": + return self._handle_initialize(request_id, params) + elif method == "tools/list": + return self._handle_list_tools(request_id) + elif method == "tools/call": + return await self._handle_call_tool(request_id, params) + else: + return self._create_error_response( + request_id, -32601, f"Method not found: {method}" + ) + + except Exception as e: + return self._create_error_response(request.get("id"), -32603, f"Internal error: {e!s}") + + def _handle_initialize(self, request_id: Any, params: dict[str, Any]) -> dict[str, Any]: + """Handle initialization request""" + return { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "protocolVersion": self.server_info["protocol_version"], + "capabilities": {"tools": {"listChanged": False}}, + "serverInfo": { + "name": self.server_info["name"], + "version": self.server_info["version"], + }, + }, + } + + def _handle_list_tools(self, request_id: Any) -> dict[str, Any]: + """Handle tools list request""" + # Convert our function tools to MCP tool format + mcp_tools = [] + for tool in self.tools: + func_def = tool["function"] + mcp_tool = { + "name": func_def["name"], + "description": func_def["description"], + "inputSchema": func_def["parameters"], + } + mcp_tools.append(mcp_tool) + + return {"jsonrpc": "2.0", "id": request_id, "result": {"tools": mcp_tools}} + + async def _handle_call_tool(self, request_id: Any, params: dict[str, Any]) -> dict[str, Any]: + """Handle tool call request""" + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + + if not tool_name: + return self._create_error_response(request_id, -32602, "Missing tool name") + + # Execute the tool + try: + result = execute_tool(tool_name, arguments) + + return { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "content": [{"type": "text", "text": json.dumps(result, indent=2)}], + "isError": not result.get("success", True), + }, + } + + except Exception as e: + return self._create_error_response(request_id, -32603, f"Tool execution error: {e!s}") + + def _create_error_response(self, request_id: Any, code: int, message: str) -> dict[str, Any]: + """Create error response""" + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + + +class StdioMCPServer: + """MCP server that communicates via stdio""" + + def __init__(self): + self.server = MCPServer() + + async def run(self): + """Run the server, reading from stdin and writing to stdout""" + print("Computer Vision MCP Server starting...", file=sys.stderr) + + try: + while True: + # Read line from stdin + line = sys.stdin.readline() + if not line: + break + + line = line.strip() + if not line: + continue + + try: + # Parse JSON request + request = json.loads(line) + + # Handle request + response = await self.server.handle_request(request) + + # Write response to stdout + print(json.dumps(response), flush=True) + + except json.JSONDecodeError: + # Invalid JSON + error_response = { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": "Parse error"}, + } + print(json.dumps(error_response), flush=True) + + except KeyboardInterrupt: + print("Server shutting down...", file=sys.stderr) + except Exception as e: + print(f"Server error: {e}", file=sys.stderr) + + +def create_mcp_server() -> StdioMCPServer: + """ + Create MCP server instance + + Returns: + MCP server ready to run + + Example: + server = create_mcp_server() + await server.run() + """ + return StdioMCPServer() + + +async def start_server(): + """ + Start the MCP server + + This is the main entry point for running the server. + """ + server = create_mcp_server() + await server.run() + + +def main(): + """Command line entry point""" + print("Starting Computer Vision MCP Server...", file=sys.stderr) + + # Check if models are available + try: + import sys + from pathlib import Path + + # Add parent directory to path to import ocr_clean + parent_dir = Path(__file__).parent.parent + sys.path.append(str(parent_dir)) + + from ocr.setup_models import verify_models + + verification = verify_models() + if not all(verification.values()): + print("โš ๏ธ Warning: Some models are not available.", file=sys.stderr) + print("Run: python -m ocr_clean.setup_models", file=sys.stderr) + + except Exception as e: + print(f"โš ๏ธ Could not verify models: {e}", file=sys.stderr) + + # Run server + try: + asyncio.run(start_server()) + except KeyboardInterrupt: + print("\n๐Ÿ‘‹ Server stopped", file=sys.stderr) + except Exception as e: + print(f"โŒ Server error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/agent/mcp/tools.py b/src/agent/mcp/tools.py new file mode 100644 index 0000000..2462587 --- /dev/null +++ b/src/agent/mcp/tools.py @@ -0,0 +1,350 @@ +"""Function Tool Definitions for MCP Server + +Defines the function tools that LLMs can call through the MCP server. +Each tool corresponds to a pure OCR function with proper schema validation. +""" + +from typing import Any + + +def get_function_tools() -> list[dict[str, Any]]: + """ + Get function tool definitions for LLM integration + + Returns: + List of function tool definitions compatible with MCP protocol + + Example: + tools = get_function_tools() + for tool in tools: + print(f"Tool: {tool['function']['name']}") + """ + return [ + { + "type": "function", + "function": { + "name": "detect_ui_elements", + "description": "Detect UI elements in an image using YOLO computer vision", + "parameters": { + "type": "object", + "properties": { + "image_base64": { + "type": "string", + "description": "Base64-encoded image data (PNG, JPG, etc.)", + }, + "confidence_threshold": { + "type": "number", + "default": 0.6, + "minimum": 0.0, + "maximum": 1.0, + "description": "Minimum confidence threshold for detections (0.0-1.0)", + }, + "ui_focused": { + "type": "boolean", + "default": True, + "description": "If true, filter to UI-relevant object classes only", + }, + "max_detections": { + "type": "integer", + "default": 50, + "minimum": 1, + "maximum": 200, + "description": "Maximum number of detections to return", + }, + }, + "required": ["image_base64"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "extract_text", + "description": "Extract text from an image using PaddleOCR", + "parameters": { + "type": "object", + "properties": { + "image_base64": { + "type": "string", + "description": "Base64-encoded image data (PNG, JPG, etc.)", + }, + "language": { + "type": "string", + "default": "en", + "description": "Language code for OCR (en, ch, fr, etc.)", + }, + "confidence_threshold": { + "type": "number", + "default": 0.5, + "minimum": 0.0, + "maximum": 1.0, + "description": "Minimum confidence threshold for text results", + }, + "region": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 4, + "maxItems": 4, + "description": "Optional region [x1, y1, x2, y2] to extract text from specific area", + }, + "max_results": { + "type": "integer", + "default": 100, + "minimum": 1, + "maximum": 500, + "description": "Maximum number of text results to return", + }, + }, + "required": ["image_base64"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "find_elements_by_text", + "description": "Find UI elements that contain specific text using combined YOLO+OCR", + "parameters": { + "type": "object", + "properties": { + "image_base64": { + "type": "string", + "description": "Base64-encoded image data (PNG, JPG, etc.)", + }, + "text_query": { + "type": "string", + "description": "Text to search for (e.g., 'Submit', 'Username', 'Login')", + }, + "confidence_threshold": { + "type": "number", + "default": 0.6, + "minimum": 0.0, + "maximum": 1.0, + "description": "Minimum confidence threshold for detections", + }, + "case_sensitive": { + "type": "boolean", + "default": False, + "description": "Whether text matching should be case sensitive", + }, + "search_radius": { + "type": "integer", + "default": 100, + "minimum": 10, + "maximum": 500, + "description": "Radius in pixels to search for nearby visual elements", + }, + }, + "required": ["image_base64", "text_query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "analyze_screen_content", + "description": "Comprehensive screen analysis with natural language query", + "parameters": { + "type": "object", + "properties": { + "image_base64": { + "type": "string", + "description": "Base64-encoded image data (PNG, JPG, etc.)", + }, + "query": { + "type": "string", + "description": "Natural language description of what to analyze (e.g., 'Find all buttons and input fields')", + }, + "confidence_threshold": { + "type": "number", + "default": 0.6, + "minimum": 0.0, + "maximum": 1.0, + "description": "Minimum confidence threshold for detections", + }, + }, + "required": ["image_base64", "query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "find_clickable_elements", + "description": "Find elements that are likely to be clickable (buttons, links, etc.)", + "parameters": { + "type": "object", + "properties": { + "image_base64": { + "type": "string", + "description": "Base64-encoded image data (PNG, JPG, etc.)", + }, + "confidence_threshold": { + "type": "number", + "default": 0.6, + "minimum": 0.0, + "maximum": 1.0, + "description": "Minimum confidence threshold for detections", + }, + }, + "required": ["image_base64"], + }, + }, + }, + ] + + +def get_tool_schemas() -> dict[str, dict[str, Any]]: + """ + Get tool schemas as a dictionary for easier access + + Returns: + Dictionary mapping tool names to their schemas + """ + tools = get_function_tools() + return {tool["function"]["name"]: tool["function"] for tool in tools} + + +def validate_tool_parameters(tool_name: str, parameters: dict[str, Any]) -> dict[str, Any]: + """ + Validate parameters for a specific tool + + Args: + tool_name: Name of the tool to validate + parameters: Parameters to validate + + Returns: + Validated parameters with defaults applied + + Raises: + ValueError: If parameters are invalid + """ + schemas = get_tool_schemas() + + if tool_name not in schemas: + raise ValueError(f"Unknown tool: {tool_name}") + + schema = schemas[tool_name]["parameters"] + properties = schema.get("properties", {}) + required = schema.get("required", []) + + # Check required parameters + for req_param in required: + if req_param not in parameters: + raise ValueError(f"Missing required parameter: {req_param}") + + # Apply defaults and validate + validated = {} + for param_name, param_value in parameters.items(): + if param_name not in properties: + continue # Ignore unknown parameters + + prop_schema = properties[param_name] + + # Apply default if value is None and default exists + if param_value is None and "default" in prop_schema: + validated[param_name] = prop_schema["default"] + else: + validated[param_name] = param_value + + # Basic type validation + expected_type = prop_schema.get("type") + if expected_type == "number" and not isinstance(validated[param_name], (int, float)): + try: + validated[param_name] = float(validated[param_name]) + except (ValueError, TypeError): + raise ValueError(f"Parameter {param_name} must be a number") + + elif expected_type == "integer" and not isinstance(validated[param_name], int): + try: + validated[param_name] = int(validated[param_name]) + except (ValueError, TypeError): + raise ValueError(f"Parameter {param_name} must be an integer") + + elif expected_type == "boolean" and not isinstance(validated[param_name], bool): + if isinstance(validated[param_name], str): + validated[param_name] = validated[param_name].lower() in ("true", "1", "yes") + else: + validated[param_name] = bool(validated[param_name]) + + elif expected_type == "string" and not isinstance(validated[param_name], str): + validated[param_name] = str(validated[param_name]) + + # Range validation for numbers + if expected_type in ("number", "integer"): + if "minimum" in prop_schema and validated[param_name] < prop_schema["minimum"]: + raise ValueError(f"Parameter {param_name} must be >= {prop_schema['minimum']}") + if "maximum" in prop_schema and validated[param_name] > prop_schema["maximum"]: + raise ValueError(f"Parameter {param_name} must be <= {prop_schema['maximum']}") + + return validated + + +# Tool usage examples for documentation +TOOL_EXAMPLES = { + "detect_ui_elements": { + "description": "Detect UI elements like buttons, input fields, and interactive components", + "example": { + "image_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", + "confidence_threshold": 0.7, + "ui_focused": True, + "max_detections": 20, + }, + "response": { + "detections": [ + { + "class_name": "laptop", + "confidence": 0.85, + "bbox": [100, 200, 300, 400], + "center": [200, 300], + "area": 40000, + } + ], + "total_found": 1, + }, + }, + "extract_text": { + "description": "Extract all text from an image or specific region", + "example": { + "image_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", + "language": "en", + "confidence_threshold": 0.8, + "max_results": 50, + }, + "response": { + "text_results": [ + { + "text": "Submit", + "confidence": 0.95, + "bbox": [[120, 50], [180, 50], [180, 80], [120, 80]], + "rect_bbox": [120, 50, 180, 80], + "center": [150, 65], + "area": 1800, + } + ], + "total_found": 1, + }, + }, + "find_elements_by_text": { + "description": "Find UI elements containing specific text", + "example": { + "image_base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8...", + "text_query": "Login", + "confidence_threshold": 0.6, + "case_sensitive": False, + }, + "response": { + "elements": [ + { + "element_type": "combined", + "bbox": [100, 45, 200, 85], + "center": [150, 65], + "confidence": 0.8, + "text": "Login", + "description": "button: 'Login'", + } + ], + "total_found": 1, + }, + }, +} diff --git a/src/agent/vision_tools.py b/src/agent/vision_tools.py new file mode 100644 index 0000000..97da4fb --- /dev/null +++ b/src/agent/vision_tools.py @@ -0,0 +1,427 @@ +"""Professional AI Agent Function Tools for Computer Vision Automation + +Updated to use the unified OCR architecture instead of the old UIFinder system. +Provides standardized function tools for AI agents to automate computer interactions +using computer vision. Designed to work with any AI framework - just provide prompts. + +Example usage: + # Initialize once + tools = VisionToolsConfig() + + # Use in AI agent functions + screen_info = analyze_screen("What buttons and text are visible?") + element = find_element("Submit button") + result = click_element(element) + verify_action("Form was submitted successfully") +""" + +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np + +from ocr import ( + find_elements_by_text, + verify_click_success, + verify_element_present, +) + + +@dataclass +class VisionToolsConfig: + """Configuration for vision tools""" + + models_dir: Path = Path(__file__).parent.parent / "ocr" / "models" + confidence_threshold: float = 0.6 + ocr_language: str = "en" + + def __post_init__(self): + # Ensure models directory exists + self.models_dir.mkdir(parents=True, exist_ok=True) + + +# Global configuration instance +_config = VisionToolsConfig() + + +def configure_vision_tools(confidence_threshold: float = 0.6, ocr_language: str = "en"): + """Configure vision tools settings""" + global _config + _config.confidence_threshold = confidence_threshold + _config.ocr_language = ocr_language + + +def _capture_screen() -> np.ndarray | None: + """Capture current screen using local desktop automation""" + try: + from automation.desktop_control import DesktopControl + + desktop = DesktopControl() + success, screenshot = desktop.capture_screen() + return screenshot if success else None + except ImportError: + # Fallback for environments without local automation + return None + except Exception: + return None + + +def analyze_screen(prompt: str) -> dict[str, Any]: + """ + Analyze screen contents with natural language prompt + + Args: + prompt: Natural language description of what to analyze + + Returns: + Dict with analysis results + """ + screenshot = _capture_screen() + if screenshot is None: + return {"error": "Screen capture not available", "elements": [], "text": []} + + try: + # Use analyze_screen_content for complete analysis + from ocr import analyze_screen_content + + analysis = analyze_screen_content( + screenshot, prompt, confidence_threshold=_config.confidence_threshold + ) + + return { + "prompt": prompt, + "ui_elements": [ + { + "type": elem.element_type, + "bbox": elem.bbox, + "center": elem.center, + "confidence": elem.confidence, + "text": elem.text, + "description": elem.description, + } + for elem in analysis.ui_elements + ], + "text_elements": [ + { + "type": elem.element_type, + "bbox": elem.bbox, + "center": elem.center, + "confidence": elem.confidence, + "text": elem.text, + "description": elem.description, + } + for elem in analysis.text_elements + ], + "total_elements": len(analysis.ui_elements), + "total_text_regions": len(analysis.text_elements), + "summary": analysis.summary, + } + except Exception as e: + return {"error": f"Screen analysis failed: {e}", "elements": [], "text": []} + + +def find_element(description: str) -> dict[str, Any] | None: + """ + Find UI element by description + + Args: + description: Natural language description of element to find + + Returns: + Dict with element info or None if not found + """ + screenshot = _capture_screen() + if screenshot is None: + return {"error": "Screen capture not available"} + + try: + elements = find_elements_by_text( + screenshot, description, confidence_threshold=_config.confidence_threshold + ) + + if not elements: + return None + + # Return the most confident element + best_element = max(elements, key=lambda x: x.confidence) + + return { + "element_type": best_element.element_type, + "bbox": best_element.bbox, + "center": best_element.center, + "confidence": best_element.confidence, + "text": best_element.text, + "description": best_element.description, + } + + except Exception as e: + return {"error": f"Element search failed: {e}"} + + +def click_element(element: dict[str, Any]) -> dict[str, Any]: + """ + Click on element or coordinates + + Args: + element: Element dict with center coordinates + + Returns: + Dict with click result + """ + if "error" in element: + return element + + if "center" not in element: + return {"error": "Element missing center coordinates"} + + try: + # Take before screenshot for verification + before_screenshot = _capture_screen() + if before_screenshot is None: + return {"error": "Cannot capture screen for verification"} + + # Perform click using local desktop automation + x, y = element["center"] + try: + from automation.desktop_control import DesktopControl + + desktop = DesktopControl() + click_result = desktop.click(x, y) + if not click_result.success: + return {"error": f"Click failed: {click_result.message}"} + except ImportError: + return {"error": "Desktop automation not available"} + except Exception as e: + return {"error": f"Click failed: {e}"} + + # Wait for UI to update + time.sleep(1.0) + + # Take after screenshot and verify + after_screenshot = _capture_screen() + if after_screenshot is None: + return {"error": "Cannot capture screen after click"} + + verification = verify_click_success(before_screenshot, after_screenshot) + + return { + "action": "click", + "target": element, + "success": verification.success, + "message": verification.message, + "confidence": verification.confidence, + } + + except Exception as e: + return {"error": f"Click failed: {e}"} + + +def type_text_in_field(text: str, field: dict[str, Any]) -> dict[str, Any]: + """ + Type text in input field + + Args: + text: Text to type + field: Field element dict + + Returns: + Dict with typing result + """ + if "error" in field: + return field + + try: + # First click on field to focus it + click_result = click_element(field) + if not click_result.get("success", False): + return { + "error": f"Could not focus field: {click_result.get('message', 'Unknown error')}" + } + + # Type text using local desktop automation + try: + from automation.desktop_control import DesktopControl + + desktop = DesktopControl() + type_result = desktop.type_text(text) + if not type_result.success: + return {"error": f"Text input failed: {type_result.message}"} + except ImportError: + return {"error": "Desktop automation not available"} + except Exception as e: + return {"error": f"Text input failed: {e}"} + + # Wait for text to appear + time.sleep(0.5) + + # Verify text was entered correctly + screenshot = _capture_screen() + if screenshot is None: + return {"error": "Cannot capture screen for text verification"} + + # Use field bbox for verification if available + if "bbox" in field: + # TODO: Implement text input verification using extract_text_from_region + pass + + return { + "action": "type_text", + "text": text, + "field": field, + "success": True, # Placeholder + "message": f"Typed '{text}' in field", + } + + except Exception as e: + return {"error": f"Text input failed: {e}"} + + +def verify_action(expected: str) -> dict[str, Any]: + """ + Verify action outcomes + + Args: + expected: Expected outcome description + + Returns: + Dict with verification result + """ + screenshot = _capture_screen() + if screenshot is None: + return {"error": "Screen capture not available for verification"} + + try: + # Use the clean verification system + result = verify_element_present( + screenshot, expected, confidence_threshold=_config.confidence_threshold + ) + + return { + "action": "verify", + "expected": expected, + "success": result.success, + "message": result.message, + "confidence": result.confidence, + } + + except Exception as e: + return {"error": f"Verification failed: {e}"} + + +def wait_for_element(description: str, max_attempts: int = 10) -> dict[str, Any]: + """ + Wait for element to appear + + Args: + description: Element description to wait for + max_attempts: Maximum number of attempts + + Returns: + Dict with wait result + """ + for attempt in range(max_attempts): + element = find_element(description) + + if element and "error" not in element: + return { + "action": "wait_for_element", + "description": description, + "success": True, + "attempts": attempt + 1, + "element": element, + } + + if attempt < max_attempts - 1: + time.sleep(1.0) + + return { + "action": "wait_for_element", + "description": description, + "success": False, + "attempts": max_attempts, + "message": f"Element '{description}' not found after {max_attempts} attempts", + } + + +def scroll_screen(direction: str, pixels: int = 100) -> dict[str, Any]: + """ + Scroll screen content + + Args: + direction: Scroll direction ("up", "down", "left", "right") + pixels: Number of pixels to scroll + + Returns: + Dict with scroll result + """ + try: + # Implement scrolling using local desktop automation + from automation.desktop_control import DesktopControl + + desktop = DesktopControl() + + # Get screen center for scroll position + screenshot = _capture_screen() + if screenshot is None: + return {"error": "Cannot capture screen for scrolling"} + + height, width = screenshot.shape[:2] + center_x, center_y = width // 2, height // 2 + + scroll_result = desktop.scroll(center_x, center_y, direction, max(1, pixels // 100)) + + return { + "action": "scroll", + "direction": direction, + "pixels": pixels, + "success": scroll_result.success, + "message": scroll_result.message, + } + + except Exception as e: + return {"error": f"Scroll failed: {e}"} + + +def take_screenshot(path: str) -> dict[str, Any]: + """ + Capture screen image + + Args: + path: Path to save screenshot + + Returns: + Dict with screenshot result + """ + screenshot = _capture_screen() + if screenshot is None: + return {"error": "Screen capture not available"} + + try: + cv2.imwrite(path, screenshot) + return { + "action": "screenshot", + "path": path, + "success": True, + "message": f"Screenshot saved to {path}", + } + + except Exception as e: + return {"error": f"Screenshot save failed: {e}"} + + +# Export the main functions for AI agent use +__all__ = [ + "VisionToolsConfig", + "analyze_screen", + "click_element", + "configure_vision_tools", + "find_element", + "scroll_screen", + "take_screenshot", + "type_text_in_field", + "verify_action", + "wait_for_element", +] diff --git a/src/agents/__init__.py b/src/agents/__init__.py deleted file mode 100644 index 2f71306..0000000 --- a/src/agents/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Agents module for VM automation""" - -from .app_controller import AppControllerAgent -from .shared_context import VMConnectionInfo, VMSession, VMTarget -from .vm_navigator import VMNavigatorAgent - -__all__ = ["AppControllerAgent", "VMConnectionInfo", "VMNavigatorAgent", "VMSession", "VMTarget"] diff --git a/src/automation/__init__.py b/src/automation/__init__.py new file mode 100644 index 0000000..cb3498e --- /dev/null +++ b/src/automation/__init__.py @@ -0,0 +1,32 @@ +"""Local Desktop Automation Package + +Provides local mouse/keyboard control capabilities separate from VM/remote control. +This package handles direct interaction with the local desktop environment. + +Key Components: +- desktop_control.py: Local macOS desktop automation (moved from vm/connections/) +- form_interface.py: High-level form entry interface combining OCR + automation + +Usage: + from automation import DesktopControl, FormFiller + + # Low-level desktop automation + desktop = DesktopControl() + desktop.click(100, 200) + desktop.type_text("Hello World") + + # High-level form filling + form_filler = FormFiller() + form_filler.fill_field("Username", "john.doe") + form_filler.click_button("Submit") +""" + +from .desktop_control import DesktopControl +from .form_interface import FormFiller + +__version__ = "1.0.0" + +__all__ = [ + "DesktopControl", + "FormFiller", +] diff --git a/src/automation/desktop_control.py b/src/automation/desktop_control.py new file mode 100644 index 0000000..228da28 --- /dev/null +++ b/src/automation/desktop_control.py @@ -0,0 +1,383 @@ +"""Local macOS Desktop Automation + +Provides direct control over the local macOS desktop environment using native tools: +- AppleScript for mouse clicks and keyboard input +- screencapture for screenshots +- cliclick as alternative click method (if installed) + +This is completely separate from VM/remote control and operates on the local machine only. +""" + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import cv2 +import numpy as np + + +@dataclass +class ActionResult: + """Result of local desktop action""" + + success: bool + message: str + + +class DesktopControl: + """Local macOS desktop automation controller""" + + def __init__(self): + """Initialize desktop control""" + self.screenshot_path = Path("/tmp/desktop_screenshot.png") + self.is_active = True + + def capture_screen(self) -> tuple[bool, np.ndarray | None]: + """ + Capture desktop screenshot using macOS screencapture + + Returns: + Tuple of (success, image_array) where image_array is BGR format for OpenCV + + Example: + success, screenshot = desktop.capture_screen() + if success: + cv2.imshow("Desktop", screenshot) + """ + try: + # Use macOS screencapture for entire screen + cmd = ["screencapture", "-x", str(self.screenshot_path)] + result = subprocess.run(cmd, check=True, capture_output=True) + + if result.returncode != 0: + print(f"Screenshot command failed: {result.stderr.decode()}") + return False, None + + # Load image with OpenCV + if self.screenshot_path.exists(): + image = cv2.imread(str(self.screenshot_path)) + if image is not None: + return True, image + + return False, None + + except subprocess.CalledProcessError as e: + print(f"Desktop screen capture error: {e}") + return False, None + except Exception as e: + print(f"Desktop screen capture error: {e}") + return False, None + + def capture_window(self, interactive: bool = True) -> tuple[bool, np.ndarray | None]: + """ + Capture specific window (interactive selection) + + Args: + interactive: If True, user selects window interactively + + Returns: + Tuple of (success, image_array) + """ + try: + if interactive: + # Interactive window selection + cmd = ["screencapture", "-w", "-x", str(self.screenshot_path)] + print("Click on the window you want to capture...") + else: + # Fallback to full screen + return self.capture_screen() + + result = subprocess.run(cmd, check=True) + + if result.returncode != 0: + print("Window capture failed, falling back to full screen") + return self.capture_screen() + + # Load image with OpenCV + if self.screenshot_path.exists(): + image = cv2.imread(str(self.screenshot_path)) + if image is not None: + return True, image + + return False, None + + except subprocess.CalledProcessError as e: + print(f"Window capture error: {e}, falling back to full screen") + return self.capture_screen() + except Exception as e: + print(f"Window capture error: {e}") + return False, None + + def click(self, x: int, y: int, button: str = "left") -> ActionResult: + """ + Click at coordinates using AppleScript + + Args: + x: X coordinate + y: Y coordinate + button: "left", "right", or "middle" + + Returns: + ActionResult with success status + + Example: + result = desktop.click(100, 200) + if result.success: + print("Click successful") + """ + try: + # Use AppleScript for clicking + script = f'tell application "System Events" to click at {{{x}, {y}}}' + if button != "left": + # For non-left clicks, use control-click as approximation + script = f'tell application "System Events" to tell (click at {{{x}, {y}}}) to key down control' + + cmd = ["osascript", "-e", script] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Clicked {button} at ({x}, {y})") + else: + return ActionResult(False, f"Click failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Desktop click error: {e}") + except Exception as e: + return ActionResult(False, f"Desktop click error: {e}") + + def click_with_cliclick(self, x: int, y: int, button: str = "left") -> ActionResult: + """ + Alternative click method using cliclick (if installed) + + Install with: brew install cliclick + + Args: + x: X coordinate + y: Y coordinate + button: "left", "right", or "middle" + + Returns: + ActionResult with success status + """ + try: + # Check if cliclick is available + import shutil + + if not shutil.which("cliclick"): + return self.click(x, y, button) # Fallback to AppleScript + + # Map button names + button_map = {"left": "c", "right": "rc", "middle": "mc"} + click_type = button_map.get(button, "c") + + cmd = ["cliclick", f"{click_type}:{x},{y}"] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Clicked {button} at ({x}, {y}) via cliclick") + else: + return ActionResult(False, f"Cliclick failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Cliclick error: {e}") + except Exception: + # Fallback to AppleScript + return self.click(x, y, button) + + def double_click(self, x: int, y: int) -> ActionResult: + """ + Double-click at coordinates + + Args: + x: X coordinate + y: Y coordinate + + Returns: + ActionResult with success status + """ + result1 = self.click(x, y, "left") + if not result1.success: + return result1 + + # Small delay between clicks + import time + + time.sleep(0.1) + + result2 = self.click(x, y, "left") + if not result2.success: + return result2 + + return ActionResult(True, f"Double-clicked at ({x}, {y})") + + def type_text(self, text: str) -> ActionResult: + """ + Type text using AppleScript + + Args: + text: Text to type + + Returns: + ActionResult with success status + + Example: + result = desktop.type_text("Hello World") + """ + try: + # Escape quotes in text for AppleScript + escaped_text = text.replace('"', '\\"').replace("'", "\\'") + + script = f'tell application "System Events" to keystroke "{escaped_text}"' + cmd = ["osascript", "-e", script] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Typed: {text}") + else: + return ActionResult(False, f"Type failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Desktop type error: {e}") + except Exception as e: + return ActionResult(False, f"Desktop type error: {e}") + + def key_press(self, key: str) -> ActionResult: + """ + Press key using AppleScript + + Args: + key: Key name or combination (e.g., "enter", "cmd+c", "ctrl+v") + + Returns: + ActionResult with success status + + Example: + desktop.key_press("enter") + desktop.key_press("cmd+c") # Copy + """ + try: + # Map common key names to AppleScript key codes + key_mapping = { + "enter": "return", + "return": "return", + "escape": "escape", + "tab": "tab", + "space": "space", + "backspace": "delete", + "delete": "forward delete", + "up": "up arrow", + "down": "down arrow", + "left": "left arrow", + "right": "right arrow", + "cmd": "command", + "ctrl": "control", + "alt": "option", + "shift": "shift", + } + + apple_key = key_mapping.get(key.lower(), key) + + # Handle special key combinations + if key.lower().startswith("cmd+") or key.lower().startswith("ctrl+"): + parts = key.lower().split("+") + modifier = parts[0] + target_key = parts[1] if len(parts) > 1 else "" + + modifier_map = {"cmd": "command", "ctrl": "control"} + mod_key = modifier_map.get(modifier, modifier) + + script = f'tell application "System Events" to keystroke "{target_key}" using {mod_key} down' + else: + # Simpler approach for single keys + if apple_key in ["return", "escape", "tab", "space", "delete"]: + script = f'tell application "System Events" to keystroke "{apple_key}"' + else: + script = ( + f'tell application "System Events" to key code (key code of "{apple_key}")' + ) + + cmd = ["osascript", "-e", script] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Pressed key: {key}") + else: + return ActionResult(False, f"Key press failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Desktop key press error: {e}") + except Exception as e: + return ActionResult(False, f"Desktop key press error: {e}") + + def scroll(self, x: int, y: int, direction: str = "up", clicks: int = 3) -> ActionResult: + """ + Scroll at specific position using arrow keys + + Args: + x: X coordinate (for positioning cursor) + y: Y coordinate (for positioning cursor) + direction: "up" or "down" + clicks: Number of scroll steps + + Returns: + ActionResult with success status + """ + try: + # First click at position to focus + self.click(x, y) + + # Then use arrow keys to scroll + for _ in range(clicks): + if direction == "up": + result = self.key_press("up") + else: + result = self.key_press("down") + + if not result.success: + return result + + import time + + time.sleep(0.1) + + return ActionResult(True, f"Scrolled {direction} {clicks} times at ({x}, {y})") + + except Exception as e: + return ActionResult(False, f"Scroll failed: {e}") + + def get_desktop_info(self) -> dict: + """ + Get desktop-specific information + + Returns: + Dictionary with platform and capability information + """ + try: + # Get screen resolution + cmd = ["system_profiler", "SPDisplaysDataType", "-json"] + result = subprocess.run(cmd, capture_output=True, text=True) + + info = { + "platform": "macOS", + "type": "local_desktop", + "screenshot_capability": True, + "click_capability": True, + "keyboard_capability": True, + "cliclick_available": bool(__import__("shutil").which("cliclick")), + } + + if result.returncode == 0: + info["system_profiler_available"] = True + + return info + + except Exception as e: + return {"platform": "macOS", "type": "local_desktop", "error": str(e)} + + def cleanup(self): + """Clean up temporary files""" + try: + if self.screenshot_path.exists(): + self.screenshot_path.unlink() + except Exception: + pass diff --git a/src/automation/form_interface.py b/src/automation/form_interface.py new file mode 100644 index 0000000..66ba259 --- /dev/null +++ b/src/automation/form_interface.py @@ -0,0 +1,336 @@ +"""High-Level Form Entry Interface + +Combines OCR vision capabilities with local desktop automation to provide +easy form filling functionality. This bridges the gap between "what to click" +(from OCR) and "how to click it" (from automation). + +Usage: + form_filler = FormFiller() + form_filler.fill_field("Username", "john.doe") + form_filler.fill_field("Password", "secret123") + form_filler.click_button("Submit") +""" + +import sys +from pathlib import Path +from typing import Any + +# Add src to path for imports +sys.path.append(str(Path(__file__).parent.parent)) + +import numpy as np + +from automation.desktop_control import ActionResult, DesktopControl +from ocr import find_elements_by_text + + +class FormFiller: + """High-level interface for automated form filling using OCR + local automation""" + + def __init__(self): + """Initialize form filler with desktop control and OCR capabilities""" + self.desktop = DesktopControl() + self.last_screenshot: np.ndarray | None = None + self.screenshot_cache_time: float = 0 + + def _get_current_screenshot(self, max_age: float = 1.0) -> np.ndarray | None: + """Get current screenshot with caching to avoid excessive captures""" + import time + + current_time = time.time() + + # Use cached screenshot if recent enough + if self.last_screenshot is not None and current_time - self.screenshot_cache_time < max_age: + return self.last_screenshot + + # Capture fresh screenshot + success, screenshot = self.desktop.capture_screen() + if success: + self.last_screenshot = screenshot + self.screenshot_cache_time = current_time + return screenshot + return None + + def find_field_by_label( + self, label_text: str, confidence_threshold: float = 0.6 + ) -> dict[str, Any] | None: + """ + Find form field by its label text + + Args: + label_text: Text to search for (e.g., "Username", "Email", "Password") + confidence_threshold: Minimum OCR confidence + + Returns: + Dictionary with field information or None if not found + + Example: + field = form_filler.find_field_by_label("Username") + if field: + print(f"Found field at {field['center']}") + """ + screenshot = self._get_current_screenshot() + if screenshot is None: + return None + + try: + elements = find_elements_by_text( + screenshot, + label_text, + confidence_threshold=confidence_threshold, + case_sensitive=False, + ) + + if elements: + # Get the best match + best_element = max(elements, key=lambda x: x.confidence) + + return { + "text": best_element.text, + "center": best_element.center, + "bbox": best_element.bbox, + "confidence": best_element.confidence, + "description": best_element.description, + } + + except Exception as e: + print(f"Error finding field by label '{label_text}': {e}") + + return None + + def find_input_field_near( + self, label_center: tuple, search_radius: int = 100 + ) -> dict[str, Any] | None: + """ + Find input field near a label + + Args: + label_center: (x, y) coordinates of label + search_radius: Radius to search for input fields + + Returns: + Dictionary with input field information or None if not found + """ + screenshot = self._get_current_screenshot() + if screenshot is None: + return None + + try: + # Look for common input field indicators near the label + input_indicators = ["textfield", "input", "field", "_", "____"] + + label_x, label_y = label_center + + # Search for input elements in the vicinity + for indicator in input_indicators: + elements = find_elements_by_text( + screenshot, + indicator, + confidence_threshold=0.3, + search_radius=search_radius, + case_sensitive=False, + ) + + # Find closest element to label + closest_element = None + min_distance = float("inf") + + for element in elements: + ex, ey = element.center + distance = ((ex - label_x) ** 2 + (ey - label_y) ** 2) ** 0.5 + + if distance <= search_radius and distance < min_distance: + min_distance = distance + closest_element = element + + if closest_element: + return { + "center": closest_element.center, + "bbox": closest_element.bbox, + "confidence": closest_element.confidence, + "distance_from_label": min_distance, + } + + except Exception as e: + print(f"Error finding input field near {label_center}: {e}") + + return None + + def fill_field( + self, label_text: str, value: str, click_offset: tuple = (0, 25) + ) -> ActionResult: + """ + Fill a form field by finding its label and entering text + + Args: + label_text: Label text to search for + value: Text to enter in the field + click_offset: (x, y) offset from label to click (default: slightly below) + + Returns: + ActionResult indicating success or failure + + Example: + result = form_filler.fill_field("Username", "john.doe") + result = form_filler.fill_field("Email", "john@example.com") + """ + try: + # Find the label + field_info = self.find_field_by_label(label_text) + if not field_info: + return ActionResult(False, f"Could not find field with label: {label_text}") + + label_x, label_y = field_info["center"] + + # Try to find actual input field near label + input_field = self.find_input_field_near((label_x, label_y)) + if input_field: + # Click on the actual input field + click_x, click_y = input_field["center"] + else: + # Click near the label with offset + offset_x, offset_y = click_offset + click_x = label_x + offset_x + click_y = label_y + offset_y + + # Click to focus the field + click_result = self.desktop.click(click_x, click_y) + if not click_result.success: + return ActionResult(False, f"Failed to click field: {click_result.message}") + + # Clear field (select all and delete) + self.desktop.key_press("cmd+a") # Select all + + # Type the value + type_result = self.desktop.type_text(value) + if not type_result.success: + return ActionResult(False, f"Failed to type text: {type_result.message}") + + return ActionResult(True, f"Successfully filled '{label_text}' with '{value}'") + + except Exception as e: + return ActionResult(False, f"Error filling field '{label_text}': {e}") + + def click_button(self, button_text: str, confidence_threshold: float = 0.6) -> ActionResult: + """ + Click a button by its text + + Args: + button_text: Text on the button (e.g., "Submit", "Login", "OK") + confidence_threshold: Minimum OCR confidence + + Returns: + ActionResult indicating success or failure + + Example: + result = form_filler.click_button("Submit") + result = form_filler.click_button("Login") + """ + try: + button_info = self.find_field_by_label(button_text, confidence_threshold) + if not button_info: + return ActionResult(False, f"Could not find button: {button_text}") + + button_x, button_y = button_info["center"] + + click_result = self.desktop.click(button_x, button_y) + if click_result.success: + return ActionResult(True, f"Successfully clicked '{button_text}' button") + else: + return ActionResult(False, f"Failed to click button: {click_result.message}") + + except Exception as e: + return ActionResult(False, f"Error clicking button '{button_text}': {e}") + + def fill_form(self, form_data: dict[str, str]) -> dict[str, ActionResult]: + """ + Fill multiple form fields at once + + Args: + form_data: Dictionary mapping field labels to values + + Returns: + Dictionary mapping field labels to ActionResult objects + + Example: + results = form_filler.fill_form({ + "Username": "john.doe", + "Password": "secret123", + "Email": "john@example.com" + }) + """ + results = {} + + for label, value in form_data.items(): + # Refresh screenshot for each field to account for UI changes + self.last_screenshot = None + result = self.fill_field(label, value) + results[label] = result + + if not result.success: + print(f"Warning: Failed to fill '{label}': {result.message}") + + return results + + def get_form_fields(self, common_labels: list[str] | None = None) -> list[dict[str, Any]]: + """ + Detect form fields on current screen + + Args: + common_labels: List of common field labels to look for + + Returns: + List of detected form field information + """ + if common_labels is None: + common_labels = [ + "Username", + "Email", + "Password", + "Name", + "Address", + "Phone", + "Company", + "Title", + "Message", + "Comment", + ] + + detected_fields = [] + + for label in common_labels: + field_info = self.find_field_by_label(label, confidence_threshold=0.5) + if field_info: + detected_fields.append(field_info) + + return detected_fields + + def wait_for_element( + self, text: str, timeout: float = 10.0, check_interval: float = 1.0 + ) -> dict[str, Any] | None: + """ + Wait for an element to appear on screen + + Args: + text: Text to wait for + timeout: Maximum time to wait in seconds + check_interval: Time between checks in seconds + + Returns: + Element information if found, None if timeout + """ + import time + + start_time = time.time() + + while time.time() - start_time < timeout: + # Force fresh screenshot + self.last_screenshot = None + element = self.find_field_by_label(text) + + if element: + return element + + time.sleep(check_interval) + + return None diff --git a/src/ocr/__init__.py b/src/ocr/__init__.py new file mode 100644 index 0000000..0f5d0ea --- /dev/null +++ b/src/ocr/__init__.py @@ -0,0 +1,71 @@ +"""Pure Computer Vision Functions for Generic Use + +This module provides clean, standalone computer vision functions using: +- YOLO for UI element detection +- PaddleOCR for text recognition +- Combined search and analysis capabilities + +No dependencies on connections, sessions, workflows, or adapters. +Perfect for MCP servers and LLM function calling. + +Example usage: + import cv2 + from ocr import detect_ui_elements, extract_text, find_elements_by_text + + image = cv2.imread("screenshot.png") + elements = detect_ui_elements(image) + text_results = extract_text(image) + buttons = find_elements_by_text(image, "Submit") +""" + +from .detector import Detection, detect_ui_elements +from .finder import ( + ScreenAnalysis, + UIElement, + analyze_screen_content, + find_clickable_elements, + find_elements_by_text, +) +from .reader import TextResult, extract_text, extract_text_from_region +from .setup_models import download_models, get_model_paths, setup_models +from .verification import ( + VerificationResult, + compare_screenshots, + create_diff_visualization, + verify_click_success, + verify_element_present, + verify_page_loaded, + verify_text_input, + wait_for_element, +) + +__version__ = "1.0.0" + +__all__ = [ + # Detection functions + "detect_ui_elements", + "Detection", + # Text extraction functions + "extract_text", + "extract_text_from_region", + "TextResult", + # Combined search functions + "find_elements_by_text", + "find_clickable_elements", + "analyze_screen_content", + "UIElement", + "ScreenAnalysis", + # Verification functions + "verify_click_success", + "verify_text_input", + "verify_element_present", + "verify_page_loaded", + "wait_for_element", + "compare_screenshots", + "create_diff_visualization", + "VerificationResult", + # Model management + "setup_models", + "download_models", + "get_model_paths", +] diff --git a/src/ocr/detector.py b/src/ocr/detector.py new file mode 100644 index 0000000..6e477a7 --- /dev/null +++ b/src/ocr/detector.py @@ -0,0 +1,367 @@ +"""Pure YOLO Detection Functions + +Standalone YOLO-based UI element detection with no external dependencies. +""" + +import os +from dataclasses import dataclass +from pathlib import Path + +import cv2 +import numpy as np +import onnxruntime as ort + + +@dataclass +class Detection: + """UI element detection result""" + + class_name: str + confidence: float + bbox: tuple[int, int, int, int] # x1, y1, x2, y2 + center: tuple[int, int] + area: int + + +# UI-focused COCO classes (desktop/screen relevant objects) +UI_FOCUSED_CLASSES = { + 0: "person", # User avatars, profile pictures + 62: "tv", # Monitors, displays + 63: "laptop", # Device detection + 64: "mouse", # Peripheral detection + 65: "remote", # Control devices + 66: "keyboard", # Input device detection + 67: "cell phone", # Mobile device detection + 73: "book", # Document/reading interfaces + 74: "clock", # Time displays, widgets + 76: "scissors", # Cut/edit tools in UI +} + +# Full COCO class set for comprehensive detection +FULL_COCO_CLASSES = { + 0: "person", + 1: "bicycle", + 2: "car", + 3: "motorcycle", + 4: "airplane", + 5: "bus", + 6: "train", + 7: "truck", + 8: "boat", + 9: "traffic light", + 10: "fire hydrant", + 11: "stop sign", + 12: "parking meter", + 13: "bench", + 14: "bird", + 15: "cat", + 16: "dog", + 17: "horse", + 18: "sheep", + 19: "cow", + 20: "elephant", + 21: "bear", + 22: "zebra", + 23: "giraffe", + 24: "backpack", + 25: "umbrella", + 26: "handbag", + 27: "tie", + 28: "suitcase", + 29: "frisbee", + 30: "skis", + 31: "snowboard", + 32: "sports ball", + 33: "kite", + 34: "baseball bat", + 35: "baseball glove", + 36: "skateboard", + 37: "surfboard", + 38: "tennis racket", + 39: "bottle", + 40: "wine glass", + 41: "cup", + 42: "fork", + 43: "knife", + 44: "spoon", + 45: "bowl", + 46: "banana", + 47: "apple", + 48: "sandwich", + 49: "orange", + 50: "broccoli", + 51: "carrot", + 52: "hot dog", + 53: "pizza", + 54: "donut", + 55: "cake", + 56: "chair", + 57: "couch", + 58: "potted plant", + 59: "bed", + 60: "dining table", + 61: "toilet", + 62: "tv", + 63: "laptop", + 64: "mouse", + 65: "remote", + 66: "keyboard", + 67: "cell phone", + 68: "microwave", + 69: "oven", + 70: "toaster", + 71: "sink", + 72: "refrigerator", + 73: "book", + 74: "clock", + 75: "vase", + 76: "scissors", + 77: "teddy bear", + 78: "hair drier", + 79: "toothbrush", +} + + +def detect_ui_elements( + image: np.ndarray, + confidence_threshold: float = 0.6, + model_path: str | None = None, + ui_focused: bool = True, + max_detections: int = 100, +) -> list[Detection]: + """ + Detect UI elements in image using YOLO + + Args: + image: Input image as numpy array (BGR format from cv2) + confidence_threshold: Minimum confidence for detections (0.0-1.0) + model_path: Path to YOLO ONNX model (uses default if None) + ui_focused: If True, filter to UI-relevant classes only + max_detections: Maximum number of detections to return + + Returns: + List of Detection objects sorted by confidence + + Example: + image = cv2.imread("screenshot.png") + detections = detect_ui_elements(image, confidence_threshold=0.7) + for det in detections: + print(f"Found {det.class_name} at {det.center} with confidence {det.confidence}") + """ + # Get model path + if model_path is None: + model_path = _get_default_model_path() + + if not os.path.exists(model_path): + raise FileNotFoundError( + f"YOLO model not found at {model_path}. Run setup_models() to download the model." + ) + + # Choose class set + classes = UI_FOCUSED_CLASSES if ui_focused else FULL_COCO_CLASSES + + # Initialize ONNX session + session = ort.InferenceSession(model_path) + input_name = session.get_inputs()[0].name + input_shape = session.get_inputs()[0].shape + input_height, input_width = input_shape[2], input_shape[3] + + # Preprocess image + processed_image = _preprocess_image(image, input_width, input_height) + + # Run inference + outputs = session.run(None, {input_name: processed_image}) + + # Postprocess results + detections = _postprocess_outputs( + outputs, + image.shape[:2], # original height, width + input_width, + input_height, + classes, + confidence_threshold, + ) + + # Sort by confidence and limit results + detections.sort(key=lambda x: x.confidence, reverse=True) + return detections[:max_detections] + + +def _get_default_model_path() -> str: + """Get default YOLO model path""" + current_dir = Path(__file__).parent + models_dir = current_dir / "models" + return str(models_dir / "yolov8s.onnx") + + +def _preprocess_image(image: np.ndarray, target_width: int, target_height: int) -> np.ndarray: + """Preprocess image for YOLO inference""" + # Resize image + resized = cv2.resize(image, (target_width, target_height)) + + # Convert BGR to RGB + rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) + + # Normalize to [0, 1] and convert to float32 + normalized = rgb.astype(np.float32) / 255.0 + + # Transpose to CHW format and add batch dimension + input_tensor = normalized.transpose(2, 0, 1)[np.newaxis, ...] + + return input_tensor + + +def _postprocess_outputs( + outputs: list[np.ndarray], + original_shape: tuple[int, int], + input_width: int, + input_height: int, + classes: dict, + confidence_threshold: float, +) -> list[Detection]: + """Postprocess YOLO outputs to detections""" + predictions = outputs[0] # Shape: [1, 84, 8400] for YOLOv8s + + # Transpose to [8400, 84] + predictions = predictions.transpose(0, 2, 1)[0] + + detections = [] + orig_h, orig_w = original_shape + + for pred in predictions: + # Extract box coordinates and confidence + x_center, y_center, width, height = pred[0:4] + confidence = pred[4] + class_scores = pred[5:] + + if confidence < confidence_threshold: + continue + + # Get class with highest score + class_id = np.argmax(class_scores) + class_confidence = class_scores[class_id] * confidence + + if class_confidence < confidence_threshold: + continue + + # Skip classes not in our active set + if class_id not in classes: + continue + + # Convert to original image coordinates + x_center *= orig_w / input_width + y_center *= orig_h / input_height + width *= orig_w / input_width + height *= orig_h / input_height + + # Convert to bbox format + x1 = int(x_center - width / 2) + y1 = int(y_center - height / 2) + x2 = int(x_center + width / 2) + y2 = int(y_center + height / 2) + + # Ensure bbox is within image bounds + x1 = max(0, min(x1, orig_w)) + y1 = max(0, min(y1, orig_h)) + x2 = max(0, min(x2, orig_w)) + y2 = max(0, min(y2, orig_h)) + + # Calculate area + area = (x2 - x1) * (y2 - y1) + + detection = Detection( + class_name=classes.get(class_id, f"class_{class_id}"), + confidence=float(class_confidence), + bbox=(x1, y1, x2, y2), + center=(int(x_center), int(y_center)), + area=area, + ) + + detections.append(detection) + + # Apply Non-Maximum Suppression + detections = _apply_nms(detections, iou_threshold=0.5) + + return detections + + +def _apply_nms(detections: list[Detection], iou_threshold: float) -> list[Detection]: + """Apply Non-Maximum Suppression""" + if not detections: + return detections + + # Sort by confidence + detections.sort(key=lambda x: x.confidence, reverse=True) + + filtered_detections = [] + + for detection in detections: + # Check if this detection overlaps significantly with any kept detection + keep = True + for kept_detection in filtered_detections: + if _calculate_iou(detection.bbox, kept_detection.bbox) > iou_threshold: + keep = False + break + + if keep: + filtered_detections.append(detection) + + return filtered_detections + + +def _calculate_iou(box1: tuple[int, int, int, int], box2: tuple[int, int, int, int]) -> float: + """Calculate Intersection over Union""" + x1_1, y1_1, x2_1, y2_1 = box1 + x1_2, y1_2, x2_2, y2_2 = box2 + + # Calculate intersection area + x1_i = max(x1_1, x1_2) + y1_i = max(y1_1, y1_2) + x2_i = min(x2_1, x2_2) + y2_i = min(y2_1, y2_2) + + if x2_i <= x1_i or y2_i <= y1_i: + return 0.0 + + intersection_area = (x2_i - x1_i) * (y2_i - y1_i) + + # Calculate union area + area1 = (x2_1 - x1_1) * (y2_1 - y1_1) + area2 = (x2_2 - x1_2) * (y2_2 - y1_2) + union_area = area1 + area2 - intersection_area + + return intersection_area / union_area if union_area > 0 else 0.0 + + +def draw_detections(image: np.ndarray, detections: list[Detection]) -> np.ndarray: + """ + Draw detection bounding boxes on image + + Args: + image: Input image + detections: List of detections to draw + + Returns: + Image with drawn bounding boxes + """ + result_image = image.copy() + + for detection in detections: + x1, y1, x2, y2 = detection.bbox + + # Draw bounding box + cv2.rectangle(result_image, (x1, y1), (x2, y2), (0, 255, 0), 2) + + # Draw label + label = f"{detection.class_name}: {detection.confidence:.2f}" + label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] + cv2.rectangle( + result_image, + (x1, y1 - label_size[1] - 10), + (x1 + label_size[0], y1), + (0, 255, 0), + -1, + ) + cv2.putText(result_image, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2) + + return result_image diff --git a/src/ocr/finder.py b/src/ocr/finder.py new file mode 100644 index 0000000..a48a4cd --- /dev/null +++ b/src/ocr/finder.py @@ -0,0 +1,424 @@ +"""Combined YOLO+OCR Search and Analysis Functions + +Combines YOLO detection and PaddleOCR to provide intelligent UI element finding +and screen analysis capabilities. +""" + +import math +from dataclasses import dataclass + +import numpy as np + +from .detector import Detection, detect_ui_elements +from .reader import TextResult, extract_text + + +@dataclass +class UIElement: + """Combined UI element with visual and text information""" + + element_type: str # "visual", "text", "combined" + bbox: tuple[int, int, int, int] # x1, y1, x2, y2 + center: tuple[int, int] + confidence: float + area: int + + # Visual detection info (if available) + visual_detection: Detection | None = None + + # Text info (if available) + text_detection: TextResult | None = None + text: str | None = None + + # Metadata + description: str | None = None + + +@dataclass +class ScreenAnalysis: + """Complete screen analysis result""" + + ui_elements: list[UIElement] + text_elements: list[UIElement] + visual_elements: list[UIElement] + clickable_elements: list[UIElement] + + summary: dict + query: str + + +def find_elements_by_text( + image: np.ndarray, + text_query: str, + confidence_threshold: float = 0.6, + search_radius: int = 100, + case_sensitive: bool = False, +) -> list[UIElement]: + """ + Find UI elements that contain or are near specific text + + Args: + image: Input image + text_query: Text to search for + confidence_threshold: Minimum confidence for detections + search_radius: Radius to search for nearby visual elements + case_sensitive: Whether text search is case sensitive + + Returns: + List of UIElement objects that match the text query + + Example: + # Find all Submit buttons + submit_elements = find_elements_by_text(image, "Submit") + + # Find username input fields + username_fields = find_elements_by_text(image, "username", case_sensitive=False) + """ + # Get text detections + text_results = extract_text(image, confidence_threshold=confidence_threshold) + + # Find matching text + query = text_query.strip() + if not case_sensitive: + query = query.lower() + + matching_text = [] + for text_result in text_results: + detected_text = text_result.text.strip() + if not case_sensitive: + detected_text = detected_text.lower() + + if query in detected_text: + matching_text.append(text_result) + + if not matching_text: + return [] + + # Get visual detections + visual_detections = detect_ui_elements(image, confidence_threshold=confidence_threshold) + + elements = [] + + for text_result in matching_text: + # Create text element + text_element = UIElement( + element_type="text", + bbox=text_result.rect_bbox, + center=text_result.center, + confidence=text_result.confidence, + area=text_result.area, + text_detection=text_result, + text=text_result.text, + description=f"Text: '{text_result.text}'", + ) + + # Look for nearby visual elements + nearby_visual = None + min_distance = float("inf") + + for visual_det in visual_detections: + distance = _calculate_distance(text_result.center, visual_det.center) + if distance <= search_radius and distance < min_distance: + min_distance = distance + nearby_visual = visual_det + + if nearby_visual: + # Create combined element + combined_bbox = _merge_bboxes(text_result.rect_bbox, nearby_visual.bbox) + combined_center = ( + (text_result.center[0] + nearby_visual.center[0]) // 2, + (text_result.center[1] + nearby_visual.center[1]) // 2, + ) + combined_area = (combined_bbox[2] - combined_bbox[0]) * ( + combined_bbox[3] - combined_bbox[1] + ) + + combined_element = UIElement( + element_type="combined", + bbox=combined_bbox, + center=combined_center, + confidence=(text_result.confidence + nearby_visual.confidence) / 2, + area=combined_area, + visual_detection=nearby_visual, + text_detection=text_result, + text=text_result.text, + description=f"{nearby_visual.class_name}: '{text_result.text}'", + ) + elements.append(combined_element) + else: + elements.append(text_element) + + return elements + + +def find_clickable_elements( + image: np.ndarray, confidence_threshold: float = 0.6 +) -> list[UIElement]: + """ + Find elements that are likely to be clickable + + Args: + image: Input image + confidence_threshold: Minimum confidence for detections + + Returns: + List of potentially clickable UIElement objects + + Example: + clickable = find_clickable_elements(image) + for element in clickable: + print(f"Clickable: {element.description} at {element.center}") + """ + # Get all elements + all_elements = _get_all_ui_elements(image, confidence_threshold) + + clickable_elements = [] + + # Define clickable indicators + clickable_visual_classes = {"laptop", "mouse", "remote", "keyboard", "cell phone", "book"} + + clickable_text_terms = { + "button", + "click", + "submit", + "ok", + "cancel", + "close", + "save", + "open", + "login", + "sign", + "next", + "back", + "menu", + "settings", + } + + for element in all_elements: + is_clickable = False + + # Check visual indicators + if element.visual_detection: + if element.visual_detection.class_name.lower() in clickable_visual_classes: + is_clickable = True + + # Check text indicators + if element.text: + text_lower = element.text.lower() + if any(term in text_lower for term in clickable_text_terms): + is_clickable = True + + # Short text often indicates buttons/links + if len(element.text.split()) <= 3 and len(element.text) <= 20: + is_clickable = True + + if is_clickable: + clickable_elements.append(element) + + return clickable_elements + + +def analyze_screen_content( + image: np.ndarray, query: str, confidence_threshold: float = 0.6 +) -> ScreenAnalysis: + """ + Comprehensive screen analysis with natural language query + + Args: + image: Input image to analyze + query: Natural language description of what to analyze + confidence_threshold: Minimum confidence for detections + + Returns: + ScreenAnalysis object with comprehensive results + + Example: + analysis = analyze_screen_content(image, "Find all buttons and input fields") + print(f"Found {len(analysis.clickable_elements)} clickable elements") + print(f"Total UI elements: {analysis.summary['total_elements']}") + """ + # Get all UI elements + all_elements = _get_all_ui_elements(image, confidence_threshold) + + # Separate by type + visual_elements = [e for e in all_elements if e.element_type in ["visual", "combined"]] + text_elements = [e for e in all_elements if e.element_type in ["text", "combined"]] + clickable_elements = find_clickable_elements(image, confidence_threshold) + + # Create summary + summary = { + "total_elements": len(all_elements), + "visual_elements": len(visual_elements), + "text_elements": len(text_elements), + "clickable_elements": len(clickable_elements), + "unique_text_content": len(set(e.text for e in text_elements if e.text)), + "visual_classes": list( + set(e.visual_detection.class_name for e in visual_elements if e.visual_detection) + ), + } + + return ScreenAnalysis( + ui_elements=all_elements, + text_elements=text_elements, + visual_elements=visual_elements, + clickable_elements=clickable_elements, + summary=summary, + query=query, + ) + + +def _get_all_ui_elements(image: np.ndarray, confidence_threshold: float) -> list[UIElement]: + """Get all UI elements combining YOLO and OCR results""" + # Get detections + visual_detections = detect_ui_elements(image, confidence_threshold=confidence_threshold) + text_results = extract_text(image, confidence_threshold=confidence_threshold) + + elements = [] + + # Add visual elements + for detection in visual_detections: + element = UIElement( + element_type="visual", + bbox=detection.bbox, + center=detection.center, + confidence=detection.confidence, + area=detection.area, + visual_detection=detection, + description=f"{detection.class_name} ({detection.confidence:.2f})", + ) + elements.append(element) + + # Add text elements + for text_result in text_results: + element = UIElement( + element_type="text", + bbox=text_result.rect_bbox, + center=text_result.center, + confidence=text_result.confidence, + area=text_result.area, + text_detection=text_result, + text=text_result.text, + description=f"Text: '{text_result.text}' ({text_result.confidence:.2f})", + ) + elements.append(element) + + # Merge nearby elements (optional enhancement) + combined_elements = _combine_nearby_elements(elements, proximity_threshold=50) + + return combined_elements + + +def _combine_nearby_elements( + elements: list[UIElement], proximity_threshold: int = 50 +) -> list[UIElement]: + """Combine nearby visual and text elements""" + visual_elements = [e for e in elements if e.element_type == "visual"] + text_elements = [e for e in elements if e.element_type == "text"] + combined_elements = [] + + used_text_indices = set() + + for visual_elem in visual_elements: + # Find nearby text elements + nearby_texts = [] + for i, text_elem in enumerate(text_elements): + if i in used_text_indices: + continue + + distance = _calculate_distance(visual_elem.center, text_elem.center) + if distance <= proximity_threshold: + nearby_texts.append((i, text_elem, distance)) + + if nearby_texts: + # Sort by distance and take closest + nearby_texts.sort(key=lambda x: x[2]) + text_idx, closest_text, _ = nearby_texts[0] + used_text_indices.add(text_idx) + + # Create combined element + combined_bbox = _merge_bboxes(visual_elem.bbox, closest_text.bbox) + combined_center = ( + (visual_elem.center[0] + closest_text.center[0]) // 2, + (visual_elem.center[1] + closest_text.center[1]) // 2, + ) + combined_area = (combined_bbox[2] - combined_bbox[0]) * ( + combined_bbox[3] - combined_bbox[1] + ) + + combined_element = UIElement( + element_type="combined", + bbox=combined_bbox, + center=combined_center, + confidence=(visual_elem.confidence + closest_text.confidence) / 2, + area=combined_area, + visual_detection=visual_elem.visual_detection, + text_detection=closest_text.text_detection, + text=closest_text.text, + description=f"{visual_elem.visual_detection.class_name}: '{closest_text.text}'", + ) + combined_elements.append(combined_element) + else: + # Keep visual element as-is + combined_elements.append(visual_elem) + + # Add remaining text elements + for i, text_elem in enumerate(text_elements): + if i not in used_text_indices: + combined_elements.append(text_elem) + + return combined_elements + + +def _calculate_distance(point1: tuple[int, int], point2: tuple[int, int]) -> float: + """Calculate Euclidean distance between two points""" + return math.sqrt((point1[0] - point2[0]) ** 2 + (point1[1] - point2[1]) ** 2) + + +def _merge_bboxes( + bbox1: tuple[int, int, int, int], bbox2: tuple[int, int, int, int] +) -> tuple[int, int, int, int]: + """Merge two bounding boxes""" + x1_1, y1_1, x2_1, y2_1 = bbox1 + x1_2, y1_2, x2_2, y2_2 = bbox2 + + merged_x1 = min(x1_1, x1_2) + merged_y1 = min(y1_1, y1_2) + merged_x2 = max(x2_1, x2_2) + merged_y2 = max(y2_1, y2_2) + + return (merged_x1, merged_y1, merged_x2, merged_y2) + + +def find_elements_near_point( + image: np.ndarray, point: tuple[int, int], radius: int = 50, confidence_threshold: float = 0.6 +) -> list[UIElement]: + """ + Find UI elements near a specific point + + Args: + image: Input image + point: Center point (x, y) + radius: Search radius in pixels + confidence_threshold: Minimum confidence + + Returns: + List of nearby UIElement objects sorted by distance + """ + all_elements = _get_all_ui_elements(image, confidence_threshold) + nearby_elements = [] + + px, py = point + + for element in all_elements: + ex, ey = element.center + distance = ((ex - px) ** 2 + (ey - py) ** 2) ** 0.5 + + if distance <= radius: + # Add distance for sorting + element.distance = distance # type: ignore + nearby_elements.append(element) + + # Sort by distance + nearby_elements.sort(key=lambda x: getattr(x, "distance", float("inf"))) + + return nearby_elements diff --git a/src/models/yolov8s.onnx b/src/ocr/models/yolov8s.onnx similarity index 99% rename from src/models/yolov8s.onnx rename to src/ocr/models/yolov8s.onnx index 3ca1003..dc26b2a 100644 Binary files a/src/models/yolov8s.onnx and b/src/ocr/models/yolov8s.onnx differ diff --git a/src/ocr/models/yolov8s.pt b/src/ocr/models/yolov8s.pt new file mode 100644 index 0000000..90b4a2c Binary files /dev/null and b/src/ocr/models/yolov8s.pt differ diff --git a/src/ocr/reader.py b/src/ocr/reader.py new file mode 100644 index 0000000..ef85a15 --- /dev/null +++ b/src/ocr/reader.py @@ -0,0 +1,289 @@ +"""Pure PaddleOCR Text Extraction Functions + +Standalone PaddleOCR-based text recognition with no external dependencies. +""" + +from dataclasses import dataclass + +import cv2 +import numpy as np +import paddleocr + + +@dataclass +class TextResult: + """Text recognition result""" + + text: str + confidence: float + bbox: list[tuple[int, int]] # 4 corner points + rect_bbox: tuple[int, int, int, int] # x1, y1, x2, y2 + center: tuple[int, int] + area: int + + +def extract_text( + image: np.ndarray, + language: str = "en", + confidence_threshold: float = 0.5, + max_results: int = 100, +) -> list[TextResult]: + """ + Extract text from entire image using PaddleOCR + + Args: + image: Input image as numpy array (BGR format from cv2) + language: Language code for OCR ('en', 'ch', 'fr', etc.) + confidence_threshold: Minimum confidence for text results (0.0-1.0) + max_results: Maximum number of text results to return + + Returns: + List of TextResult objects sorted by confidence + + Example: + image = cv2.imread("screenshot.png") + text_results = extract_text(image, confidence_threshold=0.7) + for result in text_results: + print(f"Found text: '{result.text}' at {result.center}") + """ + # Initialize PaddleOCR with new API + ocr = paddleocr.PaddleOCR(use_textline_orientation=True, lang=language) + + # Run OCR using new predict method + try: + results = ocr.predict(image) + + if not results: + return [] + + text_results = [] + + # Process each page result (usually just one for single image) + for page_result in results: + if not page_result: + continue + + # Extract text data from OCRResult object + rec_texts = page_result.get("rec_texts", []) + rec_scores = page_result.get("rec_scores", []) + rec_polys = page_result.get("rec_polys", []) + + # Process each detected text + for i, (text, confidence, bbox_points) in enumerate( + zip(rec_texts, rec_scores, rec_polys, strict=False) + ): + if confidence < confidence_threshold: + continue + + # Convert bbox points to integer coordinates + bbox_points = [(int(x), int(y)) for x, y in bbox_points] + + # Calculate rectangular bounding box + x_coords = [point[0] for point in bbox_points] + y_coords = [point[1] for point in bbox_points] + + x1, y1 = min(x_coords), min(y_coords) + x2, y2 = max(x_coords), max(y_coords) + + # Calculate center and area + center_x = (x1 + x2) // 2 + center_y = (y1 + y2) // 2 + area = (x2 - x1) * (y2 - y1) + + text_result = TextResult( + text=text.strip(), + confidence=confidence, + bbox=bbox_points, + rect_bbox=(x1, y1, x2, y2), + center=(center_x, center_y), + area=area, + ) + + text_results.append(text_result) + + # Sort by confidence and limit results + text_results.sort(key=lambda x: x.confidence, reverse=True) + return text_results[:max_results] + + except Exception as e: + print(f"OCR error: {e}") + return [] + + +def extract_text_from_region( + image: np.ndarray, + region: tuple[int, int, int, int], + language: str = "en", + confidence_threshold: float = 0.5, +) -> list[TextResult]: + """ + Extract text from specific region of image + + Args: + image: Input image as numpy array + region: Region coordinates (x1, y1, x2, y2) to crop + language: Language code for OCR + confidence_threshold: Minimum confidence for results + + Returns: + List of TextResult objects with coordinates adjusted to full image + + Example: + # Extract text from top-left 300x200 region + region_text = extract_text_from_region(image, (0, 0, 300, 200)) + """ + x1, y1, x2, y2 = region + + # Crop region + cropped_image = image[y1:y2, x1:x2] + + # Extract text from cropped region + text_results = extract_text(cropped_image, language, confidence_threshold) + + # Adjust coordinates to full image + adjusted_results = [] + for result in text_results: + # Adjust bbox points + adjusted_bbox = [(x + x1, y + y1) for x, y in result.bbox] + + # Adjust rect_bbox + rx1, ry1, rx2, ry2 = result.rect_bbox + adjusted_rect_bbox = (rx1 + x1, ry1 + y1, rx2 + x1, ry2 + y1) + + # Adjust center + adjusted_center = (result.center[0] + x1, result.center[1] + y1) + + adjusted_result = TextResult( + text=result.text, + confidence=result.confidence, + bbox=adjusted_bbox, + rect_bbox=adjusted_rect_bbox, + center=adjusted_center, + area=result.area, + ) + + adjusted_results.append(adjusted_result) + + return adjusted_results + + +def find_text_by_content( + image: np.ndarray, + target_text: str, + language: str = "en", + similarity_threshold: float = 0.8, + case_sensitive: bool = False, +) -> list[TextResult]: + """ + Find specific text content in image + + Args: + image: Input image + target_text: Text to search for + language: Language for OCR + similarity_threshold: Minimum similarity for matches (0-1) + case_sensitive: Whether to match case exactly + + Returns: + List of matching TextResult objects + + Example: + # Find all occurrences of "Submit" button + submit_buttons = find_text_by_content(image, "Submit", similarity_threshold=0.9) + """ + # Extract all text + all_text = extract_text(image, language) + matches = [] + + # Prepare target text for comparison + target = target_text.strip() + if not case_sensitive: + target = target.lower() + + for text_result in all_text: + detected = text_result.text.strip() + if not case_sensitive: + detected = detected.lower() + + # Simple substring matching + if target in detected or detected in target: + matches.append(text_result) + + return matches + + +def get_text_near_point( + image: np.ndarray, point: tuple[int, int], radius: int = 50, language: str = "en" +) -> list[TextResult]: + """ + Get text near a specific point + + Args: + image: Input image + point: Center point (x, y) + radius: Search radius in pixels + language: Language for OCR + + Returns: + List of nearby TextResult objects sorted by distance + """ + # Extract all text + all_text = extract_text(image, language) + nearby_text = [] + + px, py = point + + for text_result in all_text: + tx, ty = text_result.center + distance = ((tx - px) ** 2 + (ty - py) ** 2) ** 0.5 + + if distance <= radius: + # Add distance info for sorting + text_result_with_distance = text_result + text_result_with_distance.distance = distance # type: ignore + nearby_text.append(text_result_with_distance) + + # Sort by distance + nearby_text.sort(key=lambda x: getattr(x, "distance", float("inf"))) + + return nearby_text + + +def draw_text_results(image: np.ndarray, text_results: list[TextResult]) -> np.ndarray: + """ + Draw text detection results on image + + Args: + image: Input image + text_results: List of text results to draw + + Returns: + Image with drawn text bounding boxes and labels + """ + result_image = image.copy() + + for text_result in text_results: + # Draw bounding box + bbox_points = np.array(text_result.bbox, dtype=np.int32) + cv2.polylines(result_image, [bbox_points], True, (255, 0, 0), 2) + + # Draw text label + x1, y1, x2, y2 = text_result.rect_bbox + label = f"{text_result.text} ({text_result.confidence:.2f})" + + # Background for text + label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)[0] + cv2.rectangle( + result_image, + (x1, y1 - label_size[1] - 10), + (x1 + label_size[0], y1), + (255, 0, 0), + -1, + ) + + # Text + cv2.putText( + result_image, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2 + ) + + return result_image diff --git a/src/ocr/setup_models.py b/src/ocr/setup_models.py new file mode 100644 index 0000000..7ba5b4e --- /dev/null +++ b/src/ocr/setup_models.py @@ -0,0 +1,207 @@ +"""Model Setup and Management Functions + +Downloads and manages YOLO and PaddleOCR models for computer vision tasks. +""" + +import os +from pathlib import Path + +import paddleocr +from ultralytics import YOLO + + +def get_model_paths() -> dict[str, str]: + """ + Get paths to model files + + Returns: + Dictionary with model paths + + Example: + paths = get_model_paths() + yolo_path = paths['yolo_onnx'] + """ + models_dir = Path(__file__).parent / "models" + + return { + "models_dir": str(models_dir), + "yolo_onnx": str(models_dir / "yolov8s.onnx"), + "yolo_pt": str(models_dir / "yolov8s.pt"), + } + + +def download_models() -> dict[str, str]: + """ + Download and setup all required models + + Returns: + Dictionary with downloaded model paths + + Example: + paths = download_models() + print(f"Models ready at: {paths}") + """ + print("๐Ÿ” Setting up computer vision models...") + + # Setup YOLO + yolo_path = setup_yolo_model() + + # Setup PaddleOCR + setup_paddle_ocr() + + paths = get_model_paths() + + print("โœ… All models ready!") + return paths + + +def setup_models() -> dict[str, str]: + """Alias for download_models() for backward compatibility""" + return download_models() + + +def setup_yolo_model() -> str: + """ + Download and convert YOLOv8s to ONNX format + + Returns: + Path to ONNX model file + """ + models_dir = Path(__file__).parent / "models" + models_dir.mkdir(exist_ok=True) + + onnx_path = models_dir / "yolov8s.onnx" + pt_path = models_dir / "yolov8s.pt" + + if onnx_path.exists(): + print(f"โœ… YOLOv8s ONNX already exists: {onnx_path}") + return str(onnx_path) + + print("๐Ÿ“ฅ Downloading YOLOv8s and converting to ONNX...") + + try: + # Load pretrained YOLOv8s model (will download if not present) + model = YOLO("yolov8s.pt") + + # Export to ONNX format optimized for CPU inference + model.export( + format="onnx", + imgsz=640, + optimize=True, + half=False, # Keep FP32 for CPU compatibility + dynamic=False, + simplify=True, + ) + + # Move to models directory + if os.path.exists("yolov8s.onnx"): + os.rename("yolov8s.onnx", str(onnx_path)) + if os.path.exists("yolov8s.pt"): + os.rename("yolov8s.pt", str(pt_path)) + + print(f"โœ… YOLOv8s ONNX saved to: {onnx_path}") + + except Exception as e: + print(f"โŒ YOLOv8s setup failed: {e}") + raise + + return str(onnx_path) + + +def setup_paddle_ocr() -> bool: + """ + Initialize PaddleOCR (downloads models on first use) + + Returns: + True if successful + """ + print("๐Ÿ“ฅ Setting up PaddleOCR...") + + try: + # Initialize PaddleOCR - this will download models on first use + ocr = paddleocr.PaddleOCR(use_textline_orientation=True, lang="en") + + print("โœ… PaddleOCR initialized successfully") + return True + + except Exception as e: + print(f"โŒ PaddleOCR setup failed: {e}") + raise + + +def verify_models() -> dict[str, bool]: + """ + Verify that all models are properly installed + + Returns: + Dictionary with verification status for each model + """ + print("๐Ÿ” Verifying model installations...") + + results = {"yolo_onnx": False, "paddle_ocr": False} + + # Check YOLO ONNX + try: + import onnxruntime as ort + + paths = get_model_paths() + + if os.path.exists(paths["yolo_onnx"]): + session = ort.InferenceSession(paths["yolo_onnx"]) + results["yolo_onnx"] = True + print("โœ… YOLOv8s ONNX model verified") + else: + print("โŒ YOLOv8s ONNX model not found") + + except Exception as e: + print(f"โŒ YOLOv8s ONNX verification failed: {e}") + + # Check PaddleOCR + try: + ocr = paddleocr.PaddleOCR(use_textline_orientation=True, lang="en") + results["paddle_ocr"] = True + print("โœ… PaddleOCR verified") + + except Exception as e: + print(f"โŒ PaddleOCR verification failed: {e}") + + return results + + +def cleanup_temp_files(): + """Clean up temporary files from model downloads""" + temp_files = ["yolov8s.pt", "yolov8s.onnx"] + + for temp_file in temp_files: + if os.path.exists(temp_file): + os.remove(temp_file) + print(f"๐Ÿงน Cleaned up {temp_file}") + + +if __name__ == "__main__": + # Run setup when called directly + print("Computer Vision Model Setup") + print("=" * 40) + + try: + paths = download_models() + print("\n๐Ÿ“‹ Model Summary:") + for name, path in paths.items(): + print(f" {name}: {path}") + + print("\n๐Ÿ” Verification:") + verification = verify_models() + for model, status in verification.items(): + status_icon = "โœ…" if status else "โŒ" + print(f" {model}: {status_icon}") + + cleanup_temp_files() + + if all(verification.values()): + print("\n๐ŸŽ‰ All models ready for computer vision tasks!") + else: + print("\nโš ๏ธ Some models failed verification. Check errors above.") + + except Exception as e: + print(f"\nโŒ Setup failed: {e}") + cleanup_temp_files() diff --git a/src/ocr/verification.py b/src/ocr/verification.py new file mode 100644 index 0000000..bbbec56 --- /dev/null +++ b/src/ocr/verification.py @@ -0,0 +1,319 @@ +"""Action verification using clean OCR functions + +This module provides verification tools that work with the clean OCR functions +instead of the old object-oriented UIFinder approach. +""" + +import time +from dataclasses import dataclass +from typing import Any + +import cv2 +import numpy as np + +from . import detect_ui_elements, extract_text_from_region, find_elements_by_text + + +@dataclass +class VerificationResult: + """Result of an action verification""" + + success: bool + message: str + confidence: float + screenshot: np.ndarray | None = None + found_elements: list[Any] | None = None + + +def verify_click_success( + before_screenshot: np.ndarray, + after_screenshot: np.ndarray, + expected_change: str = "any", +) -> VerificationResult: + """ + Verify that a click action was successful + + Args: + before_screenshot: Screenshot before click + after_screenshot: Screenshot after click + expected_change: Type of change expected ("any", "dialog", "page_change", etc.) + + Returns: + VerificationResult + """ + # Calculate difference between screenshots + diff = cv2.absdiff(before_screenshot, after_screenshot) + change_percentage = np.mean(diff > 10) * 100 # Pixels that changed significantly + + if change_percentage < 1.0: + return VerificationResult( + success=False, + message=f"No significant screen change detected ({change_percentage:.1f}%)", + confidence=0.1, + screenshot=after_screenshot, + ) + + # Look for specific changes based on expected_change + if expected_change == "dialog": + # Look for new dialog boxes or windows + elements_before = detect_ui_elements(before_screenshot, confidence_threshold=0.6) + elements_after = detect_ui_elements(after_screenshot, confidence_threshold=0.6) + + new_elements = len(elements_after) - len(elements_before) + if new_elements > 0: + return VerificationResult( + success=True, + message=f"Dialog appeared - {new_elements} new UI elements detected", + confidence=0.8, + screenshot=after_screenshot, + found_elements=elements_after, + ) + + # General success based on screen change + confidence = min(change_percentage / 10.0, 1.0) # Scale to 0-1 + + return VerificationResult( + success=True, + message=f"Screen changed ({change_percentage:.1f}% of pixels)", + confidence=confidence, + screenshot=after_screenshot, + ) + + +def verify_text_input( + screenshot: np.ndarray, input_region: tuple[int, int, int, int], expected_text: str +) -> VerificationResult: + """ + Verify that text was correctly entered in an input field + + Args: + screenshot: Screenshot after text input + input_region: Region of the input field (x1, y1, x2, y2) + expected_text: Text that should be present + + Returns: + VerificationResult + """ + # Read text from the input region + detected_text = extract_text_from_region(screenshot, input_region, confidence_threshold=0.5) + + if not detected_text: + return VerificationResult( + success=False, + message="No text detected in input field", + confidence=0.1, + screenshot=screenshot, + ) + + # Get the most confident text detection + best_detection = max(detected_text, key=lambda x: x.confidence) + + # Simple text matching (could be improved with fuzzy matching) + detected_clean = best_detection.text.strip().lower() + expected_clean = expected_text.strip().lower() + + if expected_clean in detected_clean or detected_clean in expected_clean: + confidence = 0.9 if detected_clean == expected_clean else 0.7 + return VerificationResult( + success=True, + message=f"Text verified: '{best_detection.text}' matches expected '{expected_text}'", + confidence=confidence, + screenshot=screenshot, + ) + + return VerificationResult( + success=False, + message=f"Text mismatch: found '{best_detection.text}', expected '{expected_text}'", + confidence=0.2, + screenshot=screenshot, + ) + + +def verify_element_present( + screenshot: np.ndarray, element_description: str, confidence_threshold: float = 0.5 +) -> VerificationResult: + """ + Verify that a specific UI element is present + + Args: + screenshot: Current screenshot + element_description: Description of element to find + confidence_threshold: Minimum confidence threshold + + Returns: + VerificationResult + """ + # Try to find element by text + matching_elements = find_elements_by_text( + screenshot, element_description, confidence_threshold=confidence_threshold + ) + + if matching_elements: + return VerificationResult( + success=True, + message=f"Element found: {element_description}", + confidence=max(elem.confidence for elem in matching_elements), + screenshot=screenshot, + found_elements=matching_elements, + ) + + # Try to find any UI elements and check if they contain keywords + ui_elements = detect_ui_elements(screenshot, confidence_threshold=confidence_threshold) + + # Simple keyword matching + keywords = element_description.lower().split() + for element in ui_elements: + if element.text and any(keyword in element.text.lower() for keyword in keywords): + return VerificationResult( + success=True, + message=f"Element found by keyword match: {element.text}", + confidence=element.confidence, + screenshot=screenshot, + found_elements=[element], + ) + + return VerificationResult( + success=False, + message=f"Element not found: {element_description}", + confidence=0.1, + screenshot=screenshot, + ) + + +def verify_page_loaded( + screenshot: np.ndarray, expected_indicators: list[str], confidence_threshold: float = 0.5 +) -> VerificationResult: + """ + Verify that a page/application has loaded completely + + Args: + screenshot: Current screenshot + expected_indicators: List of text/elements that should be present + confidence_threshold: Minimum confidence threshold + + Returns: + VerificationResult + """ + found_indicators = [] + total_confidence = 0 + + for indicator in expected_indicators: + elements = find_elements_by_text( + screenshot, indicator, confidence_threshold=confidence_threshold + ) + if elements: + found_indicators.append(indicator) + total_confidence += max(elem.confidence for elem in elements) + + if not found_indicators: + return VerificationResult( + success=False, + message=f"Page load verification failed - none of {expected_indicators} found", + confidence=0.1, + screenshot=screenshot, + ) + + success_rate = len(found_indicators) / len(expected_indicators) + avg_confidence = total_confidence / len(found_indicators) + + return VerificationResult( + success=success_rate >= 0.5, # At least half the indicators should be present + message=f"Page loaded - found {len(found_indicators)}/{len(expected_indicators)} indicators: {found_indicators}", + confidence=avg_confidence * success_rate, + screenshot=screenshot, + ) + + +def wait_for_element( + capture_func, + element_description: str, + max_attempts: int = 10, + delay: float = 1.0, + confidence_threshold: float = 0.5, +) -> VerificationResult: + """ + Wait for a specific element to appear + + Args: + capture_func: Function to capture current screenshot + element_description: Description of element to wait for + max_attempts: Maximum number of attempts + delay: Delay between attempts in seconds + confidence_threshold: Minimum confidence threshold + + Returns: + VerificationResult + """ + for attempt in range(max_attempts): + screenshot = capture_func() + if screenshot is None: + continue + + result = verify_element_present(screenshot, element_description, confidence_threshold) + if result.success: + result.message += f" (found after {attempt + 1} attempts)" + return result + + if attempt < max_attempts - 1: # Don't sleep on last attempt + time.sleep(delay) + + return VerificationResult( + success=False, + message=f"Element '{element_description}' not found after {max_attempts} attempts", + confidence=0.1, + ) + + +def compare_screenshots(screen1: np.ndarray, screen2: np.ndarray) -> dict[str, float]: + """ + Compare two screenshots and return similarity metrics + + Args: + screen1: First screenshot + screen2: Second screenshot + + Returns: + Dictionary with similarity metrics + """ + if screen1.shape != screen2.shape: + return {"similarity": 0.0, "mse": float("inf"), "change_percentage": 100.0} + + # Mean Squared Error + mse = np.mean((screen1.astype(float) - screen2.astype(float)) ** 2) + + # Change percentage + diff = cv2.absdiff(screen1, screen2) + change_percentage = np.mean(diff > 10) * 100 + + # Similarity score (inverse of normalized MSE) + max_possible_mse = 255**2 + similarity = 1.0 - (mse / max_possible_mse) + + return {"similarity": similarity, "mse": mse, "change_percentage": change_percentage} + + +def create_diff_visualization(screen1: np.ndarray, screen2: np.ndarray) -> np.ndarray: + """ + Create a visualization showing differences between two screenshots + + Args: + screen1: First screenshot + screen2: Second screenshot + + Returns: + Difference visualization image + """ + if screen1.shape != screen2.shape: + return np.zeros_like(screen1) + + # Create difference image + diff = cv2.absdiff(screen1, screen2) + + # Threshold to highlight significant changes + thresh = cv2.threshold(cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY), 10, 255, cv2.THRESH_BINARY)[1] + + # Create colored difference visualization + diff_colored = diff.copy() + diff_colored[thresh > 0] = [0, 0, 255] # Highlight changes in red + + return diff_colored diff --git a/src/setup_models.py b/src/setup_models.py deleted file mode 100644 index 406ae1d..0000000 --- a/src/setup_models.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Setup YOLOv8s ONNX model and PaddleOCR for POC""" - -import os -from pathlib import Path - -import paddleocr -from ultralytics import YOLO - - -def setup_yolo_onnx(): - """Download and export YOLOv8s to ONNX format""" - models_dir = Path(__file__).parent / "models" - models_dir.mkdir(exist_ok=True) - - onnx_path = models_dir / "yolov8s.onnx" - - if not onnx_path.exists(): - print("Downloading YOLOv8s and exporting to ONNX...") - # Load pretrained YOLOv8s model - model = YOLO("yolov8s.pt") - - # Export to ONNX format optimized for CPU inference - model.export( - format="onnx", - imgsz=640, - optimize=True, - half=False, # Keep FP32 for CPU compatibility - dynamic=False, - simplify=True, - ) - - # Move to models directory - os.rename("yolov8s.onnx", str(onnx_path)) - print(f"YOLOv8s ONNX model saved to: {onnx_path}") - else: - print(f"YOLOv8s ONNX model already exists: {onnx_path}") - - return onnx_path - - -def setup_paddle_ocr(): - """Initialize PaddleOCR with optimal settings""" - print("Setting up PaddleOCR...") - - # Initialize with English support, CPU mode for POC - ocr = paddleocr.PaddleOCR( - lang="en" # English language - ) - - print("PaddleOCR initialized successfully") - return ocr - - -def main(): - """Setup all models for POC""" - print("Setting up VM Automation POC models...") - - # Setup YOLOv8s ONNX - yolo_path = setup_yolo_onnx() - - # Setup PaddleOCR - setup_paddle_ocr() - - # Test basic functionality - print("\nTesting model setup...") - - # Test YOLO loading - try: - import onnxruntime as ort - - ort.InferenceSession(str(yolo_path)) - print("โœ“ YOLOv8s ONNX model loads successfully") - except Exception as e: - print(f"โœ— YOLOv8s ONNX model loading failed: {e}") - - # Test PaddleOCR - try: - # Small test (this will download models on first run) - print("โœ“ PaddleOCR initialized successfully") - except Exception as e: - print(f"โœ— PaddleOCR setup failed: {e}") - - print("\nModel setup complete!") - print(f"YOLOv8s ONNX: {yolo_path}") - print("PaddleOCR: Ready for inference") - - -if __name__ == "__main__": - main() diff --git a/src/tools/__init__.py b/src/tools/__init__.py deleted file mode 100644 index a580cc2..0000000 --- a/src/tools/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Tools module for VM interaction""" - -from .input_actions import ActionResult, InputActions -from .screen_capture import ScreenCapture -from .verification import ActionVerifier, VerificationResult - -__all__ = [ - "ActionResult", - "ActionVerifier", - "InputActions", - "ScreenCapture", - "VerificationResult", -] diff --git a/src/tools/verification.py b/src/tools/verification.py deleted file mode 100644 index 4d32930..0000000 --- a/src/tools/verification.py +++ /dev/null @@ -1,313 +0,0 @@ -"""Action verification and validation tools""" - -import time -from dataclasses import dataclass - -import cv2 -import numpy as np - -from src.vision.ocr_reader import OCRReader -from src.vision.ui_finder import UIElement, UIFinder - - -@dataclass -class VerificationResult: - """Result of an action verification""" - - success: bool - message: str - confidence: float - screenshot: np.ndarray | None = None - found_elements: list[UIElement] | None = None - - -class ActionVerifier: - """Verify that actions were successful""" - - def __init__(self, ui_finder: UIFinder, ocr_reader: OCRReader): - """ - Initialize action verifier - - Args: - ui_finder: UI finder instance - ocr_reader: OCR reader instance - """ - self.ui_finder = ui_finder - self.ocr_reader = ocr_reader - - def verify_click_success( - self, - before_screenshot: np.ndarray, - after_screenshot: np.ndarray, - expected_change: str = "any", - ) -> VerificationResult: - """ - Verify that a click action was successful - - Args: - before_screenshot: Screenshot before click - after_screenshot: Screenshot after click - expected_change: Type of change expected ("any", "dialog", "page_change", etc.) - - Returns: - VerificationResult - """ - # Calculate difference between screenshots - diff = cv2.absdiff(before_screenshot, after_screenshot) - change_percentage = np.mean(diff > 10) * 100 # Pixels that changed significantly - - if change_percentage < 1.0: - return VerificationResult( - success=False, - message=f"No significant screen change detected ({change_percentage:.1f}%)", - confidence=0.1, - screenshot=after_screenshot, - ) - - # Look for specific changes based on expected_change - if expected_change == "dialog": - # Look for new dialog boxes or windows - elements_before = self.ui_finder.find_ui_elements(before_screenshot) - elements_after = self.ui_finder.find_ui_elements(after_screenshot) - - new_elements = len(elements_after) - len(elements_before) - if new_elements > 0: - return VerificationResult( - success=True, - message=f"Dialog appeared - {new_elements} new UI elements detected", - confidence=0.8, - screenshot=after_screenshot, - found_elements=elements_after, - ) - - # General success based on screen change - confidence = min(change_percentage / 10.0, 1.0) # Scale to 0-1 - - return VerificationResult( - success=True, - message=f"Screen changed ({change_percentage:.1f}% of pixels)", - confidence=confidence, - screenshot=after_screenshot, - ) - - def verify_text_input( - self, screenshot: np.ndarray, input_region: tuple[int, int, int, int], expected_text: str - ) -> VerificationResult: - """ - Verify that text was correctly entered in an input field - - Args: - screenshot: Screenshot after text input - input_region: Region of the input field (x1, y1, x2, y2) - expected_text: Text that should be present - - Returns: - VerificationResult - """ - # Read text from the input region - detected_text = self.ocr_reader.read_field_value(screenshot, input_region) - - if detected_text is None: - return VerificationResult( - success=False, - message="No text detected in input field", - confidence=0.1, - screenshot=screenshot, - ) - - # Simple text matching (could be improved with fuzzy matching) - detected_clean = detected_text.strip().lower() - expected_clean = expected_text.strip().lower() - - if expected_clean in detected_clean or detected_clean in expected_clean: - confidence = 0.9 if detected_clean == expected_clean else 0.7 - return VerificationResult( - success=True, - message=f"Text verified: '{detected_text}' matches expected '{expected_text}'", - confidence=confidence, - screenshot=screenshot, - ) - - return VerificationResult( - success=False, - message=f"Text mismatch: found '{detected_text}', expected '{expected_text}'", - confidence=0.2, - screenshot=screenshot, - ) - - def verify_element_present( - self, screenshot: np.ndarray, element_description: str - ) -> VerificationResult: - """ - Verify that a specific UI element is present - - Args: - screenshot: Current screenshot - element_description: Description of element to find - - Returns: - VerificationResult - """ - # Try to find element by text - matching_elements = self.ui_finder.find_element_by_text(screenshot, element_description) - - if matching_elements: - return VerificationResult( - success=True, - message=f"Element found: {element_description}", - confidence=max(elem.confidence for elem in matching_elements), - screenshot=screenshot, - found_elements=matching_elements, - ) - - # Try to find clickable elements if text search failed - clickable_elements = self.ui_finder.find_clickable_elements(screenshot) - - # Simple keyword matching - keywords = element_description.lower().split() - for element in clickable_elements: - if element.text and any(keyword in element.text.lower() for keyword in keywords): - return VerificationResult( - success=True, - message=f"Element found by keyword match: {element.text}", - confidence=element.confidence, - screenshot=screenshot, - found_elements=[element], - ) - - return VerificationResult( - success=False, - message=f"Element not found: {element_description}", - confidence=0.1, - screenshot=screenshot, - ) - - def verify_page_loaded( - self, screenshot: np.ndarray, expected_indicators: list[str], timeout: int = 5 - ) -> VerificationResult: - """ - Verify that a page/application has loaded completely - - Args: - screenshot: Current screenshot - expected_indicators: List of text/elements that should be present - timeout: Not used in this implementation (for compatibility) - - Returns: - VerificationResult - """ - found_indicators = [] - total_confidence = 0 - - for indicator in expected_indicators: - elements = self.ui_finder.find_element_by_text(screenshot, indicator) - if elements: - found_indicators.append(indicator) - total_confidence += max(elem.confidence for elem in elements) - - if not found_indicators: - return VerificationResult( - success=False, - message=f"Page load verification failed - none of {expected_indicators} found", - confidence=0.1, - screenshot=screenshot, - ) - - success_rate = len(found_indicators) / len(expected_indicators) - avg_confidence = total_confidence / len(found_indicators) - - return VerificationResult( - success=success_rate >= 0.5, # At least half the indicators should be present - message=f"Page loaded - found {len(found_indicators)}/{len(expected_indicators)} indicators: {found_indicators}", - confidence=avg_confidence * success_rate, - screenshot=screenshot, - ) - - def wait_for_element( - self, capture_func, element_description: str, max_attempts: int = 10, delay: float = 1.0 - ) -> VerificationResult: - """ - Wait for a specific element to appear - - Args: - capture_func: Function to capture current screenshot - element_description: Description of element to wait for - max_attempts: Maximum number of attempts - delay: Delay between attempts in seconds - - Returns: - VerificationResult - """ - for attempt in range(max_attempts): - screenshot = capture_func() - if screenshot is None: - continue - - result = self.verify_element_present(screenshot, element_description) - if result.success: - result.message += f" (found after {attempt + 1} attempts)" - return result - - if attempt < max_attempts - 1: # Don't sleep on last attempt - time.sleep(delay) - - return VerificationResult( - success=False, - message=f"Element '{element_description}' not found after {max_attempts} attempts", - confidence=0.1, - ) - - def compare_screenshots(self, screen1: np.ndarray, screen2: np.ndarray) -> dict[str, float]: - """ - Compare two screenshots and return similarity metrics - - Args: - screen1: First screenshot - screen2: Second screenshot - - Returns: - Dictionary with similarity metrics - """ - if screen1.shape != screen2.shape: - return {"similarity": 0.0, "mse": float("inf"), "change_percentage": 100.0} - - # Mean Squared Error - mse = np.mean((screen1.astype(float) - screen2.astype(float)) ** 2) - - # Change percentage - diff = cv2.absdiff(screen1, screen2) - change_percentage = np.mean(diff > 10) * 100 - - # Similarity score (inverse of normalized MSE) - max_possible_mse = 255**2 - similarity = 1.0 - (mse / max_possible_mse) - - return {"similarity": similarity, "mse": mse, "change_percentage": change_percentage} - - def create_diff_visualization(self, screen1: np.ndarray, screen2: np.ndarray) -> np.ndarray: - """ - Create a visualization showing differences between two screenshots - - Args: - screen1: First screenshot - screen2: Second screenshot - - Returns: - Difference visualization image - """ - if screen1.shape != screen2.shape: - return np.zeros_like(screen1) - - # Create difference image - diff = cv2.absdiff(screen1, screen2) - - # Threshold to highlight significant changes - thresh = cv2.threshold(cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY), 10, 255, cv2.THRESH_BINARY)[ - 1 - ] - - # Create colored difference visualization - diff_colored = diff.copy() - diff_colored[thresh > 0] = [0, 0, 255] # Highlight changes in red - - return diff_colored diff --git a/src/vision/__init__.py b/src/vision/__init__.py deleted file mode 100644 index 494fe6f..0000000 --- a/src/vision/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Vision module for UI element detection and OCR""" - -from .ocr_reader import OCRReader, TextDetection -from .ui_finder import UIElement, UIFinder -from .yolo_detector import Detection, YOLODetector - -__all__ = ["Detection", "OCRReader", "TextDetection", "UIElement", "UIFinder", "YOLODetector"] diff --git a/src/vision/ocr_reader.py b/src/vision/ocr_reader.py deleted file mode 100644 index e292656..0000000 --- a/src/vision/ocr_reader.py +++ /dev/null @@ -1,210 +0,0 @@ -"""PaddleOCR wrapper for text recognition in UI elements""" - -from dataclasses import dataclass - -import cv2 -import numpy as np -import paddleocr - - -@dataclass -class TextDetection: - """Text detection and recognition result""" - - text: str - confidence: float - bbox: list[tuple[int, int]] # 4 corner points - rect_bbox: tuple[int, int, int, int] # x1, y1, x2, y2 - center: tuple[int, int] - - -class OCRReader: - """PaddleOCR wrapper for text recognition""" - - def __init__(self, use_gpu: bool = False, lang: str = "en"): - """Initialize PaddleOCR""" - self.ocr = paddleocr.PaddleOCR(lang=lang) - print(f"PaddleOCR initialized (GPU: {use_gpu}, Lang: {lang})") - - def read_text( - self, image: np.ndarray, region: tuple[int, int, int, int] | None = None - ) -> list[TextDetection]: - """ - Read text from image or specific region - - Args: - image: Input image - region: Optional region (x1, y1, x2, y2) to crop before OCR - - Returns: - List of TextDetection objects - """ - # Crop region if specified - if region: - x1, y1, x2, y2 = region - cropped_image = image[y1:y2, x1:x2] - offset_x, offset_y = x1, y1 - else: - cropped_image = image - offset_x, offset_y = 0, 0 - - # Run OCR - try: - results = self.ocr.ocr(cropped_image, cls=True) - - if not results or not results[0]: - return [] - - text_detections = [] - - for line in results[0]: - if not line: - continue - - bbox_points, (text, confidence) = line - - if confidence < 0.5: # Skip low confidence text - continue - - # Convert bbox points to absolute coordinates - abs_bbox_points = [(int(x + offset_x), int(y + offset_y)) for x, y in bbox_points] - - # Calculate rectangular bounding box - x_coords = [point[0] for point in abs_bbox_points] - y_coords = [point[1] for point in abs_bbox_points] - - x1, y1 = min(x_coords), min(y_coords) - x2, y2 = max(x_coords), max(y_coords) - - # Calculate center - center_x = (x1 + x2) // 2 - center_y = (y1 + y2) // 2 - - text_detection = TextDetection( - text=text.strip(), - confidence=confidence, - bbox=abs_bbox_points, - rect_bbox=(x1, y1, x2, y2), - center=(center_x, center_y), - ) - - text_detections.append(text_detection) - - return text_detections - - except Exception as e: - print(f"OCR error: {e}") - return [] - - def find_text( - self, image: np.ndarray, target_text: str, threshold: float = 0.8 - ) -> list[TextDetection]: - """ - Find specific text in image - - Args: - image: Input image - target_text: Text to search for - threshold: Similarity threshold (0-1) - - Returns: - List of matching TextDetection objects - """ - all_text = self.read_text(image) - matches = [] - - target_lower = target_text.lower().strip() - - for text_det in all_text: - detected_lower = text_det.text.lower().strip() - - # Simple substring matching - if target_lower in detected_lower or detected_lower in target_lower: - matches.append(text_det) - - # Fuzzy matching could be added here for better accuracy - # using libraries like fuzzywuzzy - - return matches - - def read_field_value( - self, image: np.ndarray, field_region: tuple[int, int, int, int] - ) -> str | None: - """ - Read text from a specific field region (e.g., input box) - - Args: - image: Input image - field_region: Region coordinates (x1, y1, x2, y2) - - Returns: - Detected text or None - """ - text_detections = self.read_text(image, field_region) - - if not text_detections: - return None - - # Return the text with highest confidence - best_detection = max(text_detections, key=lambda x: x.confidence) - return best_detection.text - - def draw_text_detections( - self, image: np.ndarray, text_detections: list[TextDetection] - ) -> np.ndarray: - """Draw text detection results on image""" - result_image = image.copy() - - for text_det in text_detections: - # Draw bounding box - bbox_points = np.array(text_det.bbox, dtype=np.int32) - cv2.polylines(result_image, [bbox_points], True, (255, 0, 0), 2) - - # Draw text label - x1, y1, x2, y2 = text_det.rect_bbox - label = f"{text_det.text} ({text_det.confidence:.2f})" - - # Background for text - label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)[0] - cv2.rectangle( - result_image, - (x1, y1 - label_size[1] - 10), - (x1 + label_size[0], y1), - (255, 0, 0), - -1, - ) - - # Text - cv2.putText( - result_image, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2 - ) - - return result_image - - def get_text_near_point( - self, image: np.ndarray, point: tuple[int, int], radius: int = 50 - ) -> list[TextDetection]: - """ - Get text detections near a specific point - - Args: - image: Input image - point: Center point (x, y) - radius: Search radius in pixels - - Returns: - List of nearby TextDetection objects - """ - all_text = self.read_text(image) - nearby_text = [] - - px, py = point - - for text_det in all_text: - tx, ty = text_det.center - distance = ((tx - px) ** 2 + (ty - py) ** 2) ** 0.5 - - if distance <= radius: - nearby_text.append(text_det) - - return nearby_text diff --git a/src/vision/ui_finder.py b/src/vision/ui_finder.py deleted file mode 100644 index 0b49538..0000000 --- a/src/vision/ui_finder.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Combined UI element finder using YOLO + OCR""" - -import math -from dataclasses import dataclass - -import cv2 -import numpy as np - -from .ocr_reader import OCRReader, TextDetection -from .yolo_detector import Detection, YOLODetector - - -@dataclass -class UIElement: - """Combined UI element with both visual and text information""" - - element_type: str # "detected" or "text" or "combined" - bbox: tuple[int, int, int, int] # x1, y1, x2, y2 - center: tuple[int, int] - confidence: float - - # Visual detection info (if available) - yolo_detection: Detection | None = None - - # Text info (if available) - text_detection: TextDetection | None = None - text: str | None = None - - # Metadata - element_id: str | None = None - description: str | None = None - - -class UIFinder: - """Combined UI element finder using YOLO detection + OCR""" - - def __init__(self, yolo_model_path: str, yolo_confidence: float = 0.6, use_gpu: bool = False): - """Initialize UI finder with YOLO and OCR""" - self.yolo_detector = YOLODetector(yolo_model_path, yolo_confidence) - self.ocr_reader = OCRReader(use_gpu=use_gpu) - - print("UIFinder initialized with YOLOv8s + PaddleOCR") - - def find_ui_elements(self, image: np.ndarray) -> list[UIElement]: - """Find all UI elements using both YOLO and OCR""" - # Get YOLO detections - yolo_detections = self.yolo_detector.detect(image) - - # Get OCR text detections - text_detections = self.ocr_reader.read_text(image) - - # Combine results - ui_elements = [] - - # Add YOLO detections as UI elements - for detection in yolo_detections: - ui_element = UIElement( - element_type="detected", - bbox=detection.bbox, - center=detection.center, - confidence=detection.confidence, - yolo_detection=detection, - description=f"Detected {detection.class_name}", - ) - ui_elements.append(ui_element) - - # Add text detections as UI elements - for text_det in text_detections: - ui_element = UIElement( - element_type="text", - bbox=text_det.rect_bbox, - center=text_det.center, - confidence=text_det.confidence, - text_detection=text_det, - text=text_det.text, - description=f"Text: {text_det.text}", - ) - ui_elements.append(ui_element) - - # Try to combine nearby elements (text labels with detected objects) - combined_elements = self._combine_nearby_elements(ui_elements) - - return combined_elements - - def find_element_by_text( - self, image: np.ndarray, text_query: str, search_radius: int = 100 - ) -> list[UIElement]: - """ - Find UI elements by text content or nearby text labels - - Args: - image: Screenshot image - text_query: Text to search for - search_radius: Radius to search for nearby visual elements - - Returns: - List of matching UI elements - """ - # Find text matches - text_matches = self.ocr_reader.find_text(image, text_query) - - if not text_matches: - return [] - - # Get YOLO detections for context - yolo_detections = self.yolo_detector.detect(image) - - results = [] - - for text_match in text_matches: - # Create UI element for the text itself - text_element = UIElement( - element_type="text", - bbox=text_match.rect_bbox, - center=text_match.center, - confidence=text_match.confidence, - text_detection=text_match, - text=text_match.text, - description=f"Text match: {text_match.text}", - ) - results.append(text_element) - - # Find nearby visual elements (likely the associated control) - nearby_elements = self._find_nearby_visual_elements( - text_match.center, yolo_detections, search_radius - ) - - for nearby in nearby_elements: - combined_element = UIElement( - element_type="combined", - bbox=nearby.bbox, - center=nearby.center, - confidence=(text_match.confidence + nearby.confidence) / 2, - yolo_detection=nearby, - text_detection=text_match, - text=text_match.text, - description=f"{nearby.class_name} near '{text_match.text}'", - ) - results.append(combined_element) - - return results - - def find_clickable_elements(self, image: np.ndarray) -> list[UIElement]: - """Find likely clickable elements (buttons, links, etc.)""" - # Classes that are typically clickable - clickable_classes = { - "mouse", - "remote", - "cell phone", - "laptop", - "keyboard", - "tv", - "book", - "clock", # These might represent UI elements in screenshots - } - - yolo_detections = self.yolo_detector.detect(image) - clickable_elements = [] - - for detection in yolo_detections: - if detection.class_name in clickable_classes: - ui_element = UIElement( - element_type="detected", - bbox=detection.bbox, - center=detection.center, - confidence=detection.confidence, - yolo_detection=detection, - description=f"Clickable {detection.class_name}", - ) - clickable_elements.append(ui_element) - - # Also look for text that might be buttons - text_detections = self.ocr_reader.read_text(image) - button_keywords = [ - "button", - "click", - "submit", - "ok", - "cancel", - "yes", - "no", - "save", - "delete", - ] - - for text_det in text_detections: - text_lower = text_det.text.lower() - if any(keyword in text_lower for keyword in button_keywords): - ui_element = UIElement( - element_type="text", - bbox=text_det.rect_bbox, - center=text_det.center, - confidence=text_det.confidence, - text_detection=text_det, - text=text_det.text, - description=f"Button text: {text_det.text}", - ) - clickable_elements.append(ui_element) - - return clickable_elements - - def find_input_fields(self, image: np.ndarray) -> list[UIElement]: - """Find input fields by looking for rectangular regions and nearby labels""" - # This is a simplified implementation - # In a real scenario, you might train YOLO specifically for UI elements - - # Look for text that indicates input fields - text_detections = self.ocr_reader.read_text(image) - input_indicators = ["name", "email", "password", "address", "phone", "field", "input"] - - input_fields = [] - - for text_det in text_detections: - text_lower = text_det.text.lower() - if any(indicator in text_lower for indicator in input_indicators): - # This text might be a label for an input field - # Look for rectangular regions nearby - ui_element = UIElement( - element_type="text", - bbox=text_det.rect_bbox, - center=text_det.center, - confidence=text_det.confidence, - text_detection=text_det, - text=text_det.text, - description=f"Input field label: {text_det.text}", - ) - input_fields.append(ui_element) - - return input_fields - - def _combine_nearby_elements( - self, ui_elements: list[UIElement], max_distance: int = 50 - ) -> list[UIElement]: - """Combine nearby visual and text elements""" - combined_elements = ui_elements.copy() - - # This is a simplified implementation - # In practice, you might want more sophisticated spatial reasoning - - return combined_elements - - def _find_nearby_visual_elements( - self, center_point: tuple[int, int], detections: list[Detection], max_distance: int - ) -> list[Detection]: - """Find visual elements near a text point""" - px, py = center_point - nearby = [] - - for detection in detections: - dx, dy = detection.center - distance = math.sqrt((dx - px) ** 2 + (dy - py) ** 2) - - if distance <= max_distance: - nearby.append(detection) - - return nearby - - def get_element_screenshot( - self, image: np.ndarray, element: UIElement, padding: int = 5 - ) -> np.ndarray: - """Extract screenshot of a specific UI element""" - x1, y1, x2, y2 = element.bbox - - # Add padding - x1 = max(0, x1 - padding) - y1 = max(0, y1 - padding) - x2 = min(image.shape[1], x2 + padding) - y2 = min(image.shape[0], y2 + padding) - - return image[y1:y2, x1:x2] - - def draw_ui_elements(self, image: np.ndarray, ui_elements: list[UIElement]) -> np.ndarray: - """Draw all UI elements on image for visualization""" - result_image = image.copy() - - for element in ui_elements: - x1, y1, x2, y2 = element.bbox - - # Choose color based on element type - if element.element_type == "detected": - color = (0, 255, 0) # Green for YOLO detections - elif element.element_type == "text": - color = (255, 0, 0) # Blue for OCR text - else: # combined - color = (0, 255, 255) # Yellow for combined elements - - # Draw bounding box - cv2.rectangle(result_image, (x1, y1), (x2, y2), color, 2) - - # Draw label - label = element.description or "Unknown" - label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] - cv2.rectangle( - result_image, (x1, y1 - label_size[1] - 10), (x1 + label_size[0], y1), color, -1 - ) - cv2.putText( - result_image, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2 - ) - - return result_image diff --git a/src/vision/yolo_detector.py b/src/vision/yolo_detector.py deleted file mode 100644 index bcabcd1..0000000 --- a/src/vision/yolo_detector.py +++ /dev/null @@ -1,286 +0,0 @@ -"""YOLOv8s-ONNX inference for UI element detection""" - -from dataclasses import dataclass -from pathlib import Path - -import cv2 -import numpy as np -import onnxruntime as ort - - -@dataclass -class Detection: - """UI element detection result""" - - class_name: str - confidence: float - bbox: tuple[int, int, int, int] # x1, y1, x2, y2 - center: tuple[int, int] - id: str | None = None - - -class YOLODetector: - """YOLOv8s-ONNX detector for UI elements""" - - # COCO classes relevant for UI detection - UI_CLASSES = { - 0: "person", - 1: "bicycle", - 2: "car", - 3: "motorcycle", - 4: "airplane", - 5: "bus", - 6: "train", - 7: "truck", - 8: "boat", - 9: "traffic light", - 10: "fire hydrant", - 11: "stop sign", - 12: "parking meter", - 13: "bench", - 14: "bird", - 15: "cat", - 16: "dog", - 17: "horse", - 18: "sheep", - 19: "cow", - 20: "elephant", - 21: "bear", - 22: "zebra", - 23: "giraffe", - 24: "backpack", - 25: "umbrella", - 26: "handbag", - 27: "tie", - 28: "suitcase", - 29: "frisbee", - 30: "skis", - 31: "snowboard", - 32: "sports ball", - 33: "kite", - 34: "baseball bat", - 35: "baseball glove", - 36: "skateboard", - 37: "surfboard", - 38: "tennis racket", - 39: "bottle", - 40: "wine glass", - 41: "cup", - 42: "fork", - 43: "knife", - 44: "spoon", - 45: "bowl", - 46: "banana", - 47: "apple", - 48: "sandwich", - 49: "orange", - 50: "broccoli", - 51: "carrot", - 52: "hot dog", - 53: "pizza", - 54: "donut", - 55: "cake", - 56: "chair", - 57: "couch", - 58: "potted plant", - 59: "bed", - 60: "dining table", - 61: "toilet", - 62: "tv", - 63: "laptop", - 64: "mouse", - 65: "remote", - 66: "keyboard", - 67: "cell phone", - 68: "microwave", - 69: "oven", - 70: "toaster", - 71: "sink", - 72: "refrigerator", - 73: "book", - 74: "clock", - 75: "vase", - 76: "scissors", - 77: "teddy bear", - 78: "hair drier", - 79: "toothbrush", - } - - def __init__(self, model_path: str, confidence_threshold: float = 0.6): - """Initialize YOLOv8s-ONNX detector""" - self.model_path = Path(model_path) - self.confidence_threshold = confidence_threshold - - # Initialize ONNX Runtime session - self.session = ort.InferenceSession(str(self.model_path)) - - # Get model input details - self.input_name = self.session.get_inputs()[0].name - self.input_shape = self.session.get_inputs()[0].shape - self.input_height, self.input_width = self.input_shape[2], self.input_shape[3] - - print(f"YOLOv8s ONNX loaded: {self.model_path}") - print(f"Input shape: {self.input_shape}") - - def preprocess(self, image: np.ndarray) -> np.ndarray: - """Preprocess image for YOLO inference""" - # Resize and normalize - resized = cv2.resize(image, (self.input_width, self.input_height)) - - # Convert BGR to RGB - rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) - - # Normalize to [0, 1] and convert to float32 - normalized = rgb.astype(np.float32) / 255.0 - - # Transpose to CHW format and add batch dimension - input_tensor = normalized.transpose(2, 0, 1)[np.newaxis, ...] - - return input_tensor - - def postprocess(self, outputs: np.ndarray, original_shape: tuple[int, int]) -> list[Detection]: - """Postprocess YOLO outputs to detections""" - predictions = outputs[0] # Shape: [1, 84, 8400] for YOLOv8s - - # Transpose to [8400, 84] - predictions = predictions.transpose(0, 2, 1)[0] - - detections = [] - orig_h, orig_w = original_shape - - for pred in predictions: - # Extract box coordinates and confidence - x_center, y_center, width, height = pred[0:4] - confidence = pred[4] - class_scores = pred[5:] - - if confidence < self.confidence_threshold: - continue - - # Get class with highest score - class_id = np.argmax(class_scores) - class_confidence = class_scores[class_id] * confidence - - if class_confidence < self.confidence_threshold: - continue - - # Convert to original image coordinates - x_center *= orig_w / self.input_width - y_center *= orig_h / self.input_height - width *= orig_w / self.input_width - height *= orig_h / self.input_height - - # Convert to bbox format - x1 = int(x_center - width / 2) - y1 = int(y_center - height / 2) - x2 = int(x_center + width / 2) - y2 = int(y_center + height / 2) - - # Ensure bbox is within image bounds - x1 = max(0, min(x1, orig_w)) - y1 = max(0, min(y1, orig_h)) - x2 = max(0, min(x2, orig_w)) - y2 = max(0, min(y2, orig_h)) - - detection = Detection( - class_name=self.UI_CLASSES.get(class_id, f"class_{class_id}"), - confidence=float(class_confidence), - bbox=(x1, y1, x2, y2), - center=(int(x_center), int(y_center)), - ) - - detections.append(detection) - - # Apply Non-Maximum Suppression - detections = self._apply_nms(detections, iou_threshold=0.5) - - return detections - - def _apply_nms(self, detections: list[Detection], iou_threshold: float) -> list[Detection]: - """Apply Non-Maximum Suppression""" - if not detections: - return detections - - # Sort by confidence - detections.sort(key=lambda x: x.confidence, reverse=True) - - filtered_detections = [] - - for detection in detections: - # Check if this detection overlaps significantly with any kept detection - keep = True - for kept_detection in filtered_detections: - if self._calculate_iou(detection.bbox, kept_detection.bbox) > iou_threshold: - keep = False - break - - if keep: - filtered_detections.append(detection) - - return filtered_detections - - def _calculate_iou( - self, box1: tuple[int, int, int, int], box2: tuple[int, int, int, int] - ) -> float: - """Calculate Intersection over Union""" - x1_1, y1_1, x2_1, y2_1 = box1 - x1_2, y1_2, x2_2, y2_2 = box2 - - # Calculate intersection area - x1_i = max(x1_1, x1_2) - y1_i = max(y1_1, y1_2) - x2_i = min(x2_1, x2_2) - y2_i = min(y2_1, y2_2) - - if x2_i <= x1_i or y2_i <= y1_i: - return 0.0 - - intersection_area = (x2_i - x1_i) * (y2_i - y1_i) - - # Calculate union area - area1 = (x2_1 - x1_1) * (y2_1 - y1_1) - area2 = (x2_2 - x1_2) * (y2_2 - y1_2) - union_area = area1 + area2 - intersection_area - - return intersection_area / union_area if union_area > 0 else 0.0 - - def detect(self, image: np.ndarray) -> list[Detection]: - """Detect UI elements in image""" - original_shape = image.shape[:2] # (height, width) - - # Preprocess - input_tensor = self.preprocess(image) - - # Run inference - outputs = self.session.run(None, {self.input_name: input_tensor}) - - # Postprocess - detections = self.postprocess(outputs, original_shape) - - return detections - - def draw_detections(self, image: np.ndarray, detections: list[Detection]) -> np.ndarray: - """Draw bounding boxes and labels on image""" - result_image = image.copy() - - for detection in detections: - x1, y1, x2, y2 = detection.bbox - - # Draw bounding box - cv2.rectangle(result_image, (x1, y1), (x2, y2), (0, 255, 0), 2) - - # Draw label - label = f"{detection.class_name}: {detection.confidence:.2f}" - label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] - cv2.rectangle( - result_image, - (x1, y1 - label_size[1] - 10), - (x1 + label_size[0], y1), - (0, 255, 0), - -1, - ) - cv2.putText( - result_image, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2 - ) - - return result_image diff --git a/src/agents/app_controller.py b/src/vm/automation/app_controller.py similarity index 99% rename from src/agents/app_controller.py rename to src/vm/automation/app_controller.py index 196ef33..02962d3 100644 --- a/src/agents/app_controller.py +++ b/src/vm/automation/app_controller.py @@ -4,7 +4,7 @@ import time from typing import Any -from .shared_context import VMSession, VMTarget +from vm.automation.shared_context import VMSession, VMTarget class AppControllerTools: diff --git a/src/agents/shared_context.py b/src/vm/automation/shared_context.py similarity index 100% rename from src/agents/shared_context.py rename to src/vm/automation/shared_context.py diff --git a/src/agents/vm_navigator.py b/src/vm/automation/vm_navigator.py similarity index 84% rename from src/agents/vm_navigator.py rename to src/vm/automation/vm_navigator.py index 6115b09..a931405 100644 --- a/src/agents/vm_navigator.py +++ b/src/vm/automation/vm_navigator.py @@ -1,16 +1,27 @@ """VM Navigator Agent - Production version without mocks""" import asyncio +import sys import time from pathlib import Path from typing import Any -from src.tools.input_actions import InputActions -from src.tools.screen_capture import ScreenCapture -from src.tools.verification import ActionVerifier -from src.vision.ui_finder import UIFinder +# Add src to path for clean OCR imports +sys.path.append(str(Path(__file__).parent.parent.parent)) -from .shared_context import VMSession, VMTarget +from ocr import ( + detect_ui_elements, + extract_text, + find_elements_by_text, + verify_click_success, + verify_page_loaded, +) + +# TODO: ActionVerifier needs to be updated to work with new clean OCR functions +# from ocr.verification import ActionVerifier +from vm.automation.shared_context import VMSession, VMTarget +from vm.tools.input_actions import InputActions +from vm.tools.screen_capture import ScreenCapture class VMNavigatorTools: @@ -26,17 +37,9 @@ def __init__(self, session: VMSession, vm_target: VMTarget): # InputActions will be initialized after connection is established self.input_actions = None - # Initialize vision components - models_dir = Path(__file__).parent.parent / "models" - yolo_path = models_dir / "yolov8s.onnx" - - if not yolo_path.exists(): - raise FileNotFoundError( - f"YOLO model not found: {yolo_path}. Run setup_models.py first." - ) - - self.ui_finder = UIFinder(str(yolo_path)) - self.verifier = ActionVerifier(self.ui_finder, self.ui_finder.ocr_reader) + # Vision components now use clean OCR functions + # No need for UIFinder initialization - using pure functions + # ActionVerifier will need to be updated separately def connect_to_vm(self) -> dict[str, Any]: """Connect to the VM""" @@ -123,49 +126,50 @@ def handle_intermediate_screen(self, timeout: int = 30) -> dict[str, Any]: # First check if this is an Acceptable Use screen - elements = self.ui_finder.find_element_by_text(screenshot, "Acceptable Use") + elements = find_elements_by_text( + screenshot, "Acceptable Use", confidence_threshold=0.7 + ) if elements: best_element = max(elements, key=lambda x: x.confidence) - if best_element.confidence > 0.7: - self.session.log_action("Found Acceptable Use screen") - acceptable_use_found = True - break + self.session.log_action("Found Acceptable Use screen") + acceptable_use_found = True + break # If Acceptable Use screen found, look specifically for OK button to click if acceptable_use_found: - ok_elements = self.ui_finder.find_element_by_text(screenshot, "OK") + ok_elements = find_elements_by_text(screenshot, "OK", confidence_threshold=0.7) if ok_elements: best_ok = max(ok_elements, key=lambda x: x.confidence) - if best_ok.confidence > 0.7: - if self.input_actions is None: - return {"success": False, "error": "Input actions not initialized"} + if self.input_actions is None: + return {"success": False, "error": "Input actions not initialized"} + + x, y = best_ok.center + self.session.log_action( + f"Clicking OK button at ({x}, {y}) on Acceptable Use screen" + ) + result = self.input_actions.click(x, y) - x, y = best_ok.center + if result.success: self.session.log_action( - f"Clicking OK button at ({x}, {y}) on Acceptable Use screen" + "Successfully clicked OK on Acceptable Use screen" ) - result = self.input_actions.click(x, y) - - if result.success: - self.session.log_action( - "Successfully clicked OK on Acceptable Use screen" - ) - time.sleep(3.0) # Wait for screen transition - return { - "success": True, - "message": "Handled Acceptable Use screen: clicked OK", - } - else: - self.session.log_action(f"Failed to click OK: {result.message}") + time.sleep(3.0) # Wait for screen transition + return { + "success": True, + "message": "Handled Acceptable Use screen: clicked OK", + } + else: + self.session.log_action(f"Failed to click OK: {result.message}") # If no intermediate elements found, check if desktop is already loaded - if self.vm_target.expected_desktop_elements: - result = self.verifier.verify_page_loaded( - screenshot, self.vm_target.expected_desktop_elements, timeout=2 - ) - if result.success: - self.session.log_action("Desktop already loaded, no intermediate screen") - return {"success": True, "message": "No intermediate screen, desktop ready"} + # TODO: Update ActionVerifier to work with new clean OCR functions + # if self.vm_target.expected_desktop_elements: + # result = self.verifier.verify_page_loaded( + # screenshot, self.vm_target.expected_desktop_elements, timeout=2 + # ) + # if result.success: + # self.session.log_action("Desktop already loaded, no intermediate screen") + # return {"success": True, "message": "No intermediate screen, desktop ready"} # Wait before next check time.sleep(2.0) @@ -195,17 +199,17 @@ def wait_for_desktop_loaded(self, timeout: int = 60) -> dict[str, Any]: self.session.add_screenshot(screenshot, "Desktop loading check") - # Use verifier to check for desktop elements - result = self.verifier.verify_page_loaded( - screenshot, self.vm_target.expected_desktop_elements, timeout=5 - ) + # TODO: Update ActionVerifier to work with new clean OCR functions + # For now, we'll do a basic check for any text elements + text_elements = extract_text(screenshot, confidence_threshold=0.5) + desktop_loaded = len(text_elements) > 0 - if result.success: - self.session.log_action("Desktop loaded successfully") + if desktop_loaded: + self.session.log_action("Desktop loaded successfully (basic check)") return { "success": True, - "message": result.message, - "confidence": result.confidence, + "message": "Desktop appears to be loaded (found text elements)", + "confidence": 0.8, } # Wait before next check @@ -238,7 +242,7 @@ def find_application_with_retry(self, app_name: str, max_retries: int = 3) -> di ) # Look for the application by name/text - elements = self.ui_finder.find_element_by_text(screenshot, app_name) + elements = find_elements_by_text(screenshot, app_name, confidence_threshold=0.6) if elements: best_element = max(elements, key=lambda x: x.confidence) @@ -309,10 +313,8 @@ def launch_application_verified(self, element_info: dict[str, Any]) -> dict[str, self.session.add_screenshot(after_screenshot, "After launching application") - # Verify that something changed (app launched) - verification = self.verifier.verify_click_success( - before_screenshot, after_screenshot, "page_change" - ) + # Verify that something changed (app launched) using clean verification + verification = verify_click_success(before_screenshot, after_screenshot, "page_change") if verification.success: self.session.log_action( @@ -354,8 +356,12 @@ def verify_patient_banner(self, expected_patient_info: dict[str, str]) -> dict[s height, width = screenshot.shape[:2] banner_region = (0, 0, width, int(height * 0.2)) - # Read text from patient banner area - text_detections = self.ui_finder.ocr_reader.read_text(screenshot, banner_region) + # Read text from patient banner area using clean OCR + from ocr import extract_text_from_region + + text_detections = extract_text_from_region( + screenshot, banner_region, confidence_threshold=0.5 + ) if not text_detections: return { @@ -435,21 +441,23 @@ def verify_application_loaded_enhanced(self) -> dict[str, Any]: # 1. Check for expected app elements if self.vm_target.expected_app_elements: - result = self.verifier.verify_page_loaded( - screenshot, self.vm_target.expected_app_elements + result = verify_page_loaded( + screenshot, self.vm_target.expected_app_elements, confidence_threshold=0.5 ) verification_results.append(("app_elements", result.success, result.message)) # 2. Check for any UI elements (generic check) - elements = self.ui_finder.find_ui_elements(screenshot) + elements = detect_ui_elements(screenshot, confidence_threshold=0.6) ui_elements_found = len(elements) > 0 verification_results.append( ("ui_elements", ui_elements_found, f"Found {len(elements)} UI elements") ) # 3. Check window title or specific text - app_name_elements = self.ui_finder.find_element_by_text( - screenshot, self.vm_target.target_app_name.replace(".exe", "") + app_name_elements = find_elements_by_text( + screenshot, + self.vm_target.target_app_name.replace(".exe", ""), + confidence_threshold=0.6, ) app_title_found = len(app_name_elements) > 0 verification_results.append( @@ -507,8 +515,8 @@ def __init__( self.shared_components = { "screen_capture": self.tools.screen_capture, "input_actions": self.tools.input_actions, - "ui_finder": self.tools.ui_finder, - "verifier": self.tools.verifier, + # Note: ui_finder removed - now using clean OCR functions directly + # "verifier": self.tools.verifier, # TODO: Update ActionVerifier for new OCR } async def execute_navigation( diff --git a/src/connections/__init__.py b/src/vm/connections/__init__.py similarity index 65% rename from src/connections/__init__.py rename to src/vm/connections/__init__.py index 7b5322c..71eefd3 100644 --- a/src/connections/__init__.py +++ b/src/vm/connections/__init__.py @@ -1,16 +1,22 @@ -"""VM Connection abstraction layer""" +"""VM Connection Module + +Provides connection abstractions for remote VMs via VNC and RDP protocols. +""" from .base import ActionResult, ConnectionResult, VMConnection +from .desktop_connection import DesktopConnection from .rdp_connection import RDPConnection from .vnc_connection import VNCConnection def create_connection(connection_type: str) -> VMConnection: - """Factory function to create connection based on type""" + """Factory function to create VM connections""" if connection_type.lower() == "vnc": return VNCConnection() elif connection_type.lower() == "rdp": return RDPConnection() + elif connection_type.lower() == "desktop": + return DesktopConnection() else: raise ValueError(f"Unsupported connection type: {connection_type}") @@ -18,6 +24,7 @@ def create_connection(connection_type: str) -> VMConnection: __all__ = [ "ActionResult", "ConnectionResult", + "DesktopConnection", "RDPConnection", "VMConnection", "VNCConnection", diff --git a/src/connections/base.py b/src/vm/connections/base.py similarity index 100% rename from src/connections/base.py rename to src/vm/connections/base.py diff --git a/src/vm/connections/desktop_connection.py b/src/vm/connections/desktop_connection.py new file mode 100644 index 0000000..5f6995a --- /dev/null +++ b/src/vm/connections/desktop_connection.py @@ -0,0 +1,284 @@ +"""Desktop connection implementation for local macOS desktop interaction""" + +import subprocess +from pathlib import Path + +import cv2 +import numpy as np + +from vm.connections.base import ActionResult, ConnectionResult, VMConnection + + +class DesktopConnection(VMConnection): + """Desktop connection implementation for local macOS desktop manipulation""" + + def __init__(self): + super().__init__() + self.screenshot_path = Path("/tmp/desktop_screenshot.png") + + def connect( + self, + host: str = "localhost", + port: int = 0, + username: str | None = None, + password: str | None = None, + **kwargs, + ) -> ConnectionResult: + """Connect to local desktop (always succeeds)""" + try: + # No actual connection needed for local desktop + self.is_connected = True + + self.connection_info = { + "type": "desktop", + "host": "localhost", + "platform": "macOS", + "screenshot_path": str(self.screenshot_path), + } + + return ConnectionResult(True, "Connected to local desktop") + + except Exception as e: + self.is_connected = False + return ConnectionResult(False, f"Desktop connection failed: {e}") + + def disconnect(self) -> ConnectionResult: + """Disconnect from desktop (cleanup only)""" + try: + self.is_connected = False + self.connection_info = {} + + # Clean up screenshot file if it exists + if self.screenshot_path.exists(): + self.screenshot_path.unlink() + + return ConnectionResult(True, "Desktop disconnected") + + except Exception as e: + return ConnectionResult(False, f"Desktop disconnect error: {e}") + + def capture_screen(self) -> tuple[bool, np.ndarray | None]: + """Capture desktop screenshot using macOS screencapture""" + if not self.is_connected: + return False, None + + try: + # Use macOS screencapture for entire screen + cmd = ["screencapture", "-x", str(self.screenshot_path)] + result = subprocess.run(cmd, check=True, capture_output=True) + + if result.returncode != 0: + print(f"Screenshot command failed: {result.stderr.decode()}") + return False, None + + # Load image with OpenCV + if self.screenshot_path.exists(): + image = cv2.imread(str(self.screenshot_path)) + if image is not None: + return True, image + + return False, None + + except subprocess.CalledProcessError as e: + print(f"Desktop screen capture error: {e}") + return False, None + except Exception as e: + print(f"Desktop screen capture error: {e}") + return False, None + + def capture_window(self, interactive: bool = True) -> tuple[bool, np.ndarray | None]: + """Capture specific window (interactive selection)""" + if not self.is_connected: + return False, None + + try: + if interactive: + # Interactive window selection + cmd = ["screencapture", "-w", "-x", str(self.screenshot_path)] + print("Click on the window you want to capture...") + else: + # Fallback to full screen + return self.capture_screen() + + result = subprocess.run(cmd, check=True) + + if result.returncode != 0: + print("Window capture failed, falling back to full screen") + return self.capture_screen() + + # Load image with OpenCV + if self.screenshot_path.exists(): + image = cv2.imread(str(self.screenshot_path)) + if image is not None: + return True, image + + return False, None + + except subprocess.CalledProcessError as e: + print(f"Window capture error: {e}, falling back to full screen") + return self.capture_screen() + except Exception as e: + print(f"Window capture error: {e}") + return False, None + + def click(self, x: int, y: int, button: str = "left") -> ActionResult: + """Click at coordinates using AppleScript""" + if not self.is_connected: + return ActionResult(False, "No desktop connection") + + try: + # Map button names to AppleScript button numbers + button_map = {"left": "1", "right": "2", "middle": "3"} + btn_num = button_map.get(button, "1") + + # Use AppleScript for clicking + script = f'tell application "System Events" to click at {{{x}, {y}}}' + if button != "left": + # For non-left clicks, use control-click as approximation + script = f'tell application "System Events" to tell (click at {{{x}, {y}}}) to key down control' + + cmd = ["osascript", "-e", script] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Clicked {button} at ({x}, {y})") + else: + return ActionResult(False, f"Click failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Desktop click error: {e}") + except Exception as e: + return ActionResult(False, f"Desktop click error: {e}") + + def click_with_cliclick(self, x: int, y: int, button: str = "left") -> ActionResult: + """Alternative click method using cliclick (if installed)""" + if not self.is_connected: + return ActionResult(False, "No desktop connection") + + try: + # Check if cliclick is available (brew install cliclick) + import shutil + + if not shutil.which("cliclick"): + return self.click(x, y, button) # Fallback to AppleScript + + # Map button names + button_map = {"left": "c", "right": "rc", "middle": "mc"} + click_type = button_map.get(button, "c") + + cmd = ["cliclick", f"{click_type}:{x},{y}"] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Clicked {button} at ({x}, {y}) via cliclick") + else: + return ActionResult(False, f"Cliclick failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Cliclick error: {e}") + except Exception: + # Fallback to AppleScript + return self.click(x, y, button) + + def type_text(self, text: str) -> ActionResult: + """Type text using AppleScript""" + if not self.is_connected: + return ActionResult(False, "No desktop connection") + + try: + # Escape quotes in text for AppleScript + escaped_text = text.replace('"', '\\"').replace("'", "\\'") + + script = f'tell application "System Events" to keystroke "{escaped_text}"' + cmd = ["osascript", "-e", script] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Typed: {text}") + else: + return ActionResult(False, f"Type failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Desktop type error: {e}") + except Exception as e: + return ActionResult(False, f"Desktop type error: {e}") + + def key_press(self, key: str) -> ActionResult: + """Press key using AppleScript""" + if not self.is_connected: + return ActionResult(False, "No desktop connection") + + try: + # Map common key names to AppleScript key codes + key_mapping = { + "enter": "return", + "return": "return", + "escape": "escape", + "tab": "tab", + "space": "space", + "backspace": "delete", + "delete": "forward delete", + "up": "up arrow", + "down": "down arrow", + "left": "left arrow", + "right": "right arrow", + "cmd": "command", + "ctrl": "control", + "alt": "option", + "shift": "shift", + } + + apple_key = key_mapping.get(key.lower(), key) + + # Handle special key combinations + if key.lower().startswith("cmd+") or key.lower().startswith("ctrl+"): + parts = key.lower().split("+") + modifier = parts[0] + target_key = parts[1] if len(parts) > 1 else "" + + modifier_map = {"cmd": "command", "ctrl": "control"} + mod_key = modifier_map.get(modifier, modifier) + + script = f'tell application "System Events" to keystroke "{target_key}" using {mod_key} down' + else: + script = f'tell application "System Events" to key code (key code of "{apple_key}")' + # Simpler approach for single keys + if apple_key in ["return", "escape", "tab", "space", "delete"]: + script = f'tell application "System Events" to keystroke "{apple_key}"' + + cmd = ["osascript", "-e", script] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + + if result.returncode == 0: + return ActionResult(True, f"Pressed key: {key}") + else: + return ActionResult(False, f"Key press failed: {result.stderr}") + + except subprocess.CalledProcessError as e: + return ActionResult(False, f"Desktop key press error: {e}") + except Exception as e: + return ActionResult(False, f"Desktop key press error: {e}") + + def get_desktop_info(self) -> dict: + """Get desktop-specific information""" + try: + # Get screen resolution + cmd = ["system_profiler", "SPDisplaysDataType", "-json"] + result = subprocess.run(cmd, capture_output=True, text=True) + + info = { + "platform": "macOS", + "connection_type": "desktop", + "screenshot_capability": True, + "click_capability": True, + "keyboard_capability": True, + } + + if result.returncode == 0: + # Could parse JSON for detailed display info + info["system_profiler_available"] = True + + return info + + except Exception as e: + return {"platform": "macOS", "connection_type": "desktop", "error": str(e)} diff --git a/src/connections/rdp_connection.py b/src/vm/connections/rdp_connection.py similarity index 99% rename from src/connections/rdp_connection.py rename to src/vm/connections/rdp_connection.py index 74fb058..87ddf22 100644 --- a/src/connections/rdp_connection.py +++ b/src/vm/connections/rdp_connection.py @@ -11,7 +11,7 @@ import cv2 import numpy as np -from .base import ActionResult, ConnectionResult, VMConnection +from vm.connections.base import ActionResult, ConnectionResult, VMConnection class RDPConnection(VMConnection): @@ -99,7 +99,7 @@ def connect( "brew install freerdp imagemagick xorg-server xdotool\n" "Alternative: Use VNC connection instead of RDP for better macOS compatibility.", ) - + # If we reach here, Xvfb should be available - this shouldn't happen # because the outer if-condition should have caught it return ConnectionResult(False, "Unexpected Xvfb availability state") @@ -416,7 +416,6 @@ def _capture_with_xwd_convert(self, env: dict) -> bool: except Exception: return False - def _find_free_display(self) -> int: """Find a free X11 display number""" for display_num in range(10, 100): diff --git a/src/connections/vnc_connection.py b/src/vm/connections/vnc_connection.py similarity index 98% rename from src/connections/vnc_connection.py rename to src/vm/connections/vnc_connection.py index 8c98625..7c56ff1 100644 --- a/src/connections/vnc_connection.py +++ b/src/vm/connections/vnc_connection.py @@ -6,7 +6,7 @@ import numpy as np import vncdotool.api as vnc -from .base import ActionResult, ConnectionResult, VMConnection +from vm.connections.base import ActionResult, ConnectionResult, VMConnection class VNCConnection(VMConnection): diff --git a/src/main.py b/src/vm/main.py similarity index 95% rename from src/main.py rename to src/vm/main.py index 553416c..9762550 100644 --- a/src/main.py +++ b/src/vm/main.py @@ -13,7 +13,9 @@ from pathlib import Path from typing import Any -from src.agents import AppControllerAgent, VMNavigatorAgent, VMSession, VMTarget +from vm.automation.app_controller import AppControllerAgent +from vm.automation.shared_context import VMSession, VMTarget +from vm.automation.vm_navigator import VMNavigatorAgent @dataclass @@ -100,8 +102,8 @@ def to_vm_target(self) -> VMTarget: connection_type=self.connection_type, target_app_name=self.target_app_name, target_button_text=self.target_button_text, - expected_desktop_elements=self.expected_desktop_elements, - expected_app_elements=self.expected_app_elements, + expected_desktop_elements=self.expected_desktop_elements or [], + expected_app_elements=self.expected_app_elements or [], vm_connection_timeout=self.vm_connection_timeout, desktop_load_timeout=self.desktop_load_timeout, app_launch_timeout=self.app_launch_timeout, @@ -311,39 +313,48 @@ def save_session_log(self, filepath: str | None = None) -> str: def cleanup(self): """Clean up all connections and resources""" print("\n๐Ÿงน Cleaning up connections...") - + try: # Clean up any tracked connections for connection in self._connections_to_cleanup: try: - if hasattr(connection, 'disconnect'): + if hasattr(connection, "disconnect"): result = connection.disconnect() if result.success: print(f"โœ“ {result.message}") else: print(f"โš  {result.message}") - elif hasattr(connection, 'close'): + elif hasattr(connection, "close"): connection.close() print("โœ“ Connection closed") except Exception as e: print(f"โš  Error disconnecting: {e}") - + # Clean up VM Navigator connections - if hasattr(self.vm_navigator, 'tools') and self.vm_navigator.tools and hasattr(self.vm_navigator.tools, 'screen_capture'): + if ( + hasattr(self.vm_navigator, "tools") + and self.vm_navigator.tools + and hasattr(self.vm_navigator.tools, "screen_capture") + ): try: self.vm_navigator.tools.screen_capture.disconnect() print("โœ“ VM Navigator connection cleaned up") except Exception as e: print(f"โš  Error cleaning VM Navigator: {e}") - + # Clean up App Controller connections - if self.app_controller and hasattr(self.app_controller, 'tools') and self.app_controller.tools and hasattr(self.app_controller.tools, 'screen_capture'): + if ( + self.app_controller + and hasattr(self.app_controller, "tools") + and self.app_controller.tools + and hasattr(self.app_controller.tools, "screen_capture") + ): try: self.app_controller.tools.screen_capture.disconnect() print("โœ“ App Controller connection cleaned up") except Exception as e: print(f"โš  Error cleaning App Controller: {e}") - + except Exception as e: print(f"โš  Error during cleanup: {e}") diff --git a/src/tools/input_actions.py b/src/vm/tools/input_actions.py similarity index 98% rename from src/tools/input_actions.py rename to src/vm/tools/input_actions.py index 78d49c0..be682b9 100644 --- a/src/tools/input_actions.py +++ b/src/vm/tools/input_actions.py @@ -2,8 +2,8 @@ import time -from src.connections import VMConnection -from src.connections.base import ActionResult +from vm.connections import VMConnection +from vm.connections.base import ActionResult class InputActions: diff --git a/src/tools/screen_capture.py b/src/vm/tools/screen_capture.py similarity index 98% rename from src/tools/screen_capture.py rename to src/vm/tools/screen_capture.py index 50c1df5..96f5e9f 100644 --- a/src/tools/screen_capture.py +++ b/src/vm/tools/screen_capture.py @@ -5,7 +5,7 @@ import cv2 import numpy as np -from src.connections import VMConnection, create_connection +from vm.connections import VMConnection, create_connection class ScreenCapture: diff --git a/tests/mock_components.py b/tests/mock_components.py index d295d36..321b1b5 100644 --- a/tests/mock_components.py +++ b/tests/mock_components.py @@ -7,10 +7,10 @@ import cv2 import numpy as np -from src.tools.input_actions import ActionResult, InputActions -from src.tools.screen_capture import ScreenCapture -from src.tools.verification import VerificationResult -from src.vision.ui_finder import UIElement +from ocr.verification.verification import VerificationResult +from ocr.vision.finder import UIElement +from vm.tools.input_actions import ActionResult, InputActions +from vm.tools.screen_capture import ScreenCapture class MockScreenCapture(ScreenCapture): @@ -132,7 +132,7 @@ class MockInputActions(InputActions): def __init__(self): # Create a mock connection for the parent class - from src.connections.base import ActionResult, VMConnection + from connections.base import ActionResult, VMConnection class MockConnection(VMConnection): def __init__(self): @@ -140,12 +140,12 @@ def __init__(self): self.is_connected = True def connect(self, host, port, username=None, password=None, **kwargs): - from src.connections.base import ConnectionResult + from connections.base import ConnectionResult return ConnectionResult(True, "Mock connected") def disconnect(self): - from src.connections.base import ConnectionResult + from connections.base import ConnectionResult return ConnectionResult(True, "Mock disconnected") diff --git a/uv.lock b/uv.lock index 5ca4bbf..3417667 100644 --- a/uv.lock +++ b/uv.lock @@ -158,7 +158,7 @@ wheels = [ [[package]] name = "arize-phoenix" -version = "11.31.0" +version = "11.32.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aioitertools" }, @@ -203,14 +203,14 @@ dependencies = [ { name = "uvicorn" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/5e/0752bc0885e3a9590faede3d0c59657df488c81545ea4d1353a03a660f23/arize_phoenix-11.31.0.tar.gz", hash = "sha256:6a3088076c917c0d2dfa836efae6ed1eefe80f57cc3d1b8dc478ece386474c4b", size = 2211811 } +sdist = { url = "https://files.pythonhosted.org/packages/8a/7b/9283eec95b4369a423cc7247a22f46aaef271e501e98cf2cc32a2e50efb2/arize_phoenix-11.32.1.tar.gz", hash = "sha256:bea46dbc19fe042eb667c9f8a801f26aa0a0ae6ec2b4f03e217d346729e693db", size = 2212701 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/53/f046ea4e47a30389c8122a00c9aa4c9d175ba986559b65028540986f3e8a/arize_phoenix-11.31.0-py3-none-any.whl", hash = "sha256:13ad68f3c1ee42495d7479d64eca5957af2ec91aa3f9ebfc35456ed015a8c77e", size = 2407938 }, + { url = "https://files.pythonhosted.org/packages/54/0a/96eb87290a4bb52ae0fb37aff53e4f6fc61c31ad1cd44db2a99b8d760851/arize_phoenix-11.32.1-py3-none-any.whl", hash = "sha256:2d24c3577f119e77e9d0bea48c4a29f303feb877ea8f7e45e32ec705a5a02ab1", size = 2409226 }, ] [[package]] name = "arize-phoenix-client" -version = "1.16.0" +version = "1.18.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -220,9 +220,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/fd/9ac5ea4f206cf8803af0d1b45db8cd2f8785688a0471f309c8661b8773c2/arize_phoenix_client-1.16.0.tar.gz", hash = "sha256:b3b14ed5469bd72059ee2273062c8a97ca0bc4bf4a71262250e8f606ac009eee", size = 104348 } +sdist = { url = "https://files.pythonhosted.org/packages/89/c2/432a84d7733f826e12b0c31e1d72484bd041e7554bdad103c0cf2bcfff26/arize_phoenix_client-1.18.0.tar.gz", hash = "sha256:cbce676d32e413bfe136f98b20393feecc850d9a5f48870bde4707920ab4705b", size = 114810 } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/01/14e8026d78d66673b69949d38175a49864255375f73f0b272844d675c737/arize_phoenix_client-1.16.0-py3-none-any.whl", hash = "sha256:54fbc2aefd8771d8421b5d2ccd45cb7d20322d71e7c234d1dfc5d7ad5557655e", size = 107719 }, + { url = "https://files.pythonhosted.org/packages/4c/db/70060f5b498cfd10639f79e4d4f50c3758dcb136b3d2717ef618313a39ef/arize_phoenix_client-1.18.0-py3-none-any.whl", hash = "sha256:271a2dc96f24965a847550f5f0422dcbe834c64af3a14ffc108b65fd87a5f606", size = 119389 }, ] [[package]] @@ -316,6 +316,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/1f/d3fd91808a1f4881b4072424390d38e85707edd75ed5d9cea2a0299a7a7a/bce_python_sdk-0.9.45-py3-none-any.whl", hash = "sha256:cce3ca7ad4de8be2cc0722c1d6a7db7be6f2833f8d9ca7f892c572e6ff78a959", size = 352012 }, ] +[[package]] +name = "beautifulsoup4" +version = "4.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/2e/3e5079847e653b1f6dc647aa24549d68c6addb4c595cc0d902d1b19308ad/beautifulsoup4-4.13.5.tar.gz", hash = "sha256:5e70131382930e7c3de33450a2f54a63d5e4b19386eab43a5b34d594268f3695", size = 622954 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/eb/f4151e0c7377a6e08a38108609ba5cede57986802757848688aeedd1b9e8/beautifulsoup4-4.13.5-py3-none-any.whl", hash = "sha256:642085eaa22233aceadff9c69651bc51e8bf3f874fb6d7104ece2beb24b47c4a", size = 105113 }, +] + [[package]] name = "cachetools" version = "5.5.2" @@ -451,7 +464,7 @@ dependencies = [ { name = "openai" }, { name = "openai-agents" }, { name = "opencv-python" }, - { name = "paddleocr" }, + { name = "paddleocr", extra = ["all"] }, { name = "paddlepaddle" }, { name = "pillow" }, { name = "ultralytics" }, @@ -473,31 +486,31 @@ dev = [ [package.metadata] requires-dist = [ - { name = "fastapi", specifier = ">=0.104.0" }, - { name = "numpy", specifier = ">=1.24.0" }, + { name = "fastapi", specifier = ">=0.116.1" }, + { name = "numpy", specifier = ">=2.3.2" }, { name = "onnx", specifier = ">=1.19.0" }, - { name = "onnxruntime", specifier = ">=1.16.0" }, - { name = "openai", specifier = ">=1.52.0" }, - { name = "openai-agents", specifier = ">=0.2.9" }, - { name = "opencv-python", specifier = ">=4.8.0" }, - { name = "paddleocr", specifier = ">=2.7.0" }, - { name = "paddlepaddle", specifier = ">=2.5.0" }, - { name = "pillow", specifier = ">=10.0.0" }, - { name = "ultralytics", specifier = ">=8.0.0" }, - { name = "uvicorn", specifier = ">=0.24.0" }, + { name = "onnxruntime", specifier = ">=1.22.1" }, + { name = "openai", specifier = ">=1.105.0" }, + { name = "openai-agents", specifier = ">=0.2.11" }, + { name = "opencv-python", specifier = ">=4.11.0.86" }, + { name = "paddleocr", extras = ["all"], specifier = ">=3.2.0" }, + { name = "paddlepaddle", specifier = ">=3.1.1" }, + { name = "pillow", specifier = ">=11.3.0" }, + { name = "ultralytics", specifier = ">=8.3.193" }, + { name = "uvicorn", specifier = ">=0.35.0" }, { name = "vncdotool", specifier = ">=1.2.0" }, ] [package.metadata.requires-dev] dev = [ - { name = "arize-phoenix", specifier = ">=4.0.0" }, - { name = "deepeval", specifier = ">=0.21.0" }, - { name = "pre-commit", specifier = ">=3.7.1" }, - { name = "pyright", specifier = ">=1.1.379" }, - { name = "pytest", specifier = ">=8.3.0" }, - { name = "pytest-asyncio", specifier = ">=0.23.0" }, - { name = "pytest-cov", specifier = ">=5.0.0" }, - { name = "ruff", specifier = ">=0.5.7" }, + { name = "arize-phoenix", specifier = ">=11.32.1" }, + { name = "deepeval", specifier = ">=3.4.7" }, + { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "pyright", specifier = ">=1.1.405" }, + { name = "pytest", specifier = ">=8.4.2" }, + { name = "pytest-asyncio", specifier = ">=1.1.0" }, + { name = "pytest-cov", specifier = ">=6.2.1" }, + { name = "ruff", specifier = ">=0.12.12" }, ] [[package]] @@ -586,6 +599,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/ff/026513ecad58dacd45d1d24ebe52b852165a26e287177de1d545325c0c25/cryptography-45.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:7285a89df4900ed3bfaad5679b1e668cb4b38a8de1ccbfc84b05f34512da0a90", size = 3392742 }, ] +[[package]] +name = "cssselect" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/0a/c3ea9573b1dc2e151abfe88c7fe0c26d1892fe6ed02d0cdb30f0d57029d5/cssselect-1.3.0.tar.gz", hash = "sha256:57f8a99424cfab289a1b6a816a43075a4b00948c86b4dcf3ef4ee7e15f7ab0c7", size = 42870 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/58/257350f7db99b4ae12b614a36256d9cc870d71d9e451e79c2dc3b23d7c3c/cssselect-1.3.0-py3-none-any.whl", hash = "sha256:56d1bf3e198080cc1667e137bc51de9cadfca259f03c2d4e09037b3e01e30f0d", size = 18786 }, +] + +[[package]] +name = "cssutils" +version = "2.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/9f/329d26121fe165be44b1dfff21aa0dc348f04633931f1d20ed6cf448a236/cssutils-2.11.1.tar.gz", hash = "sha256:0563a76513b6af6eebbe788c3bf3d01c920e46b3f90c8416738c5cfc773ff8e2", size = 711657 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/ec/bb273b7208c606890dc36540fe667d06ce840a6f62f9fae7e658fcdc90fb/cssutils-2.11.1-py3-none-any.whl", hash = "sha256:a67bfdfdff4f3867fab43698ec4897c1a828eca5973f4073321b3bccaf1199b1", size = 385747 }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -595,9 +629,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321 }, ] +[[package]] +name = "dataclasses-json" +version = "0.6.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marshmallow" }, + { name = "typing-inspect" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686 }, +] + [[package]] name = "deepeval" -version = "3.4.3" +version = "3.4.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -619,6 +666,7 @@ dependencies = [ { name = "pytest-repeat" }, { name = "pytest-rerunfailures" }, { name = "pytest-xdist" }, + { name = "python-dotenv" }, { name = "requests" }, { name = "rich" }, { name = "sentry-sdk" }, @@ -629,9 +677,9 @@ dependencies = [ { name = "typer" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/12/a2c615c4214da376e7ccbf2ae809ae2e7583b2e938a792bd414da9787465/deepeval-3.4.3.tar.gz", hash = "sha256:2db0c3bdd2d036eb7df26f68b8a544ef8eaaf5407ea3d282cd1c58b712478dde", size = 377336 } +sdist = { url = "https://files.pythonhosted.org/packages/65/1a/14b5d3d43a73c49eed2147735fe1c9b424094fea5f98730af72a8a4c92d0/deepeval-3.4.7.tar.gz", hash = "sha256:98bd5a6cd3d1ab58157cb24de3bf88b002134e2d7fc28c8b20cd9998421123a7", size = 382949 } wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/0e/20ed53d05df80d3bb6fcca136b8cd9693037cd705faa56969752ca036b73/deepeval-3.4.3-py3-none-any.whl", hash = "sha256:ad7035c8b5c03616d65643859fae5572bae19eedb181e94d0fd8baed6ff4c0e6", size = 560793 }, + { url = "https://files.pythonhosted.org/packages/9c/7e/e1ff99d355c1c84c26cacb3fa585d3741a60369dfb93beb4f7c8db1a3114/deepeval-3.4.7-py3-none-any.whl", hash = "sha256:955688c66f6f5ce8db13b49dbf38c8a2e0123209185b33934b7421c61f5cac1f", size = 567682 }, ] [[package]] @@ -661,6 +709,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632 }, ] +[[package]] +name = "einops" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/81/df4fbe24dff8ba3934af99044188e20a98ed441ad17a274539b74e82e126/einops-0.8.1.tar.gz", hash = "sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84", size = 54805 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/62/9773de14fe6c45c23649e98b83231fffd7b9892b6cf863251dc2afa73643/einops-0.8.1-py3-none-any.whl", hash = "sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737", size = 64359 }, +] + [[package]] name = "email-validator" version = "2.3.0" @@ -674,6 +731,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604 }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059 }, +] + [[package]] name = "execnet" version = "2.1.1" @@ -767,6 +833,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289 }, ] +[[package]] +name = "ftfy" +version = "6.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/d3/8650919bc3c7c6e90ee3fa7fd618bf373cbbe55dff043bd67353dbb20cd8/ftfy-6.3.1.tar.gz", hash = "sha256:9b3c3d90f84fb267fe64d375a07b7f8912d817cf86009ae134aa03e1819506ec", size = 308927 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821 }, +] + [[package]] name = "future" version = "1.0.0" @@ -792,7 +870,7 @@ wheels = [ [[package]] name = "google-genai" -version = "1.32.0" +version = "1.33.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -804,9 +882,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/ab/e6cdd8fa957c647ef00c4da7c59d0e734354bd49ed8d98c860732d8e1944/google_genai-1.32.0.tar.gz", hash = "sha256:349da3f5ff0e981066bd508585fcdd308d28fc4646f318c8f6d1aa6041f4c7e3", size = 240802 } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/506221067750087ba1346f0a31f6e1714fda4b612d45a54cd2164750e05a/google_genai-1.33.0.tar.gz", hash = "sha256:7d3a5ebad712d95a0d1775842505886eb43cc52f9f478aa4ab0e2d25412499a2", size = 241006 } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/55/be09472f7a656af1208196d2ef9a3d2710f3cbcf695f51acbcbe28b9472b/google_genai-1.32.0-py3-none-any.whl", hash = "sha256:c0c4b1d45adf3aa99501050dd73da2f0dea09374002231052d81a6765d15e7f6", size = 241680 }, + { url = "https://files.pythonhosted.org/packages/43/8e/55052fe488d6604309b425360beb72e6d65f11fa4cc1cdde17ccfe93e1bc/google_genai-1.33.0-py3-none-any.whl", hash = "sha256:1710e958af0a0f3d19521fabbefd86b22d1f212376103f18fed11c9d96fa48e8", size = 241753 }, ] [[package]] @@ -849,14 +927,14 @@ wheels = [ [[package]] name = "griffe" -version = "1.13.0" +version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/b5/23b91f22b7b3a7f8f62223f6664946271c0f5cb4179605a3e6bbae863920/griffe-1.13.0.tar.gz", hash = "sha256:246ea436a5e78f7fbf5f24ca8a727bb4d2a4b442a2959052eea3d0bfe9a076e0", size = 412759 } +sdist = { url = "https://files.pythonhosted.org/packages/ec/d7/6c09dd7ce4c7837e4cdb11dce980cb45ae3cd87677298dc3b781b6bce7d3/griffe-1.14.0.tar.gz", hash = "sha256:9d2a15c1eca966d68e00517de5d69dd1bc5c9f2335ef6c1775362ba5b8651a13", size = 424684 } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/8c/b7cfdd8dfe48f6b09f7353323732e1a290c388bd14f216947928dc85f904/griffe-1.13.0-py3-none-any.whl", hash = "sha256:470fde5b735625ac0a36296cd194617f039e9e83e301fcbd493e2b58382d0559", size = 139365 }, + { url = "https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl", hash = "sha256:0e9d52832cccf0f7188cfe585ba962d2674b241c01916d780925df34873bceb0", size = 144439 }, ] [[package]] @@ -1094,6 +1172,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/e8/685f47e0d754320684db4425a0967f7d3fa70126bffd76110b7009a0090f/joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241", size = 308396 }, ] +[[package]] +name = "jsonpatch" +version = "1.33" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpointer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/78/18813351fe5d63acad16aec57f94ec2b70a09e53ca98145589e185423873/jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c", size = 21699 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898 }, +] + [[package]] name = "jsonpath-ng" version = "1.7.0" @@ -1106,6 +1196,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/5a/73ecb3d82f8615f32ccdadeb9356726d6cae3a4bbc840b437ceb95708063/jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6", size = 30105 }, ] +[[package]] +name = "jsonpointer" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/0a/eebeb1fa92507ea94016a2a790b93c2ae41a7e18778f85471dc54475ed25/jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef", size = 9114 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942", size = 7595 }, +] + [[package]] name = "jsonschema" version = "4.25.1" @@ -1154,6 +1253,135 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/92/5f3068cf15ee5cb624a0c7596e67e2a0bb2adee33f71c379054a491d07da/kiwisolver-1.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:2c1a4f57df73965f3f14df20b80ee29e6a7930a57d2d9e8491a25f676e197c60", size = 64992 }, ] +[[package]] +name = "langchain" +version = "0.3.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langchain-text-splitters" }, + { name = "langsmith" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlalchemy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/f6/f4f7f3a56626fe07e2bb330feb61254dbdf06c506e6b59a536a337da51cf/langchain-0.3.27.tar.gz", hash = "sha256:aa6f1e6274ff055d0fd36254176770f356ed0a8994297d1df47df341953cec62", size = 10233809 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/d5/4861816a95b2f6993f1360cfb605aacb015506ee2090433a71de9cca8477/langchain-0.3.27-py3-none-any.whl", hash = "sha256:7b20c4f338826acb148d885b20a73a16e410ede9ee4f19bb02011852d5f98798", size = 1018194 }, +] + +[[package]] +name = "langchain-community" +version = "0.3.29" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "dataclasses-json" }, + { name = "httpx-sse" }, + { name = "langchain" }, + { name = "langchain-core" }, + { name = "langsmith" }, + { name = "numpy" }, + { name = "pydantic-settings" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlalchemy" }, + { name = "tenacity" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/71/72ddc8802d5edcd3b31bc4f0eb62cdf85c30c9b845e34cc879dab0b45679/langchain_community-0.3.29.tar.gz", hash = "sha256:1f3d37973b10458052bb3cc02dce9773a8ffbd02961698c6d395b8c8d7f9e004", size = 33238072 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/3c/107819dbed0be3f7a041245bce861c8dd4883ed08040ed482d278a274f22/langchain_community-0.3.29-py3-none-any.whl", hash = "sha256:c876ec7ef40b46353af164197f4e08e157650e8a02c9fb9d49351cdc16c839fe", size = 2530803 }, +] + +[[package]] +name = "langchain-core" +version = "0.3.75" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpatch" }, + { name = "langsmith" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "tenacity" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/63/270b71a23e849984505ddc7c5c9fd3f4bd9cb14b1a484ee44c4e51c33cc2/langchain_core-0.3.75.tar.gz", hash = "sha256:ab0eb95a06ed6043f76162e6086b45037690cb70b7f090bd83b5ebb8a05b70ed", size = 570876 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/42/0d0221cce6f168f644d7d96cb6c87c4e42fc55d2941da7a36e970e3ab8ab/langchain_core-0.3.75-py3-none-any.whl", hash = "sha256:03ca1fadf955ee3c7d5806a841f4b3a37b816acea5e61a7e6ba1298c05eea7f5", size = 443986 }, +] + +[[package]] +name = "langchain-openai" +version = "0.3.32" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "openai" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/19/167d9ad1b6bb75406c4acceda01ef0dc1101c7f629f74441fe8a787fb190/langchain_openai-0.3.32.tar.gz", hash = "sha256:782ad669bd1bdb964456d8882c5178717adcfceecb482cc20005f770e43d346d", size = 782982 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/3d/e22ee65fff79afe7bdfbd67844243eb279b440c882dad9e4262dcc87131f/langchain_openai-0.3.32-py3-none-any.whl", hash = "sha256:3354f76822f7cc76d8069831fe2a77f9bc7ff3b4f13af788bd94e4c6e853b400", size = 74531 }, +] + +[[package]] +name = "langchain-text-splitters" +version = "0.3.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/43/dcda8fd25f0b19cb2835f2f6bb67f26ad58634f04ac2d8eae00526b0fa55/langchain_text_splitters-0.3.11.tar.gz", hash = "sha256:7a50a04ada9a133bbabb80731df7f6ddac51bc9f1b9cab7fa09304d71d38a6cc", size = 46458 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/0d/41a51b40d24ff0384ec4f7ab8dd3dcea8353c05c973836b5e289f1465d4f/langchain_text_splitters-0.3.11-py3-none-any.whl", hash = "sha256:cf079131166a487f1372c8ab5d0bfaa6c0a4291733d9c43a34a16ac9bcd6a393", size = 33845 }, +] + +[[package]] +name = "langsmith" +version = "0.4.25" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/84/2f01d51a557d14a9a5590c32de5dde3b466b00a55521450c2c9e77ffa438/langsmith-0.4.25.tar.gz", hash = "sha256:56f0c45810384fba37582ca17fdbcf6ead51934d26d72672e5a810452c0d4ae3", size = 940274 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/85/8a5ca8f6044bd74acd0d364878b459d84ec460cf40aec17ed9cd5716e908/langsmith-0.4.25-py3-none-any.whl", hash = "sha256:adb61784ff58e65f0290ba45770626219fb06a776e69fbcf98aec580478b4686", size = 379416 }, +] + +[[package]] +name = "lxml" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/bd/f9d01fd4132d81c6f43ab01983caea69ec9614b913c290a26738431a015d/lxml-6.0.1.tar.gz", hash = "sha256:2b3a882ebf27dd026df3801a87cf49ff791336e0f94b0fad195db77e01240690", size = 4070214 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/a9/82b244c8198fcdf709532e39a1751943a36b3e800b420adc739d751e0299/lxml-6.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:c03ac546adaabbe0b8e4a15d9ad815a281afc8d36249c246aecf1aaad7d6f200", size = 8422788 }, + { url = "https://files.pythonhosted.org/packages/c9/8d/1ed2bc20281b0e7ed3e6c12b0a16e64ae2065d99be075be119ba88486e6d/lxml-6.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33b862c7e3bbeb4ba2c96f3a039f925c640eeba9087a4dc7a572ec0f19d89392", size = 4593547 }, + { url = "https://files.pythonhosted.org/packages/76/53/d7fd3af95b72a3493bf7fbe842a01e339d8f41567805cecfecd5c71aa5ee/lxml-6.0.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7a3ec1373f7d3f519de595032d4dcafae396c29407cfd5073f42d267ba32440d", size = 4948101 }, + { url = "https://files.pythonhosted.org/packages/9d/51/4e57cba4d55273c400fb63aefa2f0d08d15eac021432571a7eeefee67bed/lxml-6.0.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03b12214fb1608f4cffa181ec3d046c72f7e77c345d06222144744c122ded870", size = 5108090 }, + { url = "https://files.pythonhosted.org/packages/f6/6e/5f290bc26fcc642bc32942e903e833472271614e24d64ad28aaec09d5dae/lxml-6.0.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:207ae0d5f0f03b30f95e649a6fa22aa73f5825667fee9c7ec6854d30e19f2ed8", size = 5021791 }, + { url = "https://files.pythonhosted.org/packages/13/d4/2e7551a86992ece4f9a0f6eebd4fb7e312d30f1e372760e2109e721d4ce6/lxml-6.0.1-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:32297b09ed4b17f7b3f448de87a92fb31bb8747496623483788e9f27c98c0f00", size = 5358861 }, + { url = "https://files.pythonhosted.org/packages/8a/5f/cb49d727fc388bf5fd37247209bab0da11697ddc5e976ccac4826599939e/lxml-6.0.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e18224ea241b657a157c85e9cac82c2b113ec90876e01e1f127312006233756", size = 5652569 }, + { url = "https://files.pythonhosted.org/packages/ca/b8/66c1ef8c87ad0f958b0a23998851e610607c74849e75e83955d5641272e6/lxml-6.0.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a07a994d3c46cd4020c1ea566345cf6815af205b1e948213a4f0f1d392182072", size = 5252262 }, + { url = "https://files.pythonhosted.org/packages/1a/ef/131d3d6b9590e64fdbb932fbc576b81fcc686289da19c7cb796257310e82/lxml-6.0.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:2287fadaa12418a813b05095485c286c47ea58155930cfbd98c590d25770e225", size = 4710309 }, + { url = "https://files.pythonhosted.org/packages/bc/3f/07f48ae422dce44902309aa7ed386c35310929dc592439c403ec16ef9137/lxml-6.0.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b4e597efca032ed99f418bd21314745522ab9fa95af33370dcee5533f7f70136", size = 5265786 }, + { url = "https://files.pythonhosted.org/packages/11/c7/125315d7b14ab20d9155e8316f7d287a4956098f787c22d47560b74886c4/lxml-6.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9696d491f156226decdd95d9651c6786d43701e49f32bf23715c975539aa2b3b", size = 5062272 }, + { url = "https://files.pythonhosted.org/packages/8b/c3/51143c3a5fc5168a7c3ee626418468ff20d30f5a59597e7b156c1e61fba8/lxml-6.0.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e4e3cd3585f3c6f87cdea44cda68e692cc42a012f0131d25957ba4ce755241a7", size = 4786955 }, + { url = "https://files.pythonhosted.org/packages/11/86/73102370a420ec4529647b31c4a8ce8c740c77af3a5fae7a7643212d6f6e/lxml-6.0.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:45cbc92f9d22c28cd3b97f8d07fcefa42e569fbd587dfdac76852b16a4924277", size = 5673557 }, + { url = "https://files.pythonhosted.org/packages/d7/2d/aad90afaec51029aef26ef773b8fd74a9e8706e5e2f46a57acd11a421c02/lxml-6.0.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f8c9bcfd2e12299a442fba94459adf0b0d001dbc68f1594439bfa10ad1ecb74b", size = 5254211 }, + { url = "https://files.pythonhosted.org/packages/63/01/c9e42c8c2d8b41f4bdefa42ab05448852e439045f112903dd901b8fbea4d/lxml-6.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1e9dc2b9f1586e7cd77753eae81f8d76220eed9b768f337dc83a3f675f2f0cf9", size = 5275817 }, + { url = "https://files.pythonhosted.org/packages/bc/1f/962ea2696759abe331c3b0e838bb17e92224f39c638c2068bf0d8345e913/lxml-6.0.1-cp312-cp312-win32.whl", hash = "sha256:987ad5c3941c64031f59c226167f55a04d1272e76b241bfafc968bdb778e07fb", size = 3610889 }, + { url = "https://files.pythonhosted.org/packages/41/e2/22c86a990b51b44442b75c43ecb2f77b8daba8c4ba63696921966eac7022/lxml-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:abb05a45394fd76bf4a60c1b7bec0e6d4e8dfc569fc0e0b1f634cd983a006ddc", size = 4010925 }, + { url = "https://files.pythonhosted.org/packages/b2/21/dc0c73325e5eb94ef9c9d60dbb5dcdcb2e7114901ea9509735614a74e75a/lxml-6.0.1-cp312-cp312-win_arm64.whl", hash = "sha256:c4be29bce35020d8579d60aa0a4e95effd66fcfce31c46ffddf7e5422f73a299", size = 3671922 }, +] + [[package]] name = "mako" version = "1.3.10" @@ -1196,6 +1424,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601 }, ] +[[package]] +name = "marshmallow" +version = "3.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/5e/5e53d26b42ab75491cda89b871dab9e97c840bf12c63ec58a1919710cd06/marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6", size = 221825 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878 }, +] + [[package]] name = "matplotlib" version = "3.10.6" @@ -1285,6 +1525,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/f1/f3e84271e115da91d9520e65846a7176ad356f71d021c134f872687d4ffc/modelscope-1.29.2-py3-none-any.whl", hash = "sha256:27d3e1b87d1d1a1427cd99a4b1312b639aa2cd16c191308d732f9747c4d535bd", size = 5920345 }, ] +[[package]] +name = "more-itertools" +version = "10.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667 }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -1321,6 +1570,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313 }, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963 }, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -1566,7 +1824,7 @@ wheels = [ [[package]] name = "openai-agents" -version = "0.2.10" +version = "0.2.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffe" }, @@ -1577,9 +1835,9 @@ dependencies = [ { name = "types-requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/8e/99a7ad15bdd48610d2e738aecdcba9bdcca7b60b4a4f8006577e76f651b6/openai_agents-0.2.10.tar.gz", hash = "sha256:8741890b5b5f1513589aebc571776b9a17158a01f6df55ab6402668d2a204a4d", size = 1671208 } +sdist = { url = "https://files.pythonhosted.org/packages/dd/c7/e8b588851bdbb33f16397b45d182998a01e6e57ff028a143788036a89d53/openai_agents-0.2.11.tar.gz", hash = "sha256:1a2e3fade02b3d8571560dbd121bfe0d84c80f48da04c838d9d5195966714abc", size = 1677542 } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/1d/95930498ea971834f808242fb00a5574796c29adf93afb8444a6ec970a73/openai_agents-0.2.10-py3-none-any.whl", hash = "sha256:70618881d190265beefb5dac5ffd7bb99e2bed5907440e468ce03257d8994bf5", size = 178904 }, + { url = "https://files.pythonhosted.org/packages/6a/71/a8712a89502b95da64db6b0b31c12ac5039542ae8e31caddba369b6bd324/openai_agents-0.2.11-py3-none-any.whl", hash = "sha256:ed26f7bb2b08bd7607ae87eb7bcfcee8c8f4431da134252757b31120a68b9086", size = 179141 }, ] [[package]] @@ -1640,6 +1898,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/4d/092766f8e610f2c513e483c4adc892eea1634945022a73371fe01f621165/openinference_semantic_conventions-0.1.21-py3-none-any.whl", hash = "sha256:acde8282c20da1de900cdc0d6258a793ec3eb8031bfc496bd823dae17d32e326", size = 10167 }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910 }, +] + [[package]] name = "opentelemetry-api" version = "1.36.0" @@ -1811,6 +2081,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/74/72/c8218cc7489762ab3d1fb25721f8c928f10085e38feb8d52fd6583bfc592/paddleocr-3.2.0-py3-none-any.whl", hash = "sha256:2b942295ad5963de8e01d68afb15a9507d713bc7299e2dfeb198d9c3ac5cf76f", size = 75976 }, ] +[package.optional-dependencies] +all = [ + { name = "paddlex", extra = ["ie", "ocr", "trans"] }, +] + [[package]] name = "paddlepaddle" version = "3.1.1" @@ -1861,6 +2136,41 @@ wheels = [ ] [package.optional-dependencies] +ie = [ + { name = "ftfy" }, + { name = "imagesize" }, + { name = "langchain" }, + { name = "langchain-community" }, + { name = "langchain-core" }, + { name = "langchain-openai" }, + { name = "lxml" }, + { name = "openai" }, + { name = "opencv-contrib-python" }, + { name = "openpyxl" }, + { name = "premailer" }, + { name = "pyclipper" }, + { name = "pypdfium2" }, + { name = "scikit-learn" }, + { name = "shapely" }, + { name = "tokenizers" }, +] +ocr = [ + { name = "einops" }, + { name = "ftfy" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "lxml" }, + { name = "opencv-contrib-python" }, + { name = "openpyxl" }, + { name = "premailer" }, + { name = "pyclipper" }, + { name = "pypdfium2" }, + { name = "regex" }, + { name = "scikit-learn" }, + { name = "shapely" }, + { name = "tiktoken" }, + { name = "tokenizers" }, +] ocr-core = [ { name = "imagesize" }, { name = "opencv-contrib-python" }, @@ -1868,6 +2178,21 @@ ocr-core = [ { name = "pypdfium2" }, { name = "shapely" }, ] +trans = [ + { name = "beautifulsoup4" }, + { name = "ftfy" }, + { name = "imagesize" }, + { name = "lxml" }, + { name = "openai" }, + { name = "opencv-contrib-python" }, + { name = "openpyxl" }, + { name = "premailer" }, + { name = "pyclipper" }, + { name = "pypdfium2" }, + { name = "scikit-learn" }, + { name = "shapely" }, + { name = "tokenizers" }, +] [[package]] name = "pandas" @@ -1964,7 +2289,7 @@ wheels = [ [[package]] name = "posthog" -version = "6.7.1" +version = "6.7.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -1974,9 +2299,9 @@ dependencies = [ { name = "six" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/72/8a448dc2e7e5d16f52356189b4f3d1014eaa0bccca15353ff1282dba86ed/posthog-6.7.1.tar.gz", hash = "sha256:1011ddda110b65e0d080f2a5f88eb0337547ed6b5a50d0f8f6191e9357b99434", size = 106534 } +sdist = { url = "https://files.pythonhosted.org/packages/0f/40/d7f585e09e47f492ebaeb8048a8e2ce5d9f49a3896856a7a975cbc1484fa/posthog-6.7.4.tar.gz", hash = "sha256:2bfa74f321ac18efe4a48a256d62034a506ca95477af7efa32292ed488a742c5", size = 118209 } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/cd/bc1e638afa4c119f8350dfbae19fb5c63f6f36ee2a29481637645e514d0f/posthog-6.7.1-py3-none-any.whl", hash = "sha256:57d9a891ddedf690d2b294bb8b7832095c149729a3f123f47b7bcaf7b2b6e1a0", size = 124024 }, + { url = "https://files.pythonhosted.org/packages/bb/95/e795059ef73d480a7f11f1be201087f65207509525920897fb514a04914c/posthog-6.7.4-py3-none-any.whl", hash = "sha256:7f1872c53ec7e9a29b088a5a1ad03fa1be3b871d10d70c8bf6c2dafb91beaac5", size = 136409 }, ] [[package]] @@ -1995,6 +2320,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965 }, ] +[[package]] +name = "premailer" +version = "3.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "cssselect" }, + { name = "cssutils" }, + { name = "lxml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/6f/e49bd31941eff2987076383fa6d811eb785a28f498f5bb131e981bd71e13/premailer-3.10.0.tar.gz", hash = "sha256:d1875a8411f5dc92b53ef9f193db6c0f879dc378d618e0ad292723e388bfe4c2", size = 24342 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/07/4e8d94f94c7d41ca5ddf8a9695ad87b888104e2fd41a35546c1dc9ca74ac/premailer-3.10.0-py2.py3-none-any.whl", hash = "sha256:021b8196364d7df96d04f9ade51b794d0b77bcc19e998321c515633a2273be1a", size = 19544 }, +] + [[package]] name = "prettytable" version = "3.16.0" @@ -2288,15 +2629,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.404" +version = "1.1.405" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e2/6e/026be64c43af681d5632722acd100b06d3d39f383ec382ff50a71a6d5bce/pyright-1.1.404.tar.gz", hash = "sha256:455e881a558ca6be9ecca0b30ce08aa78343ecc031d37a198ffa9a7a1abeb63e", size = 4065679 } +sdist = { url = "https://files.pythonhosted.org/packages/fb/6c/ba4bbee22e76af700ea593a1d8701e3225080956753bee9750dcc25e2649/pyright-1.1.405.tar.gz", hash = "sha256:5c2a30e1037af27eb463a1cc0b9f6d65fec48478ccf092c1ac28385a15c55763", size = 4068319 } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/30/89aa7f7d7a875bbb9a577d4b1dc5a3e404e3d2ae2657354808e905e358e0/pyright-1.1.404-py3-none-any.whl", hash = "sha256:c7b7ff1fdb7219c643079e4c3e7d4125f0dafcc19d253b47e898d130ea426419", size = 5902951 }, + { url = "https://files.pythonhosted.org/packages/d5/1a/524f832e1ff1962a22a1accc775ca7b143ba2e9f5924bb6749dce566784a/pyright-1.1.405-py3-none-any.whl", hash = "sha256:a2cb13700b5508ce8e5d4546034cb7ea4aedb60215c6c33f56cec7f53996035a", size = 5905038 }, ] [[package]] @@ -2310,7 +2651,7 @@ wheels = [ [[package]] name = "pytest" -version = "8.4.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -2319,9 +2660,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714 } +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618 } wheels = [ - { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474 }, + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750 }, ] [[package]] @@ -2468,6 +2809,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775 }, ] +[[package]] +name = "regex" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/5a/4c63457fbcaf19d138d72b2e9b39405954f98c0349b31c601bfcb151582c/regex-2025.9.1.tar.gz", hash = "sha256:88ac07b38d20b54d79e704e38aa3bd2c0f8027432164226bdee201a1c0c9c9ff", size = 400852 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/ef/a0372febc5a1d44c1be75f35d7e5aff40c659ecde864d7fa10e138f75e74/regex-2025.9.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:84a25164bd8dcfa9f11c53f561ae9766e506e580b70279d05a7946510bdd6f6a", size = 486317 }, + { url = "https://files.pythonhosted.org/packages/b5/25/d64543fb7eb41a1024786d518cc57faf1ce64aa6e9ddba097675a0c2f1d2/regex-2025.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:645e88a73861c64c1af558dd12294fb4e67b5c1eae0096a60d7d8a2143a611c7", size = 289698 }, + { url = "https://files.pythonhosted.org/packages/d8/dc/fbf31fc60be317bd9f6f87daa40a8a9669b3b392aa8fe4313df0a39d0722/regex-2025.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10a450cba5cd5409526ee1d4449f42aad38dd83ac6948cbd6d7f71ca7018f7db", size = 287242 }, + { url = "https://files.pythonhosted.org/packages/0f/74/f933a607a538f785da5021acf5323961b4620972e2c2f1f39b6af4b71db7/regex-2025.9.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9dc5991592933a4192c166eeb67b29d9234f9c86344481173d1bc52f73a7104", size = 797441 }, + { url = "https://files.pythonhosted.org/packages/89/d0/71fc49b4f20e31e97f199348b8c4d6e613e7b6a54a90eb1b090c2b8496d7/regex-2025.9.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a32291add816961aab472f4fad344c92871a2ee33c6c219b6598e98c1f0108f2", size = 862654 }, + { url = "https://files.pythonhosted.org/packages/59/05/984edce1411a5685ba9abbe10d42cdd9450aab4a022271f9585539788150/regex-2025.9.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:588c161a68a383478e27442a678e3b197b13c5ba51dbba40c1ccb8c4c7bee9e9", size = 910862 }, + { url = "https://files.pythonhosted.org/packages/b2/02/5c891bb5fe0691cc1bad336e3a94b9097fbcf9707ec8ddc1dce9f0397289/regex-2025.9.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47829ffaf652f30d579534da9085fe30c171fa2a6744a93d52ef7195dc38218b", size = 801991 }, + { url = "https://files.pythonhosted.org/packages/f1/ae/fd10d6ad179910f7a1b3e0a7fde1ef8bb65e738e8ac4fd6ecff3f52252e4/regex-2025.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e978e5a35b293ea43f140c92a3269b6ab13fe0a2bf8a881f7ac740f5a6ade85", size = 786651 }, + { url = "https://files.pythonhosted.org/packages/30/cf/9d686b07bbc5bf94c879cc168db92542d6bc9fb67088d03479fef09ba9d3/regex-2025.9.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4cf09903e72411f4bf3ac1eddd624ecfd423f14b2e4bf1c8b547b72f248b7bf7", size = 856556 }, + { url = "https://files.pythonhosted.org/packages/91/9d/302f8a29bb8a49528abbab2d357a793e2a59b645c54deae0050f8474785b/regex-2025.9.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d016b0f77be63e49613c9e26aaf4a242f196cd3d7a4f15898f5f0ab55c9b24d2", size = 849001 }, + { url = "https://files.pythonhosted.org/packages/93/fa/b4c6dbdedc85ef4caec54c817cd5f4418dbfa2453214119f2538082bf666/regex-2025.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:656563e620de6908cd1c9d4f7b9e0777e3341ca7db9d4383bcaa44709c90281e", size = 788138 }, + { url = "https://files.pythonhosted.org/packages/4a/1b/91ee17a3cbf87f81e8c110399279d0e57f33405468f6e70809100f2ff7d8/regex-2025.9.1-cp312-cp312-win32.whl", hash = "sha256:df33f4ef07b68f7ab637b1dbd70accbf42ef0021c201660656601e8a9835de45", size = 264524 }, + { url = "https://files.pythonhosted.org/packages/92/28/6ba31cce05b0f1ec6b787921903f83bd0acf8efde55219435572af83c350/regex-2025.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5aba22dfbc60cda7c0853516104724dc904caa2db55f2c3e6e984eb858d3edf3", size = 275489 }, + { url = "https://files.pythonhosted.org/packages/bd/ed/ea49f324db00196e9ef7fe00dd13c6164d5173dd0f1bbe495e61bb1fb09d/regex-2025.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:ec1efb4c25e1849c2685fa95da44bfde1b28c62d356f9c8d861d4dad89ed56e9", size = 268589 }, +] + [[package]] name = "requests" version = "2.32.5" @@ -2483,6 +2846,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738 }, ] +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481 }, +] + [[package]] name = "rich" version = "14.1.0" @@ -2562,28 +2937,28 @@ wheels = [ [[package]] name = "ruff" -version = "0.12.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/55/16ab6a7d88d93001e1ae4c34cbdcfb376652d761799459ff27c1dc20f6fa/ruff-0.12.11.tar.gz", hash = "sha256:c6b09ae8426a65bbee5425b9d0b82796dbb07cb1af045743c79bfb163001165d", size = 5347103 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/a2/3b3573e474de39a7a475f3fbaf36a25600bfeb238e1a90392799163b64a0/ruff-0.12.11-py3-none-linux_armv6l.whl", hash = "sha256:93fce71e1cac3a8bf9200e63a38ac5c078f3b6baebffb74ba5274fb2ab276065", size = 11979885 }, - { url = "https://files.pythonhosted.org/packages/76/e4/235ad6d1785a2012d3ded2350fd9bc5c5af8c6f56820e696b0118dfe7d24/ruff-0.12.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b8e33ac7b28c772440afa80cebb972ffd823621ded90404f29e5ab6d1e2d4b93", size = 12742364 }, - { url = "https://files.pythonhosted.org/packages/2c/0d/15b72c5fe6b1e402a543aa9d8960e0a7e19dfb079f5b0b424db48b7febab/ruff-0.12.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d69fb9d4937aa19adb2e9f058bc4fbfe986c2040acb1a4a9747734834eaa0bfd", size = 11920111 }, - { url = "https://files.pythonhosted.org/packages/3e/c0/f66339d7893798ad3e17fa5a1e587d6fd9806f7c1c062b63f8b09dda6702/ruff-0.12.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:411954eca8464595077a93e580e2918d0a01a19317af0a72132283e28ae21bee", size = 12160060 }, - { url = "https://files.pythonhosted.org/packages/03/69/9870368326db26f20c946205fb2d0008988aea552dbaec35fbacbb46efaa/ruff-0.12.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6a2c0a2e1a450f387bf2c6237c727dd22191ae8c00e448e0672d624b2bbd7fb0", size = 11799848 }, - { url = "https://files.pythonhosted.org/packages/25/8c/dd2c7f990e9b3a8a55eee09d4e675027d31727ce33cdb29eab32d025bdc9/ruff-0.12.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ca4c3a7f937725fd2413c0e884b5248a19369ab9bdd850b5781348ba283f644", size = 13536288 }, - { url = "https://files.pythonhosted.org/packages/7a/30/d5496fa09aba59b5e01ea76775a4c8897b13055884f56f1c35a4194c2297/ruff-0.12.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4d1df0098124006f6a66ecf3581a7f7e754c4df7644b2e6704cd7ca80ff95211", size = 14490633 }, - { url = "https://files.pythonhosted.org/packages/9b/2f/81f998180ad53445d403c386549d6946d0748e536d58fce5b5e173511183/ruff-0.12.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a8dd5f230efc99a24ace3b77e3555d3fbc0343aeed3fc84c8d89e75ab2ff793", size = 13888430 }, - { url = "https://files.pythonhosted.org/packages/87/71/23a0d1d5892a377478c61dbbcffe82a3476b050f38b5162171942a029ef3/ruff-0.12.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4dc75533039d0ed04cd33fb8ca9ac9620b99672fe7ff1533b6402206901c34ee", size = 12913133 }, - { url = "https://files.pythonhosted.org/packages/80/22/3c6cef96627f89b344c933781ed38329bfb87737aa438f15da95907cbfd5/ruff-0.12.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fc58f9266d62c6eccc75261a665f26b4ef64840887fc6cbc552ce5b29f96cc8", size = 13169082 }, - { url = "https://files.pythonhosted.org/packages/05/b5/68b3ff96160d8b49e8dd10785ff3186be18fd650d356036a3770386e6c7f/ruff-0.12.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:5a0113bd6eafd545146440225fe60b4e9489f59eb5f5f107acd715ba5f0b3d2f", size = 13139490 }, - { url = "https://files.pythonhosted.org/packages/59/b9/050a3278ecd558f74f7ee016fbdf10591d50119df8d5f5da45a22c6afafc/ruff-0.12.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0d737b4059d66295c3ea5720e6efc152623bb83fde5444209b69cd33a53e2000", size = 11958928 }, - { url = "https://files.pythonhosted.org/packages/f9/bc/93be37347db854806904a43b0493af8d6873472dfb4b4b8cbb27786eb651/ruff-0.12.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:916fc5defee32dbc1fc1650b576a8fed68f5e8256e2180d4d9855aea43d6aab2", size = 11764513 }, - { url = "https://files.pythonhosted.org/packages/7a/a1/1471751e2015a81fd8e166cd311456c11df74c7e8769d4aabfbc7584c7ac/ruff-0.12.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c984f07d7adb42d3ded5be894fb4007f30f82c87559438b4879fe7aa08c62b39", size = 12745154 }, - { url = "https://files.pythonhosted.org/packages/68/ab/2542b14890d0f4872dd81b7b2a6aed3ac1786fae1ce9b17e11e6df9e31e3/ruff-0.12.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e07fbb89f2e9249f219d88331c833860489b49cdf4b032b8e4432e9b13e8a4b9", size = 13227653 }, - { url = "https://files.pythonhosted.org/packages/22/16/2fbfc61047dbfd009c58a28369a693a1484ad15441723be1cd7fe69bb679/ruff-0.12.11-py3-none-win32.whl", hash = "sha256:c792e8f597c9c756e9bcd4d87cf407a00b60af77078c96f7b6366ea2ce9ba9d3", size = 11944270 }, - { url = "https://files.pythonhosted.org/packages/08/a5/34276984705bfe069cd383101c45077ee029c3fe3b28225bf67aa35f0647/ruff-0.12.11-py3-none-win_amd64.whl", hash = "sha256:a3283325960307915b6deb3576b96919ee89432ebd9c48771ca12ee8afe4a0fd", size = 13046600 }, - { url = "https://files.pythonhosted.org/packages/84/a8/001d4a7c2b37623a3fd7463208267fb906df40ff31db496157549cfd6e72/ruff-0.12.11-py3-none-win_arm64.whl", hash = "sha256:bae4d6e6a2676f8fb0f98b74594a048bae1b944aab17e9f5d504062303c6dbea", size = 12135290 }, +version = "0.12.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/f0/e0965dd709b8cabe6356811c0ee8c096806bb57d20b5019eb4e48a117410/ruff-0.12.12.tar.gz", hash = "sha256:b86cd3415dbe31b3b46a71c598f4c4b2f550346d1ccf6326b347cc0c8fd063d6", size = 5359915 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/79/8d3d687224d88367b51c7974cec1040c4b015772bfbeffac95face14c04a/ruff-0.12.12-py3-none-linux_armv6l.whl", hash = "sha256:de1c4b916d98ab289818e55ce481e2cacfaad7710b01d1f990c497edf217dafc", size = 12116602 }, + { url = "https://files.pythonhosted.org/packages/c3/c3/6e599657fe192462f94861a09aae935b869aea8a1da07f47d6eae471397c/ruff-0.12.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7acd6045e87fac75a0b0cdedacf9ab3e1ad9d929d149785903cff9bb69ad9727", size = 12868393 }, + { url = "https://files.pythonhosted.org/packages/e8/d2/9e3e40d399abc95336b1843f52fc0daaceb672d0e3c9290a28ff1a96f79d/ruff-0.12.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:abf4073688d7d6da16611f2f126be86523a8ec4343d15d276c614bda8ec44edb", size = 12036967 }, + { url = "https://files.pythonhosted.org/packages/e9/03/6816b2ed08836be272e87107d905f0908be5b4a40c14bfc91043e76631b8/ruff-0.12.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:968e77094b1d7a576992ac078557d1439df678a34c6fe02fd979f973af167577", size = 12276038 }, + { url = "https://files.pythonhosted.org/packages/9f/d5/707b92a61310edf358a389477eabd8af68f375c0ef858194be97ca5b6069/ruff-0.12.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42a67d16e5b1ffc6d21c5f67851e0e769517fb57a8ebad1d0781b30888aa704e", size = 11901110 }, + { url = "https://files.pythonhosted.org/packages/9d/3d/f8b1038f4b9822e26ec3d5b49cf2bc313e3c1564cceb4c1a42820bf74853/ruff-0.12.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b216ec0a0674e4b1214dcc998a5088e54eaf39417327b19ffefba1c4a1e4971e", size = 13668352 }, + { url = "https://files.pythonhosted.org/packages/98/0e/91421368ae6c4f3765dd41a150f760c5f725516028a6be30e58255e3c668/ruff-0.12.12-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:59f909c0fdd8f1dcdbfed0b9569b8bf428cf144bec87d9de298dcd4723f5bee8", size = 14638365 }, + { url = "https://files.pythonhosted.org/packages/74/5d/88f3f06a142f58ecc8ecb0c2fe0b82343e2a2b04dcd098809f717cf74b6c/ruff-0.12.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ac93d87047e765336f0c18eacad51dad0c1c33c9df7484c40f98e1d773876f5", size = 14060812 }, + { url = "https://files.pythonhosted.org/packages/13/fc/8962e7ddd2e81863d5c92400820f650b86f97ff919c59836fbc4c1a6d84c/ruff-0.12.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01543c137fd3650d322922e8b14cc133b8ea734617c4891c5a9fccf4bfc9aa92", size = 13050208 }, + { url = "https://files.pythonhosted.org/packages/53/06/8deb52d48a9a624fd37390555d9589e719eac568c020b27e96eed671f25f/ruff-0.12.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afc2fa864197634e549d87fb1e7b6feb01df0a80fd510d6489e1ce8c0b1cc45", size = 13311444 }, + { url = "https://files.pythonhosted.org/packages/2a/81/de5a29af7eb8f341f8140867ffb93f82e4fde7256dadee79016ac87c2716/ruff-0.12.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0c0945246f5ad776cb8925e36af2438e66188d2b57d9cf2eed2c382c58b371e5", size = 13279474 }, + { url = "https://files.pythonhosted.org/packages/7f/14/d9577fdeaf791737ada1b4f5c6b59c21c3326f3f683229096cccd7674e0c/ruff-0.12.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a0fbafe8c58e37aae28b84a80ba1817f2ea552e9450156018a478bf1fa80f4e4", size = 12070204 }, + { url = "https://files.pythonhosted.org/packages/77/04/a910078284b47fad54506dc0af13839c418ff704e341c176f64e1127e461/ruff-0.12.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b9c456fb2fc8e1282affa932c9e40f5ec31ec9cbb66751a316bd131273b57c23", size = 11880347 }, + { url = "https://files.pythonhosted.org/packages/df/58/30185fcb0e89f05e7ea82e5817b47798f7fa7179863f9d9ba6fd4fe1b098/ruff-0.12.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f12856123b0ad0147d90b3961f5c90e7427f9acd4b40050705499c98983f489", size = 12891844 }, + { url = "https://files.pythonhosted.org/packages/21/9c/28a8dacce4855e6703dcb8cdf6c1705d0b23dd01d60150786cd55aa93b16/ruff-0.12.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:26a1b5a2bf7dd2c47e3b46d077cd9c0fc3b93e6c6cc9ed750bd312ae9dc302ee", size = 13360687 }, + { url = "https://files.pythonhosted.org/packages/c8/fa/05b6428a008e60f79546c943e54068316f32ec8ab5c4f73e4563934fbdc7/ruff-0.12.12-py3-none-win32.whl", hash = "sha256:173be2bfc142af07a01e3a759aba6f7791aa47acf3604f610b1c36db888df7b1", size = 12052870 }, + { url = "https://files.pythonhosted.org/packages/85/60/d1e335417804df452589271818749d061b22772b87efda88354cf35cdb7a/ruff-0.12.12-py3-none-win_amd64.whl", hash = "sha256:e99620bf01884e5f38611934c09dd194eb665b0109104acae3ba6102b600fd0d", size = 13178016 }, + { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762 }, ] [[package]] @@ -2627,15 +3002,15 @@ wheels = [ [[package]] name = "sentry-sdk" -version = "2.35.2" +version = "2.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bd/79/0ecb942f3f1ad26c40c27f81ff82392d85c01d26a45e3c72c2b37807e680/sentry_sdk-2.35.2.tar.gz", hash = "sha256:e9e8f3c795044beb59f2c8f4c6b9b0f9779e5e604099882df05eec525e782cc6", size = 343377 } +sdist = { url = "https://files.pythonhosted.org/packages/af/9a/0b2eafc31d5c7551b6bef54ca10d29adea471e0bd16bfe985a9dc4b6633e/sentry_sdk-2.37.0.tar.gz", hash = "sha256:2c661a482dd5accf3df58464f31733545745bb4d5cf8f5e46e0e1c4eed88479f", size = 346203 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/91/a43308dc82a0e32d80cd0dfdcfca401ecbd0f431ab45f24e48bb97b7800d/sentry_sdk-2.35.2-py2.py3-none-any.whl", hash = "sha256:38c98e3cbb620dd3dd80a8d6e39c753d453dd41f8a9df581b0584c19a52bc926", size = 363975 }, + { url = "https://files.pythonhosted.org/packages/07/d5/f9f4a2bf5db2ca8f692c46f3821fee1f302f1b76a0e2914aee5390fca565/sentry_sdk-2.37.0-py2.py3-none-any.whl", hash = "sha256:89c1ed205d5c25926558b64a9bed8a5b4fb295b007cecc32c0ec4bf7694da2e1", size = 368304 }, ] [[package]] @@ -2693,6 +3068,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, ] +[[package]] +name = "soupsieve" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/e6/21ccce3262dd4889aa3332e5a119a3491a95e8f60939870a3a035aabac0d/soupsieve-2.8.tar.gz", hash = "sha256:e2dd4a40a628cb5f28f6d4b0db8800b8f581b65bb380b97de22ba5ca8d72572f", size = 103472 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/a0/bb38d3b76b8cae341dad93a2dd83ab7462e6dbcdd84d43f54ee60a8dc167/soupsieve-2.8-py3-none-any.whl", hash = "sha256:0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c", size = 36679 }, +] + [[package]] name = "sqlalchemy" version = "2.0.43" @@ -2828,6 +3212,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638 }, ] +[[package]] +name = "tiktoken" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/86/ad0155a37c4f310935d5ac0b1ccf9bdb635dcb906e0a9a26b616dd55825a/tiktoken-0.11.0.tar.gz", hash = "sha256:3c518641aee1c52247c2b97e74d8d07d780092af79d5911a6ab5e79359d9b06a", size = 37648 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/9e/eceddeffc169fc75fe0fd4f38471309f11cb1906f9b8aa39be4f5817df65/tiktoken-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fd9e6b23e860973cf9526544e220b223c60badf5b62e80a33509d6d40e6c8f5d", size = 1055199 }, + { url = "https://files.pythonhosted.org/packages/4f/cf/5f02bfefffdc6b54e5094d2897bc80efd43050e5b09b576fd85936ee54bf/tiktoken-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a76d53cee2da71ee2731c9caa747398762bda19d7f92665e882fef229cb0b5b", size = 996655 }, + { url = "https://files.pythonhosted.org/packages/65/8e/c769b45ef379bc360c9978c4f6914c79fd432400a6733a8afc7ed7b0726a/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef72aab3ea240646e642413cb363b73869fed4e604dcfd69eec63dc54d603e8", size = 1128867 }, + { url = "https://files.pythonhosted.org/packages/d5/2d/4d77f6feb9292bfdd23d5813e442b3bba883f42d0ac78ef5fdc56873f756/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f929255c705efec7a28bf515e29dc74220b2f07544a8c81b8d69e8efc4578bd", size = 1183308 }, + { url = "https://files.pythonhosted.org/packages/7a/65/7ff0a65d3bb0fc5a1fb6cc71b03e0f6e71a68c5eea230d1ff1ba3fd6df49/tiktoken-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61f1d15822e4404953d499fd1dcc62817a12ae9fb1e4898033ec8fe3915fdf8e", size = 1244301 }, + { url = "https://files.pythonhosted.org/packages/f5/6e/5b71578799b72e5bdcef206a214c3ce860d999d579a3b56e74a6c8989ee2/tiktoken-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:45927a71ab6643dfd3ef57d515a5db3d199137adf551f66453be098502838b0f", size = 884282 }, +] + +[[package]] +name = "tokenizers" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/b4/c1ce3699e81977da2ace8b16d2badfd42b060e7d33d75c4ccdbf9dc920fa/tokenizers-0.22.0.tar.gz", hash = "sha256:2e33b98525be8453f355927f3cab312c36cd3e44f4d7e9e97da2fa94d0a49dcb", size = 362771 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/b1/18c13648edabbe66baa85fe266a478a7931ddc0cd1ba618802eb7b8d9865/tokenizers-0.22.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:eaa9620122a3fb99b943f864af95ed14c8dfc0f47afa3b404ac8c16b3f2bb484", size = 3081954 }, + { url = "https://files.pythonhosted.org/packages/c2/02/c3c454b641bd7c4f79e4464accfae9e7dfc913a777d2e561e168ae060362/tokenizers-0.22.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:71784b9ab5bf0ff3075bceeb198149d2c5e068549c0d18fe32d06ba0deb63f79", size = 2945644 }, + { url = "https://files.pythonhosted.org/packages/55/02/d10185ba2fd8c2d111e124c9d92de398aee0264b35ce433f79fb8472f5d0/tokenizers-0.22.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec5b71f668a8076802b0241a42387d48289f25435b86b769ae1837cad4172a17", size = 3254764 }, + { url = "https://files.pythonhosted.org/packages/13/89/17514bd7ef4bf5bfff58e2b131cec0f8d5cea2b1c8ffe1050a2c8de88dbb/tokenizers-0.22.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ea8562fa7498850d02a16178105b58803ea825b50dc9094d60549a7ed63654bb", size = 3161654 }, + { url = "https://files.pythonhosted.org/packages/5a/d8/bac9f3a7ef6dcceec206e3857c3b61bb16c6b702ed7ae49585f5bd85c0ef/tokenizers-0.22.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4136e1558a9ef2e2f1de1555dcd573e1cbc4a320c1a06c4107a3d46dc8ac6e4b", size = 3511484 }, + { url = "https://files.pythonhosted.org/packages/aa/27/9c9800eb6763683010a4851db4d1802d8cab9cec114c17056eccb4d4a6e0/tokenizers-0.22.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cdf5954de3962a5fd9781dc12048d24a1a6f1f5df038c6e95db328cd22964206", size = 3712829 }, + { url = "https://files.pythonhosted.org/packages/10/e3/b1726dbc1f03f757260fa21752e1921445b5bc350389a8314dd3338836db/tokenizers-0.22.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8337ca75d0731fc4860e6204cc24bb36a67d9736142aa06ed320943b50b1e7ed", size = 3408934 }, + { url = "https://files.pythonhosted.org/packages/d4/61/aeab3402c26874b74bb67a7f2c4b569dde29b51032c5384db592e7b216f4/tokenizers-0.22.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a89264e26f63c449d8cded9061adea7b5de53ba2346fc7e87311f7e4117c1cc8", size = 3345585 }, + { url = "https://files.pythonhosted.org/packages/bc/d3/498b4a8a8764cce0900af1add0f176ff24f475d4413d55b760b8cdf00893/tokenizers-0.22.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:790bad50a1b59d4c21592f9c3cf5e5cf9c3c7ce7e1a23a739f13e01fb1be377a", size = 9322986 }, + { url = "https://files.pythonhosted.org/packages/a2/62/92378eb1c2c565837ca3cb5f9569860d132ab9d195d7950c1ea2681dffd0/tokenizers-0.22.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:76cf6757c73a10ef10bf06fa937c0ec7393d90432f543f49adc8cab3fb6f26cb", size = 9276630 }, + { url = "https://files.pythonhosted.org/packages/eb/f0/342d80457aa1cda7654327460f69db0d69405af1e4c453f4dc6ca7c4a76e/tokenizers-0.22.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:1626cb186e143720c62c6c6b5371e62bbc10af60481388c0da89bc903f37ea0c", size = 9547175 }, + { url = "https://files.pythonhosted.org/packages/14/84/8aa9b4adfc4fbd09381e20a5bc6aa27040c9c09caa89988c01544e008d18/tokenizers-0.22.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:da589a61cbfea18ae267723d6b029b84598dc8ca78db9951d8f5beff72d8507c", size = 9692735 }, + { url = "https://files.pythonhosted.org/packages/bf/24/83ee2b1dc76bfe05c3142e7d0ccdfe69f0ad2f1ebf6c726cea7f0874c0d0/tokenizers-0.22.0-cp39-abi3-win32.whl", hash = "sha256:dbf9d6851bddae3e046fedfb166f47743c1c7bd11c640f0691dd35ef0bcad3be", size = 2471915 }, + { url = "https://files.pythonhosted.org/packages/d1/9b/0e0bf82214ee20231845b127aa4a8015936ad5a46779f30865d10e404167/tokenizers-0.22.0-cp39-abi3-win_amd64.whl", hash = "sha256:c78174859eeaee96021f248a56c801e36bfb6bd5b067f2e95aa82445ca324f00", size = 2680494 }, +] + [[package]] name = "torch" version = "2.8.0" @@ -2956,6 +3383,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 }, ] +[[package]] +name = "typing-inspect" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827 }, +] + [[package]] name = "typing-inspection" version = "0.4.1" @@ -2998,7 +3438,7 @@ wheels = [ [[package]] name = "ultralytics" -version = "8.3.192" +version = "8.3.193" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "matplotlib" }, @@ -3015,9 +3455,9 @@ dependencies = [ { name = "torchvision" }, { name = "ultralytics-thop" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/43/db/e019a5dc105a2221973b9ae9ed8d86e7bfcd7265f25310bb59ff30c9379a/ultralytics-8.3.192.tar.gz", hash = "sha256:b40c80b3196d40cef24abf1b21f061f2ff2493fe7c3ff5a905984cfd36ce3f5a", size = 914537 } +sdist = { url = "https://files.pythonhosted.org/packages/de/ba/300fba5b3c17527c2e6bc77af9bd149bfcaf9eb0f58ba3a99a8e6bdb2579/ultralytics-8.3.193.tar.gz", hash = "sha256:bec0001205b3298b099c94ec4988d41b225c07e7af57fb02b612eb1546ce3e6a", size = 914054 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/07/bbae3b606c484abe191751b9e0fa289f65a5a7f0469edffcf095a0753ca7/ultralytics-8.3.192-py3-none-any.whl", hash = "sha256:d425d3e7f8d8affc58525aa544db6bbe746949e180e197dc5c93e0ccd603aa3e", size = 1065270 }, + { url = "https://files.pythonhosted.org/packages/6c/8a/26f3ac48bd88862fb7309181a3a650b6693996d18dabeee5f4a9bcdc3ec0/ultralytics-8.3.193-py3-none-any.whl", hash = "sha256:e76832da9e50b93fc38137e4bd8c664b1ed7463b09f855f5cf17408f3675a958", size = 1064853 }, ] [[package]] @@ -3193,3 +3633,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702 }, { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466 }, ] + +[[package]] +name = "zstandard" +version = "0.24.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/1b/c20b2ef1d987627765dcd5bf1dadb8ef6564f00a87972635099bb76b7a05/zstandard-0.24.0.tar.gz", hash = "sha256:fe3198b81c00032326342d973e526803f183f97aa9e9a98e3f897ebafe21178f", size = 905681 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/e9/0bd281d9154bba7fc421a291e263911e1d69d6951aa80955b992a48289f6/zstandard-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a2bda8f2790add22773ee7a4e43c90ea05598bffc94c21c40ae0a9000b0133c3", size = 795710 }, + { url = "https://files.pythonhosted.org/packages/36/26/b250a2eef515caf492e2d86732e75240cdac9d92b04383722b9753590c36/zstandard-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cc76de75300f65b8eb574d855c12518dc25a075dadb41dd18f6322bda3fe15d5", size = 640336 }, + { url = "https://files.pythonhosted.org/packages/79/bf/3ba6b522306d9bf097aac8547556b98a4f753dc807a170becaf30dcd6f01/zstandard-0.24.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:d2b3b4bda1a025b10fe0269369475f420177f2cb06e0f9d32c95b4873c9f80b8", size = 5342533 }, + { url = "https://files.pythonhosted.org/packages/ea/ec/22bc75bf054e25accdf8e928bc68ab36b4466809729c554ff3a1c1c8bce6/zstandard-0.24.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b84c6c210684286e504022d11ec294d2b7922d66c823e87575d8b23eba7c81f", size = 5062837 }, + { url = "https://files.pythonhosted.org/packages/48/cc/33edfc9d286e517fb5b51d9c3210e5bcfce578d02a675f994308ca587ae1/zstandard-0.24.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c59740682a686bf835a1a4d8d0ed1eefe31ac07f1c5a7ed5f2e72cf577692b00", size = 5393855 }, + { url = "https://files.pythonhosted.org/packages/73/36/59254e9b29da6215fb3a717812bf87192d89f190f23817d88cb8868c47ac/zstandard-0.24.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6324fde5cf5120fbf6541d5ff3c86011ec056e8d0f915d8e7822926a5377193a", size = 5451058 }, + { url = "https://files.pythonhosted.org/packages/9a/c7/31674cb2168b741bbbe71ce37dd397c9c671e73349d88ad3bca9e9fae25b/zstandard-0.24.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51a86bd963de3f36688553926a84e550d45d7f9745bd1947d79472eca27fcc75", size = 5546619 }, + { url = "https://files.pythonhosted.org/packages/e6/01/1a9f22239f08c00c156f2266db857545ece66a6fc0303d45c298564bc20b/zstandard-0.24.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d82ac87017b734f2fb70ff93818c66f0ad2c3810f61040f077ed38d924e19980", size = 5046676 }, + { url = "https://files.pythonhosted.org/packages/a7/91/6c0cf8fa143a4988a0361380ac2ef0d7cb98a374704b389fbc38b5891712/zstandard-0.24.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:92ea7855d5bcfb386c34557516c73753435fb2d4a014e2c9343b5f5ba148b5d8", size = 5576381 }, + { url = "https://files.pythonhosted.org/packages/e2/77/1526080e22e78871e786ccf3c84bf5cec9ed25110a9585507d3c551da3d6/zstandard-0.24.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3adb4b5414febf074800d264ddf69ecade8c658837a83a19e8ab820e924c9933", size = 4953403 }, + { url = "https://files.pythonhosted.org/packages/6e/d0/a3a833930bff01eab697eb8abeafb0ab068438771fa066558d96d7dafbf9/zstandard-0.24.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6374feaf347e6b83ec13cc5dcfa70076f06d8f7ecd46cc71d58fac798ff08b76", size = 5267396 }, + { url = "https://files.pythonhosted.org/packages/f3/5e/90a0db9a61cd4769c06374297ecfcbbf66654f74cec89392519deba64d76/zstandard-0.24.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:13fc548e214df08d896ee5f29e1f91ee35db14f733fef8eabea8dca6e451d1e2", size = 5433269 }, + { url = "https://files.pythonhosted.org/packages/ce/58/fc6a71060dd67c26a9c5566e0d7c99248cbe5abfda6b3b65b8f1a28d59f7/zstandard-0.24.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0a416814608610abf5488889c74e43ffa0343ca6cf43957c6b6ec526212422da", size = 5814203 }, + { url = "https://files.pythonhosted.org/packages/5c/6a/89573d4393e3ecbfa425d9a4e391027f58d7810dec5cdb13a26e4cdeef5c/zstandard-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0d66da2649bb0af4471699aeb7a83d6f59ae30236fb9f6b5d20fb618ef6c6777", size = 5359622 }, + { url = "https://files.pythonhosted.org/packages/60/ff/2cbab815d6f02a53a9d8d8703bc727d8408a2e508143ca9af6c3cca2054b/zstandard-0.24.0-cp312-cp312-win32.whl", hash = "sha256:ff19efaa33e7f136fe95f9bbcc90ab7fb60648453b03f95d1de3ab6997de0f32", size = 435968 }, + { url = "https://files.pythonhosted.org/packages/ce/a3/8f96b8ddb7ad12344218fbd0fd2805702dafd126ae9f8a1fb91eef7b33da/zstandard-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc05f8a875eb651d1cc62e12a4a0e6afa5cd0cc231381adb830d2e9c196ea895", size = 505195 }, + { url = "https://files.pythonhosted.org/packages/a3/4a/bfca20679da63bfc236634ef2e4b1b4254203098b0170e3511fee781351f/zstandard-0.24.0-cp312-cp312-win_arm64.whl", hash = "sha256:b04c94718f7a8ed7cdd01b162b6caa1954b3c9d486f00ecbbd300f149d2b2606", size = 461605 }, +]