diff --git a/.circleci/config.yml b/.circleci/config.yml index b87aa96..349012d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,6 @@ jobs: name: Create config files for testing command: | cp config/config.yaml.example config/config.yaml - cp config/mam_config.json.example config/mam_config.json - run: name: Run linter command: ruff check src/ tests/ diff --git a/.env.example b/.env.example index 4379aaa..b6280ae 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,9 @@ AUTOBRR_TOKEN=your-autobrr-webhook-token # SECURITY: Never share this value - it grants full access to your MAM account MAM_ID=your-mam-session-cookie-value +# Optional: known safe torrent ID for manual download integration testing +# MAM_TEST_TID=1234567 + # ============================================================================= # QBITTORRENT SETTINGS # ============================================================================= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 207298e..dee62a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,6 @@ jobs: - name: Create config files for testing run: | cp config/config.yaml.example config/config.yaml - cp config/mam_config.json.example config/mam_config.json - name: Lint with ruff run: | diff --git a/.gitignore b/.gitignore index f8690f3..c4be7ef 100644 --- a/.gitignore +++ b/.gitignore @@ -51,9 +51,9 @@ config/config.yaml !config/config.yaml.example config/mam_config.json mam_cookies.json -!config/mam_config.json.example logs/ db.sqlite +*.sqlite-journal tests/__pycache__/ .pytest_cache/ db.sqlite @@ -79,3 +79,6 @@ logs/*_debug_*.html coverage.xml .coverage *.coverage + +# Ignore secrets directory (contains sensitive information, not to be committed) +secrets/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3be1b51..b8613d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,7 @@ repos: hooks: - id: bandit args: [-c, pyproject.toml] + exclude: ^tests/ additional_dependencies: ['bandit[toml]'] # Check for common security issues diff --git a/audiobook_dev.egg-info/PKG-INFO b/audiobook_dev.egg-info/PKG-INFO deleted file mode 100644 index 6925877..0000000 --- a/audiobook_dev.egg-info/PKG-INFO +++ /dev/null @@ -1,275 +0,0 @@ -Metadata-Version: 2.4 -Name: audiobook-dev -Version: 2.0.0 -Summary: Automated audiobook request system with MAM integration -Author-email: H2OKing89 -License: MIT -Project-URL: Homepage, https://github.com/H2OKing89/audiobook_dev -Project-URL: Repository, https://github.com/H2OKing89/audiobook_dev -Project-URL: Issues, https://github.com/H2OKing89/audiobook_dev/issues -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: End Users/Desktop -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content -Requires-Python: >=3.11 -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: fastapi -Requires-Dist: uvicorn -Requires-Dist: python-dotenv -Requires-Dist: qbittorrent-api -Requires-Dist: beautifulsoup4 -Requires-Dist: jinja2 -Requires-Dist: httpx[http2]>=0.28.1 -Requires-Dist: cachetools -Requires-Dist: lxml -Requires-Dist: html5lib -Requires-Dist: slowapi -Requires-Dist: fastapi-limiter -Requires-Dist: pydantic>=2.0.0 -Requires-Dist: python-multipart -Requires-Dist: itsdangerous -Requires-Dist: idna -Requires-Dist: cryptography -Requires-Dist: PyYAML -Provides-Extra: dev -Requires-Dist: pytest>=8.0.0; extra == "dev" -Requires-Dist: pytest-cov>=4.1.0; extra == "dev" -Requires-Dist: pytest-asyncio>=1.3.0; extra == "dev" -Requires-Dist: pytest-timeout>=2.2.0; extra == "dev" -Requires-Dist: ruff>=0.8.0; extra == "dev" -Requires-Dist: mypy>=1.8.0; extra == "dev" -Requires-Dist: pre-commit>=3.6.0; extra == "dev" -Requires-Dist: types-PyYAML; extra == "dev" -Requires-Dist: types-cachetools; extra == "dev" -Requires-Dist: types-beautifulsoup4; extra == "dev" -Dynamic: license-file - -# ๐ŸŽง Audiobook Automation System - -A modern, secure, and delightfully over-engineered FastAPI microservice for automated audiobook approval workflows. Built by Quentin with maximum automation and minimum manual intervention in mind. - -## ๐Ÿ›ก๏ธ Security Status: โœ… VERIFIED - -**Last Audit**: June 16, 2025 | **Status**: 13/13 Security Tests Passing | **UI**: Cyberpunk Theme Secured - ---- - -## โœจ Features - -- **๐Ÿ”’ Secure Webhook Endpoint** - Token-validated integration with Autobrr/MAM -- **๐Ÿ“– Metadata Enrichment** - Audnex API and Audible scraping for rich book data -- **๐Ÿ’พ Persistent Storage** - SQLite database with comprehensive audit trails -- **โฐ Time-Limited Tokens** - Cryptographically secure, single-use approval tokens -- **๐Ÿ“ฑ Multi-Platform Notifications** - Pushover, Discord, Gotify, and Ntfy support -- **๐ŸŽจ Beautiful Web Interface** - Modern, responsive UI with cyberpunk/anime aesthetics -- **๐ŸŒ Social Media Ready** - Dynamic OG/Twitter meta tags for all pages -- **โš™๏ธ qBittorrent Integration** - Automated torrent handling with MAM cookie support -- **๐Ÿš€ Async Performance** - Threadpool handling for optimal responsiveness -- **๐Ÿ“Š Comprehensive Logging** - Centralized, rotating logs with detailed audit trails -- **โ™ฟ Accessibility First** - WCAG 2.1 AA compliance with ARIA labels and keyboard navigation -- **๐Ÿงช Test Coverage** - Comprehensive unit and integration test suite - ---- - -## ๐Ÿ“š Documentation - -Complete documentation is available in the [`docs/`](docs/) directory: - -### ๐ŸŽฏ For Users - -- **[๐Ÿ“– Getting Started](docs/user-guide/getting-started.md)** - Installation and setup guide -- **[โš™๏ธ Configuration](docs/user-guide/configuration.md)** - Configuration options and examples -- **[๐ŸŒ Web Interface](docs/user-guide/web-interface.md)** - Using the web UI -- **[๐Ÿ“ฑ Notifications](docs/user-guide/notifications.md)** - Setting up notification services -- **[๐Ÿ”ง Troubleshooting](docs/user-guide/troubleshooting.md)** - Common issues and solutions - -### ๐Ÿ› ๏ธ For Developers - -- **[๐Ÿ—๏ธ Architecture](docs/development/architecture.md)** - System design and component overview -- **[๐Ÿ” Security](docs/development/SECURITY.md)** - Security implementation details -- **[๐ŸŽจ Interactive Fixes](docs/development/INTERACTIVE_FIXES.md)** - UI/UX improvements -- **[๐Ÿ“‹ Logging](docs/development/LOGGING_IMPROVEMENTS.md)** - Enhanced logging system -- **[๐Ÿงช Testing](docs/development/testing.md)** - Testing strategies and guidelines - -### ๐Ÿ”Œ API Reference - -- **[๐ŸŒ REST API](docs/api/rest-api.md)** - Complete API documentation -- **[๐Ÿ”— Webhooks](docs/api/webhooks.md)** - Webhook configuration and payloads -- **[๐Ÿ’พ Database](docs/api/database.md)** - Database schema and queries -- **[๐Ÿ“‹ Configuration](docs/api/config-reference.md)** - Complete configuration reference - ---- - -## ๐Ÿš€ Quick Start - -```bash -# Clone the repository -git clone https://github.com/kingpaging/audiobook-automation.git -cd audiobook-automation - -# Set up virtual environment -python -m venv .venv -source .venv/bin/activate # Windows: .venv\Scripts\activate - -# Install dependencies -pip install -r requirements.txt - -# Configure the system -cp config/config.yaml.example config/config.yaml -# Edit config/config.yaml with your settings - -# Initialize database -python src/db.py - -# Start the application -python src/main.py -``` - -Visit `http://localhost:8000` to access the beautiful web interface! - -For detailed setup instructions, see the [Getting Started Guide](docs/user-guide/getting-started.md). - ---- - -## ๐Ÿ—๏ธ Project Structure - -``` -audiobook_dev/ -โ”œโ”€โ”€ docs/ # ๐Ÿ“š Comprehensive documentation -โ”‚ โ”œโ”€โ”€ user-guide/ # User documentation and guides -โ”‚ โ”œโ”€โ”€ development/ # Developer and architecture docs -โ”‚ โ””โ”€โ”€ api/ # API reference and webhooks -โ”œโ”€โ”€ src/ # ๐Ÿ Python source code -โ”‚ โ”œโ”€โ”€ main.py # FastAPI application entry point -โ”‚ โ”œโ”€โ”€ webui.py # Web interface and routes -โ”‚ โ”œโ”€โ”€ metadata.py # Audiobook metadata handling -โ”‚ โ”œโ”€โ”€ token_gen.py # Secure token generation/validation -โ”‚ โ”œโ”€โ”€ notify/ # ๐Ÿ“ฑ Notification service modules -โ”‚ โ”‚ โ”œโ”€โ”€ pushover.py # Pushover notifications -โ”‚ โ”‚ โ”œโ”€โ”€ gotify.py # Gotify notifications -โ”‚ โ”‚ โ”œโ”€โ”€ discord.py # Discord notifications -โ”‚ โ”‚ โ””โ”€โ”€ ntfy.py # Ntfy notifications -โ”‚ โ”œโ”€โ”€ qbittorrent.py # qBittorrent integration -โ”‚ โ”œโ”€โ”€ db.py # SQLite database operations -โ”‚ โ”œโ”€โ”€ config.py # Configuration management -โ”‚ โ”œโ”€โ”€ html.py # Jinja2 template utilities -โ”‚ โ””โ”€โ”€ utils.py # Shared utility functions -โ”œโ”€โ”€ templates/ # ๐ŸŽจ Jinja2 HTML templates -โ”‚ โ”œโ”€โ”€ base.html # Base template with common elements -โ”‚ โ”œโ”€โ”€ index.html # Enhanced home page -โ”‚ โ”œโ”€โ”€ approval.html # Approval workflow page -โ”‚ โ”œโ”€โ”€ rejection.html # Witty rejection page -โ”‚ โ””โ”€โ”€ *.html # Additional UI templates -โ”œโ”€โ”€ static/ # ๐ŸŒ Static web assets -โ”‚ โ”œโ”€โ”€ css/style.css # Enhanced cyberpunk styling -โ”‚ โ””โ”€โ”€ js/app.js # Interactive JavaScript features -โ”œโ”€โ”€ tests/ # ๐Ÿงช Comprehensive test suite -โ”œโ”€โ”€ config/ # โš™๏ธ Configuration files -โ”‚ โ””โ”€โ”€ config.yaml # Main application configuration -โ”œโ”€โ”€ logs/ # ๐Ÿ“‹ Application logs -โ””โ”€โ”€ db.sqlite # ๐Ÿ’พ SQLite database -``` - ---- - -## Setup - -1. **Clone the repo** - - ```bash - git clone - cd audiobook_dev - ``` - -2. **Create and activate a virtualenv** - - ```bash - python3 -m venv .venv - source .venv/bin/activate - ``` - -3. **Install dependencies** - - ```bash - pip install -r requirements.txt - ``` - -4. **Copy and edit config** - - Edit `config/config.yaml` for your environment (API URLs, notification settings, etc). - - Create a `.env` file with your secrets (see `.env.example`). - ---- - -## Running - -```bash -uvicorn src.main:app --host 0.0.0.0 --port 8000 --reload -``` - -- The webhook endpoint is set in `config.yaml` (default: `/webhook/audiobook-requests`). -- The web UI is available at `/`. - ---- - -## Notifications - -- **Pushover**: Rich HTML, cover image, approval link. -- **Discord**: Embed with cover, links, and markdown. -- **Gotify**: Markdown, cover image, action links. -- **ntfy**: Markdown, cover image, action links. - -Configure each in `config/config.yaml` and `.env`. - ---- - -## Metadata - -- Uses Audnex API for fast, reliable metadata. -- Falls back to Audible scraping if needed. -- Cleans and normalizes author, narrator, series, and description fields. -- Caches lookups with LRU cache for efficiency. - ---- - -## Testing - -- Run all tests: - - ```bash - pytest -vv - ``` - -- Tests cover: - - Metadata cleaning and validation - - Notification formatting - - Web UI endpoints - - Error cases -- Fixtures in `tests/conftest.py` for isolation. - ---- - -## Development - -- Code style: Black, isort, flake8 recommended. -- Logging is configurable in `config.yaml`. -- All user input is sanitized before rendering or sending to notification services. -- For async/production, consider running with Gunicorn/Uvicorn workers. - ---- - -## Security - -- Webhook endpoints require a token (set in `.env`). -- Never commit `.env` or real secrets. -- All user input is sanitized. - ---- - -## License - -MIT License. See `LICENSE` for details. diff --git a/audiobook_dev.egg-info/SOURCES.txt b/audiobook_dev.egg-info/SOURCES.txt deleted file mode 100644 index cc0d299..0000000 --- a/audiobook_dev.egg-info/SOURCES.txt +++ /dev/null @@ -1,40 +0,0 @@ -LICENSE -README.md -pyproject.toml -audiobook_dev.egg-info/PKG-INFO -audiobook_dev.egg-info/SOURCES.txt -audiobook_dev.egg-info/dependency_links.txt -audiobook_dev.egg-info/requires.txt -audiobook_dev.egg-info/top_level.txt -src/__init__.py -src/audible_scraper.py -src/audnex_metadata.py -src/config.py -src/db.py -src/main.py -src/metadata.py -src/metadata_coordinator.py -src/qbittorrent.py -src/security.py -src/template_helpers.py -src/token_gen.py -src/utils.py -src/webui.py -tests/test_audnex_direct.py -tests/test_config.py -tests/test_database_integration.py -tests/test_end_to_end.py -tests/test_error_recovery.py -tests/test_integration.py -tests/test_main_integration.py -tests/test_mam_api.py -tests/test_mam_login.py -tests/test_metadata_extended.py -tests/test_metadata_workflow.py -tests/test_notify_formatting.py -tests/test_qbittorrent.py -tests/test_security.py -tests/test_token_gen.py -tests/test_utils_extra.py -tests/test_utils_metadata.py -tests/test_webui_extended.py diff --git a/audiobook_dev.egg-info/dependency_links.txt b/audiobook_dev.egg-info/dependency_links.txt deleted file mode 100644 index e69de29..0000000 diff --git a/audiobook_dev.egg-info/requires.txt b/audiobook_dev.egg-info/requires.txt deleted file mode 100644 index 62592b5..0000000 --- a/audiobook_dev.egg-info/requires.txt +++ /dev/null @@ -1,30 +0,0 @@ -fastapi -uvicorn -python-dotenv -qbittorrent-api -beautifulsoup4 -jinja2 -httpx[http2]>=0.28.1 -cachetools -lxml -html5lib -slowapi -fastapi-limiter -pydantic>=2.0.0 -python-multipart -itsdangerous -idna -cryptography -PyYAML - -[dev] -pytest>=8.0.0 -pytest-cov>=4.1.0 -pytest-asyncio>=1.3.0 -pytest-timeout>=2.2.0 -ruff>=0.8.0 -mypy>=1.8.0 -pre-commit>=3.6.0 -types-PyYAML -types-cachetools -types-beautifulsoup4 diff --git a/audiobook_dev.egg-info/top_level.txt b/audiobook_dev.egg-info/top_level.txt deleted file mode 100644 index 85de9cf..0000000 --- a/audiobook_dev.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -src diff --git a/config/mam_config.json.example b/config/mam_config.json.example deleted file mode 100644 index 7c90c1b..0000000 --- a/config/mam_config.json.example +++ /dev/null @@ -1,9 +0,0 @@ -{ - "username": "your_mam_username", - "password": "your_mam_password", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", - "viewport": {"width": 1920, "height": 1080}, - "timeout": 30000, - "wait_for_login": 5000, - "cookie_file": "mam_cookies.json" -} diff --git a/docs/CONCURRENCY_ANALYSIS.md b/docs/CONCURRENCY_ANALYSIS.md index 2a7a26e..3d4fa2b 100644 --- a/docs/CONCURRENCY_ANALYSIS.md +++ b/docs/CONCURRENCY_ANALYSIS.md @@ -38,9 +38,9 @@ class MetadataCoordinator: ### โš ๏ธ **Resource Contention** -- Multiple Playwright browser instances -- Concurrent MAM logins/cookie sharing conflicts -- Browser memory usage spikes +- Multiple HTTP clients and open connections +- Concurrent MAM API calls sharing the same session cookie +- Connection pool pressure during bursts ### โš ๏ธ **API Overload** diff --git a/docs/MAM_API_MIGRATION.md b/docs/MAM_API_MIGRATION.md index e3c4664..cd827dc 100644 --- a/docs/MAM_API_MIGRATION.md +++ b/docs/MAM_API_MIGRATION.md @@ -1,9 +1,9 @@ # MAM (MyAnonamouse) API Migration -**Status**: โœ… Ready for Testing -**Branch**: `mam-api-migration` +**Status**: โœ… API Search and Download Verified +**Branch**: `refact/mam_api` **Created**: December 23, 2025 -**Last Updated**: December 23, 2025 +**Last Updated**: May 5, 2026 ## Overview @@ -19,7 +19,7 @@ This document tracks the migration from HTML scraping to MAM's JSON API for torr ### What's Changing | Before | After | -|--------|-------| +| -------- | ------- | | HTML scraping via Playwright | JSON API via httpx (HTTP/2) | | `mam_config.json` with credentials | `MAM_ID` cookie from `.env` | | Complex browser session management | Simple cookie-based auth | @@ -108,25 +108,26 @@ Several fields are JSON-encoded strings that must be parsed: ### Phase 3: Integration โœ… -- [x] Create `src/mam_api/adapter.py` - backward-compatible adapter -- [x] Add `MAMApiAdapter` with same interface as `MAMScraper` -- [x] Export `MAMScraper` alias for drop-in replacement +- [x] Create `src/mam_api/adapter.py` - API metadata adapter +- [x] Add `MAMApiAdapter` for ASIN and full metadata lookups +- [x] Remove old `MAMScraper` compatibility alias - [x] Update `metadata_coordinator.py` import to use new adapter -- [ ] Test with real MAM data +- [x] Test with real MAM data ### Phase 4: Cleanup โœ… -- [x] Remove `config/mam_config.json` (contains sensitive data!) +- [x] Remove `config/mam_config.json.example` - [x] Remove `src/mam_login_only.py` (no longer needed) -- [x] Archive old `src/mam_scraper.py` (keep for reference) +- [x] Remove archived Playwright scraper login code - [x] Update remaining documentation ### Phase 5: Testing โœ… - [x] Add unit tests for Pydantic models - [x] Add integration tests for API client -- [x] Add adapter tests (40 tests total, all passing) -- [ ] Verify torrent download functionality (requires real MAM_ID) +- [x] Add adapter and client tests +- [x] Add optional skipped-by-default torrent download integration test +- [x] Verify torrent download functionality (requires real MAM_ID and MAM_TEST_TID) --- @@ -137,7 +138,7 @@ Several fields are JSON-encoded strings that must be parsed: - `src/mam_api/__init__.py` โœ… - `src/mam_api/models.py` - Pydantic models for API responses โœ… - `src/mam_api/client.py` - httpx HTTP/2 client โœ… -- `src/mam_api/adapter.py` - Backward-compatible adapter โœ… +- `src/mam_api/adapter.py` - API metadata adapter โœ… ### Modified Files @@ -146,12 +147,9 @@ Several fields are JSON-encoded strings that must be parsed: ### Files Removed โœ… -- `config/mam_config.json` - Sensitive! Contains credentials โœ… +- `config/mam_config.json.example` - Old web-login credential template โœ… - `src/mam_login_only.py` - No longer needed โœ… - -### Files Archived โœ… - -- `src/mam_scraper.py` โ†’ `src/archive/mam_scraper.py.old` โœ… +- `src/archive/mam_scraper.py.old` - Old Playwright login scraper โœ… --- @@ -189,6 +187,14 @@ with MamClient(mam_id=os.getenv("MAM_ID")) as mam: torrent_bytes = mam.download_torrent_by_tid(tid) ``` +### Optional Download Integration Test + +The real download test is skipped by default. To run it, set `MAM_ID` and a known safe torrent ID: + +```bash +MAM_ID=your_cookie_value MAM_TEST_TID=1207719 pytest tests/test_mam_api.py -k real_download --no-cov +``` + --- ## Progress Log @@ -199,3 +205,10 @@ with MamClient(mam_id=os.getenv("MAM_ID")) as mam: - Created migration documentation - Implemented core Pydantic models - Implemented httpx client (sync + async) + +### May 5, 2026 + +- Removed old web portal login/scraper code and Playwright dependency +- Verified real MAM API search with refreshed `MAM_ID` +- Added optional `MAM_TEST_TID` integration test for real `.torrent` download verification +- Verified real `.torrent` download with `MAM_TEST_TID=1234567` (replace with actual safe torrent ID) diff --git a/docs/PR1_REVIEW_FIXES.md b/docs/PR1_REVIEW_FIXES.md index 9290c28..58d7a5e 100644 --- a/docs/PR1_REVIEW_FIXES.md +++ b/docs/PR1_REVIEW_FIXES.md @@ -210,7 +210,7 @@ Most Round 3 review comments re-flagged issues already resolved in Round 2: - **File**: [requirements.txt](https://github.com/H2OKing89/audiobook_dev/pull/1#discussion_r2642728717) - **Lines**: 7, 21-22 -- **Issue**: jinja2, PyYAML, playwright not pinned to specific versions +- **Issue**: jinja2 and PyYAML not pinned to specific versions - **Note**: Deferred - requires testing across version ranges - **Status**: โณ Future Enhancement diff --git a/docs/SYSTEM_COMPLETION_SUMMARY.md b/docs/SYSTEM_COMPLETION_SUMMARY.md index 109cf62..4944019 100644 --- a/docs/SYSTEM_COMPLETION_SUMMARY.md +++ b/docs/SYSTEM_COMPLETION_SUMMARY.md @@ -7,7 +7,7 @@ ### ๐Ÿ”ง **Core System Refactoring** - **Modular Architecture**: Refactored metadata workflow into separate, focused modules - - `mam_scraper.py` - Async Playwright-based MAM scraping + - `mam_api/` - MAM JSON API client, models, and metadata adapter - `audnex_metadata.py` - Comprehensive metadata cleaning and enrichment - `audible_scraper.py` - Audible fallback scraping - `metadata_coordinator.py` - Orchestrates the entire workflow @@ -17,7 +17,7 @@ - **Global Queue System**: Implemented `asyncio.Queue` for safe, sequential webhook processing - **Background Worker**: Persistent worker thread processes requests without blocking - **Rate Limiting**: Global rate limiting across all metadata sources -- **Playwright Fix**: Converted all MAM scraping to async to resolve API conflicts +- **MAM API Client**: Uses MAM's JSON API through httpx instead of browser automation ### ๐Ÿ“Š **Monitoring & Health** @@ -62,7 +62,7 @@ - `src/main.py` - FastAPI app with queue system and endpoints - `src/metadata_coordinator.py` - Async metadata orchestration -- `src/mam_scraper.py` - Async MAM scraping +- `src/mam_api/` - MAM JSON API integration - `src/audnex_metadata.py` - Comprehensive metadata cleaning - `src/utils.py` - Enhanced notification field extraction diff --git a/docs/development/PLAYWRIGHT_ASYNC_FIX_SUMMARY.md b/docs/development/PLAYWRIGHT_ASYNC_FIX_SUMMARY.md deleted file mode 100644 index 63b02f2..0000000 --- a/docs/development/PLAYWRIGHT_ASYNC_FIX_SUMMARY.md +++ /dev/null @@ -1,131 +0,0 @@ -# Playwright Async/Sync API Fix Summary - -## Problem Identified - -When the main FastAPI application tried to use the MAM scraper, it encountered a Playwright error: - -``` -Error scraping MAM: It looks like you are using Playwright Sync API inside the asyncio loop. -Please use the Async API instead. -``` - -This happened because: - -1. FastAPI webhook handlers run in an async context -2. The MAM scraper was using `sync_playwright()` (synchronous API) -3. Playwright doesn't allow mixing sync and async APIs in the same event loop - -## Solution Implemented - -### 1. Converted MAM Scraper to Async - -**File**: `src/mam_scraper.py` - -- Changed import from `playwright.sync_api` to `playwright.async_api` -- Made `scrape_asin_from_url()` async -- Made `login_and_get_cookies()` async -- Made `_scrape_with_cookies()` async -- Updated all Playwright API calls to use `await` -- Updated main function to use `asyncio.run()` - -### 2. Updated Metadata Coordinator - -**File**: `src/metadata_coordinator.py` - -- Made `get_metadata_from_webhook()` async -- Added `await` for MAM scraper call -- Updated main function to use `asyncio.run()` - -### 3. Updated Main Application - -**File**: `src/main.py` - -- Added `await` for coordinator call in webhook handler -- Webhook handler was already async, so this was compatible - -## Key Changes Made - -### MAM Scraper (`src/mam_scraper.py`) - -```python -# Before -from playwright.sync_api import sync_playwright -def scrape_asin_from_url(self, url: str, force_login: bool = False) -> Optional[str]: - with sync_playwright() as p: - # sync operations - -# After -from playwright.async_api import async_playwright -async def scrape_asin_from_url(self, url: str, force_login: bool = False) -> Optional[str]: - async with async_playwright() as p: - # await async operations -``` - -### Metadata Coordinator (`src/metadata_coordinator.py`) - -```python -# Before -def get_metadata_from_webhook(self, webhook_payload: Dict[str, Any]) -> Optional[Dict[str, Any]]: - asin = self.mam_scraper.scrape_asin_from_url(url) - -# After -async def get_metadata_from_webhook(self, webhook_payload: Dict[str, Any]) -> Optional[Dict[str, Any]]: - asin = await self.mam_scraper.scrape_asin_from_url(url) -``` - -### Main Application (`src/main.py`) - -```python -# Before -metadata = coordinator.get_metadata_from_webhook(payload) - -# After -metadata = await coordinator.get_metadata_from_webhook(payload) -``` - -## Results Verified - -### โœ… Async Playwright Working - -``` -2025-06-17 22:15:17,987 - INFO - ASIN successfully extracted: B0F8PKCTCW -``` - -- No more "Sync API inside asyncio loop" errors -- MAM scraping works from FastAPI webhook handlers -- Comprehensive metadata pipeline fully functional - -### โœ… Comprehensive Metadata Still Working - -``` -Title: In Another World with My Smartphone: Volume 6 -Author: Patora Fuyuhara -Narrator: Gabriel Michael, Ashely Biski -Publisher: Podium Audio -Duration: 395 minutes -Series: In Another World With My Smartphone Series #6 -``` - -- All metadata fields preserved -- Notifications have rich content -- Backend has comprehensive book information - -### โœ… Workflow Performance - -- MAM ASIN extraction: โœ… Working async -- Audnex metadata: โœ… Working -- Rate limiting: โœ… Working (30s test mode) -- Enhanced metadata: โœ… Working (chapters, etc.) - -## Impact - -**Before**: Playwright sync/async conflict prevented MAM scraping in production -**After**: Full async pipeline works seamlessly in FastAPI application - -The webhook processing now works end-to-end: - -1. Webhook received by FastAPI (async) -2. MAM ASIN extraction (async Playwright) -3. Audnex metadata fetching (requests) -4. Comprehensive metadata with 50+ fields -5. Rich notifications with all book details diff --git a/docs/development/contributing.md b/docs/development/contributing.md index 7e58ecf..ceff50f 100644 --- a/docs/development/contributing.md +++ b/docs/development/contributing.md @@ -78,7 +78,7 @@ python tests/test_metadata_workflow.py python tests/test_mam_integration.py # Run specific tests -python test_mam_login.py +pytest tests/test_mam_api.py -k Integration --no-cov python test_audnex_direct.py ``` @@ -206,7 +206,7 @@ python tests/test_metadata_workflow.py python tests/test_mam_integration.py # Run development/debug tests -python test_mam_login.py +pytest tests/test_mam_api.py -k Integration --no-cov python test_audnex_direct.py # Run with pytest (if configured) diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 011e16a..068c35f 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -6,12 +6,10 @@ This guide covers all configuration options for the Audiobook Automation System. All configuration files are located in the `config/` directory: -``` +```text config/ โ”œโ”€โ”€ config.yaml # Main application configuration -โ”œโ”€โ”€ config.yaml.example # Template for main config -โ”œโ”€โ”€ mam_config.json # MAM credentials (optional) -โ””โ”€โ”€ mam_config.json.example # Template for MAM config +โ””โ”€โ”€ config.yaml.example # Template for main config ``` ## ๐Ÿ”ง Main Configuration (`config.yaml`) @@ -93,40 +91,22 @@ PUSHOVER_API_TOKEN=your-pushover-api-token GOTIFY_URL=https://gotify.example.com GOTIFY_TOKEN=your-gotify-token NTFY_URL=https://ntfy.sh/your-topic + +# MAM API auth (optional, required for MAM metadata lookups) +MAM_ID=your-mam-session-cookie-value ``` -## ๐Ÿ” MAM Configuration (Optional) +## ๐Ÿ” MAM API Configuration (Optional) -For full MAM integration with ASIN extraction: +For full MAM integration with ASIN extraction, set `MAM_ID` in `.env` to the value of your MAM `mam_id` browser cookie. The application uses MAM's JSON API directly and does not log in through the website. -### 1. Setup MAM Config +### Find the Cookie Value ```bash -# Copy the example -cp config/mam_config.json.example config/mam_config.json - -# Or use the setup helper -python setup_mam_config.py -``` - -### 2. Configure MAM Credentials - -Edit `config/mam_config.json`: - -```json -{ - "email": "your-mam-email@example.com", - "password": "your-mam-password", - "base_url": "https://www.myanonamouse.net", - "timeout_seconds": 30, - "browser_settings": { - "headless": true, - "user_agent": "Mozilla/5.0 (compatible; AudiobookBot/1.0)" - } -} +MAM_ID=your-mam-session-cookie-value ``` -โš ๏ธ **Security Note**: `mam_config.json` is excluded from git for security. +Security note: `MAM_ID` is a session token. Keep it only in `.env`, never commit it, and rotate it if it is shared or exposed. ## ๐ŸŽฏ Configuration Examples @@ -159,8 +139,8 @@ Test your configuration: # Test main config python -c "from src.config import load_config; print('โœ… Config valid')" -# Test MAM config (if configured) -python test_mam_login.py +# Test MAM API auth (if configured) +pytest tests/test_mam_api.py -k Integration --no-cov # Test metadata workflow python tests/test_metadata_workflow.py @@ -176,11 +156,11 @@ python tests/test_metadata_workflow.py cp config/config.yaml.example config/config.yaml ``` -**MAM login fails:** +**MAM API auth fails:** -- Verify credentials in `config/mam_config.json` -- Check if MAM requires 2FA (not currently supported) -- Test login manually on MAM website +- Verify `MAM_ID` in `.env` is the current `mam_id` cookie value +- Log in to MAM in your browser and refresh the cookie value if the API reports authentication failure +- Make sure the value is not URL-encoded twice or surrounded by quotes **Rate limiting too slow:** @@ -196,7 +176,7 @@ cp config/config.yaml.example config/config.yaml - [ ] `config/config.yaml` created and configured - [ ] `.env` file created with required tokens -- [ ] `config/mam_config.json` created (if using MAM) +- [ ] `MAM_ID` set in `.env` (if using MAM) - [ ] Configuration validated with test scripts - [ ] Notification services tested (if enabled) - [ ] Rate limiting configured appropriately diff --git a/docs/user-guide/troubleshooting.md b/docs/user-guide/troubleshooting.md index 5e7a326..d31e5b4 100644 --- a/docs/user-guide/troubleshooting.md +++ b/docs/user-guide/troubleshooting.md @@ -134,33 +134,24 @@ curl http://localhost:8080 ## ๐Ÿ“Š Metadata Workflow Issues -### MAM Login Failed +### MAM API Auth Failed -**Error:** `MAM login failed` or `Not logged in` +**Error:** `MAM API authentication failed; update MAM_ID` **Solutions:** -1. **Check credentials:** +1. **Check API cookie:** ```bash -# Verify MAM config exists -ls -la config/mam_config.json +# Confirm MAM_ID is present without printing the value +test -n "$MAM_ID" && echo "MAM_ID is set" -# Test MAM login -python test_mam_login.py +pytest tests/test_mam_api.py -k Integration --no-cov ``` -2. **Update credentials:** +2. **Refresh the cookie value:** -```bash -# Recreate MAM config -python setup_mam_config.py -``` - -3. **Check for MAM issues:** - - Verify account is active - - Check if 2FA is enabled (not supported) - - Try logging in manually on MAM website +Verify the account is active, log in to MAM in your browser, copy the current `mam_id` cookie value into `.env` as `MAM_ID`, and restart the app so the environment reloads. ### ASIN Not Found @@ -408,8 +399,8 @@ tail -f logs/audiobook_requests.log # Metadata workflow tail -f logs/metadata_coordinator.log -# MAM scraper -tail -f logs/mam_scraper.log +# MAM API client +tail -f logs/audiobook_requests.log # Notifications tail -f logs/notifications.log @@ -447,7 +438,7 @@ python --version uname -a # Package versions -pip freeze | grep -E "(flask|requests|playwright)" +pip freeze | grep -E "(fastapi|httpx|pydantic)" ``` 2. **Configuration (sanitized):** diff --git a/pyproject.toml b/pyproject.toml index e887155..27b8494 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,6 +193,9 @@ module = "tests.*" disallow_untyped_defs = false check_untyped_defs = false +[tool.bandit] +exclude_dirs = ["tests"] + [tool.black] line-length = 120 target-version = ['py311', 'py312', 'py313'] diff --git a/requirements.txt b/requirements.txt index 916a39a..be9d103 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,6 @@ itsdangerous idna cryptography PyYAML -playwright # Structured logging structlog>=24.4.0 diff --git a/src/archive/mam_scraper.py.old b/src/archive/mam_scraper.py.old deleted file mode 100644 index 68da70b..0000000 --- a/src/archive/mam_scraper.py.old +++ /dev/null @@ -1,414 +0,0 @@ -""" -MyAnonamouse.net ASIN scraper -Extracts ASIN from MAM torrent pages using Playwright -""" - -import argparse -import asyncio -import json -import logging -import os -import re -import sys -import time -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, Optional - -from playwright.async_api import async_playwright - -# Add parent directory to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent)) -from src.config import load_config - -# Ensure logs directory exists before configuring FileHandler -_log_dir = Path("logs") -_log_dir.mkdir(parents=True, exist_ok=True) - -# Configure logging with fallback if FileHandler fails -_handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] -try: - _handlers.append(logging.FileHandler(_log_dir / "mam_scraper.log")) -except Exception: - pass # Fall back to stdout only - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=_handlers -) - - -class MAMScraper: - def __init__(self) -> None: - self.config = load_config() - self.mam_config = self.config.get('metadata', {}).get('mam', {}) - self.config_file = self.mam_config.get('config_file', 'mam_config.json') - self.base_url = self.mam_config.get('base_url', 'https://www.myanonamouse.net') - self.login_url = self.mam_config.get('login_url', 'https://www.myanonamouse.net/loggedin.php') - self.global_rate_limit = self.config.get('metadata', {}).get('rate_limit_seconds', 120) - self.last_global_request_time = 0 - - async def _check_global_rate_limit(self) -> None: - """Check if we need to wait for global rate limit.""" - current_time = time.time() - time_since_last_global = current_time - self.last_global_request_time - - if time_since_last_global < self.global_rate_limit: - wait_time = self.global_rate_limit - time_since_last_global - logging.info(f"Global rate limit: waiting {wait_time:.1f} seconds...") - await asyncio.sleep(wait_time) - - self.last_global_request_time = time.time() - - def load_mam_config(self) -> Dict[str, Any]: - """Load MAM-specific configuration from JSON file.""" - try: - with open(self.config_file, 'r') as f: - config = json.load(f) - logging.info("MAM configuration loaded successfully") - return config - except FileNotFoundError: - logging.error(f"MAM config file {self.config_file} not found") - raise - except json.JSONDecodeError as e: - logging.error(f"Invalid JSON in MAM config file: {e}") - raise - - def save_mam_config(self, config: Dict[str, Any]) -> None: - """Save MAM configuration to JSON file.""" - try: - with open(self.config_file, 'w') as f: - json.dump(config, f, indent=2) - logging.info("MAM configuration saved successfully") - except Exception as e: - logging.error(f"Failed to save MAM config: {e}") - raise - - async def login_and_get_cookies(self, email: str, password: str) -> Dict[str, str]: - """Login to MAM and retrieve session cookies.""" - # Validate credentials before launching browser - if not email or not password: - logging.warning("Login attempted with missing credentials") - raise ValueError("Email and password must be provided for login") - - logging.info("Starting MAM login process...") - - async with async_playwright() as p: - browser = await p.chromium.launch(headless=True) - context = await browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" - ) - - try: - page = await context.new_page() - - # Go to login page - logging.info(f"Navigating to login page: {self.login_url}") - await page.goto(self.login_url) - await page.wait_for_load_state("networkidle") - - # Check if we're already on the login form - if await page.locator("input[name='email']").count() == 0: - # Try going to the main page - logging.info(f"Trying main page: {self.base_url}") - await page.goto(self.base_url) - await page.wait_for_load_state("networkidle") - - # Look for login form - email_input = page.locator("input[name='email'], input[type='email'], input[name='username']") - password_input = page.locator("input[name='password'], input[type='password']") - - if await email_input.count() == 0: - logging.error("Could not find email input field") - # Ensure logs directory exists before writing debug file - os.makedirs('logs', exist_ok=True) - with open('logs/login_page_debug.html', 'w', encoding='utf-8') as f: - f.write(await page.content()) - raise Exception("Login form not found") - - logging.info("Found login form, filling credentials...") - - # Fill in credentials - await email_input.fill(email) - await password_input.fill(password) - - # Look for "Keep me logged in" checkbox - keep_logged_in_selectors = [ - "input[type='checkbox']", - "input[name='autolog']", - "input[name='remember']", - "input[name='stay_logged_in']" - ] - - for selector in keep_logged_in_selectors: - checkbox = page.locator(selector) - if await checkbox.count() > 0: - logging.info(f"Found keep-logged-in checkbox with selector: {selector}") - await checkbox.check() - break - - # Submit the form - submit_button = page.locator("input[type='submit'], button[type='submit']").first - - if await submit_button.count() > 0: - logging.info("Submitting login form...") - await submit_button.click() - else: - logging.info("No submit button found, trying Enter key") - await password_input.press("Enter") - - # Wait for navigation after login - await page.wait_for_load_state("networkidle", timeout=10000) - - # Check if login was successful - current_url = page.url - logging.info(f"After login, current URL: {current_url}") - - # Look for signs of successful login - if await page.locator("text=logout").count() > 0 or await page.locator("text=Logout").count() > 0: - logging.info("โœ… Login successful! Found logout link") - elif "login" in current_url.lower(): - logging.error("โŒ Login failed - still on login page") - raise Exception("Login failed - check credentials") - else: - logging.info("Login appears successful (URL changed)") - - # Get cookies - cookies = await context.cookies() - logging.info(f"Retrieved {len(cookies)} cookies") - - # Extract session cookies - session_cookies = {} - for cookie in cookies: - cookie_name = cookie.get('name', '') - cookie_value = cookie.get('value', '') - if cookie_name in ['mam_id', 'session', 'PHPSESSID'] or 'session' in cookie_name.lower(): - session_cookies[cookie_name] = cookie_value - logging.info(f"Found session cookie: {cookie_name}") - - if not session_cookies: - logging.warning("No obvious session cookies found, saving all cookies") - session_cookies = {cookie.get('name', ''): cookie.get('value', '') for cookie in cookies} - - return session_cookies - - except Exception as e: - logging.error(f"Login process failed: {e}") - raise - finally: - await browser.close() - - def extract_asin_from_page(self, page_source: str) -> Optional[str]: - """Extract ASIN from page source.""" - logging.info("Starting ASIN extraction...") - logging.info(f"Page source length: {len(page_source)} characters") - - # Primary pattern: ASIN:B0XXXXXXXX - pattern = r'ASIN:([A-Z0-9]{10})' - match = re.search(pattern, page_source) - if match: - asin = match.group(1) - logging.info(f"ASIN successfully extracted: {asin}") - return asin - - # Alternative patterns - alternative_patterns = [ - r'asin[:\s=]+([A-Z0-9]{10})', # Case insensitive - r'ASIN[:\s=]+([A-Z0-9]{10})', # Different separators - r'B0[A-Z0-9]{8}', # Direct B0 pattern - ] - - for i, alt_pattern in enumerate(alternative_patterns): - logging.info(f"Trying alternative pattern {i+1}: {alt_pattern}") - alt_match = re.search(alt_pattern, page_source, re.IGNORECASE) - if alt_match: - asin = alt_match.group(1) if alt_match.groups() else alt_match.group(0) - logging.info(f"ASIN found with alternative pattern: {asin}") - return asin - - logging.warning("No ASIN found in page source") - return None - - async def scrape_asin_from_url(self, url: str, force_login: bool = False) -> Optional[str]: - """Main method to scrape ASIN from MAM URL.""" - logging.info(f"Starting ASIN scraping for URL: {url}") - - # Load MAM config - try: - mam_config = self.load_mam_config() - except Exception as e: - logging.error(f"Failed to load MAM config: {e}") - return None - - # Check if we need to login - cookies = mam_config.get('cookies', {}) - if not cookies or not cookies.get('mam_id') or force_login: - logging.info("No valid cookies found or forced login, logging in...") - - email = mam_config.get('email') - password = mam_config.get('password') - - if not email or not password: - logging.error("Email or password not found in MAM config file") - return None - - try: - cookies = await self.login_and_get_cookies(email, password) - mam_config['cookies'] = cookies - mam_config['last_login'] = datetime.now().isoformat() - self.save_mam_config(mam_config) - logging.info("Login successful, cookies saved") - except Exception as e: - logging.error(f"Login failed: {e}") - return None - - # Try to scrape with current cookies - result = await self._scrape_with_cookies(url, cookies) - - # If we got a login-required result and haven't already tried to re-login, attempt auto re-login - if result is None and not force_login: - result = await self._auto_relogin_and_retry(url, mam_config, cookies) - - return result - - async def _auto_relogin_and_retry( - self, url: str, mam_config: Dict[str, Any], cookies: Dict[str, str] - ) -> Optional[str]: - """Attempt automatic re-login and retry scraping when cookies are expired. - - Args: - url: The MAM URL to scrape - mam_config: The MAM configuration dict - cookies: The current (possibly expired) cookies - - Returns: - The ASIN if found after re-login, None otherwise - """ - logging.info("Scraping failed, possibly due to expired cookies. Attempting auto re-login...") - - email = mam_config.get('email') - password = mam_config.get('password') - - if not email or not password: - logging.error("Cannot perform auto re-login: missing email or password in config") - return None - - try: - logging.info("Performing automatic re-login to refresh cookies...") - fresh_cookies = await self.login_and_get_cookies(email, password) - mam_config['cookies'] = fresh_cookies - mam_config['last_login'] = datetime.now().isoformat() - self.save_mam_config(mam_config) - logging.info("Auto re-login successful, retrying scrape with fresh cookies...") - - # Retry with fresh cookies - result = await self._scrape_with_cookies(url, fresh_cookies) - if result: - logging.info("โœ… Auto re-login and retry successful!") - else: - logging.warning("โŒ Retry after auto re-login still failed") - return result - except Exception as e: - logging.error(f"Auto re-login failed: {e}") - return None - - async def _scrape_with_cookies(self, url: str, cookies: Dict[str, str]) -> Optional[str]: - """Scrape the torrent page using cookies.""" - async with async_playwright() as p: - browser = None - try: - browser = await p.chromium.launch(headless=True) - context = await browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" - ) - - # Set cookies - cookie_list = [] - for name, value in cookies.items(): - cookie_list.append({ - 'name': name, - 'value': value, - 'domain': '.myanonamouse.net', - 'path': '/' - }) - - await context.add_cookies(cookie_list) - logging.info(f"Set {len(cookie_list)} cookies") - - page = await context.new_page() - logging.info(f"Navigating to {url}") - - # Apply global rate limiting (async) - await self._check_global_rate_limit() - - response = await page.goto(url, wait_until="domcontentloaded", timeout=30000) - - if response: - status_code = response.status - logging.info(f"Page loaded with status: {status_code}") - - if status_code != 200: - logging.warning(f"Unexpected status code: {status_code}") - else: - logging.warning("No response object received") - - # Wait for page to fully load - await page.wait_for_timeout(2000) - - # Check if we're logged in - if await page.locator("text=Login").count() > 0: - logging.error("โŒ Not logged in - login required") - return None - elif await page.locator("text=logout").count() > 0 or await page.locator("text=Logout").count() > 0: - logging.info("โœ… Successfully logged in") - else: - # Additional check for login indicators in page content - page_content = await page.content() - if "login" in page_content.lower() and "password" in page_content.lower(): - logging.error("โŒ Login form detected in page - not logged in") - return None - else: - logging.info("โœ… Login status unclear but proceeding (no obvious login form)") - - # Get page content - source = await page.content() - logging.info(f"Page content retrieved, length: {len(source)} characters") - - # Extract ASIN - asin = self.extract_asin_from_page(source) - return asin - - except Exception as e: - logging.error(f"Scraping error: {e}") - return None - finally: - if browser: - await browser.close() - - -def main() -> int | None: - """Main function for command line usage.""" - - async def async_main() -> str | None: - parser = argparse.ArgumentParser(description="MAM ASIN Scraper") - parser.add_argument("url", help="MAM torrent URL to scrape") - parser.add_argument("--force-login", action="store_true", help="Force re-login even if cookies exist") - args = parser.parse_args() - - scraper = MAMScraper() - asin = await scraper.scrape_asin_from_url(args.url, force_login=args.force_login) - - if asin: - print(f"โœ… ASIN found: {asin}") - return asin - else: - print("โŒ No ASIN found") - return None - - result = asyncio.run(async_main()) - return 0 if result else 1 - - -if __name__ == "__main__": - main() diff --git a/src/http_client.py b/src/http_client.py index f3655e2..aa52a2c 100644 --- a/src/http_client.py +++ b/src/http_client.py @@ -81,7 +81,7 @@ class HttpClientConfig: # Region configuration default_region: str = "us" - regions: list[str] = field(default_factory=lambda: DEFAULT_REGIONS.copy()) + regions: list[str] = field(default_factory=DEFAULT_REGIONS.copy) max_regions_to_try: int = 10 @classmethod diff --git a/src/logging_setup.py b/src/logging_setup.py index 48cfbd5..272e6e7 100644 --- a/src/logging_setup.py +++ b/src/logging_setup.py @@ -266,9 +266,9 @@ def _get_version() -> str: """Get application version (git sha or package version).""" # Try git first try: - import subprocess + import subprocess # nosec B404 - result = subprocess.run( + result = subprocess.run( # nosec B603, B607 ["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, @@ -279,10 +279,14 @@ def _get_version() -> str: if result.returncode == 0: return result.stdout.strip() except Exception: - # Git command failed or not in a git repo, silently fall through to package version - pass + return _get_package_version() + + return _get_package_version() + + +def _get_package_version() -> str: + """Get installed package version or development fallback.""" - # Fall back to package version try: from importlib.metadata import version diff --git a/src/main.py b/src/main.py index ee2098c..1f05fd7 100644 --- a/src/main.py +++ b/src/main.py @@ -816,7 +816,7 @@ async def rejection_css_test(): server_config = load_config().get("server", {}) uvicorn.run( "src.main:app", - host=server_config.get("host", "0.0.0.0"), + host=server_config.get("host", "0.0.0.0"), # nosec B104 port=server_config.get("port", 8000), reload=server_config.get("reload", True), ) diff --git a/src/mam_api/__init__.py b/src/mam_api/__init__.py index d8c2714..c28a3f3 100644 --- a/src/mam_api/__init__.py +++ b/src/mam_api/__init__.py @@ -3,8 +3,6 @@ This package provides a clean interface to MAM's JSON API using httpx for HTTP/2 support and Pydantic for response validation. - -The adapter module provides backward compatibility with the old MAMScraper interface. """ from src.mam_api.adapter import MAMApiAdapter @@ -17,12 +15,8 @@ ) -# Alias for backward compatibility with old MAMScraper code -MAMScraper = MAMApiAdapter - __all__ = [ "MAMApiAdapter", - "MAMScraper", "MamApiError", "MamAsyncClient", "MamClient", diff --git a/src/mam_api/adapter.py b/src/mam_api/adapter.py index ae9649e..5874689 100644 --- a/src/mam_api/adapter.py +++ b/src/mam_api/adapter.py @@ -1,45 +1,40 @@ -""" -MAM API Adapter - -Provides backward-compatible interface for existing code that used MAMScraper. -This adapter uses the new MAM JSON API instead of Playwright HTML scraping. - -Usage: - # Drop-in replacement for MAMScraper - from src.mam_api.adapter import MAMApiAdapter - - adapter = MAMApiAdapter() - asin = await adapter.scrape_asin_from_url(url) -""" +"""MAM API adapter for torrent metadata lookups.""" import asyncio import os import re import time from typing import Any -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, urlparse, urlunparse import httpx from pydantic import ValidationError from src.logging_setup import get_logger -from .client import MamApiError, MamAsyncClient +from .client import MAM_AUTH_ERROR_MESSAGE, MamApiError, MamAsyncClient from .models import MamTorrentRaw log = get_logger(__name__) +def _sanitize_url_for_log(url: str) -> str: + parsed = urlparse(url) + return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", "")) + + +def _is_mam_auth_error(exc: MamApiError) -> bool: + message = str(exc) + return MAM_AUTH_ERROR_MESSAGE in message or "MAM_ID not configured" in message + + class MAMApiAdapter: """ - Backward-compatible adapter that provides the same interface as MAMScraper - but uses the MAM JSON API instead of Playwright. + Adapter that retrieves MAM metadata through the JSON API. - Key differences from MAMScraper: - - No browser automation (faster, lighter) - - Uses mam_id from environment instead of mam_config.json - - Gets more data than just ASIN (author, narrator, series, etc.) + Auth uses the MAM_ID environment variable, which should contain the + mam_id session cookie value. """ def __init__(self, mam_id: str | None = None, rate_limit_seconds: float = 2.0) -> None: @@ -94,7 +89,7 @@ async def _check_rate_limit(self) -> None: self._last_api_call_time = time.time() @staticmethod - def extract_tid_from_url(url: str) -> int | None: + def extract_tid_from_url(url: str | None) -> int | None: """ Extract torrent ID (tid) from MAM URL. @@ -132,7 +127,8 @@ def extract_tid_from_url(url: str) -> int | None: except (ValueError, IndexError): pass - log.warning("mam.adapter.tid_extract_failed", url=url) + safe_url = _sanitize_url_for_log(url) + log.warning("mam.adapter.tid_extract_failed", url=safe_url) return None async def get_torrent_data(self, url: str) -> MamTorrentRaw | None: @@ -164,7 +160,11 @@ async def get_torrent_data(self, url: str) -> MamTorrentRaw | None: log.warning("mam.adapter.torrent_not_found", tid=tid) return None - except MamApiError: + except MamApiError as exc: + if _is_mam_auth_error(exc): + log.exception("mam.adapter.auth_error", tid=tid) + raise + log.exception("mam.adapter.api_error", tid=tid) return None except httpx.HTTPError: @@ -174,23 +174,16 @@ async def get_torrent_data(self, url: str) -> MamTorrentRaw | None: log.exception("mam.adapter.validation_error", tid=tid) return None - async def scrape_asin_from_url(self, url: str, force_login: bool = False) -> str | None: + async def get_asin_from_url(self, url: str) -> str | None: """ Get ASIN from MAM URL using the JSON API. - This method provides backward compatibility with MAMScraper.scrape_asin_from_url(). - The force_login parameter is ignored since we use cookie-based auth. - Args: url: MAM torrent URL - force_login: Ignored (kept for backward compatibility) Returns: ASIN string if found, None otherwise """ - if force_login: - log.debug("mam.adapter.force_login_ignored") - torrent = await self.get_torrent_data(url) if not torrent: return None @@ -209,7 +202,7 @@ async def get_full_metadata(self, url: str) -> dict[str, Any] | None: """ Get full metadata from MAM URL. - This is an enhanced method that returns more data than scrape_asin_from_url(). + This is an enhanced method that returns more data than get_asin_from_url(). Args: url: MAM torrent URL @@ -255,8 +248,3 @@ async def get_full_metadata(self, url: str) -> dict[str, Any] | None: "mam_id": normalized.tid, "source": "mam_api", } - - -# For backward compatibility, also expose as MAMScraper alias -# This allows gradual migration without changing all imports at once -MAMScraper = MAMApiAdapter diff --git a/src/mam_api/client.py b/src/mam_api/client.py index a510064..a0db796 100644 --- a/src/mam_api/client.py +++ b/src/mam_api/client.py @@ -25,6 +25,8 @@ MAM_BASE_URL = "https://www.myanonamouse.net" MAM_SEARCH_PATH = "/tor/js/loadSearchJSONbasic.php" MAM_DOWNLOAD_PATH = "/tor/download.php" +MAM_LOGIN_PATHS = ("/login.php", "/loggedin.php") +MAM_AUTH_ERROR_MESSAGE = "MAM API authentication failed; update MAM_ID" # Regex to extract tid from MAM URLs like /t/1207719 _TID_RE = re.compile(r"(?:https?://www\.myanonamouse\.net)?/t/(\d+)") @@ -48,6 +50,45 @@ def extract_tid_from_irc(line: str) -> int | None: return None +def _url_path_looks_like_login(url: str | None) -> bool: + if not url: + return False + + path = str(httpx.URL(url).path) + return path.endswith(MAM_LOGIN_PATHS) + + +def _raise_for_api_response(response: httpx.Response) -> None: + if response.status_code in (401, 403): + raise MamApiError(MAM_AUTH_ERROR_MESSAGE) + + if response.is_redirect and _url_path_looks_like_login(response.headers.get("location")): + raise MamApiError(MAM_AUTH_ERROR_MESSAGE) + + if _url_path_looks_like_login(str(response.url)): + raise MamApiError(MAM_AUTH_ERROR_MESSAGE) + + response.raise_for_status() + + +def _api_response_json(response: httpx.Response) -> Any: + _raise_for_api_response(response) + + try: + return response.json() + except ValueError as exc: + content_type = response.headers.get("content-type", "") + if "html" in content_type.lower(): + raise MamApiError(MAM_AUTH_ERROR_MESSAGE) from exc + raise MamApiError("MAM API returned invalid JSON") from exc + + +def _validated_torrent_content(content: bytes) -> bytes: + if not content.startswith(b"d") or b"4:info" not in content: + raise MamApiError("MAM download did not return a valid .torrent file") + return content + + class MamApiError(RuntimeError): """Error from MAM API operations.""" @@ -82,7 +123,7 @@ def __init__( timeout=httpx.Timeout(timeout), headers={"User-Agent": user_agent}, cookies={"mam_id": mam_id}, - follow_redirects=True, + follow_redirects=False, ) # Log initialization without exposing cookie value log.debug("mam.client.init", base_url=base_url, http2=http2) @@ -149,9 +190,7 @@ def search( ) r = self._client.post(MAM_SEARCH_PATH, json=payload, params={"perpage": str(perpage)}) - r.raise_for_status() - - data = r.json() + data = _api_response_json(r) response = MamSearchResponseRaw.model_validate(data) log.debug("mam.search.response", results=len(response.data), found=response.found) @@ -224,10 +263,11 @@ def download_torrent_by_tid(self, tid: int) -> bytes: Raw bytes of the .torrent file """ log.info("mam.download.tid", tid=tid) - r = self._client.get(MAM_DOWNLOAD_PATH, params={"tid": str(tid)}) - r.raise_for_status() - log.debug("mam.download.complete", tid=tid, size=len(r.content)) - return r.content # type: ignore[no-any-return] + r = self._client.get(MAM_DOWNLOAD_PATH, params={"tid": str(tid)}, follow_redirects=True) + _raise_for_api_response(r) + content = _validated_torrent_content(r.content) + log.debug("mam.download.complete", tid=tid, size=len(content)) + return content def download_torrent_by_dl(self, dl_token: str) -> bytes: """ @@ -244,10 +284,11 @@ def download_torrent_by_dl(self, dl_token: str) -> bytes: path = f"{MAM_DOWNLOAD_PATH}/{dl_token}" log.info("mam.download.dl_token") - r = self._client.get(path) - r.raise_for_status() - log.debug("mam.download.dl_complete", size=len(r.content)) - return r.content # type: ignore[no-any-return] + r = self._client.get(path, follow_redirects=True) + _raise_for_api_response(r) + content = _validated_torrent_content(r.content) + log.debug("mam.download.dl_complete", size=len(content)) + return content class MamAsyncClient: @@ -278,7 +319,7 @@ def __init__( timeout=httpx.Timeout(timeout), headers={"User-Agent": user_agent}, cookies={"mam_id": mam_id}, - follow_redirects=True, + follow_redirects=False, ) log.debug("mam.async_client.init", base_url=base_url, http2=http2) @@ -345,9 +386,8 @@ async def search( ) r = await self._client.post(MAM_SEARCH_PATH, json=payload, params={"perpage": str(perpage)}) - r.raise_for_status() - response = MamSearchResponseRaw.model_validate(r.json()) + response = MamSearchResponseRaw.model_validate(_api_response_json(r)) log.debug("mam.async_search.response", results=len(response.data)) return response # type: ignore[no-any-return] @@ -397,10 +437,11 @@ async def get_torrent( async def download_torrent_by_tid(self, tid: int) -> bytes: """Download .torrent file using session cookie (async).""" log.info("mam.async_download.tid", tid=tid) - r = await self._client.get(MAM_DOWNLOAD_PATH, params={"tid": str(tid)}) - r.raise_for_status() - log.debug("mam.async_download.complete", tid=tid, size=len(r.content)) - return r.content # type: ignore[no-any-return] + r = await self._client.get(MAM_DOWNLOAD_PATH, params={"tid": str(tid)}, follow_redirects=True) + _raise_for_api_response(r) + content = _validated_torrent_content(r.content) + log.debug("mam.async_download.complete", tid=tid, size=len(content)) + return content async def download_torrent_by_dl(self, dl_token: str) -> bytes: """Download .torrent using dl token (async).""" @@ -409,7 +450,8 @@ async def download_torrent_by_dl(self, dl_token: str) -> bytes: path = f"{MAM_DOWNLOAD_PATH}/{dl_token}" log.info("mam.async_download.dl_token") - r = await self._client.get(path) - r.raise_for_status() - log.debug("mam.async_download.dl_complete", size=len(r.content)) - return r.content # type: ignore[no-any-return] + r = await self._client.get(path, follow_redirects=True) + _raise_for_api_response(r) + content = _validated_torrent_content(r.content) + log.debug("mam.async_download.dl_complete", size=len(content)) + return content diff --git a/src/metadata_coordinator.py b/src/metadata_coordinator.py index 40c0b1f..741cf4d 100644 --- a/src/metadata_coordinator.py +++ b/src/metadata_coordinator.py @@ -22,7 +22,7 @@ from src.audnex_metadata import AudnexMetadata from src.config import load_config from src.logging_setup import get_logger -from src.mam_api import MAMApiAdapter # New API-based adapter (was MAMScraper) +from src.mam_api import MAMApiAdapter, MamApiError log = get_logger(__name__) @@ -31,7 +31,7 @@ class MetadataCoordinator: def __init__(self): self.config = load_config() - self.mam_scraper = MAMApiAdapter() # New API-based adapter + self.mam_adapter = MAMApiAdapter() self.audnex = AudnexMetadata() self.audible = AudibleScraper() @@ -62,11 +62,14 @@ async def get_metadata_from_webhook(self, webhook_payload: dict[str, Any]) -> di if url and "myanonamouse.net" in url: log.info("coordinator.step1.mam_extract") try: - asin = await self.mam_scraper.scrape_asin_from_url(url) + asin = await self.mam_adapter.get_asin_from_url(url) if asin: log.info("coordinator.step1.asin_found", asin=asin) else: log.warning("coordinator.step1.no_asin", reason="mam_torrent_has_no_asin") + except MamApiError: + log.exception("coordinator.step1.mam_auth_error") + raise except httpx.RequestError: log.exception("coordinator.step1.network_error") except ValueError: diff --git a/src/qbittorrent.py b/src/qbittorrent.py index 9f987b7..628cea4 100644 --- a/src/qbittorrent.py +++ b/src/qbittorrent.py @@ -80,14 +80,9 @@ def from_env(cls) -> "QBittorrentConfig": password = os.getenv("QBITTORRENT_PASSWORD") verify = os.getenv("QBITTORRENT_VERIFY_SSL", "true").lower() == "true" - if not all([host, username, password]): + if not host or not username or not password: raise ValueError("QBITTORRENT_URL, QBITTORRENT_USERNAME, and QBITTORRENT_PASSWORD must be set") - # Type narrowing: all() check above ensures these are not None - assert isinstance(host, str), f"host must be a str, got {type(host)}" - assert isinstance(username, str), f"username must be a str, got {type(username)}" - assert isinstance(password, str), f"password must be a str, got {type(password)}" - return cls( host=host, username=username, @@ -215,8 +210,8 @@ def find_info_bounds(data: bytes) -> tuple[int, int] | None: info_start, info_end = bounds info_bytes = torrent_data[info_start:info_end] - # SHA1 hash of the info dict is the torrent's info hash - return hashlib.sha1(info_bytes).hexdigest().lower() + # SHA1 is required by the BitTorrent v1 info-hash specification. + return hashlib.sha1(info_bytes, usedforsecurity=False).hexdigest().lower() except (ValueError, IndexError): return None @@ -315,21 +310,23 @@ def client(self) -> Client: if self._config is None: self.configure() - assert self._config is not None + config = self._config + if config is None: + raise QBittorrentConnectionError("qBittorrent configuration unavailable") log.debug( "qbittorrent.client.init", - host=self._config.host, - username=self._config.username, + host=config.host, + username=config.username, ) try: self._client = Client( - host=self._config.host, - username=self._config.username, - password=self._config.password, - VERIFY_WEBUI_CERTIFICATE=self._config.verify_certificate, - REQUESTS_ARGS={"timeout": self._config.timeout}, + host=config.host, + username=config.username, + password=config.password, + VERIFY_WEBUI_CERTIFICATE=config.verify_certificate, + REQUESTS_ARGS={"timeout": config.timeout}, DISABLE_LOGGING_DEBUG_OUTPUT=True, ) # Verify connection works by fetching version diff --git a/src/utils.py b/src/utils.py index d0e3ed2..319e43b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -27,7 +27,7 @@ def format_release_date(date_str: str) -> str: if not date_str: return "" if "T" in date_str: - return date_str.split("T")[0] + return date_str.split("T", maxsplit=1)[0] return date_str diff --git a/tests/test_audnex_direct.py b/tests/test_audnex_direct.py index b641ebc..a62a28d 100644 --- a/tests/test_audnex_direct.py +++ b/tests/test_audnex_direct.py @@ -180,9 +180,9 @@ def main(): logging.info("") logging.info("๐Ÿ“ To get the full workflow working:") - logging.info("1. Run: python setup_mam_config.py") - logging.info("2. Add your MAM credentials") - logging.info("3. Test with: python test_mam_asin.py") + logging.info("1. Set MAM_ID in .env from your mam_id browser cookie") + logging.info("2. Restart the app or shell so the environment reloads") + logging.info("3. Test with: pytest tests/test_mam_api.py -k Integration --no-cov") if __name__ == "__main__": diff --git a/tests/test_mam_api.py b/tests/test_mam_api.py index 4ea9f0f..2858347 100644 --- a/tests/test_mam_api.py +++ b/tests/test_mam_api.py @@ -5,7 +5,7 @@ - Pydantic model parsing (including JSON-inside-string fields) - URL parsing and torrent ID extraction - API client functionality (mocked) -- Adapter backward compatibility +- API adapter behavior """ import inspect @@ -18,7 +18,7 @@ from pydantic import ValidationError from src.mam_api.adapter import MAMApiAdapter -from src.mam_api.client import MamAsyncClient, MamClient, extract_tid_from_irc +from src.mam_api.client import MamApiError, MamAsyncClient, MamClient, extract_tid_from_irc from src.mam_api.models import ( MamMediaInfo, MamSearchResponseRaw, @@ -31,6 +31,9 @@ ) +TORRENT_BYTES = b"d8:announce13:http://test4:infod4:name4:teste" + + # ============================================================================= # Test Data / Fixtures # ============================================================================= @@ -361,6 +364,7 @@ def test_client_initialization(self): # Client init should succeed with valid mam_id (no network call during init) client = MamClient(mam_id="test_id") assert client._client is not None + assert client._client.follow_redirects is False client.close() def test_client_no_mam_id(self): @@ -368,6 +372,60 @@ def test_client_no_mam_id(self): with pytest.raises(ValueError, match="mam_id"): MamClient(mam_id="") # Empty string is falsy + def test_search_login_redirect_raises_auth_error(self): + """Test stale mam_id redirects are treated as API auth failures.""" + client = MamClient(mam_id="test_id") + request = httpx.Request("POST", "https://www.myanonamouse.net/tor/js/loadSearchJSONbasic.php") + response = httpx.Response(302, headers={"Location": "/login.php"}, request=request) + + with patch.object(client._client, "post", return_value=response): + with pytest.raises(MamApiError, match="MAM API authentication failed"): + client.search(tor={"text": "test"}, perpage=5) + + client.close() + + def test_search_html_response_raises_auth_error(self): + """Test login HTML from the API endpoint is reported as auth failure.""" + client = MamClient(mam_id="test_id") + request = httpx.Request("POST", "https://www.myanonamouse.net/tor/js/loadSearchJSONbasic.php") + response = httpx.Response( + 200, + headers={"Content-Type": "text/html; charset=utf-8"}, + content=b"
", + request=request, + ) + + with patch.object(client._client, "post", return_value=response): + with pytest.raises(MamApiError, match="MAM API authentication failed"): + client.search(tor={"text": "test"}, perpage=5) + + client.close() + + def test_download_torrent_by_tid_follows_redirects_and_validates(self): + """Test tid download follows valid redirects and returns torrent bytes.""" + client = MamClient(mam_id="test_id") + request = httpx.Request("GET", "https://www.myanonamouse.net/tor/download.php?tid=123") + response = httpx.Response(200, content=TORRENT_BYTES, request=request) + + with patch.object(client._client, "get", return_value=response) as mock_get: + content = client.download_torrent_by_tid(123) + + assert content == TORRENT_BYTES + mock_get.assert_called_once_with("/tor/download.php", params={"tid": "123"}, follow_redirects=True) + client.close() + + def test_download_torrent_by_dl_rejects_html_response(self): + """Test dl-token download rejects non-torrent content.""" + client = MamClient(mam_id="test_id") + request = httpx.Request("GET", "https://www.myanonamouse.net/tor/download.php/token") + response = httpx.Response(200, content=b"not a torrent", request=request) + + with patch.object(client._client, "get", return_value=response): + with pytest.raises(MamApiError, match=r"valid \.torrent"): + client.download_torrent_by_dl("token") + + client.close() + class TestMamAsyncClient: """Test async MAM client.""" @@ -378,6 +436,21 @@ async def test_async_client_initialization(self): # Async client init should succeed with valid mam_id (no network call during init) client = MamAsyncClient(mam_id="test_id") assert client._client is not None + assert client._client.follow_redirects is False + await client.aclose() + + @pytest.mark.asyncio + async def test_async_search_login_redirect_raises_auth_error(self): + """Test async client treats login redirects as auth failures.""" + client = MamAsyncClient(mam_id="test_id") + request = httpx.Request("POST", "https://www.myanonamouse.net/tor/js/loadSearchJSONbasic.php") + response = httpx.Response(302, headers={"Location": "/login.php"}, request=request) + + with patch.object(client._client, "post", new_callable=AsyncMock) as mock_post: + mock_post.return_value = response + with pytest.raises(MamApiError, match="MAM API authentication failed"): + await client.search(tor={"text": "test"}, perpage=5) + await client.aclose() @pytest.mark.asyncio @@ -394,6 +467,21 @@ async def test_search_returns_response(self): await client.aclose() + @pytest.mark.asyncio + async def test_async_download_torrent_by_tid_follows_redirects_and_validates(self): + """Test async tid download follows valid redirects and validates content.""" + client = MamAsyncClient(mam_id="test_id") + request = httpx.Request("GET", "https://www.myanonamouse.net/tor/download.php?tid=123") + response = httpx.Response(200, content=TORRENT_BYTES, request=request) + + with patch.object(client._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = response + content = await client.download_torrent_by_tid(123) + + assert content == TORRENT_BYTES + mock_get.assert_called_once_with("/tor/download.php", params={"tid": "123"}, follow_redirects=True) + await client.aclose() + # ============================================================================= # Adapter Tests @@ -401,7 +489,7 @@ async def test_search_returns_response(self): class TestMAMApiAdapter: - """Test backward-compatible adapter.""" + """Test MAM API adapter.""" def test_extract_tid_from_url_t_format(self): """Test extracting tid from /t/ URL format.""" @@ -427,6 +515,18 @@ def test_extract_tid_from_url_invalid(self): tid = MAMApiAdapter.extract_tid_from_url(url) assert tid is None + def test_extract_tid_from_url_warning_strips_query_and_fragment(self): + """Test failed tid extraction logs a URL without sensitive query data.""" + url = "https://example.com/not-mam?token=secret-value#fragment" + + with patch("src.mam_api.adapter.log.warning") as mock_warning: + tid = MAMApiAdapter.extract_tid_from_url(url) + + assert tid is None + mock_warning.assert_called_once() + assert mock_warning.call_args.kwargs["url"] == "https://example.com/not-mam" + assert "secret-value" not in str(mock_warning.call_args) + def test_extract_tid_from_url_empty(self): """Test extracting tid from empty/None URL.""" assert MAMApiAdapter.extract_tid_from_url("") is None @@ -445,8 +545,8 @@ def test_extract_tid_from_url_torrents_php_no_id(self): assert tid is None @pytest.mark.asyncio - async def test_scrape_asin_backward_compat(self): - """Test that scrape_asin_from_url maintains interface.""" + async def test_get_asin_from_url(self): + """Test ASIN lookup from MAM API torrent data.""" adapter = MAMApiAdapter(mam_id="test_id") # Create a real MamTorrentRaw object with isbn field @@ -456,28 +556,25 @@ async def test_scrape_asin_backward_compat(self): with patch.object(adapter, "get_torrent_data", new_callable=AsyncMock) as mock_get: mock_get.return_value = mock_torrent - asin = await adapter.scrape_asin_from_url( - "https://www.myanonamouse.net/t/12345", - force_login=True, # Should be ignored but shouldn't error - ) + asin = await adapter.get_asin_from_url("https://www.myanonamouse.net/t/12345") assert asin == "B0TESTMOCK" @pytest.mark.asyncio - async def test_scrape_asin_no_torrent(self): - """Test scrape_asin_from_url when no torrent found.""" + async def test_get_asin_from_url_no_torrent(self): + """Test ASIN lookup when no torrent found.""" adapter = MAMApiAdapter(mam_id="test_id") with patch.object(adapter, "get_torrent_data", new_callable=AsyncMock) as mock_get: mock_get.return_value = None - asin = await adapter.scrape_asin_from_url("https://www.myanonamouse.net/t/12345") + asin = await adapter.get_asin_from_url("https://www.myanonamouse.net/t/12345") assert asin is None @pytest.mark.asyncio - async def test_scrape_asin_torrent_no_asin(self): - """Test scrape_asin_from_url when torrent has no ASIN.""" + async def test_get_asin_from_url_torrent_no_asin(self): + """Test ASIN lookup when torrent has no ASIN.""" adapter = MAMApiAdapter(mam_id="test_id") # Create a torrent without ASIN @@ -486,7 +583,7 @@ async def test_scrape_asin_torrent_no_asin(self): with patch.object(adapter, "get_torrent_data", new_callable=AsyncMock) as mock_get: mock_get.return_value = mock_torrent - asin = await adapter.scrape_asin_from_url("https://www.myanonamouse.net/t/12345") + asin = await adapter.get_asin_from_url("https://www.myanonamouse.net/t/12345") assert asin is None @@ -585,8 +682,6 @@ async def test_get_client_no_mam_id_raises(self): adapter = MAMApiAdapter(mam_id=None) adapter.mam_id = None # Explicitly set to None - from src.mam_api.client import MamApiError - with pytest.raises(MamApiError, match="MAM_ID not configured"): await adapter._get_client() @@ -653,8 +748,6 @@ async def test_get_torrent_data_api_error(self): """Test get_torrent_data handles MamApiError.""" adapter = MAMApiAdapter(mam_id="test_id", rate_limit_seconds=0) - from src.mam_api.client import MamApiError - with patch.object(adapter, "_get_client", new_callable=AsyncMock) as mock_get_client: mock_client = MagicMock() mock_client.get_torrent = AsyncMock(side_effect=MamApiError("API error")) @@ -664,6 +757,19 @@ async def test_get_torrent_data_api_error(self): assert result is None + @pytest.mark.asyncio + async def test_get_torrent_data_auth_error_raises(self): + """Test get_torrent_data surfaces MAM auth failures.""" + adapter = MAMApiAdapter(mam_id="test_id", rate_limit_seconds=0) + + with patch.object(adapter, "_get_client", new_callable=AsyncMock) as mock_get_client: + mock_client = MagicMock() + mock_client.get_torrent = AsyncMock(side_effect=MamApiError("MAM API authentication failed; update MAM_ID")) + mock_get_client.return_value = mock_client + + with pytest.raises(MamApiError, match="MAM API authentication failed"): + await adapter.get_torrent_data("https://www.myanonamouse.net/t/12345") + @pytest.mark.asyncio async def test_get_torrent_data_http_error(self): """Test get_torrent_data handles HTTP errors.""" @@ -698,12 +804,6 @@ def test_adapter_init_without_mam_id_from_env(self): adapter = MAMApiAdapter() assert adapter.mam_id is None - def test_adapter_alias_mam_scraper(self): - """Test that MAMScraper is an alias for MAMApiAdapter.""" - from src.mam_api.adapter import MAMScraper - - assert MAMScraper is MAMApiAdapter - # ============================================================================= # Integration Tests (require MAM_ID) @@ -716,6 +816,7 @@ class TestIntegration: These are skipped if MAM_ID is not set in the environment. Run with: MAM_ID=your_cookie_value pytest tests/test_mam_api.py -k Integration + Set MAM_TEST_TID to also run the real torrent download check. """ @pytest.fixture @@ -726,6 +827,17 @@ def mam_id(self): pytest.skip("MAM_ID not set - skipping integration test") return mam_id + @pytest.fixture + def mam_test_tid(self): + """Get optional MAM_TEST_TID from environment or skip download test.""" + tid = os.getenv("MAM_TEST_TID") + if not tid: + pytest.skip("MAM_TEST_TID not set - skipping torrent download integration test") + try: + return int(tid) + except ValueError: + pytest.skip("MAM_TEST_TID must be an integer torrent ID") + @pytest.mark.asyncio async def test_real_search(self, mam_id): """Test real search against MAM API.""" @@ -736,3 +848,14 @@ async def test_real_search(self, mam_id): assert results.data[0].title is not None finally: await client.aclose() + + @pytest.mark.asyncio + async def test_real_download_torrent_by_tid(self, mam_id, mam_test_tid): + """Test real .torrent download against MAM API when MAM_TEST_TID is set.""" + client = MamAsyncClient(mam_id=mam_id) + try: + torrent_bytes = await client.download_torrent_by_tid(mam_test_tid) + assert torrent_bytes.startswith(b"d") + assert b"4:info" in torrent_bytes + finally: + await client.aclose() diff --git a/tests/test_metadata_coordinator.py b/tests/test_metadata_coordinator.py index 13f4e3a..bc99963 100644 --- a/tests/test_metadata_coordinator.py +++ b/tests/test_metadata_coordinator.py @@ -17,6 +17,7 @@ import httpx import pytest +from src.mam_api import MamApiError from src.metadata_coordinator import MetadataCoordinator, main @@ -64,7 +65,7 @@ def coordinator(mock_config): coord = MetadataCoordinator() # Ensure our mock instances are assigned - coord.mam_scraper = mock_mam_instance + coord.mam_adapter = mock_mam_instance coord.audnex = mock_audnex_instance coord.audible = mock_audible_instance @@ -183,7 +184,7 @@ class TestGetMetadataFromWebhook: async def test_webhook_mam_url_success(self, coordinator, sample_webhook_payload, sample_audnex_metadata): """Test successful ASIN extraction from MAM URL.""" # MAM returns ASIN - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value="B0TEST1234") + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B0TEST1234") # Audnex returns metadata coordinator.audnex.get_book_by_asin = AsyncMock(return_value=sample_audnex_metadata.copy()) @@ -193,7 +194,7 @@ async def test_webhook_mam_url_success(self, coordinator, sample_webhook_payload assert result["asin"] == "B0TEST1234" assert result["source"] == "audnex" assert result["asin_source"] == "mam" - coordinator.mam_scraper.scrape_asin_from_url.assert_called_once() + coordinator.mam_adapter.get_asin_from_url.assert_called_once() @pytest.mark.asyncio async def test_webhook_mam_no_asin_falls_back_to_audible( @@ -201,7 +202,7 @@ async def test_webhook_mam_no_asin_falls_back_to_audible( ): """Test fallback to Audible search when MAM returns no ASIN.""" # MAM returns no ASIN - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value=None) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) # Audible search returns results coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) @@ -226,7 +227,7 @@ async def test_webhook_no_mam_url_goes_to_audible(self, coordinator, sample_audi assert result is not None assert result["source"] == "audible" # MAM should not be called for non-MAM URLs - coordinator.mam_scraper.scrape_asin_from_url.assert_not_called() + coordinator.mam_adapter.get_asin_from_url.assert_not_called() @pytest.mark.asyncio async def test_webhook_audnex_failure_falls_back_to_audible( @@ -234,7 +235,7 @@ async def test_webhook_audnex_failure_falls_back_to_audible( ): """Test Audnex failure falls back to Audible.""" # MAM returns ASIN - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value="B0TEST1234") + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B0TEST1234") # Audnex fails coordinator.audnex.get_book_by_asin = AsyncMock(return_value=None) # Audible search succeeds @@ -248,7 +249,7 @@ async def test_webhook_audnex_failure_falls_back_to_audible( @pytest.mark.asyncio async def test_webhook_all_sources_fail(self, coordinator, sample_webhook_payload): """Test when all metadata sources fail.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value=None) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=None) result = await coordinator.get_metadata_from_webhook(sample_webhook_payload) @@ -258,7 +259,7 @@ async def test_webhook_all_sources_fail(self, coordinator, sample_webhook_payloa @pytest.mark.asyncio async def test_webhook_mam_network_error(self, coordinator, sample_webhook_payload, sample_audible_metadata): """Test MAM network error is handled gracefully.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(side_effect=httpx.RequestError("Network error")) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(side_effect=httpx.RequestError("Network error")) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) result = await coordinator.get_metadata_from_webhook(sample_webhook_payload) @@ -267,10 +268,21 @@ async def test_webhook_mam_network_error(self, coordinator, sample_webhook_paylo assert result is not None assert result["source"] == "audible" + @pytest.mark.asyncio + async def test_webhook_mam_auth_error_raises(self, coordinator, sample_webhook_payload): + """Test MAM auth errors are surfaced instead of falling back to Audible.""" + coordinator.mam_adapter.get_asin_from_url = AsyncMock(side_effect=MamApiError("Auth failed")) + coordinator.audible.search_from_webhook_name = AsyncMock() + + with pytest.raises(MamApiError, match="Auth failed"): + await coordinator.get_metadata_from_webhook(sample_webhook_payload) + + coordinator.audible.search_from_webhook_name.assert_not_called() + @pytest.mark.asyncio async def test_webhook_audnex_network_error(self, coordinator, sample_webhook_payload, sample_audible_metadata): """Test Audnex network error falls back to Audible.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value="B0TEST1234") + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B0TEST1234") coordinator.audnex.get_book_by_asin = AsyncMock(side_effect=httpx.RequestError("Network error")) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) @@ -282,7 +294,7 @@ async def test_webhook_audnex_network_error(self, coordinator, sample_webhook_pa @pytest.mark.asyncio async def test_webhook_audnex_value_error(self, coordinator, sample_webhook_payload, sample_audible_metadata): """Test Audnex ValueError (malformed response) falls back.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value="B0TEST1234") + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B0TEST1234") coordinator.audnex.get_book_by_asin = AsyncMock(side_effect=ValueError("Malformed response")) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) @@ -294,7 +306,7 @@ async def test_webhook_audnex_value_error(self, coordinator, sample_webhook_payl @pytest.mark.asyncio async def test_webhook_audnex_unexpected_error(self, coordinator, sample_webhook_payload, sample_audible_metadata): """Test Audnex unexpected error is handled.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value="B0TEST1234") + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B0TEST1234") coordinator.audnex.get_book_by_asin = AsyncMock(side_effect=RuntimeError("Unexpected")) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) @@ -306,7 +318,7 @@ async def test_webhook_audnex_unexpected_error(self, coordinator, sample_webhook @pytest.mark.asyncio async def test_webhook_audible_network_error_raises(self, coordinator, sample_webhook_payload): """Test Audible network error raises ValueError.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value=None) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) coordinator.audible.search_from_webhook_name = AsyncMock(side_effect=httpx.RequestError("Network error")) with pytest.raises(ValueError, match="Could not fetch metadata"): @@ -315,7 +327,7 @@ async def test_webhook_audible_network_error_raises(self, coordinator, sample_we @pytest.mark.asyncio async def test_webhook_audible_value_error_raises(self, coordinator, sample_webhook_payload): """Test Audible ValueError raises.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value=None) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) coordinator.audible.search_from_webhook_name = AsyncMock(side_effect=ValueError("Malformed response")) with pytest.raises(ValueError, match="Could not fetch metadata"): @@ -324,7 +336,7 @@ async def test_webhook_audible_value_error_raises(self, coordinator, sample_webh @pytest.mark.asyncio async def test_webhook_audible_unexpected_error_returns_none(self, coordinator, sample_webhook_payload): """Test Audible unexpected error returns None.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value=None) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) coordinator.audible.search_from_webhook_name = AsyncMock(side_effect=RuntimeError("Unexpected")) result = await coordinator.get_metadata_from_webhook(sample_webhook_payload) @@ -338,7 +350,7 @@ async def test_webhook_passes_seed_authors_and_update( """Test that seed_authors and update params are passed to Audnex.""" coordinator.seed_authors = True coordinator.force_update = True - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value="B0TEST1234") + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B0TEST1234") coordinator.audnex.get_book_by_asin = AsyncMock(return_value=sample_audnex_metadata.copy()) await coordinator.get_metadata_from_webhook(sample_webhook_payload) @@ -356,7 +368,7 @@ async def test_webhook_empty_name_still_works(self, coordinator, sample_audible_ "name": "", "url": "https://www.myanonamouse.net/t/12345", } - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value=None) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) result = await coordinator.get_metadata_from_webhook(payload) @@ -367,7 +379,7 @@ async def test_webhook_empty_name_still_works(self, coordinator, sample_audible_ @pytest.mark.asyncio async def test_webhook_mam_value_error(self, coordinator, sample_webhook_payload, sample_audible_metadata): """Test MAM ValueError (malformed response) continues to Audible.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(side_effect=ValueError("Malformed response")) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(side_effect=ValueError("Malformed response")) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) result = await coordinator.get_metadata_from_webhook(sample_webhook_payload) @@ -378,7 +390,7 @@ async def test_webhook_mam_value_error(self, coordinator, sample_webhook_payload @pytest.mark.asyncio async def test_webhook_mam_unexpected_error(self, coordinator, sample_webhook_payload, sample_audible_metadata): """Test MAM unexpected error continues to Audible.""" - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(side_effect=RuntimeError("Unexpected")) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(side_effect=RuntimeError("Unexpected")) coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()]) result = await coordinator.get_metadata_from_webhook(sample_webhook_payload) @@ -916,7 +928,7 @@ async def test_full_workflow_mam_to_audnex_to_chapters( ): """Test complete workflow: webhook โ†’ MAM โ†’ Audnex โ†’ chapters.""" # Setup mocks - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(return_value="B0TEST1234") + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B0TEST1234") coordinator.audnex.get_book_by_asin = AsyncMock(return_value=sample_audnex_metadata.copy()) coordinator.audnex.get_chapters_by_asin = AsyncMock(return_value=sample_chapters) @@ -936,7 +948,7 @@ async def test_full_workflow_fallback_to_audible( ): """Test workflow with fallback: MAM fails โ†’ Audnex fails โ†’ Audible succeeds.""" # MAM fails - coordinator.mam_scraper.scrape_asin_from_url = AsyncMock(side_effect=httpx.RequestError("MAM down")) + coordinator.mam_adapter.get_asin_from_url = AsyncMock(side_effect=httpx.RequestError("MAM down")) # Audible succeeds coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[sample_audible_metadata.copy()])