diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..62c8935 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ \ No newline at end of file diff --git a/crawler/.dockerignore b/crawler/.dockerignore new file mode 100644 index 0000000..5bf1e41 --- /dev/null +++ b/crawler/.dockerignore @@ -0,0 +1,5 @@ +node_modules +npm-debug.log +.git +.gitignore +*.md diff --git a/crawler/.env.example b/crawler/.env.example new file mode 100644 index 0000000..917fa2f --- /dev/null +++ b/crawler/.env.example @@ -0,0 +1,56 @@ +# STAC Crawler Configuration +# Copy this file to .env and adjust values as needed + +# URI of the Postgres/PostGis Host +PGHOST=example_db_URL +# Postgres/PostGis Port +PGPORT=5432 +# Postgres/PostGis user +PGUSER=example_user +# Postgres/PostGis password +PGPASSWORD=example_password +# Postgres/PostGis databse +PGDATABASE=example_db + +# For single Crawler run: +# 'catalogs', 'apis', or 'both' +CRAWL_MODE=both +# Fresh crawl - clear crawl log and re-crawl all collections (true/false, default: false) +FRESH_CRAWL=false +# Maximum Static Catalogs that are crawled (0 ist unlimited) +MAX_CATALOGS=0 +# Maximum API Catalogs that are crawled (0 ist unlimited) +MAX_APIS=0 +# Timeout of connection of crawler +TIMEOUT_MS=30000 +# Maximum recursion depth for nested catalogs (0 ist unlimited) +MAX_DEPTH=0 + +# Parallel Crawling Configuration +# Number of domains to crawl in parallel +PARALLEL_DOMAINS=5 +# Max requests per minute PER domain +MAX_REQUESTS_PER_MINUTE_PER_DOMAIN=60 +# Max concurrent requests PER domain +MAX_CONCURRENCY_PER_DOMAIN=5 + +# Scheduler Configuration +# How many days between crawl runs (default: 7) +CRAWL_DAYS_INTERVAL=7 +# Run crawler immediately on startup (true/false, default: true) +CRAWL_RUN_ON_STARTUP=true +# Retry if crawl fails but DB is ok (true/false, default: true) +CRAWL_RETRY_ON_ERROR=true +# Hours to wait before retry on crawl error (default: 2) +CRAWL_RETRY_DELAY_HOURS=2 + +# Time Window Configuration (only active when CRAWL_ENFORCE_TIME_WINDOW=true) +# By default, crawler runs anytime. Set CRAWL_ENFORCE_TIME_WINDOW=true to restrict crawling to specific hours +# Enforce time window (true/false, default: false - crawler runs anytime) +CRAWL_ENFORCE_TIME_WINDOW=false +# Hour when crawler is allowed to start (0-23, e.g. 22 for 10 PM) - only used when time window is enforced +CRAWL_ALLOWED_START_HOUR=22 +# Hour when crawler should stop (0-23, e.g. 7 for 7 AM) - only used when time window is enforced +CRAWL_ALLOWED_END_HOUR=7 +# Grace period in minutes after end hour (default: 30) - only used when time window is enforced +CRAWL_GRACE_PERIOD_MINUTES=30 \ No newline at end of file diff --git a/crawler/.gitignore b/crawler/.gitignore index e59219b..d8cbd75 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -1,3 +1,3 @@ .env node_modules -storage \ No newline at end of file +storage diff --git a/crawler/Dockerfile b/crawler/Dockerfile new file mode 100644 index 0000000..3a15606 --- /dev/null +++ b/crawler/Dockerfile @@ -0,0 +1,17 @@ +# Use official Node.js LTS image +FROM node:20-alpine + +# Set working directory +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm install --omit=dev + +# Copy application files +COPY . . + +# Run the crawler +CMD ["node", "index.js"] diff --git a/crawler/README.md b/crawler/README.md index e69de29..b0935f5 100644 --- a/crawler/README.md +++ b/crawler/README.md @@ -0,0 +1,935 @@ +# STAC Crawler + +A Node.js crawler for STAC Index API that fetches and processes catalog and collection data with configurable options. Includes an automated scheduler for periodic crawling. + +## Table of Contents + +- [Features](#features) +- [Quick Start](#quick-start) +- [Configuration](#configuration) + - [Configuration Options](#configuration-options) + - [Using Environment Variables](#using-environment-variables) + - [Using CLI Arguments](#using-cli-arguments) + - [Show Help](#show-help) +- [Running Locally](#running-locally) +- [Docker](#docker) +- [Testing](#testing) +- [Dependencies](#dependencies) + - [Core Dependencies](#core-dependencies) + - [Development Dependencies](#development-dependencies) + - [Why These Libraries?](#why-these-libraries) +- [Technical Decisions](#technical-decisions) +- [Architecture](#architecture) +- [How It Works](#how-it-works) + - [Crawling Process Overview](#crawling-process-overview) + - [What Gets Stored](#what-gets-stored) + - [Pause and Resume Functionality](#pause-and-resume-functionality) + - [Auto-Recrawling](#auto-recrawling) + - [Data Validation](#data-validation) +- [Troubleshooting](#troubleshooting) +- [Performance Tuning](#performance-tuning) +- [npm Scripts](#npm-scripts) +- [License](#license) +- [Examples](#examples) + +## Features + +- Single-run Mode: Execute crawler once and exit +- Scheduled Mode: Automated periodic crawling with configurable intervals +- Time Window Control: Optional restriction to specific hours (e.g., night-time crawling) +- Retry Logic: Automatic retry on crawl errors with configurable delay +- Environment-based Configuration: All settings configurable via `.env` file +- CLI Arguments: Override settings with command-line flags +- Database Integration: PostgreSQL storage with deadlock handling +- Parallel Execution: Efficient domain-based parallel processing with configurable rate limiting +- Graceful Shutdown: Stop after current batch with Ctrl+C, resume later +- Pause/Resume Support: Already-crawled collections are tracked and skipped on re-run +- Fresh Mode: Clear crawl log with `--fresh` flag to re-crawl everything +- STAC Validation: Validates collections using stac-node-validator +- Automatic Cleanup: Marks stale collections as inactive after 7 days without updates + +## Quick Start + +```bash +# Install dependencies +npm install + +# Copy and configure environment file +cp .env.example .env +``` +### Single Crawl Run + +```bash +# Run crawler once +npm start +``` + +### Scheduled Crawling + +```bash +# Run scheduler for automatic periodic crawling +node scheduler.js +``` + +The scheduler will: +- Run the crawler immediately on startup (configurable) +- Schedule next runs based on configured interval (default: 7 days) +- Respect time window restrictions if enabled +- Automatically retry on errors + +## Configuration + +The crawler can be configured using environment variables, CLI arguments, or a combination of both. CLI arguments take precedence over environment variables. + +### Configuration Options + +#### Crawler Configuration + +| Option | CLI Flag | Environment Variable | Default | Description | +|--------|----------|---------------------|---------|-------------| +| Mode | `-m, --mode` | `CRAWL_MODE` | `both` | Crawl mode: `catalogs`, `apis`, or `both` | +| Max Catalogs | `-c, --max-catalogs` | `MAX_CATALOGS` | `10` | Maximum number of catalogs to process (0 = unlimited) | +| Max APIs | `-a, --max-apis` | `MAX_APIS` | `5` | Maximum number of APIs to process (0 = unlimited) | +| Timeout | `-t, --timeout` | `TIMEOUT_MS` | `30000` | Timeout per operation in milliseconds | +| Max Depth | `-d, --max-depth` | `MAX_DEPTH` | `10` | Maximum recursion depth for nested catalogs (0 = unlimited) | +| Fresh | `-f, --fresh` | `FRESH_CRAWL` | `false` | Clear crawl log and re-crawl all collections | + +#### Parallel Crawling Configuration + +| Option | CLI Flag | Environment Variable | Default | Description | +|--------|----------|---------------------|---------|-------------| +| Parallel Domains | `-p, --parallel-domains` | `PARALLEL_DOMAINS` | `2` | Number of domains to crawl in parallel | +| RPM per Domain | `--rpm-per-domain` | `MAX_REQUESTS_PER_MINUTE_PER_DOMAIN` | `60` | Max requests per minute per domain | +| Concurrency per Domain | `--concurrency-per-domain` | `MAX_CONCURRENCY_PER_DOMAIN` | `5` | Max concurrent requests per domain | + +#### Legacy Rate Limiting (still supported) + +| Option | CLI Flag | Environment Variable | Default | Description | +|--------|----------|---------------------|---------|-------------| +| Max Concurrency | `--max-concurrency` | `MAX_CONCURRENCY` | `5` | Maximum concurrent requests (global) | +| Requests per Minute | `--rpm, --requests-per-minute` | `MAX_REQUESTS_PER_MINUTE` | `60` | Maximum requests per minute (global) | +| Domain Delay | `--domain-delay` | `SAME_DOMAIN_DELAY_SECS` | `1` | Delay between requests to same domain (seconds) | +| Max Retries | `--max-retries` | `MAX_REQUEST_RETRIES` | `3` | Maximum retries for failed requests | + +#### Scheduler Configuration + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `CRAWL_DAYS_INTERVAL` | `7` | Days between crawl runs | +| `CRAWL_RUN_ON_STARTUP` | `true` | Run crawler immediately on startup | +| `CRAWL_RETRY_ON_ERROR` | `true` | Retry if crawl fails but DB is ok | +| `CRAWL_RETRY_DELAY_HOURS` | `2` | Hours to wait before retry on error | +| `CRAWL_ENFORCE_TIME_WINDOW` | `false` | Enable time window restrictions | +| `CRAWL_ALLOWED_START_HOUR` | `22` | Start hour (0-23) when time window is enforced | +| `CRAWL_ALLOWED_END_HOUR` | `7` | End hour (0-23) when time window is enforced | +| `CRAWL_GRACE_PERIOD_MINUTES` | `30` | Grace period in minutes after end hour | + +#### Database Configuration + +| Environment Variable | Description | +|---------------------|-------------| +| `PGHOST` | PostgreSQL host | +| `PGPORT` | PostgreSQL port (default: 5432) | +| `PGUSER` | PostgreSQL username | +| `PGPASSWORD` | PostgreSQL password | +| `PGDATABASE` | PostgreSQL database name | + +### Using Environment Variables + +1. Copy the example environment file: +```bash +cp .env.example .env +``` + +2. Edit `.env` to customize settings: +```bash +# Database Configuration +PGHOST=localhost +PGPORT=5432 +PGUSER=postgres +PGPASSWORD=yourpassword +PGDATABASE=stac_db + +# Crawler Configuration +CRAWL_MODE=both +MAX_CATALOGS=0 # 0 = unlimited +MAX_APIS=0 # 0 = unlimited +TIMEOUT_MS=30000 +MAX_DEPTH=3 + +# Scheduler Configuration +CRAWL_DAYS_INTERVAL=7 +CRAWL_RUN_ON_STARTUP=true +CRAWL_RETRY_ON_ERROR=true +CRAWL_RETRY_DELAY_HOURS=2 + +# Time Window Configuration (optional) +# Set CRAWL_ENFORCE_TIME_WINDOW=true to restrict crawling to specific hours +CRAWL_ENFORCE_TIME_WINDOW=false +CRAWL_ALLOWED_START_HOUR=22 # 10 PM +CRAWL_ALLOWED_END_HOUR=7 # 7 AM +CRAWL_GRACE_PERIOD_MINUTES=30 +``` + +3. Run the crawler or scheduler: +```bash +# Single run +npm start + +# Scheduled runs +node scheduler.js +``` + +### Using CLI Arguments + +Run the crawler with command-line arguments to override defaults or environment variables: + +```bash +# Crawl only catalogs with custom limits +node index.js --mode catalogs --max-catalogs 20 + +# Crawl only APIs with extended timeout +node index.js -m apis -a 10 -t 60000 + +# Crawl both with all custom settings +node index.js -m both -c 50 -a 20 -t 45000 -d 5 + +# Start fresh - clear crawl log and re-crawl everything +node index.js --fresh + +# Combine fresh mode with other options +node index.js -f -m apis -a 10 + +# Configure parallel crawling for high-performance servers +node index.js -p 5 --rpm-per-domain 120 --concurrency-per-domain 10 + +# Full unlimited crawl with fresh start +node index.js -f -m both -c 0 -a 0 -d 0 +``` + +### Show Help + +Display all available options: + +```bash +node index.js --help +``` + +## Running Locally + +### Single Crawl Run + +```bash +# Install dependencies +npm install + +# Run with default configuration +npm start + +# Run with custom configuration via CLI +node index.js --mode catalogs --max-catalogs 15 +``` + +### Scheduled Crawling + +```bash +# Start the scheduler (runs in foreground) +node scheduler.js + +# The scheduler will: +# - Run crawler immediately on startup (if CRAWL_RUN_ON_STARTUP=true) +# - Schedule next run based on CRAWL_DAYS_INTERVAL +# - Wait for allowed time window (if CRAWL_ENFORCE_TIME_WINDOW=true) +# - Automatically retry on errors (if CRAWL_RETRY_ON_ERROR=true) +# - Stop gracefully with Ctrl+C +``` + +### Time Window Examples + +Example 1: Night-time only crawling (22:00 - 07:00) +```bash +CRAWL_ENFORCE_TIME_WINDOW=true +CRAWL_ALLOWED_START_HOUR=22 +CRAWL_ALLOWED_END_HOUR=7 +``` + +Example 2: Business hours crawling (09:00 - 17:00) +```bash +CRAWL_ENFORCE_TIME_WINDOW=true +CRAWL_ALLOWED_START_HOUR=9 +CRAWL_ALLOWED_END_HOUR=17 +``` + +Example 3: No restrictions (default) +```bash +CRAWL_ENFORCE_TIME_WINDOW=false +``` + +## Docker + +### Build and run with Docker + +```bash +# Build the image +docker build -t stac-crawler . + +# Run single crawl with default configuration +docker run --rm stac-crawler + +# Run with environment variables +docker run --rm \ + -e PGHOST=host.docker.internal \ + -e PGPORT=5432 \ + -e PGUSER=postgres \ + -e PGPASSWORD=yourpassword \ + -e PGDATABASE=stac_db \ + -e CRAWL_MODE=apis \ + -e MAX_APIS=10 \ + stac-crawler + +# Run with CLI arguments +docker run --rm stac-crawler --mode catalogs --max-catalogs 20 + +# Run scheduler in Docker (detached) +docker run -d \ + --name stac-scheduler \ + -e PGHOST=host.docker.internal \ + -e CRAWL_DAYS_INTERVAL=7 \ + stac-crawler node scheduler.js +``` + +Or use npm scripts: + +```bash +npm run docker:build +npm run docker:run +``` + +### Using Docker Compose + +Create a `.env` file or modify `docker-compose.yml` to set environment variables: + +```bash +# Start the crawler (single run) +docker-compose up -d + +# View logs +docker-compose logs -f + +# Stop the crawler +docker-compose down +``` + +For scheduled crawling with Docker Compose, modify `docker-compose.yml`: +```yaml +services: + crawler: + build: . + command: node scheduler.js # Use scheduler instead of single run + env_file: .env + restart: unless-stopped # Auto-restart on failure +``` + +Or use npm scripts: + +```bash +npm run docker:compose:up +npm run docker:compose:down +``` + +## Testing + +### Running Tests + +Run the complete test suite: +```bash +npm test +``` + +Run tests in watch mode during development: +```bash +npm run test:watch +``` + +Run tests with coverage report: +```bash +npm test -- --coverage +``` + +### Test Structure + +The test suite covers utility functions across four test modules: + +- `normalization.test.js` - Tests for catalog and collection normalization + - Tests `deriveCategories()`, `normalizeCatalog()`, `normalizeCollection()`, `processCatalogs()` + +- `parallel.test.js` - Tests for parallel execution utilities + - Tests `getDomain()`, `groupByDomain()`, `createDomainBatches()`, `aggregateStats()`, `executeWithConcurrency()`, `calculateRateLimits()`, `logDomainStats()` + +- `api.test.js` - Tests for API crawling utilities + - Tests batch management, URL validation, STAC API response structures + - Uses real STAC API endpoints (Microsoft Planetary Computer, Element 84, USGS, NASA CMR) + +- `is_api.test.js` - Tests for is_api field functionality + - Verifies collections are correctly marked as API or static catalog collections + - Tests `handleCatalog()` and `handleCollections()` from handlers.js + +All tests use real STAC domain names and collection IDs from production STAC APIs for realistic testing. + +## Dependencies + +The crawler uses carefully selected libraries for specific functionality: + +### Core Dependencies + +#### **Crawlee** (v3.15.3) +- **Purpose**: Advanced web crawling framework with built-in request management +- **Why chosen**: + - Automatic retry logic with exponential backoff + - Built-in rate limiting per domain + - Concurrent request handling with configurable concurrency + - Request queue management for large-scale crawling + - Automatic handling of timeouts and errors +- **Key features used**: + - `HttpCrawler` - For HTTP requests with JSON parsing + - Request/response handlers for custom processing + - Domain-based crawling strategies +- **Alternative considered**: Axios alone - rejected because it lacks built-in queue management and retry logic + +#### **axios** (v1.13.2) +- **Purpose**: HTTP client for direct API calls (non-crawling requests) +- **Why chosen**: + - Simple interface for one-off requests (e.g., fetching catalog list) + - Wide adoption and reliability + - Promise-based async/await support +- **Used for**: Initial STAC Index API calls before crawling starts + +#### **stac-js** (v0.1.9) +- **Purpose**: STAC object manipulation and metadata extraction +- **Why chosen**: + - Official STAC library with spec-compliant parsers + - Type detection (Collection, Catalog, Item) + - Built-in methods for extent extraction (`getBoundingBox()`, `getTemporalExtent()`) + - Link resolution (relative to absolute URLs) +- **Key features used**: + - `create()` - Parse JSON into STAC objects + - `isCollection()`, `isCatalog()` - Type checking + - Extent extraction methods + +#### **stac-node-validator** (v2.0.0-rc.1) +- **Purpose**: Validate STAC JSON against official schemas +- **Why chosen**: + - Uses official STAC JSON schemas + - Validates core spec + extensions (EO, SAT, Projection, etc.) + - Detailed error reporting with field-level messages + - Async validation suitable for high-volume crawling +- **Key features used**: + - Full STAC spec validation (v1.0.0, v1.1.0 support) + - Extension schema validation + - Error message extraction for debugging +- **Critical for**: Data quality - filters out malformed STAC metadata before database insertion + +#### **@databases/pg** (v5.5.0) +- **Purpose**: PostgreSQL database client with modern async/await support +- **Why chosen**: + - Type-safe SQL queries with tagged template literals + - Connection pooling built-in + - Better TypeScript support than `pg` alone + - Cleaner API than raw `pg` +- **Key features used**: + - Connection pool management + - Parameterized queries (SQL injection prevention) + - Transaction support + + +#### **dotenv** (v17.2.3) +- **Purpose**: Environment variable management from `.env` files +- **Why chosen**: + - Standard solution for 12-factor app configuration + - Keeps sensitive credentials out of source code + - Development/production environment separation +- **Used for**: Database credentials, crawler configuration, scheduler settings + +### Development Dependencies + +#### **Jest** (v29.7.0) +- **Purpose**: Testing framework +- **Why chosen**: + - Industry standard for Node.js testing + - Built-in assertion library + - Parallel test execution + - Coverage reporting + - Module mocking support +- **Test coverage**: 110 tests across normalization, parallel execution, and API utilities +- **Configuration**: Uses ES modules (`--experimental-vm-modules`) for modern JavaScript support + +### Implicit Dependencies + +**Node.js built-ins**: +- `pg` (Pool) - Part of `@databases/pg`, PostgreSQL connection pooling +- `process.env` - Environment variable access +- `console` - Logging (no external logger to keep dependencies minimal) + + +## Technical Decisions + +### 1. Why PostgreSQL? + +**Decision**: Use PostgreSQL as the primary database + +**Reason**: +- **PostGIS extension**: Native geospatial support for bounding box queries +- **JSONB type**: Efficient storage of STAC summaries and nested metadata +- **Robust transactions**: ACID compliance prevents data corruption during concurrent crawls +- **Indexing**: B-tree, GiST, and GIN indexes for fast spatial and text searches +- **Scalability**: Handles millions of collections without performance degradation + + +### 2. Why Domain-Based Parallel Processing? + +**Decision**: Group catalogs/APIs by domain and process domains in parallel + +**Reason**: +- **Rate limiting**: Each domain has independent rate limits - prevents throttling +- **Politeness**: Distributes load across servers, avoiding overwhelming single hosts +- **Efficiency**: Processes multiple domains simultaneously while respecting per-domain limits +- **Fairness**: Prevents slow domains from blocking fast domains + + +### 3. Why Separate Crawler and Scheduler? + +**Decision**: Keep single-run crawler (`index.js`) separate from scheduler (`scheduler.js`) + +**Reason**: +- **Flexibility**: Users can run one-off crawls or automated schedules +- **Testing**: Easier to test crawler logic without scheduler complexity +- **Resource efficiency**: Single runs exit immediately, don't hold resources +- **Debugging**: Simpler to debug individual components +- **Docker compatibility**: Can run different commands in containers + + +### 4. Why Batch Flushing to Database? + +**Decision**: Collect 25 collections in memory, then flush to database + +**Reason**: +- **Performance**: Reduces database connection overhead (25x fewer transactions) +- **Memory efficiency**: Prevents unbounded memory growth on large crawls +- **Error recovery**: Smaller batches = less data lost on errors +- **Deadlock mitigation**: Fewer concurrent transactions reduce deadlock risk + +**Batch size selection**: +- Tested on 2GB RAM servers → 25 collections = ~10MB memory footprint +- Larger batches (100+) caused OOM on constrained servers +- Smaller batches (5-10) increased database load significantly + + +### 5. Why Deadlock Retry with Exponential Backoff? + +**Decision**: Retry database deadlocks up to 3 times with exponential backoff + +**Reason**: +- **PostgreSQL behavior**: Concurrent inserts on related tables (keywords, extensions) can deadlock +- **Automatic recovery**: Transient deadlocks resolve after retry +- **Exponential backoff**: Reduces contention by spreading out retry attempts +- **Max retries**: Prevents infinite loops on persistent deadlocks + + + + + + +## Architecture + +### Core Components + +- **`index.js`** - Main crawler entry point for single runs +- **`scheduler.js`** - Scheduler for periodic automated crawling +- **`utils/db.js`** - Database helper with PostgreSQL connection pool +- **`utils/normalization.js`** - Data normalization and processing +- **`utils/parallel.js`** - Parallel execution utilities with domain-based batching +- **`utils/config.js`** - Configuration management (env vars + CLI) +- **`utils/time.js`** - Time formatting utilities +- **`utils/handlers.js`** - Request handlers for catalogs and collections with STAC validation +- **`utils/endpoints.js`** - STAC API endpoint discovery utilities +- **`catalogs/catalog.js`** - Static catalog crawling logic +- **`apis/api.js`** - STAC API crawling logic + +## How It Works + +### Crawling Process Overview + +The crawler operates in two modes: **static catalog crawling** and **STAC API crawling**. Both modes follow a similar workflow but use different strategies to discover and process STAC collections. + +#### Static Catalog Crawling + +1. **Initialization**: Fetch the list of static catalogs from STAC Index API (`https://www.stacindex.org/api/catalogs`) +2. **Domain Grouping**: Group catalogs by domain to enable parallel processing while respecting rate limits +3. **Parallel Execution**: Process multiple domains simultaneously with configurable concurrency +4. **Recursive Traversal**: For each catalog: + - Fetch the catalog JSON from its URL + - Validate STAC structure using `stac-node-validator` + - Migrate to normalized format using `stac-js` + - Extract child links (catalogs and collections) + - Recursively follow catalog links up to `MAX_DEPTH` (default: 3) + - Process collection links to extract metadata +5. **Link Following**: The crawler follows STAC link relations: + - `rel=child` - Navigate to child catalogs/collections + - `rel=item` - Skip (items are not processed, only collections) + - `rel=self` - Used to determine the source URL + +#### STAC API Crawling + +1. **Initialization**: Fetch the list of STAC APIs from STAC Index API +2. **Domain Grouping**: Same as static catalog crawling +3. **API Discovery**: For each API: + - Fetch the API root endpoint + - Validate STAC API compliance + - Discover `/collections` endpoint from API conformance or links + - Try multiple endpoint variations if needed (`/collections`, `/search`, etc.) +4. **Collection Enumeration**: + - Fetch all collections from `/collections` endpoint + - Handle pagination if the API returns paged results + - Process each collection individually +5. **Nested Catalog Support**: If a collection contains child catalog links, recursively crawl them (up to `MAX_DEPTH`) + +#### What Gets Stored + +The crawler stores the following data in PostgreSQL: + +**Collections** (main data): +- **Core metadata**: `stac_id` (generated from slug + collection ID), `title`, `description`, `license` +- **Spatial extent**: Bounding box (`bbox`) stored as PostGIS geometry +- **Temporal extent**: Start and end dates +- **STAC version**: Version of STAC specification used +- **Source tracking**: `source_url` (original collection URL), `crawllog_catalog_id` (reference to source catalog) + +**Related data** (linked tables): +- **Keywords**: Extracted from collection metadata, stored in `collection_keywords` with many-to-many relation +- **STAC Extensions**: List of STAC extensions used (e.g., `eo`, `sat`, `proj`), stored in `collection_stac_extension` +- **Providers**: Data providers with name, description, roles, and URL +- **Assets**: Collection-level assets (thumbnails, documentation, etc.) +- **Summaries**: Statistical summaries of collection properties + +**Crawl tracking** (for pause/resume): +- **`crawllog_catalog`**: Stores the catalog/API URLs and slugs for future re-crawling +- **`crawllog_collection`**: Records which collection URLs have been processed and when + +**What is NOT stored**: +- **Individual items**: The crawler only processes collections, not individual STAC items +- **Catalog metadata**: Static catalogs are only used for traversal, not saved to the database +- **Full link arrays**: Only essential links (self, root) are preserved + +#### Pause and Resume Functionality + +**How Pausing Works**: +1. **Graceful Shutdown**: Press `Ctrl+C` once to trigger graceful shutdown +2. **Batch Completion**: The crawler finishes the current batch of requests before stopping +3. **Progress Saved**: All processed collections are saved to `crawllog_collection` with their source URLs +4. **Safe Exit**: Database connections are properly closed + +**How Resuming Works**: +1. **URL Lookup**: When restarting, the crawler queries `crawllog_collection` for already-processed URLs +2. **Skip Logic**: URLs in the crawl log are skipped during traversal +3. **Continue from Interruption**: Only new/unprocessed collections are fetched +4. **Idempotent**: Running the crawler multiple times is safe - duplicates are handled via `ON CONFLICT` clauses + +**Force Stop**: Press `Ctrl+C` twice for immediate termination (may leave incomplete transactions) + +#### Auto-Recrawling + +The scheduler (`scheduler.js`) provides automated periodic crawling: + +1. **Interval-based**: Runs every `CRAWL_DAYS_INTERVAL` days (default: 7) +2. **Time Window Enforcement**: Optional restriction to specific hours (e.g., night-time only) +3. **Startup Behavior**: Configurable immediate run on startup (`CRAWL_RUN_ON_STARTUP`) +4. **Error Recovery**: Automatic retry on crawl errors with configurable delay +5. **Recrawl Strategy**: Full re-crawl of all catalogs/APIs - `ON CONFLICT` ensures updates rather than duplicates + +**Scheduling Logic**: +``` +Startup → DB Check → Time Window Check → Run Crawler → Success? + ↓ Yes ↓ No (crawl error) + Schedule Next Wait RETRY_DELAY → Retry + ↓ + Wait Until Next → Run Crawler +``` + +### Data Validation + +The crawler implements multi-layer validation to ensure data quality: + +#### 1. STAC Specification Validation + +**Library**: `stac-node-validator` (v2.0.0-rc.1) + +**What it validates**: +- STAC JSON structure compliance with official STAC schemas +- Required fields presence (id, type, stac_version, etc.) +- Field types and formats +- STAC extension schemas (e.g., EO, SAT, Projection) +- Link relation requirements + +**When it runs**: Before processing any catalog or collection + +**Error handling**: +- Non-compliant structures are logged with detailed error messages +- Collections with validation errors are skipped +- Statistics track compliant vs. non-compliant items + + + +#### 2. STAC Migration Validation + +**Library**: `stac-js` (v0.1.9) + +**What it validates**: +- Converts raw JSON to typed STAC objects +- Validates object type (Collection, Catalog, Item) +- Validates link structure and relationships +- Extracts spatial/temporal extents using STAC-aware parsers +- Resolves relative URLs to absolute URLs + +**When it runs**: After STAC spec validation passes + +**Error handling**: +- Migration failures indicate malformed STAC structures +- Failed migrations are logged and skipped +- `stac-js` methods return null for invalid data (e.g., `getBoundingBox()`) + + + +#### 3. Custom Data Normalization + +**Module**: `utils/normalization.js` + +**What it normalizes**: +- **Categories/Keywords**: Derives from multiple possible fields (categories, keywords, tags) +- **Temporal extents**: Handles null values, open-ended intervals +- **Bounding boxes**: Validates array structure, handles missing coordinates +- **URLs**: Extracts self links, resolves relative paths +- **Provider roles**: Normalizes role names (producer, processor, host, licensor) +- **Fallback strategy**: Uses multiple fallback levels to extract data + + +#### 4. URL and HTTP Validation + +**Validation checks**: +- **URL format**: Ensures valid HTTP/HTTPS URLs before making requests +- **Response status**: Checks for 200 OK status codes +- **Content-Type**: Accepts JSON, GeoJSON, and some binary/text types +- **Timeout enforcement**: Requests timeout after configured duration +- **Retry logic**: Automatic retry with exponential backoff for failed requests + +**Rate limiting**: +- Per-domain rate limits prevent overwhelming servers +- Configurable requests per minute per domain +- Crawler respects HTTP 429 (Too Many Requests) responses + +#### Validation Statistics + +The crawler tracks validation results: +- `stacCompliant` - Collections passing STAC validation +- `nonCompliant` - Collections failing STAC validation +- `collectionsSaved` - Successfully saved to database +- `collectionsFailed` - Failed database insertion + +**Example output**: +``` +Validation Results: + STAC Compliant: 450 + Non-compliant: 12 + Saved to DB: 448 + Failed to save: 2 +``` + +## Troubleshooting + +### Scheduler Not Running + +Check that: +1. Database connection is configured correctly in `.env` +2. Database is accessible and running +3. Time window settings allow execution (if `CRAWL_ENFORCE_TIME_WINDOW=true`) + +View scheduler status: +```bash +node scheduler.js +# Output shows current configuration and time window status +``` + +### Crawler Runs Too Frequently + +Increase `CRAWL_DAYS_INTERVAL`: +```bash +CRAWL_DAYS_INTERVAL=7 # Run every week +``` + +### Crawler Only Runs at Specific Times + +This is controlled by time window enforcement. To allow crawling anytime: +```bash +CRAWL_ENFORCE_TIME_WINDOW=false +``` + +### Database Connection Errors + +Verify database configuration: +```bash +# Test connection manually +psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE +``` + +Check environment variables are loaded: +```bash +node -e "require('dotenv').config(); console.log(process.env.PGHOST)" +``` + +### Deadlock Errors + +The crawler has automatic deadlock retry logic with exponential backoff. If deadlocks persist: +- Reduce parallel execution settings +- Increase database connection pool size +- Check database load and indexing + +## Performance Tuning + +### Parallel Execution Settings + +The defaults are optimized for 2GB RAM servers. Control parallel processing via environment variables or CLI: + +| Setting | Default | Description | +|---------|---------|-------------| +| `PARALLEL_DOMAINS` | `2` | Number of domains to process simultaneously | +| `MAX_REQUESTS_PER_MINUTE_PER_DOMAIN` | `60` | Rate limit per domain | +| `MAX_CONCURRENCY_PER_DOMAIN` | `5` | Max concurrent requests per domain | + +Theoretical max throughput = `PARALLEL_DOMAINS` x `MAX_REQUESTS_PER_MINUTE_PER_DOMAIN` requests/min + +Example for higher-resource servers: +```bash +# High-performance settings (4+ GB RAM) +PARALLEL_DOMAINS=5 +MAX_REQUESTS_PER_MINUTE_PER_DOMAIN=120 +MAX_CONCURRENCY_PER_DOMAIN=10 +# Theoretical throughput: 600 req/min +``` + +### Database Connection Pool + +Adjust pool size in `utils/db.js`: +```javascript +const pool = new Pool({ + // ... other settings + max: 10, // Increase for higher parallelism +}); +``` + +### Timeout Configuration + +Increase timeouts for slow endpoints: +```bash +TIMEOUT_MS=120000 # 2 minutes +``` + +## npm Scripts + +```bash +npm start # Run crawler once +npm test # Run all tests +npm run test:watch # Run tests in watch mode +npm run docker:build # Build Docker image +npm run docker:run # Run Docker container +npm run docker:compose:up # Start with docker-compose +npm run docker:compose:down # Stop docker-compose +``` + +## License + +See LICENSE file in the project root. + +## Examples + +### Single-Run Examples + +#### Example 1: Quick API Test +Crawl only the first 3 APIs with a short timeout: +```bash +node index.js -m apis -a 3 -t 15000 +``` + +#### Example 2: Deep Catalog Exploration +Crawl 100 catalogs with maximum depth and extended timeout: +```bash +node index.js -m catalogs -c 100 -d 10 -t 120000 +``` + +#### Example 3: Balanced Crawl +Crawl both catalogs and APIs with moderate settings: +```bash +node index.js -m both -c 25 -a 15 -t 45000 -d 4 +``` + +### Scheduler Examples + +#### Example 1: Weekly Full Crawl (Default) +Run complete crawl every 7 days, anytime: +```bash +CRAWL_DAYS_INTERVAL=7 +CRAWL_RUN_ON_STARTUP=true +CRAWL_ENFORCE_TIME_WINDOW=false +``` + +#### Example 2: Night-time Weekly Crawl +Run every 7 days, only between 22:00 and 07:00: +```bash +CRAWL_DAYS_INTERVAL=7 +CRAWL_ENFORCE_TIME_WINDOW=true +CRAWL_ALLOWED_START_HOUR=22 +CRAWL_ALLOWED_END_HOUR=7 +CRAWL_GRACE_PERIOD_MINUTES=30 +``` + +#### Example 3: Daily Updates +Run every day with retry on errors: +```bash +CRAWL_DAYS_INTERVAL=1 +CRAWL_RUN_ON_STARTUP=true +CRAWL_RETRY_ON_ERROR=true +CRAWL_RETRY_DELAY_HOURS=2 +``` + +#### Example 4: Production Setup +Full production configuration in `.env`: +```bash +# Database +PGHOST=db.production.com +PGPORT=5432 +PGUSER=crawler_user +PGPASSWORD=secure_password +PGDATABASE=stac_production + +# Crawler - Full scan +CRAWL_MODE=both +MAX_CATALOGS=0 # Unlimited +MAX_APIS=0 # Unlimited +TIMEOUT_MS=60000 +MAX_DEPTH=5 + +# Scheduler - Weekly night crawls +CRAWL_DAYS_INTERVAL=7 +CRAWL_RUN_ON_STARTUP=false # Wait for scheduled time +CRAWL_RETRY_ON_ERROR=true +CRAWL_RETRY_DELAY_HOURS=2 + +# Time Window - Night time only +CRAWL_ENFORCE_TIME_WINDOW=true +CRAWL_ALLOWED_START_HOUR=22 +CRAWL_ALLOWED_END_HOUR=7 +CRAWL_GRACE_PERIOD_MINUTES=30 +``` + +Then run the scheduler: +```bash +node scheduler.js +``` diff --git a/crawler/__tests__/api.test.js b/crawler/__tests__/api.test.js new file mode 100644 index 0000000..3ee8c9c --- /dev/null +++ b/crawler/__tests__/api.test.js @@ -0,0 +1,574 @@ +/** + * @fileoverview Unit tests for API crawling utilities + * Tests the actual checkAndFlushApi function with mocked dependencies + */ + +import { jest } from '@jest/globals'; + +// Mock the handlers module before importing +const mockFlushCollectionsToDb = jest.fn(); + +jest.unstable_mockModule('../utils/handlers.js', () => ({ + flushCollectionsToDb: mockFlushCollectionsToDb, + handleCollections: jest.fn() +})); + +// Mock db module +jest.unstable_mockModule('../utils/db.js', () => ({ + default: { + isCollectionUrlCrawled: jest.fn().mockResolvedValue(false), + getCrawledCollectionUrls: jest.fn().mockResolvedValue(new Set()) + } +})); + +// Mock index.js to avoid side effects from main module +jest.unstable_mockModule('../index.js', () => ({ + isShutdownRequested: jest.fn().mockReturnValue(false) +})); + +// Import the actual module to test +const { checkAndFlushApi, BATCH_SIZE, API_CLEAR_BATCH_SIZE } = await import('../apis/api.js'); + +describe('checkAndFlushApi - Batch Management', () => { + beforeEach(() => { + mockFlushCollectionsToDb.mockClear(); + mockFlushCollectionsToDb.mockResolvedValue({ saved: 0, failed: 0 }); + }); + + test('should flush collections when BATCH_SIZE is reached', async () => { + const results = { + collections: new Array(BATCH_SIZE).fill(null).map((_, i) => ({ + id: `sentinel-2-l2a-${i}`, + title: `Sentinel-2 Collection ${i}` + })), + apis: [], + stats: { + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn() + }; + + mockFlushCollectionsToDb.mockResolvedValueOnce({ saved: 25, failed: 0 }); + await checkAndFlushApi(results, mockLog); + + expect(mockFlushCollectionsToDb).toHaveBeenCalledTimes(1); + expect(mockFlushCollectionsToDb).toHaveBeenCalledWith(results, mockLog, false); + expect(results.stats.collectionsSaved).toBe(25); + expect(results.stats.collectionsFailed).toBe(0); + }); + + test('should not flush when below BATCH_SIZE', async () => { + const results = { + collections: [ + { id: 'landsat-c2-l2', title: 'Landsat Collection 2' } + ], + apis: [], + stats: { + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + const mockLog = { info: jest.fn(), warning: jest.fn() }; + await checkAndFlushApi(results, mockLog); + + expect(mockFlushCollectionsToDb).not.toHaveBeenCalled(); + }); + + test('should clear APIs array when API_CLEAR_BATCH_SIZE is reached', async () => { + const results = { + collections: [], + apis: new Array(API_CLEAR_BATCH_SIZE).fill(null).map((_, i) => ({ id: `api-${i}` })), + stats: { + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + const mockLog = { info: jest.fn(), warning: jest.fn() }; + await checkAndFlushApi(results, mockLog); + + expect(results.apis.length).toBe(0); + expect(mockLog.info).toHaveBeenCalledWith( + expect.stringContaining('[MEMORY] Clearing') + ); + }); + + test('should handle both flush and clear simultaneously', async () => { + const results = { + collections: new Array(BATCH_SIZE).fill({}).map((_, i) => ({ id: `col-${i}` })), + apis: new Array(API_CLEAR_BATCH_SIZE).fill({}).map((_, i) => ({ id: `api-${i}` })), + stats: { + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + const mockLog = { info: jest.fn(), warning: jest.fn() }; + mockFlushCollectionsToDb.mockResolvedValueOnce({ saved: 20, failed: 5 }); + await checkAndFlushApi(results, mockLog); + + expect(mockFlushCollectionsToDb).toHaveBeenCalled(); + expect(results.apis.length).toBe(0); + expect(results.stats.collectionsSaved).toBe(20); + expect(results.stats.collectionsFailed).toBe(5); + }); + + test('should accumulate stats from multiple flushes', async () => { + const results = { + collections: new Array(BATCH_SIZE).fill({}).map((_, i) => ({ id: `col-${i}` })), + apis: [], + stats: { + collectionsSaved: 10, + collectionsFailed: 2 + } + }; + + const mockLog = { info: jest.fn(), warning: jest.fn() }; + mockFlushCollectionsToDb.mockResolvedValueOnce({ saved: 15, failed: 10 }); + await checkAndFlushApi(results, mockLog); + + expect(results.stats.collectionsSaved).toBe(25); // 10 + 15 + expect(results.stats.collectionsFailed).toBe(12); // 2 + 10 + }); +}); + +describe('API Endpoint URL Validation', () => { + test('should recognize valid STAC API URLs', () => { + const validApis = [ + 'https://planetarycomputer.microsoft.com/api/stac/v1', + 'https://earth-search.aws.element84.com/v1', + 'https://landsatlook.usgs.gov/stac-server', + 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD', + 'https://catalogue.dataspace.copernicus.eu/stac', + 'https://stac.terria.io', + 'https://data.lpdaac.earthdatacloud.nasa.gov/stac' + ]; + + validApis.forEach(url => { + expect(() => new URL(url)).not.toThrow(); + expect(new URL(url).protocol).toBe('https:'); + }); + }); + + test('should parse STAC API domains correctly', () => { + const apiUrls = [ + { url: 'https://planetarycomputer.microsoft.com/api/stac/v1', domain: 'planetarycomputer.microsoft.com' }, + { url: 'https://earth-search.aws.element84.com/v1', domain: 'earth-search.aws.element84.com' }, + { url: 'https://landsatlook.usgs.gov/stac-server', domain: 'landsatlook.usgs.gov' }, + { url: 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD', domain: 'cmr.earthdata.nasa.gov' } + ]; + + apiUrls.forEach(({ url, domain }) => { + const parsed = new URL(url); + expect(parsed.hostname).toBe(domain); + }); + }); + + test('should construct collections endpoint from API root', () => { + const apiRoots = [ + 'https://planetarycomputer.microsoft.com/api/stac/v1', + 'https://earth-search.aws.element84.com/v1', + 'https://landsatlook.usgs.gov/stac-server' + ]; + + apiRoots.forEach(root => { + const baseUrl = root.endsWith('/') ? root.slice(0, -1) : root; + const collectionsUrl = `${baseUrl}/collections`; + + expect(collectionsUrl).toContain('/collections'); + expect(() => new URL(collectionsUrl)).not.toThrow(); + }); + }); + + test('should handle API URLs with trailing slashes', () => { + const urls = [ + { with: 'https://planetarycomputer.microsoft.com/api/stac/v1/', without: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { with: 'https://earth-search.aws.element84.com/v1/', without: 'https://earth-search.aws.element84.com/v1' } + ]; + + urls.forEach(({ with: withSlash, without }) => { + const normalized = withSlash.endsWith('/') ? withSlash.slice(0, -1) : withSlash; + expect(normalized).toBe(without); + }); + }); +}); + +describe('STAC API Response Structures', () => { + test('should validate Microsoft Planetary Computer API root structure', () => { + const apiRoot = { + type: 'Catalog', + id: 'microsoft-pc', + title: 'Microsoft Planetary Computer STAC API', + description: 'Catalog of datasets on the Microsoft Planetary Computer', + stac_version: '1.0.0', + conformsTo: [ + 'https://api.stacspec.org/v1.0.0/core', + 'https://api.stacspec.org/v1.0.0/collections', + 'https://api.stacspec.org/v1.0.0/ogcapi-features' + ], + links: [ + { rel: 'self', href: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { rel: 'root', href: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { rel: 'data', href: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections' }, + { rel: 'conformance', href: 'https://planetarycomputer.microsoft.com/api/stac/v1/conformance' } + ] + }; + + expect(apiRoot.type).toBe('Catalog'); + expect(apiRoot.stac_version).toBeDefined(); + expect(apiRoot.conformsTo).toBeInstanceOf(Array); + expect(apiRoot.links).toBeInstanceOf(Array); + + const dataLink = apiRoot.links.find(l => l.rel === 'data'); + expect(dataLink).toBeDefined(); + expect(dataLink.href).toContain('/collections'); + }); + + test('should validate Earth Search API collections response structure', () => { + const collectionsResponse = { + collections: [ + { + id: 'sentinel-2-l2a', + type: 'Collection', + title: 'Sentinel-2 Level-2A', + description: 'Sentinel-2 Level-2A, orthorectified atmosphere-corrected surface reflectance', + stac_version: '1.0.0', + license: 'proprietary', + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2015-06-27T10:25:31Z', null]] } + }, + links: [ + { rel: 'self', href: 'https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a' } + ] + } + ], + links: [ + { rel: 'self', href: 'https://earth-search.aws.element84.com/v1/collections' }, + { rel: 'root', href: 'https://earth-search.aws.element84.com/v1' } + ] + }; + + expect(collectionsResponse.collections).toBeInstanceOf(Array); + expect(collectionsResponse.collections.length).toBeGreaterThan(0); + + const collection = collectionsResponse.collections[0]; + expect(collection.type).toBe('Collection'); + expect(collection.id).toBeDefined(); + expect(collection.extent).toBeDefined(); + expect(collection.extent.spatial).toBeDefined(); + expect(collection.extent.temporal).toBeDefined(); + }); + + test('should validate USGS Landsat collection metadata', () => { + const collection = { + id: 'landsat-c2-l2', + type: 'Collection', + title: 'Landsat Collection 2 Level-2', + description: 'Landsat Collection 2 Level-2 Science Products', + stac_version: '1.0.0', + license: 'proprietary', + keywords: ['landsat', 'usgs', 'nasa', 'satellite', 'global'], + providers: [ + { + name: 'NASA', + roles: ['producer'], + url: 'https://landsat.gsfc.nasa.gov/' + }, + { + name: 'USGS', + roles: ['processor', 'host'], + url: 'https://www.usgs.gov/landsat-missions' + } + ], + extent: { + spatial: { + bbox: [[-180, -90, 180, 90]] + }, + temporal: { + interval: [['1972-07-25T00:00:00Z', null]] + } + }, + summaries: { + platform: ['landsat-4', 'landsat-5', 'landsat-7', 'landsat-8', 'landsat-9'], + instruments: ['tm', 'etm+', 'oli', 'tirs'] + } + }; + + expect(collection.id).toBe('landsat-c2-l2'); + expect(collection.keywords).toContain('landsat'); + expect(collection.providers).toBeInstanceOf(Array); + expect(collection.providers.length).toBeGreaterThan(0); + expect(collection.summaries).toBeDefined(); + expect(collection.summaries.platform).toBeInstanceOf(Array); + }); + + test('should validate NASA CMR STAC API structure', () => { + const cmrCollection = { + id: 'HLSL30.v2.0', + type: 'Collection', + title: 'HLS Landsat Operational Land Imager Surface Reflectance and TOA Brightness Daily Global 30m v2.0', + description: 'The Harmonized Landsat Sentinel-2 (HLS) project provides consistent surface reflectance data from Landsat 8 and Sentinel-2 satellites.', + stac_version: '1.0.0', + license: 'not-provided', + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2013-04-11T00:00:00Z', null]] } + }, + links: [ + { rel: 'self', href: 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD/collections/HLSL30.v2.0' }, + { rel: 'parent', href: 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD' }, + { rel: 'root', href: 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD' } + ] + }; + + expect(cmrCollection.id).toContain('.'); + expect(cmrCollection.title).toContain('HLS'); + expect(cmrCollection.links.some(l => l.rel === 'parent')).toBe(true); + expect(cmrCollection.links[0].href).toContain('cmr.earthdata.nasa.gov'); + }); +}); + +describe('API Collections Extraction', () => { + test('should extract collection IDs from API responses', () => { + const responses = [ + { + api: 'Microsoft Planetary Computer', + collections: ['landsat-c2-l2', 'sentinel-2-l2a', 'naip', 'cop-dem-glo-30'] + }, + { + api: 'Earth Search', + collections: ['sentinel-2-l2a', 'sentinel-2-l1c', 'landsat-c2-l2', 'cop-dem-glo-30'] + }, + { + api: 'USGS Landsat', + collections: ['landsat-c2l1', 'landsat-c2l2-sr', 'landsat-c2l2-st'] + } + ]; + + responses.forEach(({ api, collections }) => { + expect(collections).toBeInstanceOf(Array); + expect(collections.length).toBeGreaterThan(0); + collections.forEach(id => { + expect(typeof id).toBe('string'); + expect(id.length).toBeGreaterThan(0); + }); + }); + }); + + test('should track API processing statistics', () => { + const stats = { + totalRequests: 15, + successfulRequests: 14, + failedRequests: 1, + apisProcessed: 3, + stacCompliant: 3, + nonCompliant: 0, + collectionsFound: 25, + collectionsSaved: 25, + collectionsFailed: 0 + }; + + expect(stats.successfulRequests + stats.failedRequests).toBe(stats.totalRequests); + expect(stats.apisProcessed).toBe(3); + expect(stats.stacCompliant).toBeGreaterThan(0); + expect(stats.collectionsFound).toBeGreaterThan(stats.apisProcessed); + }); +}); + +describe('Batch Flushing for API Collections', () => { + const BATCH_SIZE = 25; + + test('should check if collections reach batch size threshold', () => { + const results = { + collections: new Array(BATCH_SIZE).fill(null).map((_, i) => ({ + id: `sentinel-2-l2a-item-${i}`, + title: `Sentinel-2 Item ${i}`, + bbox: [-180, -90, 180, 90] + })), + stats: { + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + // Verify we have enough collections to trigger a flush + expect(results.collections.length).toBe(BATCH_SIZE); + expect(results.collections.length >= BATCH_SIZE).toBe(true); + }); + + test('should not flush collections below batch size', () => { + const results = { + collections: [ + { id: 'landsat-c2-l2-1', title: 'Landsat 1' }, + { id: 'landsat-c2-l2-2', title: 'Landsat 2' } + ], + stats: { + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + expect(results.collections.length).toBeLessThan(BATCH_SIZE); + expect(results.collections.length >= BATCH_SIZE).toBe(false); + }); + + test('should track batch statistics correctly', () => { + const stats = { + collectionsSaved: 25, + collectionsFailed: 2, + collectionsFound: 27 + }; + + expect(stats.collectionsSaved + stats.collectionsFailed).toBe(stats.collectionsFound); + expect(stats.collectionsSaved).toBeGreaterThan(0); + }); +}); + +describe('API Discovery and Link Following', () => { + test('should identify collections endpoint from API links', () => { + const apiLinks = [ + { rel: 'self', href: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { rel: 'root', href: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { rel: 'data', href: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections' }, + { rel: 'search', href: 'https://planetarycomputer.microsoft.com/api/stac/v1/search' } + ]; + + const collectionsLink = apiLinks.find(l => l.rel === 'data' || l.rel === 'collections'); + + expect(collectionsLink).toBeDefined(); + expect(collectionsLink.href).toContain('/collections'); + }); + + test('should handle child catalog links in API responses', () => { + const apiWithChildren = { + type: 'Catalog', + id: 'root-catalog', + links: [ + { rel: 'self', href: 'https://stac.terria.io' }, + { rel: 'child', href: 'https://stac.terria.io/catalogs/cbers', title: 'CBERS' }, + { rel: 'child', href: 'https://stac.terria.io/catalogs/dem', title: 'DEM' }, + { rel: 'child', href: 'https://stac.terria.io/catalogs/aster', title: 'ASTER' } + ] + }; + + const childLinks = apiWithChildren.links.filter(l => l.rel === 'child'); + + expect(childLinks.length).toBe(3); + childLinks.forEach(link => { + expect(link.href).toContain('stac.terria.io'); + expect(() => new URL(link.href)).not.toThrow(); + }); + }); + + test('should construct absolute URLs from relative API links', () => { + const baseUrl = 'https://earth-search.aws.element84.com/v1'; + const relativeLinks = [ + { relative: './collections', expected: 'https://earth-search.aws.element84.com/collections' }, + { relative: 'collections/sentinel-2-l2a', expected: 'https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a' } + ]; + + relativeLinks.forEach(({ relative, expected }) => { + let absoluteUrl; + if (!relative.startsWith('http')) { + const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); + absoluteUrl = relative.startsWith('./') + ? `${basePath}/${relative.slice(2)}` + : `${baseUrl}/${relative}`; + } + + // Basic check that it's now absolute + expect(absoluteUrl || relative).toContain('https://'); + }); + }); +}); + +describe('API Rate Limiting and Concurrency', () => { + test('should calculate rate limits per domain', () => { + const maxRequestsPerMinute = 120; + const rateLimits = { + maxRequestsPerMinute: maxRequestsPerMinute + }; + + expect(rateLimits.maxRequestsPerMinute).toBe(120); + expect(rateLimits.maxRequestsPerMinute).toBeGreaterThan(0); + }); + + test('should group API URLs by domain for parallel crawling', () => { + const apis = [ + { url: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { url: 'https://earth-search.aws.element84.com/v1' }, + { url: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections' }, + { url: 'https://landsatlook.usgs.gov/stac-server' } + ]; + + const domainMap = new Map(); + apis.forEach(api => { + const domain = new URL(api.url).hostname; + if (!domainMap.has(domain)) { + domainMap.set(domain, []); + } + domainMap.get(domain).push(api); + }); + + expect(domainMap.size).toBe(3); + expect(domainMap.get('planetarycomputer.microsoft.com').length).toBe(2); + expect(domainMap.get('earth-search.aws.element84.com').length).toBe(1); + expect(domainMap.get('landsatlook.usgs.gov').length).toBe(1); + }); + + test('should respect parallel domain concurrency limits', () => { + const config = { + parallelDomains: 5, + maxRequestsPerMinutePerDomain: 120, + maxConcurrencyPerDomain: 20 + }; + + expect(config.parallelDomains).toBeLessThanOrEqual(10); + expect(config.maxConcurrencyPerDomain).toBeGreaterThan(0); + + // Theoretical max throughput + const maxThroughput = config.parallelDomains * config.maxRequestsPerMinutePerDomain; + expect(maxThroughput).toBe(600); + }); +}); + +describe('S3 URL Handling in API Responses', () => { + test('should convert S3 URLs to HTTPS', () => { + const s3Urls = [ + { s3: 's3://usgs-landsat/collection02', expected: 'https://usgs-landsat.s3.amazonaws.com/collection02' }, + { s3: 's3://sentinel-s2-l2a/tiles/10/T/FK', expected: 'https://sentinel-s2-l2a.s3.amazonaws.com/tiles/10/T/FK' } + ]; + + s3Urls.forEach(({ s3, expected }) => { + const s3Match = s3.match(/^s3:\/\/([^/]+)\/(.*)$/); + if (s3Match) { + const [, bucket, path] = s3Match; + const httpsUrl = `https://${bucket}.s3.amazonaws.com/${path}`; + expect(httpsUrl).toBe(expected); + } + }); + }); + + test('should handle malformed S3 URLs gracefully', () => { + const malformedUrls = [ + 's3://', + 's3://bucket-only', + 's3:invalid', + 'not-s3://something' + ]; + + malformedUrls.forEach(url => { + if (url.startsWith('s3://')) { + const s3Match = url.match(/^s3:\/\/([^/]+)\/(.*)$/); + expect(s3Match).toBeFalsy(); + } + }); + }); +}); diff --git a/crawler/__tests__/deactivateStaleCollections.test.js b/crawler/__tests__/deactivateStaleCollections.test.js new file mode 100644 index 0000000..edfaa8b --- /dev/null +++ b/crawler/__tests__/deactivateStaleCollections.test.js @@ -0,0 +1,33 @@ +import { jest } from '@jest/globals'; +import db from '../utils/db.js'; + +describe('deactivateStaleCollections', () => { + beforeEach(() => { + jest.restoreAllMocks(); + }); + + test('sets is_active=false for collections older than crawl start (7 days window)', async () => { + const querySpy = jest + .spyOn(db.pool, 'query') + .mockResolvedValue({ rowCount: 3 }); + + const count = await db.deactivateStaleCollections(); + + expect(querySpy).toHaveBeenCalledTimes(1); + + const sql = querySpy.mock.calls[0][0]; + expect(sql).toMatch(/UPDATE\s+collection/i); + expect(sql).toMatch(/SET\s+is_active\s*=\s*false/i); + expect(sql).toMatch(/updated_at\s*<\s*NOW\(\)\s*-\s*INTERVAL\s*'7 days'/i); + expect(sql).toMatch(/AND\s+is_active\s*=\s*true/i); + expect(count).toBe(3); + }); + + test('returns 0 when no collections are deactivated', async () => { + jest.spyOn(db.pool, 'query').mockResolvedValue({ rowCount: 0 }); + + const count = await db.deactivateStaleCollections(); + + expect(count).toBe(0); + }); +}); diff --git a/crawler/__tests__/is_api.test.js b/crawler/__tests__/is_api.test.js new file mode 100644 index 0000000..4853204 --- /dev/null +++ b/crawler/__tests__/is_api.test.js @@ -0,0 +1,562 @@ +/** + * @fileoverview Unit tests for is_api field functionality + * Tests that collections are correctly marked as API or static catalog collections + */ + +import { jest } from '@jest/globals'; +import create from 'stac-js'; + +// Mock normalizeCollection to return a simple object +jest.unstable_mockModule('../utils/normalization.js', () => ({ + normalizeCollection: jest.fn((stacObj, index) => ({ + id: stacObj.id || `collection-${index}`, + title: stacObj.title || 'Test Collection', + description: stacObj.description || 'Test Description' + })) +})); + +// Mock db module +const mockInsertOrUpdateCollection = jest.fn(); +const mockInsertOrUpdateCatalog = jest.fn(); +const mockIsCollectionUrlCrawled = jest.fn().mockResolvedValue(false); +const mockGetCrawledCollectionUrls = jest.fn().mockResolvedValue(new Set()); +jest.unstable_mockModule('../utils/db.js', () => ({ + default: { + insertOrUpdateCollection: mockInsertOrUpdateCollection, + insertOrUpdateCatalog: mockInsertOrUpdateCatalog, + isCollectionUrlCrawled: mockIsCollectionUrlCrawled, + getCrawledCollectionUrls: mockGetCrawledCollectionUrls + } +})); + +// Mock endpoints module +jest.unstable_mockModule('../utils/endpoints.js', () => ({ + tryCollectionEndpoints: jest.fn() +})); + +// Import the modules to test +const { handleCatalog, handleCollections } = await import('../utils/handlers.js'); +const { normalizeCollection } = await import('../utils/normalization.js'); + +describe('is_api field - handleCatalog (static catalogs)', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('should set is_api=false for collections extracted from static catalogs', async () => { + // Real Sentinel-2 collection from static catalog + const collectionJson = { + stac_version: '1.0.0', + type: 'Collection', + id: 'sentinel-2-l2a', + title: 'Sentinel-2 Level-2A', + description: 'Sentinel-2 Level-2A, orthorectified atmosphere-corrected surface reflectance', + license: 'proprietary', + keywords: ['sentinel', 'esa', 'copernicus', 'satellite', 'global'], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2015-06-27T10:25:31Z', null]] } + }, + links: [ + { rel: 'self', href: './sentinel-2-l2a/collection.json' }, + { rel: 'root', href: '../catalog.json' } + ] + }; + + const mockRequest = { + url: 'https://example.com/catalog/collection.json', + userData: { + depth: 1, + catalogId: 'test-catalog', + catalogSlug: 'test-catalog-slug' + } + }; + + const mockCrawler = { + addRequests: jest.fn() + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn(), + debug: jest.fn(), + error: jest.fn() + }; + + const results = { + collections: [], + catalogs: [], + stats: { + stacCompliant: 0, + catalogsProcessed: 0, + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + await handleCatalog({ + request: mockRequest, + json: collectionJson, + crawler: mockCrawler, + log: mockLog, + indent: '', + results, + config: {} + }); + + // Verify that a collection was added + expect(results.collections.length).toBe(1); + + // Verify that is_api is set to false for static catalog collection + expect(results.collections[0].is_api).toBe(false); + + // Verify other fields are set correctly + expect(results.collections[0].sourceSlug).toBe('test-catalog-slug'); + expect(results.collections[0].crawledUrl).toBe('https://example.com/catalog/collection.json'); + }); + + test('should set is_api=false for STAC catalog (not collection)', async () => { + // Real static STAC catalog structure + const catalogJson = { + stac_version: '1.0.0', + type: 'Catalog', + id: 'earth-observation-catalog', + title: 'Earth Observation Data Catalog', + description: 'A catalog of Earth observation satellite imagery collections', + links: [ + { rel: 'self', href: './catalog.json' }, + { rel: 'root', href: './catalog.json' }, + { rel: 'child', href: './sentinel-2/catalog.json', title: 'Sentinel-2' }, + { rel: 'child', href: './landsat/catalog.json', title: 'Landsat' } + ] + }; + + const mockRequest = { + url: 'https://example.com/catalog.json', + userData: { + depth: 0, + catalogId: 'root-catalog', + catalogSlug: 'root-slug' + } + }; + + const mockCrawler = { + addRequests: jest.fn() + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn(), + debug: jest.fn(), + error: jest.fn() + }; + + const results = { + collections: [], + catalogs: [], + stats: { + stacCompliant: 0, + catalogsProcessed: 0, + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + await handleCatalog({ + request: mockRequest, + json: catalogJson, + crawler: mockCrawler, + log: mockLog, + indent: '', + results, + config: {} + }); + + // Verify that no collections were added (it's a catalog, not a collection) + expect(results.collections.length).toBe(0); + + // Verify catalog was processed + expect(results.catalogs.length).toBe(1); + expect(results.stats.catalogsProcessed).toBe(1); + }); +}); + +describe('is_api field - handleCollections', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('should set is_api=false when isApi parameter is false (static catalog)', async () => { + // Real static catalog collections response + const collectionsJson = { + collections: [ + { + stac_version: '1.0.0', + type: 'Collection', + id: 'landsat-c2-l2', + title: 'Landsat Collection 2 Level-2', + description: 'Landsat Collection 2 Level-2 Science Products', + license: 'proprietary', + keywords: ['landsat', 'usgs', 'nasa', 'satellite', 'global'], + providers: [ + { name: 'NASA', roles: ['producer'], url: 'https://landsat.gsfc.nasa.gov/' }, + { name: 'USGS', roles: ['processor', 'host'], url: 'https://www.usgs.gov/landsat-missions' } + ], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['1972-07-25T00:00:00Z', null]] } + }, + summaries: { + platform: ['landsat-4', 'landsat-5', 'landsat-7', 'landsat-8', 'landsat-9'], + instruments: ['tm', 'etm+', 'oli', 'tirs'] + }, + links: [ + { rel: 'self', href: './landsat-c2-l2/collection.json' } + ] + }, + { + stac_version: '1.0.0', + type: 'Collection', + id: 'cop-dem-glo-30', + title: 'Copernicus DEM GLO-30', + description: 'Global 30m Digital Elevation Model', + license: 'proprietary', + keywords: ['dem', 'elevation', 'copernicus'], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2021-04-22T00:00:00Z', '2021-04-22T23:59:59Z']] } + }, + links: [ + { rel: 'self', href: './cop-dem-glo-30/collection.json' } + ] + } + ] + }; + + const mockRequest = { + url: 'https://example.com/collections', + userData: { + catalogId: 'test-catalog', + catalogSlug: 'test-catalog-slug' + } + }; + + const mockCrawler = { + addRequests: jest.fn() + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn(), + debug: jest.fn() + }; + + const results = { + collections: [], + stats: { + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + await handleCollections({ + request: mockRequest, + json: collectionsJson, + crawler: mockCrawler, + log: mockLog, + indent: '', + results, + isApi: false // Static catalog + }); + + // Verify collections were added + expect(results.collections.length).toBe(2); + + // Verify all collections have is_api=false + results.collections.forEach(collection => { + expect(collection.is_api).toBe(false); + }); + }); + + test('should set is_api=true when isApi parameter is true (API)', async () => { + // Real Earth Search API collections response + const collectionsJson = { + collections: [ + { + stac_version: '1.0.0', + type: 'Collection', + id: 'sentinel-2-l2a', + title: 'Sentinel-2 Level-2A', + description: 'Sentinel-2 Level-2A, orthorectified atmosphere-corrected surface reflectance', + license: 'proprietary', + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2015-06-27T10:25:31Z', null]] } + }, + links: [ + { rel: 'self', href: 'https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a' }, + { rel: 'root', href: 'https://earth-search.aws.element84.com/v1' } + ] + }, + { + stac_version: '1.0.0', + type: 'Collection', + id: 'sentinel-2-l1c', + title: 'Sentinel-2 Level-1C', + description: 'Sentinel-2 Level-1C Top-of-Atmosphere reflectance', + license: 'proprietary', + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2015-06-27T10:25:31Z', null]] } + }, + links: [ + { rel: 'self', href: 'https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c' }, + { rel: 'root', href: 'https://earth-search.aws.element84.com/v1' } + ] + } + ], + links: [ + { rel: 'self', href: 'https://earth-search.aws.element84.com/v1/collections' }, + { rel: 'root', href: 'https://earth-search.aws.element84.com/v1' } + ] + }; + + const mockRequest = { + url: 'https://api.example.com/stac/v1/collections', + userData: { + apiId: 'test-api', + catalogSlug: 'test-api-slug' + } + }; + + const mockCrawler = { + addRequests: jest.fn() + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn(), + debug: jest.fn() + }; + + const results = { + collections: [], + stats: { + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + await handleCollections({ + request: mockRequest, + json: collectionsJson, + crawler: mockCrawler, + log: mockLog, + indent: '', + results, + isApi: true // API endpoint + }); + + // Verify collections were added + expect(results.collections.length).toBe(2); + + // Verify all collections have is_api=true + results.collections.forEach(collection => { + expect(collection.is_api).toBe(true); + }); + }); + + test('should default to is_api=false when isApi parameter is not provided', async () => { + // Real NASA CMR STAC collection + const collectionsJson = { + collections: [ + { + stac_version: '1.0.0', + type: 'Collection', + id: 'HLSL30.v2.0', + title: 'HLS Landsat Operational Land Imager Surface Reflectance and TOA Brightness Daily Global 30m v2.0', + description: 'The Harmonized Landsat Sentinel-2 (HLS) project provides consistent surface reflectance data from Landsat 8 and Sentinel-2 satellites.', + license: 'not-provided', + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2013-04-11T00:00:00Z', null]] } + }, + links: [ + { rel: 'self', href: 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD/collections/HLSL30.v2.0' }, + { rel: 'parent', href: 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD' }, + { rel: 'root', href: 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD' } + ] + } + ] + }; + + const mockRequest = { + url: 'https://example.com/collections', + userData: { + catalogId: 'test-catalog', + catalogSlug: 'test-catalog-slug' + } + }; + + const mockCrawler = { + addRequests: jest.fn() + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn(), + debug: jest.fn() + }; + + const results = { + collections: [], + stats: { + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0 + } + }; + + // Call without isApi parameter - should default to false + await handleCollections({ + request: mockRequest, + json: collectionsJson, + crawler: mockCrawler, + log: mockLog, + indent: '', + results + // isApi parameter omitted + }); + + // Verify collections were added + expect(results.collections.length).toBe(1); + + // Verify is_api defaults to false + expect(results.collections[0].is_api).toBe(false); + }); +}); + +describe('is_api field - Database integration', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockInsertOrUpdateCollection.mockResolvedValue(1); + }); + + test('should pass is_api=true to database for API collections', async () => { + const { flushCollectionsToDb } = await import('../utils/handlers.js'); + + const results = { + collections: [ + { + id: 'sentinel-2-l2a', + title: 'Sentinel-2 Level-2A', + description: 'Sentinel-2 Level-2A, orthorectified atmosphere-corrected surface reflectance', + license: 'proprietary', + is_api: true, // API collection from Microsoft Planetary Computer + sourceSlug: 'microsoft-planetary-computer', + crawledUrl: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a' + } + ] + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn() + }; + + await flushCollectionsToDb(results, mockLog, true); + + // Verify insertOrUpdateCollection was called + expect(mockInsertOrUpdateCollection).toHaveBeenCalledTimes(1); + + // Verify the collection passed has is_api=true + const passedCollection = mockInsertOrUpdateCollection.mock.calls[0][0]; + expect(passedCollection.is_api).toBe(true); + }); + + test('should pass is_api=false to database for static catalog collections', async () => { + const { flushCollectionsToDb } = await import('../utils/handlers.js'); + + const results = { + collections: [ + { + id: 'landsat-c2-l2', + title: 'Landsat Collection 2 Level-2', + description: 'Landsat Collection 2 Level-2 Science Products', + license: 'proprietary', + is_api: false, // Static catalog collection + sourceSlug: 'usgs-landsat-catalog', + crawledUrl: 'https://landsatlook.usgs.gov/stac-browser/landsat-c2-l2/collection.json' + } + ] + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn() + }; + + await flushCollectionsToDb(results, mockLog, true); + + // Verify insertOrUpdateCollection was called + expect(mockInsertOrUpdateCollection).toHaveBeenCalledTimes(1); + + // Verify the collection passed has is_api=false + const passedCollection = mockInsertOrUpdateCollection.mock.calls[0][0]; + expect(passedCollection.is_api).toBe(false); + }); + + test('should handle mixed API and static catalog collections in batch', async () => { + const { flushCollectionsToDb } = await import('../utils/handlers.js'); + + const results = { + collections: [ + { + id: 'sentinel-2-l2a', + title: 'Sentinel-2 Level-2A', + is_api: true, // From Earth Search API + sourceSlug: 'earth-search' + }, + { + id: 'landsat-c2-l2', + title: 'Landsat Collection 2 Level-2', + is_api: false, // From static catalog + sourceSlug: 'usgs-catalog' + }, + { + id: 'naip', + title: 'NAIP: National Agriculture Imagery Program', + is_api: true, // From Microsoft Planetary Computer API + sourceSlug: 'microsoft-pc' + }, + { + id: 'cop-dem-glo-30', + title: 'Copernicus DEM GLO-30', + is_api: false, // From static catalog + sourceSlug: 'copernicus-catalog' + } + ] + }; + + const mockLog = { + info: jest.fn(), + warning: jest.fn() + }; + + await flushCollectionsToDb(results, mockLog, true); + + // Verify all collections were processed + expect(mockInsertOrUpdateCollection).toHaveBeenCalledTimes(4); + + // Verify correct is_api values were passed + const calls = mockInsertOrUpdateCollection.mock.calls; + expect(calls[0][0].is_api).toBe(true); // sentinel-2-l2a (API) + expect(calls[1][0].is_api).toBe(false); // landsat-c2-l2 (static) + expect(calls[2][0].is_api).toBe(true); // naip (API) + expect(calls[3][0].is_api).toBe(false); // cop-dem-glo-30 (static) + }); +}); diff --git a/crawler/__tests__/normalization.test.js b/crawler/__tests__/normalization.test.js new file mode 100644 index 0000000..5080828 --- /dev/null +++ b/crawler/__tests__/normalization.test.js @@ -0,0 +1,448 @@ +/** + * @fileoverview Unit tests for normalization utilities + */ + +import { jest } from '@jest/globals'; +import { + deriveCategories, + normalizeCatalog, + normalizeCollection, + processCatalogs +} from '../utils/normalization.js'; + +describe('deriveCategories', () => { + test('should return empty array for null input', () => { + expect(deriveCategories(null)).toEqual([]); + }); + + test('should return empty array for undefined input', () => { + expect(deriveCategories(undefined)).toEqual([]); + }); + + test('should return empty array for non-object input', () => { + expect(deriveCategories('string')).toEqual([]); + expect(deriveCategories(123)).toEqual([]); + }); + + test('should extract categories from categories field', () => { + const catalog = { categories: ['imagery', 'satellite'] }; + expect(deriveCategories(catalog)).toEqual(['imagery', 'satellite']); + }); + + test('should filter out falsy values from categories', () => { + const catalog = { categories: ['imagery', null, '', 'satellite', undefined] }; + expect(deriveCategories(catalog)).toEqual(['imagery', 'satellite']); + }); + + test('should extract categories from keywords field', () => { + const catalog = { keywords: ['landsat', 'modis'] }; + expect(deriveCategories(catalog)).toEqual(['landsat', 'modis']); + }); + + test('should extract categories from tags field', () => { + const catalog = { tags: ['climate', 'weather'] }; + expect(deriveCategories(catalog)).toEqual(['climate', 'weather']); + }); + + test('should extract category from access field', () => { + const catalog = { access: 'public' }; + expect(deriveCategories(catalog)).toEqual(['public']); + }); + + test('should trim whitespace from access field', () => { + const catalog = { access: ' restricted ' }; + expect(deriveCategories(catalog)).toEqual(['restricted']); + }); + + test('should ignore empty access field', () => { + const catalog = { access: ' ' }; + expect(deriveCategories(catalog)).toEqual([]); + }); + + test('should prioritize categories over keywords', () => { + const catalog = { + categories: ['cat1'], + keywords: ['key1'] + }; + expect(deriveCategories(catalog)).toEqual(['cat1']); + }); + + test('should prioritize keywords over tags', () => { + const catalog = { + keywords: ['key1'], + tags: ['tag1'] + }; + expect(deriveCategories(catalog)).toEqual(['key1']); + }); + + test('should prioritize tags over access', () => { + const catalog = { + tags: ['tag1'], + access: 'public' + }; + expect(deriveCategories(catalog)).toEqual(['tag1']); + }); + + test('should convert non-string array elements to strings', () => { + const catalog = { categories: [1, 2, true, 'test'] }; + expect(deriveCategories(catalog)).toEqual(['1', '2', 'true', 'test']); + }); +}); + +describe('normalizeCatalog', () => { + test('should normalize a basic catalog object', () => { + const catalog = { + id: 'microsoft-pc', + url: 'https://planetarycomputer.microsoft.com/api/stac/v1', + slug: 'microsoft-planetary-computer', + title: 'Microsoft Planetary Computer STAC API', + summary: 'A test catalog', + access: 'public', + created: '2024-01-01', + updated: '2024-01-02', + isPrivate: false, + isApi: true, + accessInfo: 'Free access' + }; + + const result = normalizeCatalog(catalog, 5); + + expect(result.index).toBe(5); + expect(result.id).toBe('microsoft-pc'); + expect(result.url).toBe('https://planetarycomputer.microsoft.com/api/stac/v1'); + expect(result.slug).toBe('microsoft-planetary-computer'); + expect(result.title).toBe('Microsoft Planetary Computer STAC API'); + expect(result.summary).toBe('A test catalog'); + expect(result.access).toBe('public'); + expect(result.created).toBe('2024-01-01'); + expect(result.updated).toBe('2024-01-02'); + expect(result.isPrivate).toBe(false); + expect(result.isApi).toBe(true); + expect(result.accessInfo).toBe('Free access'); + }); + + test('should derive categories from catalog', () => { + const catalog = { + id: 'usgs-landsat', + url: 'https://landsatlook.usgs.gov/stac-server', + categories: ['imagery', 'satellite'] + }; + + const result = normalizeCatalog(catalog, 0); + expect(result.categories).toEqual(['imagery', 'satellite']); + }); + + test('should preserve additional dynamic properties', () => { + const catalog = { + id: 'test', + url: 'https://example.com', + customField: 'custom value', + anotherField: 123 + }; + + const result = normalizeCatalog(catalog, 0); + expect(result.customField).toBe('custom value'); + expect(result.anotherField).toBe(123); + }); + + test('should not duplicate standard properties in dynamic properties', () => { + const catalog = { + id: 'test', + url: 'https://example.com', + title: 'Test' + }; + + const result = normalizeCatalog(catalog, 0); + const keys = Object.keys(result); + const idCount = keys.filter(k => k === 'id').length; + expect(idCount).toBe(1); + }); +}); + +describe('normalizeCollection', () => { + test('should normalize a plain collection object', () => { + const collection = { + id: 'sentinel-2-l2a', + title: 'Sentinel-2 Level-2A', + description: 'Sentinel-2 Level-2A, orthorectified atmosphere-corrected surface reflectance', + license: 'proprietary', + keywords: ['sentinel', 'copernicus', 'esa', 'msi', 'reflectance'], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2015-06-27T10:25:31Z', null]] } + }, + links: [ + { rel: 'self', href: 'https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a' } + ], + stac_version: '1.0.0', + type: 'Collection', + summaries: { 'eo:bands': [] }, + stac_extensions: ['https://stac-extensions.github.io/eo/v1.0.0/schema.json'], + providers: [{ name: 'Test Provider' }], + assets: {} + }; + + const result = normalizeCollection(collection, 0); + + expect(result.index).toBe(0); + expect(result.id).toBe('sentinel-2-l2a'); + expect(result.title).toBe('Sentinel-2 Level-2A'); + expect(result.description).toBe('Sentinel-2 Level-2A, orthorectified atmosphere-corrected surface reflectance'); + expect(result.license).toBe('proprietary'); + expect(result.keywords).toEqual(['sentinel', 'copernicus', 'esa', 'msi', 'reflectance']); + expect(result.bbox).toEqual([-180, -90, 180, 90]); + expect(result.temporal).toEqual(['2015-06-27T10:25:31Z', null]); + expect(result.url).toBe('https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a'); + expect(result.stac_version).toBe('1.0.0'); + expect(result.type).toBe('Collection'); + }); + + test('should handle stac-js object with getBoundingBox method', () => { + const collection = { + id: 'test', + getBoundingBox: () => [0, 0, 10, 10], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] } + } + }; + + const result = normalizeCollection(collection, 0); + expect(result.bbox).toEqual([0, 0, 10, 10]); + }); + + test('should handle stac-js object with getTemporalExtent method', () => { + const collection = { + id: 'test', + getTemporalExtent: () => ['2020-01-01', '2023-12-31'], + extent: { + temporal: { interval: [['2019-01-01', '2022-12-31']] } + } + }; + + const result = normalizeCollection(collection, 0); + expect(result.temporal).toEqual(['2020-01-01', '2023-12-31']); + }); + + test('should fallback to extent.spatial.bbox when methods unavailable', () => { + const collection = { + id: 'test', + extent: { + spatial: { bbox: [[1, 2, 3, 4]] } + } + }; + + const result = normalizeCollection(collection, 0); + expect(result.bbox).toEqual([1, 2, 3, 4]); + }); + + test('should fallback to extent.temporal.interval when methods unavailable', () => { + const collection = { + id: 'test', + extent: { + temporal: { interval: [['2020-01-01', null]] } + } + }; + + const result = normalizeCollection(collection, 0); + expect(result.temporal).toEqual(['2020-01-01', null]); + }); + + test('should handle stac-js object with getAbsoluteUrl method', () => { + const collection = { + id: 'test', + getAbsoluteUrl: () => 'https://example.com/absolute' + }; + + const result = normalizeCollection(collection, 0); + expect(result.url).toBe('https://example.com/absolute'); + }); + + test('should extract self link from links array', () => { + const collection = { + id: 'landsat-c2-l2', + links: [ + { rel: 'root', href: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { rel: 'self', href: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/landsat-c2-l2' } + ] + }; + + const result = normalizeCollection(collection, 0); + expect(result.url).toBe('https://planetarycomputer.microsoft.com/api/stac/v1/collections/landsat-c2-l2'); + }); + + test('should use summary field if description is missing', () => { + const collection = { + id: 'test', + summary: 'This is a summary' + }; + + const result = normalizeCollection(collection, 0); + expect(result.description).toBe('This is a summary'); + }); + + test('should prefer description over summary', () => { + const collection = { + id: 'test', + description: 'Description text', + summary: 'Summary text' + }; + + const result = normalizeCollection(collection, 0); + expect(result.description).toBe('Description text'); + }); + + test('should handle stac-js object with toJSON method', () => { + const collection = { + id: 'test-from-method', + toJSON: () => ({ + id: 'test-from-json', + title: 'JSON Title', + extent: { + spatial: { bbox: [[5, 6, 7, 8]] } + } + }) + }; + + const result = normalizeCollection(collection, 0); + expect(result.id).toBe('test-from-method'); // Direct property takes precedence + expect(result.bbox).toEqual([5, 6, 7, 8]); // Fallback from toJSON + }); + + test('should convert stac-js link objects to plain objects', () => { + const collection = { + id: 'cop-dem-glo-30', + links: [ + { rel: 'self', href: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/cop-dem-glo-30', type: 'application/json', title: 'Self' } + ] + }; + + const result = normalizeCollection(collection, 0); + expect(result.links).toEqual([ + { rel: 'self', href: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/cop-dem-glo-30', type: 'application/json', title: 'Self' } + ]); + }); + + test('should default to Unknown for missing id', () => { + const collection = {}; + + const result = normalizeCollection(collection, 0); + expect(result.id).toBe('Unknown'); + }); + + test('should default to Collection for missing type', () => { + const collection = { id: 'test' }; + + const result = normalizeCollection(collection, 0); + expect(result.type).toBe('Collection'); + }); + + test('should handle null values gracefully', () => { + const collection = { + id: 'test', + title: null, + description: null, + license: null, + bbox: null, + temporal: null + }; + + const result = normalizeCollection(collection, 0); + expect(result.title).toBeNull(); + expect(result.description).toBeNull(); + expect(result.license).toBeNull(); + expect(result.bbox).toBeNull(); + expect(result.temporal).toBeNull(); + }); + + test('should default empty array for keywords', () => { + const collection = { id: 'test' }; + + const result = normalizeCollection(collection, 0); + expect(result.keywords).toEqual([]); + }); + + test('should default empty array for stac_extensions', () => { + const collection = { id: 'test' }; + + const result = normalizeCollection(collection, 0); + expect(result.stac_extensions).toEqual([]); + }); + + test('should default empty array for providers', () => { + const collection = { id: 'test' }; + + const result = normalizeCollection(collection, 0); + expect(result.providers).toEqual([]); + }); +}); + +describe('processCatalogs', () => { + // Mock console.log to avoid clutter in test output + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + console.log.mockRestore(); + }); + + test('should throw error for non-array input', () => { + expect(() => processCatalogs('not an array')).toThrow('Expected an array'); + expect(() => processCatalogs(null)).toThrow('Expected an array'); + expect(() => processCatalogs({})).toThrow('Expected an array'); + }); + + test('should process empty array', () => { + const result = processCatalogs([]); + expect(result).toEqual([]); + expect(result.length).toBe(0); + }); + + test('should normalize all catalogs in array', () => { + const catalogs = [ + { id: 'microsoft-pc', url: 'https://planetarycomputer.microsoft.com/api/stac/v1', title: 'Microsoft Planetary Computer' }, + { id: 'earth-search', url: 'https://earth-search.aws.element84.com/v1', title: 'Earth Search by Element 84' }, + { id: 'usgs-landsat', url: 'https://landsatlook.usgs.gov/stac-server', title: 'USGS Landsat' } + ]; + + const result = processCatalogs(catalogs); + expect(result.length).toBe(3); + expect(result[0].id).toBe('microsoft-pc'); + expect(result[0].index).toBe(0); + expect(result[1].id).toBe('earth-search'); + expect(result[1].index).toBe(1); + expect(result[2].id).toBe('usgs-landsat'); + expect(result[2].index).toBe(2); + }); + + test('should maintain index order', () => { + const catalogs = [ + { id: 'planetary-computer', url: 'https://planetarycomputer.microsoft.com/api/stac/v1' }, + { id: 'earth-search', url: 'https://earth-search.aws.element84.com/v1' }, + { id: 'copernicus', url: 'https://catalogue.dataspace.copernicus.eu/stac' } + ]; + + const result = processCatalogs(catalogs); + expect(result[0].index).toBe(0); + expect(result[1].index).toBe(1); + expect(result[2].index).toBe(2); + }); + + test('should log summary information', () => { + const catalogs = [ + { id: 'nasa-cmr', url: 'https://cmr.earthdata.nasa.gov/stac', title: 'NASA CMR STAC', isApi: true, categories: ['satellite', 'nasa'] } + ]; + + processCatalogs(catalogs); + + expect(console.log).toHaveBeenCalledWith('Total: 1 catalogs found\n'); + expect(console.log).toHaveBeenCalledWith('Example - First Catalog:'); + }); + + test('should not log example for empty array', () => { + processCatalogs([]); + + expect(console.log).toHaveBeenCalledWith('Total: 0 catalogs found\n'); + expect(console.log).not.toHaveBeenCalledWith('Example - First Catalog:'); + }); +}); diff --git a/crawler/__tests__/parallel.test.js b/crawler/__tests__/parallel.test.js new file mode 100644 index 0000000..07ca3a0 --- /dev/null +++ b/crawler/__tests__/parallel.test.js @@ -0,0 +1,555 @@ +/** + * @fileoverview Unit tests for parallel execution utilities + */ + +import { jest } from '@jest/globals'; +import { + getDomain, + groupByDomain, + createDomainBatches, + aggregateStats, + executeWithConcurrency, + calculateRateLimits, + logDomainStats +} from '../utils/parallel.js'; + +describe('getDomain', () => { + test('should extract domain from valid URL', () => { + expect(getDomain('https://planetarycomputer.microsoft.com/api/stac/v1')).toBe('planetarycomputer.microsoft.com'); + expect(getDomain('http://earth-search.aws.element84.com/v1')).toBe('earth-search.aws.element84.com'); + expect(getDomain('https://landsatlook.usgs.gov/stac-server')).toBe('landsatlook.usgs.gov'); + }); + + test('should handle URLs with ports', () => { + expect(getDomain('https://planetarycomputer.microsoft.com:8080/path')).toBe('planetarycomputer.microsoft.com'); + expect(getDomain('http://localhost:8080/stac')).toBe('localhost'); + }); + + test('should handle URLs with query parameters', () => { + expect(getDomain('https://earth-search.aws.element84.com/v1/search?limit=10')).toBe('earth-search.aws.element84.com'); + }); + + test('should handle URLs with hash fragments', () => { + expect(getDomain('https://catalogue.dataspace.copernicus.eu/stac#collections')).toBe('catalogue.dataspace.copernicus.eu'); + }); + + test('should return "unknown" for invalid URLs', () => { + expect(getDomain('not a url')).toBe('unknown'); + expect(getDomain('')).toBe('unknown'); + expect(getDomain('//invalid')).toBe('unknown'); + }); + + test('should handle different protocols', () => { + expect(getDomain('ftp://data.lpdaac.earthdatacloud.nasa.gov')).toBe('data.lpdaac.earthdatacloud.nasa.gov'); + expect(getDomain('ws://stac-api.terria.io')).toBe('stac-api.terria.io'); + }); +}); + +describe('groupByDomain', () => { + test('should group items by domain', () => { + const items = [ + { url: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/landsat-c2-l2' }, + { url: 'https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a' }, + { url: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a' } + ]; + + const result = groupByDomain(items); + + expect(result.size).toBe(2); + expect(result.get('planetarycomputer.microsoft.com').length).toBe(2); + expect(result.get('earth-search.aws.element84.com').length).toBe(1); + }); + + test('should handle empty array', () => { + const result = groupByDomain([]); + expect(result.size).toBe(0); + }); + + test('should handle single domain', () => { + const items = [ + { url: 'https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l1' }, + { url: 'https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-st' }, + { url: 'https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-sr' } + ]; + + const result = groupByDomain(items); + + expect(result.size).toBe(1); + expect(result.get('landsatlook.usgs.gov').length).toBe(3); + }); + + test('should preserve item data', () => { + const items = [ + { url: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/landsat-c2-l2', id: 'landsat-c2-l2', data: 'test' }, + { url: 'https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a', id: 'sentinel-2-l2a', data: 'test2' } + ]; + + const result = groupByDomain(items); + const domainItems = result.get('planetarycomputer.microsoft.com'); + + expect(domainItems[0].id).toBe('landsat-c2-l2'); + expect(domainItems[0].data).toBe('test'); + expect(domainItems[1].id).toBe('sentinel-2-l2a'); + }); + + test('should handle invalid URLs by grouping under "unknown"', () => { + const items = [ + { url: 'invalid url 1' }, + { url: 'invalid url 2' }, + { url: 'https://earth-search.aws.element84.com/v1' } + ]; + + const result = groupByDomain(items); + + expect(result.has('unknown')).toBe(true); + expect(result.get('unknown').length).toBe(2); + expect(result.get('earth-search.aws.element84.com').length).toBe(1); + }); + + test('should handle subdomains as separate domains', () => { + const items = [ + { url: 'https://stac.terria.io/catalogs/cbers' }, + { url: 'https://data.lpdaac.earthdatacloud.nasa.gov/stac' }, + { url: 'https://cmr.earthdata.nasa.gov/stac' } + ]; + + const result = groupByDomain(items); + + expect(result.size).toBe(3); + expect(result.has('stac.terria.io')).toBe(true); + expect(result.has('data.lpdaac.earthdatacloud.nasa.gov')).toBe(true); + expect(result.has('cmr.earthdata.nasa.gov')).toBe(true); + }); +}); + +describe('createDomainBatches', () => { + test('should create batches of specified size', () => { + const domainMap = new Map([ + ['domain1.com', [1, 2, 3]], + ['domain2.com', [4, 5]], + ['domain3.com', [6]], + ['domain4.com', [7, 8]], + ['domain5.com', [9]], + ['domain6.com', [10]] + ]); + + const batches = createDomainBatches(domainMap, 2); + + expect(batches.length).toBe(3); + expect(batches[0].length).toBe(2); + expect(batches[1].length).toBe(2); + expect(batches[2].length).toBe(2); + }); + + test('should handle remainder in last batch', () => { + const domainMap = new Map([ + ['domain1.com', []], + ['domain2.com', []], + ['domain3.com', []] + ]); + + const batches = createDomainBatches(domainMap, 2); + + expect(batches.length).toBe(2); + expect(batches[0].length).toBe(2); + expect(batches[1].length).toBe(1); + }); + + test('should use default batch size of 5', () => { + const domainMap = new Map([ + ['d1', []], ['d2', []], ['d3', []], ['d4', []], ['d5', []], + ['d6', []], ['d7', []], ['d8', []], ['d9', []], ['d10', []] + ]); + + const batches = createDomainBatches(domainMap); + + expect(batches.length).toBe(2); + expect(batches[0].length).toBe(5); + expect(batches[1].length).toBe(5); + }); + + test('should handle empty domain map', () => { + const domainMap = new Map(); + const batches = createDomainBatches(domainMap, 5); + + expect(batches.length).toBe(0); + }); + + test('should handle domain map smaller than batch size', () => { + const domainMap = new Map([ + ['domain1.com', [1, 2]], + ['domain2.com', [3]] + ]); + + const batches = createDomainBatches(domainMap, 5); + + expect(batches.length).toBe(1); + expect(batches[0].length).toBe(2); + }); + + test('should preserve domain-items pairs correctly', () => { + const domainMap = new Map([ + ['planetarycomputer.microsoft.com', ['landsat-c2-l2', 'sentinel-2-l2a']], + ['earth-search.aws.element84.com', ['sentinel-2-l1c', 'landsat-c2-l1']] + ]); + + const batches = createDomainBatches(domainMap, 2); + + expect(batches[0][0][0]).toBe('planetarycomputer.microsoft.com'); + expect(batches[0][0][1]).toEqual(['landsat-c2-l2', 'sentinel-2-l2a']); + expect(batches[0][1][0]).toBe('earth-search.aws.element84.com'); + expect(batches[0][1][1]).toEqual(['sentinel-2-l1c', 'landsat-c2-l1']); + }); +}); + +describe('aggregateStats', () => { + test('should aggregate statistics from multiple results', () => { + const results = [ + { + stats: { + totalRequests: 10, + successfulRequests: 8, + failedRequests: 2, + collectionsFound: 5, + collectionsSaved: 4, + collectionsFailed: 1 + } + }, + { + stats: { + totalRequests: 20, + successfulRequests: 18, + failedRequests: 2, + collectionsFound: 10, + collectionsSaved: 9, + collectionsFailed: 1 + } + } + ]; + + const aggregated = aggregateStats(results); + + expect(aggregated.totalRequests).toBe(30); + expect(aggregated.successfulRequests).toBe(26); + expect(aggregated.failedRequests).toBe(4); + expect(aggregated.collectionsFound).toBe(15); + expect(aggregated.collectionsSaved).toBe(13); + expect(aggregated.collectionsFailed).toBe(2); + }); + + test('should handle empty results array', () => { + const aggregated = aggregateStats([]); + + expect(aggregated.totalRequests).toBe(0); + expect(aggregated.successfulRequests).toBe(0); + expect(aggregated.failedRequests).toBe(0); + }); + + test('should handle results with missing stats', () => { + const results = [ + { stats: { totalRequests: 10 } }, + { stats: null }, + { otherField: 'value' } + ]; + + const aggregated = aggregateStats(results); + + expect(aggregated.totalRequests).toBe(10); + expect(aggregated.successfulRequests).toBe(0); + }); + + test('should include all standard stat fields', () => { + const results = [ + { + stats: { + totalRequests: 5, + successfulRequests: 4, + failedRequests: 1, + collectionsFound: 2, + collectionsSaved: 2, + collectionsFailed: 0, + catalogsProcessed: 1, + apisProcessed: 0, + stacCompliant: 1, + nonCompliant: 0 + } + } + ]; + + const aggregated = aggregateStats(results); + + expect(aggregated).toHaveProperty('totalRequests'); + expect(aggregated).toHaveProperty('successfulRequests'); + expect(aggregated).toHaveProperty('failedRequests'); + expect(aggregated).toHaveProperty('collectionsFound'); + expect(aggregated).toHaveProperty('collectionsSaved'); + expect(aggregated).toHaveProperty('collectionsFailed'); + expect(aggregated).toHaveProperty('catalogsProcessed'); + expect(aggregated).toHaveProperty('apisProcessed'); + expect(aggregated).toHaveProperty('stacCompliant'); + expect(aggregated).toHaveProperty('nonCompliant'); + }); + + test('should ignore non-numeric values', () => { + const results = [ + { + stats: { + totalRequests: 10, + successfulRequests: 'invalid', + failedRequests: null, + collectionsFound: undefined + } + } + ]; + + const aggregated = aggregateStats(results); + + expect(aggregated.totalRequests).toBe(10); + expect(aggregated.successfulRequests).toBe(0); + expect(aggregated.failedRequests).toBe(0); + expect(aggregated.collectionsFound).toBe(0); + }); + + test('should handle partial stats objects', () => { + const results = [ + { stats: { totalRequests: 5 } }, + { stats: { successfulRequests: 10, collectionsFound: 3 } } + ]; + + const aggregated = aggregateStats(results); + + expect(aggregated.totalRequests).toBe(5); + expect(aggregated.successfulRequests).toBe(10); + expect(aggregated.collectionsFound).toBe(3); + expect(aggregated.failedRequests).toBe(0); + }); +}); + +describe('executeWithConcurrency', () => { + test('should execute tasks with concurrency limit', async () => { + let concurrentCount = 0; + let maxConcurrent = 0; + + const createTask = (delay) => async () => { + concurrentCount++; + maxConcurrent = Math.max(maxConcurrent, concurrentCount); + await new Promise(resolve => setTimeout(resolve, delay)); + concurrentCount--; + return delay; + }; + + const tasks = [ + createTask(50), + createTask(50), + createTask(50), + createTask(50), + createTask(50) + ]; + + const results = await executeWithConcurrency(tasks, 2); + + expect(results.length).toBe(5); + expect(maxConcurrent).toBeLessThanOrEqual(2); + }); + + test('should return results in correct order', async () => { + const tasks = [ + async () => 'first', + async () => 'second', + async () => 'third' + ]; + + const results = await executeWithConcurrency(tasks, 2); + + expect(results).toEqual(['first', 'second', 'third']); + }); + + test('should handle empty task array', async () => { + const results = await executeWithConcurrency([], 5); + expect(results).toEqual([]); + }); + + test('should handle single task', async () => { + const tasks = [async () => 'result']; + const results = await executeWithConcurrency(tasks, 5); + + expect(results).toEqual(['result']); + }); + + test('should handle task errors gracefully', async () => { + const tasks = [ + async () => 'success', + async () => { throw new Error('Task failed'); }, + async () => 'success2' + ]; + + const results = await executeWithConcurrency(tasks, 2); + + expect(results[0]).toBe('success'); + expect(results[1]).toHaveProperty('error', 'Task failed'); + expect(results[1]).toHaveProperty('stats', {}); + expect(results[2]).toBe('success2'); + }); + + test('should call progress callback with correct values', async () => { + const progressUpdates = []; + const onProgress = (completed, total) => { + progressUpdates.push({ completed, total }); + }; + + const tasks = [ + async () => 'a', + async () => 'b', + async () => 'c' + ]; + + await executeWithConcurrency(tasks, 2, onProgress); + + expect(progressUpdates.length).toBe(3); + expect(progressUpdates[0]).toEqual({ completed: 1, total: 3 }); + expect(progressUpdates[1]).toEqual({ completed: 2, total: 3 }); + expect(progressUpdates[2]).toEqual({ completed: 3, total: 3 }); + }); + + test('should work without progress callback', async () => { + const tasks = [async () => 'result']; + const results = await executeWithConcurrency(tasks, 1); + + expect(results).toEqual(['result']); + }); + + test('should handle concurrency of 1', async () => { + let executing = 0; + + const createTask = () => async () => { + executing++; + expect(executing).toBe(1); + await new Promise(resolve => setTimeout(resolve, 10)); + executing--; + return 'done'; + }; + + const tasks = [createTask(), createTask(), createTask()]; + await executeWithConcurrency(tasks, 1); + }); + + test('should handle concurrency greater than task count', async () => { + const tasks = [ + async () => 'a', + async () => 'b' + ]; + + const results = await executeWithConcurrency(tasks, 10); + expect(results).toEqual(['a', 'b']); + }); +}); + +describe('calculateRateLimits', () => { + test('should return rate limit configuration', () => { + const config = calculateRateLimits(120); + + expect(config).toHaveProperty('maxRequestsPerMinute'); + expect(config.maxRequestsPerMinute).toBe(120); + }); + + test('should use default value of 120', () => { + const config = calculateRateLimits(); + + expect(config.maxRequestsPerMinute).toBe(120); + }); + + test('should accept different rate values', () => { + expect(calculateRateLimits(60).maxRequestsPerMinute).toBe(60); + expect(calculateRateLimits(300).maxRequestsPerMinute).toBe(300); + expect(calculateRateLimits(1).maxRequestsPerMinute).toBe(1); + }); + + test('should handle zero and negative values', () => { + expect(calculateRateLimits(0).maxRequestsPerMinute).toBe(0); + expect(calculateRateLimits(-10).maxRequestsPerMinute).toBe(-10); + }); +}); + +describe('logDomainStats', () => { + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + console.log.mockRestore(); + }); + + test('should log domain statistics', () => { + const domainMap = new Map([ + ['example.com', [1, 2, 3]], + ['test.org', [4, 5]] + ]); + + logDomainStats(domainMap, 'catalogs'); + + expect(console.log).toHaveBeenCalledWith('\n=== Domain Distribution for catalogs ==='); + expect(console.log).toHaveBeenCalledWith('Total domains: 2'); + }); + + test('should sort domains by item count', () => { + const domainMap = new Map([ + ['landsatlook.usgs.gov', [1]], + ['planetarycomputer.microsoft.com', [1, 2, 3, 4, 5]], + ['earth-search.aws.element84.com', [1, 2, 3]] + ]); + + logDomainStats(domainMap); + + const calls = console.log.mock.calls.map(call => call[0]); + const largeDomainIndex = calls.findIndex(c => c.includes('planetarycomputer.microsoft.com')); + const mediumDomainIndex = calls.findIndex(c => c.includes('earth-search.aws.element84.com')); + const smallDomainIndex = calls.findIndex(c => c.includes('landsatlook.usgs.gov')); + + expect(largeDomainIndex).toBeLessThan(mediumDomainIndex); + expect(mediumDomainIndex).toBeLessThan(smallDomainIndex); + }); + + test('should show only top 10 domains', () => { + const domainMap = new Map(); + for (let i = 0; i < 15; i++) { + domainMap.set(`domain${i}.com`, [1, 2]); + } + + logDomainStats(domainMap); + + const calls = console.log.mock.calls.map(call => call[0]); + const moreDomainsMessage = calls.find(c => c.includes('and 5 more domains')); + + expect(moreDomainsMessage).toBeDefined(); + }); + + test('should not show "more domains" message for 10 or fewer domains', () => { + const domainMap = new Map(); + for (let i = 0; i < 8; i++) { + domainMap.set(`domain${i}.com`, [1]); + } + + logDomainStats(domainMap); + + const calls = console.log.mock.calls.map(call => call[0]); + const moreDomainsMessage = calls.find(c => c.includes('more domains')); + + expect(moreDomainsMessage).toBeUndefined(); + }); + + test('should use default item type of "items"', () => { + const domainMap = new Map([['catalogue.dataspace.copernicus.eu', [1, 2]]]); + + logDomainStats(domainMap); + + expect(console.log).toHaveBeenCalledWith('\n=== Domain Distribution for items ==='); + }); + + test('should handle empty domain map', () => { + const domainMap = new Map(); + + logDomainStats(domainMap, 'test'); + + expect(console.log).toHaveBeenCalledWith('Total domains: 0'); + }); +}); diff --git a/crawler/apis/api.js b/crawler/apis/api.js new file mode 100644 index 0000000..0eac760 --- /dev/null +++ b/crawler/apis/api.js @@ -0,0 +1,654 @@ +/** + * @fileoverview API crawling functionality for STAC Index using Crawlee + * Supports parallel crawling of multiple domains simultaneously + * @module apis/api + */ + +import { HttpCrawler, Configuration, log as crawleeLog } from 'crawlee'; +import create from 'stac-js'; +import { normalizeCollection } from '../utils/normalization.js'; +import { handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; +import { + groupByDomain, + executeWithConcurrency, + aggregateStats, + calculateRateLimits, + logDomainStats +} from '../utils/parallel.js'; +import globalStats from '../utils/globalStats.js'; +import db from '../utils/db.js'; +import { isShutdownRequested } from '../index.js'; + +/** + * Batch size for saving collections to database during API crawling + * Set low (25) for servers with limited RAM (2GB) + * @type {number} + */ +const BATCH_SIZE = 25; + +/** + * Batch size for clearing apis array to free memory + * Set low (25) for servers with limited RAM (2GB) + * @type {number} + */ +const API_CLEAR_BATCH_SIZE = 25; + +/** + * Checks if batch size is reached and flushes if necessary + * @async + * @param {Object} results - Results object containing collections array + * @param {Object} log - Logger instance + */ +async function checkAndFlushApi(results, log) { + if (results.collections.length >= BATCH_SIZE) { + const { saved, failed } = await flushCollectionsToDb(results, log, false); + results.stats.collectionsSaved = (results.stats.collectionsSaved || 0) + saved; + results.stats.collectionsFailed = (results.stats.collectionsFailed || 0) + failed; + } + + if (results.apis && results.apis.length >= API_CLEAR_BATCH_SIZE) { + log.info(`[MEMORY] Clearing ${results.apis.length} APIs from memory`); + results.apis.length = 0; + } +} + +/** + * Creates and runs a single Crawlee HttpCrawler for a specific domain + * @async + * @param {Array} apis - Array of API objects with url, slug, and title for this domain + * @param {string} domain - The domain being crawled + * @param {Object} config - Configuration object + * @returns {Promise} Crawl results with collections and statistics + */ +async function crawlSingleApiDomain(apis, domain, config = {}) { + // Set unique storage directory for this crawler to avoid conflicts + const safeDomain = domain.replace(/[^a-zA-Z0-9]/g, '_'); + const storageDir = `/tmp/crawlee-api-${safeDomain}-${Date.now()}-${Math.random().toString(36).slice(2)}`; + Configuration.getGlobalConfig().set('storageDir', storageDir); + Configuration.getGlobalConfig().set('persistStorage', false); + + const timeoutSecs = config.timeout && config.timeout !== Infinity + ? Math.ceil(config.timeout / 1000) + : 60; + + // Calculate rate limits for this domain + const rateLimits = calculateRateLimits(config.maxRequestsPerMinutePerDomain || 120); + + // Store results + const results = { + collections: [], + apis: [], + stats: { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + } + }; + + // Maximum depth for nested catalog crawling (0 = unlimited) + const maxDepth = config.maxDepth || 10; + + const concurrency = config.maxConcurrencyPerDomain || 20; + + const DB_QUEUE_TARGET = 1000; + const DB_QUEUE_LOW_WATERMARK = 100; + const DB_QUEUE_BATCH_SIZE = 900; + const domainApiIds = apis.map(api => api.crawllogCatalogId).filter(Boolean); + + function getApiQueueLabel(url) { + if (typeof url !== 'string') return 'API_ROOT'; + if (/\/collections\/?$/.test(url)) return 'API_COLLECTIONS'; + if (/\/collections\//.test(url)) return 'API_COLLECTION'; + return 'API_ROOT'; + } + + async function ensureDbQueueBuffer(crawler, log) { + if (!crawler?.requestQueue?.getInfo) return; + + const info = await crawler.requestQueue.getInfo(); + const pending = info?.pendingRequestCount ?? 0; + + if (pending > DB_QUEUE_LOW_WATERMARK) return; + + const toFetch = Math.min(DB_QUEUE_BATCH_SIZE, Math.max(DB_QUEUE_TARGET - pending, 0)); + if (toFetch <= 0) return; + + const batch = await db.claimCollectionQueueBatch({ + limit: toFetch, + isApi: true, + crawllogCatalogIds: domainApiIds.length > 0 ? domainApiIds : undefined + }); + if (batch.length === 0) return; + + const requests = batch.map((item, idx) => ({ + url: item.url, + label: getApiQueueLabel(item.url), + userData: { + apiId: `queued-collection-${idx}`, + apiUrl: item.url, + apiSlug: item.slug || null, + catalogSlug: item.slug || null, + crawllogCatalogId: item.crawllogCatalogId || null, + depth: 0 + } + })); + + await crawler.addRequests(requests); + log.info(`[QUEUE] Pulled ${requests.length} API collection URLs from DB queue (pending: ${pending})`); + } + + const crawler = new HttpCrawler({ + requestHandlerTimeoutSecs: timeoutSecs, + + // Rate limiting + maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, + maxRequestRetries: config.maxRequestRetries || 3, + + // High concurrency for throughput + maxConcurrency: concurrency, + + // Reduce periodic statistics logging (we have our own end statistics) + statisticsOptions: { + logIntervalSecs: 60, + }, + + // Accept additional MIME types + additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], + + async requestHandler({ request, json, body, crawler, log }) { + results.stats.totalRequests++; + globalStats.increment('totalRequests'); + const depth = request.userData?.depth || 0; + const indent = ' '.repeat(Math.min(depth, 5)); + + // Fallback: manually parse JSON if Crawlee's automatic parsing failed + if (!json && body) { + try { + const bodyStr = typeof body === 'string' ? body : body.toString('utf8'); + json = JSON.parse(bodyStr); + log.debug(`${indent}Manually parsed JSON for ${request.url} (${bodyStr.length} bytes)`); + } catch (parseError) { + log.warning(`${indent}Failed to parse response body as JSON: ${parseError.message}`); + } + } + + try { + if (request.label === 'API_ROOT') { + await handleApiRoot({ request, json, crawler, log, indent, results, maxDepth }); + } else if (request.label === 'API_COLLECTIONS') { + await handleCollections({ request, json, crawler, log, indent, results, isApi: true }); + } else if (request.label === 'API_COLLECTION') { + await handleApiCollection({ request, json, crawler, log, indent, results }); + } + + results.stats.successfulRequests++; + globalStats.increment('successfulRequests'); + await ensureDbQueueBuffer(crawler, log); + } catch (error) { + log.error(`${indent}Error handling ${request.label} at ${request.url}: ${error.message}`); + throw error; + } + }, + + async failedRequestHandler({ request, error, log }) { + results.stats.failedRequests++; + globalStats.increment('failedRequests'); + const indent = ' '; + const apiId = request.userData?.apiId || 'unknown'; + + if (error.message.includes('STAC validation')) { + log.info(`${indent}[STAC VALIDATION FAILED] ${apiId} at ${request.url}`); + log.info(`${indent} Reason: ${error.message}`); + results.stats.nonCompliant++; + globalStats.increment('nonCompliant'); + } else if (error.message.includes('timeout')) { + log.warning(`${indent}[TIMEOUT] ${apiId} at ${request.url}`); + } else if (error.message.includes('ENOTFOUND') || error.message.includes('ECONNREFUSED')) { + log.warning(`${indent}[CONNECTION FAILED] ${apiId} at ${request.url}`); + } else if (error.statusCode === 429) { + const retryAfter = error.response?.headers?.['retry-after'] || 'unknown'; + log.warning(`${indent}[RATE LIMITED] ${apiId} at ${request.url} - Retry-After: ${retryAfter}s`); + } else if (error.code === 'ERR_NON_2XX_3XX_RESPONSE') { + log.warning(`${indent}[HTTP ERROR] ${apiId} at ${request.url} - Status: ${error.statusCode}`); + } else { + log.warning(`${indent}[FAILED] ${apiId} at ${request.url}`); + log.warning(`${indent} Error: ${error.message}`); + } + } + }); + + // Seed the crawler with initial API requests + const initialRequests = apis + .filter(api => !api.hasPendingQueue) + .map((api, index) => ({ + url: api.url, + label: 'API_ROOT', + userData: { + apiId: `${domain}-api-${index}`, + apiUrl: api.url, + apiSlug: api.slug, + crawllogCatalogId: api.crawllogCatalogId, // Link to crawllog_catalog for collections + depth: 0 + } + })); + + await crawler.addRequests(initialRequests); + + await ensureDbQueueBuffer(crawler, crawleeLog); + + // Register domain as active in global stats + globalStats.domainStarted(domain); + + console.log(` [${domain}] Starting: ${initialRequests.length} APIs, max ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent`); + await crawler.run(); + + // Flush any remaining collections to database + const finalFlush = await flushCollectionsToDb(results, crawleeLog, true); + results.stats.collectionsSaved += finalFlush.saved; + results.stats.collectionsFailed += finalFlush.failed; + + // Update global stats with final counts + globalStats.increment('collectionsSaved', results.stats.collectionsSaved); + globalStats.increment('collectionsFailed', results.stats.collectionsFailed); + globalStats.increment('collectionsFound', results.stats.collectionsFound); + globalStats.increment('apisProcessed', results.stats.apisProcessed); + globalStats.increment('stacCompliant', results.stats.stacCompliant); + + // Register domain as completed + globalStats.domainCompleted(domain); + + // Clear apis array to free memory + results.apis.length = 0; + + console.log(` [${domain}] Finished: ${results.stats.collectionsFound} collections, ${results.stats.successfulRequests}/${results.stats.totalRequests} requests`); + + return results; +} + +/** + * Crawls STAC APIs to retrieve collection information without fetching items. + * Groups APIs by domain and crawls multiple domains simultaneously. + * + * @param {Array} apis - Array of API objects with url, slug, and title + * @param {boolean} isApi - Boolean flag indicating if the URLs are APIs + * @param {Object} config - Configuration object with timeout settings + * @returns {Promise} Results object with collections array and statistics + */ +async function crawlApis(apis, isApi, config = {}) { + if (!isApi || !Array.isArray(apis) || apis.length === 0) { + return { + collections: [], + stats: { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + } + }; + } + + // Group API objects by domain (keeps slug intact) + const domainMap = groupByDomain(apis); + + // Log domain distribution + logDomainStats(domainMap, 'APIs'); + + // Number of domains to crawl in parallel (default: 5) + const parallelDomains = config.parallelDomains || 5; + const maxRequestsPerMinutePerDomain = config.maxRequestsPerMinutePerDomain || 120; + + console.log(`\n=== Parallel API Crawling Configuration ===`); + console.log(`Parallel domains: ${parallelDomains}`); + console.log(`Max requests/min per domain: ${maxRequestsPerMinutePerDomain}`); + console.log(`Theoretical max throughput: ${parallelDomains * maxRequestsPerMinutePerDomain} req/min across all domains`); + console.log(`============================================\n`); + + // Create tasks for each domain (pass full API objects including slug), with shutdown check + const domainTasks = Array.from(domainMap.entries()).map(([domain, domainApis]) => { + return async () => { + // Check if shutdown was requested before starting this domain + if (isShutdownRequested()) { + console.log(` [${domain}] Skipped (shutdown requested)`); + return { stats: { totalRequests: 0, successfulRequests: 0, failedRequests: 0, collectionsFound: 0, collectionsSaved: 0, collectionsFailed: 0, apisProcessed: 0, stacCompliant: 0, nonCompliant: 0 } }; + } + return crawlSingleApiDomain(domainApis, domain, config); + }; + }); + + console.log(`Starting parallel API crawl of ${domainMap.size} domains (${parallelDomains} at a time)...\n`); + console.log(`Press Ctrl+C to pause (will stop after current batch and resume on next run)\n`); + + // Track total runtime for throughput calculation + const crawlStartTime = Date.now(); + + // Execute with concurrency limit + const allResults = await executeWithConcurrency( + domainTasks, + parallelDomains, + (completed, total) => { + if (isShutdownRequested()) { + console.log(`\n>>> Shutdown requested. Stopping after current domains complete... <<<\n`); + } else { + console.log(`\n>>> Domain progress: ${completed}/${total} domains completed <<<\n`); + } + } + ); + + const crawlEndTime = Date.now(); + const totalRuntimeMs = crawlEndTime - crawlStartTime; + const totalRuntimeMinutes = totalRuntimeMs / 60000; + + // Aggregate all statistics + const aggregatedStats = aggregateStats(allResults); + + // Calculate actual throughput + const requestsPerMinute = totalRuntimeMinutes > 0 + ? Math.round(aggregatedStats.totalRequests / totalRuntimeMinutes) + : 0; + + console.log('\n=== API Crawl Statistics ==='); + console.log(` Domains Processed: ${domainMap.size}`); + console.log(` Total Runtime: ${Math.round(totalRuntimeMs / 1000)}s`); + console.log(` Total Requests: ${aggregatedStats.totalRequests}`); + console.log(` Requests/Min (actual): ${requestsPerMinute}`); + console.log(` Successful: ${aggregatedStats.successfulRequests}`); + console.log(` Failed: ${aggregatedStats.failedRequests}`); + console.log(` STAC Compliant: ${aggregatedStats.stacCompliant}`); + console.log(` Non-Compliant: ${aggregatedStats.nonCompliant}`); + console.log(` APIs Processed: ${aggregatedStats.apisProcessed}`); + console.log(` Collections Found: ${aggregatedStats.collectionsFound}`); + console.log(` Collections Saved to DB: ${aggregatedStats.collectionsSaved}`); + console.log(` Collections Failed: ${aggregatedStats.collectionsFailed}`); + console.log('=====================================\n'); + + return { + collections: [], + apis: [], + stats: aggregatedStats + }; +} + + +/** + * Handles API root endpoint - validates STAC, discovers collections endpoints + * @async + */ +async function handleApiRoot({ request, json, crawler, log, indent, results, maxDepth = 10 }) { + const apiId = request.userData?.apiId || 'unknown'; + const apiUrl = request.userData?.apiUrl || request.url; + const apiSlug = request.userData?.apiSlug || null; + const crawllogCatalogId = request.userData?.crawllogCatalogId || null; + const depth = request.userData?.depth || 0; + + log.info(`${indent}Processing API: ${apiId} at ${apiUrl} (depth: ${depth})`); + + if (!json || typeof json !== 'object') { + log.warning(`${indent}Invalid JSON response for ${apiId} at ${request.url}`); + throw new Error('Invalid JSON response: null or not an object'); + } + + let stacObj; + try { + stacObj = create(json, true); + results.stats.stacCompliant++; + + if (typeof stacObj.isCatalog === 'function' && stacObj.isCatalog()) { + log.info(`${indent}STAC Catalog/API validated: ${apiId}`); + } else if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { + log.info(`${indent}STAC Collection validated: ${apiId}`); + } + } catch (parseError) { + log.warning(`${indent}Non-compliant STAC API ${apiId} at ${request.url}`); + log.warning(`${indent}Error details: ${parseError.message}`); + throw new Error(`STAC validation failed: ${parseError.message}`); + } + + results.stats.apisProcessed++; + // Only track minimal info to reduce memory + results.apis.push({ + id: apiId + }); + + // If this is a STAC Collection directly, extract and store it + if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { + // Persist collection URL in crawllog_collection queue + try { + await db.enqueueCollectionUrl({ + sourceUrl: request.url, + crawllogCatalogId: crawllogCatalogId + }); + } catch (err) { + log.warning(`${indent}Failed to enqueue collection URL: ${err.message}`); + } + + // Check if this collection URL was already crawled (pause/resume support) + const alreadyCrawled = await db.isCollectionUrlCrawled(request.url); + if (alreadyCrawled) { + log.info(`${indent}Skipping already-crawled collection: ${stacObj.id} (resume mode)`); + try { + await db.markCatalogCrawled(crawllogCatalogId); + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + return; + } + + const collection = normalizeCollection(stacObj, results.collections.length); + // Add the API slug to the collection for unique stac_id generation + collection.sourceSlug = apiSlug; + // Mark as API collection + collection.is_api = true; + // Link to crawllog_catalog + collection.crawllogCatalogId = crawllogCatalogId; + // Store the crawled URL + collection.crawledUrl = request.url; + results.collections.push(collection); + results.stats.collectionsFound++; + log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + + await checkAndFlushApi(results, log); + try { + await db.markCatalogCrawled(crawllogCatalogId); + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + return; + } + + // Try to find collections endpoint using stac-js + let collectionsEndpoint = null; + + if (typeof stacObj.getApiCollectionsLink === 'function') { + const collectionsLink = stacObj.getApiCollectionsLink(); + if (collectionsLink && collectionsLink.href) { + collectionsEndpoint = collectionsLink.href; + log.info(`${indent}Found collections link via stac-js: ${collectionsEndpoint}`); + } + } + + // Fallback: use standard /collections endpoint + if (!collectionsEndpoint) { + const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; + collectionsEndpoint = `${baseUrl}/collections`; + log.debug(`${indent}No collections link found, using fallback: ${collectionsEndpoint}`); + } + + // Persist collections endpoint in DB queue (batch-loaded into RAM) + try { + await db.enqueueCollectionUrl({ + sourceUrl: collectionsEndpoint, + crawllogCatalogId: crawllogCatalogId + }); + } catch (err) { + log.warning(`${indent}Failed to enqueue collections endpoint: ${err.message}`); + } + + // Also check for child links (nested catalogs) + if (typeof stacObj.getChildLinks === 'function') { + const childLinks = stacObj.getChildLinks(); + + if (childLinks.length > 0) { + log.info(`${indent}Found ${childLinks.length} child catalog links`); + + const nextDepth = depth + 1; + if (maxDepth > 0 && nextDepth > maxDepth) { + log.warning(`${indent}Skipping ${childLinks.length} child catalogs - max depth (${maxDepth}) reached`); + try { + await db.markCatalogCrawled(crawllogCatalogId); + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + return; + } + + const enqueuePromises = []; + childLinks + .map((link, idx) => { + let childUrl; + try { + childUrl = typeof link.getAbsoluteUrl === 'function' + ? link.getAbsoluteUrl() + : link.href; + } catch (err) { + log.warning(`${indent}Error getting URL for link ${idx}: ${err.message}`); + return null; + } + + // Handle S3 protocol URLs - convert to HTTPS + if (childUrl && typeof childUrl === 'string' && childUrl.startsWith('s3://')) { + const s3Match = childUrl.match(/^s3:\/\/([^/]+)\/(.*)$/); + if (s3Match) { + const [, bucket, path] = s3Match; + childUrl = `https://${bucket}.s3.amazonaws.com/${path}`; + log.debug(`${indent}Converted S3 URL: ${link.href} -> ${childUrl}`); + } else { + log.warning(`${indent}Skipping malformed S3 URL at index ${idx}: ${childUrl}`); + return null; + } + } + + // If URL is relative, make it absolute using the API URL + if (childUrl && typeof childUrl === 'string' && !childUrl.startsWith('http')) { + const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; + const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); + childUrl = `${basePath}/${childUrl}`; + } + + // Validate URL + if (!childUrl || typeof childUrl !== 'string' || !childUrl.startsWith('http')) { + log.warning(`${indent}Skipping invalid URL at index ${idx}: ${childUrl}`); + return null; + } + + enqueuePromises.push(db.enqueueCollectionUrl({ + sourceUrl: childUrl, + crawllogCatalogId: crawllogCatalogId + })); + + return childUrl; + }) + .filter(Boolean); + + if (enqueuePromises.length > 0) { + try { + await Promise.all(enqueuePromises); + log.info(`${indent}Queued ${enqueuePromises.length} child catalogs/collections into DB queue`); + } catch (err) { + log.warning(`${indent}Failed to enqueue child catalog/collection URLs: ${err.message}`); + } + } + } + } + + // Help garbage collector by dereferencing large objects + stacObj = null; + + try { + await db.markCatalogCrawled(crawllogCatalogId); + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } +} + +/** + * Handles individual API collection endpoint + * @async + */ +async function handleApiCollection({ request, json, crawler, log, indent, results }) { + const apiId = request.userData?.apiId || 'unknown'; + const apiSlug = request.userData?.apiSlug || request.userData?.catalogSlug || null; + const crawllogCatalogId = request.userData?.crawllogCatalogId || null; + + // Persist collection URL in crawllog_collection queue + try { + await db.enqueueCollectionUrl({ + sourceUrl: request.url, + crawllogCatalogId: crawllogCatalogId + }); + } catch (err) { + log.warning(`${indent}Failed to enqueue collection URL: ${err.message}`); + } + + // Check if this collection URL was already crawled (pause/resume support) + const alreadyCrawled = await db.isCollectionUrlCrawled(request.url); + if (alreadyCrawled) { + log.info(`${indent}Skipping already-crawled collection at ${request.url} (resume mode)`); + try { + await db.markCatalogCrawled(crawllogCatalogId); + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + return; + } + + let stacObj; + try { + stacObj = create(json, true); + + if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { + const collection = normalizeCollection(stacObj, results.collections.length); + // Add the API slug to the collection for unique stac_id generation + collection.sourceSlug = apiSlug; + // Mark as API collection + collection.is_api = true; + // Link to crawllog_catalog + collection.crawllogCatalogId = crawllogCatalogId; + // Store the crawled URL + collection.crawledUrl = request.url; + results.collections.push(collection); + results.stats.collectionsFound++; + log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + + await checkAndFlushApi(results, log); + } else { + log.warning(`${indent}Expected collection but got: ${json.type || 'unknown type'}`); + } + } catch (parseError) { + log.warning(`${indent}Skipping non-compliant STAC collection at ${request.url}`); + } + + // Help garbage collector + stacObj = null; + + try { + await db.markCatalogCrawled(crawllogCatalogId); + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } +} + +export { + crawlApis, + checkAndFlushApi, + BATCH_SIZE, + API_CLEAR_BATCH_SIZE +}; diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js new file mode 100644 index 0000000..ede62f2 --- /dev/null +++ b/crawler/catalogs/catalog.js @@ -0,0 +1,325 @@ +/** + * @fileoverview Catalog crawling functionality for STAC Index using Crawlee + * Supports parallel crawling of multiple domains simultaneously + * @module catalogs/catalog + */ + +import { HttpCrawler, log as crawleeLog, Configuration } from 'crawlee'; +import { handleCatalog, handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; +import { + groupByDomain, + executeWithConcurrency, + aggregateStats, + calculateRateLimits, + logDomainStats +} from '../utils/parallel.js'; +import globalStats from '../utils/globalStats.js'; +import { isShutdownRequested } from '../index.js'; +import db from '../utils/db.js'; + +/** + * Creates and runs a single Crawlee HttpCrawler for a specific domain + * @async + * @param {Array} catalogs - Array of catalog objects for this domain + * @param {string} domain - The domain being crawled + * @param {Object} config - Configuration object + * @returns {Promise} Crawl results with collections and statistics + */ +async function crawlSingleDomain(catalogs, domain, config = {}) { + // Set unique storage directory for this crawler to avoid conflicts + const safeDomain = domain.replace(/[^a-zA-Z0-9]/g, '_'); + const storageDir = `/tmp/crawlee-catalog-${safeDomain}-${Date.now()}-${Math.random().toString(36).slice(2)}`; + Configuration.getGlobalConfig().set('storageDir', storageDir); + Configuration.getGlobalConfig().set('persistStorage', false); + + const timeoutSecs = config.timeout && config.timeout !== Infinity + ? Math.ceil(config.timeout / 1000) + : 60; + + // Calculate rate limits for this domain + const rateLimits = calculateRateLimits(config.maxRequestsPerMinutePerDomain || 120); + + // Store results + const results = { + collections: [], + catalogs: [], + stats: { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0, + catalogsProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + } + }; + + const concurrency = config.maxConcurrencyPerDomain || 20; + + const DB_QUEUE_TARGET = 1000; + const DB_QUEUE_LOW_WATERMARK = 100; + const DB_QUEUE_BATCH_SIZE = 900; + const domainCatalogIds = catalogs.map(catalog => catalog.crawllogCatalogId).filter(Boolean); + + function getCatalogQueueLabel(url) { + if (typeof url === 'string' && /\/collections\/?$/.test(url)) { + return 'COLLECTIONS'; + } + return 'CATALOG'; + } + + async function ensureDbQueueBuffer(crawler, log) { + if (!crawler?.requestQueue?.getInfo) return; + + const info = await crawler.requestQueue.getInfo(); + const pending = info?.pendingRequestCount ?? 0; + + if (pending > DB_QUEUE_LOW_WATERMARK) return; + + const toFetch = Math.min(DB_QUEUE_BATCH_SIZE, Math.max(DB_QUEUE_TARGET - pending, 0)); + if (toFetch <= 0) return; + + const batch = await db.claimCollectionQueueBatch({ + limit: toFetch, + isApi: false, + crawllogCatalogIds: domainCatalogIds.length > 0 ? domainCatalogIds : undefined + }); + if (batch.length === 0) return; + + const requests = batch.map((item, idx) => ({ + url: item.url, + label: getCatalogQueueLabel(item.url), + userData: { + depth: 0, + catalogId: `queued-collection-${idx}`, + catalogSlug: item.slug || null, + crawllogCatalogId: item.crawllogCatalogId || null + } + })); + + await crawler.addRequests(requests); + log.info(`[QUEUE] Pulled ${requests.length} collection URLs from DB queue (pending: ${pending})`); + } + + const crawler = new HttpCrawler({ + requestHandlerTimeoutSecs: timeoutSecs, + + // Rate limiting + maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, + maxRequestRetries: config.maxRequestRetries || 3, + + // High concurrency for throughput + maxConcurrency: concurrency, + + // Reduce periodic statistics logging (we have our own end statistics) + statisticsOptions: { + logIntervalSecs: 60, + }, + + // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) + additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], + + async requestHandler({ request, json, body, crawler, log }) { + results.stats.totalRequests++; + globalStats.increment('totalRequests'); + const depth = request.userData?.depth || 0; + const indent = ' '.repeat(depth); + + // Fallback: manually parse JSON if Crawlee's automatic parsing failed + if (!json && body) { + try { + const bodyStr = typeof body === 'string' ? body : body.toString('utf8'); + json = JSON.parse(bodyStr); + log.debug(`${indent}Manually parsed JSON for ${request.url} (${bodyStr.length} bytes)`); + } catch (parseError) { + log.warning(`${indent}Failed to parse response body as JSON: ${parseError.message}`); + } + } + + try { + // Route based on request label + if (request.label === 'CATALOG') { + await handleCatalog({ request, json, crawler, log, indent, results, config }); + } else if (request.label === 'COLLECTIONS') { + await handleCollections({ request, json, crawler, log, indent, results }); + } + + results.stats.successfulRequests++; + globalStats.increment('successfulRequests'); + await ensureDbQueueBuffer(crawler, log); + } catch (error) { + log.error(`${indent}Error handling ${request.label} at ${request.url}: ${error.message}`); + throw error; + } + }, + + async failedRequestHandler({ request, error, log }) { + results.stats.failedRequests++; + globalStats.increment('failedRequests'); + const depth = request.userData?.depth || 0; + const indent = ' '.repeat(depth); + const catalogId = request.userData?.catalogId || 'unknown'; + + if (error.message.includes('STAC validation')) { + log.info(`${indent}[STAC VALIDATION FAILED] ${catalogId} at ${request.url}`); + log.info(`${indent} Reason: ${error.message}`); + results.stats.nonCompliant++; + globalStats.increment('nonCompliant'); + } else if (error.message.includes('timeout')) { + log.warning(`${indent}[TIMEOUT] ${catalogId} at ${request.url}`); + } else if (error.message.includes('ENOTFOUND') || error.message.includes('ECONNREFUSED')) { + log.warning(`${indent}[CONNECTION FAILED] ${catalogId} at ${request.url}`); + } else if (error.statusCode === 429) { + const retryAfter = error.response?.headers?.['retry-after'] || 'unknown'; + log.warning(`${indent}[RATE LIMITED] ${catalogId} at ${request.url} - Retry-After: ${retryAfter}s`); + } else if (error.code === 'ERR_NON_2XX_3XX_RESPONSE') { + log.warning(`${indent}[HTTP ERROR] ${catalogId} at ${request.url} - Status: ${error.statusCode}`); + } else { + log.warning(`${indent}[FAILED] ${catalogId} at ${request.url}`); + log.warning(`${indent} Error: ${error.message}`); + } + + } + }); + + // Seed the crawler with catalog requests for this domain + const initialRequests = catalogs + .filter(catalog => !catalog.hasPendingQueue) + .map(catalog => ({ + url: catalog.url, + label: 'CATALOG', + userData: { + depth: 0, + catalogId: catalog.id, + catalogTitle: catalog.title, + catalogSlug: catalog.slug, + crawllogCatalogId: catalog.crawllogCatalogId // Pass for linking collections to crawllog_catalog + } + })); + + await crawler.addRequests(initialRequests); + + await ensureDbQueueBuffer(crawler, crawleeLog); + + // Register domain as active in global stats + globalStats.domainStarted(domain); + + console.log(` [${domain}] Starting: ${initialRequests.length} catalogs, max ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent`); + await crawler.run(); + + // Flush any remaining collections to database + const finalFlush = await flushCollectionsToDb(results, crawleeLog, true); + results.stats.collectionsSaved += finalFlush.saved; + results.stats.collectionsFailed += finalFlush.failed; + + // Update global stats with final counts + globalStats.increment('collectionsSaved', results.stats.collectionsSaved); + globalStats.increment('collectionsFailed', results.stats.collectionsFailed); + globalStats.increment('collectionsFound', results.stats.collectionsFound); + globalStats.increment('catalogsProcessed', results.stats.catalogsProcessed); + globalStats.increment('stacCompliant', results.stats.stacCompliant); + + // Register domain as completed + globalStats.domainCompleted(domain); + + // Clear catalogs array to free memory + results.catalogs.length = 0; + + console.log(` [${domain}] Finished: ${results.stats.collectionsFound} collections, ${results.stats.successfulRequests}/${results.stats.totalRequests} requests`); + + return results; +} + +/** + * Creates and runs parallel Crawlee HttpCrawlers to crawl STAC catalogs + * Groups catalogs by domain and crawls multiple domains simultaneously + * @async + * @param {Array} initialCatalogs - Array of catalog objects to start crawling from + * @param {Object} config - Configuration object with timeout, depth, and parallel settings + * @returns {Promise} Crawl results with collections and statistics + */ +async function crawlCatalogs(initialCatalogs, config = {}) { + // Group catalogs by domain + const domainMap = groupByDomain(initialCatalogs); + + // Log domain distribution + logDomainStats(domainMap, 'catalogs'); + + // Number of domains to crawl in parallel (default: 5) + const parallelDomains = config.parallelDomains || 5; + const maxRequestsPerMinutePerDomain = config.maxRequestsPerMinutePerDomain || 120; + + console.log(`\n=== Parallel Catalog Crawling Configuration ===`); + console.log(`Parallel domains: ${parallelDomains}`); + console.log(`Max requests/min per domain: ${maxRequestsPerMinutePerDomain}`); + console.log(`Theoretical max throughput: ${parallelDomains * maxRequestsPerMinutePerDomain} req/min across all domains`); + console.log(`===============================================\n`); + + // Create tasks for each domain, with shutdown check + const domainTasks = Array.from(domainMap.entries()).map(([domain, catalogs]) => { + return async () => { + // Check if shutdown was requested before starting this domain + if (isShutdownRequested()) { + console.log(` [${domain}] Skipped (shutdown requested)`); + return { stats: { totalRequests: 0, successfulRequests: 0, failedRequests: 0, collectionsFound: 0, collectionsSaved: 0, collectionsFailed: 0, catalogsProcessed: 0, stacCompliant: 0, nonCompliant: 0 } }; + } + return crawlSingleDomain(catalogs, domain, config); + }; + }); + + console.log(`Starting parallel crawl of ${domainMap.size} domains (${parallelDomains} at a time)...\n`); + console.log(`Press Ctrl+C to pause (will stop after current batch and resume on next run)\n`); + + // Track total runtime for throughput calculation + const crawlStartTime = Date.now(); + + // Execute with concurrency limit + const allResults = await executeWithConcurrency( + domainTasks, + parallelDomains, + (completed, total) => { + if (isShutdownRequested()) { + console.log(`\n>>> Shutdown requested. Stopping after current domains complete... <<<\n`); + } else { + console.log(`\n>>> Domain progress: ${completed}/${total} domains completed <<<\n`); + } + } + ); + + const crawlEndTime = Date.now(); + const totalRuntimeMs = crawlEndTime - crawlStartTime; + const totalRuntimeMinutes = totalRuntimeMs / 60000; + + // Aggregate all statistics + const aggregatedStats = aggregateStats(allResults); + + // Calculate actual throughput + const requestsPerMinute = totalRuntimeMinutes > 0 + ? Math.round(aggregatedStats.totalRequests / totalRuntimeMinutes) + : 0; + + console.log('\n=== Catalog Crawl Statistics ==='); + console.log(` Domains Processed: ${domainMap.size}`); + console.log(` Total Runtime: ${Math.round(totalRuntimeMs / 1000)}s`); + console.log(` Total Requests: ${aggregatedStats.totalRequests}`); + console.log(` Requests/Min (actual): ${requestsPerMinute}`); + console.log(` Successful: ${aggregatedStats.successfulRequests}`); + console.log(` Failed: ${aggregatedStats.failedRequests}`); + console.log(` STAC Compliant: ${aggregatedStats.stacCompliant}`); + console.log(` Non-Compliant: ${aggregatedStats.nonCompliant}`); + console.log(` Catalogs Processed: ${aggregatedStats.catalogsProcessed}`); + console.log(` Collections Found: ${aggregatedStats.collectionsFound}`); + console.log(` Collections Saved to DB: ${aggregatedStats.collectionsSaved}`); + console.log(` Collections Failed: ${aggregatedStats.collectionsFailed}`); + console.log('=========================================\n'); + + return { + collections: [], + catalogs: [], + stats: aggregatedStats + }; +} + +export { crawlCatalogs }; diff --git a/crawler/docker-compose.yml b/crawler/docker-compose.yml new file mode 100644 index 0000000..f154ad5 --- /dev/null +++ b/crawler/docker-compose.yml @@ -0,0 +1,15 @@ +services: + crawler: + build: + context: . + dockerfile: Dockerfile + container_name: stac-crawler + # restart: unless-stopped + environment: + - NODE_ENV=production + networks: + - stac-network + +networks: + stac-network: + external: true diff --git a/crawler/index.js b/crawler/index.js new file mode 100644 index 0000000..f57c279 --- /dev/null +++ b/crawler/index.js @@ -0,0 +1,337 @@ +/** + * @fileoverview STAC Index API crawler that fetches and processes catalog data + * @module crawler + */ + +import axios from 'axios'; +import { processCatalogs } from './utils/normalization.js'; +import { crawlCatalogs } from './catalogs/catalog.js'; +import { crawlApis } from './apis/api.js'; +import { getConfig, isStaticCatalogUrl } from './utils/config.js'; +import { formatDuration } from './utils/time.js'; +import db from './utils/db.js'; +import globalStats from './utils/globalStats.js'; + +/** + * URL of the STAC Index API endpoint + * @type {string} + */ +const targetUrl = 'https://www.stacindex.org/api/catalogs'; + +/** + * Flag to track if shutdown was requested + */ +let shutdownRequested = false; + +/** + * Check if shutdown was requested (can be used by crawlers to stop early) + * @returns {boolean} True if shutdown was requested + */ +export function isShutdownRequested() { + return shutdownRequested; +} + +/** + * Request a graceful shutdown of the crawler + * The crawler will stop after completing the current batch + */ +export function requestShutdown() { + if (!shutdownRequested) { + console.log('\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.log('GRACEFUL SHUTDOWN REQUESTED'); + console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.log('The crawler will stop after the current batch completes.'); + console.log('Already-crawled collections are saved in crawllog_collection.'); + console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'); + shutdownRequested = true; + } +} + +/** + * Reset the shutdown flag (for scheduler to start a new crawl) + */ +export function resetShutdownFlag() { + shutdownRequested = false; +} + +/** + * Fetches catalog data from the STAC Index API and processes it + * @async + * @function crawler + * @returns {Promise} Returns statistics about the crawl including success status and runtime + */ +export const crawler = async () => { + // Start the timer + const startTime = Date.now(); + let dbError = false; + let crawlError = false; + + // Setup graceful shutdown handler + const shutdownHandler = async (signal) => { + if (shutdownRequested) { + console.log('\nForce shutdown requested. Exiting immediately...'); + process.exit(1); + } + + console.log(`\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`); + console.log(`PAUSE REQUESTED (${signal})`); + console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`); + console.log(`The crawler will stop after the current batch completes.`); + console.log(`Already-crawled collections are saved in crawllog_collection.`); + console.log(`Re-run the crawler to resume from where it left off.`); + console.log(`Press Ctrl+C again to force immediate exit.`); + console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n`); + + shutdownRequested = true; + }; + + process.on('SIGINT', shutdownHandler); + process.on('SIGTERM', shutdownHandler); + + try { + // Load configuration + const config = getConfig(); + + // Initialize database connection + try { + await db.initDb(); + } catch (err) { + console.error(`\nDatabase initialization failed: ${err.message}`); + dbError = true; + throw err; + } + + // Clear crawllog if fresh mode is enabled (allows re-crawling everything) + if (config.fresh) { + console.log('\n=== Fresh mode enabled - Clearing crawllog ==='); + try { + await db.clearCrawllogCollection(); + console.log('Crawllog collection entries cleared. All URLs will be re-crawled.'); + } catch (err) { + console.error(`Warning: Failed to clear crawllog: ${err.message}`); + } + } + + // Display configuration + console.log('\n=== STAC Crawler Configuration ==='); + console.log(`Mode: ${config.mode}`); + console.log(`Fresh Mode: ${config.fresh ? 'enabled (re-crawl everything)' : 'disabled (resume/skip crawled URLs)'}`); + console.log(`Max Catalogs: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs} (debugging limit)`); + console.log(`Max APIs: ${config.maxApis === 0 ? 'unlimited' : config.maxApis} (debugging limit)`); + console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); + console.log(`Max Depth: ${config.maxDepth === 0 ? 'unlimited' : config.maxDepth} levels`); + console.log('--- Parallel Crawling ---'); + console.log(`Parallel Domains: ${config.parallelDomains}`); + console.log(`Max Requests/Min per Domain: ${config.maxRequestsPerMinutePerDomain}`); + console.log(`Max Concurrency per Domain: ${config.maxConcurrencyPerDomain}`); + console.log(`Theoretical Max Throughput: ${config.parallelDomains * config.maxRequestsPerMinutePerDomain} req/min`); + console.log('==================================\n'); + + const response = await axios.get(targetUrl); + const catalogs = processCatalogs(response.data); + + // Save catalogs from STAC Index to crawllog_catalog table + // This creates the URL queue for re-crawling and stores the slug for stac_id generation + console.log('\n=== Saving catalogs to crawllog_catalog ==='); + let catalogsSaved = 0; + let catalogsFailed = 0; + + for (const catalog of catalogs) { + try { + const isApi = catalog.isApi === true && !isStaticCatalogUrl(catalog.url); + const crawllogId = await db.saveCrawllogCatalog({ + slug: catalog.slug, + url: catalog.url, + isApi: isApi + }); + console.log(`Saved: ${catalog.title || catalog.slug} (crawllog_id: ${crawllogId}, isApi: ${isApi})`); + catalogsSaved++; + } catch (err) { + console.error(`Failed: ${catalog.title || catalog.slug} - ${err.message}`); + catalogsFailed++; + } + } + + console.log(`\nCrawllog Catalogs: ${catalogsSaved} saved, ${catalogsFailed} failed\n`); + + // Now fetch the URL queue from crawllog_catalog for re-crawling + // This allows us to re-crawl existing catalogs without fetching from STAC Index again + console.log('\n=== Loading catalogs from crawllog_catalog for crawling ==='); + const crawllogCatalogs = await db.getCrawllogCatalogs({ isApi: false }); + const crawllogApis = await db.getCrawllogCatalogs({ isApi: true }); + const pendingCatalogIds = new Set(await db.getCrawllogCatalogIdsWithPendingQueue({ isApi: false })); + const pendingApiIds = new Set(await db.getCrawllogCatalogIdsWithPendingQueue({ isApi: true })); + + console.log(`Loaded ${crawllogCatalogs.length} catalogs and ${crawllogApis.length} APIs from crawllog_catalog\n`); + + // Merge original catalog metadata with crawllog entries for crawling + // We need the full catalog info (title, etc.) for processing + const catalogUrlMap = new Map(catalogs.map(c => [c.url, c])); + + const regularCatalogs = crawllogCatalogs.map(cl => { + const original = catalogUrlMap.get(cl.url) || {}; + return { + ...original, + id: cl.id, + slug: cl.slug, + url: cl.url, + crawllogCatalogId: cl.id, // Pass the crawllog_catalog id for linking + createdAt: cl.createdAt, + updatedAt: cl.updatedAt, + hasPendingQueue: pendingCatalogIds.has(cl.id) + }; + }); + + + const realApis = crawllogApis.map(cl => { + const original = catalogUrlMap.get(cl.url) || {}; + return { + ...original, + id: cl.id, + slug: cl.slug, + url: cl.url, + crawllogCatalogId: cl.id, // Pass the crawllog_catalog id for linking + createdAt: cl.createdAt, + updatedAt: cl.updatedAt, + hasPendingQueue: pendingApiIds.has(cl.id) + }; + }); + + + console.log(`\nCatalog Classification (from crawllog_catalog):`); + console.log(` Catalogs: ${regularCatalogs.length}`); + console.log(` APIs: ${realApis.length}\n`); + + // Start global statistics tracking (no periodic logging, only final stats) + const totalItems = [...regularCatalogs, ...realApis].length; + globalStats.start(totalItems); + + const shouldCrawlSeed = (seed) => { + if (seed.hasPendingQueue) return true; + if (!seed.createdAt || !seed.updatedAt) return true; + const createdAt = new Date(seed.createdAt).getTime(); + const updatedAt = new Date(seed.updatedAt).getTime(); + if (Number.isNaN(createdAt) || Number.isNaN(updatedAt)) return true; + return updatedAt <= createdAt; + }; + + // Crawl catalogs if mode is 'catalogs' or 'both' + if (config.mode === 'catalogs' || config.mode === 'both') { + console.log('\nCrawling collections and nested catalogs with Crawlee...\n'); + + const allCatalogsToProcess = regularCatalogs.filter(shouldCrawlSeed); + const skippedCatalogs = regularCatalogs.length - allCatalogsToProcess.length; + if (skippedCatalogs > 0) { + console.log(`Skipping ${skippedCatalogs} catalogs already fully crawled (no pending queue)`); + } + + // Note: MAX_CATALOGS limit is for debugging purposes only + // Set maxCatalogs to 0 or use --max-catalogs 0 for unlimited catalog crawling + const catalogsToProcess = config.maxCatalogs === 0 + ? allCatalogsToProcess + : allCatalogsToProcess.slice(0, config.maxCatalogs); + + console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs})\n`); + + try { + const results = await crawlCatalogs(catalogsToProcess, config); + console.log(`\nTotal collections found across all catalogs: ${results.stats.collectionsFound}`); + } catch (error) { + console.error(`Failed to crawl catalogs: ${error.message}`); + } + } else { + console.log('\nSkipping catalog crawling (mode: apis)\n'); + } + + // Crawl APIs if mode is 'apis' or 'both' + if (config.mode === 'apis' || config.mode === 'both') { + console.log('\nCrawling APIs...'); + // Pass full API objects (including slug and crawllogCatalogId) instead of just URLs + const apiObjects = realApis + .filter(shouldCrawlSeed) + .map(api => ({ + url: api.url, + slug: api.slug, + title: api.title, + crawllogCatalogId: api.crawllogCatalogId, // Link to crawllog_catalog for collections + hasPendingQueue: api.hasPendingQueue + })); + const skippedApis = realApis.length - apiObjects.length; + if (skippedApis > 0) { + console.log(`Skipping ${skippedApis} APIs already fully crawled (no pending queue)`); + } + + if (apiObjects.length > 0) { + // Note: MAX_APIS limit is for debugging purposes only + // Set maxApis to 0 or use --max-apis 0 for unlimited API crawling + const apisToProcess = config.maxApis === 0 ? apiObjects : apiObjects.slice(0, config.maxApis); + console.log(`Found ${apiObjects.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === 0 ? 'unlimited' : config.maxApis})...`); + + try { + await crawlApis(apisToProcess, true, config); + } catch (error) { + console.error(`Failed to crawl APIs: ${error.message}`); + } + } else { + console.log('No APIs found to crawl.'); + } + } else { + console.log('\nSkipping API crawling (mode: catalogs)\n'); + } + + } catch (error) { + console.error(`Error fetching ${targetUrl}: ${error.message}`); + if (!dbError) { + crawlError = true; + } + } finally { + // Stop global statistics tracking and log final stats + globalStats.stop(); + + // Deactivate collections that haven't been updated in the last 7 days + if (!dbError) { + try { + console.log('\nChecking for stale collections...'); + await db.deactivateStaleCollections(); + } catch (err) { + console.error(`Error deactivating stale collections: ${err.message}`); + } + } + + // Close database connection + if (!dbError) { + try { + await db.close(); + console.log('\nDatabase connection closed.'); + } catch (err) { + console.error(`Error closing database: ${err.message}`); + } + } + + // Display total running time + const endTime = Date.now(); + const elapsedTime = endTime - startTime; + + console.log('\n=== Crawler Time Statistics ==='); + console.log(`Total Running Time: ${formatDuration(elapsedTime)}`); + console.log(`Total Running Time (ms): ${elapsedTime}ms`); + console.log(`Status: ${dbError ? 'Database Error' : crawlError ? 'Crawl Error' : 'Success'}`); + console.log('================================\n'); + + // Return statistics + return { + success: !dbError && !crawlError, + dbError, + crawlError, + elapsedTime, + startTime, + endTime + }; + } +}; + +// Run crawler if this file is executed directly +const isMainModule = import.meta.url === `file://${process.argv[1]}`; +if (isMainModule || import.meta.url === `file:///${process.argv[1].replace(/\\/g, '/')}`) { + crawler(); +} \ No newline at end of file diff --git a/crawler/jest.config.js b/crawler/jest.config.js new file mode 100644 index 0000000..8c35124 --- /dev/null +++ b/crawler/jest.config.js @@ -0,0 +1,21 @@ +export default { + testEnvironment: 'node', + transform: {}, + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, + testMatch: [ + '**/__tests__/**/*.test.js' + ], + collectCoverageFrom: [ + 'utils/**/*.js', + 'apis/**/*.js', + 'catalogs/**/*.js', + '!**/node_modules/**', + '!**/__tests__/**' + ], + coveragePathIgnorePatterns: [ + '/node_modules/', + '/__tests__/' + ] +}; diff --git a/crawler/package-lock.json b/crawler/package-lock.json new file mode 100644 index 0000000..a56d247 --- /dev/null +++ b/crawler/package-lock.json @@ -0,0 +1,7552 @@ +{ + "name": "crawler", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "crawler", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@databases/pg": "^5.5.0", + "axios": "^1.13.2", + "crawlee": "^3.15.3", + "dotenv": "^17.2.3", + "stac-js": "^0.1.9", + "stac-node-validator": "^2.0.0-rc.1" + }, + "devDependencies": { + "jest": "^29.7.0" + } + }, + "node_modules/@apify/consts": { + "version": "2.48.0", + "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.48.0.tgz", + "integrity": "sha512-a0HeYDxAbbkRxc9z2N6beMFAmAJSgBw8WuKUwV+KmCuPyGUVLp54fYzjQ63p9Gv5IVFC88/HMXpAzI29ARgO5w==", + "license": "Apache-2.0" + }, + "node_modules/@apify/datastructures": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@apify/datastructures/-/datastructures-2.0.3.tgz", + "integrity": "sha512-E6yQyc/XZDqJopbaGmhzZXMJqwGf96ELtDANZa0t68jcOAJZS+pF7YUfQOLszXq6JQAdnRvTH2caotL6urX7HA==", + "license": "Apache-2.0" + }, + "node_modules/@apify/log": { + "version": "2.5.28", + "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.28.tgz", + "integrity": "sha512-jU8qIvU+Crek8glBjFl3INjJQWWDR9n2z9Dr0WvUI8KJi0LG9fMdTvV+Aprf9z1b37CbHXgiZkA1iPlNYxKOEQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.48.0", + "ansi-colors": "^4.1.1" + } + }, + "node_modules/@apify/ps-tree": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@apify/ps-tree/-/ps-tree-1.2.0.tgz", + "integrity": "sha512-VHIswI7rD/R4bToeIDuJ9WJXt+qr5SdhfoZ9RzdjmCs9mgy7l0P4RugQEUCcU+WB4sfImbd4CKwzXcn0uYx1yw==", + "license": "MIT", + "dependencies": { + "event-stream": "3.3.4" + }, + "bin": { + "ps-tree": "bin/ps-tree.js" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/@apify/pseudo_url": { + "version": "2.0.69", + "resolved": "https://registry.npmjs.org/@apify/pseudo_url/-/pseudo_url-2.0.69.tgz", + "integrity": "sha512-p/jZpaITBbFX8uVqz5MeY0uvOsMSV0SKbxrkTd8ZkmF8L7+LU93aOb/G/AnAEozgzjPV8Tf1ihkHnP2aY09y6Q==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.5.28" + } + }, + "node_modules/@apify/timeout": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@apify/timeout/-/timeout-0.3.2.tgz", + "integrity": "sha512-JnOLIOpqfm366q7opKrA6HrL0iYRpYYDn8Mi77sMR2GZ1fPbwMWCVzN23LJWfJV7izetZbCMrqRUXsR1etZ7dA==", + "license": "Apache-2.0" + }, + "node_modules/@apify/utilities": { + "version": "2.23.4", + "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.23.4.tgz", + "integrity": "sha512-1tLXOJBJR1SUSp/iEj6kcvV+9B5dn1mvIWDtRYwevJXXURyJdPwzJApi0F0DZz/Vk2HeCC381gnSqASzXN8MLA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.48.0", + "@apify/log": "^2.5.28" + } + }, + "node_modules/@asamuzakjp/css-color": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", + "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.3", + "@csstools/css-color-parser": "^3.0.9", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.28.6.tgz", + "integrity": "sha512-JYgintcMjRiCvS8mMECzaEn+m3PfoQiyqukOMCCVQtoJGYJw8j/8LBJEiqkHLkfwCcs74E3pbAUFNg7d9VNJ+Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.6.tgz", + "integrity": "sha512-2lfu57JtzctfIrcGMz992hyLlByuzgIk58+hhGCxjKZ3rWI82NnVLjXcaTqkI2NvlcvOskZaiZ5kjUALo3Lpxg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.6.tgz", + "integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/generator": "^7.28.6", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.6.tgz", + "integrity": "sha512-lOoVRwADj8hjf7al89tvQ2a1lf53Z+7tiXMgpZJL3maQPDxh0DgLMN62B2MKUOFcoodBHLMbDM6WAbKgNy5Suw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", + "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.28.6", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", + "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", + "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.28.6", + "@babel/helper-validator-identifier": "^7.28.5", + "@babel/traverse": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", + "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", + "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.6.tgz", + "integrity": "sha512-TeR9zWR18BvbfPmGbLampPMW+uW1NZnJlRuuHso8i87QZNq2JRF9i6RgxRqtEq+wQGsS19NNTWr2duhnE49mfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.6" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.28.6.tgz", + "integrity": "sha512-jiLC0ma9XkQT3TKJ9uYvlakm66Pamywo+qwL+oL8HJOvc6TWdZXVfhqJr8CCzbSGUAbDOzlGHJC1U+vRfLQDvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.28.6.tgz", + "integrity": "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.28.6.tgz", + "integrity": "sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.6.tgz", + "integrity": "sha512-fgWX62k02qtjqdSNTAGxmKYY/7FSL9WAS1o2Hu5+I5m9T0yxZzr4cnrfXQ/MX0rIifthCSs6FKTlzYbJcPtMNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/generator": "^7.28.6", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.6", + "@babel/template": "^7.28.6", + "@babel/types": "^7.28.6", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.6.tgz", + "integrity": "sha512-0ZrskXVEHSWIqZM/sQZ4EV3jZJXRkio/WCxaqKZP1g//CEWEPSfeZFcms4XeKBCHU0ZKnIkdJeU/kF+eRp5lBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@borewit/text-codec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.1.1.tgz", + "integrity": "sha512-5L/uBxmjaCIX5h8Z+uu+kA9BQLkc/Wl06UGR5ajNRxu+/XjonB5i8JpgFMrPj3LXTCPA0pv8yxUvbUi+QthGGA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@crawlee/basic": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.15.3.tgz", + "integrity": "sha512-+j0rhP16Gx84eFFXnG2t0YxmwkIwz5cWFnJ6CFyj1F7ElQ5JmVkzxyIWoyKBWCmLpPlafySht1tKq7L/4EZ1AQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "csv-stringify": "^6.2.0", + "fs-extra": "^11.0.0", + "got-scraping": "^4.0.0", + "ow": "^0.28.1", + "tldts": "^7.0.0", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/browser": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.15.3.tgz", + "integrity": "sha512-PtRzsurFO/A+puXg9oFUcP5LmEYNXkGyyQ2RQUJdg9exN1kbRwaaSrx4IUVi55waC1Z1Vkr5Ycq6nkVGTE6OcQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@crawlee/basic": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "ow": "^0.28.1", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/@crawlee/browser-pool": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.15.3.tgz", + "integrity": "sha512-a+QPQyHhLOO2cVzqjA8c9nuu++omzS9PWXRq248z46F+CnmAyYbClhuKiuBwhaNSTDKv7mgD8u1HikpzSN1duA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.0", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "fingerprint-generator": "^2.1.68", + "fingerprint-injector": "^2.1.68", + "lodash.merge": "^4.6.2", + "nanoid": "^3.3.4", + "ow": "^0.28.1", + "p-limit": "^3.1.0", + "proxy-chain": "^2.0.1", + "quick-lru": "^5.1.1", + "tiny-typed-emitter": "^2.1.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/@crawlee/cheerio": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.15.3.tgz", + "integrity": "sha512-yYbaUkV7meXtHLN9AW/Loo6BfZonp8ma2GvTZAlWXmQbiq3nmZ/npVWvR4UHWVDj0VsRe/IWsl8jgUzJFxD2/Q==", + "license": "Apache-2.0", + "dependencies": { + "@crawlee/http": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "cheerio": "1.0.0-rc.12", + "htmlparser2": "^9.0.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/cli": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.15.3.tgz", + "integrity": "sha512-cWo0NeF96WGO9sl5Q6BDFthvtqLky0CaCK2NtUvPJ7/EoXaiKm5D8o//lo1coMQmy72JEEgCGnSc/Xo8SDNUhA==", + "license": "Apache-2.0", + "dependencies": { + "@crawlee/templates": "3.15.3", + "ansi-colors": "^4.1.3", + "fs-extra": "^11.0.0", + "inquirer": "^8.2.4", + "tslib": "^2.4.0", + "yargonaut": "^1.1.4", + "yargs": "^17.5.1" + }, + "bin": { + "crawlee": "index.js" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/core": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.15.3.tgz", + "integrity": "sha512-cBglpY4KVlKnozeO8K4lw6/TDWajtJjPj4aNfckxUeEzapJgvTaxg6ZI4zxir6vic8sTeK9Olp3qG3wLnlrtXw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.20.0", + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@apify/pseudo_url": "^2.0.30", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/memory-storage": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@sapphire/async-queue": "^1.5.1", + "@vladfrangu/async_event_emitter": "^2.2.2", + "csv-stringify": "^6.2.0", + "fs-extra": "^11.0.0", + "got-scraping": "^4.0.0", + "json5": "^2.2.3", + "minimatch": "^9.0.0", + "ow": "^0.28.1", + "stream-json": "^1.8.0", + "tldts": "^7.0.0", + "tough-cookie": "^6.0.0", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/http": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.15.3.tgz", + "integrity": "sha512-NvD9khVsji6gX/t1YNBgTSlVCMRgzVhkL/oygyXPjfihfX42adAWJGi/ztK5/drA+7nNZlGY304W9Kheo5SqCQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/basic": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@types/content-type": "^1.1.5", + "cheerio": "1.0.0-rc.12", + "content-type": "^1.0.4", + "got-scraping": "^4.0.0", + "iconv-lite": "^0.7.0", + "mime-types": "^2.1.35", + "ow": "^0.28.1", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/jsdom": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.15.3.tgz", + "integrity": "sha512-SmsJcaLW12C35Myy2e0jdZpG1HhbAA/QwF+P1Op0AB0lerDYT8sGQJXARczLJceu+zhZeM/90g1oRuH8N3wB7g==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/http": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@types/jsdom": "^21.0.0", + "cheerio": "1.0.0-rc.12", + "jsdom": "^26.0.0", + "ow": "^0.28.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/linkedom": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.15.3.tgz", + "integrity": "sha512-pCmfjMuRDAdDqJiHL/Ph9rOZC9I4tFxXhveNUS0X3suOj/5y67m5t9CsV6Bv3XuwAEVXDc8MNOCz0FUN9dl3jw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/http": "3.15.3", + "@crawlee/types": "3.15.3", + "linkedom": "^0.18.0", + "ow": "^0.28.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/memory-storage": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.15.3.tgz", + "integrity": "sha512-iOUOGBTZNyl2srDsrAJwhYu4+leOxQSlx9uAtGt88kC7srlrn2B/OgXjhzTv0Vo7+kkAiTkxUMAfZ2eNW+UbSw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@crawlee/types": "3.15.3", + "@sapphire/async-queue": "^1.5.0", + "@sapphire/shapeshift": "^3.0.0", + "content-type": "^1.0.4", + "fs-extra": "^11.0.0", + "json5": "^2.2.3", + "mime-types": "^2.1.35", + "proper-lockfile": "^4.1.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">= 16" + } + }, + "node_modules/@crawlee/playwright": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.15.3.tgz", + "integrity": "sha512-PTIqiE0gTdIBJIJ9GC7VZYSOjyMNjg156nhsLKoJJK3dT/M0dKhoM36zgchghagcjuP+fLZYmx+TMDgpAfWfxQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.1", + "@crawlee/browser": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "cheerio": "1.0.0-rc.12", + "idcac-playwright": "^0.1.2", + "jquery": "^3.6.0", + "lodash.isequal": "^4.5.0", + "ml-logistic-regression": "^2.0.0", + "ml-matrix": "^6.11.0", + "ow": "^0.28.1", + "string-comparison": "^1.3.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + } + } + }, + "node_modules/@crawlee/puppeteer": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.15.3.tgz", + "integrity": "sha512-hiNrXwCPLaEEqejlXPWf567KnArwhZx4HHs16YqiB6wElf2eptvPO6jdeAnQX7BXyV3NWP4QPVKyieOXa/d51A==", + "license": "Apache-2.0", + "dependencies": { + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@crawlee/browser": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "cheerio": "1.0.0-rc.12", + "devtools-protocol": "*", + "idcac-playwright": "^0.1.2", + "jquery": "^3.6.0", + "ow": "^0.28.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "puppeteer": { + "optional": true + } + } + }, + "node_modules/@crawlee/templates": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.15.3.tgz", + "integrity": "sha512-7VKwdKYFf8yF4uaZU626cdZDpVQs5jv9bK//q94JK5IzpRdkwRedD2N93fYBrGVYyGqNhlEJz1nEIdAe+d6Knw==", + "license": "Apache-2.0", + "dependencies": { + "ansi-colors": "^4.1.3", + "inquirer": "^9.0.0", + "tslib": "^2.4.0", + "yargonaut": "^1.1.4", + "yargs": "^17.5.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/templates/node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/@crawlee/templates/node_modules/inquirer": { + "version": "9.3.8", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.3.8.tgz", + "integrity": "sha512-pFGGdaHrmRKMh4WoDDSowddgjT1Vkl90atobmTeSmcPGdYiwikch/m/Ef5wRaiamHejtw0cUUMMerzDUXCci2w==", + "license": "MIT", + "dependencies": { + "@inquirer/external-editor": "^1.0.2", + "@inquirer/figures": "^1.0.3", + "ansi-escapes": "^4.3.2", + "cli-width": "^4.1.0", + "mute-stream": "1.0.0", + "ora": "^5.4.1", + "run-async": "^3.0.0", + "rxjs": "^7.8.1", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@crawlee/templates/node_modules/mute-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/@crawlee/templates/node_modules/run-async": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", + "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/@crawlee/templates/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@crawlee/types": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.15.3.tgz", + "integrity": "sha512-RvgVPXrsQw4GQIUXrC1z1aNOedUPJnZ/U/8n+jZ0fu1Iw9moJVMuiuIxSI8q1P6BA84aWZdalyfDWBZ3FMjsiw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/utils": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.15.3.tgz", + "integrity": "sha512-guldTIfG+No6zoNmi5CKwABJDnrN8NqgwB9PFMR8kD+5r//TPFENfU9I3w4tQXx/pefnSZ99JrVZMUL3zenpJA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/ps-tree": "^1.2.0", + "@crawlee/types": "3.15.3", + "@types/sax": "^1.2.7", + "cheerio": "1.0.0-rc.12", + "file-type": "^20.0.0", + "got-scraping": "^4.0.3", + "ow": "^0.28.1", + "robots-parser": "^3.0.1", + "sax": "^1.4.1", + "tslib": "^2.4.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", + "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", + "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", + "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.1.0", + "@csstools/css-calc": "^2.1.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", + "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", + "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@databases/connection-pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@databases/connection-pool/-/connection-pool-1.1.0.tgz", + "integrity": "sha512-/12/SNgl0V77mJTo5SX3yGPz4c9XGQwAlCfA0vlfs/0HcaErNpYXpmhj0StET07w6TmTJTnaUgX2EPcQK9ez5A==", + "license": "MIT", + "dependencies": { + "@databases/queue": "^1.0.0", + "is-promise": "^4.0.0" + } + }, + "node_modules/@databases/escape-identifier": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@databases/escape-identifier/-/escape-identifier-1.0.3.tgz", + "integrity": "sha512-Su36iSVzaHxpVdISVMViUX/32sLvzxVgjZpYhzhotxZUuLo11GVWsiHwqkvUZijTLUxcDmUqEwGJO3O/soLuZA==", + "license": "MIT", + "dependencies": { + "@databases/validate-unicode": "^1.0.0" + } + }, + "node_modules/@databases/lock": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@databases/lock/-/lock-2.1.0.tgz", + "integrity": "sha512-ReWnFE5qeCuO2SA5h5fDh/hE/vMolA+Epe6xkAQP1FL2nhnsTCYwN2JACk/kWctR4OQoh0njBjPZ0yfIptclcA==", + "license": "MIT", + "dependencies": { + "@databases/queue": "^1.0.0" + } + }, + "node_modules/@databases/pg": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@databases/pg/-/pg-5.5.0.tgz", + "integrity": "sha512-WIojK9AYIlNi5YRfc5YUOow3PQ82ClmwT9HG3nEsKLUERYieoVmHMYDQLS0ry6FjgJx+2yFs7LCw4kZpWu1TBw==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "@databases/escape-identifier": "^1.0.3", + "@databases/pg-config": "^3.2.0", + "@databases/pg-connection-string": "^1.0.0", + "@databases/pg-data-type-id": "^3.0.0", + "@databases/pg-errors": "^1.0.0", + "@databases/push-to-async-iterable": "^3.0.0", + "@databases/shared": "^3.1.0", + "@databases/split-sql-query": "^1.0.4", + "@databases/sql": "^3.3.0", + "assert-never": "^1.2.1", + "pg": "^8.4.2", + "pg-cursor": "^2.4.2" + } + }, + "node_modules/@databases/pg-config": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@databases/pg-config/-/pg-config-3.4.0.tgz", + "integrity": "sha512-4dYiTbHjzyQfEfaIGkh3uCBNBRWPs5Jcws94cFagLAGnjO/TcghC7oexzC81+bIADLDlpCw7DEWJAK/gNSQxkw==", + "license": "MIT", + "dependencies": { + "cosmiconfig": "^8.1.0", + "funtypes": "^4.1.0" + } + }, + "node_modules/@databases/pg-connection-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@databases/pg-connection-string/-/pg-connection-string-1.0.0.tgz", + "integrity": "sha512-8czOF9jlv7PlS7BPjnL82ynpDs1t8cu+C2jvdtMr37e8daPKMS7n1KfNE9xtr2Gq4QYKjynep097eYa5yIwcLA==", + "license": "MIT" + }, + "node_modules/@databases/pg-data-type-id": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@databases/pg-data-type-id/-/pg-data-type-id-3.0.0.tgz", + "integrity": "sha512-VqW1csN8pRsWJxjPsGIC9FQ8wyenfmGv0P//BaeDMAu/giM3IXKxKM8fkScUSQ00uqFK/L1iHS5g6dgodF3XzA==", + "license": "MIT" + }, + "node_modules/@databases/pg-errors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@databases/pg-errors/-/pg-errors-1.0.0.tgz", + "integrity": "sha512-Yz3exbptZwOn4ZD/MSwY6z++XVyOFsMh5DERvSw3awRwJFnfdaqdeiIxxX0MVjM6KPihF0xxp8lPO7vTc5ydpw==", + "license": "MIT" + }, + "node_modules/@databases/push-to-async-iterable": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@databases/push-to-async-iterable/-/push-to-async-iterable-3.0.0.tgz", + "integrity": "sha512-xwu/yNgINdMU+fn6UwFsxh+pa6UrVPafY+0qm0RK0/nKyjllfDqSbwK4gSmdmLEwPYxKwch9CAE3P8NxN1hPSg==", + "license": "MIT", + "dependencies": { + "@databases/queue": "^1.0.0" + } + }, + "node_modules/@databases/queue": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@databases/queue/-/queue-1.0.1.tgz", + "integrity": "sha512-dqRU+/aQ4lhFzjPIkIhjB0+UEKMb76FoBgHOJUTcEblgatr/IhdhHliT3VVwcImXh35Mz297PAXE4yFM4eYWUQ==", + "license": "MIT" + }, + "node_modules/@databases/shared": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@databases/shared/-/shared-3.1.0.tgz", + "integrity": "sha512-bO1DIYAYDiWOCqVPvBio1JqZQYh4dph2M1av2w/REeFT6WBd64mTrOFlcxKV0CUAYT0UiJsDfPqEfw0/APRzWg==", + "license": "MIT", + "dependencies": { + "@databases/connection-pool": "^1.1.0", + "@databases/lock": "^2.1.0", + "@databases/queue": "^1.0.1", + "@databases/split-sql-query": "^1.0.4", + "@databases/sql": "^3.3.0" + } + }, + "node_modules/@databases/split-sql-query": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@databases/split-sql-query/-/split-sql-query-1.0.4.tgz", + "integrity": "sha512-lDqDQvH34NNjLs0knaDvL6HKgPtishQlDYHfOkvbAd5VQOEhcDvvmG2zbBuFvS2HQAz5NsyLj5erGaxibkxhvQ==", + "license": "MIT", + "peerDependencies": { + "@databases/sql": "*" + } + }, + "node_modules/@databases/sql": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@databases/sql/-/sql-3.3.0.tgz", + "integrity": "sha512-vj9huEy4mjJ48GS1Z8yvtMm4BYAnFYACUds25ym6Gd/gsnngkJ17fo62a6mmbNNwCBS/8467PmZR01Zs/06TjA==", + "license": "MIT" + }, + "node_modules/@databases/validate-unicode": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@databases/validate-unicode/-/validate-unicode-1.0.0.tgz", + "integrity": "sha512-dLKqxGcymeVwEb/6c44KjOnzaAafFf0Wxa8xcfEjx/qOl3rdijsKYBAtIGhtVtOlpPf/PFKfgTuFurSPn/3B/g==", + "license": "MIT" + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", + "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.15.tgz", + "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/source-map": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "license": "MIT" + }, + "node_modules/@multiformats/base-x": { + "version": "4.0.1", + "license": "MIT" + }, + "node_modules/@radiantearth/stac-migrate": { + "version": "2.0.2", + "license": "Apache-2.0", + "dependencies": { + "compare-versions": "^3.6.0", + "multihashes": "^3.1.2", + "yargs": "^17.6.2" + }, + "bin": { + "stac-migrate": "bin/cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/m-mohr" + } + }, + "node_modules/@sapphire/async-queue": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@sapphire/async-queue/-/async-queue-1.5.5.tgz", + "integrity": "sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==", + "license": "MIT", + "engines": { + "node": ">=v14.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@sapphire/shapeshift": { + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-3.9.7.tgz", + "integrity": "sha512-4It2mxPSr4OGn4HSQWGmhFMsNFGfFVhWeRPCRwbH972Ek2pzfGRZtb0pJ4Ze6oIzcyh2jw7nUDa6qGlWofgd9g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=v16" + } + }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "license": "MIT" + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.1.1.tgz", + "integrity": "sha512-rO92VvpgMc3kfiTjGT52LEtJ8Yc5kCWhZjLQ3LwlA4pSgPpQO7bVpYXParOD8Jwf+cVQECJo3yP/4I8aZtUQTQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@tokenizer/inflate": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", + "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "fflate": "^0.8.2", + "token-types": "^6.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT" + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/content-type": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@types/content-type/-/content-type-1.1.9.tgz", + "integrity": "sha512-Hq9IMnfekuOCsEmYl4QX2HBrT+XsfXiupfrLLY8Dcf3Puf4BkBOxSbWYTITSOQAhJoYPBez+b4MJRpIYL65z8A==", + "license": "MIT" + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/http-cache-semantics": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", + "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/jsdom": { + "version": "21.1.7", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.7.tgz", + "integrity": "sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^7.0.0" + } + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, + "node_modules/@types/yargs": { + "version": "17.0.35", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", + "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vladfrangu/async_event_emitter": { + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/@vladfrangu/async_event_emitter/-/async_event_emitter-2.4.7.tgz", + "integrity": "sha512-Xfe6rpCTxSxfbswi/W/Pz7zp1WWSNn4A0eW4mLkQUewCrXXtMj31lCg+iQyTkh/CkusZSq9eDflu7tjEDXUY6g==", + "license": "MIT", + "engines": { + "node": ">=v14.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@zxing/text-encoding": { + "version": "0.9.0", + "license": "(Unlicense OR Apache-2.0)", + "optional": true + }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/assert-never": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/assert-never/-/assert-never-1.4.0.tgz", + "integrity": "sha512-5oJg84os6NMQNl27T9LnZkvvqzvAnHu03ShCnoj6bsJwS7L8AO4lf+C/XjK/nvzEqQB744moC6V128RucQd1jA==", + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "license": "MIT" + }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/axios": { + "version": "1.13.2", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.4", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", + "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0 || ^8.0.0-0" + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.8.31", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.31.tgz", + "integrity": "sha512-a28v2eWrrRWPpJSzxc+mKwm0ZtVx/G8SepdQZDArnXYU/XS+IF6mp8aB/4E+hH1tyGCoDo3KlUCdlSxGDsRkAw==", + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.1.4" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/byte-counter": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/byte-counter/-/byte-counter-0.1.0.tgz", + "integrity": "sha512-jheRLVMeUKrDBjVw2O5+k4EvR4t9wtxHL+bo/LxfkxsVeuGMy3a5SEGgXdAFA4FSzTrU8rQXQIrsZ3oBq5a0pQ==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cacheable-lookup": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", + "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + } + }, + "node_modules/cacheable-request": { + "version": "13.0.15", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-13.0.15.tgz", + "integrity": "sha512-NjiSrjv37X73FmGGU5ec/M83vWQ6q1Ae3BFe+ABfdeeMy4LOMKYTpfEjrBnLedu43clKZtsYbKrHTIQE7vKq+A==", + "license": "MIT", + "dependencies": { + "@types/http-cache-semantics": "^4.0.4", + "get-stream": "^9.0.1", + "http-cache-semantics": "^4.2.0", + "keyv": "^5.5.4", + "mimic-response": "^4.0.0", + "normalize-url": "^8.1.0", + "responselike": "^4.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/call-bind": { + "version": "1.0.8", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-define-property": "^1.0.0", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001757", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001757.tgz", + "integrity": "sha512-r0nnL/I28Zi/yjk1el6ilj27tKcdjLsNqAOZr0yVjWPrSQyHgKI2INaEWw21bAQSv2LXRt1XuCS/GomNpWOxsQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "license": "MIT" + }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cheerio/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "license": "ISC", + "engines": { + "node": ">= 10" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", + "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/color-convert": { + "version": "2.0.1", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", + "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/compare-versions": { + "version": "3.6.0", + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "license": "MIT", + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/crawlee": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.15.3.tgz", + "integrity": "sha512-l+l1Fs4fEKUqKn9Vuw+tHiraWIVbRpSXFa09JeTdZID/xUlPHVLkKrqGLNa0cvZc7dqX2s9+1xLXid+pRn851w==", + "license": "Apache-2.0", + "dependencies": { + "@crawlee/basic": "3.15.3", + "@crawlee/browser": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/cheerio": "3.15.3", + "@crawlee/cli": "3.15.3", + "@crawlee/core": "3.15.3", + "@crawlee/http": "3.15.3", + "@crawlee/jsdom": "3.15.3", + "@crawlee/linkedom": "3.15.3", + "@crawlee/playwright": "3.15.3", + "@crawlee/puppeteer": "3.15.3", + "@crawlee/utils": "3.15.3", + "import-local": "^3.1.0", + "tslib": "^2.4.0" + }, + "bin": { + "crawlee": "cli.js" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" + }, + "node_modules/cssstyle": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", + "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^3.2.0", + "rrweb-cssom": "^0.8.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/csv-stringify": { + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.6.0.tgz", + "integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==", + "license": "MIT" + }, + "node_modules/data-urls": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "license": "MIT" + }, + "node_modules/decompress-response": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-10.0.0.tgz", + "integrity": "sha512-oj7KWToJuuxlPr7VV0vabvxEIiqNMo+q0NueIiL3XhtwC6FVOX7Hr1c0C4eD0bmf7Zr+S/dSf2xvkH3Ad6sU3Q==", + "license": "MIT", + "dependencies": { + "mimic-response": "^4.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dedent": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.1.tgz", + "integrity": "sha512-9JmrhGZpOlEgOLdQgSm0zxFaYoQon408V1v49aqTWuXENVlnCuY9JBZcXZiCsZQWDjTm5Qf/nIvAy77mXDAjEg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1551306", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1551306.tgz", + "integrity": "sha512-CFx8QdSim8iIv+2ZcEOclBKTQY6BI1IEDa7Tm9YkwAXzEWFndTEzpTo5jAUhSnq24IC7xaDw0wvGcm96+Y3PEg==", + "license": "BSD-3-Clause" + }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "license": "MIT", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.262", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.262.tgz", + "integrity": "sha512-NlAsMteRHek05jRUxUR0a5jpjYq9ykk6+kO0yRaMi5moe7u0fVIOeQ3Y30A8dIiWFBNUoQGi1ljb1i5VtS9WQQ==", + "license": "ISC" + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "license": "MIT" + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/event-stream": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "integrity": "sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==", + "license": "MIT", + "dependencies": { + "duplexer": "~0.1.1", + "from": "~0", + "map-stream": "~0.1.0", + "pause-stream": "0.0.11", + "split": "0.3", + "stream-combiner": "~0.0.4", + "through": "~2.3.1" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/execa/node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, + "node_modules/figlet": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/figlet/-/figlet-1.9.4.tgz", + "integrity": "sha512-uN6QE+TrzTAHC1IWTyrc4FfGo2KH/82J8Jl1tyKB7+z5DBit/m3D++Iu5lg91qJMnQQ3vpJrj5gxcK/pk4R9tQ==", + "license": "MIT", + "dependencies": { + "commander": "^14.0.0" + }, + "bin": { + "figlet": "bin/index.js" + }, + "engines": { + "node": ">= 17.0.0" + } + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/file-type": { + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.5.0.tgz", + "integrity": "sha512-BfHZtG/l9iMm4Ecianu7P8HRD2tBHLtjXinm4X62XBOYzi7CYA7jyqfJzOvXHqzVrVPYqBo2/GvbARMaaJkKVg==", + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.2.6", + "strtok3": "^10.2.0", + "token-types": "^6.0.0", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fingerprint-generator": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.1.77.tgz", + "integrity": "sha512-wR15VUEZnwozFiSDRV+40zxlEt3ZV3JNYvLx0CSF9D9smov4pUC6MJZJnlxtDr+Ir4oppU8vn1JXApLk/Qr5Uw==", + "license": "Apache-2.0", + "dependencies": { + "generative-bayesian-network": "^2.1.77", + "header-generator": "^2.1.77", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/fingerprint-injector": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.1.77.tgz", + "integrity": "sha512-R778SIyrqgWO0P+UWKzIFWUWZz13EGu6UmV7CX3vuFDbsYIL1xiH+s+/nzPSOqFdhXyLo7d8aTOjbGbRLULoQQ==", + "license": "Apache-2.0", + "dependencies": { + "fingerprint-generator": "^2.1.77", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "^1.22.2", + "puppeteer": ">= 9.x" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/for-each": { + "version": "0.3.5", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.1.0.tgz", + "integrity": "sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", + "license": "MIT" + }, + "node_modules/fs-extra": { + "version": "11.3.2", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.2.tgz", + "integrity": "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/funtypes": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/funtypes/-/funtypes-4.2.0.tgz", + "integrity": "sha512-DvOtjiKvkeuXGV0O8LQh9quUP3bSOTEQPGv537Sao8kDq2rDbg48UsSJ7wlBLPzR2Mn0pV7cyAiq5pYG1oUyCQ==", + "license": "MIT" + }, + "node_modules/generative-bayesian-network": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.77.tgz", + "integrity": "sha512-viU4CRPsmgiklR94LhvdMndaY73BkCH1pGjmOjWbLR/ZwcUd06gKF3TCcsS3npRl74o33YSInSixxm16wIukcA==", + "license": "Apache-2.0", + "dependencies": { + "adm-zip": "^0.5.9", + "tslib": "^2.4.0" + } + }, + "node_modules/generator-function": { + "version": "2.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "license": "MIT", + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/got": { + "version": "14.6.5", + "resolved": "https://registry.npmjs.org/got/-/got-14.6.5.tgz", + "integrity": "sha512-Su87c0NNeg97de1sO02gy9I8EmE7DCJ1gzcFLcgGpYeq2PnLg4xz73MWrp6HjqbSsjb6Glf4UBDW6JNyZA6uSg==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^7.0.1", + "byte-counter": "^0.1.0", + "cacheable-lookup": "^7.0.0", + "cacheable-request": "^13.0.12", + "decompress-response": "^10.0.0", + "form-data-encoder": "^4.0.2", + "http2-wrapper": "^2.2.1", + "keyv": "^5.5.3", + "lowercase-keys": "^3.0.0", + "p-cancelable": "^4.0.1", + "responselike": "^4.0.2", + "type-fest": "^4.26.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, + "node_modules/got-scraping": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/got-scraping/-/got-scraping-4.1.2.tgz", + "integrity": "sha512-LtVwPM5YLnNY7HVT/AK/yDBUg/4yOZSlAjjug2ovrHQseS43QCmO1XosKKXcXrfc6OMX8OnDbAWIauFMcaJ5TQ==", + "license": "Apache-2.0", + "dependencies": { + "got": "^14.2.1", + "header-generator": "^2.1.41", + "http2-wrapper": "^2.2.0", + "mimic-response": "^4.0.0", + "ow": "^1.1.1", + "quick-lru": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/got-scraping/node_modules/@sindresorhus/is": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", + "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/got-scraping/node_modules/callsites": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.2.0.tgz", + "integrity": "sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/dot-prop": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-7.2.0.tgz", + "integrity": "sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==", + "license": "MIT", + "dependencies": { + "type-fest": "^2.11.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/ow": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ow/-/ow-1.1.1.tgz", + "integrity": "sha512-sJBRCbS5vh1Jp9EOgwp1Ws3c16lJrUkJYlvWTYC03oyiYVwS/ns7lKRWow4w4XjDyTrA2pplQv4B2naWSR6yDA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^5.3.0", + "callsites": "^4.0.0", + "dot-prop": "^7.2.0", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/quick-lru": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz", + "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-ansi/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/header-generator": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.77.tgz", + "integrity": "sha512-ggSG/mfkFMu8CO7xP591G8kp1IJCBvgXu7M8oxTjC9u914JsIzE6zIfoFsXzA+pf0utWJhUsdqU0oV/DtQ4DFQ==", + "license": "Apache-2.0", + "dependencies": { + "browserslist": "^4.21.1", + "generative-bayesian-network": "^2.1.77", + "ow": "^0.28.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, + "node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http2-wrapper": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", + "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", + "license": "MIT", + "dependencies": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.2.0" + }, + "engines": { + "node": ">=10.19.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/idcac-playwright": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/idcac-playwright/-/idcac-playwright-0.1.3.tgz", + "integrity": "sha512-VVYQ4sv6OrUJKVzYaIP1hq0qAHd1O22HW5LnL1Wf6zkrLStQ/QEg4iJ0rllIOEpd+Rmm+635AJD59A+Vw+2PgQ==", + "license": "ISC" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "license": "ISC" + }, + "node_modules/inquirer": { + "version": "8.2.7", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz", + "integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==", + "license": "MIT", + "dependencies": { + "@inquirer/external-editor": "^1.0.0", + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.1", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "figures": "^3.0.0", + "lodash": "^4.17.21", + "mute-stream": "0.0.8", + "ora": "^5.4.1", + "run-async": "^2.4.0", + "rxjs": "^7.5.5", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6", + "wrap-ansi": "^6.0.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/inquirer/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-any-array": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", + "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==", + "license": "MIT" + }, + "node_modules/is-arguments": { + "version": "1.2.0", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-callable": { + "version": "1.2.7", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-generator-function": { + "version": "1.1.2", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "license": "MIT" + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-regex": { + "version": "1.2.1", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-reports/node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "execa": "^5.0.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-circus": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-config": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", + "parse-json": "^5.2.0", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/jquery": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.7.1.tgz", + "integrity": "sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==", + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsdom": { + "version": "26.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz", + "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", + "license": "MIT", + "dependencies": { + "cssstyle": "^4.2.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.5.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.16", + "parse5": "^7.2.1", + "rrweb-cssom": "^0.8.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.1.1", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.1.1", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jsdom/node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/jsdom/node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "license": "MIT" + }, + "node_modules/jsdom/node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/keyv": { + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.5.4.tgz", + "integrity": "sha512-eohl3hKTiVyD1ilYdw9T0OiB4hnjef89e3dMYKz+mVKDzj+5IteTseASUsOB+EU9Tf6VNTCjDePcP6wkDGmLKQ==", + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/klaw": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-4.1.0.tgz", + "integrity": "sha512-1zGZ9MF9H22UnkpVeuaGKOjfA2t6WrfdrJmGjy16ykcjnKQDmHVX+KI477rpbGevz/5FD4MC3xf1oxylBgcaQw==", + "license": "MIT", + "engines": { + "node": ">=14.14.0" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/linkedom": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz", + "integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==", + "license": "ISC", + "dependencies": { + "css-select": "^5.1.0", + "cssom": "^0.5.0", + "html-escaper": "^3.0.3", + "htmlparser2": "^10.0.0", + "uhyphen": "^0.2.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "canvas": ">= 2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/linkedom/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/linkedom/node_modules/htmlparser2": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", + "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.1", + "entities": "^6.0.0" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lowercase-keys": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", + "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/map-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", + "integrity": "sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", + "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ml-array-max": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-1.2.4.tgz", + "integrity": "sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0" + } + }, + "node_modules/ml-array-min": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-1.2.3.tgz", + "integrity": "sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0" + } + }, + "node_modules/ml-array-rescale": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-1.3.7.tgz", + "integrity": "sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0", + "ml-array-max": "^1.2.4", + "ml-array-min": "^1.2.3" + } + }, + "node_modules/ml-logistic-regression": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-logistic-regression/-/ml-logistic-regression-2.0.0.tgz", + "integrity": "sha512-xHhB91ut8GRRbJyB1ZQfKsl1MHmE1PqMeRjxhks96M5BGvCbC9eEojf4KgRMKM2LxFblhVUcVzweAoPB48Nt0A==", + "license": "MIT", + "dependencies": { + "ml-matrix": "^6.5.0" + } + }, + "node_modules/ml-matrix": { + "version": "6.12.1", + "resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.12.1.tgz", + "integrity": "sha512-TJ+8eOFdp+INvzR4zAuwBQJznDUfktMtOB6g/hUcGh3rcyjxbz4Te57Pgri8Q9bhSQ7Zys4IYOGhFdnlgeB6Lw==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.1", + "ml-array-rescale": "^1.3.7" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/multibase": { + "version": "3.1.2", + "license": "MIT", + "dependencies": { + "@multiformats/base-x": "^4.0.1", + "web-encoding": "^1.0.6" + }, + "engines": { + "node": ">=10.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/multiformats": { + "version": "9.9.0", + "license": "(Apache-2.0 AND MIT)" + }, + "node_modules/multihashes": { + "version": "3.1.2", + "license": "MIT", + "dependencies": { + "multibase": "^3.1.0", + "uint8arrays": "^2.0.5", + "varint": "^6.0.0" + }, + "engines": { + "node": ">=10.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "license": "ISC" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-url": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.1.0.tgz", + "integrity": "sha512-X06Mfd/5aKsRHc0O0J5CUedwnPmnDtLF2+nq+KN9KSDlJHkPuh0JUviWjEWMe0SW/9TDdSLVPuk7L5gGTIA1/w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/nwsapi": { + "version": "2.2.22", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.22.tgz", + "integrity": "sha512-ujSMe1OWVn55euT1ihwCI1ZcAaAU3nxUiDwfDQldc51ZXaB9m2AyOn6/jh1BLe2t/G8xd6uKG1UBF2aZJeg2SQ==", + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", + "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.1.0", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.5.0", + "is-interactive": "^1.0.0", + "is-unicode-supported": "^0.1.0", + "log-symbols": "^4.1.0", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/ow/-/ow-0.28.2.tgz", + "integrity": "sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^4.2.0", + "callsites": "^3.1.0", + "dot-prop": "^6.0.1", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow/node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/p-cancelable": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz", + "integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parent-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parent-require/-/parent-require-1.0.0.tgz", + "integrity": "sha512-2MXDNZC4aXdkkap+rBBMv0lUsfJqvX5/2FiYYnfCnorZt3Pk06/IOR5KeaoghgS2w07MLWgjbsnyaq6PdHn2LQ==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "license": [ + "MIT", + "Apache2" + ], + "dependencies": { + "through": "~2.3" + } + }, + "node_modules/pg": { + "version": "8.16.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", + "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.3", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.2.7" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz", + "integrity": "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "license": "MIT" + }, + "node_modules/pg-cursor": { + "version": "2.15.3", + "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.15.3.tgz", + "integrity": "sha512-eHw63TsiGtFEfAd7tOTZ+TLy+i/2ePKS20H84qCQ+aQ60pve05Okon9tKMC+YN3j6XyeFoHnaim7Lt9WVafQsA==", + "license": "MIT", + "peerDependencies": { + "pg": "^8" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", + "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proxy-chain": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/proxy-chain/-/proxy-chain-2.6.0.tgz", + "integrity": "sha512-+NpVKSk68j8sQJG2tBbFuJxMzKTlqeCXXFbqvlyiFhnmxdcYJSv4XZzUSIfwIUwR3D0T8fEJqrA4C7yykU40Pw==", + "license": "Apache-2.0", + "dependencies": { + "socks": "^2.8.3", + "socks-proxy-agent": "^8.0.3", + "tslib": "^2.3.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "license": "MIT" + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.11", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", + "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", + "license": "MIT" + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-cwd/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/responselike": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-4.0.2.tgz", + "integrity": "sha512-cGk8IbWEAnaCpdAt1BHzJ3Ahz5ewDJa0KseTsE3qIRMJ3C698W8psM7byCeWVpd/Ha7FUYzuRVzXoKoM6nRUbA==", + "license": "MIT", + "dependencies": { + "lowercase-keys": "^3.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/robots-parser": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz", + "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "license": "MIT" + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.3.tgz", + "integrity": "sha512-yqYn1JhPczigF94DMS+shiDMjDowYO6y9+wB/4WgO0Y19jWYk0lQ4tuG5KI7kj4FTp1wxPj5IFfcrz/s1c3jjQ==", + "license": "BlueOak-1.0.0" + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", + "license": "MIT", + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/stac-js": { + "version": "0.1.9", + "license": "Apache-2.0", + "dependencies": { + "@radiantearth/stac-migrate": "^2.0.2", + "urijs": "^1.19.11" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/m-mohr" + } + }, + "node_modules/stac-node-validator": { + "version": "2.0.0-rc.1", + "resolved": "https://registry.npmjs.org/stac-node-validator/-/stac-node-validator-2.0.0-rc.1.tgz", + "integrity": "sha512-qY0NfFZhkmTP1TQ+usaVtqpm0MHPEg0qfesY9VVAPNsiMSVWSW0tzZnBmUeLOsdwJo23y6UmrC1LnpDuzN2VrQ==", + "license": "Apache-2.0", + "dependencies": { + "ajv": "^8.8.2", + "ajv-formats": "^2.1.1", + "axios": "^1.7.4", + "compare-versions": "^6.1.0", + "fs-extra": "^10.0.0", + "jest-diff": "^29.0.1", + "klaw": "^4.0.1", + "stac-js": "^0.1.4", + "uri-js": "^4.4.1", + "yargs": "^17.7.2" + }, + "bin": { + "stac-node-validator": "bin/cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/m-mohr" + } + }, + "node_modules/stac-node-validator/node_modules/compare-versions": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-6.1.1.tgz", + "integrity": "sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==", + "license": "MIT" + }, + "node_modules/stac-node-validator/node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/stack-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-combiner": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", + "integrity": "sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==", + "license": "MIT", + "dependencies": { + "duplexer": "~0.1.1" + } + }, + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-comparison": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string-comparison/-/string-comparison-1.3.0.tgz", + "integrity": "sha512-46aD+slEwybxAMPRII83ATbgMgTiz5P8mVd7Z6VJsCzSHFjdt1hkAVLeFxPIyEb11tc6ihpJTlIqoO0MCF6NPw==", + "license": "MIT", + "engines": { + "node": "^16.0.0 || >=18.0.0" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strtok3": { + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.4.tgz", + "integrity": "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "license": "MIT" + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==", + "license": "MIT" + }, + "node_modules/tldts": { + "version": "7.0.19", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.19.tgz", + "integrity": "sha512-8PWx8tvC4jDB39BQw1m4x8y5MH1BcQ5xHeL2n7UVFulMPH/3Q0uiamahFJ3lXA0zO2SUyRXuVVbWSDmstlt9YA==", + "license": "MIT", + "dependencies": { + "tldts-core": "^7.0.19" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.0.19", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.19.tgz", + "integrity": "sha512-lJX2dEWx0SGH4O6p+7FPwYmJ/bu1JbcGJ8RLaG9b7liIgZ85itUVEPbMtWRVrde/0fnDPEPHW10ZsKW3kVsE9A==", + "license": "MIT" + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/token-types": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.1.tgz", + "integrity": "sha512-kh9LVIWH5CnL63Ipf0jhlBIy0UsrMj/NJDfpsy1SqOXlLKEVyXXYrnFxFT1yOOYVGBSApeVnjPw/sBz5BfEjAQ==", + "license": "MIT", + "dependencies": { + "@borewit/text-codec": "^0.1.0", + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tough-cookie": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz", + "integrity": "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/uhyphen": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==", + "license": "ISC" + }, + "node_modules/uint8array-extras": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/uint8arrays": { + "version": "2.1.10", + "license": "MIT", + "dependencies": { + "multiformats": "^9.4.2" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", + "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/urijs": { + "version": "1.19.11", + "license": "MIT" + }, + "node_modules/util": { + "version": "0.12.5", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/vali-date": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/vali-date/-/vali-date-1.0.0.tgz", + "integrity": "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/varint": { + "version": "6.0.0", + "license": "MIT" + }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/web-encoding": { + "version": "1.1.5", + "license": "MIT", + "dependencies": { + "util": "^0.12.3" + }, + "optionalDependencies": { + "@zxing/text-encoding": "0.9.0" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which-typed-array": { + "version": "1.1.19", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargonaut": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/yargonaut/-/yargonaut-1.1.4.tgz", + "integrity": "sha512-rHgFmbgXAAzl+1nngqOcwEljqHGG9uUZoPjsdZEs1w5JW9RXYzrSvH/u70C1JE5qFi0qjsdhnUX/dJRpWqitSA==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "^1.1.1", + "figlet": "^1.1.1", + "parent-require": "^1.0.0" + } + }, + "node_modules/yargonaut/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yargonaut/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yargonaut/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yargonaut/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yargonaut/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/yoctocolors-cjs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.3.tgz", + "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/crawler/package.json b/crawler/package.json new file mode 100644 index 0000000..3e4ae16 --- /dev/null +++ b/crawler/package.json @@ -0,0 +1,29 @@ +{ + "name": "crawler", + "version": "1.0.0", + "description": "STAC Index crawler", + "main": "index.js", + "scripts": { + "start": "node index.js", + "docker:build": "docker build -t stac-crawler .", + "docker:run": "docker run --rm stac-crawler", + "docker:compose:up": "docker-compose up -d", + "docker:compose:down": "docker-compose down", + "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js", + "test:watch": "node --experimental-vm-modules node_modules/jest/bin/jest.js --watch" + }, + "author": "", + "license": "ISC", + "type": "module", + "dependencies": { + "@databases/pg": "^5.5.0", + "axios": "^1.13.2", + "crawlee": "^3.15.3", + "dotenv": "^17.2.3", + "stac-js": "^0.1.9", + "stac-node-validator": "^2.0.0-rc.1" + }, + "devDependencies": { + "jest": "^29.7.0" + } +} diff --git a/crawler/scheduler.js b/crawler/scheduler.js new file mode 100644 index 0000000..19ce9a0 --- /dev/null +++ b/crawler/scheduler.js @@ -0,0 +1,306 @@ +/** + * @fileoverview Scheduler for running STAC crawler at configurable intervals + * Optionally restricts crawling to specified time windows + * Skips scheduling if database errors occur + * @module scheduler + */ + +import dotenv from 'dotenv'; +import { crawler, requestShutdown, resetShutdownFlag } from './index.js'; +import { formatDuration } from './utils/time.js'; + +dotenv.config(); + +/** + * Configuration + */ +const DAYS_INTERVAL = parseInt(process.env.CRAWL_DAYS_INTERVAL, 10) || 7; // Run every N days +const RUN_ON_STARTUP = process.env.CRAWL_RUN_ON_STARTUP !== 'false'; // Set to false to wait N days before first run +const RETRY_ON_CRAWL_ERROR = process.env.CRAWL_RETRY_ON_ERROR !== 'false'; // Retry if crawl fails but DB is ok +const RETRY_DELAY_HOURS = parseInt(process.env.CRAWL_RETRY_DELAY_HOURS, 10) || 2; // Hours to wait before retry on crawl error + +// Time window configuration (crawler only starts between these hours) +const ALLOWED_START_HOUR = parseInt(process.env.CRAWL_ALLOWED_START_HOUR, 10) || 0; // Default: 00:00 (Midnight) +const ALLOWED_END_HOUR = parseInt(process.env.CRAWL_ALLOWED_END_HOUR, 10) || 23; // Default: 23:00 (11 PM) +const ENFORCE_TIME_WINDOW = process.env.CRAWL_ENFORCE_TIME_WINDOW === 'true'; // Set to true to enable time window check +const GRACE_PERIOD_MINUTES = parseInt(process.env.CRAWL_GRACE_PERIOD_MINUTES, 10) || 30; // Minutes to allow crawler to finish gracefully after end hour + +/** + * Check if current time is within allowed time window + * @returns {boolean} True if within allowed window + */ +const isWithinAllowedTimeWindow = () => { + if (!ENFORCE_TIME_WINDOW) return true; + + const now = new Date(); + const currentHour = now.getHours(); + + // Handle time window that spans midnight (e.g., 22:00 - 07:00) + if (ALLOWED_START_HOUR > ALLOWED_END_HOUR) { + return currentHour >= ALLOWED_START_HOUR || currentHour < ALLOWED_END_HOUR; + } else { + // Normal time window (e.g., 09:00 - 17:00) + return currentHour >= ALLOWED_START_HOUR && currentHour < ALLOWED_END_HOUR; + } +}; + +/** + * Calculate milliseconds until next allowed start time + * @returns {number} Milliseconds to wait + */ +const getMillisecondsUntilAllowedTime = () => { + if (!ENFORCE_TIME_WINDOW) return 0; + + const now = new Date(); + const currentHour = now.getHours(); + + // Already in allowed window + if (isWithinAllowedTimeWindow()) { + return 0; + } + + // Calculate next allowed start time + const nextAllowedTime = new Date(now); + nextAllowedTime.setHours(ALLOWED_START_HOUR, 0, 0, 0); + + // If allowed start hour is later today + if (currentHour < ALLOWED_START_HOUR && ALLOWED_START_HOUR < ALLOWED_END_HOUR) { + // Same day, later + } else if (currentHour >= ALLOWED_END_HOUR && currentHour < ALLOWED_START_HOUR) { + // Same day, wait until ALLOWED_START_HOUR + } else { + // Next day + nextAllowedTime.setDate(nextAllowedTime.getDate() + 1); + } + + const msToWait = nextAllowedTime.getTime() - now.getTime(); + return msToWait > 0 ? msToWait : 0; +}; + +/** + * Calculate milliseconds until the end of allowed time window + * @returns {number} Milliseconds until end hour + */ +const getMillisecondsUntilEndTime = () => { + const now = new Date(); + const endTime = new Date(now); + endTime.setHours(ALLOWED_END_HOUR, 0, 0, 0); + + // If end hour is earlier than current hour, it's tomorrow + if (now.getHours() >= ALLOWED_END_HOUR && ALLOWED_START_HOUR > ALLOWED_END_HOUR) { + endTime.setDate(endTime.getDate() + 1); + } + + const msUntilEnd = endTime.getTime() - now.getTime(); + return msUntilEnd > 0 ? msUntilEnd : 0; +}; + +/** + * Runs the crawler and returns statistics + * Monitors time and warns if approaching end of allowed window + * @async + * @function runCrawler + * @returns {Promise} Crawler statistics + */ +const runCrawler = async () => { + // Reset shutdown flag before starting a new crawl + resetShutdownFlag(); + + const timestamp = new Date().toISOString(); + console.log(`\n${'='.repeat(60)}`); + console.log(`[${timestamp}] Starting crawler run...`); + console.log('='.repeat(60)); + + // Set up shutdown timer if we're approaching end time + let shutdownTimer = null; + let gracePeriodTimer = null; + + if (ENFORCE_TIME_WINDOW) { + const msUntilEnd = getMillisecondsUntilEndTime(); + const msUntilGraceEnd = msUntilEnd + (GRACE_PERIOD_MINUTES * 60 * 1000); + + if (msUntilEnd > 0 && msUntilEnd < 12 * 60 * 60 * 1000) { // Less than 12 hours + const endTime = new Date(Date.now() + msUntilEnd); + console.log(`Note: Crawl should complete before ${endTime.toLocaleTimeString()} (${formatDuration(msUntilEnd)} remaining)`); + console.log(`Grace period: ${GRACE_PERIOD_MINUTES} minutes after end time\n`); + + // Set warning timer for end time + shutdownTimer = setTimeout(() => { + console.warn(`\n${'!'.repeat(60)}`); + console.warn(`WARNING: End time (${ALLOWED_END_HOUR}:00) reached!`); + console.warn(`Crawler is still running. Grace period: ${GRACE_PERIOD_MINUTES} minutes`); + console.warn(`The crawler will continue to finish current operations.`); + console.warn('!'.repeat(60) + '\n'); + }, msUntilEnd); + + // Set forced shutdown timer (end time + grace period) + // Instead of process.exit(), we request graceful shutdown + gracePeriodTimer = setTimeout(() => { + console.error(`\n${'!'.repeat(60)}`); + console.error(`CRITICAL: Grace period expired! (${ALLOWED_END_HOUR}:00 + ${GRACE_PERIOD_MINUTES}min)`); + console.error(`Requesting graceful shutdown to respect time window.`); + console.error(`Next run will be scheduled for ${ALLOWED_START_HOUR}:00`); + console.error('!'.repeat(60) + '\n'); + requestShutdown(); // Request graceful shutdown instead of hard exit + }, msUntilGraceEnd); + } + } + + try { + const stats = await crawler(); + + // Clear timers if crawler finished in time + if (shutdownTimer) clearTimeout(shutdownTimer); + if (gracePeriodTimer) clearTimeout(gracePeriodTimer); + + console.log(`\n[${new Date().toISOString()}] Crawler finished`); + return stats; + } catch (error) { + // Clear timers on error + if (shutdownTimer) clearTimeout(shutdownTimer); + if (gracePeriodTimer) clearTimeout(gracePeriodTimer); + + console.error(`\n[${new Date().toISOString()}] Crawler encountered an error:`, error.message); + return { + success: false, + dbError: true, + crawlError: true, + elapsedTime: 0, + error: error.message + }; + } +}; + +/** + * Schedule next run exactly 7 days after the START of the last crawl + * Adjusts timing to fit within allowed time window if configured + * @param {boolean} isRetry - Whether this is a retry after an error + */ +const scheduleNextRun = (isRetry = false) => { + let delayMs; + let intervalDescription; + + if (isRetry) { + delayMs = RETRY_DELAY_HOURS * 60 * 60 * 1000; + intervalDescription = `${RETRY_DELAY_HOURS} hour(s) (retry)`; + } else { + // Schedule next run: exactly 7 days from now (start of last crawl) + const intervalMs = DAYS_INTERVAL * 24 * 60 * 60 * 1000; + delayMs = intervalMs; + intervalDescription = `${DAYS_INTERVAL} days`; + } + + // Check if scheduled time falls within allowed window + if (ENFORCE_TIME_WINDOW) { + const scheduledTime = new Date(Date.now() + delayMs); + const scheduledHour = scheduledTime.getHours(); + + // Check if the scheduled time is outside the window + const isScheduledTimeAllowed = ALLOWED_START_HOUR > ALLOWED_END_HOUR + ? (scheduledHour >= ALLOWED_START_HOUR || scheduledHour < ALLOWED_END_HOUR) + : (scheduledHour >= ALLOWED_START_HOUR && scheduledHour < ALLOWED_END_HOUR); + + if (!isScheduledTimeAllowed) { + // Calculate how much to add to reach the next allowed window + const hoursUntilAllowed = ALLOWED_START_HOUR > scheduledHour + ? ALLOWED_START_HOUR - scheduledHour + : (24 - scheduledHour) + ALLOWED_START_HOUR; + const additionalMs = hoursUntilAllowed * 60 * 60 * 1000; + delayMs += additionalMs; + console.log(`\nTime window enforcement: Next run moved to allowed window (${ALLOWED_START_HOUR}:00 - ${ALLOWED_END_HOUR}:00)`); + } + } + + const nextRun = new Date(Date.now() + delayMs); + + console.log(`\nNext crawl scheduled for: ${nextRun.toLocaleString()}`); + console.log(` Interval: ${intervalDescription}`); + console.log(` Wait time: ${formatDuration(delayMs)}\n`); + + setTimeout(async () => { + const stats = await runCrawler(); + + if (stats.dbError) { + console.error('\nDATABASE ERROR DETECTED - Scheduler stopped to prevent data issues.'); + console.error(' Please fix the database connection and restart the scheduler.\n'); + process.exit(1); + } else if (stats.crawlError && RETRY_ON_CRAWL_ERROR) { + console.warn('\nCrawl error detected but database is OK - scheduling retry...'); + scheduleNextRun(true); // Retry after 2 hours + } else if (stats.success) { + console.log('\nCrawl completed successfully - scheduling next run...'); + scheduleNextRun(false); // Schedule next run in exactly 7 days + } else { + console.error('\nCrawl failed - scheduler stopped.\n'); + process.exit(1); + } + }, delayMs); +}; + +/** + * Start the scheduler + */ +const startScheduler = async () => { + console.log('\n╔═══════════════════════════════════════════════════════════╗'); + console.log('║ STAC Crawler Scheduler Started ║'); + console.log('╚═══════════════════════════════════════════════════════════╝\n'); + console.log(`Interval: Every ${DAYS_INTERVAL} days`); + console.log(`Run on startup: ${RUN_ON_STARTUP}`); + console.log(`Retry on crawl error: ${RETRY_ON_CRAWL_ERROR}`); + if (RETRY_ON_CRAWL_ERROR) { + console.log(` Retry delay: ${RETRY_DELAY_HOURS} hour(s)`); + } + console.log(`Time window enforcement: ${ENFORCE_TIME_WINDOW ? 'ENABLED' : 'DISABLED'}`); + if (ENFORCE_TIME_WINDOW) { + console.log(` Allowed start hours: ${ALLOWED_START_HOUR}:00 - ${ALLOWED_END_HOUR}:00`); + console.log(` Currently in window: ${isWithinAllowedTimeWindow() ? 'YES' : 'NO'}`); + } + console.log(`Current time: ${new Date().toLocaleString()}\n`); + + // Run immediately if configured + if (RUN_ON_STARTUP) { + // Check if we need to wait for allowed time window + if (ENFORCE_TIME_WINDOW && !isWithinAllowedTimeWindow()) { + const waitMs = getMillisecondsUntilAllowedTime(); + const waitUntil = new Date(Date.now() + waitMs); + console.log(`Current time is outside allowed window (${ALLOWED_START_HOUR}:00 - ${ALLOWED_END_HOUR}:00)`); + console.log(` Waiting until: ${waitUntil.toLocaleString()}`); + console.log(` Wait time: ${formatDuration(waitMs)}\n`); + + await new Promise(resolve => setTimeout(resolve, waitMs)); + } + + console.log('Running initial crawl on startup...'); + const stats = await runCrawler(); + + if (stats.dbError) { + console.error('\nDATABASE ERROR - Cannot start scheduler.'); + console.error(' Please fix the database connection and try again.\n'); + process.exit(1); + } else if (stats.crawlError && RETRY_ON_CRAWL_ERROR) { + console.warn('\nInitial crawl had errors but database is OK - scheduling retry...'); + scheduleNextRun(true); + } else if (stats.success) { + console.log('\nInitial crawl completed - scheduling next run...'); + scheduleNextRun(false); + } else { + console.error('\nInitial crawl failed - exiting.\n'); + process.exit(1); + } + } else { + // Schedule first run without running now + scheduleNextRun(false); + } + + console.log('Scheduler is running. Press Ctrl+C to stop.\n'); + + // Graceful shutdown + process.on('SIGINT', () => { + console.log('\n\nStopping scheduler...'); + console.log('Scheduler stopped. See ya later Aligator!\n'); + process.exit(0); + }); +}; + +// Start the scheduler +startScheduler(); diff --git a/crawler/utils/cli.js b/crawler/utils/cli.js new file mode 100644 index 0000000..cf98e54 --- /dev/null +++ b/crawler/utils/cli.js @@ -0,0 +1,118 @@ +/** + * @fileoverview CLI argument parsing for STAC crawler (temporary debugging file) + * @module utils/cli + * @note This file can be easily removed after debugging is complete + */ + +/** + * Parse command line arguments + * @returns {Object} Parsed CLI arguments + */ +export function parseCliArgs() { + const args = process.argv.slice(2); + const config = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--mode' || arg === '-m') { + config.mode = args[++i]; + } else if (arg === '--max-catalogs' || arg === '-c') { + config.maxCatalogs = parseInt(args[++i], 10); + } else if (arg === '--max-apis' || arg === '-a') { + config.maxApis = parseInt(args[++i], 10); + } else if (arg === '--timeout' || arg === '-t') { + config.timeout = parseInt(args[++i], 10); + } else if (arg === '--max-depth' || arg === '-d') { + config.maxDepth = parseInt(args[++i], 10); + } else if (arg === '--max-concurrency') { + config.maxConcurrency = parseInt(args[++i], 10); + } else if (arg === '--requests-per-minute' || arg === '--rpm') { + config.maxRequestsPerMinute = parseInt(args[++i], 10); + } else if (arg === '--domain-delay') { + config.sameDomainDelaySecs = parseFloat(args[++i]); + } else if (arg === '--max-retries') { + config.maxRequestRetries = parseInt(args[++i], 10); + // New parallel crawling options + } else if (arg === '--parallel-domains' || arg === '-p') { + config.parallelDomains = parseInt(args[++i], 10); + } else if (arg === '--rpm-per-domain') { + config.maxRequestsPerMinutePerDomain = parseInt(args[++i], 10); + } else if (arg === '--concurrency-per-domain') { + config.maxConcurrencyPerDomain = parseInt(args[++i], 10); + } else if (arg === '--fresh' || arg === '-f') { + config.fresh = true; + } else if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } + } + + return config; +} + +/** + * Print help message + */ +export function printHelp() { + console.log(` +STAC Crawler Configuration Options: + + -m, --mode Crawl mode: 'catalogs', 'apis', or 'both' (default: 'both') + -c, --max-catalogs Maximum number of catalogs to crawl (default: 10, use 0 for unlimited) + Note: Limits are for debugging purposes only + -a, --max-apis Maximum number of APIs to crawl (default: 5, use 0 for unlimited) + Note: Limits are for debugging purposes only + -t, --timeout Timeout for each crawl operation in ms (default: 30000) + + Parallel Crawling Options (NEW): + -p, --parallel-domains Number of domains to crawl in parallel (default: 5) + Each domain gets its own crawler instance + --rpm-per-domain Max requests per minute PER DOMAIN (default: 120) + Total throughput = parallel-domains × rpm-per-domain + --concurrency-per-domain Max concurrent requests per domain (default: 10) + + Resume/Fresh Options: + -f, --fresh Clear crawl log and start fresh (ignore previous progress) + Without this flag, crawler resumes from where it left off + + Legacy Rate Limiting Options (still supported): + --max-concurrency Maximum concurrent requests (default: 5) + --requests-per-minute, --rpm Maximum requests per minute (default: 60) + --domain-delay Delay between requests to same domain (default: 1) + --max-retries Maximum retries for failed requests (default: 3) + + -d, --max-depth Maximum recursion depth for nested catalogs (default: 10, use 0 for unlimited) + Prevents memory issues from deeply nested catalog hierarchies + -h, --help Show this help message + +Environment Variables: + CRAWL_MODE Same as --mode + MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) + MAX_APIS Same as --max-apis (use 0 for unlimited) + TIMEOUT_MS Same as --timeout + PARALLEL_DOMAINS Same as --parallel-domains + MAX_REQUESTS_PER_MINUTE_PER_DOMAIN Same as --rpm-per-domain + MAX_CONCURRENCY_PER_DOMAIN Same as --concurrency-per-domain + MAX_CONCURRENCY Same as --max-concurrency + MAX_REQUESTS_PER_MINUTE Same as --requests-per-minute + SAME_DOMAIN_DELAY_SECS Same as --domain-delay + MAX_REQUEST_RETRIES Same as --max-retries + MAX_DEPTH Same as --max-depth (use 0 for unlimited) + +Examples: + # Basic usage + node index.js --mode catalogs --max-catalogs 20 + node index.js -m apis -a 10 -t 60000 + + # Parallel crawling (recommended for performance) + node index.js -p 5 --rpm-per-domain 120 # 5 domains × 120 req/min = 600 req/min max + node index.js -p 10 --rpm-per-domain 60 # 10 domains × 60 req/min = 600 req/min max + + # Unlimited mode (no debugging limits) + node index.js -m both -c 0 -a 0 + + # With environment variables + PARALLEL_DOMAINS=5 MAX_REQUESTS_PER_MINUTE_PER_DOMAIN=120 node index.js + `); +} diff --git a/crawler/utils/config.js b/crawler/utils/config.js new file mode 100644 index 0000000..3e83563 --- /dev/null +++ b/crawler/utils/config.js @@ -0,0 +1,149 @@ +/** + * @fileoverview Configuration management for STAC crawler + * @module utils/config + */ + +import dotenv from 'dotenv'; +import { parseCliArgs } from './cli.js'; + +/** + * Checks if a URL points to a static catalog file rather than an API endpoint + * @param {string} url - URL to check + * @returns {boolean} True if URL appears to be a static file + */ +export function isStaticCatalogUrl(url) { + if (!url || typeof url !== 'string') return false; + + // Check if URL ends with common static catalog file patterns + const staticPatterns = [ + /\.json$/i, // ends with .json + /\/collection\.json/i, // collection.json file + /\/catalog\.json/i, // catalog.json file + /\/stac\.json/i // stac.json file + ]; + + return staticPatterns.some(pattern => pattern.test(url)); +} + +// Load environment variables from .env file +dotenv.config(); + +/** + * Get configuration from environment variables, CLI args, and defaults + * CLI args take precedence over env vars, which take precedence over defaults + * @returns {Object} Configuration object + */ +function getConfig() { + const cliArgs = parseCliArgs(); + + // Default configuration (optimized for 2GB RAM servers) + const defaults = { + mode: 'both', // 'catalogs', 'apis', or 'both' + maxCatalogs: 10, // Maximum number of catalogs to crawl + maxApis: 5, // Maximum number of APIs to crawl + timeout: 30000, // Timeout in milliseconds (30 seconds) + maxDepth: 10, // Maximum recursion depth for nested catalogs (0 = unlimited) + + // Parallel crawling options (reduced for 2GB RAM servers) + parallelDomains: 2, // Number of domains to crawl in parallel (reduced from 5) + maxRequestsPerMinutePerDomain: 60, // Max requests per minute PER domain (reduced from 120) + maxConcurrencyPerDomain: 5, // Max concurrent requests per domain (reduced from 20) + + // Legacy rate limiting options (still supported but parallel options are preferred) + maxConcurrency: 5, // Maximum number of concurrent requests (global) + maxRequestsPerMinute: 60, // Maximum requests per minute (global) + sameDomainDelaySecs: 1, // Delay between requests to the same domain + maxRequestRetries: 3 // Maximum number of retries for failed requests + }; + + // Build configuration with precedence: CLI > ENV > Defaults + const config = { + mode: cliArgs.mode || process.env.CRAWL_MODE || defaults.mode, + maxCatalogs: cliArgs.maxCatalogs !== undefined ? cliArgs.maxCatalogs : + (process.env.MAX_CATALOGS ? parseInt(process.env.MAX_CATALOGS, 10) : defaults.maxCatalogs), + maxApis: cliArgs.maxApis !== undefined ? cliArgs.maxApis : + (process.env.MAX_APIS ? parseInt(process.env.MAX_APIS, 10) : defaults.maxApis), + timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : + (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), + maxDepth: cliArgs.maxDepth !== undefined ? cliArgs.maxDepth : + (process.env.MAX_DEPTH ? parseInt(process.env.MAX_DEPTH, 10) : defaults.maxDepth), + + // NEW: Parallel crawling options + parallelDomains: cliArgs.parallelDomains !== undefined ? cliArgs.parallelDomains : + (process.env.PARALLEL_DOMAINS ? parseInt(process.env.PARALLEL_DOMAINS, 10) : defaults.parallelDomains), + maxRequestsPerMinutePerDomain: cliArgs.maxRequestsPerMinutePerDomain !== undefined ? cliArgs.maxRequestsPerMinutePerDomain : + (process.env.MAX_REQUESTS_PER_MINUTE_PER_DOMAIN ? parseInt(process.env.MAX_REQUESTS_PER_MINUTE_PER_DOMAIN, 10) : defaults.maxRequestsPerMinutePerDomain), + maxConcurrencyPerDomain: cliArgs.maxConcurrencyPerDomain !== undefined ? cliArgs.maxConcurrencyPerDomain : + (process.env.MAX_CONCURRENCY_PER_DOMAIN ? parseInt(process.env.MAX_CONCURRENCY_PER_DOMAIN, 10) : defaults.maxConcurrencyPerDomain), + + // Fresh start option - clear crawl log and recrawl everything + fresh: cliArgs.fresh || process.env.FRESH_CRAWL === 'true' || false, + + // Legacy rate limiting options + maxConcurrency: cliArgs.maxConcurrency !== undefined ? cliArgs.maxConcurrency : + (process.env.MAX_CONCURRENCY ? parseInt(process.env.MAX_CONCURRENCY, 10) : defaults.maxConcurrency), + maxRequestsPerMinute: cliArgs.maxRequestsPerMinute !== undefined ? cliArgs.maxRequestsPerMinute : + (process.env.MAX_REQUESTS_PER_MINUTE ? parseInt(process.env.MAX_REQUESTS_PER_MINUTE, 10) : defaults.maxRequestsPerMinute), + sameDomainDelaySecs: cliArgs.sameDomainDelaySecs !== undefined ? cliArgs.sameDomainDelaySecs : + (process.env.SAME_DOMAIN_DELAY_SECS ? parseFloat(process.env.SAME_DOMAIN_DELAY_SECS) : defaults.sameDomainDelaySecs), + maxRequestRetries: cliArgs.maxRequestRetries !== undefined ? cliArgs.maxRequestRetries : + (process.env.MAX_REQUEST_RETRIES ? parseInt(process.env.MAX_REQUEST_RETRIES, 10) : defaults.maxRequestRetries) + }; + + // Validate mode + const validModes = ['catalogs', 'apis', 'both']; + if (!validModes.includes(config.mode)) { + console.error(`Invalid mode: ${config.mode}. Must be one of: ${validModes.join(', ')}`); + process.exit(1); + } + + // Validate numeric values (0 means unlimited for some options) + if ((config.maxCatalogs < 0) || + (config.maxApis < 0) || + (config.timeout !== Infinity && config.timeout < 0) || + (config.parallelDomains < 1) || + (config.maxRequestsPerMinutePerDomain < 1)) { + console.error('Invalid configuration: parallelDomains and maxRequestsPerMinutePerDomain must be >= 1'); + process.exit(1); + } + + return config; +} + +/** + * Create a timeout promise that rejects after the specified time + * @param {number} ms - Timeout in milliseconds + * @param {string} operation - Description of the operation for error message + * @returns {Promise} Promise that rejects after timeout + */ +function createTimeout(ms, operation = 'Operation') { + return new Promise((_, reject) => { + setTimeout(() => { + reject(new Error(`${operation} timed out after ${ms}ms`)); + }, ms); + }); +} + +/** + * Wrap a promise with a timeout + * @param {Promise} promise - Promise to wrap + * @param {number} ms - Timeout in milliseconds (Infinity for no timeout) + * @param {string} operation - Description of the operation + * @returns {Promise} Promise that races against timeout + */ +async function withTimeout(promise, ms, operation = 'Operation') { + // If timeout is Infinity, just return the promise without racing + if (ms === Infinity) { + return promise; + } + return Promise.race([ + promise, + createTimeout(ms, operation) + ]); +} + +export { + getConfig, + withTimeout, + createTimeout +}; diff --git a/crawler/utils/db.js b/crawler/utils/db.js new file mode 100644 index 0000000..86f3a14 --- /dev/null +++ b/crawler/utils/db.js @@ -0,0 +1,924 @@ +/** + * @fileoverview Database helper module for STAC crawler using PostgreSQL connection pool + * Provides functions for database initialization, collection/catalog management, and connection handling + * @module utils/db + * + * Exports: + * - initDb() - Initialize and test database connection + * - insertOrUpdateCatalog() - Process catalog (currently skips saving) + * - insertOrUpdateCollection() - Insert or update STAC collection with retry logic + * - close() - Close database connection pool + * - pool - PostgreSQL connection pool instance + */ +import pkg from 'pg'; +const { Pool } = pkg; +import dotenv from 'dotenv'; +dotenv.config(); + +const pool = new Pool({ + host: process.env.PGHOST, + port: parseInt(process.env.PGPORT, 10), + user: process.env.PGUSER , + password: process.env.PGPASSWORD , + database: process.env.PGDATABASE , + max: 10, +}); + +/** + * Initialize and test database connection + * Tests the connection by executing a simple query and logs the result + * @async + * @function initDb + * @returns {Promise} + * @throws {Error} If database connection fails + */ +async function initDb() { + const host = process.env.PGHOST; + const port = parseInt(process.env.PGPORT, 10); + const database = process.env.PGDATABASE; + const user = process.env.PGUSER; + + // Test database connection + let client; + try { + client = await pool.connect(); + await client.query('SELECT 1'); + console.log(`DB connection established successfully to ${host}:${port}/${database}`); + } catch (error) { + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.error('DATABASE CONNECTION FAILED'); + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.error(` Host: ${host}`); + console.error(` Port: ${port}`); + console.error(` Database: ${database}`); + console.error(` User: ${user}`); + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.error(` Error: ${error.message}`); + if (error.code) { + console.error(` Code: ${error.code}`); + } + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + throw error; + } finally { + if (client) { + client.release(); + } + } +} + +/** + * Process a catalog for traversal only - catalogs are not saved to database + * Only collections are saved. Catalogs are only used to traverse deeper into the tree. + * @param {Object} catalog - STAC catalog object + * @returns {Promise} always returns null (no catalog saved) + */ +async function insertOrUpdateCatalog(catalog) { + if (!catalog || typeof catalog !== 'object') return null; + + // Catalogs are not saved to database - only used for tree traversal + // Only collections will be saved + console.log(`Skipping catalog save (used for traversal only): ${catalog.title || catalog.id}`); + + return null; +} + +/** + * Insert or update a catalog/API entry in crawllog_catalog table + * This stores the URL queue for re-crawling and the slug for stac_id generation + * @param {Object} catalogInfo - Catalog info object + * @param {string} catalogInfo.slug - The STAC Index slug for this catalog + * @param {string} catalogInfo.url - The source URL of the catalog/API + * @param {boolean} catalogInfo.isApi - Whether this is an API (true) or static catalog (false) + * @returns {Promise} The crawllog_catalog id + */ +async function saveCrawllogCatalog(catalogInfo) { + if (!catalogInfo || !catalogInfo.url) { + throw new Error('Catalog info with url is required'); + } + + const { slug, url, isApi = false } = catalogInfo; + + const result = await pool.query( + `INSERT INTO crawllog_catalog (slug, source_url, is_api, updated_at) + VALUES ($1, $2, $3, now()) + ON CONFLICT (source_url) DO UPDATE SET + slug = COALESCE(EXCLUDED.slug, crawllog_catalog.slug), + is_api = EXCLUDED.is_api + RETURNING id`, + [slug || null, url, isApi] + ); + + return result.rows[0].id; +} + +/** + * Get all catalogs from crawllog_catalog for re-crawling + * @param {Object} options - Query options + * @param {boolean} options.isApi - If provided, filter by API status + * @returns {Promise} Array of catalog objects with id, slug, source_url, is_api + */ +async function getCrawllogCatalogs(options = {}) { + let query = 'SELECT id, slug, source_url, is_api, created_at, updated_at FROM crawllog_catalog'; + const params = []; + + if (options.isApi !== undefined) { + query += ' WHERE is_api = $1'; + params.push(options.isApi); + } + + query += ' ORDER BY id'; + + const result = await pool.query(query, params); + return result.rows.map(row => ({ + id: row.id, + slug: row.slug, + url: row.source_url, + isApi: row.is_api, + createdAt: row.created_at, + updatedAt: row.updated_at + })); +} + +/** + * Get crawllog_catalog ids that still have pending queue entries + * @param {Object} options + * @param {boolean} options.isApi - If provided, filter by API status + * @returns {Promise>} Array of crawllog_catalog ids + */ +async function getCrawllogCatalogIdsWithPendingQueue(options = {}) { + let query = ` + SELECT DISTINCT cc.crawllog_catalog_id + FROM crawllog_collection cc + JOIN crawllog_catalog c ON c.id = cc.crawllog_catalog_id + WHERE cc.source_url IS NOT NULL + AND cc.collection_id IS NULL + `; + const params = []; + + if (options.isApi !== undefined) { + query += ' AND c.is_api = $1'; + params.push(options.isApi); + } + + const result = await pool.query(query, params); + return result.rows.map(row => row.crawllog_catalog_id); +} + +/** + * Get the crawllog_catalog id for a given source URL + * @param {string} sourceUrl - The source URL to look up + * @returns {Promise} The crawllog_catalog id or null if not found + */ +async function getCrawllogCatalogIdByUrl(sourceUrl) { + if (!sourceUrl) return null; + + const result = await pool.query( + 'SELECT id FROM crawllog_catalog WHERE source_url = $1', + [sourceUrl] + ); + + return result.rows.length > 0 ? result.rows[0].id : null; +} + +/** + * Get the slug for a given crawllog_catalog id + * Used to generate stac_id for collections + * @param {number} crawllogCatalogId - The crawllog_catalog id + * @returns {Promise} The slug or null if not found + */ +async function getSlugByCrawllogCatalogId(crawllogCatalogId) { + if (!crawllogCatalogId) return null; + + const result = await pool.query( + 'SELECT slug FROM crawllog_catalog WHERE id = $1', + [crawllogCatalogId] + ); + + return result.rows.length > 0 ? result.rows[0].slug : null; +} + +/** + * Get already-crawled collection URLs for a given catalog (best-effort) + * Used for pause/resume functionality - skip URLs that have already been processed + * NOTE: crawllog_collection is used as a queue only; crawled URLs live in collection.source_url + * @param {number} crawllogCatalogId - The crawllog_catalog id + * @returns {Promise>} Set of source URLs already in collection + */ +async function getCrawledCollectionUrls(crawllogCatalogId) { + if (!crawllogCatalogId) return new Set(); + + const catalogResult = await pool.query( + 'SELECT source_url FROM crawllog_catalog WHERE id = $1', + [crawllogCatalogId] + ); + + if (catalogResult.rows.length === 0) return new Set(); + + const catalogUrl = catalogResult.rows[0].source_url; + const likePattern = `${catalogUrl.replace(/\/$/, '')}/collections/%`; + + const result = await pool.query( + 'SELECT source_url FROM collection WHERE source_url IS NOT NULL AND (source_url = $1 OR source_url LIKE $2)', + [catalogUrl, likePattern] + ); + + return new Set(result.rows.map(row => row.source_url)); +} + +/** + * Check if a specific collection URL has already been crawled + * @param {string} sourceUrl - The source URL to check + * @returns {Promise} True if URL exists in collection table (already crawled) + */ +async function isCollectionUrlCrawled(sourceUrl) { + if (!sourceUrl) return false; + + const result = await pool.query( + 'SELECT 1 FROM collection WHERE source_url = $1 LIMIT 1', + [sourceUrl] + ); + + return result.rows.length > 0; +} + +/** + * Enqueue a collection URL into crawllog_collection without marking it as crawled + * Used to persist newly discovered collection links for later processing + * @param {Object} params + * @param {string} params.sourceUrl - Collection URL to enqueue + * @param {number|null} params.crawllogCatalogId - Parent crawllog_catalog id + * @returns {Promise} + */ +async function enqueueCollectionUrl({ sourceUrl, crawllogCatalogId = null }) { + if (!sourceUrl) return; + + await pool.query( + `INSERT INTO crawllog_collection (collection_id, source_url, crawllog_catalog_id) + VALUES (NULL, $1, $2) + ON CONFLICT (source_url) DO UPDATE SET + crawllog_catalog_id = COALESCE(EXCLUDED.crawllog_catalog_id, crawllog_collection.crawllog_catalog_id)`, + [sourceUrl, crawllogCatalogId] + ); +} + +/** + * Get pending (not yet crawled) collection URLs from crawllog_collection + * Joined with crawllog_catalog to determine API vs catalog context + * @param {Object} options + * @param {boolean} options.isApi - If provided, filter by API status + * @returns {Promise} Array of pending collection seed objects + */ +async function getPendingCollectionSeeds(options = {}) { + let query = ` + SELECT cc.source_url, cc.crawllog_catalog_id, c.slug, c.is_api + FROM crawllog_collection cc + JOIN crawllog_catalog c ON c.id = cc.crawllog_catalog_id + WHERE cc.collection_id IS NULL + AND cc.source_url IS NOT NULL + `; + const params = []; + + if (options.isApi !== undefined) { + query += ' AND c.is_api = $1'; + params.push(options.isApi); + } + + query += ' ORDER BY cc.id'; + + const result = await pool.query(query, params); + return result.rows.map(row => ({ + url: row.source_url, + crawllogCatalogId: row.crawllog_catalog_id, + slug: row.slug, + isApi: row.is_api + })); +} + +/** + * Claim and remove a batch of pending collection URLs from crawllog_collection + * Used to feed the in-memory queue in controlled batches + * @param {Object} options + * @param {number} options.limit - Maximum number of URLs to claim + * @param {boolean} options.isApi - If provided, filter by API status + * @returns {Promise} Array of claimed queue items + */ +async function claimCollectionQueueBatch({ limit = 900, isApi, crawllogCatalogIds } = {}) { + if (!limit || limit <= 0) return []; + + const params = []; + let apiFilter = ''; + let catalogFilter = ''; + let limitParam = '$1'; + + if (isApi !== undefined) { + apiFilter = 'AND c.is_api = $1'; + params.push(isApi); + limitParam = '$2'; + } + + if (Array.isArray(crawllogCatalogIds) && crawllogCatalogIds.length > 0) { + params.push(crawllogCatalogIds); + catalogFilter = `AND cc.crawllog_catalog_id = ANY($${params.length})`; + limitParam = `$${params.length + 1}`; + } + + params.push(limit); + + const query = ` + WITH cte AS ( + SELECT cc.id + FROM crawllog_collection cc + JOIN crawllog_catalog c ON c.id = cc.crawllog_catalog_id + WHERE cc.source_url IS NOT NULL + ${apiFilter} + ${catalogFilter} + ORDER BY cc.id + LIMIT ${limitParam} + ) + DELETE FROM crawllog_collection cc + USING cte, crawllog_catalog c + WHERE cc.id = cte.id + AND c.id = cc.crawllog_catalog_id + RETURNING cc.source_url, cc.crawllog_catalog_id, c.slug, c.is_api; + `; + + const result = await pool.query(query, params); + return result.rows.map(row => ({ + url: row.source_url, + crawllogCatalogId: row.crawllog_catalog_id, + slug: row.slug, + isApi: row.is_api + })); +} + +/** + * Remove a URL from crawllog_collection queue + * Used when a URL was processed outside of DB batch claiming + * @param {string} sourceUrl - URL to remove + * @returns {Promise} Number of rows deleted + */ +async function removeFromCollectionQueue(sourceUrl) { + if (!sourceUrl) return 0; + + const result = await pool.query( + 'DELETE FROM crawllog_collection WHERE source_url = $1', + [sourceUrl] + ); + + return result.rowCount; +} + +/** + * Update the updated_at timestamp for a crawllog_catalog entry + * Called when a catalog has been fully processed + * @param {number} crawllogCatalogId - The crawllog_catalog id + */ +async function markCatalogCrawled(crawllogCatalogId) { + if (!crawllogCatalogId) return; + + await pool.query( + 'UPDATE crawllog_catalog SET updated_at = now() WHERE id = $1', + [crawllogCatalogId] + ); +} + +/** + * Clear all entries from crawllog_collection table + * Used for fresh crawl - forces re-crawling of all collections + * @returns {Promise} Number of rows deleted + */ +async function clearCrawllogCollection() { + const result = await pool.query('DELETE FROM crawllog_collection'); + return result.rowCount; +} + +/** + * Clear all entries from both crawllog tables + * Used for complete fresh start + * @returns {Promise<{catalogs: number, collections: number}>} Number of rows deleted from each table + */ +async function clearAllCrawllogs() { + // Delete collections first (foreign key constraint) + const collectionsResult = await pool.query('DELETE FROM crawllog_collection'); + const catalogsResult = await pool.query('DELETE FROM crawllog_catalog'); + + return { + catalogs: catalogsResult.rowCount, + collections: collectionsResult.rowCount + }; +} + +/** + * Check if an error is a PostgreSQL deadlock error + * @param {Error} error - The error to check + * @returns {boolean} true if it's a deadlock error + */ +function isDeadlockError(error) { + // PostgreSQL deadlock error code is '40P01' + return error.code === '40P01' || error.message?.includes('deadlock detected'); +} + +/** + * Sleep for a given number of milliseconds + * @param {number} ms - Milliseconds to sleep + * @returns {Promise} + */ +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +/** + * Insert or update a collection in the database with deadlock retry logic + * @param {Object} collection - STAC collection object + * @param {number} maxRetries - Maximum number of retry attempts for deadlocks (default: 3) + * @returns {Promise} collection ID + */ +async function insertOrUpdateCollection(collection, maxRetries = 3) { + let lastError; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + return await _insertOrUpdateCollectionInternal(collection); + } catch (error) { + lastError = error; + + if (isDeadlockError(error) && attempt < maxRetries) { + // Exponential backoff: 100ms, 200ms, 400ms, ... + const delayMs = 100 * Math.pow(2, attempt - 1) + Math.random() * 50; + console.warn(`WARN [DB] Deadlock detected for collection "${collection.title || collection.id}", retrying in ${Math.round(delayMs)}ms (attempt ${attempt}/${maxRetries})`); + await sleep(delayMs); + continue; + } + + // Not a deadlock or max retries reached, throw the error + throw error; + } + } + + // Should not reach here, but just in case + throw lastError; +} + +/** + * Internal implementation of insertOrUpdateCollection + * @param {Object} collection - STAC collection object + * @returns {Promise} collection ID + */ +async function _insertOrUpdateCollectionInternal(collection) { + const client = await pool.connect(); + try { + await client.query('BEGIN'); + + // Parse spatial extent (bbox) + // Support both normalized format (bbox) and original STAC format (extent.spatial.bbox) + let spatialExtent = null; + let bbox = null; + + // Try normalized format first (from normalizeCollection) + if (collection.bbox && Array.isArray(collection.bbox)) { + bbox = collection.bbox; + } + // Fallback to original STAC format + else if (collection.extent?.spatial?.bbox && collection.extent.spatial.bbox[0]) { + bbox = collection.extent.spatial.bbox[0]; + } + + if (bbox && bbox.length === 4) { + // Create polygon from bbox [west, south, east, north] + // EWKT format requires SRID=4326, not EPSG:4326 + spatialExtent = `SRID=4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; + } + + // Parse temporal extent + // Support both normalized format (temporal) and original STAC format (extent.temporal.interval) + let temporalStart = null; + let temporalEnd = null; + let interval = null; + + // Try normalized format first (from normalizeCollection) + if (collection.temporal && Array.isArray(collection.temporal)) { + interval = collection.temporal; + } + // Fallback to original STAC format + else if (collection.extent?.temporal?.interval && collection.extent.temporal.interval[0]) { + interval = collection.extent.temporal.interval[0]; + } + + if (interval) { + temporalStart = interval[0] ? new Date(interval[0]) : null; + temporalEnd = interval[1] ? new Date(interval[1]) : null; + } + + // Insert or update collection + const collectionTitle = collection.title || collection.id || 'Unnamed Collection'; + + // Construct unique stac_id from sourceSlug and collection id + // Format: {sourceSlug}_{collection_id} for uniqueness across different sources + let stacId = null; + if (collection.sourceSlug && collection.id) { + stacId = `${collection.sourceSlug}_${collection.id}`; + } else if (collection.id) { + stacId = collection.id; + } + + // Extract source URL - prefer crawledUrl (the actual absolute URL the collection was fetched from) + // Fall back to links only if crawledUrl is not available + let sourceUrl = null; + if (collection.crawledUrl) { + // Use the absolute URL from the crawler (most reliable) + sourceUrl = collection.crawledUrl; + } else if (collection.links && Array.isArray(collection.links)) { + // Fallback to self/root links (may be relative URLs) + const selfLink = collection.links.find(link => link.rel === 'self'); + const rootLink = collection.links.find(link => link.rel === 'root'); + sourceUrl = selfLink?.href || rootLink?.href || null; + } + + // Check if collection with same stac_id already exists - stac_id is the unique key for upsert + // stac_id format: {sourceSlug}_{collection.id} ensures uniqueness across sources + let existingCollection; + if (stacId) { + // Primary matching: stac_id is unique, so match by stac_id alone + existingCollection = await client.query( + 'SELECT id FROM collection WHERE stac_id = $1', + [stacId] + ); + } else { + // Fallback for collections without stac_id: match by title + source_url + existingCollection = await client.query( + 'SELECT id FROM collection WHERE stac_id IS NULL AND title = $1 AND source_url = $2', + [collectionTitle, sourceUrl] + ); + } + + // Use originalJson if available (from normalizeCollection), otherwise use the collection object + // This ensures the full original STAC JSON is stored, not the normalized version + const fullJsonData = collection.originalJson || collection; + + // Determine is_api based on source_url + // If source_url ends with .json, it's NOT an API (static file) + // Otherwise, it's an API endpoint + let isApi = false; + if (sourceUrl) { + isApi = !sourceUrl.toLowerCase().endsWith('.json'); + } + + let collectionId; + if (existingCollection.rows.length > 0) { + // Update existing collection + collectionId = existingCollection.rows[0].id; + await client.query( + `UPDATE collection SET + stac_id = $1, + stac_version = $2, + title = $3, + description = $4, + license = $5, + spatial_extent = ST_GeomFromEWKT($6), + temporal_extent_start = $7, + temporal_extent_end = $8, + is_active = $9, + source_url = $10, + full_json = $11, + is_api = $12, + updated_at = now() + WHERE id = $13`, + [ + stacId, + collection.stac_version || null, + collectionTitle, + collection.description || null, + collection.license || null, + spatialExtent, + temporalStart, + temporalEnd, + true, // is_active + sourceUrl, + JSON.stringify(fullJsonData), + isApi, + collectionId + ] + ); + } else { + // Insert new collection - updated_at defaults to now() (same as created_at) + // since we know the data is current as of this crawl + const collectionResult = await client.query( + `INSERT INTO collection ( + stac_id, stac_version, title, description, license, + spatial_extent, temporal_extent_start, temporal_extent_end, + is_active, source_url, full_json, is_api + ) + VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, $12) + RETURNING id`, + [ + stacId, + collection.stac_version || null, + collectionTitle, + collection.description || null, + collection.license || null, + spatialExtent, + temporalStart, + temporalEnd, + true, // is_active + sourceUrl, + JSON.stringify(fullJsonData), + isApi + ] + ); + collectionId = collectionResult.rows[0].id; + } + + // Insert summaries + if (collection.summaries && typeof collection.summaries === 'object') { + await client.query('DELETE FROM collection_summaries WHERE collection_id = $1', [collectionId]); + for (const [name, value] of Object.entries(collection.summaries)) { + await insertSummary(client, collectionId, name, value); + } + } + + // Insert keywords + if (collection.keywords && Array.isArray(collection.keywords)) { + await insertKeywords(client, collectionId, collection.keywords, 'collection'); + } + + // Insert STAC extensions + if (collection.stac_extensions && Array.isArray(collection.stac_extensions)) { + await insertStacExtensions(client, collectionId, collection.stac_extensions, 'collection'); + } + + // Insert providers + if (collection.providers && Array.isArray(collection.providers)) { + await insertProviders(client, collectionId, collection.providers); + } + + // Insert assets + if (collection.assets && typeof collection.assets === 'object') { + await insertAssets(client, collectionId, collection.assets); + } + + // Remove from crawllog_collection queue once crawled + if (sourceUrl) { + await client.query( + 'DELETE FROM crawllog_collection WHERE source_url = $1', + [sourceUrl] + ); + } + + await client.query('COMMIT'); + + return collectionId; + } catch (error) { + await client.query('ROLLBACK'); + // Only log non-deadlock errors here, deadlocks are handled by retry wrapper + if (!isDeadlockError(error)) { + console.error('Error inserting collection:', error.message); + } + throw error; + } finally { + client.release(); + } +} + + + +/** + * Insert or update keywords for a collection + * Deletes existing keywords for the parent and inserts new ones + * @async + * @function insertKeywords + * @param {Object} client - PostgreSQL client from connection pool + * @param {number} parentId - Parent entity ID (collection ID) + * @param {string[]} keywords - Array of keyword strings + * @param {string} type - Entity type ('collection') + * @returns {Promise} + */ +async function insertKeywords(client, parentId, keywords, type) { + await client.query( + `DELETE FROM ${type}_keywords WHERE ${type}_id = $1`, + [parentId] + ); + + for (const keyword of keywords) { + if (!keyword) continue; + + // Insert keyword if not exists + const keywordResult = await client.query( + 'INSERT INTO keywords (keyword) VALUES ($1) ON CONFLICT (keyword) DO UPDATE SET keyword = EXCLUDED.keyword RETURNING id', + [keyword] + ); + const keywordId = keywordResult.rows[0].id; + + // Link keyword to parent + await client.query( + `INSERT INTO ${type}_keywords (${type}_id, keyword_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, + [parentId, keywordId] + ); + } +} + +/** + * Insert or update STAC extensions for a collection + * Deletes existing extensions for the parent and inserts new ones + * @async + * @function insertStacExtensions + * @param {Object} client - PostgreSQL client from connection pool + * @param {number} parentId - Parent entity ID (collection ID) + * @param {string[]} extensions - Array of STAC extension URLs + * @param {string} type - Entity type ('collection') + * @returns {Promise} + */ +async function insertStacExtensions(client, parentId, extensions, type) { + await client.query( + `DELETE FROM ${type}_stac_extension WHERE ${type}_id = $1`, + [parentId] + ); + + for (const extension of extensions) { + if (!extension) continue; + + // Insert extension if not exists + const extResult = await client.query( + 'INSERT INTO stac_extensions (stac_extension) VALUES ($1) ON CONFLICT (stac_extension) DO UPDATE SET stac_extension = EXCLUDED.stac_extension RETURNING id', + [extension] + ); + const extId = extResult.rows[0].id; + + // Link extension to parent + await client.query( + `INSERT INTO ${type}_stac_extension (${type}_id, stac_extension_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, + [parentId, extId] + ); + } +} + + +/** + * Insert a single collection summary entry + * Automatically determines the summary type (range, set, schema, or value) based on the value + * @async + * @function insertSummary + * @param {Object} client - PostgreSQL client from connection pool + * @param {number} collectionId - Collection ID + * @param {string} name - Summary property name + * @param {*} value - Summary value (can be array, object, or primitive) + * @returns {Promise} + */ +async function insertSummary(client, collectionId, name, value) { + let kind = 'unknown'; + let rangeMin = null; + let rangeMax = null; + let setValue = null; + let jsonSchema = null; + + if (Array.isArray(value)) { + if (value.length === 2 && typeof value[0] === 'number' && typeof value[1] === 'number') { + kind = 'range'; + rangeMin = value[0]; + rangeMax = value[1]; + } else { + kind = 'set'; + setValue = JSON.stringify(value); + } + } else if (typeof value === 'object') { + kind = 'schema'; + jsonSchema = JSON.stringify(value); + } else { + kind = 'value'; + setValue = String(value); + } + + await client.query( + 'INSERT INTO collection_summaries (collection_id, name, kind, range_min, range_max, set_value, json_schema) VALUES ($1, $2, $3, $4, $5, $6, $7)', + [collectionId, name, kind, rangeMin, rangeMax, setValue, jsonSchema] + ); +} + +/** + * Insert or update providers for a collection + * Deletes existing provider links and creates new ones + * @async + * @function insertProviders + * @param {Object} client - PostgreSQL client from connection pool + * @param {number} collectionId - Collection ID + * @param {Object[]} providers - Array of provider objects with name and roles + * @returns {Promise} + */ +async function insertProviders(client, collectionId, providers) { + await client.query('DELETE FROM collection_providers WHERE collection_id = $1', [collectionId]); + + for (const provider of providers) { + if (!provider.name) continue; + + // Insert provider if not exists + const providerResult = await client.query( + 'INSERT INTO providers (provider) VALUES ($1) ON CONFLICT (provider) DO UPDATE SET provider = EXCLUDED.provider RETURNING id', + [provider.name] + ); + const providerId = providerResult.rows[0].id; + + // Link provider to collection + const roles = provider.roles ? provider.roles.join(',') : null; + await client.query( + 'INSERT INTO collection_providers (collection_id, provider_id, collection_provider_roles) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING', + [collectionId, providerId, roles] + ); + } +} + +/** + * Insert or update assets for a collection + * Deletes existing asset links and creates new ones + * @async + * @function insertAssets + * @param {Object} client - PostgreSQL client from connection pool + * @param {number} collectionId - Collection ID + * @param {Object} assets - Object mapping asset names to asset data (href, type, roles, metadata) + * @returns {Promise} + */ +async function insertAssets(client, collectionId, assets) { + await client.query('DELETE FROM collection_assets WHERE collection_id = $1', [collectionId]); + + for (const [assetName, assetData] of Object.entries(assets)) { + if (!assetData) continue; + + // Insert asset + const assetResult = await client.query( + 'INSERT INTO assets (name, href, type, roles, metadata) VALUES ($1, $2, $3, $4, $5) RETURNING id', + [ + assetName, + assetData.href || null, + assetData.type || null, + assetData.roles || null, + JSON.stringify(assetData) + ] + ); + const assetId = assetResult.rows[0].id; + + // Link asset to collection + const roles = assetData.roles ? assetData.roles.join(',') : null; + await client.query( + 'INSERT INTO collection_assets (collection_id, asset_id, collection_asset_roles) VALUES ($1, $2, $3)', + [collectionId, assetId, roles] + ); + } +} + + + + +/** + * Mark collections as inactive if they haven't been updated in the last 7 days + * Should be called after a crawl completes to deactivate stale collections + * @async + * @function deactivateStaleCollections + * @returns {Promise} Number of collections marked as inactive + */ +async function deactivateStaleCollections() { + const result = await pool.query(` + UPDATE collection + SET is_active = false + WHERE updated_at < NOW() - INTERVAL '7 days' + AND is_active = true + `); + + const count = result.rowCount; + if (count > 0) { + console.log(`Marked ${count} collection(s) as inactive (not updated in last 7 days)`); + } + + return count; +} + +/** + * Close the database connection pool + * Should be called when the application shuts down + * @async + * @function close + * @returns {Promise} + */ +async function close() { + await pool.end(); +} + +export default { + initDb, + insertOrUpdateCatalog, + insertOrUpdateCollection, + saveCrawllogCatalog, + getCrawllogCatalogs, + getCrawllogCatalogIdsWithPendingQueue, + getCrawllogCatalogIdByUrl, + getSlugByCrawllogCatalogId, + getCrawledCollectionUrls, + isCollectionUrlCrawled, + enqueueCollectionUrl, + getPendingCollectionSeeds, + claimCollectionQueueBatch, + removeFromCollectionQueue, + markCatalogCrawled, + clearCrawllogCollection, + clearAllCrawllogs, + deactivateStaleCollections, + close, + pool +}; diff --git a/crawler/utils/endpoints.js b/crawler/utils/endpoints.js new file mode 100644 index 0000000..ea3e6e1 --- /dev/null +++ b/crawler/utils/endpoints.js @@ -0,0 +1,89 @@ +/** + * @fileoverview Endpoint utilities for STAC collections + * @module utils/endpoints + */ + +import db from './db.js'; + +/** + * Finds the collection endpoint from STAC catalog links + * STAC catalogs should advertise their collection endpoint via rel="data" or rel="collections" + * Falls back to a single /collections endpoint if no link is found + * @async + * @param {Object} stacCatalog - Parsed STAC catalog object from stac-js + * @param {string} baseUrl - Base catalog URL + * @param {string} catalogId - Catalog ID for logging + * @param {number} depth - Current depth + * @param {Object} crawler - Crawlee crawler instance + * @param {Object} log - Logger + * @param {string} indent - Indentation for logging + * @param {string} catalogSlug - Slug of the source catalog for unique ID generation + * @param {number} crawllogCatalogId - ID from crawllog_catalog for linking collections + */ +export async function tryCollectionEndpoints(stacCatalog, baseUrl, catalogId, depth, crawler, log, indent, catalogSlug = null, crawllogCatalogId = null) { + let collectionUrl = null; + + // Try to find collection endpoint from STAC links (proper STAC discovery) + if (stacCatalog && typeof stacCatalog.getLinks === 'function') { + const links = stacCatalog.getLinks(); + + // Look for rel="data" (STAC API) or rel="collections" link + const collectionLink = links.find(link => + link.rel === 'data' || link.rel === 'collections' + ); + + if (collectionLink) { + try { + collectionUrl = typeof collectionLink.getAbsoluteUrl === 'function' + ? collectionLink.getAbsoluteUrl() + : collectionLink.href; + + // Handle S3 protocol URLs - convert to HTTPS + if (collectionUrl && collectionUrl.startsWith('s3://')) { + const s3Match = collectionUrl.match(/^s3:\/\/([^/]+)\/(.*)$/); + if (s3Match) { + const [, bucket, path] = s3Match; + collectionUrl = `https://${bucket}.s3.amazonaws.com/${path}`; + log.debug(`${indent}Converted S3 URL: ${collectionLink.href} -> ${collectionUrl}`); + } + } + + // Handle relative URLs + if (collectionUrl && !collectionUrl.startsWith('http')) { + const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); + collectionUrl = `${basePath}/${collectionUrl}`; + } + + log.info(`${indent}Found collection endpoint via STAC link (rel="${collectionLink.rel}"): ${collectionUrl}`); + } catch (err) { + log.warning(`${indent}Error resolving collection link: ${err.message}`); + } + } + } + + // Fallback: if no link found, try the standard /collections endpoint + if (!collectionUrl) { + // Remove trailing filename (like catalog.json) from base URL + const urlParts = baseUrl.split('/'); + const lastPart = urlParts[urlParts.length - 1]; + + if (lastPart.includes('.json') || lastPart.includes('.')) { + urlParts.pop(); + } + + collectionUrl = urlParts.join('/') + '/collections'; + log.debug(`${indent}No collection link found, using fallback: ${collectionUrl}`); + } + + // Persist collection endpoint in DB queue + try { + await db.enqueueCollectionUrl({ + sourceUrl: collectionUrl, + crawllogCatalogId + }); + } catch (err) { + log.warning(`${indent}Failed to enqueue collections endpoint: ${err.message}`); + } + + // Collection request will be pulled from DB queue in batch mode +} diff --git a/crawler/utils/globalStats.js b/crawler/utils/globalStats.js new file mode 100644 index 0000000..bc36382 --- /dev/null +++ b/crawler/utils/globalStats.js @@ -0,0 +1,180 @@ +/** + * @fileoverview Global statistics tracker for aggregated crawler metrics + * Provides real-time statistics across all parallel crawlers + * @module utils/globalStats + */ + +import { log as crawleeLog } from 'crawlee'; + +/** + * Global statistics singleton that aggregates metrics from all crawlers + */ +class GlobalStatistics { + constructor() { + this.reset(); + this.intervalId = null; + this.intervalSecs = 60; // Log every 60 seconds like Crawlee + } + + /** + * Reset all statistics + */ + reset() { + this.startTime = null; + this.stats = { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0, + catalogsProcessed: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + }; + this.activeDomains = new Set(); + this.completedDomains = 0; + this.totalDomains = 0; + } + + /** + * Start the statistics tracking + * @param {number} totalDomains - Total number of domains to process + * @param {number} intervalSecs - Logging interval in seconds (0 or null to disable periodic logging) + */ + start(totalDomains = 0, intervalSecs = 0) { + this.reset(); + this.startTime = Date.now(); + this.totalDomains = totalDomains; + this.intervalSecs = intervalSecs; + + // Only start periodic logging if intervalSecs > 0 + if (intervalSecs && intervalSecs > 0) { + this.intervalId = setInterval(() => { + this.logStatistics(); + }, this.intervalSecs * 1000); + } + } + + /** + * Stop the statistics tracking + */ + stop() { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + // Log final statistics + this.logStatistics(true); + } + + /** + * Register a domain as active + * @param {string} domain - Domain name + */ + domainStarted(domain) { + this.activeDomains.add(domain); + } + + /** + * Register a domain as completed + * @param {string} domain - Domain name + */ + domainCompleted(domain) { + this.activeDomains.delete(domain); + this.completedDomains++; + } + + /** + * Increment a statistic counter (thread-safe for single-threaded Node.js) + * @param {string} stat - Statistic name + * @param {number} amount - Amount to increment (default: 1) + */ + increment(stat, amount = 1) { + if (this.stats.hasOwnProperty(stat)) { + this.stats[stat] += amount; + } + } + + /** + * Add stats from a completed domain crawl + * @param {Object} domainStats - Statistics object from a domain crawl + */ + addDomainStats(domainStats) { + if (!domainStats) return; + + for (const [key, value] of Object.entries(domainStats)) { + if (typeof value === 'number' && this.stats.hasOwnProperty(key)) { + this.stats[key] += value; + } + } + } + + /** + * Get current runtime in milliseconds + * @returns {number} Runtime in milliseconds + */ + getRuntimeMs() { + if (!this.startTime) return 0; + return Date.now() - this.startTime; + } + + /** + * Calculate requests per minute + * @returns {number} Requests per minute + */ + getRequestsPerMinute() { + const runtimeMinutes = this.getRuntimeMs() / 60000; + if (runtimeMinutes <= 0) return 0; + return Math.round(this.stats.totalRequests / runtimeMinutes); + } + + /** + * Log current statistics using Crawlee's logger + * @param {boolean} isFinal - Whether this is the final log + */ + logStatistics(isFinal = false) { + const runtimeMs = this.getRuntimeMs(); + const runtimeSecs = Math.round(runtimeMs / 1000); + const reqPerMin = this.getRequestsPerMinute(); + + const prefix = isFinal ? 'GlobalStatistics: Final' : 'GlobalStatistics'; + + const statsObj = { + requestsFinishedPerMinute: reqPerMin, + requestsTotal: this.stats.totalRequests, + requestsSuccessful: this.stats.successfulRequests, + requestsFailed: this.stats.failedRequests, + collectionsFound: this.stats.collectionsFound, + collectionsSaved: this.stats.collectionsSaved, + domainsActive: this.activeDomains.size, + domainsCompleted: this.completedDomains, + domainsTotal: this.totalDomains, + crawlerRuntimeSecs: runtimeSecs + }; + + crawleeLog.info(`${prefix}: ${JSON.stringify(statsObj)}`); + } + + /** + * Get current statistics snapshot + * @returns {Object} Current statistics + */ + getStats() { + return { + ...this.stats, + runtimeMs: this.getRuntimeMs(), + requestsPerMinute: this.getRequestsPerMinute(), + activeDomains: this.activeDomains.size, + completedDomains: this.completedDomains, + totalDomains: this.totalDomains + }; + } +} + +// Export singleton instance +const globalStats = new GlobalStatistics(); + +export default globalStats; +export { GlobalStatistics }; diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js new file mode 100644 index 0000000..b850956 --- /dev/null +++ b/crawler/utils/handlers.js @@ -0,0 +1,613 @@ +/** + * @fileoverview Request handlers for catalog and collection crawling + * @module utils/handlers + */ + +import create from 'stac-js'; +import validate from 'stac-node-validator'; +import { normalizeCollection } from './normalization.js'; +import { tryCollectionEndpoints } from './endpoints.js'; +import db from './db.js'; + +/** + * Batch size for saving collections to database + * After this many collections are collected, they will be flushed to DB + * Set low (25) for servers with limited RAM (2GB) + * @type {number} + */ +const BATCH_SIZE = 25; + +/** + * Validates STAC structure using stac-node-validator before attempting migration + * @async + * @param {Object} json - JSON object to validate + * @param {Object} log - Logger instance + * @param {string} indent - Indentation for logging + * @returns {Promise} Validation result with valid flag, errors, and warnings + */ +async function validateStacStructure(json, log, indent = '') { + if (!json || typeof json !== 'object') { + return { + valid: false, + error: 'Invalid JSON: null or not an object', + errors: ['Invalid JSON structure'] + }; + } + + try { + // Use stac-node-validator for full STAC spec validation + const result = await validate(json); + + if (result.valid) { + log.debug(`${indent}STAC validation passed (version: ${result.version}, type: ${result.type})`); + return { valid: true, version: result.version, type: result.type }; + } else { + // Collect all validation errors + const errors = []; + + // Core schema errors + if (result.results.core && result.results.core.length > 0) { + errors.push(...result.results.core.map(err => + `${err.instancePath || 'root'}: ${err.message}` + )); + } + + // Extension errors + if (result.results.extensions) { + Object.entries(result.results.extensions).forEach(([ext, extErrors]) => { + if (extErrors.length > 0) { + errors.push(...extErrors.map(err => + `[${ext}] ${err.instancePath || 'root'}: ${err.message}` + )); + } + }); + } + + // Custom validation errors + if (result.results.custom && result.results.custom.length > 0) { + errors.push(...result.results.custom.map(err => err.message || String(err))); + } + + return { + valid: false, + error: `STAC validation failed with ${errors.length} error(s)`, + errors: errors.slice(0, 5), // Limit to first 5 errors for logging + totalErrors: errors.length + }; + } + } catch (validationError) { + // If validator itself fails, return error + return { + valid: false, + error: `Validator error: ${validationError.message}`, + errors: [validationError.message] + }; + } +} + +/** + * Batch size for clearing catalogs array to free memory + * The catalogs array is only used for statistics, so we clear it periodically + * Set low (25) for servers with limited RAM (2GB) + * @type {number} + */ +const CATALOG_CLEAR_BATCH_SIZE = 25; + +/** + * Flushes collected collections to the database and clears the array + * @async + * @param {Object} results - Results object containing collections array + * @param {Object} log - Logger instance + * @param {boolean} force - If true, flush even if below batch size (used at end of crawl) + * @returns {Promise<{saved: number, failed: number}>} Count of saved and failed collections + */ +export async function flushCollectionsToDb(results, log, force = false) { + if (!force && results.collections.length < BATCH_SIZE) { + return { saved: 0, failed: 0 }; + } + + if (results.collections.length === 0) { + return { saved: 0, failed: 0 }; + } + + const collectionsToSave = [...results.collections]; + results.collections.length = 0; // Clear the array to free memory + + let saved = 0; + let failed = 0; + + log.info(`[BATCH] Flushing ${collectionsToSave.length} collections to database...`); + + for (const collection of collectionsToSave) { + try { + await db.insertOrUpdateCollection(collection); + saved++; + } catch (err) { + log.warning(`[BATCH] Failed to save collection ${collection.id}: ${err.message}`); + failed++; + } + } + + log.info(`[BATCH] Saved ${saved} collections, ${failed} failed`); + + return { saved, failed }; +} + +/** + * Checks if batch size is reached and flushes if necessary + * Also clears the catalogs array periodically to free memory + * @async + * @param {Object} results - Results object containing collections array + * @param {Object} log - Logger instance + */ +async function checkAndFlush(results, log) { + if (results.collections.length >= BATCH_SIZE) { + const { saved, failed } = await flushCollectionsToDb(results, log, false); + results.stats.collectionsSaved += saved; + results.stats.collectionsFailed += failed; + } + + // Clear catalogs array periodically to free memory + // The catalogs array is only used for end statistics, which we track in stats object + // Note: catalogs may not exist when called from API crawler (which uses apis instead) + if (results.catalogs && results.catalogs.length >= CATALOG_CLEAR_BATCH_SIZE) { + log.info(`[MEMORY] Clearing ${results.catalogs.length} catalogs from memory`); + results.catalogs.length = 0; + } +} + +/** + * Handles catalog requests - validates STAC, extracts child catalogs and collections + * @async + * @param {Object} context - Request handler context + * @param {Object} context.request - Crawlee request object + * @param {Object} context.json - Parsed JSON response + * @param {Object} context.crawler - Crawlee crawler instance + * @param {Object} context.log - Logger instance + * @param {string} context.indent - Indentation for logging + * @param {Object} context.results - Results object to store data + * @param {Object} context.config - Configuration object with maxDepth + */ +export async function handleCatalog({ request, json, crawler, log, indent, results, config = {} }) { + const depth = request.userData?.depth || 0; + const catalogId = request.userData?.catalogId || 'unknown'; + const catalogSlug = request.userData?.catalogSlug || null; + const crawllogCatalogId = request.userData?.crawllogCatalogId || null; + const maxDepth = config.maxDepth || 0; // 0 = unlimited + + log.info(`${indent}Processing catalog: ${catalogId} (depth: ${depth}${maxDepth > 0 ? `/${maxDepth}` : ''})`); + + // Validate STAC structure before attempting migration + const validation = await validateStacStructure(json, log, indent); + if (!validation.valid) { + log.warning(`${indent}Pre-validation failed for catalog ${catalogId} at ${request.url}`); + log.warning(`${indent}Validation error: ${validation.error}`); + log.debug(`${indent}Response preview: ${JSON.stringify(json).substring(0, 200)}...`); + results.stats.nonCompliant++; + throw new Error(`STAC pre-validation failed: ${validation.error}`); + } + + // Migrate and validate with stac-js + // Note: create(data, migrate, updateVersionNumber) - second param enables migration + // Migration will upgrade older STAC versions (>= 0.6.0) to latest version (1.1.0) + let stacCatalog; + try { + stacCatalog = create(json, true); + results.stats.stacCompliant++; + + // Log STAC object type + if (typeof stacCatalog.isCatalog === 'function' && stacCatalog.isCatalog()) { + log.info(`${indent}STAC Catalog validated: ${catalogId}`); + } else if (typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection()) { + log.info(`${indent}STAC Collection validated: ${catalogId}`); + } + } catch (parseError) { + log.warning(`${indent}Non-compliant STAC catalog ${catalogId} at ${request.url}`); + log.warning(`${indent}Error details: ${parseError.message}`); + log.debug(`${indent}Response preview: ${JSON.stringify(json).substring(0, 200)}...`); + throw new Error(`STAC validation failed: ${parseError.message}`); + } + + results.stats.catalogsProcessed++; + // Only track minimal info to reduce memory - don't store full catalog data + results.catalogs.push({ + id: catalogId, + depth + }); + + // Save catalog to database (only for actual Catalogs, not Collections) + const isCollection = typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection(); + if (!isCollection) { + try { + + + await db.insertOrUpdateCatalog({ + id: stacCatalog.id, + title: stacCatalog.title || catalogId, + description: stacCatalog.description, + stac_version: stacCatalog.stac_version, + type: stacCatalog.type || 'Catalog', + keywords: stacCatalog.keywords, + stac_extensions: stacCatalog.stac_extensions, + links: stacCatalog.links + }); + } catch (err) { + log.warning(`${indent}Failed to save catalog ${catalogId} to database: ${err.message}`); + } + } + + // If this is a STAC Collection (not a catalog), extract and store it + // Collections don't have /collections endpoints, so we skip tryCollectionEndpoints for them + if (isCollection) { + // Persist collection URL in crawllog_collection queue + try { + if (typeof db.enqueueCollectionUrl === 'function') { + await db.enqueueCollectionUrl({ + sourceUrl: request.url, + crawllogCatalogId: crawllogCatalogId + }); + } + } catch (err) { + log.warning(`${indent}Failed to enqueue collection URL: ${err.message}`); + } + + // Check if this collection URL was already crawled (pause/resume support) + const alreadyCrawled = await db.isCollectionUrlCrawled(request.url); + if (alreadyCrawled) { + log.info(`${indent}Skipping already-crawled collection: ${stacCatalog.id} (resume mode)`); + try { + if (typeof db.markCatalogCrawled === 'function') { + await db.markCatalogCrawled(crawllogCatalogId); + } + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + return; + } + + const collection = normalizeCollection(stacCatalog, results.collections.length); + // Add the catalog slug to the collection for unique stac_id generation + collection.sourceSlug = catalogSlug; + // Store the actual crawled URL as the source URL (not relative links from the JSON) + collection.crawledUrl = request.url; + // Mark as non-API collection (from static catalog) + collection.is_api = false; + // Link to the crawllog_catalog for the parent catalog + collection.crawllogCatalogId = crawllogCatalogId; + results.collections.push(collection); + results.stats.collectionsFound++; + log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + + // Check if we should flush to database + await checkAndFlush(results, log); + } else { + // Only try /collections endpoint for Catalogs, not Collections + // Static STAC catalogs don't have /collections endpoints - they use rel="child" links + // STAC APIs have /collections endpoints and advertise them via rel="data" or rel="collections" + await tryCollectionEndpoints(stacCatalog, request.url, catalogId, depth, crawler, log, indent, catalogSlug, crawllogCatalogId); + } + + // Extract and enqueue child catalog links using stac-js + if (stacCatalog && typeof stacCatalog.getChildLinks === 'function') { + const childLinks = stacCatalog.getChildLinks(); + + if (childLinks.length > 0) { + log.info(`${indent}Found ${childLinks.length} child catalog links`); + + // Check maxDepth before enqueueing children + if (maxDepth > 0 && depth >= maxDepth) { + log.info(`${indent}Max depth (${maxDepth}) reached, skipping ${childLinks.length} child catalogs`); + // Clear memory and return early - don't enqueue children + await checkAndFlush(results, log); + try { + if (typeof db.markCatalogCrawled === 'function') { + await db.markCatalogCrawled(crawllogCatalogId); + } + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + return; + } + + // Log first child link structure for debugging + if (childLinks[0]) { + log.debug(`${indent}Sample child link structure:`, { + hasGetAbsoluteUrl: typeof childLinks[0].getAbsoluteUrl === 'function', + href: childLinks[0].href, + title: childLinks[0].title, + rel: childLinks[0].rel + }); + } + + let queuedCount = 0; + childLinks + .map((link, idx) => { + let childUrl; + try { + childUrl = typeof link.getAbsoluteUrl === 'function' + ? link.getAbsoluteUrl() + : link.href; + } catch (err) { + log.warning(`${indent}Error getting URL for link ${idx}: ${err.message}`); + return null; + } + + // Handle S3 protocol URLs - convert to HTTPS + if (childUrl && typeof childUrl === 'string' && childUrl.startsWith('s3://')) { + // s3://bucket-name/path -> https://bucket-name.s3.amazonaws.com/path + const s3Match = childUrl.match(/^s3:\/\/([^/]+)\/(.*)$/); + if (s3Match) { + const [, bucket, path] = s3Match; + childUrl = `https://${bucket}.s3.amazonaws.com/${path}`; + log.debug(`${indent}Converted S3 URL: ${link.href} -> ${childUrl}`); + } else { + log.warning(`${indent}Skipping malformed S3 URL at index ${idx}: ${childUrl}`); + return null; + } + } + + // If URL is relative, make it absolute using the catalog URL + if (childUrl && typeof childUrl === 'string' && !childUrl.startsWith('http')) { + const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; + const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); + childUrl = `${basePath}/${childUrl}`; + } + + // Validate URL is a string and looks like a URL + if (!childUrl || typeof childUrl !== 'string' || !childUrl.startsWith('http')) { + log.warning(`${indent}Skipping invalid URL at index ${idx}: ${childUrl}`); + return null; + } + + // Get title as string + const linkTitle = typeof link.title === 'string' && link.title.length > 0 + ? link.title + : `child-${idx}`; + + // Persist child catalog URL in DB queue + if (childUrl) { + queuedCount++; + if (typeof db.enqueueCollectionUrl === 'function') { + db.enqueueCollectionUrl({ + sourceUrl: childUrl, + crawllogCatalogId: crawllogCatalogId + }).catch(err => { + log.warning(`${indent}Failed to enqueue child catalog URL: ${err.message}`); + }); + } + } + + return { + url: childUrl, + label: 'CATALOG', + userData: { + depth: depth + 1, + catalogId: linkTitle, + parentId: catalogId, + catalogSlug: catalogSlug, + crawllogCatalogId: crawllogCatalogId // Pass through for linking collections + } + }; + }) + .filter(Boolean); // Remove null entries + + log.info(`${indent}Queued ${queuedCount}/${childLinks.length} child catalogs into DB queue`); + } + } + + // Ensure memory is cleared periodically even if no collections were found + await checkAndFlush(results, log); + + // Remove processed catalog URL from DB queue (if present) + try { + if (typeof db.removeFromCollectionQueue === 'function') { + await db.removeFromCollectionQueue(request.url); + } + } catch (err) { + log.warning(`${indent}Failed to remove catalog URL from queue: ${err.message}`); + } + + try { + if (typeof db.markCatalogCrawled === 'function') { + await db.markCatalogCrawled(crawllogCatalogId); + } + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + + // Help garbage collector by dereferencing large objects + stacCatalog = null; +} + +/** + * Handles collection endpoint requests + * @async + * @param {Object} context - Request handler context + * @param {Object} context.request - Crawlee request object + * @param {Object} context.json - Parsed JSON response + * @param {Object} context.crawler - Crawlee crawler instance + * @param {Object} context.log - Logger instance + * @param {string} context.indent - Indentation for logging + * @param {Object} context.results - Results object to store data + * @param {boolean} context.isApi - Whether this is an API collections endpoint (default: false) + */ +export async function handleCollections({ request, json, crawler, log, indent, results, isApi = false }) { + const catalogId = request.userData?.catalogId || 'unknown'; + const catalogSlug = request.userData?.catalogSlug || null; + const crawllogCatalogId = request.userData?.crawllogCatalogId || null; + + // Validate STAC structure before attempting migration using stac-node-validator + const validation = await validateStacStructure(json, log, indent); + if (!validation.valid) { + log.warning(`${indent}STAC validation failed for collections at ${request.url}`); + log.warning(`${indent}Error: ${validation.error}`); + if (validation.errors && validation.errors.length > 0) { + validation.errors.forEach((err, idx) => { + log.warning(`${indent} [${idx + 1}] ${err}`); + }); + } + results.stats.nonCompliant++; + return; + } + + // Parse and migrate response with stac-js + let stacObj; + try { + stacObj = create(json, true); + } catch (parseError) { + log.warning(`${indent}Migration failed for collections at ${request.url}: ${parseError.message}`); + results.stats.nonCompliant++; + return; + } + + let collectionsData = []; + + // Check if this is a CollectionCollection (STAC API response) + if (stacObj && typeof stacObj.getAll === 'function') { + collectionsData = stacObj.getAll(); + } else if (Array.isArray(json)) { + // Handle array of collections + collectionsData = json.map(col => { + try { + return create(col, true); + } catch { + return null; + } + }).filter(Boolean); + } else if (json.collections) { + // Handle nested collections property + collectionsData = json.collections.map(col => { + try { + return create(col, true); + } catch { + return null; + } + }).filter(Boolean); + } else if (typeof stacObj?.isCatalog === 'function' && stacObj.isCatalog()) { + log.warning(`${indent}Collections endpoint returned a Catalog at ${request.url}; skipping`); + results.stats.nonCompliant++; + return; + } + + if (collectionsData.length > 0) { + const filteredCollections = []; + let nonCollectionCount = 0; + + for (const colObj of collectionsData) { + const raw = typeof colObj?.toJSON === 'function' ? colObj.toJSON() : colObj; + const isCollection = (typeof colObj?.isCollection === 'function' && colObj.isCollection()) + || raw?.type === 'Collection'; + + if (!isCollection) { + nonCollectionCount++; + continue; + } + + filteredCollections.push(colObj); + } + + if (nonCollectionCount > 0) { + log.warning(`${indent}Skipped ${nonCollectionCount} non-Collection object(s) at ${request.url}`); + } + + if (filteredCollections.length === 0) { + log.warning(`${indent}No valid Collection objects found at ${request.url}`); + results.stats.nonCompliant++; + return; + } + + log.info(`${indent}Found ${filteredCollections.length} collections for catalog ${catalogId}`); + + // Get base URL for constructing absolute collection URLs + // Remove trailing /collections from the request URL to get the API base + const baseUrl = request.url.replace(/\/collections\/?$/, ''); + + // Note: crawllog_collection is a queue only; already-crawled URLs are stored in collection table + + // Normalize and store collections, skipping already-crawled ones + let skippedCount = 0; + const collections = []; + for (let index = 0; index < filteredCollections.length; index++) { + const colObj = filteredCollections[index]; + const collection = normalizeCollection(colObj, index); + // Add the catalog slug to the collection for unique stac_id generation + collection.sourceSlug = catalogSlug; + // Link to the crawllog_catalog for the parent catalog + collection.crawllogCatalogId = crawllogCatalogId; + + // Store the absolute URL as crawledUrl + // Use stac-js getAbsoluteUrl() if available, otherwise construct from base + id + if (typeof colObj.getAbsoluteUrl === 'function') { + try { + collection.crawledUrl = colObj.getAbsoluteUrl(); + + } catch { + // Fallback to constructing URL from base + collection.crawledUrl = `${baseUrl}/collections/${collection.id}`; + } + } else { + collection.crawledUrl = `${baseUrl}/collections/${collection.id}`; + } + + // Persist discovered collection URL in crawllog_collection queue + try { + if (typeof db.enqueueCollectionUrl === 'function') { + await db.enqueueCollectionUrl({ + sourceUrl: collection.crawledUrl, + crawllogCatalogId: crawllogCatalogId + }); + } + } catch (err) { + log.warning(`${indent}Failed to enqueue collection URL: ${err.message}`); + } + + // Skip if this URL was already crawled (pause/resume support) + const alreadyCrawled = await db.isCollectionUrlCrawled(collection.crawledUrl); + if (alreadyCrawled) { + skippedCount++; + continue; + } + + // Mark collection as API or static catalog based on context + collection.is_api = isApi; + + collections.push(collection); + } + + if (skippedCount > 0) { + log.info(`${indent}Skipped ${skippedCount} already-crawled collections (resume mode)`); + } + + results.collections.push(...collections); + results.stats.collectionsFound += collections.length; + + // Display sample + if (collections.length > 0) { + log.info(`${indent} Sample: ${collections[0].id} - ${collections[0].title}`); + } + + // Check if we should flush to database + await checkAndFlush(results, log); + } + + // Remove processed collections endpoint URL from DB queue (if present) + try { + if (typeof db.removeFromCollectionQueue === 'function') { + await db.removeFromCollectionQueue(request.url); + } + } catch (err) { + log.warning(`${indent}Failed to remove collections URL from queue: ${err.message}`); + } + + try { + if (typeof db.markCatalogCrawled === 'function') { + await db.markCatalogCrawled(crawllogCatalogId); + } + } catch (err) { + log.warning(`${indent}Failed to mark catalog as crawled: ${err.message}`); + } + + // Help garbage collector by dereferencing large objects + stacObj = null; + collectionsData = null; +} \ No newline at end of file diff --git a/crawler/utils/normalization.js b/crawler/utils/normalization.js new file mode 100644 index 0000000..c70dbf4 --- /dev/null +++ b/crawler/utils/normalization.js @@ -0,0 +1,203 @@ +/** + * @fileoverview Normalization utilities for STAC catalogs and collections + * @module utils/normalization + */ + +/** + * Derives categories from a catalog object by checking various possible fields + * @param {Object} catalog - Catalog object to extract categories from + * @returns {Array} Array of category strings, empty array if none found + */ +export function deriveCategories(catalog) { + if (!catalog || typeof catalog !== 'object') { + return []; + } + + if (Array.isArray(catalog.categories)) { + return catalog.categories.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.keywords)) { + return catalog.keywords.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.tags)) { + return catalog.tags.filter(Boolean).map(String); + } + + if (typeof catalog.access === 'string' && catalog.access.trim().length) { + return [catalog.access.trim()]; + } + + return []; +} + +/** + * Normalizes a catalog object from STAC Index API format + * @param {Object} catalog - Catalog object from the STAC Index API + * @param {number} index - Index position in the original array + * @returns {Object} Normalized catalog object with standard properties + */ +export function normalizeCatalog(catalog, index) { + return { + index, + id: catalog.id, + url: catalog.url, + slug: catalog.slug, + title: catalog.title, + summary: catalog.summary, + access: catalog.access, + created: catalog.created, + updated: catalog.updated, + isPrivate: catalog.isPrivate, + isApi: catalog.isApi, + accessInfo: catalog.accessInfo, + categories: deriveCategories(catalog), + // Preserve any additional dynamic properties + ...Object.fromEntries( + Object.entries(catalog).filter(([key]) => + !['id', 'url', 'slug', 'title', 'summary', 'access', 'created', + 'updated', 'isPrivate', 'isApi', 'accessInfo', 'stac_version'].includes(key) + ) + ) + }; +} + +/** + * Normalizes a collection object using stac-js methods for metadata extraction + * Preserves all fields needed for database insertion including summaries, extensions, etc. + * @param {Object} colObj - Collection object (stac-js or plain object) + * @param {number} index - Index position + * @returns {Object} Normalized collection object with all fields for db.js + */ +export function normalizeCollection(colObj, index) { + // Get raw data from stac-js object if available + // stac-js stores the original data in toJSON() or we can access it directly + const rawData = typeof colObj.toJSON === 'function' ? colObj.toJSON() : colObj; + + // Determine the STAC type using stac-js methods if available + // This is more reliable than trusting the type field in the JSON + let stacType = null; + if (typeof colObj.isCollection === 'function' && colObj.isCollection()) { + stacType = 'Collection'; + } else if (typeof colObj.isCatalog === 'function' && colObj.isCatalog()) { + stacType = 'Catalog'; + } else { + // Fallback to the type field in the data, or default to 'Collection' + stacType = colObj.type || rawData?.type || 'Collection'; + } + + // Extract bbox: try stac-js method first, then fallback to raw data + let bbox = null; + if (typeof colObj.getBoundingBox === 'function') { + bbox = colObj.getBoundingBox(); + } + // Fallback to raw data if stac-js method returned null/undefined + if (!bbox && rawData?.extent?.spatial?.bbox?.[0]) { + bbox = rawData.extent.spatial.bbox[0]; + } + // Final fallback: direct access on colObj + if (!bbox && colObj?.extent?.spatial?.bbox?.[0]) { + bbox = colObj.extent.spatial.bbox[0]; + } + + // Extract temporal: try stac-js method first, then fallback to raw data + let temporal = null; + if (typeof colObj.getTemporalExtent === 'function') { + temporal = colObj.getTemporalExtent(); + } + // Fallback to raw data if stac-js method returned null/undefined + if (!temporal && rawData?.extent?.temporal?.interval?.[0]) { + temporal = rawData.extent.temporal.interval[0]; + } + // Final fallback: direct access on colObj + if (!temporal && colObj?.extent?.temporal?.interval?.[0]) { + temporal = colObj.extent.temporal.interval[0]; + } + + // Get self URL using stac-js link navigation + let selfUrl = null; + if (typeof colObj.getAbsoluteUrl === 'function') { + selfUrl = colObj.getAbsoluteUrl(); + } else if (colObj.links) { + const selfLink = colObj.links.find(l => l.rel === 'self'); + selfUrl = selfLink?.href || null; + } + // Fallback to raw data for URL + if (!selfUrl && rawData?.links) { + const selfLink = rawData.links.find(l => l.rel === 'self'); + selfUrl = selfLink?.href || null; + } + + // Extract links array (needed for source_url extraction in db.js) + let links = null; + if (Array.isArray(colObj.links)) { + // Convert stac-js link objects to plain objects if needed + // Filter out null/undefined and ensure at least rel or href exists + links = colObj.links + .filter(l => l && (l.rel || l.href)) + .map(l => ({ + rel: l.rel || undefined, + href: l.href || undefined, + type: l.type || undefined, + title: l.title || undefined + })); + } else if (Array.isArray(rawData?.links)) { + links = rawData.links; + } + + + + return { + index, + id: colObj.id || rawData?.id || 'Unknown', + url: selfUrl, + title: colObj.title || rawData?.title || null, + description: colObj.description || colObj.summary || rawData?.description || rawData?.summary || null, + bbox, + temporal, + license: colObj.license || rawData?.license || null, + keywords: colObj.keywords || rawData?.keywords || [], + + // Additional fields needed for db.js - pass through from raw data + links, + stac_version: colObj.stac_version || rawData?.stac_version || null, + type: stacType, + summaries: colObj.summaries || rawData?.summaries || null, + stac_extensions: colObj.stac_extensions || rawData?.stac_extensions || [], + providers: colObj.providers || rawData?.providers || [], + assets: colObj.assets || rawData?.assets || null, + + // Preserve the original STAC JSON for full_json storage in database + // This ensures nothing is lost during normalization + originalJson: rawData + }; +} + +/** + * Processes an array of catalogs from the STAC Index API + * @param {Array} catalogs - Array of catalog objects from the STAC Index API + * @returns {Array} Array of normalized catalog objects + * @throws {Error} Throws error if input is not an array + */ +export function processCatalogs(catalogs) { + if (!Array.isArray(catalogs)) { + throw new Error('Expected an array'); + } + + const normalized = catalogs.map((catalog, index) => normalizeCatalog(catalog, index)); + + console.log(`Total: ${normalized.length} catalogs found\n`); + + if (normalized.length > 0) { + console.log('Example - First Catalog:'); + const first = normalized[0]; + console.log(` ID: ${first.id}`); + console.log(` URL: ${first.url}`); + console.log(` Title: ${first.title}`); + console.log(` Is API: ${first.isApi}`); + console.log(` Categories: ${JSON.stringify(first.categories)}`); + } + + return normalized; +} diff --git a/crawler/utils/parallel.js b/crawler/utils/parallel.js new file mode 100644 index 0000000..f6bcd0a --- /dev/null +++ b/crawler/utils/parallel.js @@ -0,0 +1,183 @@ +/** + * @fileoverview Parallel execution utilities for domain-based crawling + * Allows crawling multiple domains simultaneously while respecting per-domain rate limits + * @module utils/parallel + */ + +/** + * Extracts the domain from a URL + * @param {string} url - URL to extract domain from + * @returns {string} The domain (hostname) of the URL + */ +export function getDomain(url) { + try { + const urlObj = new URL(url); + return urlObj.hostname; + } catch { + return 'unknown'; + } +} + +/** + * Groups items by their URL domain + * @param {Array} items - Array of objects with url property + * @returns {Map>} Map of domain -> items + */ +export function groupByDomain(items) { + const domainMap = new Map(); + + for (const item of items) { + const domain = getDomain(item.url); + if (!domainMap.has(domain)) { + domainMap.set(domain, []); + } + domainMap.get(domain).push(item); + } + + return domainMap; +} + +/** + * Creates batches of domains for parallel processing + * @param {Map} domainMap - Map of domain -> items + * @param {number} batchSize - Number of domains to process in parallel + * @returns {Array>} Array of batches, each containing [domain, items] pairs + */ +export function createDomainBatches(domainMap, batchSize = 5) { + const entries = Array.from(domainMap.entries()); + const batches = []; + + for (let i = 0; i < entries.length; i += batchSize) { + batches.push(entries.slice(i, i + batchSize)); + } + + return batches; +} + +/** + * Aggregates statistics from multiple crawler results + * @param {Array} results - Array of result objects with stats + * @returns {Object} Aggregated statistics + */ +export function aggregateStats(results) { + const aggregated = { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0, + catalogsProcessed: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + }; + + for (const result of results) { + if (!result || !result.stats) continue; + + for (const key of Object.keys(aggregated)) { + if (typeof result.stats[key] === 'number') { + aggregated[key] += result.stats[key]; + } + } + } + + return aggregated; +} + +/** + * Executes async functions in parallel with a concurrency limit + * @param {Array} tasks - Array of async functions to execute + * @param {number} concurrency - Maximum number of tasks to run in parallel + * @param {Function} onProgress - Optional callback for progress updates + * @returns {Promise} Array of results from all tasks + */ +export async function executeWithConcurrency(tasks, concurrency, onProgress = null) { + const results = []; + let completed = 0; + let running = 0; + let index = 0; + + return new Promise((resolve) => { + const runNext = async () => { + if (index >= tasks.length) { + if (running === 0) { + resolve(results); + } + return; + } + + const currentIndex = index++; + running++; + + try { + const result = await tasks[currentIndex](); + results[currentIndex] = result; + } catch (error) { + console.error(`[executeWithConcurrency] Task ${currentIndex} failed: ${error.message}`); + console.error(error.stack); + results[currentIndex] = { error: error.message, stats: {} }; + } + + running--; + completed++; + + if (onProgress) { + onProgress(completed, tasks.length); + } + + runNext(); + }; + + // Start initial batch + const initialBatch = Math.min(concurrency, tasks.length); + for (let i = 0; i < initialBatch; i++) { + runNext(); + } + + // Handle empty tasks array + if (tasks.length === 0) { + resolve(results); + } + }); +} + +/** + * Calculates optimal rate limiting based on max requests per minute + * @param {number} maxRequestsPerMinute - Maximum requests per minute (per domain) + * @returns {Object} Rate limiting configuration + */ +export function calculateRateLimits(maxRequestsPerMinute = 120) { + // Use only maxRequestsPerMinute for rate limiting + // sameDomainDelaySecs is set to 0 - we rely solely on the rate limiter + // This gives us maximum throughput while respecting the rate limit + + return { + maxRequestsPerMinute: maxRequestsPerMinute, + }; +} + +/** + * Logs domain statistics for debugging + * @param {Map} domainMap - Map of domain -> items + * @param {string} itemType - Type of items (e.g., 'catalogs', 'APIs') + */ +export function logDomainStats(domainMap, itemType = 'items') { + console.log(`\n=== Domain Distribution for ${itemType} ===`); + console.log(`Total domains: ${domainMap.size}`); + + const sorted = Array.from(domainMap.entries()) + .sort((a, b) => b[1].length - a[1].length); + + // Show top 10 domains + const top = sorted.slice(0, 10); + for (const [domain, items] of top) { + console.log(` ${domain}: ${items.length} ${itemType}`); + } + + if (sorted.length > 10) { + console.log(` ... and ${sorted.length - 10} more domains`); + } + console.log(''); +} diff --git a/crawler/utils/time.js b/crawler/utils/time.js new file mode 100644 index 0000000..b1fe747 --- /dev/null +++ b/crawler/utils/time.js @@ -0,0 +1,47 @@ +/** + * @fileoverview Time formatting utilities for STAC crawler + * @module utils/time + */ + +/** + * Format milliseconds into a human-readable duration string + * @param {number} ms - Duration in milliseconds + * @returns {string} Formatted duration string (e.g., "2h 30m 15s" or "45s") + */ +export function formatDuration(ms) { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + + const displaySeconds = seconds % 60; + const displayMinutes = minutes % 60; + const displayHours = hours % 24; + + if (days > 0) { + return `${days}d ${displayHours}h ${displayMinutes}m`; + } else if (hours > 0) { + return `${displayHours}h ${displayMinutes}m ${displaySeconds}s`; + } else if (minutes > 0) { + return `${displayMinutes}m ${displaySeconds}s`; + } else { + return `${displaySeconds}s`; + } +} + +/** + * Get formatted timestamp for logging + * @returns {string} ISO formatted timestamp + */ +export function getTimestamp() { + return new Date().toISOString(); +} + +/** + * Get localized date/time string for display + * @param {Date} date - Date object (defaults to now) + * @returns {string} Localized date/time string + */ +export function getLocalizedTime(date = new Date()) { + return date.toLocaleString(); +} diff --git a/db/ER-Diagramm_stacDB.png b/db/ER-Diagramm_stacDB.png new file mode 100644 index 0000000..93ff042 Binary files /dev/null and b/db/ER-Diagramm_stacDB.png differ diff --git a/db/README.md b/db/README.md index 05e1542..b059448 100644 --- a/db/README.md +++ b/db/README.md @@ -1,97 +1,127 @@ # STAC-Atlas Database -This directory contains the PostgreSQL database setup for STAC-Atlas, a system for managing and searching STAC (SpatioTemporal Asset Catalog) catalogs and collections. +This component contains the PostgreSQL database setup for STAC-Atlas – a system for managing and searching STAC (SpatioTemporal Asset Catalog) Collections. ## Overview -The database is built on **PostgreSQL 16** with **PostGIS 3.4** extensions, providing spatial capabilities for geospatial data management. It stores STAC catalogs, collections, and their associated metadata with full-text search and spatial indexing support. +The database is built on **PostgreSQL 16** with **PostGIS 3.4** for spatial queries. It stores STAC Collections and their metadata with full-text search and spatial indexing. ## Database Structure -### Core Tables +### Entity-Relationship Diagram -#### Catalogs -- **`catalog`**: Main catalog metadata (title, description, STAC version, type) -- **`catalog_links`**: Related links for each catalog -- **`crawllog_catalog`**: Tracks when catalogs were last crawled +![ER-Diagram](ER-Diagramm_stacDB.png) + +### Tables + +#### Crawler Tracking + +| Table | Description | +|-------|-------------| +| `crawllog_catalog` | Tracks crawler progress for catalogs. Enables resume after crash. | +| `crawllog_collection` | Tracks crawl status of individual collections with reference to catalog. | #### Collections -- **`collection`**: Collection metadata with spatial and temporal extents - - Stores spatial extent as PostGIS geometry (POLYGON, EPSG:4326) - - Includes temporal extent (start/end timestamps) - - Full JSON representation of collection stored in `full_json` (JSONB) -- **`collection_summaries`**: Collection summary statistics and ranges -- **`crawllog_collection`**: Tracks when collections were last crawled - -#### Supporting Tables -- **`keywords`**: Searchable keywords for catalogs and collections -- **`stac_extensions`**: STAC extensions used by catalogs/collections -- **`providers`**: Data providers -- **`assets`**: Assets associated with collections - -#### Relation Tables -- **`catalog_keywords`**: Many-to-many relationship between catalogs and keywords -- **`catalog_stac_extension`**: Links catalogs to STAC extensions -- **`collection_keywords`**: Many-to-many relationship between collections and keywords -- **`collection_stac_extension`**: Links collections to STAC extensions -- **`collection_providers`**: Links collections to providers with roles -- **`collection_assets`**: Links collections to assets with roles + +| Table | Description | +|-------|-------------| +| `collection` | Main metadata of STAC Collections (title, description, spatial/temporal extent, license). Stores complete JSON representation in `full_json`. | +| `collection_summaries` | Statistical summaries (value ranges, sets) for collection properties. | + +#### Lookup Tables + +| Table | Description | +|-------|-------------| +| `keywords` | Reusable keywords for search. | +| `stac_extensions` | STAC extensions (e.g., EO, SAR, Point Cloud). | +| `providers` | Data providers and organizations. | +| `assets` | Downloadable resources (files, thumbnails, metadata). | + +#### Junction Tables (n:n) + +| Table | Description | +|-------|-------------| +| `collection_keywords` | Links collections to keywords. | +| `collection_stac_extension` | Links collections to STAC extensions. | +| `collection_providers` | Links collections to providers incl. roles. | +| `collection_assets` | Links collections to assets incl. roles. | ### Extensions -The database uses the following PostgreSQL extensions: -- **PostGIS**: Spatial data types and functions +- **PostGIS**: Spatial data types and functions (geometries, bounding boxes) - **pg_trgm**: Trigram-based text search for fuzzy matching ### Indexes -Comprehensive indexing for optimal query performance: -- **Full-text search** on titles and descriptions -- **Spatial indexes** (GIST) on geographic extents -- **JSONB indexes** (GIN) for flexible JSON queries -- **Temporal indexes** on date ranges -- **Foreign key indexes** for efficient joins +Optimized indexes for fast queries: -## Getting Started +| Type | Usage | +|------|-------| +| **B-Tree** | Title, timestamps, provider names | +| **GIN** | Full-text search (`search_vector`), JSONB fields, asset roles | +| **GIST** | Spatial extent (`spatial_extent`) | -### Starting the Database +### Triggers + +- **`collection_search_vector_update`**: Automatically updates the search vector when collections are modified +- **`collection_keywords_update_vector`**: Updates the search vector when keywords are added/removed + +## Quick Start + +### Start the Database ```bash cd ./db/ +cp example.env .env +# Fill in passwords in .env file docker-compose up ``` ### Connection Details -- **Host**: `atlas.stacindex.org` -- **Port**: `5432` and `5433` +| Parameter | Value | +|-----------|-------| +| Host | choose your server | +| Port | Configurable via `DB_PORT` in `.env` | +| Database | Configurable via `POSTGRES_DB` in `.env` | + +## Configuration -## Port Configuration +### Environment Variables (.env) -This project exposes the database service on a port that can be changed. Update the port in the described place and restart the service. +| Variable | Description | +|----------|-------------| +| `POSTGRES_DB` | Database name | +| `POSTGRES_USER` | Admin user (superuser) | +| `POSTGRES_PASSWORD` | Admin password | +| `DB_PORT` | External port (host side) | +| `STAC_API_PASSWORD` | Password for API user (read-only access) | +| `STAC_CRAWLER_PASSWORD` | Password for crawler user (read-write access) | -The database uses port mapping in the format `HOST:CONTAINER`: -- **`5432:5432`** means: - - Left side (`5432`): Port on your local machine (host) (must be changed in the `.env`) - - Right side (`5432`): Port inside the Docker container +**Important**: Edit the `.env` file, not the `docker-compose.yml`. A template is provided in `example.env`. -How to change the environment parameters in the Docker Compose file -- Open the `docker-compose.yml`. -- Locate e.g. `ports:` and change the host side: -- Format: `":"` -- Example: change `5432:5432` to `5433:5432` to expose the container's 5432 on host port 5433. -- If the compose file references environment variables (e.g. `${DB_PORT}`), change the value in the corresponding `.env` file. +### User Roles -**Important**: Do not modify the `docker-compose.yml` file directly. Instead, update the port configuration in the `.env` file by changing the `${DB_PORT}`, `${POSTGRES_DB}`, `${POSTGRES_USER}` and `${POSTGRES_PASSWORD}` variable, then restart the service with `docker-compose up`. -- The change in the `.env` does not count for the ``, you can change that directly in the `docker-compose.yml` if needed. -- There is an `example.env` provided that can be renamed into `.env` and then modified. +| User | Permissions | +|------|-------------| +| `stac_api` | Read-only access (SELECT) – for the API | +| `stac_crawler` | Full read-write access – for the crawler | ## Initialization Scripts -All SQL scripts in the `./db/init/` folder are automatically executed on the start of the database. The numbering ensures guaranteed execution order: +All scripts in the `./init/` folder are automatically executed on first start in numerical order: + +| Script | Description | +|--------|-------------| +| `00_users.sh` | Creates users (`stac_api`, `stac_crawler`) with appropriate permissions | +| `01_extensions.sql` | Installs PostGIS and pg_trgm extensions | +| `02_tables_catalog.sql` | Creates `crawllog_catalog` for crawler tracking | +| `03_tables_collections.sql` | Creates collection tables and lookup tables | +| `04_relation_tables.sql` | Creates junction tables (n:n relationships) | +| `05_indexes.sql` | Creates performance indexes | +| `06_triggers.sql` | Creates triggers for full-text search | + +## Migrations + +The `./migrations/` folder contains SQL scripts for schema changes after initial setup. Those are not planed yet, but could be used in the future, when chages to the given database are required. -1. **`01_extensions.sql`** - Installs PostGIS and pg_trgm extensions -2. **`02_tables_catalog.sql`** - Creates catalog-related tables -3. **`03_tables_collections.sql`** - Creates collection-related tables -4. **`04_relation_tables.sql`** - Creates relationship n:n tables -5. **`05_indexes.sql`** - Creates the performance indexes diff --git a/db/docker-compose.yml b/db/docker-compose.yml index 3008d38..c96dfb7 100644 --- a/db/docker-compose.yml +++ b/db/docker-compose.yml @@ -21,6 +21,10 @@ services: volumes: - stac_data:/var/lib/postgresql/data - ./init:/docker-entrypoint-initdb.d + networks: [stac-network] + + networks: + - stac-network networks: - stac-network @@ -31,4 +35,4 @@ volumes: networks: stac-network: name: stac-network - driver: bridge \ No newline at end of file + driver: bridge diff --git a/db/example.env b/db/example.env index 35158a8..ad3b4b9 100644 --- a/db/example.env +++ b/db/example.env @@ -12,4 +12,4 @@ DB_PORT= # 5432 / 5433 (at the moment both are available) STAC_API_PASSWORD= # Password for api user (read-only); add api_password here # stac_crawler: full read-write access for crawler -STAC_CRAWLER_PASSWORD= # Password for crawler user (read-write); add crawler_password here \ No newline at end of file +STAC_CRAWLER_PASSWORD= # Password for crawler user (read-write); add crawler_password here diff --git a/db/init/02_tables_catalog.sql b/db/init/02_tables_catalog.sql index dca17c4..93a692e 100644 --- a/db/init/02_tables_catalog.sql +++ b/db/init/02_tables_catalog.sql @@ -1,107 +1,13 @@ --- creates every table related to catalogs --- Main catalog table: Stores STAC catalog metadata including version, type, title, and description --- Each catalog represents a STAC catalog endpoint that has been discovered and indexed -CREATE TABLE catalog ( - id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, - stac_version TEXT, - type TEXT, - title TEXT, - description TEXT, - created_at TIMESTAMP DEFAULT now(), - updated_at TIMESTAMP DEFAULT now(), - search_vector tsvector -); - --- Catalog links table: Stores related links for catalogs (e.g., self, root, child, item links) --- Links define the navigation structure between STAC resources -CREATE TABLE catalog_links ( - id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, - catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, - source_url TEXT, - rel TEXT, - href TEXT, - type TEXT, - title TEXT -); - --- Keywords lookup table: Stores unique searchable keywords --- Used by both catalogs and collections for categorization and search -CREATE TABLE keywords ( - id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, - keyword TEXT UNIQUE -); - --- STAC extensions lookup table: Stores unique STAC extension identifiers --- Extensions provide additional standardized fields beyond core STAC spec -CREATE TABLE stac_extensions ( - id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, - stac_extension TEXT UNIQUE -); +-- The crawllog_catalog is required to save the crawler's current location. +-- If the crawler crashes, for example because the server goes down, it can +-- now restart at the correct location and does not have to crawl everything again. --- Crawl log for catalogs: Tracks when each catalog was last crawled for updates --- Used to schedule re-crawling and maintain freshness of catalog data CREATE TABLE crawllog_catalog ( id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, - catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, - last_crawled TIMESTAMP -); - --- ======================================== --- FULL-TEXT SEARCH TRIGGERS --- ======================================== - --- Trigger function to auto-update search_vector when catalog is inserted or updated --- Includes title, description, and all associated keywords for comprehensive search -CREATE OR REPLACE FUNCTION update_catalog_search_vector() -RETURNS TRIGGER AS $$ -BEGIN - NEW.search_vector := to_tsvector('simple', - coalesce(NEW.title, '') || ' ' || - coalesce(NEW.description, '') || ' ' || - coalesce( - ( - SELECT string_agg(k.keyword, ' ') - FROM catalog_keywords ck - JOIN keywords k ON k.id = ck.keyword_id - WHERE ck.catalog_id = NEW.id - ), - '' - ) - ); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER catalog_search_vector_update -BEFORE INSERT OR UPDATE ON catalog -FOR EACH ROW -EXECUTE FUNCTION update_catalog_search_vector(); - --- Trigger function to update search_vector when keywords are added/removed --- Ensures search index stays in sync with keyword changes -CREATE OR REPLACE FUNCTION update_catalog_search_vector_on_keyword_change() -RETURNS TRIGGER AS $$ -BEGIN - UPDATE catalog - SET search_vector = to_tsvector('simple', - coalesce(title, '') || ' ' || - coalesce(description, '') || ' ' || - coalesce( - ( - SELECT string_agg(k.keyword, ' ') - FROM catalog_keywords ck - JOIN keywords k ON k.id = ck.keyword_id - WHERE ck.catalog_id = catalog.id - ), - '' - ) - ) - WHERE id = COALESCE(NEW.catalog_id, OLD.catalog_id); - - RETURN COALESCE(NEW, OLD); -END; -$$ LANGUAGE plpgsql; - --- NOTE: The trigger for catalog_keywords is defined in 06_triggers.sql --- because it depends on the catalog_keywords table which is created there + slug TEXT, + source_url TEXT UNIQUE NOT NULL, + is_api BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP DEFAULT now(), + updated_at TIMESTAMP DEFAULT now() +); \ No newline at end of file diff --git a/db/init/03_tables_collections.sql b/db/init/03_tables_collections.sql index cb8f73a..076c8e6 100644 --- a/db/init/03_tables_collections.sql +++ b/db/init/03_tables_collections.sql @@ -26,6 +26,20 @@ CREATE TABLE collection ( search_vector tsvector ); +-- Keywords lookup table: Stores unique searchable keywords +-- Used by both catalogs and collections for categorization and search +CREATE TABLE keywords ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + keyword TEXT UNIQUE +); + +-- STAC extensions lookup table: Stores unique STAC extension identifiers +-- Extensions provide additional standardized fields beyond core STAC spec +CREATE TABLE stac_extensions ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + stac_extension TEXT UNIQUE +); + -- Collection summaries: Stores summaries for collection properties -- represent ranges (min/max), sets of values, or JSON schemas -- Used to describe the range of values found in collection items @@ -34,7 +48,6 @@ CREATE TABLE collection_summaries ( collection_id INTEGER REFERENCES collection(id) ON DELETE CASCADE, name TEXT, kind TEXT, - source_url TEXT, range_min NUMERIC, range_max NUMERIC, set_value TEXT, @@ -59,13 +72,13 @@ CREATE TABLE assets ( metadata JSONB ); --- Crawl log for collections: Tracks when each collection was last crawled for updates +-- Crawl log for collections: Tracks the last crawled state of each collection and references the matching catalog -- Used to schedule re-crawling and maintain freshness of collection data --- (same usecase as the crawllog for catalogs) CREATE TABLE crawllog_collection ( id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, collection_id INTEGER REFERENCES collection(id) ON DELETE CASCADE, - last_crawled TIMESTAMP + source_url TEXT UNIQUE NOT NULL, + crawllog_catalog_id INTEGER REFERENCES crawllog_catalog(id) ON DELETE CASCADE ); -- ======================================== diff --git a/db/init/04_relation_tables.sql b/db/init/04_relation_tables.sql index d95f31b..f132d7d 100644 --- a/db/init/04_relation_tables.sql +++ b/db/init/04_relation_tables.sql @@ -1,18 +1,4 @@ --- creates every table needed for relations between tables for catalogs and collections - --- Junction table: Links catalogs to their associated keywords (many-to-many) -CREATE TABLE catalog_keywords ( - catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, - keyword_id INTEGER REFERENCES keywords(id) ON DELETE CASCADE, - PRIMARY KEY (catalog_id, keyword_id) -); - --- Junction table: Links catalogs to STAC extensions they implement (many-to-many) -CREATE TABLE catalog_stac_extension ( - catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, - stac_extension_id INTEGER REFERENCES stac_extensions(id) ON DELETE CASCADE, - PRIMARY KEY (catalog_id, stac_extension_id) -); +-- creates every table needed for relations between tables for collections -- Junction table: Links collections to their associated keywords (many-to-many) CREATE TABLE collection_keywords ( diff --git a/db/init/05_indexes.sql b/db/init/05_indexes.sql index 7c0d74a..6d4cbb0 100644 --- a/db/init/05_indexes.sql +++ b/db/init/05_indexes.sql @@ -1,23 +1,6 @@ -- Performance indexes for all tables -- These indexes optimize common query patterns and improve search performance --- ======================================== --- CATALOG INDEXES --- ======================================== - --- Basic catalog lookups -CREATE INDEX idx_catalog_title ON catalog (title); -CREATE INDEX idx_catalog_updated_at ON catalog (updated_at); - --- Full-text search index on computed search_vector column (includes title, description, and keywords) -CREATE INDEX idx_catalog_search_vector ON catalog USING GIN (search_vector); - -CREATE INDEX idx_catalog_links_catalog_id ON catalog_links (catalog_id); -CREATE INDEX idx_catalog_keywords_catalog ON catalog_keywords (catalog_id); -CREATE INDEX idx_catalog_stac_ext_catalog ON catalog_stac_extension (catalog_id); - -CREATE INDEX idx_crawllog_catalog_last ON crawllog_catalog (last_crawled); - -- ======================================== -- COLLECTION INDEXES -- ======================================== @@ -41,8 +24,6 @@ CREATE INDEX idx_collection_stac_ext_collection ON collection_stac_extension (co CREATE INDEX idx_collection_providers_collection ON collection_providers (collection_id); CREATE INDEX idx_collection_assets_collection ON collection_assets (collection_id); -CREATE INDEX idx_crawllog_collection_last ON crawllog_collection (last_crawled); - -- ======================================== -- PROVIDER & ASSET INDEXES -- ======================================== diff --git a/db/init/06_triggers.sql b/db/init/06_triggers.sql index 5e167a0..720e780 100644 --- a/db/init/06_triggers.sql +++ b/db/init/06_triggers.sql @@ -3,12 +3,6 @@ -- ======================================== -- These triggers must be created here (after junction tables exist) --- Trigger to update catalog search_vector when keywords change -CREATE TRIGGER catalog_keywords_update_vector -AFTER INSERT OR DELETE ON catalog_keywords -FOR EACH ROW -EXECUTE FUNCTION update_catalog_search_vector_on_keyword_change(); - -- Trigger to update collection search_vector when keywords change CREATE TRIGGER collection_keywords_update_vector AFTER INSERT OR DELETE ON collection_keywords