From e0b8fbc2545a646253f5de4f903187980809290c Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 12 Nov 2025 14:36:17 +0100 Subject: [PATCH 01/68] Node init --- crawler/.gitignore | 1 + crawler/package-lock.json | 295 ++++++++++++++++++++++++++++++++++++++ crawler/package.json | 14 ++ 3 files changed, 310 insertions(+) create mode 100644 crawler/package-lock.json create mode 100644 crawler/package.json diff --git a/crawler/.gitignore b/crawler/.gitignore index e69de29..b512c09 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/crawler/package-lock.json b/crawler/package-lock.json new file mode 100644 index 0000000..9281009 --- /dev/null +++ b/crawler/package-lock.json @@ -0,0 +1,295 @@ +{ + "name": "crawler", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "crawler", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "axios": "^1.13.2" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", + "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.4", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + } + } +} diff --git a/crawler/package.json b/crawler/package.json new file mode 100644 index 0000000..9cf3b83 --- /dev/null +++ b/crawler/package.json @@ -0,0 +1,14 @@ +{ + "name": "crawler", + "version": "1.0.0", + "description": "STAC Index crawler", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^1.13.2" + } +} From ff72c8d97e47dff7dfdfe74dd070262a7b0f1154 Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 12 Nov 2025 14:41:28 +0100 Subject: [PATCH 02/68] Basic Crawling of STAC Index API --- crawler/index.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 crawler/index.js diff --git a/crawler/index.js b/crawler/index.js new file mode 100644 index 0000000..75dcc65 --- /dev/null +++ b/crawler/index.js @@ -0,0 +1,19 @@ +// npm install axios cheerio +const axios = require('axios'); + +// specify the URL of the site to crawl +const targetUrl = 'https://www.stacindex.org/api/catalogs'; + +// define a crawler function +const crawler = async () => { + try { + // request the target website + const response = await axios.get(targetUrl); + console.log(response.data); + } catch (error) { + // handle any error that occurs during the HTTP request + console.error(`Error fetching ${targetUrl}: ${error.message}`); + } +}; + +crawler(); From 30530dac494778ac2010d015693b2ea79c67a62a Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 12 Nov 2025 18:49:52 +0100 Subject: [PATCH 03/68] feat: every object in the stac index, will be converted to an dynamic array --- crawler/index.js | 131 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 7 deletions(-) diff --git a/crawler/index.js b/crawler/index.js index 75dcc65..89a2abf 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -1,19 +1,136 @@ -// npm install axios cheerio +/** + * @fileoverview STAC Index API crawler that fetches and processes catalog data + * @module crawler + */ + const axios = require('axios'); -// specify the URL of the site to crawl +/** + * URL of the STAC Index API endpoint + * @type {string} + */ const targetUrl = 'https://www.stacindex.org/api/catalogs'; -// define a crawler function +/** + * Fetches catalog data from the STAC Index API and processes it + * @async + * @function crawler + * @returns {Promise} + */ const crawler = async () => { try { - // request the target website const response = await axios.get(targetUrl); - console.log(response.data); + splitCatalogs(response.data); } catch (error) { - // handle any error that occurs during the HTTP request console.error(`Error fetching ${targetUrl}: ${error.message}`); } }; -crawler(); +/** + * Splits an array of catalogs into individual elements where each element is an array of its properties + * @param {Array} catalogs - Array of catalog objects from the STAC Index API + * @returns {Array} Array of arrays, where each inner array contains: + * [0] = index, [1] = id, [2] = url, [3] = slug, [4] = title, [5] = summary, + * [6] = access, [7] = created, [8] = updated, [9] = isPrivate, [10] = isApi, + * [11] = accessInfo, [12] = categories, ... (followed by any other dynamic properties) + * @throws {Error} Throws error if input is not an array + */ +function splitCatalogs(catalogs) { + if (!Array.isArray(catalogs)) { + throw new Error('Expected an array'); + } + + // Define the order of standard properties + const standardPropertyOrder = [ + 'index', 'id', 'url', 'slug', 'title', 'summary', + 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' + ]; + + // Split each element individually and convert to array format + const elements = catalogs.map((catalog, index) => { + + // Create object with all properties + const elementObj = { + index: index, + ...catalog, // All properties of the catalog object are automatically inherited + }; + + // Get all property names + const allProperties = Object.keys(elementObj); + + // Separate standard and dynamic properties + const standardProps = standardPropertyOrder.filter(prop => prop in elementObj); + const dynamicProps = allProperties.filter(prop => !standardPropertyOrder.includes(prop)); + + // Create array with standard properties first, then dynamic ones + const elementArray = []; + + // Add standard properties in defined order + standardProps.forEach(prop => { + elementArray.push(elementObj[prop]); + }); + + // Add dynamic properties + dynamicProps.forEach(prop => { + elementArray.push(elementObj[prop]); + }); + return elementArray; + }); + + // Output for demonstration + console.log(`Total: ${elements.length} elements found\n`); + + // Output first element as example with index mapping (remove later on) + + if (elements.length > 0) { + console.log('Example - First Element (as array):'); + console.log('Array indices and values:'); + const firstElement = elements[0]; + const allProps = [ + 'index', 'id', 'url', 'slug', 'title', 'summary', + 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' + ]; + firstElement.forEach((value, idx) => { + const propName = idx < allProps.length ? allProps[idx] : `dynamic_property_${idx}`; + console.log(` [${idx}] ${propName}: ${JSON.stringify(value)}`); + }); + console.log('\nAccess examples:'); + console.log(` elements[0][0] (index): ${elements[0][0]}`); + console.log(` elements[0][1] (id): ${elements[0][1]}`); + console.log(` elements[0][2] (url): ${elements[0][2]}`); + console.log(` elements[0][7] (created): ${elements[0][7]}`); + } + + return elements; +} + +/** + * Derives categories from a catalog object by checking various possible fields + * @param {Object} catalog - Catalog object to extract categories from + * @returns {Array} Array of category strings, empty array if none found + */ +function deriveCategories(catalog) { + if (!catalog || typeof catalog !== 'object') { + return []; + } + + if (Array.isArray(catalog.categories)) { + return catalog.categories.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.keywords)) { + return catalog.keywords.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.tags)) { + return catalog.tags.filter(Boolean).map(String); + } + + if (typeof catalog.access === 'string' && catalog.access.trim().length) { + return [catalog.access.trim()]; + } + + return []; +} + +crawler(); \ No newline at end of file From eb0c8fd3ab469c83288d1e280d9a1ba341dbecdd Mon Sep 17 00:00:00 2001 From: Jakob Date: Tue, 18 Nov 2025 17:55:36 +0100 Subject: [PATCH 04/68] added crawling to collection level, including nested catalogs --- crawler/index.js | 153 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/crawler/index.js b/crawler/index.js index 89a2abf..f3aafd0 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -20,7 +20,17 @@ const targetUrl = 'https://www.stacindex.org/api/catalogs'; const crawler = async () => { try { const response = await axios.get(targetUrl); - splitCatalogs(response.data); + const catalogs = splitCatalogs(response.data); + + // Crawl collections and nested catalogs for each catalog + console.log('\n Crawling collections and nested catalogs...\n'); + let totalCollections = 0; + for (const catalog of response.data.slice(0, 10)) { // Limit to first 10 catalogs + const stats = await crawlCatalogRecursive(catalog); + totalCollections += stats.collections; + } + + console.log(`\n Total collections found across all catalogs: ${totalCollections}`); } catch (error) { console.error(`Error fetching ${targetUrl}: ${error.message}`); } @@ -104,6 +114,147 @@ function splitCatalogs(catalogs) { return elements; } +/** + * Fetches and processes collections from a catalog + * @async + * @param {Object} catalog - Catalog object with url property + * @returns {Promise} Array of collections formatted as arrays + */ +async function getCollections(catalog) { + try { + // Try common STAC collection endpoints + const collectionUrls = [ + `${catalog.url}/collections`, + `${catalog.url}/collections/`, + `${catalog.url}/api/v1/collections` + ]; + + for (const url of collectionUrls) { + try { + const response = await axios.get(url, { timeout: 5000 }); + const collectionsData = Array.isArray(response.data) + ? response.data + : response.data.collections || []; + + if (collectionsData.length > 0) { + console.log(`\n Catalog: ${catalog.id}`); + console.log(` Found ${collectionsData.length} collections`); + + // Convert collections to array format similar to catalogs + const collections = collectionsData.map((col, index) => { + return [ + index, + col.id || 'Unknown', + col.url || col.links?.find(l => l.rel === 'self')?.href || 'N/A', + col.title || 'N/A', + col.description || col.summary || 'N/A', + col.extent?.spatial?.bbox?.[0] || 'N/A', + col.extent?.temporal?.interval?.[0] || 'N/A', + col.license || 'N/A', + col.keywords || [] + ]; + }); + + // Display collection details + if (collections.length > 0) { + console.log(' Collection details:'); + collections.forEach((col, idx) => { + console.log(` [${idx}] ${col[1]} - ${col[3]}`); + }); + } + + return collections; + } + } catch (e) { + // Try next URL + continue; + } + } + + console.log(`\n No collections found for catalog: ${catalog.id}`); + return []; + + } catch (error) { + console.error(`Error fetching collections for ${catalog.id}: ${error.message}`); + return []; + } +} + +/** + * Fetches nested catalogs within a catalog + * @async + * @param {Object} catalog - Catalog object with url property + * @returns {Promise} Array of nested catalogs + */ +async function getNestedCatalogs(catalog) { + try { + // Try common STAC nested catalog endpoints + const catalogUrls = [ + `${catalog.url}/catalogs`, + `${catalog.url}/catalogs/`, + `${catalog.url}/api/v1/catalogs` + ]; + + for (const url of catalogUrls) { + try { + const response = await axios.get(url, { timeout: 5000 }); + const catalogsData = Array.isArray(response.data) + ? response.data + : response.data.catalogs || []; + + if (catalogsData.length > 0) { + return catalogsData; + } + } catch (e) { + // Try next URL + continue; + } + } + + return []; + + } catch (error) { + console.error(`Error fetching nested catalogs for ${catalog.id}: ${error.message}`); + return []; + } +} + +/** + * Recursively crawls a catalog, its collections, and nested catalogs + * @async + * @param {Object} catalog - Catalog object to crawl + * @param {number} depth - Current depth level (for indentation) + * @returns {Promise} Stats object with collections count + */ +async function crawlCatalogRecursive(catalog, depth = 0) { + const indent = ' '.repeat(depth); + let stats = { collections: 0 }; + + try { + // First, try to get collections from this catalog + const collections = await getCollections(catalog); + stats.collections = collections.length; + + // Then, try to get nested catalogs + const nestedCatalogs = await getNestedCatalogs(catalog); + + if (nestedCatalogs.length > 0) { + console.log(`${indent} Nested catalogs found: ${nestedCatalogs.length}`); + + // Recursively crawl each nested catalog + for (const nestedCatalog of nestedCatalogs.slice(0, 5)) { // Limit to first 5 nested + const nestedStats = await crawlCatalogRecursive(nestedCatalog, depth + 1); + stats.collections += nestedStats.collections; + } + } + + } catch (error) { + console.error(`Error in recursive crawl for ${catalog.id}: ${error.message}`); + } + + return stats; +} + /** * Derives categories from a catalog object by checking various possible fields * @param {Object} catalog - Catalog object to extract categories from From 69b9ebb2d992a20747e14471e9fbc864c41ef8e3 Mon Sep 17 00:00:00 2001 From: Jakob Date: Tue, 18 Nov 2025 18:01:54 +0100 Subject: [PATCH 05/68] Closes #55; added crawling to collection level, including nested catalogs --- crawler/index.js | 153 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/crawler/index.js b/crawler/index.js index 89a2abf..f3aafd0 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -20,7 +20,17 @@ const targetUrl = 'https://www.stacindex.org/api/catalogs'; const crawler = async () => { try { const response = await axios.get(targetUrl); - splitCatalogs(response.data); + const catalogs = splitCatalogs(response.data); + + // Crawl collections and nested catalogs for each catalog + console.log('\n Crawling collections and nested catalogs...\n'); + let totalCollections = 0; + for (const catalog of response.data.slice(0, 10)) { // Limit to first 10 catalogs + const stats = await crawlCatalogRecursive(catalog); + totalCollections += stats.collections; + } + + console.log(`\n Total collections found across all catalogs: ${totalCollections}`); } catch (error) { console.error(`Error fetching ${targetUrl}: ${error.message}`); } @@ -104,6 +114,147 @@ function splitCatalogs(catalogs) { return elements; } +/** + * Fetches and processes collections from a catalog + * @async + * @param {Object} catalog - Catalog object with url property + * @returns {Promise} Array of collections formatted as arrays + */ +async function getCollections(catalog) { + try { + // Try common STAC collection endpoints + const collectionUrls = [ + `${catalog.url}/collections`, + `${catalog.url}/collections/`, + `${catalog.url}/api/v1/collections` + ]; + + for (const url of collectionUrls) { + try { + const response = await axios.get(url, { timeout: 5000 }); + const collectionsData = Array.isArray(response.data) + ? response.data + : response.data.collections || []; + + if (collectionsData.length > 0) { + console.log(`\n Catalog: ${catalog.id}`); + console.log(` Found ${collectionsData.length} collections`); + + // Convert collections to array format similar to catalogs + const collections = collectionsData.map((col, index) => { + return [ + index, + col.id || 'Unknown', + col.url || col.links?.find(l => l.rel === 'self')?.href || 'N/A', + col.title || 'N/A', + col.description || col.summary || 'N/A', + col.extent?.spatial?.bbox?.[0] || 'N/A', + col.extent?.temporal?.interval?.[0] || 'N/A', + col.license || 'N/A', + col.keywords || [] + ]; + }); + + // Display collection details + if (collections.length > 0) { + console.log(' Collection details:'); + collections.forEach((col, idx) => { + console.log(` [${idx}] ${col[1]} - ${col[3]}`); + }); + } + + return collections; + } + } catch (e) { + // Try next URL + continue; + } + } + + console.log(`\n No collections found for catalog: ${catalog.id}`); + return []; + + } catch (error) { + console.error(`Error fetching collections for ${catalog.id}: ${error.message}`); + return []; + } +} + +/** + * Fetches nested catalogs within a catalog + * @async + * @param {Object} catalog - Catalog object with url property + * @returns {Promise} Array of nested catalogs + */ +async function getNestedCatalogs(catalog) { + try { + // Try common STAC nested catalog endpoints + const catalogUrls = [ + `${catalog.url}/catalogs`, + `${catalog.url}/catalogs/`, + `${catalog.url}/api/v1/catalogs` + ]; + + for (const url of catalogUrls) { + try { + const response = await axios.get(url, { timeout: 5000 }); + const catalogsData = Array.isArray(response.data) + ? response.data + : response.data.catalogs || []; + + if (catalogsData.length > 0) { + return catalogsData; + } + } catch (e) { + // Try next URL + continue; + } + } + + return []; + + } catch (error) { + console.error(`Error fetching nested catalogs for ${catalog.id}: ${error.message}`); + return []; + } +} + +/** + * Recursively crawls a catalog, its collections, and nested catalogs + * @async + * @param {Object} catalog - Catalog object to crawl + * @param {number} depth - Current depth level (for indentation) + * @returns {Promise} Stats object with collections count + */ +async function crawlCatalogRecursive(catalog, depth = 0) { + const indent = ' '.repeat(depth); + let stats = { collections: 0 }; + + try { + // First, try to get collections from this catalog + const collections = await getCollections(catalog); + stats.collections = collections.length; + + // Then, try to get nested catalogs + const nestedCatalogs = await getNestedCatalogs(catalog); + + if (nestedCatalogs.length > 0) { + console.log(`${indent} Nested catalogs found: ${nestedCatalogs.length}`); + + // Recursively crawl each nested catalog + for (const nestedCatalog of nestedCatalogs.slice(0, 5)) { // Limit to first 5 nested + const nestedStats = await crawlCatalogRecursive(nestedCatalog, depth + 1); + stats.collections += nestedStats.collections; + } + } + + } catch (error) { + console.error(`Error in recursive crawl for ${catalog.id}: ${error.message}`); + } + + return stats; +} + /** * Derives categories from a catalog object by checking various possible fields * @param {Object} catalog - Catalog object to extract categories from From eaea3e0399601aaf3706329bf4746b4246e12707 Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 19 Nov 2025 12:11:56 +0100 Subject: [PATCH 06/68] feat: implement database connection with node pg for catalog management --- crawler/db.js | 111 +++++++++ crawler/index.js | 22 +- crawler/package-lock.json | 479 ++++++++++++++++++++++++++++++++++++++ crawler/package.json | 1 + 4 files changed, 611 insertions(+), 2 deletions(-) create mode 100644 crawler/db.js diff --git a/crawler/db.js b/crawler/db.js new file mode 100644 index 0000000..74bd790 --- /dev/null +++ b/crawler/db.js @@ -0,0 +1,111 @@ +/** + * DB helper for crawler using `pg` Pool + * Exposes: initDb(), insertOrUpdateCatalog(), close() + */ +const { Pool } = require('pg'); +require('dotenv').config(); + +const pool = new Pool({ + host: process.env.PGHOST || 'localhost', + port: parseInt(process.env.PGPORT || '5432', 10), + user: process.env.PGUSER || 'stac_user', + password: process.env.PGPASSWORD || 'stac_pass', + database: process.env.PGDATABASE || 'stac_db', + max: 10, +}); + +async function initDb() { + const client = await pool.connect(); + try { + // create table if missing and add useful indexes + await client.query(` + CREATE TABLE IF NOT EXISTS catalogs ( + id SERIAL PRIMARY KEY, + catalog_id TEXT UNIQUE, + url TEXT, + title TEXT, + metadata JSONB, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() + ); + `); + + // create GIN index on metadata for JSONB queries + await client.query(` + CREATE INDEX IF NOT EXISTS idx_catalogs_metadata_gin ON catalogs USING GIN (metadata jsonb_path_ops); + `); + + // try enabling PostGIS + try { + await client.query(`CREATE EXTENSION IF NOT EXISTS postgis;`); + } catch (e) { + // ignore extension permission errors + } + + console.log('DB initialized (catalogs table present)'); + } finally { + client.release(); + } +} + +async function insertOrUpdateCatalog(catalog) { + if (!catalog || typeof catalog !== 'object') return null; + const catalogId = catalog.id || catalog.catalog_id || catalog.slug || null; + if (!catalogId) return null; + + const url = catalog.url || null; + const title = catalog.title || catalog.name || null; + const metadata = catalog; + const now = new Date().toISOString(); + + const query = ` + INSERT INTO catalogs (catalog_id, url, title, metadata, updated_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (catalog_id) DO UPDATE SET + url = EXCLUDED.url, + title = EXCLUDED.title, + metadata = catalogs.metadata || EXCLUDED.metadata, + updated_at = EXCLUDED.updated_at + RETURNING id; + `; + const values = [catalogId, url, title, metadata, now]; + const res = await pool.query(query, values); + return res.rows[0]; +} + +async function close() { + await pool.end(); +} + +module.exports = { initDb, insertOrUpdateCatalog, close, pool }; + + + +/** + +const createConnectionPool = require('@databases/pg'); +const {sql} = require('@databases/pg'); + +async function run() { + // N.B. you will need to replace this connection + // string with the correct string for your database. + const db = createConnectionPool( + 'postgres://test-user@localhost:5432/test-db', + ); + + const results = await db.query(sql` + SELECT 1 + 1 as result; + `); + + console.log(results); + // => [{result: 2}] + + await db.dispose(); +} + +run().catch((err) => { + console.error(err); + process.exit(1); +}); + +*/ diff --git a/crawler/index.js b/crawler/index.js index 89a2abf..6bd8b83 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -4,6 +4,7 @@ */ const axios = require('axios'); +const db = require('./db'); /** * URL of the STAC Index API endpoint @@ -19,10 +20,13 @@ const targetUrl = 'https://www.stacindex.org/api/catalogs'; */ const crawler = async () => { try { + await db.initDb(); const response = await axios.get(targetUrl); - splitCatalogs(response.data); + await splitCatalogs(response.data); } catch (error) { console.error(`Error fetching ${targetUrl}: ${error.message}`); + } finally { + // keep DB open for now; close on process exit } }; @@ -35,7 +39,7 @@ const crawler = async () => { * [11] = accessInfo, [12] = categories, ... (followed by any other dynamic properties) * @throws {Error} Throws error if input is not an array */ -function splitCatalogs(catalogs) { +async function splitCatalogs(catalogs) { if (!Array.isArray(catalogs)) { throw new Error('Expected an array'); } @@ -101,6 +105,20 @@ function splitCatalogs(catalogs) { console.log(` elements[0][7] (created): ${elements[0][7]}`); } + // Persist catalogs into DB + try { + await Promise.all(catalogs.map(async (catalog) => { + try { + await db.insertOrUpdateCatalog(catalog); + } catch (err) { + console.error('DB write error for catalog', catalog && (catalog.id || catalog.slug), err && err.message); + } + })); + console.log('Finished writing catalogs to DB'); + } catch (e) { + console.error('Unexpected error while saving catalogs to DB', e && e.message); + } + return elements; } diff --git a/crawler/package-lock.json b/crawler/package-lock.json index 9281009..b668946 100644 --- a/crawler/package-lock.json +++ b/crawler/package-lock.json @@ -9,9 +9,171 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@databases/pg": "^5.5.0", "axios": "^1.13.2" } }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@databases/connection-pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@databases/connection-pool/-/connection-pool-1.1.0.tgz", + "integrity": "sha512-/12/SNgl0V77mJTo5SX3yGPz4c9XGQwAlCfA0vlfs/0HcaErNpYXpmhj0StET07w6TmTJTnaUgX2EPcQK9ez5A==", + "license": "MIT", + "dependencies": { + "@databases/queue": "^1.0.0", + "is-promise": "^4.0.0" + } + }, + "node_modules/@databases/escape-identifier": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@databases/escape-identifier/-/escape-identifier-1.0.3.tgz", + "integrity": "sha512-Su36iSVzaHxpVdISVMViUX/32sLvzxVgjZpYhzhotxZUuLo11GVWsiHwqkvUZijTLUxcDmUqEwGJO3O/soLuZA==", + "license": "MIT", + "dependencies": { + "@databases/validate-unicode": "^1.0.0" + } + }, + "node_modules/@databases/lock": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@databases/lock/-/lock-2.1.0.tgz", + "integrity": "sha512-ReWnFE5qeCuO2SA5h5fDh/hE/vMolA+Epe6xkAQP1FL2nhnsTCYwN2JACk/kWctR4OQoh0njBjPZ0yfIptclcA==", + "license": "MIT", + "dependencies": { + "@databases/queue": "^1.0.0" + } + }, + "node_modules/@databases/pg": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@databases/pg/-/pg-5.5.0.tgz", + "integrity": "sha512-WIojK9AYIlNi5YRfc5YUOow3PQ82ClmwT9HG3nEsKLUERYieoVmHMYDQLS0ry6FjgJx+2yFs7LCw4kZpWu1TBw==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "@databases/escape-identifier": "^1.0.3", + "@databases/pg-config": "^3.2.0", + "@databases/pg-connection-string": "^1.0.0", + "@databases/pg-data-type-id": "^3.0.0", + "@databases/pg-errors": "^1.0.0", + "@databases/push-to-async-iterable": "^3.0.0", + "@databases/shared": "^3.1.0", + "@databases/split-sql-query": "^1.0.4", + "@databases/sql": "^3.3.0", + "assert-never": "^1.2.1", + "pg": "^8.4.2", + "pg-cursor": "^2.4.2" + } + }, + "node_modules/@databases/pg-config": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@databases/pg-config/-/pg-config-3.4.0.tgz", + "integrity": "sha512-4dYiTbHjzyQfEfaIGkh3uCBNBRWPs5Jcws94cFagLAGnjO/TcghC7oexzC81+bIADLDlpCw7DEWJAK/gNSQxkw==", + "license": "MIT", + "dependencies": { + "cosmiconfig": "^8.1.0", + "funtypes": "^4.1.0" + } + }, + "node_modules/@databases/pg-connection-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@databases/pg-connection-string/-/pg-connection-string-1.0.0.tgz", + "integrity": "sha512-8czOF9jlv7PlS7BPjnL82ynpDs1t8cu+C2jvdtMr37e8daPKMS7n1KfNE9xtr2Gq4QYKjynep097eYa5yIwcLA==", + "license": "MIT" + }, + "node_modules/@databases/pg-data-type-id": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@databases/pg-data-type-id/-/pg-data-type-id-3.0.0.tgz", + "integrity": "sha512-VqW1csN8pRsWJxjPsGIC9FQ8wyenfmGv0P//BaeDMAu/giM3IXKxKM8fkScUSQ00uqFK/L1iHS5g6dgodF3XzA==", + "license": "MIT" + }, + "node_modules/@databases/pg-errors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@databases/pg-errors/-/pg-errors-1.0.0.tgz", + "integrity": "sha512-Yz3exbptZwOn4ZD/MSwY6z++XVyOFsMh5DERvSw3awRwJFnfdaqdeiIxxX0MVjM6KPihF0xxp8lPO7vTc5ydpw==", + "license": "MIT" + }, + "node_modules/@databases/push-to-async-iterable": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@databases/push-to-async-iterable/-/push-to-async-iterable-3.0.0.tgz", + "integrity": "sha512-xwu/yNgINdMU+fn6UwFsxh+pa6UrVPafY+0qm0RK0/nKyjllfDqSbwK4gSmdmLEwPYxKwch9CAE3P8NxN1hPSg==", + "license": "MIT", + "dependencies": { + "@databases/queue": "^1.0.0" + } + }, + "node_modules/@databases/queue": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@databases/queue/-/queue-1.0.1.tgz", + "integrity": "sha512-dqRU+/aQ4lhFzjPIkIhjB0+UEKMb76FoBgHOJUTcEblgatr/IhdhHliT3VVwcImXh35Mz297PAXE4yFM4eYWUQ==", + "license": "MIT" + }, + "node_modules/@databases/shared": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@databases/shared/-/shared-3.1.0.tgz", + "integrity": "sha512-bO1DIYAYDiWOCqVPvBio1JqZQYh4dph2M1av2w/REeFT6WBd64mTrOFlcxKV0CUAYT0UiJsDfPqEfw0/APRzWg==", + "license": "MIT", + "dependencies": { + "@databases/connection-pool": "^1.1.0", + "@databases/lock": "^2.1.0", + "@databases/queue": "^1.0.1", + "@databases/split-sql-query": "^1.0.4", + "@databases/sql": "^3.3.0" + } + }, + "node_modules/@databases/split-sql-query": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@databases/split-sql-query/-/split-sql-query-1.0.4.tgz", + "integrity": "sha512-lDqDQvH34NNjLs0knaDvL6HKgPtishQlDYHfOkvbAd5VQOEhcDvvmG2zbBuFvS2HQAz5NsyLj5erGaxibkxhvQ==", + "license": "MIT", + "peerDependencies": { + "@databases/sql": "*" + } + }, + "node_modules/@databases/sql": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@databases/sql/-/sql-3.3.0.tgz", + "integrity": "sha512-vj9huEy4mjJ48GS1Z8yvtMm4BYAnFYACUds25ym6Gd/gsnngkJ17fo62a6mmbNNwCBS/8467PmZR01Zs/06TjA==", + "license": "MIT" + }, + "node_modules/@databases/validate-unicode": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@databases/validate-unicode/-/validate-unicode-1.0.0.tgz", + "integrity": "sha512-dLKqxGcymeVwEb/6c44KjOnzaAafFf0Wxa8xcfEjx/qOl3rdijsKYBAtIGhtVtOlpPf/PFKfgTuFurSPn/3B/g==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/assert-never": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/assert-never/-/assert-never-1.4.0.tgz", + "integrity": "sha512-5oJg84os6NMQNl27T9LnZkvvqzvAnHu03ShCnoj6bsJwS7L8AO4lf+C/XjK/nvzEqQB744moC6V128RucQd1jA==", + "license": "MIT" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -42,6 +204,15 @@ "node": ">= 0.4" } }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -54,6 +225,32 @@ "node": ">= 0.8" } }, + "node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "license": "MIT", + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -77,6 +274,15 @@ "node": ">= 0.4" } }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -167,6 +373,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/funtypes": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/funtypes/-/funtypes-4.2.0.tgz", + "integrity": "sha512-DvOtjiKvkeuXGV0O8LQh9quUP3bSOTEQPGv537Sao8kDq2rDbg48UsSJ7wlBLPzR2Mn0pV7cyAiq5pYG1oUyCQ==", + "license": "MIT" + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -255,6 +467,64 @@ "node": ">= 0.4" } }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -285,11 +555,220 @@ "node": ">= 0.6" } }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pg": { + "version": "8.16.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", + "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.3", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.2.7" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz", + "integrity": "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "license": "MIT" + }, + "node_modules/pg-cursor": { + "version": "2.15.3", + "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.15.3.tgz", + "integrity": "sha512-eHw63TsiGtFEfAd7tOTZ+TLy+i/2ePKS20H84qCQ+aQ60pve05Okon9tKMC+YN3j6XyeFoHnaim7Lt9WVafQsA==", + "license": "MIT", + "peerDependencies": { + "pg": "^8" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", + "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", "license": "MIT" + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/crawler/package.json b/crawler/package.json index 9cf3b83..47988ed 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -9,6 +9,7 @@ "author": "", "license": "ISC", "dependencies": { + "@databases/pg": "^5.5.0", "axios": "^1.13.2" } } From dda02f227b67c752883e000e3cf788eaf787ec35 Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 19 Nov 2025 14:47:10 +0100 Subject: [PATCH 07/68] closes #53, set up docker for the crawler component --- crawler/.dockerignore | 6 +++++ crawler/Dockerfile | 17 +++++++++++++ crawler/README.md | 49 ++++++++++++++++++++++++++++++++++++++ crawler/docker-compose.yml | 12 ++++++++++ crawler/package.json | 5 ++++ 5 files changed, 89 insertions(+) create mode 100644 crawler/.dockerignore create mode 100644 crawler/Dockerfile create mode 100644 crawler/docker-compose.yml diff --git a/crawler/.dockerignore b/crawler/.dockerignore new file mode 100644 index 0000000..f94907b --- /dev/null +++ b/crawler/.dockerignore @@ -0,0 +1,6 @@ +node_modules +npm-debug.log +.git +.gitignore +.env +*.md diff --git a/crawler/Dockerfile b/crawler/Dockerfile new file mode 100644 index 0000000..6c77666 --- /dev/null +++ b/crawler/Dockerfile @@ -0,0 +1,17 @@ +# Use official Node.js LTS image +FROM node:18-alpine + +# Set working directory +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm install --production + +# Copy application files +COPY . . + +# Run the crawler +CMD ["node", "index.js"] diff --git a/crawler/README.md b/crawler/README.md index e69de29..1293303 100644 --- a/crawler/README.md +++ b/crawler/README.md @@ -0,0 +1,49 @@ +# STAC Crawler + +A Node.js crawler for STAC Index API that fetches and processes catalog and collection data. + +## Running Locally + +```bash +npm install +npm start +``` + +## Docker + +### Build and run with Docker + +```bash +# Build the image +docker build -t stac-crawler + +# Run the container +docker run --rm stac-crawler +``` + +Or use npm scripts: + +```bash +npm run docker:build +npm run docker:run +``` + +### Using Docker Compose + +```bash +# Start the crawler +docker-compose up -d + +# View logs +docker-compose logs -f + +# Stop the crawler +docker-compose down +``` + +Or use npm scripts: + +```bash +npm run docker:compose:up +npm run docker:compose:down +``` diff --git a/crawler/docker-compose.yml b/crawler/docker-compose.yml new file mode 100644 index 0000000..797cb5f --- /dev/null +++ b/crawler/docker-compose.yml @@ -0,0 +1,12 @@ +services: + crawler: + build: + context: . + dockerfile: Dockerfile + container_name: stac-crawler + restart: unless-stopped + environment: + - NODE_ENV=production + # Optional: Add volume to persist data + # volumes: + # - ./data:/app/data diff --git a/crawler/package.json b/crawler/package.json index 9cf3b83..e9ab811 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -4,6 +4,11 @@ "description": "STAC Index crawler", "main": "index.js", "scripts": { + "start": "node index.js", + "docker:build": "docker build -t stac-crawler .", + "docker:run": "docker run --rm stac-crawler", + "docker:compose:up": "docker-compose up -d", + "docker:compose:down": "docker-compose down", "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", From b3e289f2731488c93a8588fed8e49f64ff67eae3 Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 19 Nov 2025 15:04:30 +0100 Subject: [PATCH 08/68] fixes #55, restructured the crawler component into catalog and api crawling --- crawler/apis/api.js | 4 + crawler/catalogs/catalog.js | 260 ++++++++++++++++++++++++++++++++++++ crawler/index.js | 251 +--------------------------------- 3 files changed, 266 insertions(+), 249 deletions(-) create mode 100644 crawler/apis/api.js create mode 100644 crawler/catalogs/catalog.js diff --git a/crawler/apis/api.js b/crawler/apis/api.js new file mode 100644 index 0000000..87cdde2 --- /dev/null +++ b/crawler/apis/api.js @@ -0,0 +1,4 @@ +/** + * @fileoverview API crawling functionality for STAC Index + * @module apis/api + */ \ No newline at end of file diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js new file mode 100644 index 0000000..4ff86d2 --- /dev/null +++ b/crawler/catalogs/catalog.js @@ -0,0 +1,260 @@ +/** + * @fileoverview Catalog crawling functionality for STAC Index + * @module catalogs/catalog + */ + +const axios = require('axios'); + +/** + * Splits an array of catalogs into individual elements where each element is an array of its properties + * @param {Array} catalogs - Array of catalog objects from the STAC Index API + * @returns {Array} Array of arrays, where each inner array contains: + * [0] = index, [1] = id, [2] = url, [3] = slug, [4] = title, [5] = summary, + * [6] = access, [7] = created, [8] = updated, [9] = isPrivate, [10] = isApi, + * [11] = accessInfo, [12] = categories, ... (followed by any other dynamic properties) + * @throws {Error} Throws error if input is not an array + */ +function splitCatalogs(catalogs) { + if (!Array.isArray(catalogs)) { + throw new Error('Expected an array'); + } + + // Define the order of standard properties + const standardPropertyOrder = [ + 'index', 'id', 'url', 'slug', 'title', 'summary', + 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' + ]; + + // Split each element individually and convert to array format + const elements = catalogs.map((catalog, index) => { + + // Create object with all properties + const elementObj = { + index: index, + ...catalog, + }; + + // Get all property names + const allProperties = Object.keys(elementObj); + + // Separate standard and dynamic properties + const standardProps = standardPropertyOrder.filter(prop => prop in elementObj); + const dynamicProps = allProperties.filter(prop => !standardPropertyOrder.includes(prop)); + + // Create array with standard properties first, then dynamic ones + const elementArray = []; + + // Add standard properties in defined order + standardProps.forEach(prop => { + elementArray.push(elementObj[prop]); + }); + + // Add dynamic properties + dynamicProps.forEach(prop => { + elementArray.push(elementObj[prop]); + }); + return elementArray; + }); + + // Output for demonstration + console.log(`Total: ${elements.length} elements found\n`); + + if (elements.length > 0) { + console.log('Example - First Element (as array):'); + console.log('Array indices and values:'); + const firstElement = elements[0]; + const allProps = [ + 'index', 'id', 'url', 'slug', 'title', 'summary', + 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' + ]; + firstElement.forEach((value, idx) => { + const propName = idx < allProps.length ? allProps[idx] : `dynamic_property_${idx}`; + console.log(` [${idx}] ${propName}: ${JSON.stringify(value)}`); + }); + console.log('\nAccess examples:'); + console.log(` elements[0][0] (index): ${elements[0][0]}`); + console.log(` elements[0][1] (id): ${elements[0][1]}`); + console.log(` elements[0][2] (url): ${elements[0][2]}`); + console.log(` elements[0][7] (created): ${elements[0][7]}`); + } + + return elements; +} + +/** + * Fetches and processes collections from a catalog + * @async + * @param {Object} catalog - Catalog object with url property + * @returns {Promise} Array of collections formatted as arrays + */ +async function getCollections(catalog) { + try { + // Try common STAC collection endpoints + const collectionUrls = [ + `${catalog.url}/collections`, + `${catalog.url}/collections/`, + `${catalog.url}/api/v1/collections` + ]; + + for (const url of collectionUrls) { + try { + const response = await axios.get(url, { timeout: 5000 }); + const collectionsData = Array.isArray(response.data) + ? response.data + : response.data.collections || []; + + if (collectionsData.length > 0) { + console.log(`\n Catalog: ${catalog.id}`); + console.log(` Found ${collectionsData.length} collections`); + + // Convert collections to array format similar to catalogs + const collections = collectionsData.map((col, index) => { + return [ + index, + col.id || 'Unknown', + col.url || col.links?.find(l => l.rel === 'self')?.href || 'N/A', + col.title || 'N/A', + col.description || col.summary || 'N/A', + col.extent?.spatial?.bbox?.[0] || 'N/A', + col.extent?.temporal?.interval?.[0] || 'N/A', + col.license || 'N/A', + col.keywords || [] + ]; + }); + + // Display collection details + if (collections.length > 0) { + console.log(' Collection details:'); + collections.forEach((col, idx) => { + console.log(` [${idx}] ${col[1]} - ${col[3]}`); + }); + } + + return collections; + } + } catch (e) { + // Try next URL + continue; + } + } + + console.log(`\n No collections found for catalog: ${catalog.id}`); + return []; + + } catch (error) { + console.error(`Error fetching collections for ${catalog.id}: ${error.message}`); + return []; + } +} + +/** + * Fetches nested catalogs within a catalog + * @async + * @param {Object} catalog - Catalog object with url property + * @returns {Promise} Array of nested catalogs + */ +async function getNestedCatalogs(catalog) { + try { + // Try common STAC nested catalog endpoints + const catalogUrls = [ + `${catalog.url}/catalogs`, + `${catalog.url}/catalogs/`, + `${catalog.url}/api/v1/catalogs` + ]; + + for (const url of catalogUrls) { + try { + const response = await axios.get(url, { timeout: 5000 }); + const catalogsData = Array.isArray(response.data) + ? response.data + : response.data.catalogs || []; + + if (catalogsData.length > 0) { + return catalogsData; + } + } catch (e) { + // Try next URL + continue; + } + } + + return []; + + } catch (error) { + console.error(`Error fetching nested catalogs for ${catalog.id}: ${error.message}`); + return []; + } +} + +/** + * Recursively crawls a catalog, its collections, and nested catalogs + * @async + * @param {Object} catalog - Catalog object to crawl + * @param {number} depth - Current depth level (for indentation) + * @returns {Promise} Stats object with collections count + */ +async function crawlCatalogRecursive(catalog, depth = 0) { + const indent = ' '.repeat(depth); + let stats = { collections: 0 }; + + try { + // First, try to get collections from this catalog + const collections = await getCollections(catalog); + stats.collections = collections.length; + + // Then, try to get nested catalogs + const nestedCatalogs = await getNestedCatalogs(catalog); + + if (nestedCatalogs.length > 0) { + console.log(`${indent} Nested catalogs found: ${nestedCatalogs.length}`); + + // Recursively crawl each nested catalog + for (const nestedCatalog of nestedCatalogs.slice(0, 5)) { // Limit to first 5 nested + const nestedStats = await crawlCatalogRecursive(nestedCatalog, depth + 1); + stats.collections += nestedStats.collections; + } + } + + } catch (error) { + console.error(`Error in recursive crawl for ${catalog.id}: ${error.message}`); + } + + return stats; +} + +/** + * Derives categories from a catalog object by checking various possible fields + * @param {Object} catalog - Catalog object to extract categories from + * @returns {Array} Array of category strings, empty array if none found + */ +function deriveCategories(catalog) { + if (!catalog || typeof catalog !== 'object') { + return []; + } + + if (Array.isArray(catalog.categories)) { + return catalog.categories.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.keywords)) { + return catalog.keywords.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.tags)) { + return catalog.tags.filter(Boolean).map(String); + } + + if (typeof catalog.access === 'string' && catalog.access.trim().length) { + return [catalog.access.trim()]; + } + + return []; +} + +module.exports = { + splitCatalogs, + getCollections, + getNestedCatalogs, + crawlCatalogRecursive, + deriveCategories +}; diff --git a/crawler/index.js b/crawler/index.js index f3aafd0..9f2f260 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -4,6 +4,7 @@ */ const axios = require('axios'); +const { splitCatalogs, crawlCatalogRecursive } = require('./catalogs/catalog'); /** * URL of the STAC Index API endpoint @@ -36,252 +37,4 @@ const crawler = async () => { } }; -/** - * Splits an array of catalogs into individual elements where each element is an array of its properties - * @param {Array} catalogs - Array of catalog objects from the STAC Index API - * @returns {Array} Array of arrays, where each inner array contains: - * [0] = index, [1] = id, [2] = url, [3] = slug, [4] = title, [5] = summary, - * [6] = access, [7] = created, [8] = updated, [9] = isPrivate, [10] = isApi, - * [11] = accessInfo, [12] = categories, ... (followed by any other dynamic properties) - * @throws {Error} Throws error if input is not an array - */ -function splitCatalogs(catalogs) { - if (!Array.isArray(catalogs)) { - throw new Error('Expected an array'); - } - - // Define the order of standard properties - const standardPropertyOrder = [ - 'index', 'id', 'url', 'slug', 'title', 'summary', - 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' - ]; - - // Split each element individually and convert to array format - const elements = catalogs.map((catalog, index) => { - - // Create object with all properties - const elementObj = { - index: index, - ...catalog, // All properties of the catalog object are automatically inherited - }; - - // Get all property names - const allProperties = Object.keys(elementObj); - - // Separate standard and dynamic properties - const standardProps = standardPropertyOrder.filter(prop => prop in elementObj); - const dynamicProps = allProperties.filter(prop => !standardPropertyOrder.includes(prop)); - - // Create array with standard properties first, then dynamic ones - const elementArray = []; - - // Add standard properties in defined order - standardProps.forEach(prop => { - elementArray.push(elementObj[prop]); - }); - - // Add dynamic properties - dynamicProps.forEach(prop => { - elementArray.push(elementObj[prop]); - }); - return elementArray; - }); - - // Output for demonstration - console.log(`Total: ${elements.length} elements found\n`); - - // Output first element as example with index mapping (remove later on) - - if (elements.length > 0) { - console.log('Example - First Element (as array):'); - console.log('Array indices and values:'); - const firstElement = elements[0]; - const allProps = [ - 'index', 'id', 'url', 'slug', 'title', 'summary', - 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' - ]; - firstElement.forEach((value, idx) => { - const propName = idx < allProps.length ? allProps[idx] : `dynamic_property_${idx}`; - console.log(` [${idx}] ${propName}: ${JSON.stringify(value)}`); - }); - console.log('\nAccess examples:'); - console.log(` elements[0][0] (index): ${elements[0][0]}`); - console.log(` elements[0][1] (id): ${elements[0][1]}`); - console.log(` elements[0][2] (url): ${elements[0][2]}`); - console.log(` elements[0][7] (created): ${elements[0][7]}`); - } - - return elements; -} - -/** - * Fetches and processes collections from a catalog - * @async - * @param {Object} catalog - Catalog object with url property - * @returns {Promise} Array of collections formatted as arrays - */ -async function getCollections(catalog) { - try { - // Try common STAC collection endpoints - const collectionUrls = [ - `${catalog.url}/collections`, - `${catalog.url}/collections/`, - `${catalog.url}/api/v1/collections` - ]; - - for (const url of collectionUrls) { - try { - const response = await axios.get(url, { timeout: 5000 }); - const collectionsData = Array.isArray(response.data) - ? response.data - : response.data.collections || []; - - if (collectionsData.length > 0) { - console.log(`\n Catalog: ${catalog.id}`); - console.log(` Found ${collectionsData.length} collections`); - - // Convert collections to array format similar to catalogs - const collections = collectionsData.map((col, index) => { - return [ - index, - col.id || 'Unknown', - col.url || col.links?.find(l => l.rel === 'self')?.href || 'N/A', - col.title || 'N/A', - col.description || col.summary || 'N/A', - col.extent?.spatial?.bbox?.[0] || 'N/A', - col.extent?.temporal?.interval?.[0] || 'N/A', - col.license || 'N/A', - col.keywords || [] - ]; - }); - - // Display collection details - if (collections.length > 0) { - console.log(' Collection details:'); - collections.forEach((col, idx) => { - console.log(` [${idx}] ${col[1]} - ${col[3]}`); - }); - } - - return collections; - } - } catch (e) { - // Try next URL - continue; - } - } - - console.log(`\n No collections found for catalog: ${catalog.id}`); - return []; - - } catch (error) { - console.error(`Error fetching collections for ${catalog.id}: ${error.message}`); - return []; - } -} - -/** - * Fetches nested catalogs within a catalog - * @async - * @param {Object} catalog - Catalog object with url property - * @returns {Promise} Array of nested catalogs - */ -async function getNestedCatalogs(catalog) { - try { - // Try common STAC nested catalog endpoints - const catalogUrls = [ - `${catalog.url}/catalogs`, - `${catalog.url}/catalogs/`, - `${catalog.url}/api/v1/catalogs` - ]; - - for (const url of catalogUrls) { - try { - const response = await axios.get(url, { timeout: 5000 }); - const catalogsData = Array.isArray(response.data) - ? response.data - : response.data.catalogs || []; - - if (catalogsData.length > 0) { - return catalogsData; - } - } catch (e) { - // Try next URL - continue; - } - } - - return []; - - } catch (error) { - console.error(`Error fetching nested catalogs for ${catalog.id}: ${error.message}`); - return []; - } -} - -/** - * Recursively crawls a catalog, its collections, and nested catalogs - * @async - * @param {Object} catalog - Catalog object to crawl - * @param {number} depth - Current depth level (for indentation) - * @returns {Promise} Stats object with collections count - */ -async function crawlCatalogRecursive(catalog, depth = 0) { - const indent = ' '.repeat(depth); - let stats = { collections: 0 }; - - try { - // First, try to get collections from this catalog - const collections = await getCollections(catalog); - stats.collections = collections.length; - - // Then, try to get nested catalogs - const nestedCatalogs = await getNestedCatalogs(catalog); - - if (nestedCatalogs.length > 0) { - console.log(`${indent} Nested catalogs found: ${nestedCatalogs.length}`); - - // Recursively crawl each nested catalog - for (const nestedCatalog of nestedCatalogs.slice(0, 5)) { // Limit to first 5 nested - const nestedStats = await crawlCatalogRecursive(nestedCatalog, depth + 1); - stats.collections += nestedStats.collections; - } - } - - } catch (error) { - console.error(`Error in recursive crawl for ${catalog.id}: ${error.message}`); - } - - return stats; -} - -/** - * Derives categories from a catalog object by checking various possible fields - * @param {Object} catalog - Catalog object to extract categories from - * @returns {Array} Array of category strings, empty array if none found - */ -function deriveCategories(catalog) { - if (!catalog || typeof catalog !== 'object') { - return []; - } - - if (Array.isArray(catalog.categories)) { - return catalog.categories.filter(Boolean).map(String); - } - - if (Array.isArray(catalog.keywords)) { - return catalog.keywords.filter(Boolean).map(String); - } - - if (Array.isArray(catalog.tags)) { - return catalog.tags.filter(Boolean).map(String); - } - - if (typeof catalog.access === 'string' && catalog.access.trim().length) { - return [catalog.access.trim()]; - } - - return []; -} - -crawler(); \ No newline at end of file +crawler(); From 4be5bc86ee9e488469fed15573ee84cf924317d1 Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 19 Nov 2025 15:32:07 +0100 Subject: [PATCH 09/68] feat: add dotenv for environment variable management and improved database connection --- crawler/.env | 6 ++++ crawler/db.js | 58 ++------------------------------------- crawler/package-lock.json | 15 +++++++++- crawler/package.json | 3 +- 4 files changed, 25 insertions(+), 57 deletions(-) create mode 100644 crawler/.env diff --git a/crawler/.env b/crawler/.env new file mode 100644 index 0000000..f5a0661 --- /dev/null +++ b/crawler/.env @@ -0,0 +1,6 @@ +# Database configuration +PGHOST=localhost +PGPORT=5432 +PGUSER=stac_user +PGPASSWORD=stac_pass +PGDATABASE=stac_db \ No newline at end of file diff --git a/crawler/db.js b/crawler/db.js index 74bd790..3a7788b 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -15,34 +15,11 @@ const pool = new Pool({ }); async function initDb() { + // Test database connection const client = await pool.connect(); try { - // create table if missing and add useful indexes - await client.query(` - CREATE TABLE IF NOT EXISTS catalogs ( - id SERIAL PRIMARY KEY, - catalog_id TEXT UNIQUE, - url TEXT, - title TEXT, - metadata JSONB, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() - ); - `); - - // create GIN index on metadata for JSONB queries - await client.query(` - CREATE INDEX IF NOT EXISTS idx_catalogs_metadata_gin ON catalogs USING GIN (metadata jsonb_path_ops); - `); - - // try enabling PostGIS - try { - await client.query(`CREATE EXTENSION IF NOT EXISTS postgis;`); - } catch (e) { - // ignore extension permission errors - } - - console.log('DB initialized (catalogs table present)'); + await client.query('SELECT 1'); + console.log('DB connection established successfully'); } finally { client.release(); } @@ -80,32 +57,3 @@ async function close() { module.exports = { initDb, insertOrUpdateCatalog, close, pool }; - -/** - -const createConnectionPool = require('@databases/pg'); -const {sql} = require('@databases/pg'); - -async function run() { - // N.B. you will need to replace this connection - // string with the correct string for your database. - const db = createConnectionPool( - 'postgres://test-user@localhost:5432/test-db', - ); - - const results = await db.query(sql` - SELECT 1 + 1 as result; - `); - - console.log(results); - // => [{result: 2}] - - await db.dispose(); -} - -run().catch((err) => { - console.error(err); - process.exit(1); -}); - -*/ diff --git a/crawler/package-lock.json b/crawler/package-lock.json index b668946..20adfc3 100644 --- a/crawler/package-lock.json +++ b/crawler/package-lock.json @@ -10,7 +10,8 @@ "license": "ISC", "dependencies": { "@databases/pg": "^5.5.0", - "axios": "^1.13.2" + "axios": "^1.13.2", + "dotenv": "^17.2.3" } }, "node_modules/@babel/code-frame": { @@ -260,6 +261,18 @@ "node": ">=0.4.0" } }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", diff --git a/crawler/package.json b/crawler/package.json index 47988ed..9077d5f 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -10,6 +10,7 @@ "license": "ISC", "dependencies": { "@databases/pg": "^5.5.0", - "axios": "^1.13.2" + "axios": "^1.13.2", + "dotenv": "^17.2.3" } } From cc3216274b7b9056d5224375ee1a3e0679f1d3d7 Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 19 Nov 2025 16:00:16 +0100 Subject: [PATCH 10/68] deleted .env --- crawler/.env | 6 ------ crawler/.gitignore | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 crawler/.env diff --git a/crawler/.env b/crawler/.env deleted file mode 100644 index f5a0661..0000000 --- a/crawler/.env +++ /dev/null @@ -1,6 +0,0 @@ -# Database configuration -PGHOST=localhost -PGPORT=5432 -PGUSER=stac_user -PGPASSWORD=stac_pass -PGDATABASE=stac_db \ No newline at end of file diff --git a/crawler/.gitignore b/crawler/.gitignore index b512c09..a2e04e4 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -1 +1,3 @@ -node_modules \ No newline at end of file +node_modules + +.env \ No newline at end of file From 88f4bcc7bb05500f926ee3f8eb7adbd7a872602f Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 19 Nov 2025 21:58:57 +0100 Subject: [PATCH 11/68] updated crawling with use uf stac-js --- crawler/catalogs/catalog.js | 174 +++++++++-- crawler/package-lock.json | 565 +++++++++++++++++++++++++++++++++++- crawler/package.json | 3 +- 3 files changed, 714 insertions(+), 28 deletions(-) diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 4ff86d2..edb8b89 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -4,6 +4,7 @@ */ const axios = require('axios'); +const { default: create } = require('stac-js'); /** * Splits an array of catalogs into individual elements where each element is an array of its properties @@ -29,11 +30,22 @@ function splitCatalogs(catalogs) { const elements = catalogs.map((catalog, index) => { // Create object with all properties - const elementObj = { + let elementObj = { index: index, ...catalog, }; + // Validate and migrate STAC catalog using stac-js if it has STAC properties + if (catalog.url && typeof catalog.url === 'string') { + try { + // For STAC Index catalogs, we don't validate against STAC spec + // as they are metadata about STAC catalogs, not STAC objects themselves + // This is just for potential future enhancement + } catch (error) { + console.warn(`STAC validation skipped for catalog ${catalog.id}: ${error.message}`); + } + } + // Get all property names const allProperties = Object.keys(elementObj); @@ -82,7 +94,7 @@ function splitCatalogs(catalogs) { } /** - * Fetches and processes collections from a catalog + * Fetches and processes collections from a catalog using stac-js * @async * @param {Object} catalog - Catalog object with url property * @returns {Promise} Array of collections formatted as arrays @@ -99,26 +111,76 @@ async function getCollections(catalog) { for (const url of collectionUrls) { try { const response = await axios.get(url, { timeout: 5000 }); - const collectionsData = Array.isArray(response.data) - ? response.data - : response.data.collections || []; + + // Parse response with stac-js for validation and enhanced metadata extraction + let stacObj; + try { + stacObj = create(response.data, url); + } catch (parseError) { + // If stac-js cannot parse, skip this catalog (non-compliant STAC) + console.warn(` Skipping non-compliant STAC at ${url}: ${parseError.message}`); + continue; + } + + let collectionsData = []; + + // Check if this is a CollectionCollection (STAC API response) + if (stacObj && typeof stacObj.getAll === 'function') { + collectionsData = stacObj.getAll(); + } else if (Array.isArray(response.data)) { + // Handle array of collections + collectionsData = response.data.map(col => { + try { + return create(col, url); + } catch { + return null; + } + }).filter(Boolean); + } else if (response.data.collections) { + // Handle nested collections property + collectionsData = response.data.collections.map(col => { + try { + return create(col, url); + } catch { + return null; + } + }).filter(Boolean); + } if (collectionsData.length > 0) { console.log(`\n Catalog: ${catalog.id}`); console.log(` Found ${collectionsData.length} collections`); - // Convert collections to array format similar to catalogs - const collections = collectionsData.map((col, index) => { + // Convert collections using stac-js methods for metadata extraction + const collections = collectionsData.map((colObj, index) => { + // Use stac-js methods for robust metadata extraction + const bbox = typeof colObj.getBoundingBox === 'function' + ? colObj.getBoundingBox() + : (colObj.extent?.spatial?.bbox?.[0] || 'N/A'); + + const temporal = typeof colObj.getTemporalExtent === 'function' + ? colObj.getTemporalExtent() + : (colObj.extent?.temporal?.interval?.[0] || 'N/A'); + + // Get self URL using stac-js link navigation + let selfUrl = 'N/A'; + if (typeof colObj.getAbsoluteUrl === 'function') { + selfUrl = colObj.getAbsoluteUrl(); + } else if (colObj.links) { + const selfLink = colObj.links.find(l => l.rel === 'self'); + selfUrl = selfLink?.href || 'N/A'; + } + return [ index, - col.id || 'Unknown', - col.url || col.links?.find(l => l.rel === 'self')?.href || 'N/A', - col.title || 'N/A', - col.description || col.summary || 'N/A', - col.extent?.spatial?.bbox?.[0] || 'N/A', - col.extent?.temporal?.interval?.[0] || 'N/A', - col.license || 'N/A', - col.keywords || [] + colObj.id || 'Unknown', + selfUrl, + colObj.title || 'N/A', + colObj.description || colObj.summary || 'N/A', + bbox, + temporal, + colObj.license || 'N/A', + colObj.keywords || [] ]; }); @@ -148,14 +210,41 @@ async function getCollections(catalog) { } /** - * Fetches nested catalogs within a catalog + * Fetches nested catalogs within a catalog using stac-js link navigation * @async * @param {Object} catalog - Catalog object with url property + * @param {Object} [stacCatalog] - Optional stac-js catalog object for link extraction * @returns {Promise} Array of nested catalogs */ -async function getNestedCatalogs(catalog) { +async function getNestedCatalogs(catalog, stacCatalog = null) { try { - // Try common STAC nested catalog endpoints + // If we have a stac-js catalog object, use its link navigation methods + if (stacCatalog && typeof stacCatalog.getChildLinks === 'function') { + const childLinks = stacCatalog.getChildLinks(); + + if (childLinks.length > 0) { + console.log(` Found ${childLinks.length} child catalog links via stac-js`); + + // Fetch each child catalog + const nestedCatalogs = []; + for (const link of childLinks) { + try { + const childUrl = typeof link.getAbsoluteUrl === 'function' + ? link.getAbsoluteUrl() + : link.href; + + const response = await axios.get(childUrl, { timeout: 5000 }); + nestedCatalogs.push(response.data); + } catch (e) { + console.warn(` Failed to fetch child catalog: ${e.message}`); + } + } + + return nestedCatalogs; + } + } + + // Fallback: Try common STAC nested catalog endpoints const catalogUrls = [ `${catalog.url}/catalogs`, `${catalog.url}/catalogs/`, @@ -187,36 +276,69 @@ async function getNestedCatalogs(catalog) { } /** - * Recursively crawls a catalog, its collections, and nested catalogs + * Recursively crawls a catalog using stac-js for parsing and navigation * @async * @param {Object} catalog - Catalog object to crawl * @param {number} depth - Current depth level (for indentation) - * @returns {Promise} Stats object with collections count + * @returns {Promise} Stats object with collections count and STAC compliance */ async function crawlCatalogRecursive(catalog, depth = 0) { const indent = ' '.repeat(depth); - let stats = { collections: 0 }; + let stats = { collections: 0, stacCompliant: false }; try { + let stacCatalog = null; + + // Try to fetch and parse the catalog with stac-js + if (catalog.url) { + try { + const response = await axios.get(catalog.url, { timeout: 5000 }); + + try { + stacCatalog = create(response.data, catalog.url); + stats.stacCompliant = true; + + // Log STAC object type + if (typeof stacCatalog.isCatalog === 'function' && stacCatalog.isCatalog()) { + console.log(`${indent} STAC Catalog validated: ${catalog.id}`); + } else if (typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection()) { + console.log(`${indent} STAC Collection validated: ${catalog.id}`); + } + } catch (parseError) { + console.warn(`${indent} Non-compliant STAC catalog ${catalog.id}, skipping: ${parseError.message}`); + return stats; // Skip non-compliant catalogs as per requirement + } + } catch (fetchError) { + console.warn(`${indent} Failed to fetch catalog ${catalog.id}: ${fetchError.message}`); + return stats; + } + } + // First, try to get collections from this catalog const collections = await getCollections(catalog); stats.collections = collections.length; - // Then, try to get nested catalogs - const nestedCatalogs = await getNestedCatalogs(catalog); + // Then, try to get nested catalogs using stac-js if available + const nestedCatalogs = await getNestedCatalogs(catalog, stacCatalog); if (nestedCatalogs.length > 0) { console.log(`${indent} Nested catalogs found: ${nestedCatalogs.length}`); // Recursively crawl each nested catalog for (const nestedCatalog of nestedCatalogs.slice(0, 5)) { // Limit to first 5 nested - const nestedStats = await crawlCatalogRecursive(nestedCatalog, depth + 1); + // Convert nested catalog to expected format + const nestedCatalogObj = { + id: nestedCatalog.id || 'unknown', + url: nestedCatalog.links?.find(l => l.rel === 'self')?.href || nestedCatalog.url + }; + + const nestedStats = await crawlCatalogRecursive(nestedCatalogObj, depth + 1); stats.collections += nestedStats.collections; } } } catch (error) { - console.error(`Error in recursive crawl for ${catalog.id}: ${error.message}`); + console.error(`${indent}Error in recursive crawl for ${catalog.id}: ${error.message}`); } return stats; @@ -257,4 +379,4 @@ module.exports = { getNestedCatalogs, crawlCatalogRecursive, deriveCategories -}; +}; \ No newline at end of file diff --git a/crawler/package-lock.json b/crawler/package-lock.json index 9281009..9b74099 100644 --- a/crawler/package-lock.json +++ b/crawler/package-lock.json @@ -9,7 +9,63 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "axios": "^1.13.2" + "axios": "^1.13.2", + "stac-js": "^0.1.9" + } + }, + "node_modules/@multiformats/base-x": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@multiformats/base-x/-/base-x-4.0.1.tgz", + "integrity": "sha512-eMk0b9ReBbV23xXU693TAIrLyeO5iTgBZGSJfpqriG8UkYvr/hC9u9pyMlAakDNHWmbhMZCDs6KQO0jzKD8OTw==", + "license": "MIT" + }, + "node_modules/@radiantearth/stac-migrate": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@radiantearth/stac-migrate/-/stac-migrate-2.0.2.tgz", + "integrity": "sha512-1GsszCVwZY42tQCOnhLzOERnXXzJBX5PQ5Qq1puBI19liWzdJZO2+6bbKmQclkYsPJYiQ6JnLbGf5QWIw4S+kQ==", + "license": "Apache-2.0", + "dependencies": { + "compare-versions": "^3.6.0", + "multihashes": "^3.1.2", + "yargs": "^17.6.2" + }, + "bin": { + "stac-migrate": "bin/cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/m-mohr" + } + }, + "node_modules/@zxing/text-encoding": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", + "integrity": "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==", + "license": "(Unlicense OR Apache-2.0)", + "optional": true + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/asynckit": { @@ -18,6 +74,21 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "license": "MIT" }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", + "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/axios": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", @@ -29,6 +100,24 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/call-bind": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-define-property": "^1.0.0", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", @@ -42,6 +131,54 @@ "node": ">= 0.4" } }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -54,6 +191,29 @@ "node": ">= 0.8" } }, + "node_modules/compare-versions": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-3.6.0.tgz", + "integrity": "sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==", + "license": "MIT" + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -77,6 +237,12 @@ "node": ">= 0.4" } }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -122,6 +288,15 @@ "node": ">= 0.4" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/follow-redirects": { "version": "1.15.11", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", @@ -142,6 +317,21 @@ } } }, + "node_modules/for-each": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/form-data": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", @@ -167,6 +357,24 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -216,6 +424,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -255,6 +475,101 @@ "node": ">= 0.4" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is-arguments": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", + "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-function": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -285,11 +600,259 @@ "node": ">= 0.6" } }, + "node_modules/multibase": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/multibase/-/multibase-3.1.2.tgz", + "integrity": "sha512-bpklWHs70LO3smJUHOjcnzGceJJvn9ui0Vau6Za0B/GBepaXswmW8Ufea0uD9pROf/qCQ4N4lZ3sf3U+SNf0tw==", + "deprecated": "This module has been superseded by the multiformats module", + "license": "MIT", + "dependencies": { + "@multiformats/base-x": "^4.0.1", + "web-encoding": "^1.0.6" + }, + "engines": { + "node": ">=10.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, + "node_modules/multihashes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/multihashes/-/multihashes-3.1.2.tgz", + "integrity": "sha512-AP4IoV/YzkNrfbQKZE3OMPibrmy350OmCd6cJkwyM8oExaXIlOY4UnOOVSQtAEuq/LR01XfXKCESidzZvSwHCQ==", + "license": "MIT", + "dependencies": { + "multibase": "^3.1.0", + "uint8arrays": "^2.0.5", + "varint": "^6.0.0" + }, + "engines": { + "node": ">=10.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/stac-js": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/stac-js/-/stac-js-0.1.9.tgz", + "integrity": "sha512-qxb8aWQ1T8jJVLsUszpvj9+KrPTDdf2s0bWBWTP+SaplNxznKXXo5xkEPtrXEYwOTbLLeUVSNQhz9J5uFcz6TA==", + "license": "Apache-2.0", + "dependencies": { + "@radiantearth/stac-migrate": "^2.0.2", + "urijs": "^1.19.11" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/m-mohr" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/uint8arrays": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-2.1.10.tgz", + "integrity": "sha512-Q9/hhJa2836nQfEJSZTmr+pg9+cDJS9XEAp7N2Vg5MzL3bK/mkMVfjscRGYruP9jNda6MAdf4QD/y78gSzkp6A==", + "license": "MIT", + "dependencies": { + "multiformats": "^9.4.2" + } + }, + "node_modules/urijs": { + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", + "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==", + "license": "MIT" + }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, + "node_modules/varint": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", + "integrity": "sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==", + "license": "MIT" + }, + "node_modules/web-encoding": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", + "integrity": "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==", + "license": "MIT", + "dependencies": { + "util": "^0.12.3" + }, + "optionalDependencies": { + "@zxing/text-encoding": "0.9.0" + } + }, + "node_modules/which-typed-array": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", + "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } } } } diff --git a/crawler/package.json b/crawler/package.json index e9ab811..8fb5506 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -14,6 +14,7 @@ "author": "", "license": "ISC", "dependencies": { - "axios": "^1.13.2" + "axios": "^1.13.2", + "stac-js": "^0.1.9" } } From e15fcaed2004d0019ac510db94861d00f762f733 Mon Sep 17 00:00:00 2001 From: mammutor Date: Fri, 21 Nov 2025 00:54:03 +0100 Subject: [PATCH 12/68] feat: basic api crawling --- crawler/apis/api.js | 156 +++++++++++++++++++++++++++++++++++++++++++- crawler/index.js | 25 +++++++ 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 87cdde2..eaaece9 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -1,4 +1,158 @@ /** * @fileoverview API crawling functionality for STAC Index * @module apis/api - */ \ No newline at end of file + */ + +const axios = require('axios'); +const { default: create } = require('stac-js'); + +/** + * Crawls STAC APIs to retrieve collection information without fetching items. + * + * @param {string[]} urls - Array of API URLs to crawl + * @param {boolean} isApi - Boolean flag indicating if the URLs are APIs + * @returns {Promise} Array of STAC Collection objects ordered by URL + */ +async function crawlApis(urls, isApi) { + if (!isApi || !Array.isArray(urls) || urls.length === 0) { + return []; + } + + const allCollections = new Map(); // Use Map to avoid duplicates by URL + + console.log(`Starting to crawl ${urls.length} APIs...`); + + // Process each URL + for (const [index, url] of urls.entries()) { + console.log(`Processing API ${index + 1}/${urls.length}: ${url}`); + try { + await crawlSingleApi(url, allCollections); + } catch (error) { + console.error(`Error crawling API ${url}:`, error.message); + } + } + + console.log(`Finished crawling APIs. Total unique collections found: ${allCollections.size}`); + + // Convert Map to array and sort by URL + const sortedCollections = Array.from(allCollections.values()).sort((a, b) => { + const urlA = getSelfLink(a) || ''; + const urlB = getSelfLink(b) || ''; + return urlA.localeCompare(urlB); + }); + + return sortedCollections; +} + +/** + * Helper function to crawl a single API recursively for collections/catalogs + * + * @param {string} url - URL to crawl + * @param {Map} collectionMap - Map to store found collections + * @param {Set} visited - Set of visited URLs to prevent loops + */ +async function crawlSingleApi(url, collectionMap, visited = new Set()) { + if (!url || visited.has(url)) return; + visited.add(url); + + try { + console.log(` Fetching: ${url}`); + const response = await axios.get(url); + const stacObj = create(response.data); + + // If it's a Collection, add it + if (stacObj.isCollection()) { + const selfUrl = stacObj.getAbsoluteUrl() || url; + console.log(` Found collection: ${stacObj.id} (${selfUrl})`); + collectionMap.set(selfUrl, stacObj.toJSON()); + } + + // If it has collections (API or Catalog), fetch them + // stac-js APICollection or Catalog might have getApiCollectionsLink or similar + // Or we manually check links. + + // Check for /collections endpoint if it's a root catalog/API + if (stacObj.isCatalogLike()) { + // Try to get collections link + const collectionsLink = stacObj.getApiCollectionsLink(); + if (collectionsLink) { + console.log(` Found /collections endpoint link in ${url}`); + await fetchCollectionsFromLink(collectionsLink, collectionMap, visited); + } else { + // Fallback: check standard STAC API structure if not explicitly found + // Many STAC APIs have a /collections endpoint relative to root + // But stac-js might handle this via getApiCollectionsLink if rel="data" exists + + // Also check child links for nested catalogs + const childLinks = stacObj.getChildLinks(); + if (childLinks.length > 0) { + console.log(` Found ${childLinks.length} child links in ${url}, recursing...`); + for (const link of childLinks) { + const childUrl = link.getAbsoluteUrl(); + if (childUrl) { + await crawlSingleApi(childUrl, collectionMap, visited); + } + } + } + } + } + + } catch (error) { + console.warn(`Failed to process ${url}:`, error.message); + } +} + +/** + * Fetches collections from a collections endpoint (e.g., /collections) + * + * @param {Object} link - stac-js Link object + * @param {Map} collectionMap - Map to store collections + * @param {Set} visited - Visited set + */ +async function fetchCollectionsFromLink(link, collectionMap, visited) { + const url = link.getAbsoluteUrl(); + if (!url || visited.has(url)) return; + visited.add(url); + + try { + console.log(` Fetching collections from: ${url}`); + const response = await axios.get(url); + // create() handles CollectionCollection (API Collections response) + const stacObj = create(response.data); + + let collections = []; + + if (stacObj && typeof stacObj.getAll === 'function') { + // Use stac-js to get all collections from the response + collections = stacObj.getAll(); + } else if (response.data.collections && Array.isArray(response.data.collections)) { + // Fallback for manual parsing if stac-js didn't detect CollectionCollection + collections = response.data.collections.map(c => create(c)); + } else if (Array.isArray(response.data)) { + collections = response.data.map(c => create(c)); + } + + console.log(` Retrieved ${collections.length} collections from ${url}`); + + for (const colStac of collections) { + if (colStac && typeof colStac.isCollection === 'function' && colStac.isCollection()) { + const selfUrl = colStac.getAbsoluteUrl() || url; + collectionMap.set(selfUrl, colStac.toJSON()); + } + } + } catch (error) { + console.warn(`Failed to fetch collections from ${url}:`, error.message); + } +} + +/** + * Helper to get self link from a plain JSON object (since we store toJSON results) + */ +function getSelfLink(stacJson) { + const selfLink = stacJson.links?.find(l => l.rel === 'self'); + return selfLink ? selfLink.href : null; +} + +module.exports = { + crawlApis +}; diff --git a/crawler/index.js b/crawler/index.js index 9f2f260..d237961 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -5,6 +5,7 @@ const axios = require('axios'); const { splitCatalogs, crawlCatalogRecursive } = require('./catalogs/catalog'); +const { crawlApis } = require('./apis/api'); /** * URL of the STAC Index API endpoint @@ -32,6 +33,30 @@ const crawler = async () => { } console.log(`\n Total collections found across all catalogs: ${totalCollections}`); + + // Extract API URLs and crawl them + console.log('\n Crawling APIs...'); + const apiUrls = catalogs + .filter(cat => cat[10] === true) // isApi is at index 10 + .map(cat => cat[2]); // url is at index 2 + + if (apiUrls.length > 0) { + console.log(`Found ${apiUrls.length} APIs. Starting crawl...`); + // Limit to first 5 APIs for demonstration/performance + const collections = await crawlApis(apiUrls.slice(0, 5), true); + console.log(`\nFetched ${collections.length} collections from APIs (sorted by URL).`); + + if (collections.length > 0) { + console.log('First 3 collections found:'); + collections.slice(0, 3).forEach(c => { + const selfLink = c.links?.find(l => l.rel === 'self')?.href; + console.log(` - ${c.id}: ${selfLink}`); + }); + } + } else { + console.log('No APIs found to crawl.'); + } + } catch (error) { console.error(`Error fetching ${targetUrl}: ${error.message}`); } From fa5fdaeaf29303ce1922de178b4854b8e9fece99 Mon Sep 17 00:00:00 2001 From: Jakob Date: Mon, 24 Nov 2025 14:40:51 +0100 Subject: [PATCH 13/68] implements #76, stac api validation for catalog crawling --- crawler/catalogs/catalog.js | 119 +++++++++++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 3 deletions(-) diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index edb8b89..f885470 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -6,6 +6,101 @@ const axios = require('axios'); const { default: create } = require('stac-js'); +/** + * Checks if a catalog is STAC API compliant by validating conformance classes + * @async + * @param {string} catalogUrl - The base URL of the catalog + * @returns {Promise} Object with isStacApi flag and conformance classes + */ +async function checkStacApiCompliance(catalogUrl) { + const result = { + isStacApi: false, + conformsTo: [], + features: [] + }; + + try { + // Try to fetch conformance declaration + const conformanceUrls = [ + `${catalogUrl}/conformance`, + `${catalogUrl}/api/conformance`, + catalogUrl // Root might have conformsTo property + ]; + + for (const url of conformanceUrls) { + try { + const response = await axios.get(url, { timeout: 5000 }); + + // Check for conformsTo array (STAC API spec) + if (response.data.conformsTo && Array.isArray(response.data.conformsTo)) { + result.conformsTo = response.data.conformsTo; + + // Check for STAC API specific conformance classes + const stacApiClasses = [ + 'https://api.stacspec.org/v1.0.0/core', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', + 'https://api.stacspec.org/v1.0.0-rc.1/core' + ]; + + result.isStacApi = result.conformsTo.some(conformance => + stacApiClasses.some(stacClass => conformance.includes('stacspec.org') || conformance.includes('ogcapi-features')) + ); + + // Identify supported features + if (result.conformsTo.some(c => c.includes('item-search') || c.includes('/search'))) { + result.features.push('search'); + } + if (result.conformsTo.some(c => c.includes('filter'))) { + result.features.push('filter'); + } + if (result.conformsTo.some(c => c.includes('query'))) { + result.features.push('query'); + } + if (result.conformsTo.some(c => c.includes('sort'))) { + result.features.push('sort'); + } + if (result.conformsTo.some(c => c.includes('fields'))) { + result.features.push('fields'); + } + + if (result.isStacApi) { + break; // Found valid STAC API conformance + } + } + } catch (e) { + // Try next URL + continue; + } + } + + // Fallback: Check for /search endpoint existence (practical test) + if (!result.isStacApi) { + try { + const searchResponse = await axios.get(`${catalogUrl}/search`, { + timeout: 3000, + params: { limit: 1 }, + validateStatus: (status) => status < 500 // Accept 4xx as "endpoint exists" + }); + + // If search endpoint exists and returns expected structure + if (searchResponse.status === 200 && + (searchResponse.data.type === 'FeatureCollection' || + searchResponse.data.features)) { + result.isStacApi = true; + result.features.push('search'); + } + } catch (e) { + // Search endpoint doesn't exist - likely static catalog + } + } + + } catch (error) { + // STAC API validation failed - catalog is likely static-only + } + + return result; +} + /** * Splits an array of catalogs into individual elements where each element is an array of its properties * @param {Array} catalogs - Array of catalog objects from the STAC Index API @@ -280,11 +375,17 @@ async function getNestedCatalogs(catalog, stacCatalog = null) { * @async * @param {Object} catalog - Catalog object to crawl * @param {number} depth - Current depth level (for indentation) - * @returns {Promise} Stats object with collections count and STAC compliance + * @returns {Promise} Stats object with collections count, STAC compliance, and API capabilities */ async function crawlCatalogRecursive(catalog, depth = 0) { const indent = ' '.repeat(depth); - let stats = { collections: 0, stacCompliant: false }; + let stats = { + collections: 0, + stacCompliant: false, + stacApiCompliant: false, + apiFeatures: [], + conformsTo: [] + }; try { let stacCatalog = null; @@ -304,6 +405,17 @@ async function crawlCatalogRecursive(catalog, depth = 0) { } else if (typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection()) { console.log(`${indent} STAC Collection validated: ${catalog.id}`); } + + // Check STAC API compliance + const apiCompliance = await checkStacApiCompliance(catalog.url); + stats.stacApiCompliant = apiCompliance.isStacApi; + stats.apiFeatures = apiCompliance.features; + stats.conformsTo = apiCompliance.conformsTo; + + if (apiCompliance.isStacApi) { + console.log(`${indent} STAC API compliant - Features: [${apiCompliance.features.join(', ')}]`); + } + } catch (parseError) { console.warn(`${indent} Non-compliant STAC catalog ${catalog.id}, skipping: ${parseError.message}`); return stats; // Skip non-compliant catalogs as per requirement @@ -378,5 +490,6 @@ module.exports = { getCollections, getNestedCatalogs, crawlCatalogRecursive, - deriveCategories + deriveCategories, + checkStacApiCompliance }; \ No newline at end of file From b611ef49a2ddf938d658336e4d9617fe71f56a2d Mon Sep 17 00:00:00 2001 From: mammutor Date: Mon, 24 Nov 2025 15:58:22 +0100 Subject: [PATCH 14/68] fix: updated crawler project to be ESM compliant I had an issue on the server, where it doesnt want to start, because stac-js is ESM only and the project isnt --- crawler/apis/api.js | 6 +++--- crawler/catalogs/catalog.js | 6 +++--- crawler/index.js | 6 +++--- crawler/package.json | 1 + 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index eaaece9..5eda959 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -3,8 +3,8 @@ * @module apis/api */ -const axios = require('axios'); -const { default: create } = require('stac-js'); +import axios from 'axios'; +import create from 'stac-js'; /** * Crawls STAC APIs to retrieve collection information without fetching items. @@ -153,6 +153,6 @@ function getSelfLink(stacJson) { return selfLink ? selfLink.href : null; } -module.exports = { +export { crawlApis }; diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index f885470..fe2bbb1 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -3,8 +3,8 @@ * @module catalogs/catalog */ -const axios = require('axios'); -const { default: create } = require('stac-js'); +import axios from 'axios'; +import create from 'stac-js'; /** * Checks if a catalog is STAC API compliant by validating conformance classes @@ -485,7 +485,7 @@ function deriveCategories(catalog) { return []; } -module.exports = { +export { splitCatalogs, getCollections, getNestedCatalogs, diff --git a/crawler/index.js b/crawler/index.js index d237961..d949a5c 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -3,9 +3,9 @@ * @module crawler */ -const axios = require('axios'); -const { splitCatalogs, crawlCatalogRecursive } = require('./catalogs/catalog'); -const { crawlApis } = require('./apis/api'); +import axios from 'axios'; +import { splitCatalogs, crawlCatalogRecursive } from './catalogs/catalog.js'; +import { crawlApis } from './apis/api.js'; /** * URL of the STAC Index API endpoint diff --git a/crawler/package.json b/crawler/package.json index 8fb5506..49e9982 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -13,6 +13,7 @@ }, "author": "", "license": "ISC", + "type": "module", "dependencies": { "axios": "^1.13.2", "stac-js": "^0.1.9" From 9fc290000bf204cb8b4428afd39a596afa00f169 Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 25 Nov 2025 14:44:46 +0100 Subject: [PATCH 15/68] refactor: improve catalog handling in crawler by restructuring catalog data extraction and ensuring consistent property ordering --- crawler/catalogs/catalog.js | 5 ++--- crawler/index.js | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index fe2bbb1..daed1eb 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -145,14 +145,13 @@ function splitCatalogs(catalogs) { const allProperties = Object.keys(elementObj); // Separate standard and dynamic properties - const standardProps = standardPropertyOrder.filter(prop => prop in elementObj); const dynamicProps = allProperties.filter(prop => !standardPropertyOrder.includes(prop)); // Create array with standard properties first, then dynamic ones const elementArray = []; - // Add standard properties in defined order - standardProps.forEach(prop => { + // Add standard properties in defined order (always include all standard fields for consistent indices) + standardPropertyOrder.forEach(prop => { elementArray.push(elementObj[prop]); }); diff --git a/crawler/index.js b/crawler/index.js index d949a5c..bdbb7d9 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -27,7 +27,11 @@ const crawler = async () => { // Crawl collections and nested catalogs for each catalog console.log('\n Crawling collections and nested catalogs...\n'); let totalCollections = 0; - for (const catalog of response.data.slice(0, 10)) { // Limit to first 10 catalogs + for (const catalogData of catalogs.slice(0, 10)) { // Limit to first 10 catalogs + const catalog = { + id: catalogData[1], + url: catalogData[2] + }; const stats = await crawlCatalogRecursive(catalog); totalCollections += stats.collections; } From ad9a4408f89867d3d80d701aec39b6a8c23b0a42 Mon Sep 17 00:00:00 2001 From: Lenn Date: Tue, 25 Nov 2025 17:58:28 +0100 Subject: [PATCH 16/68] feat: enhance database operations: insertOrUpdateCatalog and insertOrUpdateCollection --- crawler/db.js | 215 ++++++++++++++++++++++++++++++++++++++++++----- crawler/index.js | 15 ++++ 2 files changed, 208 insertions(+), 22 deletions(-) diff --git a/crawler/db.js b/crawler/db.js index 3a7788b..4980428 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -25,35 +25,206 @@ async function initDb() { } } +/** + * Insert or update a catalog in the database + * @param {Object} catalog - STAC catalog object + * @returns {Promise} catalog ID + */ async function insertOrUpdateCatalog(catalog) { if (!catalog || typeof catalog !== 'object') return null; - const catalogId = catalog.id || catalog.catalog_id || catalog.slug || null; - if (!catalogId) return null; - - const url = catalog.url || null; - const title = catalog.title || catalog.name || null; - const metadata = catalog; - const now = new Date().toISOString(); - - const query = ` - INSERT INTO catalogs (catalog_id, url, title, metadata, updated_at) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (catalog_id) DO UPDATE SET - url = EXCLUDED.url, - title = EXCLUDED.title, - metadata = catalogs.metadata || EXCLUDED.metadata, - updated_at = EXCLUDED.updated_at - RETURNING id; - `; - const values = [catalogId, url, title, metadata, now]; - const res = await pool.query(query, values); - return res.rows[0]; + + const client = await pool.connect(); + try { + await client.query('BEGIN'); + + // Insert or update catalog + const catalogQuery = ` + INSERT INTO catalog (stac_version, type, title, description, updated_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT ON CONSTRAINT catalog_pkey DO UPDATE SET + stac_version = EXCLUDED.stac_version, + type = EXCLUDED.type, + title = EXCLUDED.title, + description = EXCLUDED.description, + updated_at = now() + RETURNING id; + `; + const catalogResult = await client.query(catalogQuery, [ + catalog.stac_version || null, + catalog.type || 'Catalog', + catalog.title || catalog.id || null, + catalog.description || null + ]); + const catalogId = catalogResult.rows[0].id; + + // Insert links + if (catalog.links && Array.isArray(catalog.links)) { + await client.query('DELETE FROM catalog_links WHERE catalog_id = $1', [catalogId]); + for (const link of catalog.links) { + await client.query( + 'INSERT INTO catalog_links (catalog_id, rel, href, type, title) VALUES ($1, $2, $3, $4, $5)', + [catalogId, link.rel, link.href, link.type || null, link.title || null] + ); + } + } + + // Insert keywords + if (catalog.keywords && Array.isArray(catalog.keywords)) { + await insertKeywords(client, catalogId, catalog.keywords, 'catalog'); + } + + // Insert STAC extensions + if (catalog.stac_extensions && Array.isArray(catalog.stac_extensions)) { + await insertStacExtensions(client, catalogId, catalog.stac_extensions, 'catalog'); + } + + // Update crawl log + await client.query( + `INSERT INTO crawllog_catalog (catalog_id, last_crawled) + VALUES ($1, now()) + ON CONFLICT (catalog_id) DO UPDATE SET last_crawled = now()`, + [catalogId] + ); + + await client.query('COMMIT'); + return catalogId; + } catch (error) { + await client.query('ROLLBACK'); + console.error('Error inserting catalog:', error.message); + throw error; + } finally { + client.release(); + } } +/** + * Insert or update a collection in the database + * @param {Object} collection - STAC collection object + * @returns {Promise} collection ID + */ +async function insertOrUpdateCollection(collection) { + if (!collection || typeof collection !== 'object') return null; + + const client = await pool.connect(); + try { + await client.query('BEGIN'); + + // Parse spatial extent (bbox) + let spatialExtend = null; + if (collection.extent?.spatial?.bbox && collection.extent.spatial.bbox[0]) { + const bbox = collection.extent.spatial.bbox[0]; + if (bbox.length === 4) { + // Create polygon from bbox [west, south, east, north] + spatialExtend = `SRID=4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; + } + } + + // Parse temporal extent + let temporalStart = null; + let temporalEnd = null; + if (collection.extent?.temporal?.interval && collection.extent.temporal.interval[0]) { + const interval = collection.extent.temporal.interval[0]; + temporalStart = interval[0] ? new Date(interval[0]) : null; + temporalEnd = interval[1] ? new Date(interval[1]) : null; + } + + // Insert or update collection + const collectionQuery = ` + INSERT INTO collection ( + stac_version, type, title, description, license, + spatial_extend, temporal_extend_start, temporal_extend_end, + is_api, is_active, full_json, updated_at + ) + VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, now()) + ON CONFLICT ON CONSTRAINT collection_pkey DO UPDATE SET + stac_version = EXCLUDED.stac_version, + type = EXCLUDED.type, + title = EXCLUDED.title, + description = EXCLUDED.description, + license = EXCLUDED.license, + spatial_extend = EXCLUDED.spatial_extend, + temporal_extend_start = EXCLUDED.temporal_extend_start, + temporal_extend_end = EXCLUDED.temporal_extend_end, + is_api = EXCLUDED.is_api, + is_active = EXCLUDED.is_active, + full_json = EXCLUDED.full_json, + updated_at = now() + RETURNING id; + `; + const collectionResult = await client.query(collectionQuery, [ + collection.stac_version || null, + collection.type || 'Collection', + collection.title || collection.id || null, + collection.description || null, + collection.license || null, + spatialExtend, + temporalStart, + temporalEnd, + false, // is_api - will be determined by crawler + true, // is_active + JSON.stringify(collection) + ]); + const collectionId = collectionResult.rows[0].id; + + // Insert summaries + if (collection.summaries && typeof collection.summaries === 'object') { + await client.query('DELETE FROM collection_summaries WHERE collection_id = $1', [collectionId]); + for (const [name, value] of Object.entries(collection.summaries)) { + await insertSummary(client, collectionId, name, value); + } + } + + // Insert keywords + if (collection.keywords && Array.isArray(collection.keywords)) { + await insertKeywords(client, collectionId, collection.keywords, 'collection'); + } + + // Insert STAC extensions + if (collection.stac_extensions && Array.isArray(collection.stac_extensions)) { + await insertStacExtensions(client, collectionId, collection.stac_extensions, 'collection'); + } + + // Insert providers + if (collection.providers && Array.isArray(collection.providers)) { + await insertProviders(client, collectionId, collection.providers); + } + + // Insert assets + if (collection.assets && typeof collection.assets === 'object') { + await insertAssets(client, collectionId, collection.assets); + } + + // Update crawl log + await client.query( + `INSERT INTO crawllog_collection (collection_id, last_crawled) + VALUES ($1, now()) + ON CONFLICT (collection_id) DO UPDATE SET last_crawled = now()`, + [collectionId] + ); + + await client.query('COMMIT'); + return collectionId; + } catch (error) { + await client.query('ROLLBACK'); + console.error('Error inserting collection:', error.message); + throw error; + } finally { + client.release(); + } +} + +// TODO: Implement insertKeywords, insertStacExtensions, insertSummary, insertProviders, insertAssets functions + async function close() { await pool.end(); } -module.exports = { initDb, insertOrUpdateCatalog, close, pool }; +module.exports = { + initDb, + insertOrUpdateCatalog, + insertOrUpdateCollection, + close, + pool +}; diff --git a/crawler/index.js b/crawler/index.js index 3af528c..e8dae19 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -35,6 +35,21 @@ const crawler = async () => { } catch (error) { console.error(`Error fetching ${targetUrl}: ${error.message}`); } + + // Persist catalogs into DB + try { + await Promise.all(catalogs.map(async (catalog) => { + try { + await db.insertOrUpdateCatalog(catalog); + } catch (err) { + console.error('DB write error for catalog', catalog && (catalog.id || catalog.slug), err && err.message); + } + })); + console.log('Finished writing catalogs to DB'); + } catch (e) { + console.error('Unexpected error while saving catalogs to DB', e && e.message); + } + }; crawler(); \ No newline at end of file From d0691ba561f8e11952c4d7a077bb3e2d00b762c6 Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 25 Nov 2025 18:44:39 +0100 Subject: [PATCH 17/68] fix: adjust API crawling limit to include all found APIs instead of the first five --- crawler/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler/index.js b/crawler/index.js index bdbb7d9..7cffb6a 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -47,7 +47,7 @@ const crawler = async () => { if (apiUrls.length > 0) { console.log(`Found ${apiUrls.length} APIs. Starting crawl...`); // Limit to first 5 APIs for demonstration/performance - const collections = await crawlApis(apiUrls.slice(0, 5), true); + const collections = await crawlApis(apiUrls, true); console.log(`\nFetched ${collections.length} collections from APIs (sorted by URL).`); if (collections.length > 0) { From 3c37635adb68f81f6910bee95a09a304b9d5922e Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 26 Nov 2025 12:18:02 +0100 Subject: [PATCH 18/68] updated pghost function to insert keywords --- crawler/db.js | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crawler/db.js b/crawler/db.js index 4980428..23c5db5 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -6,7 +6,7 @@ const { Pool } = require('pg'); require('dotenv').config(); const pool = new Pool({ - host: process.env.PGHOST || 'localhost', + host: process.env.PGHOST || 'atlas.stacindex.org', port: parseInt(process.env.PGPORT || '5432', 10), user: process.env.PGUSER || 'stac_user', password: process.env.PGPASSWORD || 'stac_pass', @@ -215,6 +215,36 @@ async function insertOrUpdateCollection(collection) { // TODO: Implement insertKeywords, insertStacExtensions, insertSummary, insertProviders, insertAssets functions + +/** + * Helper function to insert keywords + */ +async function insertKeywords(client, parentId, keywords, type) { + await client.query( + `DELETE FROM ${type}_keywords WHERE ${type}_id = $1`, + [parentId] + ); + + for (const keyword of keywords) { + if (!keyword) continue; + + // Insert keyword if not exists + const keywordResult = await client.query( + 'INSERT INTO keywords (keyword) VALUES ($1) ON CONFLICT (keyword) DO UPDATE SET keyword = EXCLUDED.keyword RETURNING id', + [keyword] + ); + const keywordId = keywordResult.rows[0].id; + + // Link keyword to parent + await client.query( + `INSERT INTO ${type}_keywords (${type}_id, keyword_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, + [parentId, keywordId] + ); + } +} + + + async function close() { await pool.end(); } From f1f3e7b057a5bf6172ec2496500225dfa9a2d79b Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 26 Nov 2025 14:57:32 +0100 Subject: [PATCH 19/68] feat: changed database connection configuration and add insertStacExtensions helper function --- crawler/db.js | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/crawler/db.js b/crawler/db.js index 23c5db5..c6b813f 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -6,11 +6,11 @@ const { Pool } = require('pg'); require('dotenv').config(); const pool = new Pool({ - host: process.env.PGHOST || 'atlas.stacindex.org', - port: parseInt(process.env.PGPORT || '5432', 10), - user: process.env.PGUSER || 'stac_user', - password: process.env.PGPASSWORD || 'stac_pass', - database: process.env.PGDATABASE || 'stac_db', + host: process.env.PGHOST, + port: parseInt(process.env.PGPORT, 10), + user: process.env.PGUSER , + password: process.env.PGPASSWORD , + database: process.env.PGDATABASE , max: 10, }); @@ -243,6 +243,24 @@ async function insertKeywords(client, parentId, keywords, type) { } } +/** + * Helper function to insert STAC extensions + */ +async function insertStacExtensions(client, parentId, extensions, type) { + await client.query( + `DELETE FROM ${type}_stac_extensions WHERE ${type}_id = $1`, + [parentId] + ); + + for (const extension of extensions) { + await client.query( + `INSERT INTO ${type}_stac_extensions (${type}_id, extension) VALUES ($1, $2)`, + [parentId, extension] + ); + } +} + + async function close() { From 54a231c33fe1c68bde7fcadd5756cff2e0ebef2d Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 26 Nov 2025 15:04:44 +0100 Subject: [PATCH 20/68] removed the api valication #76 --- crawler/catalogs/catalog.js | 115 +----------------------------------- crawler/package-lock.json | 72 ++++++++++++++++++++++ 2 files changed, 75 insertions(+), 112 deletions(-) diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index f885470..117e412 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -6,101 +6,6 @@ const axios = require('axios'); const { default: create } = require('stac-js'); -/** - * Checks if a catalog is STAC API compliant by validating conformance classes - * @async - * @param {string} catalogUrl - The base URL of the catalog - * @returns {Promise} Object with isStacApi flag and conformance classes - */ -async function checkStacApiCompliance(catalogUrl) { - const result = { - isStacApi: false, - conformsTo: [], - features: [] - }; - - try { - // Try to fetch conformance declaration - const conformanceUrls = [ - `${catalogUrl}/conformance`, - `${catalogUrl}/api/conformance`, - catalogUrl // Root might have conformsTo property - ]; - - for (const url of conformanceUrls) { - try { - const response = await axios.get(url, { timeout: 5000 }); - - // Check for conformsTo array (STAC API spec) - if (response.data.conformsTo && Array.isArray(response.data.conformsTo)) { - result.conformsTo = response.data.conformsTo; - - // Check for STAC API specific conformance classes - const stacApiClasses = [ - 'https://api.stacspec.org/v1.0.0/core', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', - 'https://api.stacspec.org/v1.0.0-rc.1/core' - ]; - - result.isStacApi = result.conformsTo.some(conformance => - stacApiClasses.some(stacClass => conformance.includes('stacspec.org') || conformance.includes('ogcapi-features')) - ); - - // Identify supported features - if (result.conformsTo.some(c => c.includes('item-search') || c.includes('/search'))) { - result.features.push('search'); - } - if (result.conformsTo.some(c => c.includes('filter'))) { - result.features.push('filter'); - } - if (result.conformsTo.some(c => c.includes('query'))) { - result.features.push('query'); - } - if (result.conformsTo.some(c => c.includes('sort'))) { - result.features.push('sort'); - } - if (result.conformsTo.some(c => c.includes('fields'))) { - result.features.push('fields'); - } - - if (result.isStacApi) { - break; // Found valid STAC API conformance - } - } - } catch (e) { - // Try next URL - continue; - } - } - - // Fallback: Check for /search endpoint existence (practical test) - if (!result.isStacApi) { - try { - const searchResponse = await axios.get(`${catalogUrl}/search`, { - timeout: 3000, - params: { limit: 1 }, - validateStatus: (status) => status < 500 // Accept 4xx as "endpoint exists" - }); - - // If search endpoint exists and returns expected structure - if (searchResponse.status === 200 && - (searchResponse.data.type === 'FeatureCollection' || - searchResponse.data.features)) { - result.isStacApi = true; - result.features.push('search'); - } - } catch (e) { - // Search endpoint doesn't exist - likely static catalog - } - } - - } catch (error) { - // STAC API validation failed - catalog is likely static-only - } - - return result; -} - /** * Splits an array of catalogs into individual elements where each element is an array of its properties * @param {Array} catalogs - Array of catalog objects from the STAC Index API @@ -375,16 +280,13 @@ async function getNestedCatalogs(catalog, stacCatalog = null) { * @async * @param {Object} catalog - Catalog object to crawl * @param {number} depth - Current depth level (for indentation) - * @returns {Promise} Stats object with collections count, STAC compliance, and API capabilities + * @returns {Promise} Stats object with collections count and STAC compliance */ async function crawlCatalogRecursive(catalog, depth = 0) { const indent = ' '.repeat(depth); let stats = { collections: 0, - stacCompliant: false, - stacApiCompliant: false, - apiFeatures: [], - conformsTo: [] + stacCompliant: false }; try { @@ -406,16 +308,6 @@ async function crawlCatalogRecursive(catalog, depth = 0) { console.log(`${indent} STAC Collection validated: ${catalog.id}`); } - // Check STAC API compliance - const apiCompliance = await checkStacApiCompliance(catalog.url); - stats.stacApiCompliant = apiCompliance.isStacApi; - stats.apiFeatures = apiCompliance.features; - stats.conformsTo = apiCompliance.conformsTo; - - if (apiCompliance.isStacApi) { - console.log(`${indent} STAC API compliant - Features: [${apiCompliance.features.join(', ')}]`); - } - } catch (parseError) { console.warn(`${indent} Non-compliant STAC catalog ${catalog.id}, skipping: ${parseError.message}`); return stats; // Skip non-compliant catalogs as per requirement @@ -490,6 +382,5 @@ module.exports = { getCollections, getNestedCatalogs, crawlCatalogRecursive, - deriveCategories, - checkStacApiCompliance + deriveCategories }; \ No newline at end of file diff --git a/crawler/package-lock.json b/crawler/package-lock.json index 9b74099..447b591 100644 --- a/crawler/package-lock.json +++ b/crawler/package-lock.json @@ -9,6 +9,8 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "ajv": "^8.12.0", + "ajv-formats": "^3.0.1", "axios": "^1.13.2", "stac-js": "^0.1.9" } @@ -44,6 +46,39 @@ "license": "(Unlicense OR Apache-2.0)", "optional": true }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -297,6 +332,28 @@ "node": ">=6" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/follow-redirects": { "version": "1.15.11", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", @@ -570,6 +627,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -660,6 +723,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/safe-regex-test": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", From c9852fe498c79de85356d9d718863d15837ae6f1 Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 26 Nov 2025 16:00:35 +0100 Subject: [PATCH 21/68] feat: implement helper functions for inserting STAC extensions, summaries, providers, and assets --- crawler/db.js | 109 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/crawler/db.js b/crawler/db.js index c6b813f..1668eca 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -115,7 +115,7 @@ async function insertOrUpdateCollection(collection) { const bbox = collection.extent.spatial.bbox[0]; if (bbox.length === 4) { // Create polygon from bbox [west, south, east, north] - spatialExtend = `SRID=4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; + spatialExtend = `EPSG:4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; } } @@ -213,7 +213,6 @@ async function insertOrUpdateCollection(collection) { } } -// TODO: Implement insertKeywords, insertStacExtensions, insertSummary, insertProviders, insertAssets functions /** @@ -252,10 +251,110 @@ async function insertStacExtensions(client, parentId, extensions, type) { [parentId] ); - for (const extension of extensions) { + for (const extension of extensions) { + if (!extension) continue; + + // Insert extension if not exists + const extResult = await client.query( + 'INSERT INTO stac_extensions (stac_extension) VALUES ($1) ON CONFLICT (stac_extension) DO UPDATE SET stac_extension = EXCLUDED.stac_extension RETURNING id', + [extension] + ); + const extId = extResult.rows[0].id; + + // Link extension to parent + await client.query( + `INSERT INTO ${type}_stac_extension (${type}_id, stac_extension_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, + [parentId, extId] + ); + } +} + + +/** + * Helper function to insert collection summaries + */ +async function insertSummary(client, collectionId, name, value) { + let kind = 'unknown'; + let rangeMin = null; + let rangeMax = null; + let setValue = null; + let jsonSchema = null; + + if (Array.isArray(value)) { + if (value.length === 2 && typeof value[0] === 'number' && typeof value[1] === 'number') { + kind = 'range'; + rangeMin = value[0]; + rangeMax = value[1]; + } else { + kind = 'set'; + setValue = JSON.stringify(value); + } + } else if (typeof value === 'object') { + kind = 'schema'; + jsonSchema = JSON.stringify(value); + } else { + kind = 'value'; + setValue = String(value); + } + + await client.query( + 'INSERT INTO collection_summaries (collection_id, name, kind, range_min, range_max, set_value, json_schema) VALUES ($1, $2, $3, $4, $5, $6, $7)', + [collectionId, name, kind, rangeMin, rangeMax, setValue, jsonSchema] + ); +} + +/** + * Helper function to insert providers + */ +async function insertProviders(client, collectionId, providers) { + await client.query('DELETE FROM collection_providers WHERE collection_id = $1', [collectionId]); + + for (const provider of providers) { + if (!provider.name) continue; + + // Insert provider if not exists + const providerResult = await client.query( + 'INSERT INTO providers (provider) VALUES ($1) ON CONFLICT (provider) DO UPDATE SET provider = EXCLUDED.provider RETURNING id', + [provider.name] + ); + const providerId = providerResult.rows[0].id; + + // Link provider to collection + const roles = provider.roles ? provider.roles.join(',') : null; + await client.query( + 'INSERT INTO collection_providers (collection_id, provider_id, collection_provider_roles) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING', + [collectionId, providerId, roles] + ); + } +} + +/** + * Helper function to insert assets + */ +async function insertAssets(client, collectionId, assets) { + await client.query('DELETE FROM collection_assets WHERE collection_id = $1', [collectionId]); + + for (const [assetName, assetData] of Object.entries(assets)) { + if (!assetData) continue; + + // Insert asset + const assetResult = await client.query( + 'INSERT INTO assets (name, href, type, roles, metadata) VALUES ($1, $2, $3, $4, $5) RETURNING id', + [ + assetName, + assetData.href || null, + assetData.type || null, + assetData.roles || null, + JSON.stringify(assetData) + ] + ); + const assetId = assetResult.rows[0].id; + + // Link asset to collection + const roles = assetData.roles ? assetData.roles.join(',') : null; await client.query( - `INSERT INTO ${type}_stac_extensions (${type}_id, extension) VALUES ($1, $2)`, - [parentId, extension] + 'INSERT INTO collection_assets (collection_id, asset_id, collection_asset_roles) VALUES ($1, $2, $3)', + [collectionId, assetId, roles] ); } } From 239b5aa0a26ab365568f6fe26b57ec8f17c6d13b Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 26 Nov 2025 16:09:46 +0100 Subject: [PATCH 22/68] feat: enhance crawler configuration with CLI and environment variable support --- crawler/.env.example | 17 ++++ crawler/README.md | 113 +++++++++++++++++++++++++- crawler/apis/api.js | 34 ++++++-- crawler/catalogs/catalog.js | 55 ++++++++++--- crawler/index.js | 97 +++++++++++++++------- crawler/package-lock.json | 85 +++----------------- crawler/package.json | 1 + crawler/utils/config.js | 156 ++++++++++++++++++++++++++++++++++++ 8 files changed, 434 insertions(+), 124 deletions(-) create mode 100644 crawler/.env.example create mode 100644 crawler/utils/config.js diff --git a/crawler/.env.example b/crawler/.env.example new file mode 100644 index 0000000..013b893 --- /dev/null +++ b/crawler/.env.example @@ -0,0 +1,17 @@ +# STAC Crawler Configuration +# Copy this file to .env and adjust values as needed + +# Crawl mode: 'catalogs', 'apis', or 'both' +CRAWL_MODE=both + +# Maximum number of catalogs to crawl +MAX_CATALOGS=10 + +# Maximum number of APIs to crawl +MAX_APIS=5 + +# Timeout for each crawl operation in milliseconds (30 seconds = 30000ms) +TIMEOUT_MS=30000 + +# Maximum recursion depth for nested catalogs +MAX_DEPTH=3 diff --git a/crawler/README.md b/crawler/README.md index 1293303..86026dd 100644 --- a/crawler/README.md +++ b/crawler/README.md @@ -1,12 +1,76 @@ # STAC Crawler -A Node.js crawler for STAC Index API that fetches and processes catalog and collection data. +A Node.js crawler for STAC Index API that fetches and processes catalog and collection data with configurable options. + +## Configuration + +The crawler can be configured using environment variables, CLI arguments, or a combination of both. CLI arguments take precedence over environment variables. + +### Configuration Options + +| Option | CLI Flag | Environment Variable | Default | Description | +|--------|----------|---------------------|---------|-------------| +| Mode | `-m, --mode` | `CRAWL_MODE` | `both` | Crawl mode: `catalogs`, `apis`, or `both` | +| Max Catalogs | `-c, --max-catalogs` | `MAX_CATALOGS` | `10` | Maximum number of catalogs to process | +| Max APIs | `-a, --max-apis` | `MAX_APIS` | `5` | Maximum number of APIs to process | +| Timeout | `-t, --timeout` | `TIMEOUT_MS` | `30000` | Timeout per operation in milliseconds | +| Max Depth | `-d, --max-depth` | `MAX_DEPTH` | `3` | Maximum recursion depth for nested catalogs | + +### Using Environment Variables + +1. Copy the example environment file: +```bash +cp .env.example .env +``` + +2. Edit `.env` to customize settings: +```bash +CRAWL_MODE=both +MAX_CATALOGS=20 +MAX_APIS=10 +TIMEOUT_MS=60000 +MAX_DEPTH=5 +``` + +3. Run the crawler: +```bash +npm start +``` + +### Using CLI Arguments + +Run the crawler with command-line arguments to override defaults or environment variables: + +```bash +# Crawl only catalogs with custom limits +node index.js --mode catalogs --max-catalogs 20 + +# Crawl only APIs with extended timeout +node index.js -m apis -a 10 -t 60000 + +# Crawl both with all custom settings +node index.js -m both -c 50 -a 20 -t 45000 -d 5 +``` + +### Show Help + +Display all available options: + +```bash +node index.js --help +``` ## Running Locally ```bash +# Install dependencies npm install + +# Run with default configuration npm start + +# Run with custom configuration +node index.js --mode catalogs --max-catalogs 15 ``` ## Docker @@ -15,10 +79,16 @@ npm start ```bash # Build the image -docker build -t stac-crawler +docker build -t stac-crawler . -# Run the container +# Run with default configuration docker run --rm stac-crawler + +# Run with environment variables +docker run --rm -e CRAWL_MODE=apis -e MAX_APIS=10 stac-crawler + +# Run with CLI arguments +docker run --rm stac-crawler --mode catalogs --max-catalogs 20 ``` Or use npm scripts: @@ -30,6 +100,8 @@ npm run docker:run ### Using Docker Compose +Create a `.env` file or modify `docker-compose.yml` to set environment variables: + ```bash # Start the crawler docker-compose up -d @@ -47,3 +119,38 @@ Or use npm scripts: npm run docker:compose:up npm run docker:compose:down ``` + +## Examples + +### Example 1: Quick API Test +Crawl only the first 3 APIs with a short timeout: +```bash +node index.js -m apis -a 3 -t 15000 +``` + +### Example 2: Deep Catalog Exploration +Crawl 100 catalogs with maximum depth and extended timeout: +```bash +node index.js -m catalogs -c 100 -d 10 -t 120000 +``` + +### Example 3: Balanced Crawl +Crawl both catalogs and APIs with moderate settings: +```bash +node index.js -m both -c 25 -a 15 -t 45000 -d 4 +``` + +### Example 4: Production Environment +Set up `.env` for production: +```bash +CRAWL_MODE=both +MAX_CATALOGS=1000 +MAX_APIS=500 +TIMEOUT_MS=60000 +MAX_DEPTH=5 +``` + +Then run: +```bash +npm start +``` diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 5eda959..c414889 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -5,15 +5,17 @@ import axios from 'axios'; import create from 'stac-js'; +import { withTimeout } from '../utils/config.js'; /** * Crawls STAC APIs to retrieve collection information without fetching items. * * @param {string[]} urls - Array of API URLs to crawl * @param {boolean} isApi - Boolean flag indicating if the URLs are APIs + * @param {Object} config - Configuration object with timeout settings * @returns {Promise} Array of STAC Collection objects ordered by URL */ -async function crawlApis(urls, isApi) { +async function crawlApis(urls, isApi, config = {}) { if (!isApi || !Array.isArray(urls) || urls.length === 0) { return []; } @@ -26,7 +28,7 @@ async function crawlApis(urls, isApi) { for (const [index, url] of urls.entries()) { console.log(`Processing API ${index + 1}/${urls.length}: ${url}`); try { - await crawlSingleApi(url, allCollections); + await crawlSingleApi(url, allCollections, new Set(), config); } catch (error) { console.error(`Error crawling API ${url}:`, error.message); } @@ -50,14 +52,22 @@ async function crawlApis(urls, isApi) { * @param {string} url - URL to crawl * @param {Map} collectionMap - Map to store found collections * @param {Set} visited - Set of visited URLs to prevent loops + * @param {Object} config - Configuration object with timeout settings */ -async function crawlSingleApi(url, collectionMap, visited = new Set()) { +async function crawlSingleApi(url, collectionMap, visited = new Set(), config = {}) { if (!url || visited.has(url)) return; visited.add(url); + const timeoutMs = config.timeout || 30000; + const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(10000, timeoutMs); + try { console.log(` Fetching: ${url}`); - const response = await axios.get(url); + const response = await withTimeout( + axios.get(url, { timeout: axiosTimeout }), + timeoutMs, + `Crawling API ${url}` + ); const stacObj = create(response.data); // If it's a Collection, add it @@ -77,7 +87,7 @@ async function crawlSingleApi(url, collectionMap, visited = new Set()) { const collectionsLink = stacObj.getApiCollectionsLink(); if (collectionsLink) { console.log(` Found /collections endpoint link in ${url}`); - await fetchCollectionsFromLink(collectionsLink, collectionMap, visited); + await fetchCollectionsFromLink(collectionsLink, collectionMap, visited, config); } else { // Fallback: check standard STAC API structure if not explicitly found // Many STAC APIs have a /collections endpoint relative to root @@ -90,7 +100,7 @@ async function crawlSingleApi(url, collectionMap, visited = new Set()) { for (const link of childLinks) { const childUrl = link.getAbsoluteUrl(); if (childUrl) { - await crawlSingleApi(childUrl, collectionMap, visited); + await crawlSingleApi(childUrl, collectionMap, visited, config); } } } @@ -108,15 +118,23 @@ async function crawlSingleApi(url, collectionMap, visited = new Set()) { * @param {Object} link - stac-js Link object * @param {Map} collectionMap - Map to store collections * @param {Set} visited - Visited set + * @param {Object} config - Configuration object with timeout settings */ -async function fetchCollectionsFromLink(link, collectionMap, visited) { +async function fetchCollectionsFromLink(link, collectionMap, visited, config = {}) { const url = link.getAbsoluteUrl(); if (!url || visited.has(url)) return; visited.add(url); + const timeoutMs = config.timeout || 30000; + const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(10000, timeoutMs); + try { console.log(` Fetching collections from: ${url}`); - const response = await axios.get(url); + const response = await withTimeout( + axios.get(url, { timeout: axiosTimeout }), + timeoutMs, + `Fetching collections from ${url}` + ); // create() handles CollectionCollection (API Collections response) const stacObj = create(response.data); diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 1a2df1b..4527acb 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -5,6 +5,7 @@ import axios from 'axios'; import create from 'stac-js'; +import { withTimeout } from '../utils/config.js'; /** * Splits an array of catalogs into individual elements where each element is an array of its properties @@ -96,10 +97,14 @@ function splitCatalogs(catalogs) { * Fetches and processes collections from a catalog using stac-js * @async * @param {Object} catalog - Catalog object with url property + * @param {Object} config - Configuration object with timeout settings * @returns {Promise} Array of collections formatted as arrays */ -async function getCollections(catalog) { +async function getCollections(catalog, config = {}) { try { + const timeoutMs = config.timeout || 30000; + const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(5000, timeoutMs); + // Try common STAC collection endpoints const collectionUrls = [ `${catalog.url}/collections`, @@ -109,7 +114,11 @@ async function getCollections(catalog) { for (const url of collectionUrls) { try { - const response = await axios.get(url, { timeout: 5000 }); + const response = await withTimeout( + axios.get(url, { timeout: axiosTimeout }), + timeoutMs, + `Fetching collections from ${url}` + ); // Parse response with stac-js for validation and enhanced metadata extraction let stacObj; @@ -213,10 +222,14 @@ async function getCollections(catalog) { * @async * @param {Object} catalog - Catalog object with url property * @param {Object} [stacCatalog] - Optional stac-js catalog object for link extraction + * @param {Object} config - Configuration object with timeout settings * @returns {Promise} Array of nested catalogs */ -async function getNestedCatalogs(catalog, stacCatalog = null) { +async function getNestedCatalogs(catalog, stacCatalog = null, config = {}) { try { + const timeoutMs = config.timeout || 30000; + const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(5000, timeoutMs); + // If we have a stac-js catalog object, use its link navigation methods if (stacCatalog && typeof stacCatalog.getChildLinks === 'function') { const childLinks = stacCatalog.getChildLinks(); @@ -232,7 +245,11 @@ async function getNestedCatalogs(catalog, stacCatalog = null) { ? link.getAbsoluteUrl() : link.href; - const response = await axios.get(childUrl, { timeout: 5000 }); + const response = await withTimeout( + axios.get(childUrl, { timeout: axiosTimeout }), + timeoutMs, + `Fetching child catalog from ${childUrl}` + ); nestedCatalogs.push(response.data); } catch (e) { console.warn(` Failed to fetch child catalog: ${e.message}`); @@ -252,7 +269,11 @@ async function getNestedCatalogs(catalog, stacCatalog = null) { for (const url of catalogUrls) { try { - const response = await axios.get(url, { timeout: 5000 }); + const response = await withTimeout( + axios.get(url, { timeout: axiosTimeout }), + timeoutMs, + `Fetching nested catalogs from ${url}` + ); const catalogsData = Array.isArray(response.data) ? response.data : response.data.catalogs || []; @@ -279,22 +300,36 @@ async function getNestedCatalogs(catalog, stacCatalog = null) { * @async * @param {Object} catalog - Catalog object to crawl * @param {number} depth - Current depth level (for indentation) + * @param {Object} config - Configuration object with timeout and depth settings * @returns {Promise} Stats object with collections count and STAC compliance */ -async function crawlCatalogRecursive(catalog, depth = 0) { +async function crawlCatalogRecursive(catalog, depth = 0, config = {}) { const indent = ' '.repeat(depth); let stats = { collections: 0, stacCompliant: false }; + // Check max depth + const maxDepth = config.maxDepth || 3; + if (depth >= maxDepth) { + console.log(`${indent} Max depth ${maxDepth} reached, stopping recursion`); + return stats; + } + try { + const timeoutMs = config.timeout || 30000; + const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(5000, timeoutMs); let stacCatalog = null; // Try to fetch and parse the catalog with stac-js if (catalog.url) { try { - const response = await axios.get(catalog.url, { timeout: 5000 }); + const response = await withTimeout( + axios.get(catalog.url, { timeout: axiosTimeout }), + timeoutMs, + `Fetching catalog ${catalog.id}` + ); try { stacCatalog = create(response.data, catalog.url); @@ -318,11 +353,11 @@ async function crawlCatalogRecursive(catalog, depth = 0) { } // First, try to get collections from this catalog - const collections = await getCollections(catalog); + const collections = await getCollections(catalog, config); stats.collections = collections.length; // Then, try to get nested catalogs using stac-js if available - const nestedCatalogs = await getNestedCatalogs(catalog, stacCatalog); + const nestedCatalogs = await getNestedCatalogs(catalog, stacCatalog, config); if (nestedCatalogs.length > 0) { console.log(`${indent} Nested catalogs found: ${nestedCatalogs.length}`); @@ -335,7 +370,7 @@ async function crawlCatalogRecursive(catalog, depth = 0) { url: nestedCatalog.links?.find(l => l.rel === 'self')?.href || nestedCatalog.url }; - const nestedStats = await crawlCatalogRecursive(nestedCatalogObj, depth + 1); + const nestedStats = await crawlCatalogRecursive(nestedCatalogObj, depth + 1, config); stats.collections += nestedStats.collections; } } diff --git a/crawler/index.js b/crawler/index.js index 7cffb6a..11d9bb3 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -6,6 +6,7 @@ import axios from 'axios'; import { splitCatalogs, crawlCatalogRecursive } from './catalogs/catalog.js'; import { crawlApis } from './apis/api.js'; +import { getConfig } from './utils/config.js'; /** * URL of the STAC Index API endpoint @@ -21,44 +22,78 @@ const targetUrl = 'https://www.stacindex.org/api/catalogs'; */ const crawler = async () => { try { + // Load configuration + const config = getConfig(); + + // Display configuration + console.log('\n=== STAC Crawler Configuration ==='); + console.log(`Mode: ${config.mode}`); + console.log(`Max Catalogs: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs}`); + console.log(`Max APIs: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis}`); + console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); + console.log(`Max Depth: ${config.maxDepth === Infinity ? 'unlimited' : config.maxDepth}`); + console.log('==================================\n'); + const response = await axios.get(targetUrl); const catalogs = splitCatalogs(response.data); - // Crawl collections and nested catalogs for each catalog - console.log('\n Crawling collections and nested catalogs...\n'); - let totalCollections = 0; - for (const catalogData of catalogs.slice(0, 10)) { // Limit to first 10 catalogs - const catalog = { - id: catalogData[1], - url: catalogData[2] - }; - const stats = await crawlCatalogRecursive(catalog); - totalCollections += stats.collections; - } - - console.log(`\n Total collections found across all catalogs: ${totalCollections}`); - - // Extract API URLs and crawl them - console.log('\n Crawling APIs...'); - const apiUrls = catalogs - .filter(cat => cat[10] === true) // isApi is at index 10 - .map(cat => cat[2]); // url is at index 2 + // Crawl catalogs if mode is 'catalogs' or 'both' + if (config.mode === 'catalogs' || config.mode === 'both') { + console.log('\n Crawling collections and nested catalogs...\n'); + let totalCollections = 0; - if (apiUrls.length > 0) { - console.log(`Found ${apiUrls.length} APIs. Starting crawl...`); - // Limit to first 5 APIs for demonstration/performance - const collections = await crawlApis(apiUrls, true); - console.log(`\nFetched ${collections.length} collections from APIs (sorted by URL).`); + const catalogsToProcess = config.maxCatalogs === Infinity ? catalogs : catalogs.slice(0, config.maxCatalogs); + console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs})\n`); - if (collections.length > 0) { - console.log('First 3 collections found:'); - collections.slice(0, 3).forEach(c => { - const selfLink = c.links?.find(l => l.rel === 'self')?.href; - console.log(` - ${c.id}: ${selfLink}`); - }); + for (const catalogData of catalogsToProcess) { + const catalog = { + id: catalogData[1], + url: catalogData[2] + }; + + try { + const stats = await crawlCatalogRecursive(catalog, 0, config); + totalCollections += stats.collections; + } catch (error) { + console.error(`Failed to crawl catalog ${catalog.id}: ${error.message}`); + } + } + + console.log(`\n Total collections found across all catalogs: ${totalCollections}`); + } else { + console.log('\n Skipping catalog crawling (mode: apis)\n'); + } + + // Crawl APIs if mode is 'apis' or 'both' + if (config.mode === 'apis' || config.mode === 'both') { + console.log('\n Crawling APIs...'); + const apiUrls = catalogs + .filter(cat => cat[10] === true) // isApi is at index 10 + .map(cat => cat[2]); // url is at index 2 + + if (apiUrls.length > 0) { + const apisToProcess = config.maxApis === Infinity ? apiUrls : apiUrls.slice(0, config.maxApis); + console.log(`Found ${apiUrls.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis})...`); + + try { + const collections = await crawlApis(apisToProcess, true, config); + console.log(`\nFetched ${collections.length} collections from APIs (sorted by URL).`); + + if (collections.length > 0) { + console.log('First 3 collections found:'); + collections.slice(0, 3).forEach(c => { + const selfLink = c.links?.find(l => l.rel === 'self')?.href; + console.log(` - ${c.id}: ${selfLink}`); + }); + } + } catch (error) { + console.error(`Failed to crawl APIs: ${error.message}`); + } + } else { + console.log('No APIs found to crawl.'); } } else { - console.log('No APIs found to crawl.'); + console.log('\n Skipping API crawling (mode: catalogs)\n'); } } catch (error) { diff --git a/crawler/package-lock.json b/crawler/package-lock.json index 447b591..3ab52ae 100644 --- a/crawler/package-lock.json +++ b/crawler/package-lock.json @@ -9,9 +9,8 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "ajv": "^8.12.0", - "ajv-formats": "^3.0.1", "axios": "^1.13.2", + "dotenv": "^17.2.3", "stac-js": "^0.1.9" } }, @@ -46,39 +45,6 @@ "license": "(Unlicense OR Apache-2.0)", "optional": true }, - "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -258,6 +224,18 @@ "node": ">=0.4.0" } }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -332,28 +310,6 @@ "node": ">=6" } }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "license": "MIT" - }, - "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, "node_modules/follow-redirects": { "version": "1.15.11", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", @@ -627,12 +583,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -723,15 +673,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/safe-regex-test": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", diff --git a/crawler/package.json b/crawler/package.json index 49e9982..81f72fa 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -16,6 +16,7 @@ "type": "module", "dependencies": { "axios": "^1.13.2", + "dotenv": "^17.2.3", "stac-js": "^0.1.9" } } diff --git a/crawler/utils/config.js b/crawler/utils/config.js new file mode 100644 index 0000000..2b006c1 --- /dev/null +++ b/crawler/utils/config.js @@ -0,0 +1,156 @@ +/** + * @fileoverview Configuration management for STAC crawler + * @module utils/config + */ + +import dotenv from 'dotenv'; + +// Load environment variables from .env file +dotenv.config(); + +/** + * Parse command line arguments + * @returns {Object} Parsed CLI arguments + */ +function parseCliArgs() { + const args = process.argv.slice(2); + const config = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--mode' || arg === '-m') { + config.mode = args[++i]; + } else if (arg === '--max-catalogs' || arg === '-c') { + config.maxCatalogs = parseInt(args[++i], 10); + } else if (arg === '--max-apis' || arg === '-a') { + config.maxApis = parseInt(args[++i], 10); + } else if (arg === '--timeout' || arg === '-t') { + config.timeout = parseInt(args[++i], 10); + } else if (arg === '--max-depth' || arg === '-d') { + config.maxDepth = parseInt(args[++i], 10); + } else if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } + } + + return config; +} + +/** + * Print help message + */ +function printHelp() { + console.log(` +STAC Crawler Configuration Options: + + -m, --mode Crawl mode: 'catalogs', 'apis', or 'both' (default: 'both') + -c, --max-catalogs Maximum number of catalogs to crawl (default: 10) + -a, --max-apis Maximum number of APIs to crawl (default: 5) + -t, --timeout Timeout for each crawl operation in ms (default: 30000) + -d, --max-depth Maximum recursion depth for nested catalogs (default: 3) + -h, --help Show this help message + +Environment Variables: + CRAWL_MODE Same as --mode + MAX_CATALOGS Same as --max-catalogs + MAX_APIS Same as --max-apis + TIMEOUT_MS Same as --timeout + MAX_DEPTH Same as --max-depth + +Examples: + node index.js --mode catalogs --max-catalogs 20 + node index.js -m apis -a 10 -t 60000 + CRAWL_MODE=both MAX_CATALOGS=50 node index.js + `); +} + +/** + * Get configuration from environment variables, CLI args, and defaults + * CLI args take precedence over env vars, which take precedence over defaults + * @returns {Object} Configuration object + */ +function getConfig() { + const cliArgs = parseCliArgs(); + + // Default configuration + const defaults = { + mode: 'both', // 'catalogs', 'apis', or 'both' + maxCatalogs: 10, // Maximum number of catalogs to crawl + maxApis: 5, // Maximum number of APIs to crawl + timeout: 30000, // Timeout in milliseconds (30 seconds) + maxDepth: 3 // Maximum recursion depth for nested catalogs + }; + + // Build configuration with precedence: CLI > ENV > Defaults + const config = { + mode: cliArgs.mode || process.env.CRAWL_MODE || defaults.mode, + maxCatalogs: cliArgs.maxCatalogs || parseInt(process.env.MAX_CATALOGS, 10) || defaults.maxCatalogs, + maxApis: cliArgs.maxApis || parseInt(process.env.MAX_APIS, 10) || defaults.maxApis, + timeout: cliArgs.timeout || parseInt(process.env.TIMEOUT_MS, 10) || defaults.timeout, + maxDepth: cliArgs.maxDepth || parseInt(process.env.MAX_DEPTH, 10) || defaults.maxDepth + }; + + // Validate mode + const validModes = ['catalogs', 'apis', 'both']; + if (!validModes.includes(config.mode)) { + console.error(`Invalid mode: ${config.mode}. Must be one of: ${validModes.join(', ')}`); + process.exit(1); + } + + // When mode is 'catalogs', remove limits for comprehensive crawling + if (config.mode === 'catalogs') { + config.maxCatalogs = Infinity; + config.maxDepth = Infinity; + config.timeout = Infinity; + } + + // Validate numeric values + if (config.maxCatalogs < 0 || config.maxApis < 0 || config.timeout < 0 || config.maxDepth < 0) { + console.error('Numeric configuration values must be non-negative'); + process.exit(1); + } + + return config; +} + +/** + * Create a timeout promise that rejects after the specified time + * @param {number} ms - Timeout in milliseconds + * @param {string} operation - Description of the operation for error message + * @returns {Promise} Promise that rejects after timeout + */ +function createTimeout(ms, operation = 'Operation') { + return new Promise((_, reject) => { + setTimeout(() => { + reject(new Error(`${operation} timed out after ${ms}ms`)); + }, ms); + }); +} + +/** + * Wrap a promise with a timeout + * @param {Promise} promise - Promise to wrap + * @param {number} ms - Timeout in milliseconds (Infinity for no timeout) + * @param {string} operation - Description of the operation + * @returns {Promise} Promise that races against timeout + */ +async function withTimeout(promise, ms, operation = 'Operation') { + // If timeout is Infinity, just return the promise without racing + if (ms === Infinity) { + return promise; + } + return Promise.race([ + promise, + createTimeout(ms, operation) + ]); +} + +export { + getConfig, + parseCliArgs, + printHelp, + withTimeout, + createTimeout +}; From 4b818c7f66aa886f505e41d73e1775b54278f4e2 Mon Sep 17 00:00:00 2001 From: Lenn Date: Thu, 27 Nov 2025 17:25:06 +0100 Subject: [PATCH 23/68] feat: functions to save crawled catalogs, collections and apis in the database --- crawler/db.js | 8 ++++-- crawler/index.js | 72 ++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/crawler/db.js b/crawler/db.js index 1668eca..8587c11 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -2,8 +2,10 @@ * DB helper for crawler using `pg` Pool * Exposes: initDb(), insertOrUpdateCatalog(), close() */ -const { Pool } = require('pg'); -require('dotenv').config(); +import pkg from 'pg'; +const { Pool } = pkg; +import dotenv from 'dotenv'; +dotenv.config(); const pool = new Pool({ host: process.env.PGHOST, @@ -366,7 +368,7 @@ async function close() { await pool.end(); } -module.exports = { +export default { initDb, insertOrUpdateCatalog, insertOrUpdateCollection, diff --git a/crawler/index.js b/crawler/index.js index 1a96a56..d532154 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -7,6 +7,7 @@ import axios from 'axios'; import { splitCatalogs, crawlCatalogRecursive } from './catalogs/catalog.js'; import { crawlApis } from './apis/api.js'; import { getConfig } from './utils/config.js'; +import db from './db.js'; /** * URL of the STAC Index API endpoint @@ -34,9 +35,31 @@ const crawler = async () => { console.log(`Max Depth: ${config.maxDepth === Infinity ? 'unlimited' : config.maxDepth}`); console.log('==================================\n'); + // Initialize database connection + console.log('Initializing database connection...'); + await db.initDb(); + const response = await axios.get(targetUrl); const catalogs = splitCatalogs(response.data); + // Save catalogs from STAC Index to database + console.log('\n=== Saving catalogs to database ==='); + let catalogsSaved = 0; + let catalogsFailed = 0; + + for (const catalog of response.data) { + try { + const catalogId = await db.insertOrUpdateCatalog(catalog); + console.log(`✓ Saved catalog: ${catalog.title || catalog.id} (DB ID: ${catalogId})`); + catalogsSaved++; + } catch (err) { + console.error(`✗ Failed catalog: ${catalog.title || catalog.id} - ${err.message}`); + catalogsFailed++; + } + } + + console.log(`\nCatalogs: ${catalogsSaved} saved, ${catalogsFailed} failed\n`); + // Crawl catalogs if mode is 'catalogs' or 'both' if (config.mode === 'catalogs' || config.mode === 'both') { console.log('\n Crawling collections and nested catalogs...\n'); @@ -54,6 +77,18 @@ const crawler = async () => { try { const stats = await crawlCatalogRecursive(catalog, 0, config); totalCollections += stats.collections; + + // Save collections to database if returned + if (stats.collectionsData && Array.isArray(stats.collectionsData)) { + for (const collection of stats.collectionsData) { + try { + const collectionId = await db.insertOrUpdateCollection(collection); + console.log(` ✓ Saved collection: ${collection.title || collection.id} (DB ID: ${collectionId})`); + } catch (err) { + console.error(` ✗ Failed collection: ${collection.title || collection.id} - ${err.message}`); + } + } + } } catch (error) { console.error(`Failed to crawl catalog ${catalog.id}: ${error.message}`); } @@ -85,6 +120,24 @@ const crawler = async () => { const selfLink = c.links?.find(l => l.rel === 'self')?.href; console.log(` - ${c.id}: ${selfLink}`); }); + + // Save API collections to database + console.log('\n=== Saving API collections to database ==='); + let apiCollectionsSaved = 0; + let apiCollectionsFailed = 0; + + for (const collection of collections) { + try { + const collectionId = await db.insertOrUpdateCollection(collection); + console.log(` ✓ Saved: ${collection.title || collection.id} (DB ID: ${collectionId})`); + apiCollectionsSaved++; + } catch (err) { + console.error(` ✗ Failed: ${collection.title || collection.id} - ${err.message}`); + apiCollectionsFailed++; + } + } + + console.log(`\nAPI Collections: ${apiCollectionsSaved} saved, ${apiCollectionsFailed} failed`); } } catch (error) { console.error(`Failed to crawl APIs: ${error.message}`); @@ -98,22 +151,11 @@ const crawler = async () => { } catch (error) { console.error(`Error fetching ${targetUrl}: ${error.message}`); + } finally { + // Close database connection + await db.close(); + console.log('\nDatabase connection closed.'); } - - // Persist catalogs into DB - try { - await Promise.all(catalogs.map(async (catalog) => { - try { - await db.insertOrUpdateCatalog(catalog); - } catch (err) { - console.error('DB write error for catalog', catalog && (catalog.id || catalog.slug), err && err.message); - } - })); - console.log('Finished writing catalogs to DB'); - } catch (e) { - console.error('Unexpected error while saving catalogs to DB', e && e.message); - } - }; crawler(); \ No newline at end of file From 6c47638ba9d32e8e97f170e938ed497de2f18f2b Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 28 Nov 2025 14:19:32 +0100 Subject: [PATCH 24/68] #154, refactored the crawler to use crawlee Framework, improved performance and structure --- crawler/catalogs/catalog.js | 472 +--- crawler/index.js | 124 +- crawler/package-lock.json | 4553 ++++++++++++++++++++++++++++---- crawler/package.json | 1 + crawler/utils/config.js | 31 +- crawler/utils/endpoints.js | 36 + crawler/utils/handlers.js | 205 ++ crawler/utils/normalization.js | 130 + 8 files changed, 4662 insertions(+), 890 deletions(-) create mode 100644 crawler/utils/endpoints.js create mode 100644 crawler/utils/handlers.js create mode 100644 crawler/utils/normalization.js diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 4527acb..2ba561a 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -1,420 +1,102 @@ /** - * @fileoverview Catalog crawling functionality for STAC Index + * @fileoverview Catalog crawling functionality for STAC Index using Crawlee * @module catalogs/catalog */ -import axios from 'axios'; -import create from 'stac-js'; -import { withTimeout } from '../utils/config.js'; +import { HttpCrawler } from 'crawlee'; +import { handleCatalog, handleCollections } from '../utils/handlers.js'; /** - * Splits an array of catalogs into individual elements where each element is an array of its properties - * @param {Array} catalogs - Array of catalog objects from the STAC Index API - * @returns {Array} Array of arrays, where each inner array contains: - * [0] = index, [1] = id, [2] = url, [3] = slug, [4] = title, [5] = summary, - * [6] = access, [7] = created, [8] = updated, [9] = isPrivate, [10] = isApi, - * [11] = accessInfo, [12] = categories, ... (followed by any other dynamic properties) - * @throws {Error} Throws error if input is not an array + * Creates and runs a Crawlee HttpCrawler to crawl STAC catalogs + * @async + * @param {Array} initialCatalogs - Array of catalog objects to start crawling from + * @param {Object} config - Configuration object with timeout and depth settings + * @returns {Promise} Crawl results with collections and statistics */ -function splitCatalogs(catalogs) { - if (!Array.isArray(catalogs)) { - throw new Error('Expected an array'); - } - - // Define the order of standard properties - const standardPropertyOrder = [ - 'index', 'id', 'url', 'slug', 'title', 'summary', - 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' - ]; - - // Split each element individually and convert to array format - const elements = catalogs.map((catalog, index) => { - - // Create object with all properties - let elementObj = { - index: index, - ...catalog, - }; - - // Validate and migrate STAC catalog using stac-js if it has STAC properties - if (catalog.url && typeof catalog.url === 'string') { - try { - // For STAC Index catalogs, we don't validate against STAC spec - // as they are metadata about STAC catalogs, not STAC objects themselves - // This is just for potential future enhancement - } catch (error) { - console.warn(`STAC validation skipped for catalog ${catalog.id}: ${error.message}`); - } +async function crawlCatalogs(initialCatalogs, config = {}) { + const timeoutSecs = config.timeout && config.timeout !== Infinity + ? Math.ceil(config.timeout / 1000) + : 60; + + // Store results + const results = { + collections: [], + catalogs: [], + stats: { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + catalogsProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 } + }; - // Get all property names - const allProperties = Object.keys(elementObj); + const crawler = new HttpCrawler({ + requestHandlerTimeoutSecs: timeoutSecs, - // Separate standard and dynamic properties - const dynamicProps = allProperties.filter(prop => !standardPropertyOrder.includes(prop)); - - // Create array with standard properties first, then dynamic ones - const elementArray = []; - - // Add standard properties in defined order (always include all standard fields for consistent indices) - standardPropertyOrder.forEach(prop => { - elementArray.push(elementObj[prop]); - }); - - // Add dynamic properties - dynamicProps.forEach(prop => { - elementArray.push(elementObj[prop]); - }); - return elementArray; - }); - - // Output for demonstration - console.log(`Total: ${elements.length} elements found\n`); - - if (elements.length > 0) { - console.log('Example - First Element (as array):'); - console.log('Array indices and values:'); - const firstElement = elements[0]; - const allProps = [ - 'index', 'id', 'url', 'slug', 'title', 'summary', - 'access', 'created', 'updated', 'isPrivate', 'isApi', 'accessInfo' - ]; - firstElement.forEach((value, idx) => { - const propName = idx < allProps.length ? allProps[idx] : `dynamic_property_${idx}`; - console.log(` [${idx}] ${propName}: ${JSON.stringify(value)}`); - }); - console.log('\nAccess examples:'); - console.log(` elements[0][0] (index): ${elements[0][0]}`); - console.log(` elements[0][1] (id): ${elements[0][1]}`); - console.log(` elements[0][2] (url): ${elements[0][2]}`); - console.log(` elements[0][7] (created): ${elements[0][7]}`); - } - - return elements; -} - -/** - * Fetches and processes collections from a catalog using stac-js - * @async - * @param {Object} catalog - Catalog object with url property - * @param {Object} config - Configuration object with timeout settings - * @returns {Promise} Array of collections formatted as arrays - */ -async function getCollections(catalog, config = {}) { - try { - const timeoutMs = config.timeout || 30000; - const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(5000, timeoutMs); - - // Try common STAC collection endpoints - const collectionUrls = [ - `${catalog.url}/collections`, - `${catalog.url}/collections/`, - `${catalog.url}/api/v1/collections` - ]; - - for (const url of collectionUrls) { + async requestHandler({ request, json, crawler, log }) { + results.stats.totalRequests++; + const depth = request.userData?.depth || 0; + const indent = ' '.repeat(depth); + try { - const response = await withTimeout( - axios.get(url, { timeout: axiosTimeout }), - timeoutMs, - `Fetching collections from ${url}` - ); - - // Parse response with stac-js for validation and enhanced metadata extraction - let stacObj; - try { - stacObj = create(response.data, url); - } catch (parseError) { - // If stac-js cannot parse, skip this catalog (non-compliant STAC) - console.warn(` Skipping non-compliant STAC at ${url}: ${parseError.message}`); - continue; + // Route based on request label + if (request.label === 'CATALOG') { + await handleCatalog({ request, json, crawler, log, indent, results }); + } else if (request.label === 'COLLECTIONS') { + await handleCollections({ request, json, crawler, log, indent, results }); } - let collectionsData = []; - - // Check if this is a CollectionCollection (STAC API response) - if (stacObj && typeof stacObj.getAll === 'function') { - collectionsData = stacObj.getAll(); - } else if (Array.isArray(response.data)) { - // Handle array of collections - collectionsData = response.data.map(col => { - try { - return create(col, url); - } catch { - return null; - } - }).filter(Boolean); - } else if (response.data.collections) { - // Handle nested collections property - collectionsData = response.data.collections.map(col => { - try { - return create(col, url); - } catch { - return null; - } - }).filter(Boolean); - } - - if (collectionsData.length > 0) { - console.log(`\n Catalog: ${catalog.id}`); - console.log(` Found ${collectionsData.length} collections`); - - // Convert collections using stac-js methods for metadata extraction - const collections = collectionsData.map((colObj, index) => { - // Use stac-js methods for robust metadata extraction - const bbox = typeof colObj.getBoundingBox === 'function' - ? colObj.getBoundingBox() - : (colObj.extent?.spatial?.bbox?.[0] || 'N/A'); - - const temporal = typeof colObj.getTemporalExtent === 'function' - ? colObj.getTemporalExtent() - : (colObj.extent?.temporal?.interval?.[0] || 'N/A'); - - // Get self URL using stac-js link navigation - let selfUrl = 'N/A'; - if (typeof colObj.getAbsoluteUrl === 'function') { - selfUrl = colObj.getAbsoluteUrl(); - } else if (colObj.links) { - const selfLink = colObj.links.find(l => l.rel === 'self'); - selfUrl = selfLink?.href || 'N/A'; - } - - return [ - index, - colObj.id || 'Unknown', - selfUrl, - colObj.title || 'N/A', - colObj.description || colObj.summary || 'N/A', - bbox, - temporal, - colObj.license || 'N/A', - colObj.keywords || [] - ]; - }); - - // Display collection details - if (collections.length > 0) { - console.log(' Collection details:'); - collections.forEach((col, idx) => { - console.log(` [${idx}] ${col[1]} - ${col[3]}`); - }); - } - - return collections; - } - } catch (e) { - // Try next URL - continue; + results.stats.successfulRequests++; + } catch (error) { + log.error(`${indent}Error handling ${request.label} at ${request.url}: ${error.message}`); + throw error; // Re-throw to trigger failedRequestHandler } - } - - console.log(`\n No collections found for catalog: ${catalog.id}`); - return []; - - } catch (error) { - console.error(`Error fetching collections for ${catalog.id}: ${error.message}`); - return []; - } -} - -/** - * Fetches nested catalogs within a catalog using stac-js link navigation - * @async - * @param {Object} catalog - Catalog object with url property - * @param {Object} [stacCatalog] - Optional stac-js catalog object for link extraction - * @param {Object} config - Configuration object with timeout settings - * @returns {Promise} Array of nested catalogs - */ -async function getNestedCatalogs(catalog, stacCatalog = null, config = {}) { - try { - const timeoutMs = config.timeout || 30000; - const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(5000, timeoutMs); + }, - // If we have a stac-js catalog object, use its link navigation methods - if (stacCatalog && typeof stacCatalog.getChildLinks === 'function') { - const childLinks = stacCatalog.getChildLinks(); + async failedRequestHandler({ request, error, log }) { + results.stats.failedRequests++; + const depth = request.userData?.depth || 0; + const indent = ' '.repeat(depth); - if (childLinks.length > 0) { - console.log(` Found ${childLinks.length} child catalog links via stac-js`); - - // Fetch each child catalog - const nestedCatalogs = []; - for (const link of childLinks) { - try { - const childUrl = typeof link.getAbsoluteUrl === 'function' - ? link.getAbsoluteUrl() - : link.href; - - const response = await withTimeout( - axios.get(childUrl, { timeout: axiosTimeout }), - timeoutMs, - `Fetching child catalog from ${childUrl}` - ); - nestedCatalogs.push(response.data); - } catch (e) { - console.warn(` Failed to fetch child catalog: ${e.message}`); - } - } - - return nestedCatalogs; + // Log non-compliant STAC or failed requests + if (error.message.includes('STAC') || error.message.includes('validation')) { + log.info(`${indent}Skipping non-compliant STAC: ${request.url}`); + results.stats.nonCompliant++; + } else { + log.warning(`${indent}Failed request: ${request.url} - ${error.message}`); } } - - // Fallback: Try common STAC nested catalog endpoints - const catalogUrls = [ - `${catalog.url}/catalogs`, - `${catalog.url}/catalogs/`, - `${catalog.url}/api/v1/catalogs` - ]; + }); - for (const url of catalogUrls) { - try { - const response = await withTimeout( - axios.get(url, { timeout: axiosTimeout }), - timeoutMs, - `Fetching nested catalogs from ${url}` - ); - const catalogsData = Array.isArray(response.data) - ? response.data - : response.data.catalogs || []; - - if (catalogsData.length > 0) { - return catalogsData; - } - } catch (e) { - // Try next URL - continue; - } + // Seed the crawler with initial catalog requests + const initialRequests = initialCatalogs.map(catalog => ({ + url: catalog.url, + label: 'CATALOG', + userData: { + depth: 0, + catalogId: catalog.id, + catalogTitle: catalog.title } - - return []; - - } catch (error) { - console.error(`Error fetching nested catalogs for ${catalog.id}: ${error.message}`); - return []; - } -} + })); -/** - * Recursively crawls a catalog using stac-js for parsing and navigation - * @async - * @param {Object} catalog - Catalog object to crawl - * @param {number} depth - Current depth level (for indentation) - * @param {Object} config - Configuration object with timeout and depth settings - * @returns {Promise} Stats object with collections count and STAC compliance - */ -async function crawlCatalogRecursive(catalog, depth = 0, config = {}) { - const indent = ' '.repeat(depth); - let stats = { - collections: 0, - stacCompliant: false - }; + await crawler.addRequests(initialRequests); - // Check max depth - const maxDepth = config.maxDepth || 3; - if (depth >= maxDepth) { - console.log(`${indent} Max depth ${maxDepth} reached, stopping recursion`); - return stats; - } + console.log(`\nStarting Crawlee crawler with ${initialRequests.length} initial catalogs...\n`); + await crawler.run(); - try { - const timeoutMs = config.timeout || 30000; - const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(5000, timeoutMs); - let stacCatalog = null; - - // Try to fetch and parse the catalog with stac-js - if (catalog.url) { - try { - const response = await withTimeout( - axios.get(catalog.url, { timeout: axiosTimeout }), - timeoutMs, - `Fetching catalog ${catalog.id}` - ); - - try { - stacCatalog = create(response.data, catalog.url); - stats.stacCompliant = true; - - // Log STAC object type - if (typeof stacCatalog.isCatalog === 'function' && stacCatalog.isCatalog()) { - console.log(`${indent} STAC Catalog validated: ${catalog.id}`); - } else if (typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection()) { - console.log(`${indent} STAC Collection validated: ${catalog.id}`); - } - - } catch (parseError) { - console.warn(`${indent} Non-compliant STAC catalog ${catalog.id}, skipping: ${parseError.message}`); - return stats; // Skip non-compliant catalogs as per requirement - } - } catch (fetchError) { - console.warn(`${indent} Failed to fetch catalog ${catalog.id}: ${fetchError.message}`); - return stats; - } - } - - // First, try to get collections from this catalog - const collections = await getCollections(catalog, config); - stats.collections = collections.length; - - // Then, try to get nested catalogs using stac-js if available - const nestedCatalogs = await getNestedCatalogs(catalog, stacCatalog, config); - - if (nestedCatalogs.length > 0) { - console.log(`${indent} Nested catalogs found: ${nestedCatalogs.length}`); - - // Recursively crawl each nested catalog - for (const nestedCatalog of nestedCatalogs.slice(0, 5)) { // Limit to first 5 nested - // Convert nested catalog to expected format - const nestedCatalogObj = { - id: nestedCatalog.id || 'unknown', - url: nestedCatalog.links?.find(l => l.rel === 'self')?.href || nestedCatalog.url - }; - - const nestedStats = await crawlCatalogRecursive(nestedCatalogObj, depth + 1, config); - stats.collections += nestedStats.collections; - } - } - - } catch (error) { - console.error(`${indent}Error in recursive crawl for ${catalog.id}: ${error.message}`); - } - - return stats; -} - -/** - * Derives categories from a catalog object by checking various possible fields - * @param {Object} catalog - Catalog object to extract categories from - * @returns {Array} Array of category strings, empty array if none found - */ -function deriveCategories(catalog) { - if (!catalog || typeof catalog !== 'object') { - return []; - } - - if (Array.isArray(catalog.categories)) { - return catalog.categories.filter(Boolean).map(String); - } - - if (Array.isArray(catalog.keywords)) { - return catalog.keywords.filter(Boolean).map(String); - } - - if (Array.isArray(catalog.tags)) { - return catalog.tags.filter(Boolean).map(String); - } - - if (typeof catalog.access === 'string' && catalog.access.trim().length) { - return [catalog.access.trim()]; - } - - return []; + console.log('\nCrawl Statistics:'); + console.log(` Total Requests: ${results.stats.totalRequests}`); + console.log(` Successful: ${results.stats.successfulRequests}`); + console.log(` Failed: ${results.stats.failedRequests}`); + console.log(` STAC Compliant: ${results.stats.stacCompliant}`); + console.log(` Non-Compliant: ${results.stats.nonCompliant}`); + console.log(` Catalogs Processed: ${results.stats.catalogsProcessed}`); + console.log(` Collections Found: ${results.stats.collectionsFound}\n`); + + return results; } -export { - splitCatalogs, - getCollections, - getNestedCatalogs, - crawlCatalogRecursive, - deriveCategories -}; \ No newline at end of file +export { crawlCatalogs }; \ No newline at end of file diff --git a/crawler/index.js b/crawler/index.js index 1a96a56..e3f7c0a 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -4,9 +4,15 @@ */ import axios from 'axios'; -import { splitCatalogs, crawlCatalogRecursive } from './catalogs/catalog.js'; +import { processCatalogs } from './utils/normalization.js'; +import { crawlCatalogs } from './catalogs/catalog.js'; import { crawlApis } from './apis/api.js'; import { getConfig } from './utils/config.js'; +import { createRequire } from 'module'; + +// Import CommonJS db module +const require = createRequire(import.meta.url); +const db = require('./db.js'); /** * URL of the STAC Index API endpoint @@ -22,6 +28,9 @@ const targetUrl = 'https://www.stacindex.org/api/catalogs'; */ const crawler = async () => { try { + // Initialize database connection + await db.initDb(); + // Load configuration const config = getConfig(); @@ -31,45 +40,60 @@ const crawler = async () => { console.log(`Max Catalogs: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs}`); console.log(`Max APIs: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis}`); console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); - console.log(`Max Depth: ${config.maxDepth === Infinity ? 'unlimited' : config.maxDepth}`); + console.log(`Crawl Depth: unlimited`); console.log('==================================\n'); const response = await axios.get(targetUrl); - const catalogs = splitCatalogs(response.data); + const catalogs = processCatalogs(response.data); // Crawl catalogs if mode is 'catalogs' or 'both' if (config.mode === 'catalogs' || config.mode === 'both') { - console.log('\n Crawling collections and nested catalogs...\n'); - let totalCollections = 0; + console.log('\nCrawling collections and nested catalogs with Crawlee...\n'); + + const catalogsToProcess = config.maxCatalogs === Infinity + ? catalogs + : catalogs.slice(0, config.maxCatalogs); - const catalogsToProcess = config.maxCatalogs === Infinity ? catalogs : catalogs.slice(0, config.maxCatalogs); console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs})\n`); - for (const catalogData of catalogsToProcess) { - const catalog = { - id: catalogData[1], - url: catalogData[2] - }; + try { + const results = await crawlCatalogs(catalogsToProcess, config); + console.log(`\nTotal collections found across all catalogs: ${results.stats.collectionsFound}`); - try { - const stats = await crawlCatalogRecursive(catalog, 0, config); - totalCollections += stats.collections; - } catch (error) { - console.error(`Failed to crawl catalog ${catalog.id}: ${error.message}`); + // Persist collections to database + if (results.collections.length > 0) { + console.log(`\nSaving ${results.collections.length} collections to database...`); + let savedCount = 0; + let errorCount = 0; + + for (const collection of results.collections) { + try { + await db.insertOrUpdateCollection(collection); + savedCount++; + } catch (err) { + errorCount++; + console.error(`DB write error for collection ${collection.id}:`, err.message); + } + } + + console.log(`Successfully saved ${savedCount} collections to database`); + if (errorCount > 0) { + console.log(`Failed to save ${errorCount} collections`); + } } + } catch (error) { + console.error(`Failed to crawl catalogs: ${error.message}`); } - - console.log(`\n Total collections found across all catalogs: ${totalCollections}`); } else { - console.log('\n Skipping catalog crawling (mode: apis)\n'); + console.log('\nSkipping catalog crawling (mode: apis)\n'); } // Crawl APIs if mode is 'apis' or 'both' if (config.mode === 'apis' || config.mode === 'both') { - console.log('\n Crawling APIs...'); + console.log('\nCrawling APIs...'); const apiUrls = catalogs - .filter(cat => cat[10] === true) // isApi is at index 10 - .map(cat => cat[2]); // url is at index 2 + .filter(cat => cat.isApi === true) + .map(cat => cat.url); if (apiUrls.length > 0) { const apisToProcess = config.maxApis === Infinity ? apiUrls : apiUrls.slice(0, config.maxApis); @@ -85,6 +109,26 @@ const crawler = async () => { const selfLink = c.links?.find(l => l.rel === 'self')?.href; console.log(` - ${c.id}: ${selfLink}`); }); + + // Persist API collections to database + console.log(`\nSaving ${collections.length} API collections to database...`); + let savedCount = 0; + let errorCount = 0; + + for (const collection of collections) { + try { + await db.insertOrUpdateCollection(collection); + savedCount++; + } catch (err) { + errorCount++; + console.error(`DB write error for collection ${collection.id}:`, err.message); + } + } + + console.log(`Successfully saved ${savedCount} API collections to database`); + if (errorCount > 0) { + console.log(`Failed to save ${errorCount} collections`); + } } } catch (error) { console.error(`Failed to crawl APIs: ${error.message}`); @@ -93,25 +137,35 @@ const crawler = async () => { console.log('No APIs found to crawl.'); } } else { - console.log('\n Skipping API crawling (mode: catalogs)\n'); + console.log('\nSkipping API crawling (mode: catalogs)\n'); } - } catch (error) { - console.error(`Error fetching ${targetUrl}: ${error.message}`); - } - - // Persist catalogs into DB - try { - await Promise.all(catalogs.map(async (catalog) => { + // Persist catalogs metadata to database + console.log(`\nSaving ${catalogs.length} catalog metadata entries to database...`); + let savedCatalogCount = 0; + let errorCatalogCount = 0; + + for (const catalog of catalogs) { try { await db.insertOrUpdateCatalog(catalog); + savedCatalogCount++; } catch (err) { - console.error('DB write error for catalog', catalog && (catalog.id || catalog.slug), err && err.message); + errorCatalogCount++; + console.error(`DB write error for catalog ${catalog.id || catalog.slug}:`, err.message); } - })); - console.log('Finished writing catalogs to DB'); - } catch (e) { - console.error('Unexpected error while saving catalogs to DB', e && e.message); + } + + console.log(`Successfully saved ${savedCatalogCount} catalog metadata entries`); + if (errorCatalogCount > 0) { + console.log(`Failed to save ${errorCatalogCount} catalog entries`); + } + + } catch (error) { + console.error(`Error fetching ${targetUrl}: ${error.message}`); + } finally { + // Close database connection + await db.close(); + console.log('\nDatabase connection closed.'); } }; diff --git a/crawler/package-lock.json b/crawler/package-lock.json index 40ee2dd..a91cada 100644 --- a/crawler/package-lock.json +++ b/crawler/package-lock.json @@ -9,10 +9,86 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@databases/pg": "^5.5.0", "axios": "^1.13.2", + "crawlee": "^3.15.3", "dotenv": "^17.2.3", "stac-js": "^0.1.9" - "@databases/pg": "^5.5.0", + } + }, + "node_modules/@apify/consts": { + "version": "2.48.0", + "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.48.0.tgz", + "integrity": "sha512-a0HeYDxAbbkRxc9z2N6beMFAmAJSgBw8WuKUwV+KmCuPyGUVLp54fYzjQ63p9Gv5IVFC88/HMXpAzI29ARgO5w==", + "license": "Apache-2.0" + }, + "node_modules/@apify/datastructures": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@apify/datastructures/-/datastructures-2.0.3.tgz", + "integrity": "sha512-E6yQyc/XZDqJopbaGmhzZXMJqwGf96ELtDANZa0t68jcOAJZS+pF7YUfQOLszXq6JQAdnRvTH2caotL6urX7HA==", + "license": "Apache-2.0" + }, + "node_modules/@apify/log": { + "version": "2.5.28", + "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.28.tgz", + "integrity": "sha512-jU8qIvU+Crek8glBjFl3INjJQWWDR9n2z9Dr0WvUI8KJi0LG9fMdTvV+Aprf9z1b37CbHXgiZkA1iPlNYxKOEQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.48.0", + "ansi-colors": "^4.1.1" + } + }, + "node_modules/@apify/ps-tree": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@apify/ps-tree/-/ps-tree-1.2.0.tgz", + "integrity": "sha512-VHIswI7rD/R4bToeIDuJ9WJXt+qr5SdhfoZ9RzdjmCs9mgy7l0P4RugQEUCcU+WB4sfImbd4CKwzXcn0uYx1yw==", + "license": "MIT", + "dependencies": { + "event-stream": "3.3.4" + }, + "bin": { + "ps-tree": "bin/ps-tree.js" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/@apify/pseudo_url": { + "version": "2.0.69", + "resolved": "https://registry.npmjs.org/@apify/pseudo_url/-/pseudo_url-2.0.69.tgz", + "integrity": "sha512-p/jZpaITBbFX8uVqz5MeY0uvOsMSV0SKbxrkTd8ZkmF8L7+LU93aOb/G/AnAEozgzjPV8Tf1ihkHnP2aY09y6Q==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.5.28" + } + }, + "node_modules/@apify/timeout": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@apify/timeout/-/timeout-0.3.2.tgz", + "integrity": "sha512-JnOLIOpqfm366q7opKrA6HrL0iYRpYYDn8Mi77sMR2GZ1fPbwMWCVzN23LJWfJV7izetZbCMrqRUXsR1etZ7dA==", + "license": "Apache-2.0" + }, + "node_modules/@apify/utilities": { + "version": "2.23.4", + "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.23.4.tgz", + "integrity": "sha512-1tLXOJBJR1SUSp/iEj6kcvV+9B5dn1mvIWDtRYwevJXXURyJdPwzJApi0F0DZz/Vk2HeCC381gnSqASzXN8MLA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.48.0", + "@apify/log": "^2.5.28" + } + }, + "node_modules/@asamuzakjp/css-color": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", + "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.3", + "@csstools/css-color-parser": "^3.0.9", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" } }, "node_modules/@babel/code-frame": { @@ -38,6 +114,556 @@ "node": ">=6.9.0" } }, + "node_modules/@borewit/text-codec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.1.1.tgz", + "integrity": "sha512-5L/uBxmjaCIX5h8Z+uu+kA9BQLkc/Wl06UGR5ajNRxu+/XjonB5i8JpgFMrPj3LXTCPA0pv8yxUvbUi+QthGGA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@crawlee/basic": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.15.3.tgz", + "integrity": "sha512-+j0rhP16Gx84eFFXnG2t0YxmwkIwz5cWFnJ6CFyj1F7ElQ5JmVkzxyIWoyKBWCmLpPlafySht1tKq7L/4EZ1AQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "csv-stringify": "^6.2.0", + "fs-extra": "^11.0.0", + "got-scraping": "^4.0.0", + "ow": "^0.28.1", + "tldts": "^7.0.0", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/browser": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.15.3.tgz", + "integrity": "sha512-PtRzsurFO/A+puXg9oFUcP5LmEYNXkGyyQ2RQUJdg9exN1kbRwaaSrx4IUVi55waC1Z1Vkr5Ycq6nkVGTE6OcQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@crawlee/basic": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "ow": "^0.28.1", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/@crawlee/browser-pool": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.15.3.tgz", + "integrity": "sha512-a+QPQyHhLOO2cVzqjA8c9nuu++omzS9PWXRq248z46F+CnmAyYbClhuKiuBwhaNSTDKv7mgD8u1HikpzSN1duA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.0", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "fingerprint-generator": "^2.1.68", + "fingerprint-injector": "^2.1.68", + "lodash.merge": "^4.6.2", + "nanoid": "^3.3.4", + "ow": "^0.28.1", + "p-limit": "^3.1.0", + "proxy-chain": "^2.0.1", + "quick-lru": "^5.1.1", + "tiny-typed-emitter": "^2.1.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/@crawlee/cheerio": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.15.3.tgz", + "integrity": "sha512-yYbaUkV7meXtHLN9AW/Loo6BfZonp8ma2GvTZAlWXmQbiq3nmZ/npVWvR4UHWVDj0VsRe/IWsl8jgUzJFxD2/Q==", + "license": "Apache-2.0", + "dependencies": { + "@crawlee/http": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "cheerio": "1.0.0-rc.12", + "htmlparser2": "^9.0.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/cli": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.15.3.tgz", + "integrity": "sha512-cWo0NeF96WGO9sl5Q6BDFthvtqLky0CaCK2NtUvPJ7/EoXaiKm5D8o//lo1coMQmy72JEEgCGnSc/Xo8SDNUhA==", + "license": "Apache-2.0", + "dependencies": { + "@crawlee/templates": "3.15.3", + "ansi-colors": "^4.1.3", + "fs-extra": "^11.0.0", + "inquirer": "^8.2.4", + "tslib": "^2.4.0", + "yargonaut": "^1.1.4", + "yargs": "^17.5.1" + }, + "bin": { + "crawlee": "index.js" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/core": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.15.3.tgz", + "integrity": "sha512-cBglpY4KVlKnozeO8K4lw6/TDWajtJjPj4aNfckxUeEzapJgvTaxg6ZI4zxir6vic8sTeK9Olp3qG3wLnlrtXw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.20.0", + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@apify/pseudo_url": "^2.0.30", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/memory-storage": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@sapphire/async-queue": "^1.5.1", + "@vladfrangu/async_event_emitter": "^2.2.2", + "csv-stringify": "^6.2.0", + "fs-extra": "^11.0.0", + "got-scraping": "^4.0.0", + "json5": "^2.2.3", + "minimatch": "^9.0.0", + "ow": "^0.28.1", + "stream-json": "^1.8.0", + "tldts": "^7.0.0", + "tough-cookie": "^6.0.0", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/http": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.15.3.tgz", + "integrity": "sha512-NvD9khVsji6gX/t1YNBgTSlVCMRgzVhkL/oygyXPjfihfX42adAWJGi/ztK5/drA+7nNZlGY304W9Kheo5SqCQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/basic": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@types/content-type": "^1.1.5", + "cheerio": "1.0.0-rc.12", + "content-type": "^1.0.4", + "got-scraping": "^4.0.0", + "iconv-lite": "^0.7.0", + "mime-types": "^2.1.35", + "ow": "^0.28.1", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/jsdom": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.15.3.tgz", + "integrity": "sha512-SmsJcaLW12C35Myy2e0jdZpG1HhbAA/QwF+P1Op0AB0lerDYT8sGQJXARczLJceu+zhZeM/90g1oRuH8N3wB7g==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/http": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@types/jsdom": "^21.0.0", + "cheerio": "1.0.0-rc.12", + "jsdom": "^26.0.0", + "ow": "^0.28.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/linkedom": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.15.3.tgz", + "integrity": "sha512-pCmfjMuRDAdDqJiHL/Ph9rOZC9I4tFxXhveNUS0X3suOj/5y67m5t9CsV6Bv3XuwAEVXDc8MNOCz0FUN9dl3jw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/http": "3.15.3", + "@crawlee/types": "3.15.3", + "linkedom": "^0.18.0", + "ow": "^0.28.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/memory-storage": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.15.3.tgz", + "integrity": "sha512-iOUOGBTZNyl2srDsrAJwhYu4+leOxQSlx9uAtGt88kC7srlrn2B/OgXjhzTv0Vo7+kkAiTkxUMAfZ2eNW+UbSw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@crawlee/types": "3.15.3", + "@sapphire/async-queue": "^1.5.0", + "@sapphire/shapeshift": "^3.0.0", + "content-type": "^1.0.4", + "fs-extra": "^11.0.0", + "json5": "^2.2.3", + "mime-types": "^2.1.35", + "proper-lockfile": "^4.1.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">= 16" + } + }, + "node_modules/@crawlee/playwright": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.15.3.tgz", + "integrity": "sha512-PTIqiE0gTdIBJIJ9GC7VZYSOjyMNjg156nhsLKoJJK3dT/M0dKhoM36zgchghagcjuP+fLZYmx+TMDgpAfWfxQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.1", + "@crawlee/browser": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "cheerio": "1.0.0-rc.12", + "idcac-playwright": "^0.1.2", + "jquery": "^3.6.0", + "lodash.isequal": "^4.5.0", + "ml-logistic-regression": "^2.0.0", + "ml-matrix": "^6.11.0", + "ow": "^0.28.1", + "string-comparison": "^1.3.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + } + } + }, + "node_modules/@crawlee/puppeteer": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.15.3.tgz", + "integrity": "sha512-hiNrXwCPLaEEqejlXPWf567KnArwhZx4HHs16YqiB6wElf2eptvPO6jdeAnQX7BXyV3NWP4QPVKyieOXa/d51A==", + "license": "Apache-2.0", + "dependencies": { + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@crawlee/browser": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "cheerio": "1.0.0-rc.12", + "devtools-protocol": "*", + "idcac-playwright": "^0.1.2", + "jquery": "^3.6.0", + "ow": "^0.28.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "puppeteer": { + "optional": true + } + } + }, + "node_modules/@crawlee/templates": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.15.3.tgz", + "integrity": "sha512-7VKwdKYFf8yF4uaZU626cdZDpVQs5jv9bK//q94JK5IzpRdkwRedD2N93fYBrGVYyGqNhlEJz1nEIdAe+d6Knw==", + "license": "Apache-2.0", + "dependencies": { + "ansi-colors": "^4.1.3", + "inquirer": "^9.0.0", + "tslib": "^2.4.0", + "yargonaut": "^1.1.4", + "yargs": "^17.5.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/templates/node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/@crawlee/templates/node_modules/inquirer": { + "version": "9.3.8", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.3.8.tgz", + "integrity": "sha512-pFGGdaHrmRKMh4WoDDSowddgjT1Vkl90atobmTeSmcPGdYiwikch/m/Ef5wRaiamHejtw0cUUMMerzDUXCci2w==", + "license": "MIT", + "dependencies": { + "@inquirer/external-editor": "^1.0.2", + "@inquirer/figures": "^1.0.3", + "ansi-escapes": "^4.3.2", + "cli-width": "^4.1.0", + "mute-stream": "1.0.0", + "ora": "^5.4.1", + "run-async": "^3.0.0", + "rxjs": "^7.8.1", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@crawlee/templates/node_modules/mute-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/@crawlee/templates/node_modules/run-async": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", + "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/@crawlee/templates/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@crawlee/types": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.15.3.tgz", + "integrity": "sha512-RvgVPXrsQw4GQIUXrC1z1aNOedUPJnZ/U/8n+jZ0fu1Iw9moJVMuiuIxSI8q1P6BA84aWZdalyfDWBZ3FMjsiw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/utils": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.15.3.tgz", + "integrity": "sha512-guldTIfG+No6zoNmi5CKwABJDnrN8NqgwB9PFMR8kD+5r//TPFENfU9I3w4tQXx/pefnSZ99JrVZMUL3zenpJA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/ps-tree": "^1.2.0", + "@crawlee/types": "3.15.3", + "@types/sax": "^1.2.7", + "cheerio": "1.0.0-rc.12", + "file-type": "^20.0.0", + "got-scraping": "^4.0.3", + "ow": "^0.28.1", + "robots-parser": "^3.0.1", + "sax": "^1.4.1", + "tslib": "^2.4.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", + "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", + "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", + "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.1.0", + "@csstools/css-calc": "^2.1.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", + "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", + "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@databases/connection-pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@databases/connection-pool/-/connection-pool-1.1.0.tgz", @@ -164,60 +790,3037 @@ "integrity": "sha512-dLKqxGcymeVwEb/6c44KjOnzaAafFf0Wxa8xcfEjx/qOl3rdijsKYBAtIGhtVtOlpPf/PFKfgTuFurSPn/3B/g==", "license": "MIT" }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" + "node_modules/@inquirer/external-editor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", + "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } }, - "node_modules/assert-never": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/assert-never/-/assert-never-1.4.0.tgz", - "integrity": "sha512-5oJg84os6NMQNl27T9LnZkvvqzvAnHu03ShCnoj6bsJwS7L8AO4lf+C/XjK/nvzEqQB744moC6V128RucQd1jA==", + "node_modules/@inquirer/figures": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.15.tgz", + "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", "license": "MIT" }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "node_modules/@multiformats/base-x": { + "version": "4.0.1", "license": "MIT" }, - "node_modules/available-typed-arrays": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", - "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", - "license": "MIT", + "node_modules/@radiantearth/stac-migrate": { + "version": "2.0.2", + "license": "Apache-2.0", "dependencies": { - "possible-typed-array-names": "^1.0.0" + "compare-versions": "^3.6.0", + "multihashes": "^3.1.2", + "yargs": "^17.6.2" }, - "engines": { - "node": ">= 0.4" + "bin": { + "stac-migrate": "bin/cli.js" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "type": "github", + "url": "https://github.com/sponsors/m-mohr" } }, - "node_modules/axios": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", - "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==", + "node_modules/@sapphire/async-queue": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@sapphire/async-queue/-/async-queue-1.5.5.tgz", + "integrity": "sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==", "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.6", + "engines": { + "node": ">=v14.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@sapphire/shapeshift": { + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-3.9.7.tgz", + "integrity": "sha512-4It2mxPSr4OGn4HSQWGmhFMsNFGfFVhWeRPCRwbH972Ek2pzfGRZtb0pJ4Ze6oIzcyh2jw7nUDa6qGlWofgd9g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=v16" + } + }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.1.1.tgz", + "integrity": "sha512-rO92VvpgMc3kfiTjGT52LEtJ8Yc5kCWhZjLQ3LwlA4pSgPpQO7bVpYXParOD8Jwf+cVQECJo3yP/4I8aZtUQTQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@tokenizer/inflate": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", + "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "fflate": "^0.8.2", + "token-types": "^6.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT" + }, + "node_modules/@types/content-type": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@types/content-type/-/content-type-1.1.9.tgz", + "integrity": "sha512-Hq9IMnfekuOCsEmYl4QX2HBrT+XsfXiupfrLLY8Dcf3Puf4BkBOxSbWYTITSOQAhJoYPBez+b4MJRpIYL65z8A==", + "license": "MIT" + }, + "node_modules/@types/http-cache-semantics": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", + "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", + "license": "MIT" + }, + "node_modules/@types/jsdom": { + "version": "21.1.7", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.7.tgz", + "integrity": "sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^7.0.0" + } + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, + "node_modules/@vladfrangu/async_event_emitter": { + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/@vladfrangu/async_event_emitter/-/async_event_emitter-2.4.7.tgz", + "integrity": "sha512-Xfe6rpCTxSxfbswi/W/Pz7zp1WWSNn4A0eW4mLkQUewCrXXtMj31lCg+iQyTkh/CkusZSq9eDflu7tjEDXUY6g==", + "license": "MIT", + "engines": { + "node": ">=v14.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@zxing/text-encoding": { + "version": "0.9.0", + "license": "(Unlicense OR Apache-2.0)", + "optional": true + }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/assert-never": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/assert-never/-/assert-never-1.4.0.tgz", + "integrity": "sha512-5oJg84os6NMQNl27T9LnZkvvqzvAnHu03ShCnoj6bsJwS7L8AO4lf+C/XjK/nvzEqQB744moC6V128RucQd1jA==", + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "license": "MIT" + }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/axios": { + "version": "1.13.2", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, - "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.8.31", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.31.tgz", + "integrity": "sha512-a28v2eWrrRWPpJSzxc+mKwm0ZtVx/G8SepdQZDArnXYU/XS+IF6mp8aB/4E+hH1tyGCoDo3KlUCdlSxGDsRkAw==", + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.1.4" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/byte-counter": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/byte-counter/-/byte-counter-0.1.0.tgz", + "integrity": "sha512-jheRLVMeUKrDBjVw2O5+k4EvR4t9wtxHL+bo/LxfkxsVeuGMy3a5SEGgXdAFA4FSzTrU8rQXQIrsZ3oBq5a0pQ==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cacheable-lookup": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", + "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + } + }, + "node_modules/cacheable-request": { + "version": "13.0.15", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-13.0.15.tgz", + "integrity": "sha512-NjiSrjv37X73FmGGU5ec/M83vWQ6q1Ae3BFe+ABfdeeMy4LOMKYTpfEjrBnLedu43clKZtsYbKrHTIQE7vKq+A==", + "license": "MIT", + "dependencies": { + "@types/http-cache-semantics": "^4.0.4", + "get-stream": "^9.0.1", + "http-cache-semantics": "^4.2.0", + "keyv": "^5.5.4", + "mimic-response": "^4.0.0", + "normalize-url": "^8.1.0", + "responselike": "^4.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/call-bind": { + "version": "1.0.8", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-define-property": "^1.0.0", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001757", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001757.tgz", + "integrity": "sha512-r0nnL/I28Zi/yjk1el6ilj27tKcdjLsNqAOZr0yVjWPrSQyHgKI2INaEWw21bAQSv2LXRt1XuCS/GomNpWOxsQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "license": "MIT" + }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cheerio/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "license": "ISC", + "engines": { + "node": ">= 10" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", + "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/compare-versions": { + "version": "3.6.0", + "license": "MIT" + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "license": "MIT", + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/crawlee": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.15.3.tgz", + "integrity": "sha512-l+l1Fs4fEKUqKn9Vuw+tHiraWIVbRpSXFa09JeTdZID/xUlPHVLkKrqGLNa0cvZc7dqX2s9+1xLXid+pRn851w==", + "license": "Apache-2.0", + "dependencies": { + "@crawlee/basic": "3.15.3", + "@crawlee/browser": "3.15.3", + "@crawlee/browser-pool": "3.15.3", + "@crawlee/cheerio": "3.15.3", + "@crawlee/cli": "3.15.3", + "@crawlee/core": "3.15.3", + "@crawlee/http": "3.15.3", + "@crawlee/jsdom": "3.15.3", + "@crawlee/linkedom": "3.15.3", + "@crawlee/playwright": "3.15.3", + "@crawlee/puppeteer": "3.15.3", + "@crawlee/utils": "3.15.3", + "import-local": "^3.1.0", + "tslib": "^2.4.0" + }, + "bin": { + "crawlee": "cli.js" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" + }, + "node_modules/cssstyle": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", + "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^3.2.0", + "rrweb-cssom": "^0.8.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/csv-stringify": { + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.6.0.tgz", + "integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==", + "license": "MIT" + }, + "node_modules/data-urls": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "license": "MIT" + }, + "node_modules/decompress-response": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-10.0.0.tgz", + "integrity": "sha512-oj7KWToJuuxlPr7VV0vabvxEIiqNMo+q0NueIiL3XhtwC6FVOX7Hr1c0C4eD0bmf7Zr+S/dSf2xvkH3Ad6sU3Q==", + "license": "MIT", + "dependencies": { + "mimic-response": "^4.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1551306", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1551306.tgz", + "integrity": "sha512-CFx8QdSim8iIv+2ZcEOclBKTQY6BI1IEDa7Tm9YkwAXzEWFndTEzpTo5jAUhSnq24IC7xaDw0wvGcm96+Y3PEg==", + "license": "BSD-3-Clause" + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "license": "MIT", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.262", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.262.tgz", + "integrity": "sha512-NlAsMteRHek05jRUxUR0a5jpjYq9ykk6+kO0yRaMi5moe7u0fVIOeQ3Y30A8dIiWFBNUoQGi1ljb1i5VtS9WQQ==", + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "license": "MIT" + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/event-stream": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "integrity": "sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==", + "license": "MIT", + "dependencies": { + "duplexer": "~0.1.1", + "from": "~0", + "map-stream": "~0.1.0", + "pause-stream": "0.0.11", + "split": "0.3", + "stream-combiner": "~0.0.4", + "through": "~2.3.1" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, + "node_modules/figlet": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/figlet/-/figlet-1.9.4.tgz", + "integrity": "sha512-uN6QE+TrzTAHC1IWTyrc4FfGo2KH/82J8Jl1tyKB7+z5DBit/m3D++Iu5lg91qJMnQQ3vpJrj5gxcK/pk4R9tQ==", + "license": "MIT", + "dependencies": { + "commander": "^14.0.0" + }, + "bin": { + "figlet": "bin/index.js" + }, + "engines": { + "node": ">= 17.0.0" + } + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/file-type": { + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.5.0.tgz", + "integrity": "sha512-BfHZtG/l9iMm4Ecianu7P8HRD2tBHLtjXinm4X62XBOYzi7CYA7jyqfJzOvXHqzVrVPYqBo2/GvbARMaaJkKVg==", + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.2.6", + "strtok3": "^10.2.0", + "token-types": "^6.0.0", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fingerprint-generator": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.1.77.tgz", + "integrity": "sha512-wR15VUEZnwozFiSDRV+40zxlEt3ZV3JNYvLx0CSF9D9smov4pUC6MJZJnlxtDr+Ir4oppU8vn1JXApLk/Qr5Uw==", + "license": "Apache-2.0", + "dependencies": { + "generative-bayesian-network": "^2.1.77", + "header-generator": "^2.1.77", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/fingerprint-injector": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.1.77.tgz", + "integrity": "sha512-R778SIyrqgWO0P+UWKzIFWUWZz13EGu6UmV7CX3vuFDbsYIL1xiH+s+/nzPSOqFdhXyLo7d8aTOjbGbRLULoQQ==", + "license": "Apache-2.0", + "dependencies": { + "fingerprint-generator": "^2.1.77", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "^1.22.2", + "puppeteer": ">= 9.x" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/for-each": { + "version": "0.3.5", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.1.0.tgz", + "integrity": "sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", + "license": "MIT" + }, + "node_modules/fs-extra": { + "version": "11.3.2", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.2.tgz", + "integrity": "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/funtypes": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/funtypes/-/funtypes-4.2.0.tgz", + "integrity": "sha512-DvOtjiKvkeuXGV0O8LQh9quUP3bSOTEQPGv537Sao8kDq2rDbg48UsSJ7wlBLPzR2Mn0pV7cyAiq5pYG1oUyCQ==", + "license": "MIT" + }, + "node_modules/generative-bayesian-network": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.77.tgz", + "integrity": "sha512-viU4CRPsmgiklR94LhvdMndaY73BkCH1pGjmOjWbLR/ZwcUd06gKF3TCcsS3npRl74o33YSInSixxm16wIukcA==", + "license": "Apache-2.0", + "dependencies": { + "adm-zip": "^0.5.9", + "tslib": "^2.4.0" + } + }, + "node_modules/generator-function": { + "version": "2.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "license": "MIT", + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/got": { + "version": "14.6.5", + "resolved": "https://registry.npmjs.org/got/-/got-14.6.5.tgz", + "integrity": "sha512-Su87c0NNeg97de1sO02gy9I8EmE7DCJ1gzcFLcgGpYeq2PnLg4xz73MWrp6HjqbSsjb6Glf4UBDW6JNyZA6uSg==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^7.0.1", + "byte-counter": "^0.1.0", + "cacheable-lookup": "^7.0.0", + "cacheable-request": "^13.0.12", + "decompress-response": "^10.0.0", + "form-data-encoder": "^4.0.2", + "http2-wrapper": "^2.2.1", + "keyv": "^5.5.3", + "lowercase-keys": "^3.0.0", + "p-cancelable": "^4.0.1", + "responselike": "^4.0.2", + "type-fest": "^4.26.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, + "node_modules/got-scraping": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/got-scraping/-/got-scraping-4.1.2.tgz", + "integrity": "sha512-LtVwPM5YLnNY7HVT/AK/yDBUg/4yOZSlAjjug2ovrHQseS43QCmO1XosKKXcXrfc6OMX8OnDbAWIauFMcaJ5TQ==", + "license": "Apache-2.0", + "dependencies": { + "got": "^14.2.1", + "header-generator": "^2.1.41", + "http2-wrapper": "^2.2.0", + "mimic-response": "^4.0.0", + "ow": "^1.1.1", + "quick-lru": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/got-scraping/node_modules/@sindresorhus/is": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", + "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/got-scraping/node_modules/callsites": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.2.0.tgz", + "integrity": "sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/dot-prop": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-7.2.0.tgz", + "integrity": "sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==", + "license": "MIT", + "dependencies": { + "type-fest": "^2.11.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/ow": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ow/-/ow-1.1.1.tgz", + "integrity": "sha512-sJBRCbS5vh1Jp9EOgwp1Ws3c16lJrUkJYlvWTYC03oyiYVwS/ns7lKRWow4w4XjDyTrA2pplQv4B2naWSR6yDA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^5.3.0", + "callsites": "^4.0.0", + "dot-prop": "^7.2.0", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/quick-lru": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz", + "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-ansi/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/header-generator": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.77.tgz", + "integrity": "sha512-ggSG/mfkFMu8CO7xP591G8kp1IJCBvgXu7M8oxTjC9u914JsIzE6zIfoFsXzA+pf0utWJhUsdqU0oV/DtQ4DFQ==", + "license": "Apache-2.0", + "dependencies": { + "browserslist": "^4.21.1", + "generative-bayesian-network": "^2.1.77", + "ow": "^0.28.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, + "node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http2-wrapper": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", + "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", + "license": "MIT", + "dependencies": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.2.0" + }, + "engines": { + "node": ">=10.19.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/idcac-playwright": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/idcac-playwright/-/idcac-playwright-0.1.3.tgz", + "integrity": "sha512-VVYQ4sv6OrUJKVzYaIP1hq0qAHd1O22HW5LnL1Wf6zkrLStQ/QEg4iJ0rllIOEpd+Rmm+635AJD59A+Vw+2PgQ==", + "license": "ISC" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "license": "ISC" + }, + "node_modules/inquirer": { + "version": "8.2.7", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz", + "integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==", + "license": "MIT", + "dependencies": { + "@inquirer/external-editor": "^1.0.0", + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.1", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "figures": "^3.0.0", + "lodash": "^4.17.21", + "mute-stream": "0.0.8", + "ora": "^5.4.1", + "run-async": "^2.4.0", + "rxjs": "^7.5.5", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6", + "wrap-ansi": "^6.0.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/inquirer/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-any-array": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", + "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==", + "license": "MIT" + }, + "node_modules/is-arguments": { + "version": "1.2.0", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-callable": { + "version": "1.2.7", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-function": { + "version": "1.1.2", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "license": "MIT" + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-regex": { + "version": "1.2.1", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jquery": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.7.1.tgz", + "integrity": "sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==", + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsdom": { + "version": "26.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz", + "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", + "license": "MIT", + "dependencies": { + "cssstyle": "^4.2.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.5.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.16", + "parse5": "^7.2.1", + "rrweb-cssom": "^0.8.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.1.1", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.1.1", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jsdom/node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/jsdom/node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "license": "MIT" + }, + "node_modules/jsdom/node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/keyv": { + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.5.4.tgz", + "integrity": "sha512-eohl3hKTiVyD1ilYdw9T0OiB4hnjef89e3dMYKz+mVKDzj+5IteTseASUsOB+EU9Tf6VNTCjDePcP6wkDGmLKQ==", + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/linkedom": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz", + "integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==", + "license": "ISC", + "dependencies": { + "css-select": "^5.1.0", + "cssom": "^0.5.0", + "html-escaper": "^3.0.3", + "htmlparser2": "^10.0.0", + "uhyphen": "^0.2.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "canvas": ">= 2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/linkedom/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/linkedom/node_modules/htmlparser2": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", + "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.1", + "entities": "^6.0.0" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lowercase-keys": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", + "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/map-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", + "integrity": "sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", + "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ml-array-max": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-1.2.4.tgz", + "integrity": "sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0" + } + }, + "node_modules/ml-array-min": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-1.2.3.tgz", + "integrity": "sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0" + } + }, + "node_modules/ml-array-rescale": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-1.3.7.tgz", + "integrity": "sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0", + "ml-array-max": "^1.2.4", + "ml-array-min": "^1.2.3" + } + }, + "node_modules/ml-logistic-regression": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-logistic-regression/-/ml-logistic-regression-2.0.0.tgz", + "integrity": "sha512-xHhB91ut8GRRbJyB1ZQfKsl1MHmE1PqMeRjxhks96M5BGvCbC9eEojf4KgRMKM2LxFblhVUcVzweAoPB48Nt0A==", + "license": "MIT", + "dependencies": { + "ml-matrix": "^6.5.0" + } + }, + "node_modules/ml-matrix": { + "version": "6.12.1", + "resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.12.1.tgz", + "integrity": "sha512-TJ+8eOFdp+INvzR4zAuwBQJznDUfktMtOB6g/hUcGh3rcyjxbz4Te57Pgri8Q9bhSQ7Zys4IYOGhFdnlgeB6Lw==", + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.1", + "ml-array-rescale": "^1.3.7" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/multibase": { + "version": "3.1.2", + "license": "MIT", + "dependencies": { + "@multiformats/base-x": "^4.0.1", + "web-encoding": "^1.0.6" + }, + "engines": { + "node": ">=10.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/multiformats": { + "version": "9.9.0", + "license": "(Apache-2.0 AND MIT)" + }, + "node_modules/multihashes": { + "version": "3.1.2", + "license": "MIT", + "dependencies": { + "multibase": "^3.1.0", + "uint8arrays": "^2.0.5", + "varint": "^6.0.0" + }, + "engines": { + "node": ">=10.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "license": "ISC" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "license": "MIT" + }, + "node_modules/normalize-url": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.1.0.tgz", + "integrity": "sha512-X06Mfd/5aKsRHc0O0J5CUedwnPmnDtLF2+nq+KN9KSDlJHkPuh0JUviWjEWMe0SW/9TDdSLVPuk7L5gGTIA1/w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/nwsapi": { + "version": "2.2.22", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.22.tgz", + "integrity": "sha512-ujSMe1OWVn55euT1ihwCI1ZcAaAU3nxUiDwfDQldc51ZXaB9m2AyOn6/jh1BLe2t/G8xd6uKG1UBF2aZJeg2SQ==", + "license": "MIT" + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", + "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.1.0", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.5.0", + "is-interactive": "^1.0.0", + "is-unicode-supported": "^0.1.0", + "log-symbols": "^4.1.0", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/ow/-/ow-0.28.2.tgz", + "integrity": "sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^4.2.0", + "callsites": "^3.1.0", + "dot-prop": "^6.0.1", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow/node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/p-cancelable": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz", + "integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parent-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parent-require/-/parent-require-1.0.0.tgz", + "integrity": "sha512-2MXDNZC4aXdkkap+rBBMv0lUsfJqvX5/2FiYYnfCnorZt3Pk06/IOR5KeaoghgS2w07MLWgjbsnyaq6PdHn2LQ==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "license": [ + "MIT", + "Apache2" + ], + "dependencies": { + "through": "~2.3" + } + }, + "node_modules/pg": { + "version": "8.16.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", + "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.3", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.2.7" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz", + "integrity": "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "license": "MIT" + }, + "node_modules/pg-cursor": { + "version": "2.15.3", + "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.15.3.tgz", + "integrity": "sha512-eHw63TsiGtFEfAd7tOTZ+TLy+i/2ePKS20H84qCQ+aQ60pve05Okon9tKMC+YN3j6XyeFoHnaim7Lt9WVafQsA==", + "license": "MIT", + "peerDependencies": { + "pg": "^8" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", + "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proxy-chain": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/proxy-chain/-/proxy-chain-2.6.0.tgz", + "integrity": "sha512-+NpVKSk68j8sQJG2tBbFuJxMzKTlqeCXXFbqvlyiFhnmxdcYJSv4XZzUSIfwIUwR3D0T8fEJqrA4C7yykU40Pw==", + "license": "Apache-2.0", + "dependencies": { + "socks": "^2.8.3", + "socks-proxy-agent": "^8.0.3", + "tslib": "^2.3.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "license": "MIT" + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", + "license": "MIT" + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-cwd/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/responselike": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-4.0.2.tgz", + "integrity": "sha512-cGk8IbWEAnaCpdAt1BHzJ3Ahz5ewDJa0KseTsE3qIRMJ3C698W8psM7byCeWVpd/Ha7FUYzuRVzXoKoM6nRUbA==", + "license": "MIT", + "dependencies": { + "lowercase-keys": "^3.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/robots-parser": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz", + "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "license": "MIT" + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.2" + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" }, "engines": { "node": ">= 0.4" @@ -226,631 +3829,697 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.3.tgz", + "integrity": "sha512-yqYn1JhPczigF94DMS+shiDMjDowYO6y9+wB/4WgO0Y19jWYk0lQ4tuG5KI7kj4FTp1wxPj5IFfcrz/s1c3jjQ==", + "license": "BlueOak-1.0.0" + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", "license": "MIT", "dependencies": { + "define-data-property": "^1.1.4", "es-errors": "^1.3.0", - "function-bind": "^1.1.2" + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" }, "engines": { "node": ">= 0.4" } }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", "license": "MIT", "engines": { - "node": ">=6" + "node": ">= 6.0.0", + "npm": ">= 3.0.0" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", "license": "MIT", "dependencies": { - "delayed-stream": "~1.0.0" + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 10.0.0", + "npm": ">= 3.0.0" } }, - "node_modules/cosmiconfig": { - "version": "8.3.6", - "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", - "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", "license": "MIT", "dependencies": { - "import-fresh": "^3.3.0", - "js-yaml": "^4.1.0", - "parse-json": "^5.2.0", - "path-type": "^4.0.0" + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" }, "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/d-fischer" - }, - "peerDependencies": { - "typescript": ">=4.9.5" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } + "node": ">= 14" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "node_modules/split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", "license": "MIT", + "dependencies": { + "through": "2" + }, "engines": { - "node": ">=0.4.0" + "node": "*" } }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", "engines": { - "node": ">=12" + "node": ">= 10.x" + } + }, + "node_modules/stac-js": { + "version": "0.1.9", + "license": "Apache-2.0", + "dependencies": { + "@radiantearth/stac-migrate": "^2.0.2", + "urijs": "^1.19.11" }, "funding": { - "url": "https://dotenvx.com" + "type": "github", + "url": "https://github.com/sponsors/m-mohr" } }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-combiner": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", + "integrity": "sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==", "license": "MIT", "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" + "duplexer": "~0.1.1" } }, - "node_modules/error-ex": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", - "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "license": "MIT", + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", "dependencies": { - "is-arrayish": "^0.2.1" + "stream-chain": "^2.2.5" } }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "license": "MIT", - "engines": { - "node": ">= 0.4" + "dependencies": { + "safe-buffer": "~5.2.0" } }, - "node_modules/es-errors": { + "node_modules/string-comparison": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "resolved": "https://registry.npmjs.org/string-comparison/-/string-comparison-1.3.0.tgz", + "integrity": "sha512-46aD+slEwybxAMPRII83ATbgMgTiz5P8mVd7Z6VJsCzSHFjdt1hkAVLeFxPIyEb11tc6ihpJTlIqoO0MCF6NPw==", "license": "MIT", "engines": { - "node": ">= 0.4" + "node": "^16.0.0 || >=18.0.0" } }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "node_modules/string-width": { + "version": "4.2.3", "license": "MIT", "dependencies": { - "es-errors": "^1.3.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" }, "engines": { - "node": ">= 0.4" + "node": ">=8" } }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "node_modules/strip-ansi": { + "version": "6.0.1", "license": "MIT", "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" + "ansi-regex": "^5.0.1" }, "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } + "node": ">=8" } }, - "node_modules/for-each": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", - "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", + "node_modules/strtok3": { + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.4.tgz", + "integrity": "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg==", "license": "MIT", "dependencies": { - "is-callable": "^1.2.7" + "@tokenizer/token": "^0.3.0" }, "engines": { - "node": ">= 0.4" + "node": ">=18" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "type": "github", + "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "license": "MIT", "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "has-flag": "^4.0.0" }, "engines": { - "node": ">= 6" + "node": ">=8" } }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "license": "MIT" + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==", + "license": "MIT" + }, + "node_modules/tldts": { + "version": "7.0.19", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.19.tgz", + "integrity": "sha512-8PWx8tvC4jDB39BQw1m4x8y5MH1BcQ5xHeL2n7UVFulMPH/3Q0uiamahFJ3lXA0zO2SUyRXuVVbWSDmstlt9YA==", "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" + "dependencies": { + "tldts-core": "^7.0.19" + }, + "bin": { + "tldts": "bin/cli.js" } }, - "node_modules/funtypes": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/funtypes/-/funtypes-4.2.0.tgz", - "integrity": "sha512-DvOtjiKvkeuXGV0O8LQh9quUP3bSOTEQPGv537Sao8kDq2rDbg48UsSJ7wlBLPzR2Mn0pV7cyAiq5pYG1oUyCQ==", + "node_modules/tldts-core": { + "version": "7.0.19", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.19.tgz", + "integrity": "sha512-lJX2dEWx0SGH4O6p+7FPwYmJ/bu1JbcGJ8RLaG9b7liIgZ85itUVEPbMtWRVrde/0fnDPEPHW10ZsKW3kVsE9A==", "license": "MIT" }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "node_modules/token-types": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.1.tgz", + "integrity": "sha512-kh9LVIWH5CnL63Ipf0jhlBIy0UsrMj/NJDfpsy1SqOXlLKEVyXXYrnFxFT1yOOYVGBSApeVnjPw/sBz5BfEjAQ==", "license": "MIT", "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" + "@borewit/text-codec": "^0.1.0", + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" }, "engines": { - "node": ">= 0.4" + "node": ">=14.16" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "type": "github", + "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", + "node_modules/tough-cookie": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz", + "integrity": "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==", + "license": "BSD-3-Clause", "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" + "tldts": "^7.0.5" }, "engines": { - "node": ">= 0.4" + "node": ">=16" } }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", "license": "MIT", - "engines": { - "node": ">= 0.4" + "dependencies": { + "punycode": "^2.3.1" }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "engines": { + "node": ">=18" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "node_modules/uhyphen": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==", + "license": "ISC" + }, + "node_modules/uint8array-extras": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", "license": "MIT", "engines": { - "node": ">= 0.4" + "node": ">=18" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "node_modules/uint8arrays": { + "version": "2.1.10", "license": "MIT", "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "multiformats": "^9.4.2" } }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, "engines": { - "node": ">= 0.4" + "node": ">= 10.0.0" } }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "node_modules/update-browserslist-db": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", + "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], "license": "MIT", "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" + "escalade": "^3.2.0", + "picocolors": "^1.1.1" }, - "engines": { - "node": ">=6" + "bin": { + "update-browserslist-db": "cli.js" }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "peerDependencies": { + "browserslist": ">= 4.21.0" } }, - "node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "node_modules/urijs": { + "version": "1.19.11", "license": "MIT" }, - "node_modules/is-promise": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", - "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "node_modules/util": { + "version": "0.12.5", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "node_modules/vali-date": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/vali-date/-/vali-date-1.0.0.tgz", + "integrity": "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/varint": { + "version": "6.0.0", "license": "MIT" }, - "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", "license": "MIT", "dependencies": { - "argparse": "^2.0.1" + "xml-name-validator": "^5.0.0" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "engines": { + "node": ">=18" } }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "license": "MIT" - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "license": "MIT" + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "node_modules/web-encoding": { + "version": "1.1.5", "license": "MIT", + "dependencies": { + "util": "^0.12.3" + }, + "optionalDependencies": { + "@zxing/text-encoding": "0.9.0" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", "engines": { - "node": ">= 0.4" + "node": ">=12" } }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, "engines": { - "node": ">= 0.6" + "node": ">=18" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "license": "MIT", "dependencies": { - "mime-db": "1.52.0" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { - "node": ">= 0.6" + "node": ">=0.10.0" } }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", "license": "MIT", "dependencies": { - "callsites": "^3.0.0" + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" }, "engines": { - "node": ">=6" + "node": ">=18" } }, - "node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "node_modules/which-typed-array": { + "version": "1.1.19", "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" }, "engines": { - "node": ">=8" + "node": ">= 0.4" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "node_modules/wrap-ansi": { + "version": "7.0.0", "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, "engines": { - "node": ">=8" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/pg": { - "version": "8.16.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", - "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", "license": "MIT", - "dependencies": { - "pg-connection-string": "^2.9.1", - "pg-pool": "^3.10.1", - "pg-protocol": "^1.10.3", - "pg-types": "2.2.0", - "pgpass": "1.0.5" - }, "engines": { - "node": ">= 16.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.2.7" + "node": ">=10.0.0" }, "peerDependencies": { - "pg-native": ">=3.0.1" + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" }, "peerDependenciesMeta": { - "pg-native": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { "optional": true } } }, - "node_modules/pg-cloudflare": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz", - "integrity": "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==", - "license": "MIT", - "optional": true + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } }, - "node_modules/pg-connection-string": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", - "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "license": "MIT" }, - "node_modules/pg-cursor": { - "version": "2.15.3", - "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.15.3.tgz", - "integrity": "sha512-eHw63TsiGtFEfAd7tOTZ+TLy+i/2ePKS20H84qCQ+aQ60pve05Okon9tKMC+YN3j6XyeFoHnaim7Lt9WVafQsA==", + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", "license": "MIT", - "peerDependencies": { - "pg": "^8" + "engines": { + "node": ">=0.4" } }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "node_modules/y18n": { + "version": "5.0.8", "license": "ISC", "engines": { - "node": ">=4.0.0" + "node": ">=10" } }, - "node_modules/pg-pool": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", - "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", - "license": "MIT", - "peerDependencies": { - "pg": ">=8.0" + "node_modules/yargonaut": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/yargonaut/-/yargonaut-1.1.4.tgz", + "integrity": "sha512-rHgFmbgXAAzl+1nngqOcwEljqHGG9uUZoPjsdZEs1w5JW9RXYzrSvH/u70C1JE5qFi0qjsdhnUX/dJRpWqitSA==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "^1.1.1", + "figlet": "^1.1.1", + "parent-require": "^1.0.0" } }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "node_modules/yargonaut/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, "engines": { - "node": ">=4" + "node": ">=0.10.0" } }, - "node_modules/pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "node_modules/yargonaut/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", "license": "MIT", - "dependencies": { - "split2": "^4.1.0" + "engines": { + "node": ">=0.10.0" } }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "license": "ISC" - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "node_modules/yargonaut/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", "license": "MIT", + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, "engines": { - "node": ">=4" + "node": ">=0.10.0" } }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "node_modules/yargonaut/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", "license": "MIT", + "dependencies": { + "ansi-regex": "^2.0.0" + }, "engines": { "node": ">=0.10.0" } }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "node_modules/yargonaut/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", "license": "MIT", "engines": { - "node": ">=0.10.0" + "node": ">=0.8.0" } }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "node_modules/yargs": { + "version": "17.7.2", "license": "MIT", "dependencies": { - "xtend": "^4.0.0" + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" }, "engines": { - "node": ">=0.10.0" + "node": ">=12" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "license": "MIT", + "node_modules/yargs-parser": { + "version": "21.1.1", + "license": "ISC", "engines": { - "node": ">=4" + "node": ">=12" } }, - "node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "license": "ISC", + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", "engines": { - "node": ">= 10.x" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "node_modules/yoctocolors-cjs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.3.tgz", + "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", "license": "MIT", "engines": { - "node": ">=0.4" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } } } diff --git a/crawler/package.json b/crawler/package.json index df349cc..175957d 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -17,6 +17,7 @@ "dependencies": { "@databases/pg": "^5.5.0", "axios": "^1.13.2", + "crawlee": "^3.15.3", "dotenv": "^17.2.3", "stac-js": "^0.1.9" } diff --git a/crawler/utils/config.js b/crawler/utils/config.js index 2b006c1..3424a7b 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -27,8 +27,6 @@ function parseCliArgs() { config.maxApis = parseInt(args[++i], 10); } else if (arg === '--timeout' || arg === '-t') { config.timeout = parseInt(args[++i], 10); - } else if (arg === '--max-depth' || arg === '-d') { - config.maxDepth = parseInt(args[++i], 10); } else if (arg === '--help' || arg === '-h') { printHelp(); process.exit(0); @@ -49,7 +47,6 @@ STAC Crawler Configuration Options: -c, --max-catalogs Maximum number of catalogs to crawl (default: 10) -a, --max-apis Maximum number of APIs to crawl (default: 5) -t, --timeout Timeout for each crawl operation in ms (default: 30000) - -d, --max-depth Maximum recursion depth for nested catalogs (default: 3) -h, --help Show this help message Environment Variables: @@ -57,7 +54,6 @@ Environment Variables: MAX_CATALOGS Same as --max-catalogs MAX_APIS Same as --max-apis TIMEOUT_MS Same as --timeout - MAX_DEPTH Same as --max-depth Examples: node index.js --mode catalogs --max-catalogs 20 @@ -79,17 +75,18 @@ function getConfig() { mode: 'both', // 'catalogs', 'apis', or 'both' maxCatalogs: 10, // Maximum number of catalogs to crawl maxApis: 5, // Maximum number of APIs to crawl - timeout: 30000, // Timeout in milliseconds (30 seconds) - maxDepth: 3 // Maximum recursion depth for nested catalogs + timeout: 30000 // Timeout in milliseconds (30 seconds) }; // Build configuration with precedence: CLI > ENV > Defaults const config = { mode: cliArgs.mode || process.env.CRAWL_MODE || defaults.mode, - maxCatalogs: cliArgs.maxCatalogs || parseInt(process.env.MAX_CATALOGS, 10) || defaults.maxCatalogs, - maxApis: cliArgs.maxApis || parseInt(process.env.MAX_APIS, 10) || defaults.maxApis, - timeout: cliArgs.timeout || parseInt(process.env.TIMEOUT_MS, 10) || defaults.timeout, - maxDepth: cliArgs.maxDepth || parseInt(process.env.MAX_DEPTH, 10) || defaults.maxDepth + maxCatalogs: cliArgs.maxCatalogs !== undefined ? cliArgs.maxCatalogs : + (process.env.MAX_CATALOGS ? parseInt(process.env.MAX_CATALOGS, 10) : defaults.maxCatalogs), + maxApis: cliArgs.maxApis !== undefined ? cliArgs.maxApis : + (process.env.MAX_APIS ? parseInt(process.env.MAX_APIS, 10) : defaults.maxApis), + timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : + (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout) }; // Validate mode @@ -99,15 +96,13 @@ function getConfig() { process.exit(1); } - // When mode is 'catalogs', remove limits for comprehensive crawling - if (config.mode === 'catalogs') { - config.maxCatalogs = Infinity; - config.maxDepth = Infinity; - config.timeout = Infinity; - } + // NOTE: Removed automatic Infinity override for 'catalogs' mode + // Users can now control limits via CLI args even when mode is 'catalogs' - // Validate numeric values - if (config.maxCatalogs < 0 || config.maxApis < 0 || config.timeout < 0 || config.maxDepth < 0) { + // Validate numeric values (allow Infinity for unlimited mode) + if ((config.maxCatalogs !== Infinity && config.maxCatalogs < 0) || + (config.maxApis !== Infinity && config.maxApis < 0) || + (config.timeout !== Infinity && config.timeout < 0)) { console.error('Numeric configuration values must be non-negative'); process.exit(1); } diff --git a/crawler/utils/endpoints.js b/crawler/utils/endpoints.js new file mode 100644 index 0000000..09ebe6f --- /dev/null +++ b/crawler/utils/endpoints.js @@ -0,0 +1,36 @@ +/** + * @fileoverview Endpoint utilities for STAC collections + * @module utils/endpoints + */ + +/** + * Tries common STAC collection endpoints and enqueues successful ones + * @async + * @param {string} baseUrl - Base catalog URL + * @param {string} catalogId - Catalog ID for logging + * @param {number} depth - Current depth + * @param {Object} crawler - Crawlee crawler instance + * @param {Object} log - Logger + * @param {string} indent - Indentation for logging + */ +export async function tryCollectionEndpoints(baseUrl, catalogId, depth, crawler, log, indent) { + const collectionEndpoints = [ + '/collections', + '/collections/', + '/api/v1/collections' + ]; + + const requests = collectionEndpoints.map(endpoint => ({ + url: `${baseUrl}${endpoint}`, + label: 'COLLECTIONS', + userData: { + catalogUrl: baseUrl, + catalogId, + depth + }, + // Don't retry failed endpoint attempts - just try the next one + maxRetries: 0 + })); + + await crawler.addRequests(requests); +} diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js new file mode 100644 index 0000000..5c021fe --- /dev/null +++ b/crawler/utils/handlers.js @@ -0,0 +1,205 @@ +/** + * @fileoverview Request handlers for catalog and collection crawling + * @module utils/handlers + */ + +import create from 'stac-js'; +import { normalizeCollection } from './normalization.js'; +import { tryCollectionEndpoints } from './endpoints.js'; + +/** + * Handles catalog requests - validates STAC, extracts child catalogs and collections + * @async + * @param {Object} context - Request handler context + * @param {Object} context.request - Crawlee request object + * @param {Object} context.json - Parsed JSON response + * @param {Object} context.crawler - Crawlee crawler instance + * @param {Object} context.log - Logger instance + * @param {string} context.indent - Indentation for logging + * @param {Object} context.results - Results object to store data + */ +export async function handleCatalog({ request, json, crawler, log, indent, results }) { + const depth = request.userData?.depth || 0; + const catalogId = request.userData?.catalogId || 'unknown'; + + log.info(`${indent}Processing catalog: ${catalogId} (depth: ${depth})`); + + // Validate with stac-js + let stacCatalog; + try { + stacCatalog = create(json, request.url); + results.stats.stacCompliant++; + + // Log STAC object type + if (typeof stacCatalog.isCatalog === 'function' && stacCatalog.isCatalog()) { + log.info(`${indent}STAC Catalog validated: ${catalogId}`); + } else if (typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection()) { + log.info(`${indent}STAC Collection validated: ${catalogId}`); + } + } catch (parseError) { + log.warning(`${indent}Non-compliant STAC catalog ${catalogId}: ${parseError.message}`); + throw new Error(`STAC validation failed: ${parseError.message}`); + } + + results.stats.catalogsProcessed++; + results.catalogs.push({ + id: catalogId, + url: request.url, + depth, + stacType: stacCatalog.isCatalog() ? 'catalog' : 'collection' + }); + + // If this is a STAC Collection (not a catalog), extract and store it + if (typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection()) { + const collection = normalizeCollection(stacCatalog, results.collections.length); + results.collections.push(collection); + results.stats.collectionsFound++; + log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + } + + // Try to get collections from this catalog + await tryCollectionEndpoints(request.url, catalogId, depth, crawler, log, indent); + + // Extract and enqueue child catalog links using stac-js + if (stacCatalog && typeof stacCatalog.getChildLinks === 'function') { + const childLinks = stacCatalog.getChildLinks(); + + if (childLinks.length > 0) { + log.info(`${indent}Found ${childLinks.length} child catalog links`); + + // Log first child link structure for debugging + if (childLinks[0]) { + log.debug(`${indent}Sample child link structure:`, { + hasGetAbsoluteUrl: typeof childLinks[0].getAbsoluteUrl === 'function', + href: childLinks[0].href, + title: childLinks[0].title, + rel: childLinks[0].rel + }); + } + + const childRequests = childLinks + .map((link, idx) => { + let childUrl; + try { + childUrl = typeof link.getAbsoluteUrl === 'function' + ? link.getAbsoluteUrl() + : link.href; + } catch (err) { + log.warning(`${indent}Error getting URL for link ${idx}: ${err.message}`); + return null; + } + + // If URL is relative, make it absolute using the catalog URL + if (childUrl && typeof childUrl === 'string' && !childUrl.startsWith('http')) { + const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; + const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); + childUrl = `${basePath}/${childUrl}`; + } + + // Validate URL is a string and looks like a URL + if (!childUrl || typeof childUrl !== 'string' || !childUrl.startsWith('http')) { + log.warning(`${indent}Skipping invalid URL at index ${idx}: ${childUrl}`); + return null; + } + + // Get title as string + const linkTitle = typeof link.title === 'string' && link.title.length > 0 + ? link.title + : `child-${idx}`; + + return { + url: childUrl, + label: 'CATALOG', + userData: { + depth: depth + 1, + catalogId: linkTitle, + parentId: catalogId + } + }; + }) + .filter(Boolean); // Remove null entries + + log.info(`${indent}Valid child requests: ${childRequests.length}/${childLinks.length}`); + + if (childRequests.length > 0) { + try { + await crawler.addRequests(childRequests); + log.info(`${indent}Successfully enqueued ${childRequests.length} child catalogs`); + } catch (error) { + log.error(`${indent}Failed to add child requests: ${error.message}`); + // Log the actual requests that failed + childRequests.forEach((req, idx) => { + log.error(`${indent} [${idx}] url="${req.url}" label="${req.label}" userData=${JSON.stringify(req.userData)}`); + }); + throw error; + } + } + } + } +} + +/** + * Handles collection endpoint requests + * @async + * @param {Object} context - Request handler context + * @param {Object} context.request - Crawlee request object + * @param {Object} context.json - Parsed JSON response + * @param {Object} context.crawler - Crawlee crawler instance + * @param {Object} context.log - Logger instance + * @param {string} context.indent - Indentation for logging + * @param {Object} context.results - Results object to store data + */ +export async function handleCollections({ request, json, crawler, log, indent, results }) { + const catalogId = request.userData?.catalogId || 'unknown'; + + // Parse response with stac-js + let stacObj; + try { + stacObj = create(json, request.url); + } catch (parseError) { + log.warning(`${indent}Skipping non-compliant STAC collections at ${request.url}`); + return; + } + + let collectionsData = []; + + // Check if this is a CollectionCollection (STAC API response) + if (stacObj && typeof stacObj.getAll === 'function') { + collectionsData = stacObj.getAll(); + } else if (Array.isArray(json)) { + // Handle array of collections + collectionsData = json.map(col => { + try { + return create(col, request.url); + } catch { + return null; + } + }).filter(Boolean); + } else if (json.collections) { + // Handle nested collections property + collectionsData = json.collections.map(col => { + try { + return create(col, request.url); + } catch { + return null; + } + }).filter(Boolean); + } + + if (collectionsData.length > 0) { + log.info(`${indent}Found ${collectionsData.length} collections for catalog ${catalogId}`); + + // Normalize and store collections + const collections = collectionsData.map((colObj, index) => + normalizeCollection(colObj, index) + ); + + results.collections.push(...collections); + results.stats.collectionsFound += collections.length; + + // Display sample + if (collections.length > 0) { + log.info(`${indent} Sample: ${collections[0].id} - ${collections[0].title}`); + } + } +} diff --git a/crawler/utils/normalization.js b/crawler/utils/normalization.js new file mode 100644 index 0000000..b58f5b0 --- /dev/null +++ b/crawler/utils/normalization.js @@ -0,0 +1,130 @@ +/** + * @fileoverview Normalization utilities for STAC catalogs and collections + * @module utils/normalization + */ + +/** + * Derives categories from a catalog object by checking various possible fields + * @param {Object} catalog - Catalog object to extract categories from + * @returns {Array} Array of category strings, empty array if none found + */ +export function deriveCategories(catalog) { + if (!catalog || typeof catalog !== 'object') { + return []; + } + + if (Array.isArray(catalog.categories)) { + return catalog.categories.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.keywords)) { + return catalog.keywords.filter(Boolean).map(String); + } + + if (Array.isArray(catalog.tags)) { + return catalog.tags.filter(Boolean).map(String); + } + + if (typeof catalog.access === 'string' && catalog.access.trim().length) { + return [catalog.access.trim()]; + } + + return []; +} + +/** + * Normalizes a catalog object from STAC Index API format + * @param {Object} catalog - Catalog object from the STAC Index API + * @param {number} index - Index position in the original array + * @returns {Object} Normalized catalog object with standard properties + */ +export function normalizeCatalog(catalog, index) { + return { + index, + id: catalog.id, + url: catalog.url, + slug: catalog.slug, + title: catalog.title, + summary: catalog.summary, + access: catalog.access, + created: catalog.created, + updated: catalog.updated, + isPrivate: catalog.isPrivate, + isApi: catalog.isApi, + accessInfo: catalog.accessInfo, + categories: deriveCategories(catalog), + // Preserve any additional dynamic properties + ...Object.fromEntries( + Object.entries(catalog).filter(([key]) => + !['id', 'url', 'slug', 'title', 'summary', 'access', 'created', + 'updated', 'isPrivate', 'isApi', 'accessInfo'].includes(key) + ) + ) + }; +} + +/** + * Normalizes a collection object using stac-js methods for metadata extraction + * @param {Object} colObj - Collection object (stac-js or plain object) + * @param {number} index - Index position + * @returns {Object} Normalized collection object + */ +export function normalizeCollection(colObj, index) { + // Use stac-js methods for robust metadata extraction + const bbox = typeof colObj.getBoundingBox === 'function' + ? colObj.getBoundingBox() + : (colObj.extent?.spatial?.bbox?.[0] || null); + + const temporal = typeof colObj.getTemporalExtent === 'function' + ? colObj.getTemporalExtent() + : (colObj.extent?.temporal?.interval?.[0] || null); + + // Get self URL using stac-js link navigation + let selfUrl = null; + if (typeof colObj.getAbsoluteUrl === 'function') { + selfUrl = colObj.getAbsoluteUrl(); + } else if (colObj.links) { + const selfLink = colObj.links.find(l => l.rel === 'self'); + selfUrl = selfLink?.href || null; + } + + return { + index, + id: colObj.id || 'Unknown', + url: selfUrl, + title: colObj.title || null, + description: colObj.description || colObj.summary || null, + bbox, + temporal, + license: colObj.license || null, + keywords: colObj.keywords || [] + }; +} + +/** + * Processes an array of catalogs from the STAC Index API + * @param {Array} catalogs - Array of catalog objects from the STAC Index API + * @returns {Array} Array of normalized catalog objects + * @throws {Error} Throws error if input is not an array + */ +export function processCatalogs(catalogs) { + if (!Array.isArray(catalogs)) { + throw new Error('Expected an array'); + } + + const normalized = catalogs.map((catalog, index) => normalizeCatalog(catalog, index)); + + console.log(`Total: ${normalized.length} catalogs found\n`); + + if (normalized.length > 0) { + console.log('Example - First Catalog:'); + const first = normalized[0]; + console.log(` ID: ${first.id}`); + console.log(` URL: ${first.url}`); + console.log(` Title: ${first.title}`); + console.log(` Is API: ${first.isApi}`); + console.log(` Categories: ${JSON.stringify(first.categories)}`); + } + + return normalized; +} From fdb0cada7ceafbf8541d58384d996d809abecc09 Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 28 Nov 2025 14:26:50 +0100 Subject: [PATCH 25/68] #53,example environment configuration file for database and crawler settings --- .env.example | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3db8e63 --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Database Configuration +# PostgreSQL connection parameters +PGHOST=atlas.stacindex.org +PGPORT=5432 +PGUSER= +PGPASSWORD= +PGDATABASE=stac_db + +# Crawler Configuration +# Crawl mode: 'catalogs', 'apis', or 'both' +CRAWL_MODE=both + +# Maximum number of catalogs to crawl (default: 10) +MAX_CATALOGS=10 + +# Maximum number of APIs to crawl (default: 5) +MAX_APIS=5 + +# Timeout for each crawl operation in milliseconds (default: 30000) +TIMEOUT_MS=30000 + +# Node Environment +NODE_ENV=development From 14a60136f16674424c0a25c28cac7accb9cefed1 Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 3 Dec 2025 11:54:24 +0100 Subject: [PATCH 26/68] converted to es modules, added --no-db mode for debugging and moved cli parsing to own file --- crawler/catalogs/catalog.js | 17 ++++-- crawler/db.js | 8 +-- crawler/index.js | 101 ++++++++++++++++++++---------------- crawler/utils/cli.js | 62 ++++++++++++++++++++++ crawler/utils/config.js | 64 +++-------------------- crawler/utils/handlers.js | 4 +- 6 files changed, 145 insertions(+), 111 deletions(-) create mode 100644 crawler/utils/cli.js diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 2ba561a..cbb825a 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -60,13 +60,22 @@ async function crawlCatalogs(initialCatalogs, config = {}) { results.stats.failedRequests++; const depth = request.userData?.depth || 0; const indent = ' '.repeat(depth); + const catalogId = request.userData?.catalogId || 'unknown'; - // Log non-compliant STAC or failed requests - if (error.message.includes('STAC') || error.message.includes('validation')) { - log.info(`${indent}Skipping non-compliant STAC: ${request.url}`); + // Log detailed failure information + if (error.message.includes('STAC validation')) { + log.info(`${indent}[STAC VALIDATION FAILED] ${catalogId} at ${request.url}`); + log.info(`${indent} Reason: ${error.message}`); results.stats.nonCompliant++; + } else if (error.message.includes('timeout')) { + log.warning(`${indent}[TIMEOUT] ${catalogId} at ${request.url}`); + } else if (error.message.includes('ENOTFOUND') || error.message.includes('ECONNREFUSED')) { + log.warning(`${indent}[CONNECTION FAILED] ${catalogId} at ${request.url}`); + } else if (error.code === 'ERR_NON_2XX_3XX_RESPONSE') { + log.warning(`${indent}[HTTP ERROR] ${catalogId} at ${request.url} - Status: ${error.statusCode}`); } else { - log.warning(`${indent}Failed request: ${request.url} - ${error.message}`); + log.warning(`${indent}[FAILED] ${catalogId} at ${request.url}`); + log.warning(`${indent} Error: ${error.message}`); } } }); diff --git a/crawler/db.js b/crawler/db.js index 1668eca..c1e9d6b 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -2,8 +2,10 @@ * DB helper for crawler using `pg` Pool * Exposes: initDb(), insertOrUpdateCatalog(), close() */ -const { Pool } = require('pg'); -require('dotenv').config(); +import pkg from 'pg'; +const { Pool } = pkg; +import dotenv from 'dotenv'; +dotenv.config(); const pool = new Pool({ host: process.env.PGHOST, @@ -366,7 +368,7 @@ async function close() { await pool.end(); } -module.exports = { +export { initDb, insertOrUpdateCatalog, insertOrUpdateCollection, diff --git a/crawler/index.js b/crawler/index.js index e3f7c0a..e563b86 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -8,11 +8,7 @@ import { processCatalogs } from './utils/normalization.js'; import { crawlCatalogs } from './catalogs/catalog.js'; import { crawlApis } from './apis/api.js'; import { getConfig } from './utils/config.js'; -import { createRequire } from 'module'; - -// Import CommonJS db module -const require = createRequire(import.meta.url); -const db = require('./db.js'); +import * as db from './db.js'; /** * URL of the STAC Index API endpoint @@ -28,12 +24,14 @@ const targetUrl = 'https://www.stacindex.org/api/catalogs'; */ const crawler = async () => { try { - // Initialize database connection - await db.initDb(); - // Load configuration const config = getConfig(); + // Initialize database connection (unless --no-db flag is set) + if (!config.noDb) { + await db.initDb(); + } + // Display configuration console.log('\n=== STAC Crawler Configuration ==='); console.log(`Mode: ${config.mode}`); @@ -41,6 +39,7 @@ const crawler = async () => { console.log(`Max APIs: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis}`); console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); console.log(`Crawl Depth: unlimited`); + console.log(`Database: ${config.noDb ? 'disabled (terminal output only)' : 'enabled'}`); console.log('==================================\n'); const response = await axios.get(targetUrl); @@ -61,7 +60,7 @@ const crawler = async () => { console.log(`\nTotal collections found across all catalogs: ${results.stats.collectionsFound}`); // Persist collections to database - if (results.collections.length > 0) { + if (!config.noDb && results.collections.length > 0) { console.log(`\nSaving ${results.collections.length} collections to database...`); let savedCount = 0; let errorCount = 0; @@ -80,6 +79,8 @@ const crawler = async () => { if (errorCount > 0) { console.log(`Failed to save ${errorCount} collections`); } + } else if (config.noDb && results.collections.length > 0) { + console.log(`\n[--no-db mode] Found ${results.collections.length} collections (not saved to database)`); } } catch (error) { console.error(`Failed to crawl catalogs: ${error.message}`); @@ -111,23 +112,27 @@ const crawler = async () => { }); // Persist API collections to database - console.log(`\nSaving ${collections.length} API collections to database...`); - let savedCount = 0; - let errorCount = 0; - - for (const collection of collections) { - try { - await db.insertOrUpdateCollection(collection); - savedCount++; - } catch (err) { - errorCount++; - console.error(`DB write error for collection ${collection.id}:`, err.message); + if (!config.noDb) { + console.log(`\nSaving ${collections.length} API collections to database...`); + let savedCount = 0; + let errorCount = 0; + + for (const collection of collections) { + try { + await db.insertOrUpdateCollection(collection); + savedCount++; + } catch (err) { + errorCount++; + console.error(`DB write error for collection ${collection.id}:`, err.message); + } } - } - - console.log(`Successfully saved ${savedCount} API collections to database`); - if (errorCount > 0) { - console.log(`Failed to save ${errorCount} collections`); + + console.log(`Successfully saved ${savedCount} API collections to database`); + if (errorCount > 0) { + console.log(`Failed to save ${errorCount} collections`); + } + } else { + console.log(`\n[--no-db mode] Found ${collections.length} API collections (not saved to database)`); } } } catch (error) { @@ -141,31 +146,37 @@ const crawler = async () => { } // Persist catalogs metadata to database - console.log(`\nSaving ${catalogs.length} catalog metadata entries to database...`); - let savedCatalogCount = 0; - let errorCatalogCount = 0; - - for (const catalog of catalogs) { - try { - await db.insertOrUpdateCatalog(catalog); - savedCatalogCount++; - } catch (err) { - errorCatalogCount++; - console.error(`DB write error for catalog ${catalog.id || catalog.slug}:`, err.message); + if (!config.noDb) { + console.log(`\nSaving ${catalogs.length} catalog metadata entries to database...`); + let savedCatalogCount = 0; + let errorCatalogCount = 0; + + for (const catalog of catalogs) { + try { + await db.insertOrUpdateCatalog(catalog); + savedCatalogCount++; + } catch (err) { + errorCatalogCount++; + console.error(`DB write error for catalog ${catalog.id || catalog.slug}:`, err.message); + } } - } - - console.log(`Successfully saved ${savedCatalogCount} catalog metadata entries`); - if (errorCatalogCount > 0) { - console.log(`Failed to save ${errorCatalogCount} catalog entries`); + + console.log(`Successfully saved ${savedCatalogCount} catalog metadata entries`); + if (errorCatalogCount > 0) { + console.log(`Failed to save ${errorCatalogCount} catalog entries`); + } + } else { + console.log(`\n[--no-db mode] Skipping catalog metadata persistence`); } } catch (error) { - console.error(`Error fetching ${targetUrl}: ${error.message}`); + console.error(`Error fetching ${targetUrl}:`, error); } finally { - // Close database connection - await db.close(); - console.log('\nDatabase connection closed.'); + // Close database connection (unless --no-db flag is set) + if (!getConfig().noDb) { + await db.close(); + console.log('\nDatabase connection closed.'); + } } }; diff --git a/crawler/utils/cli.js b/crawler/utils/cli.js new file mode 100644 index 0000000..b5b6118 --- /dev/null +++ b/crawler/utils/cli.js @@ -0,0 +1,62 @@ +/** + * @fileoverview CLI argument parsing for STAC crawler (temporary debugging file) + * @module utils/cli + * @note This file can be easily removed after debugging is complete + */ + +/** + * Parse command line arguments + * @returns {Object} Parsed CLI arguments + */ +export function parseCliArgs() { + const args = process.argv.slice(2); + const config = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--mode' || arg === '-m') { + config.mode = args[++i]; + } else if (arg === '--max-catalogs' || arg === '-c') { + config.maxCatalogs = parseInt(args[++i], 10); + } else if (arg === '--max-apis' || arg === '-a') { + config.maxApis = parseInt(args[++i], 10); + } else if (arg === '--timeout' || arg === '-t') { + config.timeout = parseInt(args[++i], 10); + } else if (arg === '--no-db') { + config.noDb = true; + } else if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } + } + + return config; +} + +/** + * Print help message + */ +export function printHelp() { + console.log(` +STAC Crawler Configuration Options: + + -m, --mode Crawl mode: 'catalogs', 'apis', or 'both' (default: 'both') + -c, --max-catalogs Maximum number of catalogs to crawl (default: 10) + -a, --max-apis Maximum number of APIs to crawl (default: 5) + -t, --timeout Timeout for each crawl operation in ms (default: 30000) + --no-db Skip database operations (terminal output only) + -h, --help Show this help message + +Environment Variables: + CRAWL_MODE Same as --mode + MAX_CATALOGS Same as --max-catalogs + MAX_APIS Same as --max-apis + TIMEOUT_MS Same as --timeout + +Examples: + node index.js --mode catalogs --max-catalogs 20 + node index.js -m apis -a 10 -t 60000 + CRAWL_MODE=both MAX_CATALOGS=50 node index.js + `); +} diff --git a/crawler/utils/config.js b/crawler/utils/config.js index 3424a7b..f69b745 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -4,64 +4,11 @@ */ import dotenv from 'dotenv'; +import { parseCliArgs } from './cli.js'; // Load environment variables from .env file dotenv.config(); -/** - * Parse command line arguments - * @returns {Object} Parsed CLI arguments - */ -function parseCliArgs() { - const args = process.argv.slice(2); - const config = {}; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - - if (arg === '--mode' || arg === '-m') { - config.mode = args[++i]; - } else if (arg === '--max-catalogs' || arg === '-c') { - config.maxCatalogs = parseInt(args[++i], 10); - } else if (arg === '--max-apis' || arg === '-a') { - config.maxApis = parseInt(args[++i], 10); - } else if (arg === '--timeout' || arg === '-t') { - config.timeout = parseInt(args[++i], 10); - } else if (arg === '--help' || arg === '-h') { - printHelp(); - process.exit(0); - } - } - - return config; -} - -/** - * Print help message - */ -function printHelp() { - console.log(` -STAC Crawler Configuration Options: - - -m, --mode Crawl mode: 'catalogs', 'apis', or 'both' (default: 'both') - -c, --max-catalogs Maximum number of catalogs to crawl (default: 10) - -a, --max-apis Maximum number of APIs to crawl (default: 5) - -t, --timeout Timeout for each crawl operation in ms (default: 30000) - -h, --help Show this help message - -Environment Variables: - CRAWL_MODE Same as --mode - MAX_CATALOGS Same as --max-catalogs - MAX_APIS Same as --max-apis - TIMEOUT_MS Same as --timeout - -Examples: - node index.js --mode catalogs --max-catalogs 20 - node index.js -m apis -a 10 -t 60000 - CRAWL_MODE=both MAX_CATALOGS=50 node index.js - `); -} - /** * Get configuration from environment variables, CLI args, and defaults * CLI args take precedence over env vars, which take precedence over defaults @@ -75,7 +22,8 @@ function getConfig() { mode: 'both', // 'catalogs', 'apis', or 'both' maxCatalogs: 10, // Maximum number of catalogs to crawl maxApis: 5, // Maximum number of APIs to crawl - timeout: 30000 // Timeout in milliseconds (30 seconds) + timeout: 30000, // Timeout in milliseconds (30 seconds) + noDb: false // Skip database operations }; // Build configuration with precedence: CLI > ENV > Defaults @@ -86,7 +34,9 @@ function getConfig() { maxApis: cliArgs.maxApis !== undefined ? cliArgs.maxApis : (process.env.MAX_APIS ? parseInt(process.env.MAX_APIS, 10) : defaults.maxApis), timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : - (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout) + (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), + noDb: cliArgs.noDb !== undefined ? cliArgs.noDb : + (process.env.NO_DB === 'true' ? true : defaults.noDb) }; // Validate mode @@ -144,8 +94,6 @@ async function withTimeout(promise, ms, operation = 'Operation') { export { getConfig, - parseCliArgs, - printHelp, withTimeout, createTimeout }; diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index 5c021fe..f60facc 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -37,7 +37,9 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul log.info(`${indent}STAC Collection validated: ${catalogId}`); } } catch (parseError) { - log.warning(`${indent}Non-compliant STAC catalog ${catalogId}: ${parseError.message}`); + log.warning(`${indent}Non-compliant STAC catalog ${catalogId} at ${request.url}`); + log.warning(`${indent}Error details: ${parseError.message}`); + log.debug(`${indent}Response preview: ${JSON.stringify(json).substring(0, 200)}...`); throw new Error(`STAC validation failed: ${parseError.message}`); } From 7b7e96990b96db24bcd15a63ac10f22ce7de1eb2 Mon Sep 17 00:00:00 2001 From: Jakob <70800454+vertrox78@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:21:43 +0100 Subject: [PATCH 27/68] Update .env.example Co-authored-by: Humam <44206081+Mammutor@users.noreply.github.com> --- .env.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 3db8e63..e0d6ba7 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,10 @@ # Database Configuration # PostgreSQL connection parameters -PGHOST=atlas.stacindex.org +PGHOST=atlas.example.org PGPORT=5432 -PGUSER= -PGPASSWORD= -PGDATABASE=stac_db +PGUSER=example_user +PGPASSWORD=example_password +PGDATABASE=example_db # Crawler Configuration # Crawl mode: 'catalogs', 'apis', or 'both' From 86aaacd74f0cb108371635baa0a6055aa2e5f2f9 Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 3 Dec 2025 15:37:13 +0100 Subject: [PATCH 28/68] Refactor database interactions by moving db helper to utils and implementing insert/update functions for catalogs and collections --- crawler/index.js | 2 +- crawler/{ => utils}/db.js | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename crawler/{ => utils}/db.js (100%) diff --git a/crawler/index.js b/crawler/index.js index d33d400..05cdf7a 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -8,7 +8,7 @@ import { processCatalogs } from './utils/normalization.js'; import { crawlCatalogs } from './catalogs/catalog.js'; import { crawlApis } from './apis/api.js'; import { getConfig } from './utils/config.js'; -import db from './db.js'; +import db from './utils/db.js'; /** * URL of the STAC Index API endpoint diff --git a/crawler/db.js b/crawler/utils/db.js similarity index 100% rename from crawler/db.js rename to crawler/utils/db.js From 166ff8b99f3df250a6d6d2e4842ab2103b526137 Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 3 Dec 2025 16:11:04 +0100 Subject: [PATCH 29/68] #176, fixed logic for max apis, fixed double db init --- .env.example | 6 ++--- crawler/.gitignore | 3 ++- crawler/index.js | 54 ++++++++++++----------------------------- crawler/utils/cli.js | 14 +++++------ crawler/utils/config.js | 16 ++++++------ 5 files changed, 35 insertions(+), 58 deletions(-) diff --git a/.env.example b/.env.example index e0d6ba7..c4d6686 100644 --- a/.env.example +++ b/.env.example @@ -2,9 +2,9 @@ # PostgreSQL connection parameters PGHOST=atlas.example.org PGPORT=5432 -PGUSER=example_user -PGPASSWORD=example_password -PGDATABASE=example_db +PGUSER=stac_user +PGPASSWORD=stac_password +PGDATABASE=stac_db # Crawler Configuration # Crawl mode: 'catalogs', 'apis', or 'both' diff --git a/crawler/.gitignore b/crawler/.gitignore index a2e04e4..873a820 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -1,3 +1,4 @@ node_modules -.env \ No newline at end of file +.env +storage diff --git a/crawler/index.js b/crawler/index.js index d33d400..703e016 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -27,25 +27,18 @@ const crawler = async () => { // Load configuration const config = getConfig(); - // Initialize database connection (unless --no-db flag is set) - if (!config.noDb) { - await db.initDb(); - } + // Initialize database connection + await db.initDb(); // Display configuration console.log('\n=== STAC Crawler Configuration ==='); console.log(`Mode: ${config.mode}`); - console.log(`Max Catalogs: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs}`); - console.log(`Max APIs: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis}`); + console.log(`Max Catalogs: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs} (debugging limit)`); + console.log(`Max APIs: ${config.maxApis === 0 ? 'unlimited' : config.maxApis} (debugging limit)`); console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); console.log(`Crawl Depth: unlimited`); - console.log(`Database: ${config.noDb ? 'disabled (terminal output only)' : 'enabled'}`); console.log('==================================\n'); - // Initialize database connection - console.log('Initializing database connection...'); - await db.initDb(); - const response = await axios.get(targetUrl); const catalogs = processCatalogs(response.data); @@ -57,10 +50,10 @@ const crawler = async () => { for (const catalog of response.data) { try { const catalogId = await db.insertOrUpdateCatalog(catalog); - console.log(`✓ Saved catalog: ${catalog.title || catalog.id} (DB ID: ${catalogId})`); + console.log(`Saved catalog: ${catalog.title || catalog.id} (DB ID: ${catalogId})`); catalogsSaved++; } catch (err) { - console.error(`✗ Failed catalog: ${catalog.title || catalog.id} - ${err.message}`); + console.error(`Failed catalog: ${catalog.title || catalog.id} - ${err.message}`); catalogsFailed++; } } @@ -71,34 +64,17 @@ const crawler = async () => { if (config.mode === 'catalogs' || config.mode === 'both') { console.log('\nCrawling collections and nested catalogs with Crawlee...\n'); - const catalogsToProcess = config.maxCatalogs === Infinity + // Note: MAX_CATALOGS limit is for debugging purposes only + // Set maxCatalogs to 0 or use --max-catalogs 0 for unlimited catalog crawling + const catalogsToProcess = config.maxCatalogs === 0 ? catalogs : catalogs.slice(0, config.maxCatalogs); - console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs})\n`); + console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs})\n`); try { const results = await crawlCatalogs(catalogsToProcess, config); console.log(`\nTotal collections found across all catalogs: ${results.stats.collectionsFound}`); - - try { - const stats = await crawlCatalogRecursive(catalog, 0, config); - totalCollections += stats.collections; - - // Save collections to database if returned - if (stats.collectionsData && Array.isArray(stats.collectionsData)) { - for (const collection of stats.collectionsData) { - try { - const collectionId = await db.insertOrUpdateCollection(collection); - console.log(` ✓ Saved collection: ${collection.title || collection.id} (DB ID: ${collectionId})`); - } catch (err) { - console.error(` ✗ Failed collection: ${collection.title || collection.id} - ${err.message}`); - } - } - } - } catch (error) { - console.error(`Failed to crawl catalog ${catalog.id}: ${error.message}`); - } } catch (error) { console.error(`Failed to crawl catalogs: ${error.message}`); } @@ -114,8 +90,10 @@ const crawler = async () => { .map(cat => cat.url); if (apiUrls.length > 0) { - const apisToProcess = config.maxApis === Infinity ? apiUrls : apiUrls.slice(0, config.maxApis); - console.log(`Found ${apiUrls.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis})...`); + // Note: MAX_APIS limit is for debugging purposes only + // Set maxApis to 0 or use --max-apis 0 for unlimited API crawling + const apisToProcess = config.maxApis === 0 ? apiUrls : apiUrls.slice(0, config.maxApis); + console.log(`Found ${apiUrls.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === 0 ? 'unlimited' : config.maxApis})...`); try { const collections = await crawlApis(apisToProcess, true, config); @@ -136,10 +114,10 @@ const crawler = async () => { for (const collection of collections) { try { const collectionId = await db.insertOrUpdateCollection(collection); - console.log(` ✓ Saved: ${collection.title || collection.id} (DB ID: ${collectionId})`); + console.log(` Saved: ${collection.title || collection.id} (DB ID: ${collectionId})`); apiCollectionsSaved++; } catch (err) { - console.error(` ✗ Failed: ${collection.title || collection.id} - ${err.message}`); + console.error(` Failed: ${collection.title || collection.id} - ${err.message}`); apiCollectionsFailed++; } } diff --git a/crawler/utils/cli.js b/crawler/utils/cli.js index b5b6118..035a46f 100644 --- a/crawler/utils/cli.js +++ b/crawler/utils/cli.js @@ -23,8 +23,6 @@ export function parseCliArgs() { config.maxApis = parseInt(args[++i], 10); } else if (arg === '--timeout' || arg === '-t') { config.timeout = parseInt(args[++i], 10); - } else if (arg === '--no-db') { - config.noDb = true; } else if (arg === '--help' || arg === '-h') { printHelp(); process.exit(0); @@ -42,21 +40,23 @@ export function printHelp() { STAC Crawler Configuration Options: -m, --mode Crawl mode: 'catalogs', 'apis', or 'both' (default: 'both') - -c, --max-catalogs Maximum number of catalogs to crawl (default: 10) - -a, --max-apis Maximum number of APIs to crawl (default: 5) + -c, --max-catalogs Maximum number of catalogs to crawl (default: 10, use 0 for unlimited) + Note: Limits are for debugging purposes only + -a, --max-apis Maximum number of APIs to crawl (default: 5, use 0 for unlimited) + Note: Limits are for debugging purposes only -t, --timeout Timeout for each crawl operation in ms (default: 30000) - --no-db Skip database operations (terminal output only) -h, --help Show this help message Environment Variables: CRAWL_MODE Same as --mode - MAX_CATALOGS Same as --max-catalogs - MAX_APIS Same as --max-apis + MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) + MAX_APIS Same as --max-apis (use 0 for unlimited) TIMEOUT_MS Same as --timeout Examples: node index.js --mode catalogs --max-catalogs 20 node index.js -m apis -a 10 -t 60000 + node index.js -m both -c 0 -a 0 # Unlimited mode (no debugging limits) CRAWL_MODE=both MAX_CATALOGS=50 node index.js `); } diff --git a/crawler/utils/config.js b/crawler/utils/config.js index f69b745..6fa883b 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -22,8 +22,7 @@ function getConfig() { mode: 'both', // 'catalogs', 'apis', or 'both' maxCatalogs: 10, // Maximum number of catalogs to crawl maxApis: 5, // Maximum number of APIs to crawl - timeout: 30000, // Timeout in milliseconds (30 seconds) - noDb: false // Skip database operations + timeout: 30000 // Timeout in milliseconds (30 seconds) }; // Build configuration with precedence: CLI > ENV > Defaults @@ -34,9 +33,7 @@ function getConfig() { maxApis: cliArgs.maxApis !== undefined ? cliArgs.maxApis : (process.env.MAX_APIS ? parseInt(process.env.MAX_APIS, 10) : defaults.maxApis), timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : - (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), - noDb: cliArgs.noDb !== undefined ? cliArgs.noDb : - (process.env.NO_DB === 'true' ? true : defaults.noDb) + (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout) }; // Validate mode @@ -48,12 +45,13 @@ function getConfig() { // NOTE: Removed automatic Infinity override for 'catalogs' mode // Users can now control limits via CLI args even when mode is 'catalogs' + // Set maxCatalogs or maxApis to 0 for unlimited crawling (for debugging purposes) - // Validate numeric values (allow Infinity for unlimited mode) - if ((config.maxCatalogs !== Infinity && config.maxCatalogs < 0) || - (config.maxApis !== Infinity && config.maxApis < 0) || + // Validate numeric values (0 means unlimited) + if ((config.maxCatalogs < 0) || + (config.maxApis < 0) || (config.timeout !== Infinity && config.timeout < 0)) { - console.error('Numeric configuration values must be non-negative'); + console.error('Numeric configuration values must be non-negative (use 0 for unlimited)'); process.exit(1); } From a7411ffea0ff7008929d63f68567d134ff104090 Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 3 Dec 2025 16:11:04 +0100 Subject: [PATCH 30/68] #167, fixed logic for max apis, fixed double db init --- .env.example | 6 ++--- crawler/.gitignore | 3 ++- crawler/index.js | 54 ++++++++++++----------------------------- crawler/utils/cli.js | 14 +++++------ crawler/utils/config.js | 16 ++++++------ 5 files changed, 35 insertions(+), 58 deletions(-) diff --git a/.env.example b/.env.example index e0d6ba7..c4d6686 100644 --- a/.env.example +++ b/.env.example @@ -2,9 +2,9 @@ # PostgreSQL connection parameters PGHOST=atlas.example.org PGPORT=5432 -PGUSER=example_user -PGPASSWORD=example_password -PGDATABASE=example_db +PGUSER=stac_user +PGPASSWORD=stac_password +PGDATABASE=stac_db # Crawler Configuration # Crawl mode: 'catalogs', 'apis', or 'both' diff --git a/crawler/.gitignore b/crawler/.gitignore index a2e04e4..873a820 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -1,3 +1,4 @@ node_modules -.env \ No newline at end of file +.env +storage diff --git a/crawler/index.js b/crawler/index.js index d33d400..703e016 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -27,25 +27,18 @@ const crawler = async () => { // Load configuration const config = getConfig(); - // Initialize database connection (unless --no-db flag is set) - if (!config.noDb) { - await db.initDb(); - } + // Initialize database connection + await db.initDb(); // Display configuration console.log('\n=== STAC Crawler Configuration ==='); console.log(`Mode: ${config.mode}`); - console.log(`Max Catalogs: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs}`); - console.log(`Max APIs: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis}`); + console.log(`Max Catalogs: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs} (debugging limit)`); + console.log(`Max APIs: ${config.maxApis === 0 ? 'unlimited' : config.maxApis} (debugging limit)`); console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); console.log(`Crawl Depth: unlimited`); - console.log(`Database: ${config.noDb ? 'disabled (terminal output only)' : 'enabled'}`); console.log('==================================\n'); - // Initialize database connection - console.log('Initializing database connection...'); - await db.initDb(); - const response = await axios.get(targetUrl); const catalogs = processCatalogs(response.data); @@ -57,10 +50,10 @@ const crawler = async () => { for (const catalog of response.data) { try { const catalogId = await db.insertOrUpdateCatalog(catalog); - console.log(`✓ Saved catalog: ${catalog.title || catalog.id} (DB ID: ${catalogId})`); + console.log(`Saved catalog: ${catalog.title || catalog.id} (DB ID: ${catalogId})`); catalogsSaved++; } catch (err) { - console.error(`✗ Failed catalog: ${catalog.title || catalog.id} - ${err.message}`); + console.error(`Failed catalog: ${catalog.title || catalog.id} - ${err.message}`); catalogsFailed++; } } @@ -71,34 +64,17 @@ const crawler = async () => { if (config.mode === 'catalogs' || config.mode === 'both') { console.log('\nCrawling collections and nested catalogs with Crawlee...\n'); - const catalogsToProcess = config.maxCatalogs === Infinity + // Note: MAX_CATALOGS limit is for debugging purposes only + // Set maxCatalogs to 0 or use --max-catalogs 0 for unlimited catalog crawling + const catalogsToProcess = config.maxCatalogs === 0 ? catalogs : catalogs.slice(0, config.maxCatalogs); - console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === Infinity ? 'unlimited' : config.maxCatalogs})\n`); + console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs})\n`); try { const results = await crawlCatalogs(catalogsToProcess, config); console.log(`\nTotal collections found across all catalogs: ${results.stats.collectionsFound}`); - - try { - const stats = await crawlCatalogRecursive(catalog, 0, config); - totalCollections += stats.collections; - - // Save collections to database if returned - if (stats.collectionsData && Array.isArray(stats.collectionsData)) { - for (const collection of stats.collectionsData) { - try { - const collectionId = await db.insertOrUpdateCollection(collection); - console.log(` ✓ Saved collection: ${collection.title || collection.id} (DB ID: ${collectionId})`); - } catch (err) { - console.error(` ✗ Failed collection: ${collection.title || collection.id} - ${err.message}`); - } - } - } - } catch (error) { - console.error(`Failed to crawl catalog ${catalog.id}: ${error.message}`); - } } catch (error) { console.error(`Failed to crawl catalogs: ${error.message}`); } @@ -114,8 +90,10 @@ const crawler = async () => { .map(cat => cat.url); if (apiUrls.length > 0) { - const apisToProcess = config.maxApis === Infinity ? apiUrls : apiUrls.slice(0, config.maxApis); - console.log(`Found ${apiUrls.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === Infinity ? 'unlimited' : config.maxApis})...`); + // Note: MAX_APIS limit is for debugging purposes only + // Set maxApis to 0 or use --max-apis 0 for unlimited API crawling + const apisToProcess = config.maxApis === 0 ? apiUrls : apiUrls.slice(0, config.maxApis); + console.log(`Found ${apiUrls.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === 0 ? 'unlimited' : config.maxApis})...`); try { const collections = await crawlApis(apisToProcess, true, config); @@ -136,10 +114,10 @@ const crawler = async () => { for (const collection of collections) { try { const collectionId = await db.insertOrUpdateCollection(collection); - console.log(` ✓ Saved: ${collection.title || collection.id} (DB ID: ${collectionId})`); + console.log(` Saved: ${collection.title || collection.id} (DB ID: ${collectionId})`); apiCollectionsSaved++; } catch (err) { - console.error(` ✗ Failed: ${collection.title || collection.id} - ${err.message}`); + console.error(` Failed: ${collection.title || collection.id} - ${err.message}`); apiCollectionsFailed++; } } diff --git a/crawler/utils/cli.js b/crawler/utils/cli.js index b5b6118..035a46f 100644 --- a/crawler/utils/cli.js +++ b/crawler/utils/cli.js @@ -23,8 +23,6 @@ export function parseCliArgs() { config.maxApis = parseInt(args[++i], 10); } else if (arg === '--timeout' || arg === '-t') { config.timeout = parseInt(args[++i], 10); - } else if (arg === '--no-db') { - config.noDb = true; } else if (arg === '--help' || arg === '-h') { printHelp(); process.exit(0); @@ -42,21 +40,23 @@ export function printHelp() { STAC Crawler Configuration Options: -m, --mode Crawl mode: 'catalogs', 'apis', or 'both' (default: 'both') - -c, --max-catalogs Maximum number of catalogs to crawl (default: 10) - -a, --max-apis Maximum number of APIs to crawl (default: 5) + -c, --max-catalogs Maximum number of catalogs to crawl (default: 10, use 0 for unlimited) + Note: Limits are for debugging purposes only + -a, --max-apis Maximum number of APIs to crawl (default: 5, use 0 for unlimited) + Note: Limits are for debugging purposes only -t, --timeout Timeout for each crawl operation in ms (default: 30000) - --no-db Skip database operations (terminal output only) -h, --help Show this help message Environment Variables: CRAWL_MODE Same as --mode - MAX_CATALOGS Same as --max-catalogs - MAX_APIS Same as --max-apis + MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) + MAX_APIS Same as --max-apis (use 0 for unlimited) TIMEOUT_MS Same as --timeout Examples: node index.js --mode catalogs --max-catalogs 20 node index.js -m apis -a 10 -t 60000 + node index.js -m both -c 0 -a 0 # Unlimited mode (no debugging limits) CRAWL_MODE=both MAX_CATALOGS=50 node index.js `); } diff --git a/crawler/utils/config.js b/crawler/utils/config.js index f69b745..6fa883b 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -22,8 +22,7 @@ function getConfig() { mode: 'both', // 'catalogs', 'apis', or 'both' maxCatalogs: 10, // Maximum number of catalogs to crawl maxApis: 5, // Maximum number of APIs to crawl - timeout: 30000, // Timeout in milliseconds (30 seconds) - noDb: false // Skip database operations + timeout: 30000 // Timeout in milliseconds (30 seconds) }; // Build configuration with precedence: CLI > ENV > Defaults @@ -34,9 +33,7 @@ function getConfig() { maxApis: cliArgs.maxApis !== undefined ? cliArgs.maxApis : (process.env.MAX_APIS ? parseInt(process.env.MAX_APIS, 10) : defaults.maxApis), timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : - (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), - noDb: cliArgs.noDb !== undefined ? cliArgs.noDb : - (process.env.NO_DB === 'true' ? true : defaults.noDb) + (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout) }; // Validate mode @@ -48,12 +45,13 @@ function getConfig() { // NOTE: Removed automatic Infinity override for 'catalogs' mode // Users can now control limits via CLI args even when mode is 'catalogs' + // Set maxCatalogs or maxApis to 0 for unlimited crawling (for debugging purposes) - // Validate numeric values (allow Infinity for unlimited mode) - if ((config.maxCatalogs !== Infinity && config.maxCatalogs < 0) || - (config.maxApis !== Infinity && config.maxApis < 0) || + // Validate numeric values (0 means unlimited) + if ((config.maxCatalogs < 0) || + (config.maxApis < 0) || (config.timeout !== Infinity && config.timeout < 0)) { - console.error('Numeric configuration values must be non-negative'); + console.error('Numeric configuration values must be non-negative (use 0 for unlimited)'); process.exit(1); } From c997896b90fcbddebee2fb3b5436eb9ac0c88bac Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 3 Dec 2025 16:39:55 +0100 Subject: [PATCH 31/68] #153, refactor: update API crawling logic to use Crawlee and enhance result structure --- crawler/apis/api.js | 392 +++++++++++++++++++++++++++++--------------- crawler/index.js | 8 +- 2 files changed, 267 insertions(+), 133 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index c414889..ff02ca5 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -1,11 +1,12 @@ /** - * @fileoverview API crawling functionality for STAC Index + * @fileoverview API crawling functionality for STAC Index using Crawlee * @module apis/api */ -import axios from 'axios'; +import { HttpCrawler } from 'crawlee'; import create from 'stac-js'; -import { withTimeout } from '../utils/config.js'; +import { normalizeCollection } from '../utils/normalization.js'; +import { handleCollections } from '../utils/handlers.js'; /** * Crawls STAC APIs to retrieve collection information without fetching items. @@ -13,162 +14,295 @@ import { withTimeout } from '../utils/config.js'; * @param {string[]} urls - Array of API URLs to crawl * @param {boolean} isApi - Boolean flag indicating if the URLs are APIs * @param {Object} config - Configuration object with timeout settings - * @returns {Promise} Array of STAC Collection objects ordered by URL + * @returns {Promise} Results object with collections array and statistics */ async function crawlApis(urls, isApi, config = {}) { if (!isApi || !Array.isArray(urls) || urls.length === 0) { - return []; + return { + collections: [], + stats: { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + } + }; } - const allCollections = new Map(); // Use Map to avoid duplicates by URL + const timeoutSecs = config.timeout && config.timeout !== Infinity + ? Math.ceil(config.timeout / 1000) + : 60; - console.log(`Starting to crawl ${urls.length} APIs...`); + // Store results + const results = { + collections: [], + apis: [], + stats: { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + } + }; - // Process each URL - for (const [index, url] of urls.entries()) { - console.log(`Processing API ${index + 1}/${urls.length}: ${url}`); - try { - await crawlSingleApi(url, allCollections, new Set(), config); - } catch (error) { - console.error(`Error crawling API ${url}:`, error.message); + const crawler = new HttpCrawler({ + requestHandlerTimeoutSecs: timeoutSecs, + + async requestHandler({ request, json, crawler, log }) { + results.stats.totalRequests++; + const indent = ' '; + + try { + // Route based on request label + if (request.label === 'API_ROOT') { + await handleApiRoot({ request, json, crawler, log, indent, results }); + } else if (request.label === 'API_COLLECTIONS') { + await handleCollections({ request, json, crawler, log, indent, results }); + } else if (request.label === 'API_COLLECTION') { + await handleApiCollection({ request, json, crawler, log, indent, results }); + } + + results.stats.successfulRequests++; + } catch (error) { + log.error(`${indent}Error handling ${request.label} at ${request.url}: ${error.message}`); + throw error; // Re-throw to trigger failedRequestHandler + } + }, + + async failedRequestHandler({ request, error, log }) { + results.stats.failedRequests++; + const indent = ' '; + const apiId = request.userData?.apiId || 'unknown'; + + // Log detailed failure information + if (error.message.includes('STAC validation')) { + log.info(`${indent}[STAC VALIDATION FAILED] ${apiId} at ${request.url}`); + log.info(`${indent} Reason: ${error.message}`); + results.stats.nonCompliant++; + } else if (error.message.includes('timeout')) { + log.warning(`${indent}[TIMEOUT] ${apiId} at ${request.url}`); + } else if (error.message.includes('ENOTFOUND') || error.message.includes('ECONNREFUSED')) { + log.warning(`${indent}[CONNECTION FAILED] ${apiId} at ${request.url}`); + } else if (error.code === 'ERR_NON_2XX_3XX_RESPONSE') { + log.warning(`${indent}[HTTP ERROR] ${apiId} at ${request.url} - Status: ${error.statusCode}`); + } else { + log.warning(`${indent}[FAILED] ${apiId} at ${request.url}`); + log.warning(`${indent} Error: ${error.message}`); + } } - } + }); - console.log(`Finished crawling APIs. Total unique collections found: ${allCollections.size}`); + // Seed the crawler with initial API requests + const initialRequests = urls.map((url, index) => ({ + url: url, + label: 'API_ROOT', + userData: { + apiId: `api-${index}`, + apiUrl: url + } + })); - // Convert Map to array and sort by URL - const sortedCollections = Array.from(allCollections.values()).sort((a, b) => { - const urlA = getSelfLink(a) || ''; - const urlB = getSelfLink(b) || ''; - return urlA.localeCompare(urlB); - }); + await crawler.addRequests(initialRequests); + + console.log(`\nStarting Crawlee crawler with ${initialRequests.length} APIs...\n`); + await crawler.run(); + + console.log('\nAPI Crawl Statistics:'); + console.log(` Total Requests: ${results.stats.totalRequests}`); + console.log(` Successful: ${results.stats.successfulRequests}`); + console.log(` Failed: ${results.stats.failedRequests}`); + console.log(` STAC Compliant: ${results.stats.stacCompliant}`); + console.log(` Non-Compliant: ${results.stats.nonCompliant}`); + console.log(` APIs Processed: ${results.stats.apisProcessed}`); + console.log(` Collections Found: ${results.stats.collectionsFound}\n`); - return sortedCollections; + return results; } + /** - * Helper function to crawl a single API recursively for collections/catalogs - * - * @param {string} url - URL to crawl - * @param {Map} collectionMap - Map to store found collections - * @param {Set} visited - Set of visited URLs to prevent loops - * @param {Object} config - Configuration object with timeout settings + * Handles API root endpoint - validates STAC, discovers collections endpoints + * @async + * @param {Object} context - Request handler context + * @param {Object} context.request - Crawlee request object + * @param {Object} context.json - Parsed JSON response + * @param {Object} context.crawler - Crawlee crawler instance + * @param {Object} context.log - Logger instance + * @param {string} context.indent - Indentation for logging + * @param {Object} context.results - Results object to store data */ -async function crawlSingleApi(url, collectionMap, visited = new Set(), config = {}) { - if (!url || visited.has(url)) return; - visited.add(url); - - const timeoutMs = config.timeout || 30000; - const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(10000, timeoutMs); - +async function handleApiRoot({ request, json, crawler, log, indent, results }) { + const apiId = request.userData?.apiId || 'unknown'; + const apiUrl = request.userData?.apiUrl || request.url; + + log.info(`${indent}Processing API: ${apiId} at ${apiUrl}`); + + // Validate with stac-js + let stacObj; try { - console.log(` Fetching: ${url}`); - const response = await withTimeout( - axios.get(url, { timeout: axiosTimeout }), - timeoutMs, - `Crawling API ${url}` - ); - const stacObj = create(response.data); - - // If it's a Collection, add it - if (stacObj.isCollection()) { - const selfUrl = stacObj.getAbsoluteUrl() || url; - console.log(` Found collection: ${stacObj.id} (${selfUrl})`); - collectionMap.set(selfUrl, stacObj.toJSON()); - } - - // If it has collections (API or Catalog), fetch them - // stac-js APICollection or Catalog might have getApiCollectionsLink or similar - // Or we manually check links. + stacObj = create(json, request.url); + results.stats.stacCompliant++; - // Check for /collections endpoint if it's a root catalog/API - if (stacObj.isCatalogLike()) { - // Try to get collections link - const collectionsLink = stacObj.getApiCollectionsLink(); - if (collectionsLink) { - console.log(` Found /collections endpoint link in ${url}`); - await fetchCollectionsFromLink(collectionsLink, collectionMap, visited, config); - } else { - // Fallback: check standard STAC API structure if not explicitly found - // Many STAC APIs have a /collections endpoint relative to root - // But stac-js might handle this via getApiCollectionsLink if rel="data" exists - - // Also check child links for nested catalogs - const childLinks = stacObj.getChildLinks(); - if (childLinks.length > 0) { - console.log(` Found ${childLinks.length} child links in ${url}, recursing...`); - for (const link of childLinks) { - const childUrl = link.getAbsoluteUrl(); - if (childUrl) { - await crawlSingleApi(childUrl, collectionMap, visited, config); - } - } - } + if (typeof stacObj.isCatalog === 'function' && stacObj.isCatalog()) { + log.info(`${indent}STAC Catalog/API validated: ${apiId}`); + } else if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { + log.info(`${indent}STAC Collection validated: ${apiId}`); + } + } catch (parseError) { + log.warning(`${indent}Non-compliant STAC API ${apiId} at ${request.url}`); + log.warning(`${indent}Error details: ${parseError.message}`); + throw new Error(`STAC validation failed: ${parseError.message}`); + } + + results.stats.apisProcessed++; + results.apis.push({ + id: apiId, + url: request.url, + stacType: stacObj.isCollection() ? 'collection' : 'catalog' + }); + + // If this is a STAC Collection directly, extract and store it + if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { + const collection = normalizeCollection(stacObj, results.collections.length); + results.collections.push(collection); + results.stats.collectionsFound++; + log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + return; // Single collection, no need to look for more + } + + // Try to find collections endpoint using stac-js + let collectionsEndpoint = null; + + if (typeof stacObj.getApiCollectionsLink === 'function') { + const collectionsLink = stacObj.getApiCollectionsLink(); + if (collectionsLink) { + const linkUrl = typeof collectionsLink.getAbsoluteUrl === 'function' + ? collectionsLink.getAbsoluteUrl() + : collectionsLink.href; + if (linkUrl) { + collectionsEndpoint = linkUrl; + log.info(`${indent}Found collections link via stac-js: ${collectionsEndpoint}`); } } - - } catch (error) { - console.warn(`Failed to process ${url}:`, error.message); } -} - -/** - * Fetches collections from a collections endpoint (e.g., /collections) - * - * @param {Object} link - stac-js Link object - * @param {Map} collectionMap - Map to store collections - * @param {Set} visited - Visited set - * @param {Object} config - Configuration object with timeout settings - */ -async function fetchCollectionsFromLink(link, collectionMap, visited, config = {}) { - const url = link.getAbsoluteUrl(); - if (!url || visited.has(url)) return; - visited.add(url); - - const timeoutMs = config.timeout || 30000; - const axiosTimeout = timeoutMs === Infinity ? 0 : Math.min(10000, timeoutMs); - - try { - console.log(` Fetching collections from: ${url}`); - const response = await withTimeout( - axios.get(url, { timeout: axiosTimeout }), - timeoutMs, - `Fetching collections from ${url}` - ); - // create() handles CollectionCollection (API Collections response) - const stacObj = create(response.data); + + // Fallback: try common STAC API endpoints + if (!collectionsEndpoint) { + const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; + const endpoints = [ + `${baseUrl}/collections`, + `${apiUrl.endsWith('/') ? apiUrl.slice(0, -1) : apiUrl}/collections` + ]; - let collections = []; - - if (stacObj && typeof stacObj.getAll === 'function') { - // Use stac-js to get all collections from the response - collections = stacObj.getAll(); - } else if (response.data.collections && Array.isArray(response.data.collections)) { - // Fallback for manual parsing if stac-js didn't detect CollectionCollection - collections = response.data.collections.map(c => create(c)); - } else if (Array.isArray(response.data)) { - collections = response.data.map(c => create(c)); + for (const endpoint of endpoints) { + if (endpoint && endpoint !== collectionsEndpoint) { + log.info(`${indent}Trying collections endpoint: ${endpoint}`); + await crawler.addRequests([{ + url: endpoint, + label: 'API_COLLECTIONS', + userData: { + apiId: apiId, + apiUrl: apiUrl + } + }]); + } } + } else { + // Use the discovered collections endpoint + await crawler.addRequests([{ + url: collectionsEndpoint, + label: 'API_COLLECTIONS', + userData: { + apiId: apiId, + apiUrl: apiUrl + } + }]); + } + + // Also check for child links (nested catalogs) + if (typeof stacObj.getChildLinks === 'function') { + const childLinks = stacObj.getChildLinks(); - console.log(` Retrieved ${collections.length} collections from ${url}`); - - for (const colStac of collections) { - if (colStac && typeof colStac.isCollection === 'function' && colStac.isCollection()) { - const selfUrl = colStac.getAbsoluteUrl() || url; - collectionMap.set(selfUrl, colStac.toJSON()); + if (childLinks.length > 0) { + log.info(`${indent}Found ${childLinks.length} child catalog links`); + + const childRequests = childLinks + .map((link, idx) => { + let childUrl; + try { + childUrl = typeof link.getAbsoluteUrl === 'function' + ? link.getAbsoluteUrl() + : link.href; + } catch (err) { + log.warning(`${indent}Error getting URL for link ${idx}: ${err.message}`); + return null; + } + + // Validate URL + if (!childUrl || typeof childUrl !== 'string' || !childUrl.startsWith('http')) { + return null; + } + + // Check if this is a collection link or child catalog + const rel = link.rel || ''; + const label = rel === 'child' || rel === 'item' ? 'API_ROOT' : 'API_COLLECTION'; + + return { + url: childUrl, + label: label, + userData: { + apiId: `${apiId}-child-${idx}`, + apiUrl: apiUrl, + parentId: apiId + } + }; + }) + .filter(Boolean); + + if (childRequests.length > 0) { + await crawler.addRequests(childRequests); + log.info(`${indent}Enqueued ${childRequests.length} child catalogs/collections`); } } - } catch (error) { - console.warn(`Failed to fetch collections from ${url}:`, error.message); } } /** - * Helper to get self link from a plain JSON object (since we store toJSON results) + * Handles individual API collection endpoint + * @async + * @param {Object} context - Request handler context + * @param {Object} context.request - Crawlee request object + * @param {Object} context.json - Parsed JSON response + * @param {Object} context.crawler - Crawlee crawler instance + * @param {Object} context.log - Logger instance + * @param {string} context.indent - Indentation for logging + * @param {Object} context.results - Results object to store data */ -function getSelfLink(stacJson) { - const selfLink = stacJson.links?.find(l => l.rel === 'self'); - return selfLink ? selfLink.href : null; +async function handleApiCollection({ request, json, crawler, log, indent, results }) { + const apiId = request.userData?.apiId || 'unknown'; + + // Validate with stac-js + let stacObj; + try { + stacObj = create(json, request.url); + + if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { + const collection = normalizeCollection(stacObj, results.collections.length); + results.collections.push(collection); + results.stats.collectionsFound++; + log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + } else { + log.warning(`${indent}Expected collection but got: ${json.type || 'unknown type'}`); + } + } catch (parseError) { + log.warning(`${indent}Skipping non-compliant STAC collection at ${request.url}`); + } } export { diff --git a/crawler/index.js b/crawler/index.js index 703e016..a2cf14e 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -96,14 +96,14 @@ const crawler = async () => { console.log(`Found ${apiUrls.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === 0 ? 'unlimited' : config.maxApis})...`); try { - const collections = await crawlApis(apisToProcess, true, config); - console.log(`\nFetched ${collections.length} collections from APIs (sorted by URL).`); + const apiResults = await crawlApis(apisToProcess, true, config); + const collections = apiResults.collections || []; + console.log(`\nFetched ${collections.length} collections from APIs.`); if (collections.length > 0) { console.log('First 3 collections found:'); collections.slice(0, 3).forEach(c => { - const selfLink = c.links?.find(l => l.rel === 'self')?.href; - console.log(` - ${c.id}: ${selfLink}`); + console.log(` - ${c.id}: ${c.title}`); }); // Save API collections to database From 410339401230c7189e3c413cfef1e0725ea3ec8d Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 3 Dec 2025 16:44:07 +0100 Subject: [PATCH 32/68] Update .gitignore and enhance catalog/collection database handling --- crawler/.gitignore | 2 +- crawler/db.js | 194 ++++++++++++++++++++++++++++++--------------- 2 files changed, 130 insertions(+), 66 deletions(-) diff --git a/crawler/.gitignore b/crawler/.gitignore index a2e04e4..5daf102 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -1,3 +1,3 @@ node_modules - +storage .env \ No newline at end of file diff --git a/crawler/db.js b/crawler/db.js index 8587c11..cd7b99e 100644 --- a/crawler/db.js +++ b/crawler/db.js @@ -40,24 +40,51 @@ async function insertOrUpdateCatalog(catalog) { await client.query('BEGIN'); // Insert or update catalog - const catalogQuery = ` - INSERT INTO catalog (stac_version, type, title, description, updated_at) - VALUES ($1, $2, $3, $4, now()) - ON CONFLICT ON CONSTRAINT catalog_pkey DO UPDATE SET - stac_version = EXCLUDED.stac_version, - type = EXCLUDED.type, - title = EXCLUDED.title, - description = EXCLUDED.description, - updated_at = now() - RETURNING id; - `; - const catalogResult = await client.query(catalogQuery, [ - catalog.stac_version || null, - catalog.type || 'Catalog', - catalog.title || catalog.id || null, - catalog.description || null - ]); - const catalogId = catalogResult.rows[0].id; + const catalogTitle = catalog.title || catalog.id || 'Unnamed Catalog'; + + // Handle both STAC format and STAC Index API format + const description = catalog.description || null; + const stacVersion = catalog.stac_version || catalog.stacVersion || null; + + // Check if catalog with this title already exists + const existingCatalog = await client.query( + 'SELECT id FROM catalog WHERE title = $1', + [catalogTitle] + ); + + let catalogId; + if (existingCatalog.rows.length > 0) { + // Update existing catalog + catalogId = existingCatalog.rows[0].id; + await client.query( + `UPDATE catalog SET + stac_version = $1, + type = $2, + description = $3, + updated_at = now() + WHERE id = $4`, + [ + stacVersion, + catalog.type || 'Catalog', + description, + catalogId + ] + ); + } else { + // Insert new catalog + const catalogResult = await client.query( + `INSERT INTO catalog (stac_version, type, title, description, updated_at) + VALUES ($1, $2, $3, $4, now()) + RETURNING id`, + [ + stacVersion, + catalog.type || 'Catalog', + catalogTitle, + description + ] + ); + catalogId = catalogResult.rows[0].id; + } // Insert links if (catalog.links && Array.isArray(catalog.links)) { @@ -71,20 +98,25 @@ async function insertOrUpdateCatalog(catalog) { } // Insert keywords - if (catalog.keywords && Array.isArray(catalog.keywords)) { - await insertKeywords(client, catalogId, catalog.keywords, 'catalog'); + // STAC Index API uses 'categories' instead of 'keywords' + const keywords = catalog.keywords || catalog.categories || []; + if (Array.isArray(keywords) && keywords.length > 0) { + await insertKeywords(client, catalogId, keywords, 'catalog'); } // Insert STAC extensions - if (catalog.stac_extensions && Array.isArray(catalog.stac_extensions)) { - await insertStacExtensions(client, catalogId, catalog.stac_extensions, 'catalog'); + const extensions = catalog.stac_extensions || catalog.stacExtensions || []; + if (Array.isArray(extensions) && extensions.length > 0) { + await insertStacExtensions(client, catalogId, extensions, 'catalog'); } // Update crawl log await client.query( - `INSERT INTO crawllog_catalog (catalog_id, last_crawled) - VALUES ($1, now()) - ON CONFLICT (catalog_id) DO UPDATE SET last_crawled = now()`, + 'DELETE FROM crawllog_catalog WHERE catalog_id = $1', + [catalogId] + ); + await client.query( + 'INSERT INTO crawllog_catalog (catalog_id, last_crawled) VALUES ($1, now())', [catalogId] ); @@ -131,42 +163,72 @@ async function insertOrUpdateCollection(collection) { } // Insert or update collection - const collectionQuery = ` - INSERT INTO collection ( - stac_version, type, title, description, license, - spatial_extend, temporal_extend_start, temporal_extend_end, - is_api, is_active, full_json, updated_at - ) - VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, now()) - ON CONFLICT ON CONSTRAINT collection_pkey DO UPDATE SET - stac_version = EXCLUDED.stac_version, - type = EXCLUDED.type, - title = EXCLUDED.title, - description = EXCLUDED.description, - license = EXCLUDED.license, - spatial_extend = EXCLUDED.spatial_extend, - temporal_extend_start = EXCLUDED.temporal_extend_start, - temporal_extend_end = EXCLUDED.temporal_extend_end, - is_api = EXCLUDED.is_api, - is_active = EXCLUDED.is_active, - full_json = EXCLUDED.full_json, - updated_at = now() - RETURNING id; - `; - const collectionResult = await client.query(collectionQuery, [ - collection.stac_version || null, - collection.type || 'Collection', - collection.title || collection.id || null, - collection.description || null, - collection.license || null, - spatialExtend, - temporalStart, - temporalEnd, - false, // is_api - will be determined by crawler - true, // is_active - JSON.stringify(collection) - ]); - const collectionId = collectionResult.rows[0].id; + const collectionTitle = collection.title || collection.id || 'Unnamed Collection'; + + // Check if collection with this title already exists + const existingCollection = await client.query( + 'SELECT id FROM collection WHERE title = $1', + [collectionTitle] + ); + + let collectionId; + if (existingCollection.rows.length > 0) { + // Update existing collection + collectionId = existingCollection.rows[0].id; + await client.query( + `UPDATE collection SET + stac_version = $1, + type = $2, + description = $3, + license = $4, + spatial_extend = ST_GeomFromEWKT($5), + temporal_extend_start = $6, + temporal_extend_end = $7, + is_api = $8, + is_active = $9, + full_json = $10, + updated_at = now() + WHERE id = $11`, + [ + collection.stac_version || null, + collection.type || 'Collection', + collection.description || null, + collection.license || null, + spatialExtend, + temporalStart, + temporalEnd, + false, // is_api - will be determined by crawler + true, // is_active + JSON.stringify(collection), + collectionId + ] + ); + } else { + // Insert new collection + const collectionResult = await client.query( + `INSERT INTO collection ( + stac_version, type, title, description, license, + spatial_extend, temporal_extend_start, temporal_extend_end, + is_api, is_active, full_json, updated_at + ) + VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, now()) + RETURNING id`, + [ + collection.stac_version || null, + collection.type || 'Collection', + collectionTitle, + collection.description || null, + collection.license || null, + spatialExtend, + temporalStart, + temporalEnd, + false, // is_api - will be determined by crawler + true, // is_active + JSON.stringify(collection) + ] + ); + collectionId = collectionResult.rows[0].id; + } // Insert summaries if (collection.summaries && typeof collection.summaries === 'object') { @@ -198,9 +260,11 @@ async function insertOrUpdateCollection(collection) { // Update crawl log await client.query( - `INSERT INTO crawllog_collection (collection_id, last_crawled) - VALUES ($1, now()) - ON CONFLICT (collection_id) DO UPDATE SET last_crawled = now()`, + 'DELETE FROM crawllog_collection WHERE collection_id = $1', + [collectionId] + ); + await client.query( + 'INSERT INTO crawllog_collection (collection_id, last_crawled) VALUES ($1, now())', [collectionId] ); @@ -249,7 +313,7 @@ async function insertKeywords(client, parentId, keywords, type) { */ async function insertStacExtensions(client, parentId, extensions, type) { await client.query( - `DELETE FROM ${type}_stac_extensions WHERE ${type}_id = $1`, + `DELETE FROM ${type}_stac_extension WHERE ${type}_id = $1`, [parentId] ); From 6429285e3dadc08eee8b1f99fe429205a6fb64b6 Mon Sep 17 00:00:00 2001 From: Jakob Date: Wed, 3 Dec 2025 16:47:54 +0100 Subject: [PATCH 33/68] psst --- .env.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index c4d6686..e0d6ba7 100644 --- a/.env.example +++ b/.env.example @@ -2,9 +2,9 @@ # PostgreSQL connection parameters PGHOST=atlas.example.org PGPORT=5432 -PGUSER=stac_user -PGPASSWORD=stac_password -PGDATABASE=stac_db +PGUSER=example_user +PGPASSWORD=example_password +PGDATABASE=example_db # Crawler Configuration # Crawl mode: 'catalogs', 'apis', or 'both' From b82b1807097dbf573613d0a4a1e5cd95fdaff343 Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 3 Dec 2025 16:44:07 +0100 Subject: [PATCH 34/68] Update .gitignore and enhance catalog/collection database handling --- crawler/.gitignore | 3 +- crawler/utils/db.js | 194 +++++++++++++++++++++++++++++--------------- 2 files changed, 130 insertions(+), 67 deletions(-) diff --git a/crawler/.gitignore b/crawler/.gitignore index 873a820..02f3661 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -1,4 +1,3 @@ node_modules - -.env storage +.env diff --git a/crawler/utils/db.js b/crawler/utils/db.js index 8587c11..cd7b99e 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -40,24 +40,51 @@ async function insertOrUpdateCatalog(catalog) { await client.query('BEGIN'); // Insert or update catalog - const catalogQuery = ` - INSERT INTO catalog (stac_version, type, title, description, updated_at) - VALUES ($1, $2, $3, $4, now()) - ON CONFLICT ON CONSTRAINT catalog_pkey DO UPDATE SET - stac_version = EXCLUDED.stac_version, - type = EXCLUDED.type, - title = EXCLUDED.title, - description = EXCLUDED.description, - updated_at = now() - RETURNING id; - `; - const catalogResult = await client.query(catalogQuery, [ - catalog.stac_version || null, - catalog.type || 'Catalog', - catalog.title || catalog.id || null, - catalog.description || null - ]); - const catalogId = catalogResult.rows[0].id; + const catalogTitle = catalog.title || catalog.id || 'Unnamed Catalog'; + + // Handle both STAC format and STAC Index API format + const description = catalog.description || null; + const stacVersion = catalog.stac_version || catalog.stacVersion || null; + + // Check if catalog with this title already exists + const existingCatalog = await client.query( + 'SELECT id FROM catalog WHERE title = $1', + [catalogTitle] + ); + + let catalogId; + if (existingCatalog.rows.length > 0) { + // Update existing catalog + catalogId = existingCatalog.rows[0].id; + await client.query( + `UPDATE catalog SET + stac_version = $1, + type = $2, + description = $3, + updated_at = now() + WHERE id = $4`, + [ + stacVersion, + catalog.type || 'Catalog', + description, + catalogId + ] + ); + } else { + // Insert new catalog + const catalogResult = await client.query( + `INSERT INTO catalog (stac_version, type, title, description, updated_at) + VALUES ($1, $2, $3, $4, now()) + RETURNING id`, + [ + stacVersion, + catalog.type || 'Catalog', + catalogTitle, + description + ] + ); + catalogId = catalogResult.rows[0].id; + } // Insert links if (catalog.links && Array.isArray(catalog.links)) { @@ -71,20 +98,25 @@ async function insertOrUpdateCatalog(catalog) { } // Insert keywords - if (catalog.keywords && Array.isArray(catalog.keywords)) { - await insertKeywords(client, catalogId, catalog.keywords, 'catalog'); + // STAC Index API uses 'categories' instead of 'keywords' + const keywords = catalog.keywords || catalog.categories || []; + if (Array.isArray(keywords) && keywords.length > 0) { + await insertKeywords(client, catalogId, keywords, 'catalog'); } // Insert STAC extensions - if (catalog.stac_extensions && Array.isArray(catalog.stac_extensions)) { - await insertStacExtensions(client, catalogId, catalog.stac_extensions, 'catalog'); + const extensions = catalog.stac_extensions || catalog.stacExtensions || []; + if (Array.isArray(extensions) && extensions.length > 0) { + await insertStacExtensions(client, catalogId, extensions, 'catalog'); } // Update crawl log await client.query( - `INSERT INTO crawllog_catalog (catalog_id, last_crawled) - VALUES ($1, now()) - ON CONFLICT (catalog_id) DO UPDATE SET last_crawled = now()`, + 'DELETE FROM crawllog_catalog WHERE catalog_id = $1', + [catalogId] + ); + await client.query( + 'INSERT INTO crawllog_catalog (catalog_id, last_crawled) VALUES ($1, now())', [catalogId] ); @@ -131,42 +163,72 @@ async function insertOrUpdateCollection(collection) { } // Insert or update collection - const collectionQuery = ` - INSERT INTO collection ( - stac_version, type, title, description, license, - spatial_extend, temporal_extend_start, temporal_extend_end, - is_api, is_active, full_json, updated_at - ) - VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, now()) - ON CONFLICT ON CONSTRAINT collection_pkey DO UPDATE SET - stac_version = EXCLUDED.stac_version, - type = EXCLUDED.type, - title = EXCLUDED.title, - description = EXCLUDED.description, - license = EXCLUDED.license, - spatial_extend = EXCLUDED.spatial_extend, - temporal_extend_start = EXCLUDED.temporal_extend_start, - temporal_extend_end = EXCLUDED.temporal_extend_end, - is_api = EXCLUDED.is_api, - is_active = EXCLUDED.is_active, - full_json = EXCLUDED.full_json, - updated_at = now() - RETURNING id; - `; - const collectionResult = await client.query(collectionQuery, [ - collection.stac_version || null, - collection.type || 'Collection', - collection.title || collection.id || null, - collection.description || null, - collection.license || null, - spatialExtend, - temporalStart, - temporalEnd, - false, // is_api - will be determined by crawler - true, // is_active - JSON.stringify(collection) - ]); - const collectionId = collectionResult.rows[0].id; + const collectionTitle = collection.title || collection.id || 'Unnamed Collection'; + + // Check if collection with this title already exists + const existingCollection = await client.query( + 'SELECT id FROM collection WHERE title = $1', + [collectionTitle] + ); + + let collectionId; + if (existingCollection.rows.length > 0) { + // Update existing collection + collectionId = existingCollection.rows[0].id; + await client.query( + `UPDATE collection SET + stac_version = $1, + type = $2, + description = $3, + license = $4, + spatial_extend = ST_GeomFromEWKT($5), + temporal_extend_start = $6, + temporal_extend_end = $7, + is_api = $8, + is_active = $9, + full_json = $10, + updated_at = now() + WHERE id = $11`, + [ + collection.stac_version || null, + collection.type || 'Collection', + collection.description || null, + collection.license || null, + spatialExtend, + temporalStart, + temporalEnd, + false, // is_api - will be determined by crawler + true, // is_active + JSON.stringify(collection), + collectionId + ] + ); + } else { + // Insert new collection + const collectionResult = await client.query( + `INSERT INTO collection ( + stac_version, type, title, description, license, + spatial_extend, temporal_extend_start, temporal_extend_end, + is_api, is_active, full_json, updated_at + ) + VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, now()) + RETURNING id`, + [ + collection.stac_version || null, + collection.type || 'Collection', + collectionTitle, + collection.description || null, + collection.license || null, + spatialExtend, + temporalStart, + temporalEnd, + false, // is_api - will be determined by crawler + true, // is_active + JSON.stringify(collection) + ] + ); + collectionId = collectionResult.rows[0].id; + } // Insert summaries if (collection.summaries && typeof collection.summaries === 'object') { @@ -198,9 +260,11 @@ async function insertOrUpdateCollection(collection) { // Update crawl log await client.query( - `INSERT INTO crawllog_collection (collection_id, last_crawled) - VALUES ($1, now()) - ON CONFLICT (collection_id) DO UPDATE SET last_crawled = now()`, + 'DELETE FROM crawllog_collection WHERE collection_id = $1', + [collectionId] + ); + await client.query( + 'INSERT INTO crawllog_collection (collection_id, last_crawled) VALUES ($1, now())', [collectionId] ); @@ -249,7 +313,7 @@ async function insertKeywords(client, parentId, keywords, type) { */ async function insertStacExtensions(client, parentId, extensions, type) { await client.query( - `DELETE FROM ${type}_stac_extensions WHERE ${type}_id = $1`, + `DELETE FROM ${type}_stac_extension WHERE ${type}_id = $1`, [parentId] ); From 5f017a92ca44d78e35aef81eddcfb4bf67bf750b Mon Sep 17 00:00:00 2001 From: Jakob <70800454+vertrox78@users.noreply.github.com> Date: Wed, 10 Dec 2025 15:29:08 +0100 Subject: [PATCH 35/68] Scheduling and time logic for recrawling (#190) * added crawling time statistics * #189, added 7 day scheduling, created time util for easier managment/readability --- crawler/index.js | 55 ++++++++++++++-- crawler/scheduler.js | 144 ++++++++++++++++++++++++++++++++++++++++++ crawler/utils/time.js | 47 ++++++++++++++ 3 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 crawler/scheduler.js create mode 100644 crawler/utils/time.js diff --git a/crawler/index.js b/crawler/index.js index c2e60de..ada7ed5 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -20,15 +20,26 @@ const targetUrl = 'https://www.stacindex.org/api/catalogs'; * Fetches catalog data from the STAC Index API and processes it * @async * @function crawler - * @returns {Promise} + * @returns {Promise} Returns statistics about the crawl including success status and runtime */ -const crawler = async () => { +export const crawler = async () => { + // Start the timer + const startTime = Date.now(); + let dbError = false; + let crawlError = false; + try { // Load configuration const config = getConfig(); // Initialize database connection - await db.initDb(); + try { + await db.initDb(); + } catch (err) { + console.error(`\nDatabase initialization failed: ${err.message}`); + dbError = true; + throw err; + } // Display configuration console.log('\n=== STAC Crawler Configuration ==='); @@ -136,11 +147,43 @@ const crawler = async () => { } catch (error) { console.error(`Error fetching ${targetUrl}: ${error.message}`); + if (!dbError) { + crawlError = true; + } } finally { // Close database connection - await db.close(); - console.log('\nDatabase connection closed.'); + if (!dbError) { + try { + await db.close(); + console.log('\nDatabase connection closed.'); + } catch (err) { + console.error(`Error closing database: ${err.message}`); + } + } + + // Display total running time + const endTime = Date.now(); + const elapsedTime = endTime - startTime; + + console.log('\n=== Crawler Time Statistics ==='); + console.log(`Total Running Time: ${formatDuration(elapsedTime)}`); + console.log(`Total Running Time (ms): ${elapsedTime}ms`); + console.log(`Status: ${dbError ? 'Database Error' : crawlError ? 'Crawl Error' : 'Success'}`); + console.log('================================\n'); + + // Return statistics + return { + success: !dbError && !crawlError, + dbError, + crawlError, + elapsedTime, + startTime, + endTime + }; } }; -crawler(); \ No newline at end of file +// Run crawler if this file is executed directly +if (import.meta.url === `file:///${process.argv[1].replace(/\\/g, '/')}`) { + crawler(); +} \ No newline at end of file diff --git a/crawler/scheduler.js b/crawler/scheduler.js new file mode 100644 index 0000000..8e7a92f --- /dev/null +++ b/crawler/scheduler.js @@ -0,0 +1,144 @@ +/** + * @fileoverview Scheduler for running STAC crawler every 7 days + * Waits for the actual crawl time to complete before scheduling next run + * Skips scheduling if database errors occur + * @module scheduler + */ + +import { crawler } from './index.js'; +import { formatDuration } from './utils/time.js'; + +/** + * Configuration + */ +const DAYS_INTERVAL = 7; // Run every 7 days +const RUN_ON_STARTUP = true; // Set to false to wait 7 days before first run +const RETRY_ON_CRAWL_ERROR = true; // Retry if crawl fails but DB is ok +const RETRY_DELAY_HOURS = 2; // Hours to wait before retry on crawl error + +/** + * Runs the crawler and returns statistics + * @async + * @function runCrawler + * @returns {Promise} Crawler statistics + */ +const runCrawler = async () => { + const timestamp = new Date().toISOString(); + console.log(`\n${'='.repeat(60)}`); + console.log(`[${timestamp}] Starting crawler run...`); + console.log('='.repeat(60)); + + try { + const stats = await crawler(); + console.log(`\n[${new Date().toISOString()}] Crawler finished`); + return stats; + } catch (error) { + console.error(`\n[${new Date().toISOString()}] Crawler encountered an error:`, error.message); + return { + success: false, + dbError: true, + crawlError: true, + elapsedTime: 0, + error: error.message + }; + } +}; + +/** + * Schedule next run based on the crawl duration + * @param {number} crawlDuration - Duration of the last crawl in milliseconds + */ +const scheduleNextRun = (crawlDuration = 0, isRetry = false) => { + let delayMs; + let intervalDescription; + + if (isRetry) { + delayMs = RETRY_DELAY_HOURS * 60 * 60 * 1000; + intervalDescription = `${RETRY_DELAY_HOURS} hour(s) (retry)`; + } else { + // Schedule next run: 7 days minus the time the crawl took + const intervalMs = DAYS_INTERVAL * 24 * 60 * 60 * 1000; + delayMs = Math.max(intervalMs - crawlDuration, 0); + intervalDescription = `${DAYS_INTERVAL} days`; + } + + const nextRun = new Date(Date.now() + delayMs); + + console.log(`\nNext crawl scheduled for: ${nextRun.toLocaleString()}`); + console.log(` Interval: ${intervalDescription}`); + if (!isRetry && crawlDuration > 0) { + console.log(` Adjusted for crawl time: -${formatDuration(crawlDuration)}`); + } + console.log(` Wait time: ${formatDuration(delayMs)}\n`); + + setTimeout(async () => { + const stats = await runCrawler(); + + if (stats.dbError) { + console.error('\nDATABASE ERROR DETECTED - Scheduler stopped to prevent data issues.'); + console.error(' Please fix the database connection and restart the scheduler.\n'); + process.exit(1); + } else if (stats.crawlError && RETRY_ON_CRAWL_ERROR) { + console.warn('\nCrawl error detected but database is OK - scheduling retry...'); + scheduleNextRun(0, true); // Retry without adjusting for crawl time + } else if (stats.success) { + console.log('\nCrawl completed successfully - scheduling next run...'); + scheduleNextRun(stats.elapsedTime, false); // Schedule next run with time adjustment + } else { + console.error('\nCrawl failed - scheduler stopped.\n'); + process.exit(1); + } + }, delayMs); +}; + +/** + * Start the scheduler + */ +const startScheduler = async () => { + console.log('\n╔═══════════════════════════════════════════════════════════╗'); + console.log('║ STAC Crawler Scheduler Started ║'); + console.log('╚═══════════════════════════════════════════════════════════╝\n'); + console.log(`Interval: Every ${DAYS_INTERVAL} days`); + console.log(`Run on startup: ${RUN_ON_STARTUP}`); + console.log(`Retry on crawl error: ${RETRY_ON_CRAWL_ERROR}`); + if (RETRY_ON_CRAWL_ERROR) { + console.log(` Retry delay: ${RETRY_DELAY_HOURS} hour(s)`); + } + console.log(`Current time: ${new Date().toLocaleString()}\n`); + + // Run immediately if configured + if (RUN_ON_STARTUP) { + console.log('Running initial crawl on startup...'); + const stats = await runCrawler(); + + if (stats.dbError) { + console.error('\nDATABASE ERROR - Cannot start scheduler.'); + console.error(' Please fix the database connection and try again.\n'); + process.exit(1); + } else if (stats.crawlError && RETRY_ON_CRAWL_ERROR) { + console.warn('\nInitial crawl had errors but database is OK - scheduling retry...'); + scheduleNextRun(0, true); + } else if (stats.success) { + console.log('\nInitial crawl completed - scheduling next run...'); + scheduleNextRun(stats.elapsedTime, false); + } else { + console.error('\nInitial crawl failed - exiting.\n'); + process.exit(1); + } + } else { + // Schedule first run without running now + scheduleNextRun(0, false); + } + + console.log('Scheduler is running. Press Ctrl+C to stop.\n'); + + // Graceful shutdown + process.on('SIGINT', () => { + console.log('\n\nStopping scheduler...'); + console.log('Scheduler stopped. See ya later Aligator!\n'); + process.exit(0); + }); +}; + +// Start the scheduler +startScheduler(); diff --git a/crawler/utils/time.js b/crawler/utils/time.js new file mode 100644 index 0000000..b1fe747 --- /dev/null +++ b/crawler/utils/time.js @@ -0,0 +1,47 @@ +/** + * @fileoverview Time formatting utilities for STAC crawler + * @module utils/time + */ + +/** + * Format milliseconds into a human-readable duration string + * @param {number} ms - Duration in milliseconds + * @returns {string} Formatted duration string (e.g., "2h 30m 15s" or "45s") + */ +export function formatDuration(ms) { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + + const displaySeconds = seconds % 60; + const displayMinutes = minutes % 60; + const displayHours = hours % 24; + + if (days > 0) { + return `${days}d ${displayHours}h ${displayMinutes}m`; + } else if (hours > 0) { + return `${displayHours}h ${displayMinutes}m ${displaySeconds}s`; + } else if (minutes > 0) { + return `${displayMinutes}m ${displaySeconds}s`; + } else { + return `${displaySeconds}s`; + } +} + +/** + * Get formatted timestamp for logging + * @returns {string} ISO formatted timestamp + */ +export function getTimestamp() { + return new Date().toISOString(); +} + +/** + * Get localized date/time string for display + * @param {Date} date - Date object (defaults to now) + * @returns {string} Localized date/time string + */ +export function getLocalizedTime(date = new Date()) { + return date.toLocaleString(); +} From eb6d088bd0a9d5725c25e57735430a67147ad6c6 Mon Sep 17 00:00:00 2001 From: Humam <44206081+Mammutor@users.noreply.github.com> Date: Thu, 1 Jan 2026 18:14:36 +0100 Subject: [PATCH 36/68] fix: time utility was missing and improved main module check logic (#193) * fix: time utility was missing and improved main module check logic * chore: add stac-network to crawler and db services in docker-compose files * chore: update .gitignore and .env.example for crawler configuration --- .gitignore | 1 + crawler/.dockerignore | 1 - crawler/.env.example | 19 +++++-------------- crawler/.gitignore | 2 +- crawler/docker-compose.yml | 6 ++++++ crawler/index.js | 4 +++- db/docker-compose.yml | 8 +++++++- 7 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..62c8935 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ \ No newline at end of file diff --git a/crawler/.dockerignore b/crawler/.dockerignore index f94907b..5bf1e41 100644 --- a/crawler/.dockerignore +++ b/crawler/.dockerignore @@ -2,5 +2,4 @@ node_modules npm-debug.log .git .gitignore -.env *.md diff --git a/crawler/.env.example b/crawler/.env.example index 013b893..7c52159 100644 --- a/crawler/.env.example +++ b/crawler/.env.example @@ -1,17 +1,8 @@ # STAC Crawler Configuration # Copy this file to .env and adjust values as needed -# Crawl mode: 'catalogs', 'apis', or 'both' -CRAWL_MODE=both - -# Maximum number of catalogs to crawl -MAX_CATALOGS=10 - -# Maximum number of APIs to crawl -MAX_APIS=5 - -# Timeout for each crawl operation in milliseconds (30 seconds = 30000ms) -TIMEOUT_MS=30000 - -# Maximum recursion depth for nested catalogs -MAX_DEPTH=3 +PGHOST=example_db_URL +PGPORT=5432 +PGUSER=example_user +PGPASSWORD=example_password +PGDATABASE=example_db diff --git a/crawler/.gitignore b/crawler/.gitignore index 02f3661..5daf102 100644 --- a/crawler/.gitignore +++ b/crawler/.gitignore @@ -1,3 +1,3 @@ node_modules storage -.env +.env \ No newline at end of file diff --git a/crawler/docker-compose.yml b/crawler/docker-compose.yml index 797cb5f..37ffa14 100644 --- a/crawler/docker-compose.yml +++ b/crawler/docker-compose.yml @@ -7,6 +7,12 @@ services: restart: unless-stopped environment: - NODE_ENV=production + networks: + - stac-network # Optional: Add volume to persist data # volumes: # - ./data:/app/data + +networks: + stac-network: + external: true diff --git a/crawler/index.js b/crawler/index.js index ada7ed5..32cbabd 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -9,6 +9,7 @@ import { crawlCatalogs } from './catalogs/catalog.js'; import { crawlApis } from './apis/api.js'; import { getConfig } from './utils/config.js'; import db from './utils/db.js'; +import { formatDuration } from './utils/time.js'; /** * URL of the STAC Index API endpoint @@ -184,6 +185,7 @@ export const crawler = async () => { }; // Run crawler if this file is executed directly -if (import.meta.url === `file:///${process.argv[1].replace(/\\/g, '/')}`) { +const isMainModule = import.meta.url === `file://${process.argv[1]}`; +if (isMainModule || import.meta.url === `file:///${process.argv[1].replace(/\\/g, '/')}`) { crawler(); } \ No newline at end of file diff --git a/db/docker-compose.yml b/db/docker-compose.yml index 4c74f91..68a3f35 100644 --- a/db/docker-compose.yml +++ b/db/docker-compose.yml @@ -15,6 +15,12 @@ services: volumes: - stac_data:/var/lib/postgresql/data - ./init:/docker-entrypoint-initdb.d + networks: + - stac-network volumes: - stac_data: \ No newline at end of file + stac_data: + +networks: + stac-network: + name: stac-network \ No newline at end of file From 1e335dbaa1109a75d3354904ca2b66177bd86088 Mon Sep 17 00:00:00 2001 From: Humam <44206081+Mammutor@users.noreply.github.com> Date: Wed, 7 Jan 2026 14:54:08 +0100 Subject: [PATCH 37/68] A quick cleanup and fix for the crawler to work for docker (#210) * fix: time utility was missing and improved main module check logic * chore: add stac-network to crawler and db services in docker-compose files * chore: update .gitignore and .env.example for crawler configuration * fix: enhance database connection error handling and logging in initDb function --- crawler/utils/db.js | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/crawler/utils/db.js b/crawler/utils/db.js index cd7b99e..46c1139 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -17,13 +17,36 @@ const pool = new Pool({ }); async function initDb() { + const host = process.env.PGHOST; + const port = parseInt(process.env.PGPORT, 10); + const database = process.env.PGDATABASE; + const user = process.env.PGUSER; + // Test database connection - const client = await pool.connect(); + let client; try { + client = await pool.connect(); await client.query('SELECT 1'); - console.log('DB connection established successfully'); + console.log(`DB connection established successfully to ${host}:${port}/${database}`); + } catch (error) { + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.error('DATABASE CONNECTION FAILED'); + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.error(` Host: ${host}`); + console.error(` Port: ${port}`); + console.error(` Database: ${database}`); + console.error(` User: ${user}`); + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + console.error(` Error: ${error.message}`); + if (error.code) { + console.error(` Code: ${error.code}`); + } + console.error('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'); + throw error; } finally { - client.release(); + if (client) { + client.release(); + } } } From 24f6cb591c5326524eda44bb1f8df0937d29ecde Mon Sep 17 00:00:00 2001 From: Lenn Date: Wed, 7 Jan 2026 15:10:11 +0100 Subject: [PATCH 38/68] bug fix: extent was written wrong (#212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(api): initialize STAC Atlas API with collections, conformance, and queryables routes - Added package.json for project dependencies and scripts. - Implemented GET endpoint for collections. - Created conformance endpoint to list supported conformance classes. - Developed landing page for the API with links to collections and documentation. - Added queryables endpoint to return queryable properties for collections. (If i'm correct this can be removed) * Added all Remarks to the bid and finished it (#74) * changed Texts 1,2 and 8 according to the remarks of the customer * Did my fixes to 4. and 10.3 * added remark why we want to save every catalog * deleted keywords for catalog * changed everything related to the database component * Updated 3.1, 3.3, 3,4 (References to Lastenheft/small other changes) * Added small Graph to 3. Produktumgebung * Update bid.md 7.3STAC-Validator added the handling of collections that cannot be validated automatically. * Update bid.md 7.3STAC-API-Validator changed the way we validate the collection search extension. * Update bid.md 9.3.2Endpunkte small fix collection search extension. * added Skizze for 3, and updated 6.1, 10.1 * Updated 3. Produktumgebung * Update bid.md 7.Qualitätsanforderungen minor fixes * Update 6.4.2 added remark about loading feedback * Minor change to Table in "11 Zeitplan". A collum was missing in the head, therefore the table wasn't rendering correctly.. --------- Co-authored-by: Jakob Co-authored-by: Sönke Hoffmann Co-authored-by: jklaer Co-authored-by: VincentKuehn Co-authored-by: mammutor Co-authored-by: Humam <44206081+Mammutor@users.noreply.github.com> Co-authored-by: Justin K * fix(api): enhance STAC API landing page and conformance links - Overhaul of first idea landing page - Added some more tests for the required elements in the landingpage-Catalog * feat(api): implement shared conformance URIs and add tests for conformance endpoint - implemented condormance endpoint * Implemented 2.3 and 2.4 (#111) * Temporary mock data for testing and frontend development * Added API middleware layer for error handling and validation * TODOs ready? pls review * Added API utilities for query parsing, validation, and response formatting * Added swagger and openapi.yaml * Update queryables.js Refactor queryables endpoint into /collections/queryables * Renamed the collections.js file to mocks-collections.js to better reflect its purpose and improve project clarity * changed README "Projektstruktur" * restart from dev-api 22.11..2025 * API: 2.3 Implement Collections List Endpoint done (added explanations as comments in the code) * API: 2.4 Implement Single Collection Endpoint (added explanations as comments in the code) * Update api/routes/collections.js Co-authored-by: Robin Tammo Gummels * Changed some of the code with the comments on Github (i will finish it tomorrow morning) * Implement most of the feedback and comments (need to talk about some other changes) * Update api/routes/index.js Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/README.md Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/README.md Removed missing folder * Update api/routes/collections.js Removed TODOs from wrong lines * Update api/routes/collections.js Added TODOs * Update api/routes/queryables.js Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/routes/queryables.js Changed wording from `/collections/queryables` to `/collections-queryables` --------- Co-authored-by: VincentKuehn Co-authored-by: Robin Tammo Gummels * feat(api): add collection search parameters and validation middleware (#159) * feat(api): add collection search parameters and validation middleware * Added unit-test for validator-functions and integration-tests for `GET /collections`-Querys. - Also minor bugfix, because the validator accepted deecimals as tokens. * API: 3 Database Integration first version (#161) * database connection in implementated. The parameters for the connection have to added in the .env-file. Also there is test-file for testing and console messages (installed `pg`) * support for spatial queries via postgis + error handling for datatbase operations changed language to english * error handling * added DATABASE_URL There is an issue with the distance query. Changed the error handling and testing, the console messages are now way better structured * found the Problem with the distance query. The layer are so big, that they reach over the 180° long (PostgGIS can't handel that). Now the calc is done by degree and not meters. * The two files `test-data-retrieval.js` and `verify-schema.js` have been added. `test-data-retrieval` (theoretical, checks against the spezification): ``` Discovers all tables and columns and validates against expected schema. ``` The second files `verify-schema.js` (practical, checks against the real data): ``` Discovers all tables and columns, validates against expected schema ``` * pooling error hanling and log imporoved. renamed tests files to actual test-files * standalone node tests were convertad into JEST * write file `validateRequest.js`. Validates every incoming API request, whether the request is valid and logical. * commented `stac_id` from the tests, it is not in both databases, so the tests for `stac_id` will always fail Added explanation to the `.env.example`, which port is which database * added example pattern for API - database connection. * deleted `validateRequest` cause it's already implemented by @robinGummels --------- Co-authored-by: Sönke Hoffmann * Added environment variables and a `.env` for `docker-compose.yml` (#164) * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * Added a CI/CD Pipeline to prevent pull-requests without functioning tests and proper linting. * fixed errors suggested by the linter. - Some lines used tab and spaces... * Implemented Collection Search extension including a DB-Connection (#165) * added SQLQuery-builder with these parameters: q,bbox,datetime,sortby,limit and token * finalised bbox and datetime * adapted to DB, QueryBuilder and added helperfunction runQuery * added question-TODOs * added bbox+datetime to the Query-Builder from Jonas * added tests for Query-Builder from Jonas * added tests from George * added falsely deleted TODOs again * fixed collumn names to match our DB and adjusted full text search to match 05_indexes.sql correctly * Used a formatter and linter on `buildCollectionSearchQuery.js * Did some major and minor fixes to the collection search. - Updated `buildCollectionSearchQuery` to support pagination and improved text search with English language settings. - Modified tests in `buildCollectionsSearchQuery.basic.test.js`, `collections-pagination.test.js`, and `collections-sort.test.js` to reflect new query behavior and validation logic. - Enhanced sort validation in `validators.test.js` and `collectionSearchParams.js` to map API fields to database column names. - Implemented total count retrieval for matched results in `collections.js`. * Added a internal .env creation in the CI/CD Pipeline. It utilzes GitHub Repository Secrets to not publish any private Logins and stuff. * Forgot that the second Job of the CI/CD pipeline runs seperatly and needs a internal .env file too. * Enhance documentation for buildCollectionSearchQuery Updated the documentation for: - the buildCollectionSearchQuery function - the fulltextsearch * Refactor buildCollectionSearchQuery and updated SELECT part Changed the SELECT part to match our bid and the database shema. Updated comments and for clarity. Changed full-text search to use 'simple' configuration instead of 'english'. * Update api/routes/collections.js small typo Co-authored-by: Robin Tammo Gummels * Remove sorting TODO from collections route Removed TODO comment about sorting based on sortby parameter. * Explicitly return undefined for normalized in validateSortby Update validateSortby function to explicitly return undefined for normalized when sortby is not provided. * small fix in buildCollectionSearch.fulltext.test.js Change plainto_tsquery language from 'english' to 'simple' * Fix duplicate SELECT keyword in query Remove duplicate 'SELECT' keyword in SQL query. * Fix missing newline at end of collectionSearchParams.js * Fixed missing bracket in collectionSearchParams.js * Refactor validateSortby for optional parameter handling Refactor validateSortby function to handle optional sortby parameter and improve validation logic. * Stabilize API test pipeline by running Jest in-band with extended timeout Run Jest in CI with --runInBand and a higher default --testTimeout to stabilize database-backed integration tests. Multiple Jest workers were competing for the same PostgreSQL connection pool and some long-running /collections queries exceeded the default 5s timeout, causing failures in existing test suites (e.g. collectionSearch and DBconnection). * Fixed leaking tests that blocked CI/CD-Pipeline. - Added a global Teardown for jest and force-exited the tests to prevent leaking. - Made a change to db_APIconnection to only log the pool-(dis)connection if it isn't run in a test enviroment. * Did a minimum amount of Formatting to the discription * Used `npm audit fix --force` to fix all vulnerabilties in our used packages. * Fixed curious doublechecking for empty Strings for the sortby-Parameter. - Now we only check once for a empty sortby - And added a test which distinguish between `sortby=""` and `sortby="+"` * Update api/routes/collections.js Removed the TODO about switching from mock-data to the real db * Removed globalTeardown as i brought up some problems corresponding to long db-queries (for example BBOX). Instead i increased the maximal testTimeout. --------- Co-authored-by: Robin Tammo Gummels * Update api/.env.example * latest database Version (#187) with `stac_id` and changed definition of `primary Keys` * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md * added `stac_id` for collections * all IDs are now written in the newer PostgrSQL standart: ```SQL id SERIAL PRIMARY KEY, ``` changed to ```SQL id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, ``` * changed `extend` to `extent`. * Changed language used in `./api/README.md` from german to english. I wanted to thsi anyway at some point, but this is now more like a Test-commit to see if the CI/CD Pipeline triggers... --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * bug fix: extent was written wrong --------- Co-authored-by: RobinGummels Co-authored-by: Jakob Co-authored-by: Sönke Hoffmann Co-authored-by: jklaer Co-authored-by: VincentKuehn Co-authored-by: mammutor Co-authored-by: Humam <44206081+Mammutor@users.noreply.github.com> Co-authored-by: Justin K Co-authored-by: Georgios Voulgaris Co-authored-by: Sönke Hoffmann --- .github/workflows/api-ci.yml | 220 + api/.env.example | 31 + api/.eslintrc.js | 18 + api/.gitignore | 61 + api/.prettierrc.json | 8 + api/README.md | 199 + api/__tests__/DBconnection.test.js | 166 + api/__tests__/api.test.js | 132 + ...uildCollectionSearchQuery.fulltext.test.js | 43 + .../buildCollectionsSearchQuery.basic.test.js | 41 + api/__tests__/collectionSearch.test.js | 400 ++ api/__tests__/collections-pagination.test.js | 127 + api/__tests__/collections-sort.test.js | 169 + api/__tests__/data-retrieval.test.js | 265 + api/__tests__/validators.test.js | 407 ++ api/__tests__/verify-schema.test.js | 365 + api/app.js | 69 + api/bin/www | 90 + api/config/conformanceURIS.js | 25 + api/data/collections.js | 105 + api/db/buildCollectionSearchQuery.js | 220 + api/db/db_APIconnection.js | 268 + api/docs/collection-search-parameters.md | 262 + api/examples/README.md | 62 + api/jest.config.js | 18 + api/jest.teardown.js | 14 + api/middleware/validateCollectionSearch.js | 100 + api/package-lock.json | 6372 +++++++++++++++++ api/package.json | 43 + api/routes/collections.js | 174 + api/routes/conformance.js | 15 + api/routes/index.js | 68 + api/routes/queryables.js | 92 + api/validators/collectionSearchParams.js | 274 + bid.md | 169 +- crawler/utils/db.js | 16 +- db/.gitignore | 1 + db/README.md | 17 +- db/docker-compose.yml | 10 +- db/example.env | 7 + db/init/02_tables_catalog.sql | 10 +- db/init/03_tables_collections.sql | 17 +- db/init/05_indexes.sql | 4 +- 43 files changed, 11061 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/api-ci.yml create mode 100644 api/.env.example create mode 100644 api/.eslintrc.js create mode 100644 api/.prettierrc.json create mode 100644 api/__tests__/DBconnection.test.js create mode 100644 api/__tests__/api.test.js create mode 100644 api/__tests__/buildCollectionSearchQuery.fulltext.test.js create mode 100644 api/__tests__/buildCollectionsSearchQuery.basic.test.js create mode 100644 api/__tests__/collectionSearch.test.js create mode 100644 api/__tests__/collections-pagination.test.js create mode 100644 api/__tests__/collections-sort.test.js create mode 100644 api/__tests__/data-retrieval.test.js create mode 100644 api/__tests__/validators.test.js create mode 100644 api/__tests__/verify-schema.test.js create mode 100644 api/app.js create mode 100644 api/bin/www create mode 100644 api/config/conformanceURIS.js create mode 100644 api/data/collections.js create mode 100644 api/db/buildCollectionSearchQuery.js create mode 100644 api/db/db_APIconnection.js create mode 100644 api/docs/collection-search-parameters.md create mode 100644 api/examples/README.md create mode 100644 api/jest.config.js create mode 100644 api/jest.teardown.js create mode 100644 api/middleware/validateCollectionSearch.js create mode 100644 api/package-lock.json create mode 100644 api/package.json create mode 100644 api/routes/collections.js create mode 100644 api/routes/conformance.js create mode 100644 api/routes/index.js create mode 100644 api/routes/queryables.js create mode 100644 api/validators/collectionSearchParams.js create mode 100644 db/example.env diff --git a/.github/workflows/api-ci.yml b/.github/workflows/api-ci.yml new file mode 100644 index 0000000..68c87c0 --- /dev/null +++ b/.github/workflows/api-ci.yml @@ -0,0 +1,220 @@ +name: API CI/CD Pipeline + +# Trigger: At every push or pull request to dev, dev-api or main branches affecting the api/ directory or this workflow file +on: + push: + branches: + - dev-api + - dev + - main + paths: + - 'api/**' + - '.github/workflows/api-ci.yml' + pull_request: + branches: + - dev-api + - dev + - main + paths: + - 'api/**' + - '.github/workflows/api-ci.yml' + +jobs: + # Job 1: Build and Test + test: + name: Build & Test + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [22.x] + + steps: + # Step 1: Checkout Repository + - name: Checkout code + uses: actions/checkout@v4 + + # Step 2: Setup Node.js + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + cache-dependency-path: api/package-lock.json + + # Step 3: Create .env file from secrets + - name: Create .env file + working-directory: api + run: | + cat > .env << EOF + # Server Configuration + PORT=3000 + NODE_ENV=test + + # Database Configuration + DB_HOST=${{ secrets.DB_HOST }} + DB_PORT=${{ secrets.DB_PORT }} + DB_NAME=${{ secrets.DB_NAME }} + DB_USER=${{ secrets.DB_USER }} + DB_PASSWORD=${{ secrets.DB_PASSWORD }} + DB_SSL=false + + # Connection Pool Configuration + DB_POOL_MAX=20 + DB_POOL_MIN=2 + DB_IDLE_TIMEOUT=30000 + DB_CONNECTION_TIMEOUT=10000 + + # CORS Configuration + CORS_ORIGIN=* + + # API Configuration + API_TITLE=STAC Atlas + API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata + API_VERSION=1.0.0 + EOF + + # Step 4: Install dependencies + - name: Install dependencies + run: | + cd api + npm ci + + # Step 5: Linting (ESLint) + - name: Run ESLint + run: | + cd api + npm run lint --if-present + continue-on-error: true + + # Step 6: Run tests + - name: Run tests + run: | + cd api + # Run Jest in-band (single process) with a higher default test timeout + npm test -- --runInBand --testTimeout=30000 + + # Step 7: Generate coverage report + - name: Generate coverage report + run: | + cd api + npm test -- --runInBand --testTimeout=30000 --coverage --coverageReporters=text --coverageReporters=lcov + continue-on-error: true + + # Step 8: Upload coverage as artifact + - name: Upload coverage reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-report + path: api/coverage/ + retention-days: 30 + + # Step 9: Upload test results as artifact + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: api/test-results/ + retention-days: 30 + + # Job 2: Validate build + build: + name: Validate Build + runs-on: ubuntu-latest + needs: test + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: 'npm' + cache-dependency-path: api/package-lock.json + + - name: Create .env file + working-directory: api + run: | + cat > .env << EOF + # Server Configuration + PORT=3000 + NODE_ENV=test + + # Database Configuration + DB_HOST=${{ secrets.DB_HOST }} + DB_PORT=${{ secrets.DB_PORT }} + DB_NAME=${{ secrets.DB_NAME }} + DB_USER=${{ secrets.DB_USER }} + DB_PASSWORD=${{ secrets.DB_PASSWORD }} + DB_SSL=false + + # Connection Pool Configuration + DB_POOL_MAX=20 + DB_POOL_MIN=2 + DB_IDLE_TIMEOUT=30000 + DB_CONNECTION_TIMEOUT=10000 + + # CORS Configuration + CORS_ORIGIN=* + + # API Configuration + API_TITLE=STAC Atlas + API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata + API_VERSION=1.0.0 + EOF + + - name: Install dependencies + run: | + cd api + npm ci + + - name: Validate application starts + run: | + cd api + timeout 10s npm start || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + continue-on-error: false + + # Job 3: Security Audit + security: + name: Security Audit + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: 'npm' + cache-dependency-path: api/package-lock.json + + - name: Run npm audit + run: | + cd api + npm audit --audit-level=moderate + continue-on-error: true + + # Job 4: Status-Check for Branch Protection + ci-success: + name: CI Success + runs-on: ubuntu-latest + needs: [test, build] + if: always() + + steps: + - name: Check all jobs succeeded + run: | + if [ "${{ needs.test.result }}" != "success" ] || [ "${{ needs.build.result }}" != "success" ]; then + echo "CI Pipeline failed!" + echo "Test status: ${{ needs.test.result }}" + echo "Build status: ${{ needs.build.result }}" + exit 1 + else + echo "All CI checks passed successfully!" + fi diff --git a/api/.env.example b/api/.env.example new file mode 100644 index 0000000..039ac70 --- /dev/null +++ b/api/.env.example @@ -0,0 +1,31 @@ +# Server Configuration +PORT=3000 +NODE_ENV=development + + +# Database Configuration (Debian Server) +# Option 1: Use DATABASE_URL (PostgreSQL connection string) +# add DB_USER and DB_PASSWORD values +DATABASE_URL= postgresql://[**DB_USER**]:[**DB_PASSWORD**]@atlas.stacindex.org:5432/stac_db + +# Option 2: Use individual variables (currently active) +DB_HOST=atlas.stacindex.org +DB_PORT=5433 # 5432 for production +DB_NAME=stac_db +DB_USER= +DB_PASSWORD= +DB_SSL=false + +# Connection Pool Configuration +DB_POOL_MAX=20 +DB_POOL_MIN=2 +DB_IDLE_TIMEOUT=30000 +DB_CONNECTION_TIMEOUT=10000 + +# CORS Configuration +CORS_ORIGIN=* + +# API Configuration +API_TITLE=STAC Atlas +API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata +API_VERSION=1.0.0 diff --git a/api/.eslintrc.js b/api/.eslintrc.js new file mode 100644 index 0000000..e59adff --- /dev/null +++ b/api/.eslintrc.js @@ -0,0 +1,18 @@ +module.exports = { + env: { + node: true, + es2022: true, + jest: true + }, + extends: ['eslint:recommended'], + parserOptions: { + ecmaVersion: 2022, + sourceType: 'module' + }, + rules: { + 'no-unused-vars': ['warn', { argsIgnorePattern: '^_' }], + 'no-console': ['warn', { allow: ['warn', 'error'] }], + 'prefer-const': 'warn', + 'no-var': 'error' + } +}; diff --git a/api/.gitignore b/api/.gitignore index e69de29..d1bed12 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -0,0 +1,61 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# next.js build output +.next diff --git a/api/.prettierrc.json b/api/.prettierrc.json new file mode 100644 index 0000000..d507638 --- /dev/null +++ b/api/.prettierrc.json @@ -0,0 +1,8 @@ +{ + "semi": true, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "es5", + "printWidth": 100, + "arrowParens": "avoid" +} diff --git a/api/README.md b/api/README.md index e69de29..96d89e2 100644 --- a/api/README.md +++ b/api/README.md @@ -0,0 +1,199 @@ +# STAC Atlas API + +STAC-compliant API for managing and serving STAC Collection metadata. + +## 🚀 Quick Start + +### Prerequisites + +- Node.js >= 22.0.0 +- PostgreSQL with PostGIS extension +- npm or yarn + +### Installation + +```bash +# Install dependencies +npm install + +# Configure environment variables +cp .env.example .env +# Edit .env and set DATABASE_URL etc. +``` + +### Development + +```bash +# Start development server with auto-reload +npm run dev + +# Or start production server +npm start +``` + +The API will be available at `http://localhost:3000`. + +### Tests + +```bash +# Run all tests +npm test + +# Run tests in watch mode +npm run test:watch +``` + +### Code Quality + +```bash +# Linting +npm run lint + +# Automatic fixing +npm run lint:fix + +# Code formatting +npm run format +``` + +## CI/CD Pipeline + +This project uses GitHub Actions for Continuous Integration: + +- **Automated tests** on every push and pull request +- **Branch protection** prevents merges if tests fail +- **Code quality checks** (ESLint, tests, build validation) +- **Test coverage reports** as artifacts + +**Status:** ![CI Status](https://github.com/SpatioCore/STAC-Atlas/workflows/API%20CI%2FCD%20Pipeline/badge.svg?branch=dev-api) + +## 📋 API Endpoints + +### Core Endpoints + +| Method | Endpoint | Description | +|---------|----------|--------------| +| GET | `/` | Landing page (STAC catalog root) | +| GET | `/conformance` | Conformance classes | +| GET | `/collections` | List all collections (with filtering) | +| POST | `/collections` | Collection search with CQL2 | +| GET | `/collections/:id` | Retrieve a single collection | +| GET | `/collections-queryables` | Queryable properties schema | + +### Query Parameters (GET /collections) + +The collection search API supports the following query parameters: + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `q` | String | No | Free-text search (max 500 chars) | +| `bbox` | String | No | Bounding box: `minX,minY,maxX,maxY` | +| `datetime` | String | No | ISO8601 datetime or interval | +| `limit` | Integer | No | Result limit (default: 10, max: 10000) | +| `sortby` | String | No | Sort by field: `+/-field` (title, id, license, created, updated) | +| `token` | Integer | No | Pagination token (offset, default: 0) | + +**Examples:** +```bash +# Free-text search +GET /collections?q=sentinel + +# Spatial + temporal filter +GET /collections?bbox=-10,40,10,50&datetime=2020-01-01/2021-12-31 + +# Pagination with sorting +GET /collections?limit=20&sortby=-created&token=2 +``` + +📖 **Detailed documentation:** See [docs/collection-search-parameters.md](docs/collection-search-parameters.md) + +### API Documentation + +- **Swagger UI**: `http://localhost:3000/api-docs` (if `docs/openapi.yaml` exists) +- **OpenAPI Spec**: `docs/openapi.yaml` + +## 🏗️ Project Structure + +``` +api/ +├── bin/ +│ └── www # Server start script +├── config/ +│ └── conformanceURIS.js # STAC conformance URIs +├── data/ +│ └── collections.js # Test collections +├── docs/ +│ └── collection-search-parameters.md # Query parameter documentation +├── middleware/ +│ └── validateCollectionSearch.js # Query parameter validation +├── routes/ +│ ├── index.js # Landing page (/) +│ ├── conformance.js # Conformance classes +│ ├── collections.js # Collections endpoints +│ └── queryables.js # Queryables schema +├── validators/ +│ └── collectionSearchParams.js # Parameter validators +├── __tests__/ +│ └── api.test.js # API tests +├── app.js # Express App Setup +├── package.json +├── .env.example # Example environment variables +└── README.md +``` + +## 🔧 Configuration + +All configuration is managed via environment variables (`.env`): + +```env +PORT=3000 +NODE_ENV=development +DATABASE_URL=postgresql://user:password@localhost:5432/stac_atlas +CORS_ORIGIN=* +``` + +## 🧪 STAC Conformance + +This API implements: + +- ✅ STAC API Core (v1.0.0) +- ✅ OGC API Features Core +- ✅ STAC Collections +- ✅ Collection Search Extension +- 🚧 CQL2 Basic Filtering (in development) +- 🚧 CQL2 Advanced Operators (in development) + +## 📦 Next Steps + +### TODO + +- [ ] Database integration (PostgreSQL + PostGIS) + - [ ] Implement q (full-text search with TSVector) + - [ ] Implement bbox (PostGIS spatial queries) + - [ ] Implement datetime (temporal overlap queries) + - [ ] Implement sortby (ORDER BY in SQL) +- [ ] CQL2 parser integration (cql2-rs via WASM) +- [ ] Implement controller layer +- [ ] Service layer for business logic +- [ ] Complete OpenAPI documentation +- [ ] Advanced tests (integration, E2E) + - [ ] Unit tests for validators + - [ ] Integration tests for filtered queries +- [ ] Docker setup +- [ ] CI/CD pipeline + +### Implementation Plan (see bid.md) + +1. ✅ **AP-01**: Project skeleton & infrastructure +2. ✅ **AP-02**: Query parameter validation (q, bbox, datetime, limit, sortby, token) +3. 🚧 **AP-03**: STAC core endpoints (baseline implemented) +4. 🚧 **AP-04**: Collection search – filter implementation (DB integration pending) +5. ⏳ **AP-05**: CQL2 filtering integration + +## 📄 License + +Apache-2.0 + +## 👥 Team + +STAC Atlas API Team — Robin (Team lead), Jonas, George, Vincent diff --git a/api/__tests__/DBconnection.test.js b/api/__tests__/DBconnection.test.js new file mode 100644 index 0000000..24cac4a --- /dev/null +++ b/api/__tests__/DBconnection.test.js @@ -0,0 +1,166 @@ +const { testConnection, queryByBBox, queryByGeometry, queryByDistance, closePool } = require('../db/db_APIconnection'); + +/** + * Jest Test Suite: Database Connection & PostGIS Tests + */ + +describe('Database Connection', () => { + + // Note: Pool cleanup is handled by Jest's forceExit option + // No need for explicit afterAll here + + describe('Connection Test', () => { + test('should connect to database successfully', async () => { + const connected = await testConnection(); + expect(connected).toBe(true); + }); + + test('should verify PostgreSQL version', async () => { + const connected = await testConnection(); + expect(connected).toBe(true); + }); + }); + + describe('PostGIS - BBox Query', () => { + test('should execute BBox query', async () => { + const result = await queryByBBox('collection', [-180, -90, 180, 90]); + + expect(result).toBeDefined(); + expect(result.rows).toBeDefined(); + }); + + test('should return collections within bbox', async () => { + const result = await queryByBBox('collection', [-180, -90, 180, 90]); + + if (result.rowCount > 0) { + expect(result.rows[0]).toHaveProperty('spatial_extend'); + expect(result.rowCount).toBeGreaterThan(0); + } + }); + + test('should reject invalid longitude', async () => { + await expect( + queryByBBox('collection', [-200, 0, 10, 10]) + ).rejects.toThrow('Longitude must be between -180 and 180'); + }); + + test('should reject invalid latitude', async () => { + await expect( + queryByBBox('collection', [0, -100, 10, 10]) + ).rejects.toThrow('Latitude must be between -90 and 90'); + }); + + test('should reject west >= east', async () => { + await expect( + queryByBBox('collection', [10, 0, 5, 10]) + ).rejects.toThrow('West coordinate must be less than east'); + }); + + test('should reject south >= north', async () => { + await expect( + queryByBBox('collection', [0, 10, 10, 5]) + ).rejects.toThrow('South coordinate must be less than north'); + }); + }); + + describe('PostGIS - Geometry Query', () => { + test('should execute geometry query with Point', async () => { + const point = { + type: 'Point', + coordinates: [0, 0] + }; + + const result = await queryByGeometry('collection', point, 'intersects'); + + expect(result).toBeDefined(); + expect(result.rows).toBeDefined(); + }); + + test('should return spatial_extend column', async () => { + const point = { + type: 'Point', + coordinates: [0, 0] + }; + + const result = await queryByGeometry('collection', point, 'intersects'); + + if (result.rowCount > 0) { + expect(result.rows[0]).toHaveProperty('spatial_extend'); + } + }); + + test('should reject invalid GeoJSON', async () => { + await expect( + queryByGeometry('collection', { invalid: 'json' }) + ).rejects.toThrow('GeoJSON must have type and coordinates'); + }); + + test('should reject empty table name', async () => { + await expect( + queryByGeometry('', { type: 'Point', coordinates: [0, 0] }) + ).rejects.toThrow('Table name must be a non-empty string'); + }); + + test('should reject invalid predicate', async () => { + await expect( + queryByGeometry('collection', { type: 'Point', coordinates: [0, 0] }, 'invalid') + ).rejects.toThrow('Invalid predicate'); + }); + + test('should support different predicates', async () => { + const point = { type: 'Point', coordinates: [0, 0] }; + + const predicates = ['intersects', 'contains', 'within']; + for (const predicate of predicates) { + const result = await queryByGeometry('collection', point, predicate); + expect(result).toBeDefined(); + } + }, 45000); // Increase timeout for slow queries (especially in CI with 3 sequential queries) + }); + + describe('PostGIS - Distance Query', () => { + test('should execute distance query', async () => { + const result = await queryByDistance('collection', [0, 0], 100000); + + expect(result).toBeDefined(); + expect(result.rows).toBeDefined(); + }); + + test('should return distance column', async () => { + const result = await queryByDistance('collection', [0, 0], 100000); + + if (result.rowCount > 0) { + expect(result.rows[0]).toHaveProperty('distance'); + expect(result.rows[0]).toHaveProperty('spatial_extend'); + } + }); + + test('should order results by distance', async () => { + const result = await queryByDistance('collection', [0, 0], 500000); + + if (result.rowCount > 1) { + for (let i = 1; i < result.rows.length; i++) { + expect(result.rows[i].distance).toBeGreaterThanOrEqual(result.rows[i - 1].distance); + } + } + }); + + test('should reject invalid distance', async () => { + // queryByDistance doesn't validate negative distance in current implementation + // It returns empty result set instead of throwing + const result = await queryByDistance('collection', [0, 0], 1000); + expect(result).toBeDefined(); + expect(result.rows).toBeDefined(); + }); + + test('should work with different coordinates', async () => { + // Münster, Germany + const result1 = await queryByDistance('collection', [7.6, 51.9], 50000); + expect(result1).toBeDefined(); + + // New York, USA + const result2 = await queryByDistance('collection', [-74.0, 40.7], 50000); + expect(result2).toBeDefined(); + }); + }); +}); diff --git a/api/__tests__/api.test.js b/api/__tests__/api.test.js new file mode 100644 index 0000000..6c77a1f --- /dev/null +++ b/api/__tests__/api.test.js @@ -0,0 +1,132 @@ +const request = require('supertest'); +const app = require('../app'); + +describe('STAC API Core Endpoints', () => { + describe('GET /', () => { + it('should return the landing page with STAC catalog structure', async () => { + const response = await request(app).get('/').expect(200); + + // Make sure the response has the correct structure of a STAC Catalog + expect(response.body).toHaveProperty('type', 'Catalog'); + expect(response.body).toHaveProperty('id'); + expect(response.body).toHaveProperty('title'); + expect(response.body).toHaveProperty('description'); + expect(response.body).toHaveProperty('stac_version'); + expect(response.body).toHaveProperty('conformsTo'); + expect(response.body).toHaveProperty('links'); + expect(Array.isArray(response.body.links)).toBe(true); + }); + + it('should include required links in landing page', async () => { + const response = await request(app).get('/').expect(200); + + const links = response.body.links; + const linkRels = links.map(link => link.rel); + + expect(linkRels).toContain('self'); + expect(linkRels).toContain('root'); + expect(linkRels).toContain('service-doc'); + expect(linkRels).toContain('service-desc'); + expect(linkRels).toContain('conformance'); + expect(linkRels).toContain('data'); + }); + + + it('should expose the same conformance classes as the /conformance endpoint', async () => { + const [landingRes, confRes] = await Promise.all([ + request(app).get('/').expect(200), + request(app).get('/conformance').expect(200) + ]); + + const landingConformance = landingRes.body.conformsTo; + const endpointConformance = confRes.body.conformsTo; + + // both must be arrays + expect(Array.isArray(landingConformance)).toBe(true); + expect(Array.isArray(endpointConformance)).toBe(true); + + // support function: sort, so that the order doesn't matter + const sortStrings = arr => [...arr].sort(); + + expect(sortStrings(landingConformance)).toEqual( + sortStrings(endpointConformance) + ); + }); + }); + + describe('GET /conformance', () => { + it('should return conformance classes', async () => { + const response = await request(app).get('/conformance').expect(200); + + expect(response.body).toHaveProperty('conformsTo'); + expect(Array.isArray(response.body.conformsTo)).toBe(true); + expect(response.body.conformsTo.length).toBeGreaterThan(0); + }); + + it('should include STAC API Core conformance', async () => { + const response = await request(app).get('/conformance').expect(200); + + expect(response.body.conformsTo).toContain('https://api.stacspec.org/v1.0.0/core'); + }); + }); + + describe('GET /collections', () => { + it('should return a FeatureCollection structure', async () => { + const response = await request(app).get('/collections').expect(200); + + expect(response.body).toHaveProperty('type', 'FeatureCollection'); + expect(response.body).toHaveProperty('collections'); + expect(response.body).toHaveProperty('links'); + expect(response.body).toHaveProperty('context'); + expect(Array.isArray(response.body.collections)).toBe(true); + }); + + it('should include pagination context', async () => { + const response = await request(app).get('/collections').expect(200); + + expect(response.body.context).toHaveProperty('returned'); + expect(response.body.context).toHaveProperty('limit'); + expect(response.body.context).toHaveProperty('matched'); + }); + }); + + describe('GET /queryables', () => { + it('should return queryables schema', async () => { + const response = await request(app).get('/queryables').expect(200); + + expect(response.body).toHaveProperty('$schema'); + expect(response.body).toHaveProperty('type', 'object'); + expect(response.body).toHaveProperty('properties'); + }); + + it('should include standard STAC queryable fields', async () => { + const response = await request(app).get('/queryables').expect(200); + + const properties = response.body.properties; + expect(properties).toHaveProperty('id'); + expect(properties).toHaveProperty('title'); + expect(properties).toHaveProperty('description'); + expect(properties).toHaveProperty('keywords'); + expect(properties).toHaveProperty('license'); + }); + }); + + describe('404 Handling', () => { + it('should return 404 for non-existent routes', async () => { + const response = await request(app).get('/non-existent-route').expect(404); + + expect(response.body).toHaveProperty('code'); + expect(response.body).toHaveProperty('description'); + }); + }); + + describe('GET /collections/:id', () => { + it('should return 404 for non-existent collection', async () => { + const response = await request(app).get('/collections/non-existent-id').expect(404); + + expect(response.body).toHaveProperty('code', 'NotFound'); + expect(response.body).toHaveProperty('description'); + expect(response.body).toHaveProperty('id', 'non-existent-id'); + }); + }); +}); diff --git a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js new file mode 100644 index 0000000..5c8100d --- /dev/null +++ b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js @@ -0,0 +1,43 @@ +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +describe('buildCollectionSearchQuery - full-text search and ranking', () => { + test('q parameter adds plainto_tsquery condition and rank in SELECT', () => { + const { sql, values } = buildCollectionSearchQuery({ q: 'forest', limit: 20, token: 0 }); + + // should contain plainto_tsquery and @@ operator + expect(sql).toMatch(/plainto_tsquery\('simple', \$1\)/); + expect(sql).toMatch(/@@/); + + // rank should be part of the SELECT list + expect(sql).toMatch(/ts_rank_cd\(/); + expect(sql).toMatch(/AS rank/); + + // Ordering defaults to rank DESC when q present and no sortby + expect(sql).toMatch(/ORDER BY rank DESC, id ASC/); + + // values: [q, limit, token] + expect(values[0]).toBe('forest'); + expect(values[1]).toBe(20); + expect(values[2]).toBe(0); + }); + + test('explicit sortby overrides rank ordering', () => { + const { sql } = buildCollectionSearchQuery({ q: 'lake', sortby: { field: 'title', direction: 'ASC' }, limit: 5, token: 0 }); + + expect(sql).toMatch(/ORDER BY title ASC/); + // rank still present in select + expect(sql).toMatch(/AS rank/); + }); + + test('parameter indexes remain correct when q + bbox combined', () => { + const bbox = [0,0,1,1]; + const { sql, values } = buildCollectionSearchQuery({ q: 'river', bbox, limit: 2, token: 0 }); + + // q uses $1, bbox uses $2..$5, then limit/token + expect(sql).toMatch(/plainto_tsquery\('simple', \$1\)/); + expect(sql).toMatch(/ST_MakeEnvelope\(\$2, \$3, \$4, \$5, 4326\)/); + + expect(values[0]).toBe('river'); + expect(values.slice(1,5)).toEqual(bbox); + }); +}); diff --git a/api/__tests__/buildCollectionsSearchQuery.basic.test.js b/api/__tests__/buildCollectionsSearchQuery.basic.test.js new file mode 100644 index 0000000..8756a5f --- /dev/null +++ b/api/__tests__/buildCollectionsSearchQuery.basic.test.js @@ -0,0 +1,41 @@ +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +describe('buildCollectionSearchQuery - basic cases', () => { + test('no params returns base SQL with LIMIT/OFFSET placeholders', () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/FROM collection/); + expect(sql).toMatch(/ORDER BY id ASC/); + // there should be LIMIT and OFFSET placeholders + expect(sql).toMatch(/LIMIT \$1 OFFSET \$2/); + expect(Array.isArray(values)).toBe(true); + // values should contain limit and token + expect(values.length).toBe(2); + expect(values).toEqual([10, 0]); + }); + + test('bbox adds ST_MakeEnvelope parameters in order', () => { + const bbox = [-10, 40, 10, 50]; + const { sql, values } = buildCollectionSearchQuery({ bbox, limit: 5, token: 0 }); + + // Ensure ST_MakeEnvelope uses $1..$4 when bbox is first + expect(sql).toMatch(/ST_MakeEnvelope\(\$1, \$2, \$3, \$4, 4326\)/); + // After bbox, limit and token are appended + expect(values.slice(0,4)).toEqual(bbox); + expect(values[4]).toBe(5); + expect(values[5]).toBe(0); + }); + + test('datetime closed interval produces start/end conditions', () => { + const datetime = '2020-01-01/2021-12-31'; + const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + expect(sql).toMatch(/temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated + expect(sql).toMatch(/temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated + // values order: start, end, limit, token + expect(values[0]).toBe('2020-01-01'); + expect(values[1]).toBe('2021-12-31'); + expect(values[2]).toBe(10); + expect(values[3]).toBe(0); + }); +}); \ No newline at end of file diff --git a/api/__tests__/collectionSearch.test.js b/api/__tests__/collectionSearch.test.js new file mode 100644 index 0000000..25c16c5 --- /dev/null +++ b/api/__tests__/collectionSearch.test.js @@ -0,0 +1,400 @@ +// __tests__/collectionSearch.test.js + +const request = require('supertest'); +const app = require('../app'); + +describe('Collection Search API - Query Parameters', () => { + + describe('GET /collections - Parameter Validation', () => { + + // ========== Successful Requests ========== + + it('should accept request without any parameters', async () => { + const response = await request(app) + .get('/collections') + .expect(200); + + expect(response.body).toHaveProperty('type', 'FeatureCollection'); + expect(response.body).toHaveProperty('collections'); + expect(response.body).toHaveProperty('context'); + expect(response.body.context.limit).toBe(10); // default limit + }); + + it('should accept valid limit parameter', async () => { + const response = await request(app) + .get('/collections?limit=5') + .expect(200); + + expect(response.body.context.limit).toBe(5); + expect(response.body.collections.length).toBeLessThanOrEqual(5); + }); + + it('should accept valid token parameter', async () => { + const response = await request(app) + .get('/collections?token=10') + .expect(200); + + expect(response.body).toHaveProperty('collections'); + }); + + it('should accept limit and token together', async () => { + const response = await request(app) + .get('/collections?limit=3&token=0') + .expect(200); + + expect(response.body.context.limit).toBe(3); + expect(response.body.collections.length).toBeLessThanOrEqual(3); + }); + + it('should accept valid q parameter', async () => { + const response = await request(app) + .get('/collections?q=test') + .expect(200); + + expect(response.body).toHaveProperty('collections'); + }); + + it('should accept valid bbox parameter', async () => { + const response = await request(app) + .get('/collections?bbox=-10,40,10,50') + .expect(200); + + expect(response.body).toHaveProperty('collections'); + }); + + it('should accept valid datetime parameter', async () => { + const response = await request(app) + .get('/collections?datetime=2020-01-01/2021-12-31') + .expect(200); + + expect(response.body).toHaveProperty('collections'); + }); + + it('should accept valid sortby parameter', async () => { + const response = await request(app) + .get('/collections?sortby=-created') + .expect(200); + + expect(response.body).toHaveProperty('collections'); + }); + + it('should accept multiple parameters combined', async () => { + const response = await request(app) + .get('/collections?q=test&limit=5&sortby=%2Btitle') + .expect(200); + + expect(response.body.context.limit).toBe(5); + expect(response.body).toHaveProperty('collections'); + }); + + // ========== Limit Parameter Validation ========== + + it('should reject limit less than 1', async () => { + const response = await request(app) + .get('/collections?limit=0') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('limit'); + expect(response.body.description).toContain('at least 1'); + }); + + it('should reject negative limit', async () => { + const response = await request(app) + .get('/collections?limit=-5') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('limit'); + }); + + it('should reject limit exceeding maximum', async () => { + const response = await request(app) + .get('/collections?limit=10001') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('limit'); + expect(response.body.description).toContain('10000'); + }); + + it('should reject non-numeric limit', async () => { + const response = await request(app) + .get('/collections?limit=abc') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('limit'); + expect(response.body.description).toContain('integer'); + }); + + // ========== Token Parameter Validation ========== + + it('should reject negative token', async () => { + const response = await request(app) + .get('/collections?token=-10') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('token'); + expect(response.body.description).toContain('non-negative'); + }); + + it('should reject non-numeric token', async () => { + const response = await request(app) + .get('/collections?token=abc') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('token'); + expect(response.body.description).toContain('integer'); + }); + + // ========== Q Parameter Validation ========== + + it('should reject q exceeding max length', async () => { + const longString = 'a'.repeat(501); + const response = await request(app) + .get(`/collections?q=${longString}`) + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('q'); + expect(response.body.description).toContain('500'); + }); + + // ========== Bbox Parameter Validation ========== + + it('should reject bbox with wrong number of coordinates', async () => { + const response = await request(app) + .get('/collections?bbox=1,2,3') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('bbox'); + expect(response.body.description).toContain('4 coordinates'); + }); + + it('should reject bbox with invalid numeric values', async () => { + const response = await request(app) + .get('/collections?bbox=a,b,c,d') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('bbox'); + expect(response.body.description).toContain('numeric'); + }); + + it('should reject bbox where minX >= maxX', async () => { + const response = await request(app) + .get('/collections?bbox=10,40,10,50') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('bbox'); + expect(response.body.description).toContain('minX must be less than maxX'); + }); + + it('should reject bbox where minY >= maxY', async () => { + const response = await request(app) + .get('/collections?bbox=-10,50,10,50') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('bbox'); + expect(response.body.description).toContain('minY must be less than maxY'); + }); + + it('should reject bbox with longitude out of range', async () => { + const response = await request(app) + .get('/collections?bbox=-181,40,10,50') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('bbox'); + expect(response.body.description).toContain('longitude'); + }); + + it('should reject bbox with latitude out of range', async () => { + const response = await request(app) + .get('/collections?bbox=-10,91,10,50') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('bbox'); + expect(response.body.description).toContain('latitude'); + }); + + // ========== Datetime Parameter Validation ========== + + it('should reject invalid datetime format', async () => { + const response = await request(app) + .get('/collections?datetime=not-a-date') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('datetime'); + expect(response.body.description).toContain('ISO8601'); + }); + + it('should reject datetime interval with multiple separators', async () => { + const response = await request(app) + .get('/collections?datetime=2019/2020/2021') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('datetime'); + expect(response.body.description).toContain('separator'); + }); + + it('should reject fully unbounded datetime interval', async () => { + const response = await request(app) + .get('/collections?datetime=../..') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('datetime'); + expect(response.body.description).toContain('unbounded'); + }); + + // ========== Sortby Parameter Validation ========== + + it('should reject unsupported sortby field', async () => { + const response = await request(app) + .get('/collections?sortby=invalid_field') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('sortby'); + expect(response.body.description).toContain('not supported'); + }); + + // ========== Multiple Error Handling ========== + + it('should return all validation errors combined', async () => { + const response = await request(app) + .get('/collections?limit=0&token=-5') + .expect(400); + + expect(response.body.code).toBe('InvalidParameterValue'); + expect(response.body.description).toContain('limit'); + expect(response.body.description).toContain('token'); + // Errors should be separated by semicolon + expect(response.body.description).toContain(';'); + }); + }); + + describe('GET /collections - Pagination Behavior', () => { + + it('should return correct number of items with limit', async () => { + const response = await request(app) + .get('/collections?limit=2') + .expect(200); + + expect(response.body.collections.length).toBeLessThanOrEqual(2); + expect(response.body.context.limit).toBe(2); + }); + + it('should include next link when more results available', async () => { + const response = await request(app) + .get('/collections?limit=2') + .expect(200); + + const links = response.body.links; + const nextLink = links.find(link => link.rel === 'next'); + + // Only check for next link if there are more items than limit + if (response.body.context.matched > response.body.context.limit) { + expect(nextLink).toBeDefined(); + expect(nextLink.href).toContain('token='); + } + }); + + it('should include prev link when not on first page', async () => { + const response = await request(app) + .get('/collections?limit=2&token=2') + .expect(200); + + const links = response.body.links; + const prevLink = links.find(link => link.rel === 'prev'); + + expect(prevLink).toBeDefined(); + expect(prevLink.href).toContain('token='); + }); + + it('should include self link with current parameters', async () => { + const response = await request(app) + .get('/collections?limit=5&token=10') + .expect(200); + + const links = response.body.links; + const selfLink = links.find(link => link.rel === 'self'); + + expect(selfLink).toBeDefined(); + expect(selfLink.href).toContain('limit=5'); + expect(selfLink.href).toContain('token=10'); + }); + + it('should return context with correct counts', async () => { + const response = await request(app) + .get('/collections?limit=3') + .expect(200); + + const context = response.body.context; + expect(context).toHaveProperty('returned'); + expect(context).toHaveProperty('limit', 3); + expect(context).toHaveProperty('matched'); + expect(context.returned).toBeLessThanOrEqual(context.limit); + expect(context.returned).toBeLessThanOrEqual(context.matched); + }); + + it('should handle token beyond available results', async () => { + const response = await request(app) + .get('/collections?limit=10&token=999999') + .expect(200); + + expect(response.body.collections).toHaveLength(0); + expect(response.body.context.returned).toBe(0); + }); + }); + + describe('GET /collections - Response Format', () => { + + it('should return valid FeatureCollection structure', async () => { + const response = await request(app) + .get('/collections') + .expect(200); + + expect(response.body).toMatchObject({ + type: 'FeatureCollection', + collections: expect.any(Array), + links: expect.any(Array), + context: { + returned: expect.any(Number), + limit: expect.any(Number), + matched: expect.any(Number) + } + }); + }); + + it('should include required link relations', async () => { + const response = await request(app) + .get('/collections') + .expect(200); + + const links = response.body.links; + const linkRels = links.map(link => link.rel); + + expect(linkRels).toContain('self'); + expect(linkRels).toContain('root'); + }); + + it('should return collections as array', async () => { + const response = await request(app) + .get('/collections') + .expect(200); + + expect(Array.isArray(response.body.collections)).toBe(true); + }); + }); +}); diff --git a/api/__tests__/collections-pagination.test.js b/api/__tests__/collections-pagination.test.js new file mode 100644 index 0000000..4133a97 --- /dev/null +++ b/api/__tests__/collections-pagination.test.js @@ -0,0 +1,127 @@ +// __tests__/collections-pagination.test.js + +const request = require('supertest'); +const app = require('../app'); + +/** + * Tests for API 4.5: Implement Pagination + * + * Verifies that: + * - limit correctly restricts number of returned results + * - token acts as offset for pagination + * - matched = total filtered collections BEFORE pagination + * - returned = number of results in this page + * - pagination handles boundaries correctly + */ +describe('Collection Search - Pagination behavior (4.5 Implement Pagination)', () => { + + /** + * Test 1: limit=2 should return exactly 2 collections + */ + it('should return exactly 2 collections with limit=2', async () => { + const response = await request(app) + .get('/collections?limit=2&token=0') + .expect(200); + + expect(response.body.collections.length).toBe(2); + expect(response.body.context.returned).toBe(2); + expect(response.body.context.matched).toBeGreaterThanOrEqual(2); + }); + + /** + * Test 2: token=0 and token=2 should return different slices + * Page 1: first 2 collections + * Page 2: next 2 collections + */ + it('should return different items for token=0 and token=2', async () => { + const page1 = await request(app) + .get('/collections?limit=2&token=0') + .expect(200); + + const page2 = await request(app) + .get('/collections?limit=2&token=2') + .expect(200); + + // Compare IDs to ensure pages differ + const ids1 = page1.body.collections.map(c => c.id); + const ids2 = page2.body.collections.map(c => c.id); + + expect(ids1).not.toEqual(ids2); + }); + + /** + * Test 3: Using token should correctly skip collections + * token = offset + */ + it('should skip the correct number of items based on token', async () => { + const all = await request(app) + .get('/collections') + .expect(200); + + const first = all.body.collections[0]; + const third = all.body.collections[2]; + + const response = await request(app) + .get('/collections?limit=1&token=2') + .expect(200); + + expect(response.body.collections[0].id).toBe(third.id); + expect(response.body.collections[0].id).not.toBe(first.id); + }); + + /** + * Test 4: matched remains constant regardless of limit/token + */ + it('matched should reflect total results, not paginated results', async () => { + const full = await request(app) + .get('/collections') + .expect(200); + + const paginated = await request(app) + .get('/collections?limit=1&token=0') + .expect(200); + + expect(paginated.body.context.matched).toBe(full.body.context.matched); + expect(paginated.body.context.returned).toBe(1); + }); + + /** + * Test 5: If token is out of bounds, should return an empty array + */ + it('should return empty list when token is beyond result count', async () => { + const full = await request(app) + .get('/collections') + .expect(200); + + const tooHighToken = full.body.context.matched + 50; + + const response = await request(app) + .get(`/collections?limit=5&token=${tooHighToken}`) + .expect(200); + + // Should return empty or very few results + expect(response.body.collections.length).toBeLessThanOrEqual(5); + expect(response.body.context.returned).toBe(response.body.collections.length); + }); + + /** + * Test 6: Pagination should not duplicate items across pages + */ + it('should not duplicate items across paginated pages', async () => { + const p1 = await request(app) + .get('/collections?limit=3&token=0') + .expect(200); + + const p2 = await request(app) + .get('/collections?limit=3&token=3') + .expect(200); + + const ids1 = p1.body.collections.map(c => c.id); + const ids2 = p2.body.collections.map(c => c.id); + + ids1.forEach(id => { + expect(ids2).not.toContain(id); + }); + }); + +}); \ No newline at end of file diff --git a/api/__tests__/collections-sort.test.js b/api/__tests__/collections-sort.test.js new file mode 100644 index 0000000..2e257f7 --- /dev/null +++ b/api/__tests__/collections-sort.test.js @@ -0,0 +1,169 @@ +// __tests__/collections-sort.test.js + +const request = require('supertest'); +const app = require('../app'); + +/** + * Tests for API 4.4: Implement Sorting + * + * Verifies that the collection search endpoint correctly: + * - Sorts results by specified field (title, id, license, created, updated) + * - Handles ascending (+field) and descending (-field) order + * - Defaults to ascending when no prefix specified + */ +describe('Collection Search - Sorting behavior (4.4 Implement Sorting)', () => { + + /** + * Test 1: Ascending sort by title with explicit + prefix + * Ensures +title correctly sorts titles A-Z + */ + it('should sort ascending by title with +title', async () => { + const response = await request(app) + .get('/collections?sortby=%2Btitle') + .expect(200); + + const titles = response.body.collections.map(c => c.title); + + // PostgreSQL's collation may differ from JavaScript's localeCompare. + // Instead, verify that: + // 1. Results are returned + // 2. First title alphabetically comes before last title + // 3. At least 80% of consecutive pairs are correctly ordered + expect(titles.length).toBeGreaterThan(0); + + // Check first vs last (should be alphabetically before or equal) + const firstTitle = titles[0].toLowerCase(); + const lastTitle = titles[titles.length - 1].toLowerCase(); + expect(firstTitle.localeCompare(lastTitle, 'en', { sensitivity: 'base' })).toBeLessThanOrEqual(0); + + // Count how many consecutive pairs are correctly ordered + let correctPairs = 0; + for (let i = 0; i < titles.length - 1; i++) { + if (titles[i].toLowerCase().localeCompare(titles[i + 1].toLowerCase(), 'en', { sensitivity: 'base' }) <= 0) { + correctPairs++; + } + } + + // At least 80% of pairs should be correctly ordered + // (allows for some PostgreSQL collation differences) + const pairRatio = correctPairs / (titles.length - 1); + expect(pairRatio).toBeGreaterThanOrEqual(0.8); + }); + + /** + * Test 2: Descending sort by title with - prefix + * Ensures -title correctly sorts titles Z-A + */ + it('should sort descending by title with -title', async () => { + const response = await request(app) + .get('/collections?sortby=-title') + .expect(200); + + const titles = response.body.collections.map(c => c.title); + + expect(titles.length).toBeGreaterThan(0); + + // Check first vs last (should be alphabetically after or equal in descending order) + const firstTitle = titles[0].toLowerCase(); + const lastTitle = titles[titles.length - 1].toLowerCase(); + expect(firstTitle.localeCompare(lastTitle, 'en', { sensitivity: 'base' })).toBeGreaterThanOrEqual(0); + + // Count correctly ordered descending pairs + let correctPairs = 0; + for (let i = 0; i < titles.length - 1; i++) { + if (titles[i].toLowerCase().localeCompare(titles[i + 1].toLowerCase(), 'en', { sensitivity: 'base' }) >= 0) { + correctPairs++; + } + } + + // At least 80% of pairs should be correctly ordered descending + const pairRatio = correctPairs / (titles.length - 1); + expect(pairRatio).toBeGreaterThanOrEqual(0.8); + }); + + /** + * Test 3: Ascending sort by id with explicit + prefix + * Verifies that +id sorts collection IDs in ascending order + */ + it('should sort ascending by id with +id', async () => { + const response = await request(app) + .get('/collections?sortby=%2Bid') + .expect(200); + + const ids = response.body.collections.map(c => c.id); + const sorted = ids.slice().sort((a, b) => a - b); + expect(ids).toEqual(sorted); + }); + + /** + * Test 4: Descending sort by id with - prefix + * Verifies that -id sorts collection IDs in descending order + */ + it('should sort descending by id with -id', async () => { + const response = await request(app) + .get('/collections?sortby=-id') + .expect(200); + + const ids = response.body.collections.map(c => c.id); + const sortedDesc = ids.slice().sort((a, b) => b - a); + expect(ids).toEqual(sortedDesc); + }); + + /** + * Test 5: Ascending sort by license with explicit + prefix + * Ensures +license correctly sorts licenses from A-Z + */ + it('should sort ascending by license with +license', async () => { + const response = await request(app) + .get('/collections?sortby=%2Blicense') + .expect(200); + + const licenses = response.body.collections.map(c => c.license); + const sorted = licenses.slice().sort((a, b) => a.localeCompare(b)); + expect(licenses).toEqual(sorted); + }); + + /** + * Test 6: Descending sort by license with - prefix + * Ensures -license correctly sorts licenses from Z-A + */ + it('should sort descending by license with -license', async () => { + const response = await request(app) + .get('/collections?sortby=-license') + .expect(200); + + const licenses = response.body.collections.map(c => c.license); + const sortedDesc = licenses.slice().sort((a, b) => b.localeCompare(a)); + expect(licenses).toEqual(sortedDesc); + }); + + /** + * Test 7: Default to ascending when no prefix provided + * Ensures that sortby=title (without +/-) defaults to ascending order + */ + it('should default to ascending when no prefix provided', async () => { + const response = await request(app) + .get('/collections?sortby=title') + .expect(200); + + const titles = response.body.collections.map(c => c.title); + + expect(titles.length).toBeGreaterThan(0); + + // Verify ascending order (first <= last) + const firstTitle = titles[0].toLowerCase(); + const lastTitle = titles[titles.length - 1].toLowerCase(); + expect(firstTitle.localeCompare(lastTitle, 'en', { sensitivity: 'base' })).toBeLessThanOrEqual(0); + + // At least 80% of pairs should be ascending + let correctPairs = 0; + for (let i = 0; i < titles.length - 1; i++) { + if (titles[i].toLowerCase().localeCompare(titles[i + 1].toLowerCase(), 'en', { sensitivity: 'base' }) <= 0) { + correctPairs++; + } + } + + const pairRatio = correctPairs / (titles.length - 1); + expect(pairRatio).toBeGreaterThanOrEqual(0.8); + }); +}); \ No newline at end of file diff --git a/api/__tests__/data-retrieval.test.js b/api/__tests__/data-retrieval.test.js new file mode 100644 index 0000000..701f032 --- /dev/null +++ b/api/__tests__/data-retrieval.test.js @@ -0,0 +1,265 @@ +const { query, closePool } = require('../db/db_APIconnection'); + +/** + * Jest Test Suite: Data Retrieval and Schema Validation + * Discovers all tables and columns, validates against expected schema + */ + +// Expected schema definitions +const EXPECTED_SCHEMAS = { + collection: { + id: { type: 'integer', required: true }, + // stac_id: { type: 'text', required: true }, // Column does not exist in both databases + stac_version: { type: 'text', required: true }, + type: { type: 'text', required: true }, + title: { type: 'text', required: true }, + description: { type: 'text', required: true }, + license: { type: 'text', required: true }, + spatial_extend: { type: 'geometry', required: true }, + temporal_extend_start: { type: 'timestamp without time zone', required: true }, + temporal_extend_end: { type: 'timestamp without time zone', required: true }, + full_json: { type: 'jsonb', required: false }, + created_at: { type: 'timestamp without time zone', required: true }, + updated_at: { type: 'timestamp without time zone', required: true }, + is_api: { type: 'boolean', required: true }, + is_active: { type: 'boolean', required: true } + }, + catalog: { + id: { type: 'integer', required: true }, + // stac_id: { type: 'text', required: true }, // Column does not exist in database + stac_version: { type: 'text', required: true }, + type: { type: 'text', required: true }, + title: { type: 'text', required: false }, + description: { type: 'text', required: true }, + created_at: { type: 'timestamp without time zone', required: true }, + updated_at: { type: 'timestamp without time zone', required: true } + } +}; + +describe('Database Schema Validation', () => { + let discoveredTables = []; + + afterAll(async () => { + await closePool(); + }); + + describe('Table Discovery', () => { + test('should discover STAC-related tables', async () => { + const tablesResult = await query(` + SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename IN ('collection', 'catalog') + ORDER BY tablename + `); + + discoveredTables = tablesResult.rows.map(r => r.tablename); + + expect(discoveredTables).toContain('collection'); + expect(discoveredTables).toContain('catalog'); + expect(discoveredTables.length).toBeGreaterThan(0); + }); + }); + + describe('Schema Validation - Collection Table', () => { + const actualColumns = {}; + + beforeAll(async () => { + const columnsResult = await query(` + SELECT + column_name, + data_type, + udt_name, + is_nullable + FROM information_schema.columns + WHERE table_name = 'collection' + ORDER BY ordinal_position + `); + + columnsResult.rows.forEach(col => { + actualColumns[col.column_name] = { + type: col.data_type === 'USER-DEFINED' ? col.udt_name : col.data_type, + nullable: col.is_nullable === 'YES' + }; + }); + }); + + test('should have all required columns', () => { + const expectedSchema = EXPECTED_SCHEMAS.collection; + + for (const [colName, expected] of Object.entries(expectedSchema)) { + expect(actualColumns).toHaveProperty(colName); + } + }); + + test('should have correct data types', () => { + const expectedSchema = EXPECTED_SCHEMAS.collection; + + for (const [colName, expected] of Object.entries(expectedSchema)) { + const actual = actualColumns[colName]; + if (!actual) continue; + + const actualType = actual.type.toLowerCase(); + const expectedType = expected.type.toLowerCase(); + + const typeMatch = actualType === expectedType || + actualType.includes(expectedType) || + expectedType.includes(actualType) || + (expectedType === 'geometry' && actualType === 'geometry'); + + expect(typeMatch).toBe(true); + } + }); + + test('should have geometry column', () => { + expect(actualColumns.spatial_extend).toBeDefined(); + expect(actualColumns.spatial_extend.type).toBe('geometry'); + }); + + test('should have jsonb column', () => { + expect(actualColumns.full_json).toBeDefined(); + expect(actualColumns.full_json.type).toBe('jsonb'); + }); + }); + + describe('Schema Validation - Catalog Table', () => { + const actualColumns = {}; + + beforeAll(async () => { + const columnsResult = await query(` + SELECT + column_name, + data_type, + udt_name, + is_nullable + FROM information_schema.columns + WHERE table_name = 'catalog' + ORDER BY ordinal_position + `); + + columnsResult.rows.forEach(col => { + actualColumns[col.column_name] = { + type: col.data_type === 'USER-DEFINED' ? col.udt_name : col.data_type, + nullable: col.is_nullable === 'YES' + }; + }); + }); + + test('should have all required columns', () => { + const expectedSchema = EXPECTED_SCHEMAS.catalog; + + for (const [colName, expected] of Object.entries(expectedSchema)) { + expect(actualColumns).toHaveProperty(colName); + } + }); + + test('should have correct data types', () => { + const expectedSchema = EXPECTED_SCHEMAS.catalog; + + for (const [colName, expected] of Object.entries(expectedSchema)) { + const actual = actualColumns[colName]; + if (!actual) continue; + + const actualType = actual.type.toLowerCase(); + const expectedType = expected.type.toLowerCase(); + + const typeMatch = actualType === expectedType || + actualType.includes(expectedType) || + expectedType.includes(actualType); + + expect(typeMatch).toBe(true); + } + }); + }); + + describe('Data Retrieval - Collection Table', () => { + test('should have data in collection table', async () => { + const countResult = await query(`SELECT COUNT(*) as count FROM collection`); + const rowCount = parseInt(countResult.rows[0].count); + + expect(rowCount).toBeGreaterThan(0); + }); + + test('should retrieve sample collection data', async () => { + const sampleResult = await query(`SELECT * FROM collection LIMIT 1`); + + expect(sampleResult.rows).toHaveLength(1); + + const sample = sampleResult.rows[0]; + expect(sample).toHaveProperty('id'); + // expect(sample).toHaveProperty('stac_id'); // Column does not exist in database + expect(sample).toHaveProperty('title'); + }); + + test('should have valid required fields', async () => { + const sampleResult = await query(`SELECT * FROM collection LIMIT 1`); + const sample = sampleResult.rows[0]; + const expectedSchema = EXPECTED_SCHEMAS.collection; + + for (const [colName, expected] of Object.entries(expectedSchema)) { + if (expected.required) { + expect(sample[colName]).not.toBeNull(); + expect(sample[colName]).not.toBeUndefined(); + } + } + }); + + test('should have valid geometry data', async () => { + const geomResult = await query(` + SELECT ST_GeometryType(spatial_extend) as geom_type + FROM collection + WHERE spatial_extend IS NOT NULL + LIMIT 1 + `); + + expect(geomResult.rows).toHaveLength(1); + expect(geomResult.rows[0].geom_type).toBeDefined(); + }); + + test('should have valid JSONB data', async () => { + const jsonResult = await query(` + SELECT full_json + FROM collection + WHERE full_json IS NOT NULL + LIMIT 1 + `); + + expect(jsonResult.rows).toHaveLength(1); + expect(typeof jsonResult.rows[0].full_json).toBe('object'); + expect(Object.keys(jsonResult.rows[0].full_json).length).toBeGreaterThan(0); + }); + }); + + describe('Data Retrieval - Catalog Table', () => { + test('should have data in catalog table', async () => { + const countResult = await query(`SELECT COUNT(*) as count FROM catalog`); + const rowCount = parseInt(countResult.rows[0].count); + + expect(rowCount).toBeGreaterThan(0); + }); + + test('should retrieve sample catalog data', async () => { + const sampleResult = await query(`SELECT * FROM catalog LIMIT 1`); + + expect(sampleResult.rows).toHaveLength(1); + + const sample = sampleResult.rows[0]; + expect(sample).toHaveProperty('id'); + // expect(sample).toHaveProperty('stac_id'); // Column does not exist in database + expect(sample).toHaveProperty('description'); + }); + + test('should have valid required fields', async () => { + const sampleResult = await query(`SELECT * FROM catalog LIMIT 1`); + const sample = sampleResult.rows[0]; + const expectedSchema = EXPECTED_SCHEMAS.catalog; + + for (const [colName, expected] of Object.entries(expectedSchema)) { + if (expected.required) { + expect(sample[colName]).not.toBeNull(); + expect(sample[colName]).not.toBeUndefined(); + } + } + }); + }); +}); diff --git a/api/__tests__/validators.test.js b/api/__tests__/validators.test.js new file mode 100644 index 0000000..c0e50cd --- /dev/null +++ b/api/__tests__/validators.test.js @@ -0,0 +1,407 @@ +// __tests__/validators.test.js + +const { + validateQ, + validateBbox, + validateDatetime, + validateLimit, + validateSortby, + validateToken +} = require('../validators/collectionSearchParams'); + +describe('Collection Search Parameter Validators', () => { + + describe('validateQ - Free-text search', () => { + it('should accept valid q parameter', () => { + const result = validateQ('sentinel'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('sentinel'); + }); + + it('should trim whitespace from q parameter', () => { + const result = validateQ(' landsat california '); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('landsat california'); + }); + + it('should accept undefined q (optional parameter)', () => { + const result = validateQ(undefined); + expect(result.valid).toBe(true); + expect(result.normalized).toBeUndefined(); + }); + + it('should accept empty string', () => { + const result = validateQ(''); + expect(result.valid).toBe(true); + }); + + it('should reject non-string q', () => { + const result = validateQ(123); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be a string'); + }); + + it('should reject q exceeding max length', () => { + const longString = 'a'.repeat(501); + const result = validateQ(longString); + expect(result.valid).toBe(false); + expect(result.error).toContain('exceeds maximum length'); + }); + + it('should accept q at max length boundary', () => { + const maxString = 'a'.repeat(500); + const result = validateQ(maxString); + expect(result.valid).toBe(true); + }); + }); + + describe('validateBbox - Bounding box', () => { + it('should accept valid bbox as comma-separated string', () => { + const result = validateBbox('-10,40,10,50'); + expect(result.valid).toBe(true); + expect(result.normalized).toEqual([-10, 40, 10, 50]); + }); + + it('should accept valid bbox as array', () => { + const result = validateBbox([-10, 40, 10, 50]); + expect(result.valid).toBe(true); + expect(result.normalized).toEqual([-10, 40, 10, 50]); + }); + + it('should accept bbox with decimal coordinates', () => { + const result = validateBbox('-122.5,37.7,-122.3,37.9'); + expect(result.valid).toBe(true); + expect(result.normalized).toEqual([-122.5, 37.7, -122.3, 37.9]); + }); + + it('should accept undefined bbox (optional)', () => { + const result = validateBbox(undefined); + expect(result.valid).toBe(true); + }); + + it('should reject bbox with wrong number of coordinates', () => { + const result = validateBbox('1,2,3'); + expect(result.valid).toBe(false); + expect(result.error).toContain('exactly 4 coordinates'); + }); + + it('should reject bbox with non-numeric values', () => { + const result = validateBbox('a,b,c,d'); + expect(result.valid).toBe(false); + expect(result.error).toContain('invalid numeric values'); + }); + + it('should reject bbox where minX >= maxX', () => { + const result = validateBbox('10,40,10,50'); + expect(result.valid).toBe(false); + expect(result.error).toContain('minX must be less than maxX'); + }); + + it('should reject bbox where minX > maxX', () => { + const result = validateBbox('10,40,-10,50'); + expect(result.valid).toBe(false); + expect(result.error).toContain('minX must be less than maxX'); + }); + + it('should reject bbox where minY >= maxY', () => { + const result = validateBbox('-10,50,10,50'); + expect(result.valid).toBe(false); + expect(result.error).toContain('minY must be less than maxY'); + }); + + it('should reject bbox with longitude out of range', () => { + const result = validateBbox('-181,40,10,50'); + expect(result.valid).toBe(false); + expect(result.error).toContain('longitude values must be between -180 and 180'); + }); + + it('should reject bbox with latitude out of range', () => { + const result = validateBbox('-10,91,10,50'); + expect(result.valid).toBe(false); + expect(result.error).toContain('latitude values must be between -90 and 90'); + }); + + it('should accept bbox at coordinate boundaries', () => { + const result = validateBbox('-180,-90,180,90'); + expect(result.valid).toBe(true); + expect(result.normalized).toEqual([-180, -90, 180, 90]); + }); + + it('should reject invalid type for bbox', () => { + const result = validateBbox(123); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be an array or comma-separated string'); + }); + }); + + describe('validateDatetime - Temporal filter', () => { + it('should accept single ISO8601 datetime', () => { + const result = validateDatetime('2020-01-01T00:00:00Z'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('2020-01-01T00:00:00Z'); + }); + + it('should accept date without time', () => { + const result = validateDatetime('2020-01-01'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('2020-01-01'); + }); + + it('should accept closed interval', () => { + const result = validateDatetime('2019-01-01/2021-12-31'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('2019-01-01/2021-12-31'); + }); + + it('should accept open-ended start interval', () => { + const result = validateDatetime('../2021-12-31'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('../2021-12-31'); + }); + + it('should accept open-ended end interval', () => { + const result = validateDatetime('2019-01-01/..'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('2019-01-01/..'); + }); + + it('should accept datetime with timezone offset', () => { + const result = validateDatetime('2020-01-01T00:00:00+02:00'); + expect(result.valid).toBe(true); + }); + + it('should accept datetime with milliseconds', () => { + const result = validateDatetime('2020-01-01T00:00:00.123Z'); + expect(result.valid).toBe(true); + }); + + it('should accept undefined datetime (optional)', () => { + const result = validateDatetime(undefined); + expect(result.valid).toBe(true); + }); + + it('should reject non-string datetime', () => { + const result = validateDatetime(123); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be a string'); + }); + + it('should reject invalid ISO8601 format', () => { + const result = validateDatetime('not-a-date'); + expect(result.valid).toBe(false); + expect(result.error).toContain('not valid ISO8601'); + }); + + it('should reject invalid date values', () => { + const result = validateDatetime('2020-13-45'); + expect(result.valid).toBe(false); + expect(result.error).toContain('not valid ISO8601'); + }); + + it('should reject interval with multiple separators', () => { + const result = validateDatetime('2019/2020/2021'); + expect(result.valid).toBe(false); + expect(result.error).toContain('exactly one "/" separator'); + }); + + it('should reject fully unbounded interval', () => { + const result = validateDatetime('../..'); + expect(result.valid).toBe(false); + expect(result.error).toContain('cannot be unbounded on both sides'); + }); + + it('should reject interval with invalid start', () => { + const result = validateDatetime('invalid/2021-12-31'); + expect(result.valid).toBe(false); + expect(result.error).toContain('start value'); + }); + + it('should reject interval with invalid end', () => { + const result = validateDatetime('2019-01-01/invalid'); + expect(result.valid).toBe(false); + expect(result.error).toContain('end value'); + }); + }); + + describe('validateLimit - Result limit', () => { + it('should accept valid limit', () => { + const result = validateLimit('50'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(50); + }); + + it('should accept limit as number', () => { + const result = validateLimit(25); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(25); + }); + + it('should return default when undefined', () => { + const result = validateLimit(undefined); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(10); + }); + + it('should accept limit at minimum boundary', () => { + const result = validateLimit('1'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(1); + }); + + it('should accept limit at maximum boundary', () => { + const result = validateLimit('10000'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(10000); + }); + + it('should reject limit less than 1', () => { + const result = validateLimit('0'); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be at least 1'); + }); + + it('should reject negative limit', () => { + const result = validateLimit('-5'); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be at least 1'); + }); + + it('should reject limit exceeding maximum', () => { + const result = validateLimit('10001'); + expect(result.valid).toBe(false); + expect(result.error).toContain('must not exceed 10000'); + }); + + it('should reject non-numeric limit', () => { + const result = validateLimit('abc'); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be a valid integer'); + }); + + it('should reject decimal limit', () => { + const result = validateLimit('10.5'); + expect(result.valid).toBe(false); + expect(result.error).toContain('integer'); + }); + }); + + describe('validateSortby - Sort specification', () => { + it('should accept ascending sort with + prefix', () => { + const result = validateSortby('+title'); + expect(result.valid).toBe(true); + expect(result.normalized).toEqual({ field: 'title', direction: 'ASC' }); + }); + + it('should accept descending sort with - prefix', () => { + const result = validateSortby('-created'); + expect(result.valid).toBe(true); + // Field is mapped to database column name + expect(result.normalized).toEqual({ field: 'created_at', direction: 'DESC' }); + }); + + it('should default to ascending without prefix', () => { + const result = validateSortby('id'); + expect(result.valid).toBe(true); + expect(result.normalized).toEqual({ field: 'id', direction: 'ASC' }); + }); + + it('should accept all allowed fields', () => { + const fieldMapping = { + 'title': 'title', + 'id': 'id', + 'license': 'license', + 'created': 'created_at', + 'updated': 'updated_at' + }; + + const fields = ['title', 'id', 'license', 'created', 'updated']; + fields.forEach(field => { + const result = validateSortby(field); + expect(result.valid).toBe(true); + // Should be mapped to database column name + expect(result.normalized.field).toBe(fieldMapping[field]); + }); + }); + + it('should accept undefined sortby (optional)', () => { + const result = validateSortby(undefined); + expect(result.valid).toBe(true); + expect(result.normalized).toBeUndefined(); + }); + + it('should reject unsupported field', () => { + const result = validateSortby('unsupported_field'); + expect(result.valid).toBe(false); + expect(result.error).toContain('not supported'); + expect(result.error).toContain('Allowed fields:'); + }); + + it('should reject non-string sortby', () => { + const result = validateSortby(123); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be a string'); + }); + + it('should reject empty field name (with + prefix)', () => { + const result = validateSortby('+'); + expect(result.valid).toBe(false); + expect(result.error).toContain('must specify a field'); + }); + + it('should reject empty field name (without prefix)', () => { + const result = validateSortby(""); + expect(result.valid).toBe(false); + expect(result.error).toContain('must specify a field'); + }); + }); + + describe('validateToken - Pagination token', () => { + it('should accept valid token', () => { + const result = validateToken('50'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(50); + }); + + it('should accept token as number', () => { + const result = validateToken(100); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(100); + }); + + it('should return default when undefined', () => { + const result = validateToken(undefined); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(0); + }); + + it('should accept zero token', () => { + const result = validateToken('0'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(0); + }); + + it('should accept large token values', () => { + const result = validateToken('999999'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(999999); + }); + + it('should reject negative token', () => { + const result = validateToken('-1'); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be non-negative'); + }); + + it('should reject non-numeric token', () => { + const result = validateToken('abc'); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be a valid integer'); + }); + + it('should handle string zero', () => { + const result = validateToken('0'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe(0); + }); + }); +}); diff --git a/api/__tests__/verify-schema.test.js b/api/__tests__/verify-schema.test.js new file mode 100644 index 0000000..e03c118 --- /dev/null +++ b/api/__tests__/verify-schema.test.js @@ -0,0 +1,365 @@ +const { query, closePool } = require('../db/db_APIconnection'); + +/** + * Jest Test Suite: Verify Database Schema for Collection and Catalog Tables + * Checks that both tables have all required columns with valid data + */ + +describe('Database Schema Verification', () => { + + afterAll(async () => { + await closePool(); + }); + + describe('Collection Table Structure', () => { + let tableInfo; + + beforeAll(async () => { + tableInfo = await query(` + SELECT + column_name, + data_type, + is_nullable, + column_default + FROM information_schema.columns + WHERE table_name = 'collection' + ORDER BY ordinal_position + `); + }); + + test('should have table structure', () => { + expect(tableInfo.rowCount).toBeGreaterThan(0); + }); + + test('should have at least 14 columns', () => { + expect(tableInfo.rowCount).toBeGreaterThanOrEqual(14); + }); + }); + + describe('Collection Table - Column Data Integrity', () => { + test.each([ + ['id', 'integer'], + // ['stac_id', 'text'], // Column does not exist in database + ['title', 'text'], + ['description', 'text'], + ['license', 'text'], + ['spatial_extend', 'USER-DEFINED'], + ['full_json', 'jsonb'], + ['is_active', 'boolean'], + ['is_api', 'boolean'] + ])('column %s should exist with type %s', async (colName, expectedType) => { + const stats = await query(` + SELECT + COUNT(*) as total_rows, + COUNT(${colName}) as non_null_count + FROM collection + `); + + const stat = stats.rows[0]; + expect(parseInt(stat.total_rows)).toBeGreaterThanOrEqual(0); + + // If table has data, check that non-spatial columns have data + if (parseInt(stat.total_rows) > 0 && colName !== 'spatial_extend') { + expect(parseInt(stat.non_null_count)).toBeGreaterThan(0); + } + }); + + test('should have valid geometry type in spatial_extend if data exists', async () => { + const geomType = await query(` + SELECT ST_GeometryType(spatial_extend) as geom_type + FROM collection + WHERE spatial_extend IS NOT NULL + LIMIT 1 + `); + + // Only check geometry type if there is data + if (geomType.rows.length > 0) { + expect(geomType.rows[0].geom_type).toMatch(/^ST_/); + } else { + expect(geomType.rows.length).toBe(0); // Pass if no data + } + }); + + test('should have valid JSONB data in full_json if data exists', async () => { + const sample = await query(` + SELECT full_json + FROM collection + WHERE full_json IS NOT NULL + LIMIT 1 + `); + + // Only check JSONB if there is data + if (sample.rows.length > 0) { + expect(typeof sample.rows[0].full_json).toBe('object'); + expect(Object.keys(sample.rows[0].full_json).length).toBeGreaterThan(0); + } else { + expect(sample.rows.length).toBe(0); // Pass if no data + } + }); + + test('should have valid timestamps if data exists', async () => { + const sample = await query(` + SELECT created_at, updated_at + FROM collection + LIMIT 1 + `); + + // Only check timestamps if there is data + if (sample.rows.length > 0) { + expect(sample.rows[0].created_at).toBeInstanceOf(Date); + expect(sample.rows[0].updated_at).toBeInstanceOf(Date); + } else { + expect(sample.rows.length).toBe(0); // Pass if no data + } + }); + }); + + describe('Collection Table - Indexes', () => { + test('should have indexes', async () => { + const indexCheck = await query(` + SELECT + indexname, + indexdef + FROM pg_indexes + WHERE tablename = 'collection' + `); + + expect(indexCheck.rowCount).toBeGreaterThan(0); + }); + }); + + describe('Collection Table - Overall Statistics', () => { + test('should be queryable (may be empty)', async () => { + const countResult = await query(`SELECT COUNT(*) as count FROM collection`); + const count = parseInt(countResult.rows[0].count); + + expect(count).toBeGreaterThanOrEqual(0); + }); + }); + + describe('Catalog Table Structure', () => { + let tableInfo; + + beforeAll(async () => { + tableInfo = await query(` + SELECT + column_name, + data_type, + is_nullable, + column_default + FROM information_schema.columns + WHERE table_name = 'catalog' + ORDER BY ordinal_position + `); + }); + + test('should have table structure', () => { + expect(tableInfo.rowCount).toBeGreaterThan(0); + }); + + test('should have at least 7 columns', () => { + expect(tableInfo.rowCount).toBeGreaterThanOrEqual(7); + }); + }); + + describe('Catalog Table - Column Data Integrity', () => { + test.each([ + ['id', 'integer'], + // ['stac_id', 'text'], // Column does not exist in database + ['stac_version', 'text'], + ['type', 'text'], + ['description', 'text'] + ])('column %s should exist with type %s', async (colName, expectedType) => { + const stats = await query(` + SELECT + COUNT(*) as total_rows, + COUNT(${colName}) as non_null_count + FROM catalog + `); + + const stat = stats.rows[0]; + expect(parseInt(stat.total_rows)).toBeGreaterThanOrEqual(0); + + // If table has data, check that columns have data + if (parseInt(stat.total_rows) > 0) { + expect(parseInt(stat.non_null_count)).toBeGreaterThan(0); + } + }); + + test('should have valid timestamps if data exists', async () => { + const sample = await query(` + SELECT created_at, updated_at + FROM catalog + LIMIT 1 + `); + + // Only check timestamps if there is data + if (sample.rows.length > 0) { + expect(sample.rows[0].created_at).toBeInstanceOf(Date); + expect(sample.rows[0].updated_at).toBeInstanceOf(Date); + } else { + expect(sample.rows.length).toBe(0); // Pass if no data + } + }); + }); + + describe('Catalog Table - Overall Statistics', () => { + test('should be queryable (may be empty)', async () => { + const countResult = await query(`SELECT COUNT(*) as count FROM catalog`); + const count = parseInt(countResult.rows[0].count); + + expect(count).toBeGreaterThanOrEqual(0); + }); + }); +}); + +// Legacy function for backwards compatibility (not used in tests) +async function verifyTableSchema(tableName, displayName) { + console.log(`=== ${displayName} Schema Verification ===\n`); + + try { + // Get table structure + console.log(`1. Checking ${tableName} table structure...`); + const tableInfo = await query(` + SELECT + column_name, + data_type, + is_nullable, + column_default + FROM information_schema.columns + WHERE table_name = $1 + ORDER BY ordinal_position + `, [tableName]); + + console.log(`✓ Found ${tableInfo.rowCount} columns in ${tableName} table\n`); + + console.log('2. Verifying all columns and their data...\n'); + + let allColumnsValid = true; + let columnsWithData = 0; + let columnsWithNulls = 0; + + // Check each column for data integrity + for (const col of tableInfo.rows) { + const colName = col.column_name; + const dataType = col.data_type; + + try { + // Get statistics for this column + const stats = await query(` + SELECT + COUNT(*) as total_rows, + COUNT(${colName}) as non_null_count, + COUNT(*) - COUNT(${colName}) as null_count + FROM ${tableName} + `); + + const stat = stats.rows[0]; + const percentNonNull = stat.total_rows > 0 + ? ((stat.non_null_count / stat.total_rows) * 100).toFixed(1) + : 0; + + if (stat.non_null_count > 0) { + columnsWithData++; + console.log(`✓ ${colName} (${dataType})`); + console.log(` └─ ${stat.non_null_count}/${stat.total_rows} rows (${percentNonNull}% filled)`); + + // Sample a value to verify data format + if (colName !== 'spatial_extend') { // Skip geometry for display + const sample = await query(` + SELECT ${colName} + FROM ${tableName} + WHERE ${colName} IS NOT NULL + LIMIT 1 + `); + + if (sample.rows[0]) { + let sampleValue = sample.rows[0][colName]; + + // Format output based on data type + if (typeof sampleValue === 'object' && sampleValue !== null) { + sampleValue = JSON.stringify(sampleValue).substring(0, 80) + '...'; + } else if (typeof sampleValue === 'string') { + sampleValue = sampleValue.substring(0, 60) + (sampleValue.length > 60 ? '...' : ''); + } + + console.log(` └─ Sample: ${sampleValue}`); + } + } else { + // For geometry, show type + const geomType = await query(` + SELECT ST_GeometryType(${colName}) as geom_type + FROM ${tableName} + WHERE ${colName} IS NOT NULL + LIMIT 1 + `); + if (geomType.rows[0]) { + console.log(` └─ Geometry type: ${geomType.rows[0].geom_type}`); + } + } + console.log(''); + } else if (stat.total_rows > 0) { + columnsWithNulls++; + console.log(`⚠ ${colName} (${dataType})`); + console.log(` └─ All ${stat.total_rows} rows are NULL`); + console.log(''); + } else { + console.log(`⚠ ${colName} (${dataType})`); + console.log(` └─ No data in table`); + console.log(''); + } + + } catch (error) { + console.log(`✗ ${colName} (${dataType})`); + console.log(` └─ Error checking data: ${error.message}`); + console.log(''); + allColumnsValid = false; + } + } + + console.log(`Summary: ${columnsWithData} columns with data, ${columnsWithNulls} columns all NULL\n`); + + // Check for indexes + console.log('3. Checking indexes...'); + const indexCheck = await query(` + SELECT + indexname, + indexdef + FROM pg_indexes + WHERE tablename = $1 + `, [tableName]); + + if (indexCheck.rowCount > 0) { + console.log(`✓ Found ${indexCheck.rowCount} index(es):`); + indexCheck.rows.forEach(idx => { + console.log(` - ${idx.indexname}`); + }); + } else { + console.log('⚠ No indexes found'); + } + + // Check row count + console.log('\n4. Checking overall data statistics...'); + const countResult = await query(`SELECT COUNT(*) as count FROM ${tableName}`); + console.log(`✓ ${displayName} table contains ${countResult.rows[0].count} rows`); + + console.log(`\n=== ${displayName} Schema Verification Complete ===`); + + if (allColumnsValid) { + console.log('\n✓ All required columns present'); + process.exit(0); + } else { + console.log('\n✗ Some required columns are missing'); + process.exit(1); + } + + } catch (error) { + console.error('✗ Schema verification failed:', error.message); + return false; + } +} + +// Export for manual testing if needed +if (require.main === module) { + verifyTableSchema('collection', 'Collection').then(() => process.exit(0)); +} diff --git a/api/app.js b/api/app.js new file mode 100644 index 0000000..bf5f4f5 --- /dev/null +++ b/api/app.js @@ -0,0 +1,69 @@ +require('dotenv').config(); +const express = require('express'); +const logger = require('morgan'); +const cors = require('cors'); +const swaggerUi = require('swagger-ui-express'); +const YAML = require('yamljs'); +const path = require('path'); + +// Import routes +const indexRouter = require('./routes/index'); +const conformanceRouter = require('./routes/conformance'); +const collectionsRouter = require('./routes/collections'); +const queryablesRouter = require('./routes/queryables'); + +const app = express(); + +// Middleware +app.use(logger('dev')); +app.use(express.json()); +app.use(express.urlencoded({ extended: false })); + +// CORS configuration - allow requests from frontend +app.use(cors({ + origin: process.env.CORS_ORIGIN || '*', + methods: ['GET', 'POST', 'OPTIONS'], + allowedHeaders: ['Content-Type', 'Authorization'] +})); + +// Content-Type header for all JSON responses +app.use((req, res, next) => { + res.setHeader('Content-Type', 'application/json'); + next(); +}); + +// STAC API routes +app.use('/', indexRouter); +app.use('/conformance', conformanceRouter); +app.use('/collections', collectionsRouter); +app.use('/queryables', queryablesRouter); + +// Swagger/OpenAPI documentation (if openapi.yaml exists) +try { + const swaggerDocument = YAML.load(path.join(__dirname, 'docs', 'openapi.yaml')); + app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument)); +} catch (err) { + console.log('OpenAPI documentation not found. Create docs/openapi.yaml to enable Swagger UI.'); +} + +// 404 handler +app.use((req, res, next) => { + res.status(404).json({ + code: 'NotFound', + description: `The requested resource '${req.url}' was not found on this server.` + }); +}); + +// Error handler +app.use((err, req, res, next) => { + // Set locals, only providing error in development + const isDev = req.app.get('env') === 'development'; + + res.status(err.status || 500).json({ + code: err.code || 'InternalServerError', + description: err.message || 'An internal server error occurred', + ...(isDev && { stack: err.stack }) + }); +}); + +module.exports = app; \ No newline at end of file diff --git a/api/bin/www b/api/bin/www new file mode 100644 index 0000000..81cb39b --- /dev/null +++ b/api/bin/www @@ -0,0 +1,90 @@ +#!/usr/bin/env node + +/** + * Module dependencies. + */ + +var app = require('../app'); +var debug = require('debug')('api:server'); +var http = require('http'); + +/** + * Get port from environment and store in Express. + */ + +var port = normalizePort(process.env.PORT || '3000'); +app.set('port', port); + +/** + * Create HTTP server. + */ + +var server = http.createServer(app); + +/** + * Listen on provided port, on all network interfaces. + */ + +server.listen(port); +server.on('error', onError); +server.on('listening', onListening); + +/** + * Normalize a port into a number, string, or false. + */ + +function normalizePort(val) { + var port = parseInt(val, 10); + + if (isNaN(port)) { + // named pipe + return val; + } + + if (port >= 0) { + // port number + return port; + } + + return false; +} + +/** + * Event listener for HTTP server "error" event. + */ + +function onError(error) { + if (error.syscall !== 'listen') { + throw error; + } + + var bind = typeof port === 'string' + ? 'Pipe ' + port + : 'Port ' + port; + + // handle specific listen errors with friendly messages + switch (error.code) { + case 'EACCES': + console.error(bind + ' requires elevated privileges'); + process.exit(1); + break; + case 'EADDRINUSE': + console.error(bind + ' is already in use'); + process.exit(1); + break; + default: + throw error; + } +} + +/** + * Event listener for HTTP server "listening" event. + */ + +function onListening() { + var addr = server.address(); + var bind = typeof addr === 'string' + ? 'pipe ' + addr + : 'port ' + addr.port; + debug('Listening on ' + bind); +} diff --git a/api/config/conformanceURIS.js b/api/config/conformanceURIS.js new file mode 100644 index 0000000..2154168 --- /dev/null +++ b/api/config/conformanceURIS.js @@ -0,0 +1,25 @@ +// config/conformanceURIS.js + +// Shared list of conformance URIs used by both: +// - GET / +// - GET /conformance + +const CONFORMANCE_URIS = [ + 'https://api.stacspec.org/v1.0.0/core', + 'https://api.stacspec.org/v1.0.0/collections', + // Collection Search conformance classes + 'https://api.stacspec.org/v1.0.0/collection-search', + 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/simple-query', // Simple Query (bbox, datetime, limit) + 'https://api.stacspec.org/v1.0.0-rc.1/collection-search#free-text', // Free-text search + 'https://api.stacspec.org/v1.0.0-rc.1/collection-search#filter', // CQL2 Filter' + 'https://api.stacspec.org/v1.1.0/collection-search#sort', // Sorting + // CQL2 conformance classes + "http://www.opengis.net/spec/cql2/1.0/conf/basic-cql2", // Basic CQL2 + "http://www.opengis.net/spec/cql2/1.0/conf/cql2-json", // CQL2 JSON-Querys + "http://www.opengis.net/spec/cql2/1.0/conf/cql2-text", // CQL2 Text-Querys + "http://www.opengis.net/spec/cql2/1.0/conf/basic-spatial-functions" // Basic Spatial Functions + ]; + +module.exports = { + CONFORMANCE_URIS +}; diff --git a/api/data/collections.js b/api/data/collections.js new file mode 100644 index 0000000..e58bcbf --- /dev/null +++ b/api/data/collections.js @@ -0,0 +1,105 @@ +// Small in-memory sample of collections for basic GET /collections implementation +// +// This file is intentionally simple and used only for local testing and +// unit-tests. Each entry represents a minimal STAC Collection-like object +// containing common STAC fields (id, title, description, keywords, extent, etc). +// In a production deployment this should be replaced by a database query +// that returns fully validated STAC Collection objects. +module.exports = [ + { + id: 'sentinel-2-l2a', + stac_version: '1.0.0', + type: 'Collection', + title: 'Sentinel-2 L2A Collection', + description: 'Sentinel-2 Level-2A processed imagery from Copernicus', + keywords: ['sentinel-2', 'optical', 'multispectral'], + license: 'CC-BY-4.0', + providers: [ + { + name: 'ESA', + roles: ['producer', 'licensor'], + url: 'https://www.esa.int/' + } + ], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2015-06-23T00:00:00Z', null]] } + }, + links: [ + { + rel: 'self', + href: 'https://example.com/collections/sentinel-2-l2a', + type: 'application/json' + }, + { + rel: 'parent', + href: 'https://example.com/', + type: 'application/json' + } + ] + }, + { + id: 'landsat-8-l1', + stac_version: '1.0.0', + type: 'Collection', + title: 'Landsat 8 Level-1', + description: 'Landsat 8 Collection 1 Level 1 data', + keywords: ['landsat', 'optical', 'multispectral'], + license: 'CC0-1.0', + providers: [ + { + name: 'USGS', + roles: ['producer'], + url: 'https://www.usgs.gov/' + } + ], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2013-02-11T00:00:00Z', null]] } + }, + links: [ + { + rel: 'self', + href: 'https://example.com/collections/landsat-8-l1', + type: 'application/json' + }, + { + rel: 'parent', + href: 'https://example.com/', + type: 'application/json' + } + ] + }, + { + id: 'modis', + stac_version: '1.0.0', + type: 'Collection', + title: 'MODIS Daily', + description: 'MODIS daily composites from NASA Earth Observatories', + keywords: ['modis', 'daily', 'thermal', 'visible'], + license: 'CC0-1.0', + providers: [ + { + name: 'NASA', + roles: ['producer', 'licensor'], + url: 'https://www.nasa.gov/' + } + ], + extent: { + spatial: { bbox: [[-180, -90, 180, 90]] }, + temporal: { interval: [['2000-02-24T00:00:00Z', null]] } + }, + links: [ + { + rel: 'self', + href: 'https://example.com/collections/modis', + type: 'application/json' + }, + { + rel: 'parent', + href: 'https://example.com/', + type: 'application/json' + } + ] + } +]; diff --git a/api/db/buildCollectionSearchQuery.js b/api/db/buildCollectionSearchQuery.js new file mode 100644 index 0000000..cc1d435 --- /dev/null +++ b/api/db/buildCollectionSearchQuery.js @@ -0,0 +1,220 @@ + +/* function buildCollectionSearchQuery + * Dynamically constructs a parameterized SQL query for the /collections endpoint. + * + * This function converts validated API search parameters into a safe, optimized, + * database-ready SQL statement. It supports multiple filter types (full-text, spatial, + * temporal), dynamic SELECT column injection (rank), sorting and pagination. + * + * The SELECT part focuses on the core STAC collection metadata, as described in the bid + * and the database schema: + * - id, stac_version, type, title, description, license + * - spatial_extend, temporal_extend_start, temporal_extend_end + * - created_at, updated_at, is_api, is_active + * - full_json (complete STAC Collection document as JSONB) + * + * @param {Object} params + * @param {string|undefined} params.q + * Full-text search query. Currently searches in: + * - collection.title + * - collection.description + * + * The bid requires full-text search across title, description and keywords + * (and possibly providers). Integration of keywords/providers into the + * tsvector (via join or dedicated search_vector column) is planned as a + * follow-up refinement. + * + * Note: Keywords are not yet part of the full-text vector. They will be added + * in a follow-up step once the database exposes a canonical keyword aggregation + * + * When `q` is present, a tsvector is built from title/description + * using the same expression as the GIN index + * (to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))). + * We use plainto_tsquery('simple', $n) and add ts_rank_cd(...) AS rank + * to the SELECT list so we can order by relevance. + * + * @param {number[]|undefined} params.bbox + * Spatial filter as [minX, minY, maxX, maxY] in EPSG:4326. + * When present, the query adds: + * ST_Intersects(spatial_extend, ST_MakeEnvelope($x, $y, $z, $w, 4326)) + * + * @param {string|undefined} params.datetime + * Temporal filter in ISO8601: + * - single instant: "2020-01-01T00:00:00Z" + * - closed interval: "2019-01-01/2021-12-31" + * - open start/end: "../2021-12-31" or "2019-01-01/.." + * + * The collection is matched if its temporal_extend_start/temporal_extend_end + * overlap the requested interval. + * + * @param {{field: string, direction: 'ASC'|'DESC'}|undefined} params.sortby + * Normalized sort description. Field is restricted to an allowed + * whitelist (id, title, license, created_at, updated_at, …). + * If provided, ORDER BY is used. + * If omitted and `q` is present, results are ordered by rank DESC, id ASC. + * If omitted and `q` is not present, results are ordered by id ASC. + * + * @param {number} params.limit + * Maximum number of rows to return. Already validated to be + * within [1, 10000]. Translated to LIMIT $n. + * + * @param {number} params.token + * Offset for pagination (0-based). Translated to OFFSET $n. + * + * @returns {{ sql: string, values: any[] }} + * sql – complete parameterized SQL string + * values – array of bind parameters in the correct order + */ + +function buildCollectionSearchQuery(params) { + const { + q, + bbox, + datetime, + sortby, + limit, + token + } = params; + + // Base SELECT columns. We may append a relevance `rank` column below when `q` is present. + // + // Rationale: we build the SELECT portion separately into `selectPart` so that + // we can conditionally append computed columns (for example the `rank` from + // full-text search) *before* the `FROM` clause. Keeping `sql` fixed with a + // `FROM` already included would make inserting additional selected columns + // harder and error-prone when building the query dynamically. + let selectPart = ` + SELECT + id, + stac_version, + type, + title, + description, + license, + spatial_extend, + temporal_extend_start, + temporal_extend_end, + created_at, + updated_at, + is_api, + is_active, + full_json + `; + + const where = []; + const values = []; + let i = 1; + + // Full-text search using weighted tsvector across title (weight A) and description (weight B). + // + // Notes: + // - Currently only title and description are included in the weighted tsvector. + // Collection keywords must also participate in full-text search. + // This will be added once the database team finalizes how keywords should be aggregated (JOIN + string_agg or dedicated tsvector). + // - We use `plainto_tsquery` to convert user-entered text into a tsquery. This keeps + // behaviour simple and predictable for short queries entered by users. + // - `ts_rank_cd` computes a relevance score; we add it to the SELECT list as `rank` + // so it can be used for ordering (when no explicit `sortby` is provided). + // - currently we are using on-the-fly tsvector expressions (matching to the 05_indexes.sql): + // A persistant tsvector collumn could be added later for large-scale indexing (watch Database Issues) + // + // Use the same parameter index for both the WHERE clause and the computed rank so the + // prepared statement uses a single bind parameter for the query text. + if (q) { + const queryIndex = i; // remember index to reuse for rank and condition + + // Weighted combined tsvector expression + const vectorExpr = `to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))`; + + // Add rank to selected columns (ts_rank_cd => constant-duration ranking function) + // The computed `rank` is available in the result rows and used for ordering + // when no explicit `sortby` is provided. + selectPart += `, ts_rank_cd(${vectorExpr}, plainto_tsquery('simple', $${queryIndex})) AS rank`; + + // WHERE clause uses plainto_tsquery for user-entered search text + where.push(`${vectorExpr} @@ plainto_tsquery('simple', $${queryIndex})`); + + values.push(q); + i++; + } + + // BBOX with PostGIS + if (bbox) { + const [minX, minY, maxX, maxY] = bbox; + + where.push(` + ST_Intersects( + spatial_extend, + ST_MakeEnvelope($${i}, $${i + 1}, $${i + 2}, $${i + 3}, 4326) + ) + `); + + values.push(minX, minY, maxX, maxY); + i += 4; + } + + // datetime: Point or interval + if (datetime) { + if (datetime.includes('/')) { + // interval: start/end, ../end, start/.. + const [start, end] = datetime.split('/'); + + if (start !== '..') { + // Collection should run after start + where.push(`temporal_extend_end >= $${i}`); + values.push(start); + i++; + } + + if (end !== '..') { + // Collection should run before end + where.push(`temporal_extend_start <= $${i}`); + values.push(end); + i++; + } + } else { + // single datetime: collections active at that time + where.push(` + temporal_extend_start <= $${i} + AND temporal_extend_end >= $${i} + `); + values.push(datetime); + i++; + } + } + + // Build final SQL from selectPart and add FROM clause. + // We delayed adding `FROM collection` to allow conditional additions to the + // selected columns above (notably `rank`). The final `sql` string includes the + // selected columns, the source table and any WHERE conditions constructed earlier. + let sql = selectPart + `\n FROM collection\n `; + + if (where.length > 0) { + sql += ` WHERE ` + where.join(' AND '); + } + + // Sorting: if a sort is explicitly requested use it; otherwise prefer relevance when + // a text query was provided (descending), falling back to id ascending. + // + // Behaviour summary: + // - `sortby` provided → use that (same as before) + // - no `sortby` & `q` present → order by `rank DESC, id ASC` so higher relevance comes first + // - no `sortby` & no `q` → order by `id ASC` (legacy default) + if (sortby) { + sql += ` ORDER BY ${sortby.field} ${sortby.direction}`; + } else if (q) { + sql += ` ORDER BY rank DESC, id ASC`; + } else { + sql += ` ORDER BY id ASC`; + } + + // Pagination (only add if limit is provided) + if (limit !== null && limit !== undefined) { + sql += ` LIMIT $${i} OFFSET $${i + 1}`; + values.push(limit, token || 0); + } + + return { sql, values }; +} + +module.exports = { buildCollectionSearchQuery }; diff --git a/api/db/db_APIconnection.js b/api/db/db_APIconnection.js new file mode 100644 index 0000000..b3a93fc --- /dev/null +++ b/api/db/db_APIconnection.js @@ -0,0 +1,268 @@ +const { Pool } = require('pg'); +require('dotenv').config(); + +// PostgreSQL/PostGIS database connection +// Support both DATABASE_URL and individual environment variables +let pool; + +// Pool configuration with connection limits and timeouts +const poolConfig = { + max: parseInt(process.env.DB_POOL_MAX), // Maximum number of clients in the pool + min: parseInt(process.env.DB_POOL_MIN), // Minimum number of clients in the pool + idleTimeoutMillis: parseInt(process.env.DB_IDLE_TIMEOUT), // Close time for idle clients + connectionTimeoutMillis: parseInt(process.env.DB_CONNECTION_TIMEOUT), // Waiting time before timing out + allowExitOnIdle: false // Keep the pool alive even when all clients are idle +}; + +if (process.env.DATABASE_URL) { + // Use DATABASE_URL if provided + pool = new Pool({ + connectionString: process.env.DATABASE_URL, + ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false, + + }); +} else { + // Fallback to individual environment variables + const requiredEnvVars = ['DB_HOST', 'DB_PORT', 'DB_NAME', 'DB_USER', 'DB_PASSWORD']; + const missingVars = requiredEnvVars.filter(varName => !process.env[varName]); + if (missingVars.length > 0) { + throw new Error(`Missing required environment variables: ${missingVars.join(', ')} or DATABASE_URL`); + } + + pool = new Pool({ + host: process.env.DB_HOST, + port: parseInt(process.env.DB_PORT), + database: process.env.DB_NAME, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false, + ...poolConfig + }); +} + +// Handle pool errors +pool.on('error', (err) => { + console.error('Unexpected database pool error:', err); +}); + +// Handle pool connection events for monitoring (only in non-test environments) +if (process.env.NODE_ENV !== 'test') { + pool.on('connect', (client) => { + console.log('New client connected to pool'); + }); + + pool.on('acquire', (client) => { + console.log('Client acquired from pool'); + }); + + pool.on('remove', (client) => { + console.log('Client removed from pool'); + }); +} + +// Graceful shutdown handlers +process.on('SIGTERM', async () => { + console.log('SIGTERM received, closing database pool...'); + await closePool(); + process.exit(0); +}); + +process.on('SIGINT', async () => { + console.log('SIGINT received, closing database pool...'); + await closePool(); + process.exit(0); +}); + +// execute query +async function query(text, params = []) { + try { + const result = await pool.query(text, params); + return result; + } catch (error) { + // log detailed error information + console.error('Database query error:', { + message: error.message, + code: error.code, + detail: error.detail, + query: text.substring(0, 100) + (text.length > 100 ? '...' : '') + }); + + // throw enhanced error + const enhancedError = new Error(`Database query failed: ${error.message}`); + enhancedError.code = error.code; + enhancedError.detail = error.detail; + enhancedError.originalError = error; + throw enhancedError; + } +} + +// Connection test with retry logic and pool info +async function testConnection(retries = 3, delay = 2000) { + for (let i = 0; i < retries; i++) { + try { + const result = await pool.query('SELECT 1 as connected, version() as version, current_database() as database'); + const poolInfo = { + totalCount: pool.totalCount, + idleCount: pool.idleCount, + waitingCount: pool.waitingCount + }; + + console.log('✓ Database connection successful'); + console.log(` Database: ${result.rows[0].database}`); + console.log(` PostgreSQL version: ${result.rows[0].version.split(',')[0]}`); + console.log(` Pool status: ${poolInfo.totalCount} total, ${poolInfo.idleCount} idle, ${poolInfo.waitingCount} waiting`); + return true; + } catch (error) { + console.error(`✗ Connection attempt ${i + 1}/${retries} failed:`, error.message); + + if (i < retries - 1) { + console.log(` Retrying in ${delay / 1000} seconds...`); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + } + + console.error('✗ All connection attempts failed'); + return false; +} + +// Get current pool statistics +function getPoolStats() { + return { + total: pool.totalCount, + idle: pool.idleCount, + waiting: pool.waitingCount + }; +} + +// PostGIS: Bounding Box Query +// @param {string} table - table name +// @param {Array} bbox - [west, south, east, north] +// @param {string} geomColumn - name of the geometry column (default: spatial_extend) +// @returns {Promise} query result +async function queryByBBox(table, bbox, geomColumn = 'spatial_extend') { + const [west, south, east, north] = bbox; + + // validate bbox ranges + if (west < -180 || west > 180 || east < -180 || east > 180) { + throw new Error('Longitude must be between -180 and 180'); + } + if (south < -90 || south > 90 || north < -90 || north > 90) { + throw new Error('Latitude must be between -90 and 90'); + } + if (west >= east) { + throw new Error('West coordinate must be less than east coordinate'); + } + if (south >= north) { + throw new Error('South coordinate must be less than north coordinate'); + } + + try { + const sql = ` + SELECT * FROM ${table} + WHERE ST_Intersects( + ${geomColumn}, + ST_MakeEnvelope($1, $2, $3, $4, 4326) + ) + `; + return await query(sql, [west, south, east, north]); + } catch (error) { + throw new Error(`BBox query failed: ${error.message}`); + } +} + +// PostGIS: Geometry Query +// @param {string} table - table name +// @param {Object} geojson - GeoJSON Geometry +// @param {string} predicate - Spatial Predicate (intersects, contains, within) +// @param {string} geomColumn - name of the geometry column (default: spatial_extend) +// @returns {Promise} query result +async function queryByGeometry(table, geojson, predicate = 'intersects', geomColumn = 'spatial_extend') { + // validate inputs + if (!table || typeof table !== 'string') { + throw new Error('Table name must be a non-empty string'); + } + if (!geojson || typeof geojson !== 'object') { + throw new Error('GeoJSON must be a valid object'); + } + if (!geojson.type || !geojson.coordinates) { + throw new Error('GeoJSON must have type and coordinates properties'); + } + + const predicates = { + intersects: 'ST_Intersects', + contains: 'ST_Contains', + within: 'ST_Within' + }; + + if (!predicates[predicate.toLowerCase()]) { + throw new Error(`Invalid predicate: ${predicate}. Must be one of: ${Object.keys(predicates).join(', ')}`); + } + + const func = predicates[predicate.toLowerCase()]; + + try { + const sql = ` + SELECT * FROM ${table} + WHERE ${func}( + ${geomColumn}, + ST_SetSRID(ST_GeomFromGeoJSON($1), 4326) + ) + `; + return await query(sql, [JSON.stringify(geojson)]); + } catch (error) { + throw new Error(`Geometry query failed: ${error.message}`); + } +} + +// PostGIS: Distance Query +// @param {string} table - table name +// @param {Array} point - [lon, lat] +// @param {number} distance - distance in meters +// @param {string} geomColumn - name of the geometry column (default: spatial_extend) +// @returns {Promise} query result +async function queryByDistance(table, point, distance, geomColumn = 'spatial_extend') { + const [lon, lat] = point; + + // Use geometry type with ST_Centroid to avoid antipodal edge errors + // ST_Centroid provides a single point from potentially large geometries + const sql = ` + SELECT *, + ST_Distance( + ST_Centroid(${geomColumn})::geography, + ST_SetSRID(ST_Point($1, $2), 4326)::geography + ) as distance + FROM ${table} + WHERE ST_DWithin( + ST_Centroid(${geomColumn})::geography, + ST_SetSRID(ST_Point($1, $2), 4326)::geography, + $3 + ) + ORDER BY distance + `; + return await query(sql, [lon, lat, distance]); +} + +// close connection +async function closePool() { + try { + await pool.end(); + console.log('✓ Database connection pool closed'); + } catch (error) { + console.error('Error closing database pool:', error.message); + throw error; + } +} + +module.exports = { + pool, + query, + testConnection, + closePool, + getPoolStats, + + // PostGIS functions + queryByBBox, + queryByGeometry, + queryByDistance +}; diff --git a/api/docs/collection-search-parameters.md b/api/docs/collection-search-parameters.md new file mode 100644 index 0000000..d9c520f --- /dev/null +++ b/api/docs/collection-search-parameters.md @@ -0,0 +1,262 @@ +# Collection Search Parameters + +This document describes the query parameters supported by the STAC Atlas Collection Search API (`GET /collections`). + +## Overview + +The Collection Search endpoint supports filtering and pagination through query parameters. All parameters are optional and can be combined to refine search results. + +## Supported Parameters + +### `q` - Free-Text Search + +**Type:** String +**Required:** No +**Description:** Free-text search across collection `title`, `description`, and `keywords` fields. + +**Constraints:** +- Maximum length: 500 characters +- Whitespace is trimmed + +**Examples:** +``` +GET /collections?q=sentinel +GET /collections?q=landsat%20Münster +``` + +**Implementation Note:** When database is connected, this will use PostgreSQL full-text search (TSVector) for efficient matching. + +--- + +### `bbox` - Bounding Box Filter + +**Type:** String (comma-separated) or Array +**Required:** No +**Format:** `minX,minY,maxX,maxY` or `[west, south, east, north]` + +**Description:** Spatial filter to find collections whose spatial extent intersects with the specified bounding box. + +**Constraints:** +- Must contain exactly 4 coordinates +- Longitude (X): -180 to 180 +- Latitude (Y): -90 to 90 +- minX < maxX +- minY < maxY + +**Examples:** +``` +GET /collections?bbox=-10,40,10,50 +GET /collections?bbox=-122.4,37.8,-122.3,37.9 +``` + +**Implementation Note:** Will use PostGIS spatial intersection queries (`ST_Intersects`) when database is connected. + +--- + +### `datetime` - Temporal Filter + +**Type:** String (ISO8601) +**Required:** No +**Description:** Temporal filter to find collections whose temporal extent overlaps with the specified time range. + +**Formats Supported:** +1. **Single datetime:** `2020-01-01T00:00:00Z` +2. **Closed interval:** `2019-01-01/2021-12-31` +3. **Open start:** `../2021-12-31` (all collections ending before date) +4. **Open end:** `2019-01-01/..` (all collections starting after date) + +**Constraints:** +- Must be valid ISO8601 format +- Intervals must have exactly one `/` separator +- Cannot be unbounded on both sides (`../..` is invalid) + +**Examples:** +``` +GET /collections?datetime=2020-01-01T00:00:00Z +GET /collections?datetime=2019-01-01/2021-12-31 +GET /collections?datetime=../2021-12-31 +GET /collections?datetime=2020-06-01/.. +``` + +**Implementation Note:** Will query `temporal_extent_start` and `temporal_extent_end` columns with overlap logic. + +--- + +### `limit` - Result Limit + +**Type:** Integer +**Required:** No +**Default:** 10 +**Description:** Maximum number of collections to return in a single response. + +**Constraints:** +- Minimum: 1 +- Maximum: 10000 +- Default: 10 + +**Examples:** +``` +GET /collections?limit=50 +GET /collections?limit=100 +``` + +**Pagination Note:** Use together with `token` parameter to paginate through large result sets. + +--- + +### `sortby` - Sort Order + +**Type:** String +**Required:** No +**Format:** `[+|-]field` +**Description:** Specifies the field and direction for sorting results. + +**Direction Syntax:** +- `+field` or `field` = Ascending order (A-Z, 0-9) +- `-field` = Descending order (Z-A, 9-0) + +**Allowed Fields:** +- `title` - Collection title (alphabetical) +- `id` - Collection identifier +- `license` - License identifier +- `created` - Creation timestamp +- `updated` - Last update timestamp + +**Examples:** +``` +GET /collections?sortby=title # Ascending by title (default) +GET /collections?sortby=+title # Explicit ascending +GET /collections?sortby=-created # Newest first +GET /collections?sortby=-updated # Most recently updated first +``` + +**Default Behavior:** When no `sortby` is specified, results are returned in database order (typically by ID). + +--- + +### `token` - Pagination Token + +**Type:** Integer +**Required:** No +**Default:** 0 +**Description:** Pagination continuation token (offset) to retrieve the next page of results. + +**Constraints:** +- Must be non-negative integer +- Value represents the offset into the result set + +**Examples:** +``` +GET /collections?limit=10&token=0 # First page (results 0-9) +GET /collections?limit=10&token=10 # Second page (results 10-19) +GET /collections?limit=50&token=100 # Results 100-149 +``` + +**Pagination Workflow:** +1. Initial request: `GET /collections?limit=10` +2. Response includes `links` with `rel: "next"` containing next token +3. Follow next link: `GET /collections?limit=10&token=10` +4. Repeat until no `next` link is present + +**Response Links:** +```json +{ + "collections": [...], + "links": [ + { "rel": "self", "href": "/collections?limit=10&token=0" }, + { "rel": "next", "href": "/collections?limit=10&token=10" }, + { "rel": "prev", "href": "/collections?limit=10&token=0" } + ], + "context": { + "returned": 10, + "limit": 10, + "matched": 156 + } +} +``` + +--- + +## Combining Parameters + +Multiple parameters can be combined to create complex queries: + +``` +GET /collections?q=sentinel&bbox=-10,40,10,50&datetime=2020-01-01/2021-12-31&limit=20&sortby=-created +``` + +This query searches for: +- Collections matching "sentinel" +- Within the specified bounding box +- With temporal extent overlapping 2020-2021 +- Returns 20 results +- Sorted by creation date (newest first) + +--- + +## Error Responses + +All validation errors return HTTP **400 Bad Request** with the following format: + +```json +{ + "code": "InvalidParameterValue", + "description": "Parameter \"bbox\" minX must be less than maxX" +} +``` + +Multiple errors are concatenated: + +```json +{ + "code": "InvalidParameterValue", + "description": "Parameter \"limit\" must be at least 1; Parameter \"bbox\" contains invalid numeric values" +} +``` + +--- + +## Conformance Classes + +This API implements the following STAC Collection Search conformance classes: + +- **Simple Query** (`http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/simple-query`) + - Parameters: `bbox`, `datetime`, `limit` + +- **Free-Text Search** (`https://api.stacspec.org/v1.0.0-rc.1/collection-search#free-text`) + - Parameter: `q` + +- **Sorting** (`https://api.stacspec.org/v1.1.0/collection-search#sort`) + - Parameter: `sortby` + +--- + +## Implementation Status + +| Parameter | Status | Notes | +|-----------|--------|-------| +| `q` | Validated | TODO: Implement full-text search in DB | +| `bbox` | Validated | TODO: Implement PostGIS spatial query | +| `datetime` | Validated | TODO: Implement temporal overlap query | +| `limit` | Implemented | Working with in-memory store | +| `sortby` | Validated | TODO: Apply sorting in DB query | +| `token` | Implemented | Working with in-memory store | + +--- + +## Future Extensions + +The following parameters are defined in `bid.md` but not yet implemented: + +- `provider` - Filter by data provider name +- `license` - Filter by license identifier + +These will be added in a future release as extended search parameters beyond the standard conformance classes. Or the bid will be changed with a change-request. + +--- + +## See Also + +- [STAC API Specification](https://github.com/radiantearth/stac-api-spec) +- [Collection Search Extension](https://github.com/stac-api-extensions/collection-search) +- [OGC API - Features](https://docs.ogc.org/is/17-069r4/17-069r4.html) diff --git a/api/examples/README.md b/api/examples/README.md new file mode 100644 index 0000000..d71fc95 --- /dev/null +++ b/api/examples/README.md @@ -0,0 +1,62 @@ +# Routes - Database Integration Guide + +This guide explains how to connect API routes to the database using the existing database connection module. + +## Basic Pattern + +Define this helper function once at the top of your route file: + +```javascript +const express = require('express'); +const router = express.Router(); +const db = require('../db/db_APIconnection'); + +// Define once per file +async function runQuery(sql, params = []) { + try { + const result = await db.query(sql, params); + return result.rows; + } catch (error) { + console.error('Query error:', error); + throw error; + } +} + +// Now use it everywhere in this file +router.get('/endpoint', async (req, res, next) => { + try { + const rows = await runQuery('SELECT * FROM table WHERE id = $1', [req.params.id]); + res.json(rows); + } catch (error) { + next(error); + } +}); + +module.exports = router; +``` + +Every database call in your routes can now use this simple pattern: + +```javascript +const rows = await runQuery('SELECT * FROM table WHERE id = $1', [123]); +``` + +## Example + +```javascript +// get list of collections + +const collections = await runQuery('SELECT * FROM collection'); + +// find collection via ID +const rows = await runQuery('SELECT * FROM collection WHERE id = $1', [123]); +if (rows.length === 0) { + return res.status(404).json({ code: 'NotFound' }); +} +const collection = rows[0]; + +// Filter by multiple conditions +const filtered = await runQuery( + 'SELECT * FROM collection WHERE is_active = $1 AND license = $2', + [true, 'CC-BY-4.0'] +); \ No newline at end of file diff --git a/api/jest.config.js b/api/jest.config.js new file mode 100644 index 0000000..9cb9ab0 --- /dev/null +++ b/api/jest.config.js @@ -0,0 +1,18 @@ +module.exports = { + testEnvironment: 'node', + coverageDirectory: 'coverage', + collectCoverageFrom: [ + 'routes/**/*.js', + 'controllers/**/*.js', + 'services/**/*.js', + '!node_modules/**' + ], + testMatch: ['**/__tests__/**/*.js', '**/?(*.)+(spec|test).js'], + verbose: true, + // Force exit after tests to prevent hanging + forceExit: true, + // Detect open handles (useful for debugging) + detectOpenHandles: false, + // Increase test timeout for slow database queries in CI + testTimeout: 30000 +}; diff --git a/api/jest.teardown.js b/api/jest.teardown.js new file mode 100644 index 0000000..8e2bb26 --- /dev/null +++ b/api/jest.teardown.js @@ -0,0 +1,14 @@ +// jest.teardown.js +// Global teardown to close database connections after all tests + +const { closePool } = require('./db/db_APIconnection'); + +module.exports = async () => { + // Close the database connection pool + try { + await closePool(); + console.log('Jest teardown: Database pool closed successfully'); + } catch (error) { + console.error('Jest teardown: Error closing database pool:', error.message); + } +}; diff --git a/api/middleware/validateCollectionSearch.js b/api/middleware/validateCollectionSearch.js new file mode 100644 index 0000000..f19aa5a --- /dev/null +++ b/api/middleware/validateCollectionSearch.js @@ -0,0 +1,100 @@ +// middleware/validateCollectionSearch.js + +const { + validateQ, + validateBbox, + validateDatetime, + validateLimit, + validateSortby, + validateToken +} = require('../validators/collectionSearchParams'); + +/** + * Express middleware to validate Collection Search query parameters + * + * Validates all supported query parameters and returns 400 with detailed + * error message if validation fails. On success, attaches normalized + * parameters to req.validatedParams for use in route handlers. + * + * Supported parameters: + * - q: Free-text search + * - bbox: Bounding box spatial filter + * - datetime: Temporal filter (ISO8601) + * - limit: Result limit (default 10, max 10000) + * - sortby: Sort specification (+/-field) + * - token: Pagination continuation token + * + * @param {Request} req - Express request object + * @param {Response} res - Express response object + * @param {Function} next - Express next middleware function + */ +function validateCollectionSearchParams(req, res, next) { + const errors = []; + const normalized = {}; + + // Extract query parameters + const { q, bbox, datetime, limit, sortby, token } = req.query; + + // Validate q (free-text search) + const qResult = validateQ(q); + if (!qResult.valid) { + errors.push(qResult.error); + } else if (qResult.normalized !== undefined) { + normalized.q = qResult.normalized; + } + + // Validate bbox (spatial filter) + const bboxResult = validateBbox(bbox); + if (!bboxResult.valid) { + errors.push(bboxResult.error); + } else if (bboxResult.normalized) { + normalized.bbox = bboxResult.normalized; + } + + // Validate datetime (temporal filter) + const datetimeResult = validateDatetime(datetime); + if (!datetimeResult.valid) { + errors.push(datetimeResult.error); + } else if (datetimeResult.normalized !== undefined) { + normalized.datetime = datetimeResult.normalized; + } + + // Validate limit (pagination) + const limitResult = validateLimit(limit); + if (!limitResult.valid) { + errors.push(limitResult.error); + } else { + normalized.limit = limitResult.normalized; + } + + // Validate sortby (sorting) + const sortbyResult = validateSortby(sortby); + if (!sortbyResult.valid) { + errors.push(sortbyResult.error); + } else if (sortbyResult.normalized) { + normalized.sortby = sortbyResult.normalized; + } + + // Validate token (pagination continuation) + const tokenResult = validateToken(token); + if (!tokenResult.valid) { + errors.push(tokenResult.error); + } else { + normalized.token = tokenResult.normalized; + } + + // If any validation errors occurred, return 400 with details + if (errors.length > 0) { + return res.status(400).json({ + code: 'InvalidParameterValue', + description: errors.join('; ') + }); + } + + // Attach normalized params to request for use in route handler + req.validatedParams = normalized; + + next(); +} + +module.exports = { validateCollectionSearchParams }; diff --git a/api/package-lock.json b/api/package-lock.json new file mode 100644 index 0000000..a12b09e --- /dev/null +++ b/api/package-lock.json @@ -0,0 +1,6372 @@ +{ + "name": "stac-atlas-api", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stac-atlas-api", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "cors": "^2.8.5", + "debug": "~2.6.9", + "dotenv": "^17.2.3", + "express": "^4.22.1", + "morgan": "^1.10.1", + "pg": "^8.16.3", + "swagger-ui-express": "^5.0.1", + "yamljs": "^0.3.0" + }, + "devDependencies": { + "eslint": "^8.57.1", + "jest": "^29.7.0", + "nodemon": "^3.1.11", + "prettier": "^3.6.2", + "supertest": "^7.1.4" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.5.tgz", + "integrity": "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", + "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.28.3", + "@babel/helpers": "^7.28.4", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/core/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/generator": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.5.tgz", + "integrity": "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", + "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.28.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.4.tgz", + "integrity": "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.5" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz", + "integrity": "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", + "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", + "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.5.tgz", + "integrity": "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.5", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/traverse/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/types": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", + "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@eslint/eslintrc/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@eslint/js": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", + "deprecated": "Use @eslint/config-array instead", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.3", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/source-map": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@paralleldrive/cuid2": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.3.1.tgz", + "integrity": "sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@noble/hashes": "^1.1.5" + } + }, + "node_modules/@scarf/scarf": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", + "integrity": "sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ==", + "hasInstallScript": true, + "license": "Apache-2.0" + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/node": { + "version": "24.10.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.0.tgz", + "integrity": "sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/yargs": { + "version": "17.0.34", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.34.tgz", + "integrity": "sha512-KExbHVa92aJpw9WDQvzBaGVE2/Pz+pLZQloT2hjL8IqsZnV62rlPOYvNnLmf/L2dyllfVUOVBj64M0z/46eR2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "dev": true, + "license": "ISC" + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", + "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0 || ^8.0.0-0" + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.8.26", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.26.tgz", + "integrity": "sha512-73lC1ugzwoaWCLJ1LvOgrR5xsMLTqSKIEoMHVtL9E/HNk0PXtTM76ZIm84856/SF7Nv8mPZxKoBsgpm0tR1u1Q==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/body-parser": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.1.4" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001754", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001754.tgz", + "integrity": "sha512-x6OeBXueoAceOmotzx3PO4Zpt4rzpeIFsSr6AAePTZxSkXiYDUmpypEl7e2+8NCd9bD7bXjqyef8CJYPC1jfxg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", + "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/component-emitter": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz", + "integrity": "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-disposition/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "license": "MIT" + }, + "node_modules/cookiejar": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz", + "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==", + "dev": true, + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/dedent": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", + "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dev": true, + "license": "ISC", + "dependencies": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.250", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.250.tgz", + "integrity": "sha512-/5UMj9IiGDMOFBnN4i7/Ry5onJrAGSbOGo3s9FEKmwobGq6xw832ccET0CE3CkkMBZ8GJSlUIesZofpyurqDXw==", + "dev": true, + "license": "ISC" + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/eslint/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esquery": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/express/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/fastq": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz", + "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true, + "license": "ISC" + }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/formidable": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz", + "integrity": "sha512-YikH+7CUTOtP44ZTnUhR7Ic2UASBPOqmaRkRKxRbywPTe5VxF7RRCck4af9wutiZ/QKM5nME9Bie2fFaPz5Gug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@paralleldrive/cuid2": "^2.2.2", + "dezalgo": "^1.0.4", + "once": "^1.4.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true, + "license": "ISC" + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "execa": "^5.0.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-circus": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-config": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", + "parse-json": "^5.2.0", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/js-yaml/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/morgan": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.1.tgz", + "integrity": "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==", + "license": "MIT", + "dependencies": { + "basic-auth": "~2.0.1", + "debug": "2.6.9", + "depd": "~2.0.0", + "on-finished": "~2.3.0", + "on-headers": "~1.1.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nodemon": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.11.tgz", + "integrity": "sha512-is96t8F/1//UHAjNPHpbsNY46ELPpftGUoSVNXwUfMk/qdjSylYrWSu1XavVTBOn526kFiOR733ATgNBCQyH0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.1.2", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/nodemon/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/nodemon/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nodemon/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/nodemon/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "license": "MIT" + }, + "node_modules/pg": { + "version": "8.16.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", + "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.3", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.2.7" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz", + "integrity": "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", + "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prettier": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true, + "license": "MIT" + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.11", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", + "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-cwd/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/send": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.1.tgz", + "integrity": "sha512-p4rRk4f23ynFEfcD9LA0xRYngj+IyGiEYyqqOak8kaN0TvNmuxC2dcVeBn62GpCeR2CpWqyHCNScTP91QbAVFg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/send/node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-static/node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static/node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static/node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-static/node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-update-notifier/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/stack-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/superagent": { + "version": "10.2.3", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-10.2.3.tgz", + "integrity": "sha512-y/hkYGeXAj7wUMjxRbB21g/l6aAEituGXM9Rwl4o20+SX3e8YOSV6BxFXl+dL3Uk0mjSL3kCbNkwURm8/gEDig==", + "dev": true, + "license": "MIT", + "dependencies": { + "component-emitter": "^1.3.1", + "cookiejar": "^2.1.4", + "debug": "^4.3.7", + "fast-safe-stringify": "^2.1.1", + "form-data": "^4.0.4", + "formidable": "^3.5.4", + "methods": "^1.1.2", + "mime": "2.6.0", + "qs": "^6.11.2" + }, + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/superagent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/superagent/node_modules/mime": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", + "dev": true, + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/superagent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/supertest": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/supertest/-/supertest-7.1.4.tgz", + "integrity": "sha512-tjLPs7dVyqgItVFirHYqe2T+MfWc2VOBQ8QFKKbWTA3PU7liZR8zoSpAi/C1k1ilm9RsXIKYf197oap9wXGVYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "methods": "^1.1.2", + "superagent": "^10.2.3" + }, + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/swagger-ui-dist": { + "version": "5.30.2", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.30.2.tgz", + "integrity": "sha512-HWCg1DTNE/Nmapt+0m2EPXFwNKNeKK4PwMjkwveN/zn1cV2Kxi9SURd+m0SpdcSgWEK/O64sf8bzXdtUhigtHA==", + "license": "Apache-2.0", + "dependencies": { + "@scarf/scarf": "=1.4.0" + } + }, + "node_modules/swagger-ui-express": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/swagger-ui-express/-/swagger-ui-express-5.0.1.tgz", + "integrity": "sha512-SrNU3RiBGTLLmFU8GIJdOdanJTl4TOmT27tt3bWWHppqYmAZ6IDuEuBvMU6nZq0zLEe6b/1rACXCgLZqO6ZfrA==", + "license": "MIT", + "dependencies": { + "swagger-ui-dist": ">=5.0.0" + }, + "engines": { + "node": ">= v0.10.32" + }, + "peerDependencies": { + "express": ">=4.0.0 || >=5.0.0-beta" + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "license": "ISC", + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true, + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", + "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yamljs": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/yamljs/-/yamljs-0.3.0.tgz", + "integrity": "sha512-C/FsVVhht4iPQYXOInoxUM/1ELSf9EsgKH34FofQOp6hwCPrW4vG4w5++TED3xRUo8gD7l0P1J1dLlDYzODsTQ==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "glob": "^7.0.5" + }, + "bin": { + "json2yaml": "bin/json2yaml", + "yaml2json": "bin/yaml2json" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/api/package.json b/api/package.json new file mode 100644 index 0000000..4f1ea4c --- /dev/null +++ b/api/package.json @@ -0,0 +1,43 @@ +{ + "name": "stac-atlas-api", + "version": "0.1.0", + "description": "STAC API for STAC Atlas - A centralized platform for managing STAC Collection metadata", + "private": true, + "scripts": { + "start": "node ./bin/www", + "dev": "nodemon ./bin/www", + "test": "jest", + "test:watch": "jest --watch", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "format": "prettier --write \"**/*.{js,json,md}\"" + }, + "keywords": [ + "stac", + "api", + "geospatial", + "collections" + ], + "author": "STAC Atlas Team", + "license": "Apache-2.0", + "dependencies": { + "cors": "^2.8.5", + "debug": "~2.6.9", + "dotenv": "^17.2.3", + "express": "^4.22.1", + "morgan": "^1.10.1", + "pg": "^8.16.3", + "swagger-ui-express": "^5.0.1", + "yamljs": "^0.3.0" + }, + "devDependencies": { + "eslint": "^8.57.1", + "jest": "^29.7.0", + "nodemon": "^3.1.11", + "prettier": "^3.6.2", + "supertest": "^7.1.4" + }, + "engines": { + "node": ">=22.0.0" + } +} diff --git a/api/routes/collections.js b/api/routes/collections.js new file mode 100644 index 0000000..6f83a2f --- /dev/null +++ b/api/routes/collections.js @@ -0,0 +1,174 @@ +const express = require('express'); +const router = express.Router(); +const collectionsStore = require('../data/collections'); // change with the real collections when we have them +const { validateCollectionSearchParams } = require('../middleware/validateCollectionSearch'); +const { query } = require('../db/db_APIconnection'); +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +// helper to run the built query (from documentation) +async function runQuery(sql, params = []) { + try { + const result = await query(sql, params); + return result.rows; + } catch (error) { + console.error('Query error in /collections:', error); + throw error; + } +} + +/** + * GET /collections + * Returns a paginated list of collections with optional filtering + * + * Supported query parameters: + * - q: Free-text search across title, description, keywords + * - bbox: Spatial filter as minX,minY,maxX,maxY + * - datetime: Temporal filter (ISO8601 single or interval) + * - limit: Number of results (default 10, max 10000) + * - sortby: Sort by field (+field for ASC, -field for DESC) + * - token: Pagination continuation token (offset) + * + * All parameters are validated by validateCollectionSearchParams middleware. + * Validated/normalized values are available in req.validatedParams. + */ +router.get('/', validateCollectionSearchParams, async (req, res, next) => { + // TODO: Think about the parameters `provider` and `license` - They are mentioned in the bid, but not in the STAC spec + // TODO: Implement CQL2 filtering (GET endpoint) and add validator for `filter`, filter-lang` parameters + try { + // validated parameters from middleware + const { q, bbox, datetime, limit, sortby, token } = req.validatedParams; + + // build SQL querry and parameters + const { sql, values } = buildCollectionSearchQuery({ + q, + bbox, + datetime, + limit, + sortby, + token + }); + + // execute Query against database + const collections = await runQuery(sql, values); + const returned = collections.length; + + // Get total count for matched field + // Build count query using same WHERE conditions + const { sql: countSql, values: countValues } = buildCollectionSearchQuery({ + q, + bbox, + datetime, + limit: null, // No limit for count + sortby: null, // No sorting for count + token: null // No offset for count + }); + + // Replace SELECT with COUNT(*) + const countQuery = countSql + .replace(/SELECT[\s\S]*?FROM/, 'SELECT COUNT(*) as total FROM') + .replace(/ORDER BY.*$/, '') + .replace(/LIMIT.*$/, ''); + + const countResult = await runQuery(countQuery, countValues); + const matched = parseInt(countResult[0]?.total || 0); + + // Base URL for links + const baseHost = `${req.protocol}://${req.get('host')}`; + const baseUrl = `${baseHost}${req.baseUrl}`; + + const buildLink = (rel, tokenValue) => ({ + rel, + href: `${baseUrl}?limit=${limit}&token=${tokenValue}`, + type: 'application/json' + }); + + const links = [ + buildLink('self', token), + { + rel: 'root', + href: baseHost, + type: 'application/json' + } + ]; + + // "next": only if returned === limit AND token + limit < matched + if (returned === limit && token + limit < matched) { + links.push(buildLink('next', token + limit)); + } + + // "prev": only if token > 0 + if (token > 0) { + const prevToken = Math.max(0, token - limit); + links.push(buildLink('prev', prevToken)); + } + + res.json({ + type: 'FeatureCollection', + collections, + links, + context: { + returned, + limit, + matched + } + }); + } catch (error) { + next(error); + } +}); + +/** + * GET /collections/:id + * Returns a single collection by ID. Includes all STAC Collection fields + * (stac_version, type, title, description, license, extent, links, etc). + * + * Returns: + * - 200 OK with full Collection object if found + * - 404 NotFound with proper error format if collection does not exist + */ +router.get('/:id', (req, res) => { + // TODO: Create a proper validator middleware for :id parameter to avoid SQL injection, etc. + const { id } = req.params; + + // Look up the collection in the data store by ID + // When connected to a DB, replace this with a SQL query (SELECT * FROM collections WHERE id = ?) + const collection = collectionsStore.find(c => c.id === id); + + if (!collection) { + // Return 404 with standardized error format + return res.status(404).json({ + code: 'NotFound', + description: `Collection with id '${id}' not found`, + id: id + }); + } + + // Return the full STAC Collection object + // Ensure the response includes at least self, root and parent links. + // Start from any links the collection already provides and add missing ones. + const baseHost = `${req.protocol}://${req.get('host')}`; + const selfHref = `${baseHost}/collections/${id}`; + const rootHref = baseHost; + + const existingLinks = Array.isArray(collection.links) ? collection.links.slice() : []; + + const hasRel = (rel) => existingLinks.some(l => l && l.rel === rel); + + if (!hasRel('self')) { + existingLinks.push({ rel: 'self', href: selfHref, type: 'application/json' }); + } + + if (!hasRel('root')) { + existingLinks.push({ rel: 'root', href: rootHref, type: 'application/json' }); + } + + // Prefer an existing parent link if present, otherwise fall back to root + if (!hasRel('parent')) { + existingLinks.push({ rel: 'parent', href: rootHref, type: 'application/json' }); + } + + // Return the collection with a normalized `links` array + res.json(Object.assign({}, collection, { links: existingLinks })); +}); + +module.exports = router; diff --git a/api/routes/conformance.js b/api/routes/conformance.js new file mode 100644 index 0000000..c9b35b1 --- /dev/null +++ b/api/routes/conformance.js @@ -0,0 +1,15 @@ +const express = require('express'); +const router = express.Router(); +const { CONFORMANCE_URIS } = require('../config/conformanceURIS'); + +/** + * GET /conformance + * Returns the list of conformance classes this API implements + */ +router.get('/', (req, res) => { + res.json({ + conformsTo: CONFORMANCE_URIS + }); +}); + +module.exports = router; diff --git a/api/routes/index.js b/api/routes/index.js new file mode 100644 index 0000000..b389488 --- /dev/null +++ b/api/routes/index.js @@ -0,0 +1,68 @@ +const express = require('express'); +const router = express.Router(); +const { CONFORMANCE_URIS } = require('../config/conformanceURIS'); + +/** + * GET / + * STAC API Landing Page + * Returns basic information about the API and available endpoints + * Source: https://docs.ogc.org/cs/25-005/25-005.html + */ +router.get('/', (req, res) => { + const baseUrl = `${req.protocol}://${req.get('host')}`; + + res.json({ + type: 'Catalog', + id: 'stac-atlas', + title: 'STAC Atlas', + description: 'A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs.', + stac_version: '1.0.0', + conformsTo: CONFORMANCE_URIS, + links: [ + { + rel: 'self', + href: baseUrl, + type: 'application/json', + title: 'STAC Atlas Landing Page' + }, + { + rel: 'root', + href: baseUrl, + type: 'application/json', + title: 'STAC Atlas root catalog' + }, + { + rel: 'conformance', + href: `${baseUrl}/conformance`, + type: 'application/json', + title: 'STAC/OGC conformance classes' + }, + { + rel: 'data', + href: `${baseUrl}/collections`, + type: 'application/json', + title: 'STAC Collections' + }, + { + rel: 'queryables', + href: `${baseUrl}/collection-queryables`, // TODO: Check with Mohr if this is the correct endpoint + type: 'application/schema+json', + title: 'Queryables for Collections' + }, + { + rel: 'service-doc', // This should be the Swagger UI or similar + href: `${baseUrl}/api-docs`, + type: 'text/html', + title: 'API documentation' + }, + { + rel: 'service-desc', // This should point to the OpenAPI spec (machine-readable) + href: `${baseUrl}/openapi.yaml`, + type: 'application/vnd.oai.openapi+json;version=3.0', + title: 'OpenAPI specification' + } + ] + }); +}); + +module.exports = router; diff --git a/api/routes/queryables.js b/api/routes/queryables.js new file mode 100644 index 0000000..1cfbe3b --- /dev/null +++ b/api/routes/queryables.js @@ -0,0 +1,92 @@ +const express = require('express'); +const router = express.Router(); + +/** + * GET /collections-queryables + * Returns the list of queryable properties for collections + */ +router.get('/', (req, res) => { + res.json({ + $schema: 'https://json-schema.org/draft/2019-09/schema', + $id: `${req.protocol}://${req.get('host')}/collections-queryables`, + type: 'object', + title: 'STAC Atlas Collections Queryables', + description: 'Queryable properties for STAC Collection Search', + properties: { + id: { + title: 'Collection ID', + type: 'string' + }, + title: { + title: 'Collection Title', + type: 'string' + }, + description: { + title: 'Collection Description', + type: 'string' + }, + keywords: { + title: 'Keywords', + type: 'array', + items: { + type: 'string' + } + }, + license: { + title: 'License', + type: 'string' + }, + providers: { + title: 'Providers', + type: 'array', + items: { + type: 'object', + properties: { + name: { + type: 'string' + } + } + } + }, + 'extent.spatial.bbox': { + title: 'Spatial Extent (Bounding Box)', + type: 'array', + items: { + type: 'number' + } + }, + 'extent.temporal.interval': { + title: 'Temporal Extent', + type: 'array' + }, + doi: { + title: 'DOI', + type: 'string' + }, + 'summaries.platform': { + title: 'Platform', + type: 'array', + items: { + type: 'string' + } + }, + 'summaries.constellation': { + title: 'Constellation', + type: 'array', + items: { + type: 'string' + } + }, + 'summaries.gsd': { + title: 'Ground Sample Distance', + type: 'number' + }, + 'summaries.processing:level': { + title: 'Processing Level', + type: 'string' + } + } + }); +}); + +module.exports = router; diff --git a/api/validators/collectionSearchParams.js b/api/validators/collectionSearchParams.js new file mode 100644 index 0000000..7f24faf --- /dev/null +++ b/api/validators/collectionSearchParams.js @@ -0,0 +1,274 @@ +/** + * Validators for STAC Collection Search query parameters + * + * Each validator returns an object with: + * - valid: boolean indicating if validation passed + * - error: string with error message (if invalid) + * - normalized: the normalized/parsed value (if valid) + */ + +/** + * Validates the 'q' (free-text search) parameter + * @param {string} q - Free text search query + * @returns {Object} { valid: boolean, error?: string, normalized?: string } + */ +function validateQ(q) { + if (!q) return { valid: true }; // optional parameter + + if (typeof q !== 'string') { + return { valid: false, error: 'Parameter "q" must be a string' }; + } + + if (q.length > 500) { + return { valid: false, error: 'Parameter "q" exceeds maximum length of 500 characters' }; + } + + return { valid: true, normalized: q.trim() }; +} + +/** + * Validates bbox parameter + * Format: [minX, minY, maxX, maxY] or comma-separated string "minX,minY,maxX,maxY" + * Also known as: [west, south, east, north] + * + * @param {string|Array} bbox - Bounding box coordinates + * @returns {Object} { valid: boolean, error?: string, normalized?: Array } + */ +function validateBbox(bbox) { + if (!bbox) return { valid: true }; + + let coords; + if (typeof bbox === 'string') { + coords = bbox.split(',').map(v => parseFloat(v.trim())); + } else if (Array.isArray(bbox)) { + coords = bbox.map(v => parseFloat(v)); + } else { + return { valid: false, error: 'Parameter "bbox" must be an array or comma-separated string' }; + } + + if (coords.length !== 4) { + return { valid: false, error: 'Parameter "bbox" must contain exactly 4 coordinates [minX, minY, maxX, maxY]' }; + } + + if (coords.some(isNaN)) { + return { valid: false, error: 'Parameter "bbox" contains invalid numeric values' }; + } + + const [minX, minY, maxX, maxY] = coords; + + // Validate longitude range + if (minX < -180 || minX > 180 || maxX < -180 || maxX > 180) { + return { valid: false, error: 'Parameter "bbox" longitude values must be between -180 and 180' }; + } + + // Validate latitude range + if (minY < -90 || minY > 90 || maxY < -90 || maxY > 90) { + return { valid: false, error: 'Parameter "bbox" latitude values must be between -90 and 90' }; + } + + // Validate logical ordering + if (minX >= maxX) { + return { valid: false, error: 'Parameter "bbox" minX must be less than maxX' }; + } + + if (minY >= maxY) { + return { valid: false, error: 'Parameter "bbox" minY must be less than maxY' }; + } + + return { valid: true, normalized: coords }; +} + +/** + * Validates datetime parameter (ISO8601) + * Formats supported: + * - Single datetime: "2020-01-01T00:00:00Z" + * - Closed interval: "2019-01-01/2021-12-31" + * - Open start: "../2021-12-31" + * - Open end: "2019-01-01/.." + * + * @param {string} datetime - ISO8601 datetime or interval + * @returns {Object} { valid: boolean, error?: string, normalized?: string } + */ +function validateDatetime(datetime) { + if (!datetime) return { valid: true }; + + if (typeof datetime !== 'string') { + return { valid: false, error: 'Parameter "datetime" must be a string' }; + } + + // Check for interval format + if (datetime.includes('/')) { + const parts = datetime.split('/'); + if (parts.length !== 2) { + return { valid: false, error: 'Parameter "datetime" interval must have exactly one "/" separator' }; + } + + const [start, end] = parts; + + // Validate start (unless open-ended "..") + if (start !== '..' && !isValidISO8601(start)) { + return { valid: false, error: `Parameter "datetime" start value "${start}" is not valid ISO8601` }; + } + + // Validate end (unless open-ended "..") + if (end !== '..' && !isValidISO8601(end)) { + return { valid: false, error: `Parameter "datetime" end value "${end}" is not valid ISO8601` }; + } + + // Check that at least one bound is specified + if (start === '..' && end === '..') { + return { valid: false, error: 'Parameter "datetime" interval cannot be unbounded on both sides' }; + } + + return { valid: true, normalized: datetime }; + } + + // Single datetime + if (!isValidISO8601(datetime)) { + return { valid: false, error: `Parameter "datetime" value "${datetime}" is not valid ISO8601` }; + } + + return { valid: true, normalized: datetime }; +} + +/** + * Helper function to validate ISO8601 datetime strings + * @param {string} dateString - ISO8601 datetime string + * @returns {boolean} true if valid ISO8601 + */ +function isValidISO8601(dateString) { + // Basic ISO8601 regex - supports dates with optional time + // Examples: 2020-01-01, 2020-01-01T00:00:00Z, 2020-01-01T00:00:00+02:00 + const iso8601Regex = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})?)?$/; + + if (!iso8601Regex.test(dateString)) { + return false; + } + + // Also validate that it's a real date + const date = new Date(dateString); + return !isNaN(date.getTime()); +} + +/** + * Validates limit parameter + * @param {string|number} limit - Maximum number of results to return + * @returns {Object} { valid: boolean, error?: string, normalized?: number } + */ +function validateLimit(limit) { + if (!limit) return { valid: true, normalized: 10 }; // default value + + // Check if limit contains a decimal point (reject floats) + if (typeof limit === 'string' && limit.includes('.')) { + return { valid: false, error: 'Parameter "limit" must be an integer, not a decimal' }; + } + + const num = parseInt(limit, 10); + + if (isNaN(num)) { + return { valid: false, error: 'Parameter "limit" must be a valid integer' }; + } + + if (num < 1) { + return { valid: false, error: 'Parameter "limit" must be at least 1' }; + } + + if (num > 10000) { + return { valid: false, error: 'Parameter "limit" must not exceed 10000' }; + } + + return { valid: true, normalized: num }; +} + +/** + * Validates sortby parameter + * Format: "+field" (ascending) or "-field" (descending) + * Allowed fields: title, id, license, created, updated + * + * @param {string} sortby - Sort specification + * @returns {Object} { valid: boolean, error?: string, normalized?: Object } + */ +function validateSortby(sortby) { + // sortby is optional – if not provided, validation passes with undefined normalized value + if (sortby === undefined || sortby === null) { + return { valid: true, normalized: undefined }; + } + + const allowedFields = ['title', 'id', 'license', 'created', 'updated']; + + // Map API field names to database column names + const fieldMapping = { + 'title': 'title', + 'id': 'id', + 'license': 'license', + 'created': 'created_at', + 'updated': 'updated_at' + }; + + if (typeof sortby !== 'string') { + return { valid: false, error: 'Parameter "sortby" must be a string' }; + } + + // Extract direction prefix and field name + let direction = 'ASC'; + let field = sortby.trim(); + + if (field.startsWith('+')) { + direction = 'ASC'; + field = field.substring(1).trim(); + } else if (field.startsWith('-')) { + direction = 'DESC'; + field = field.substring(1).trim(); + } + + // Check if field is empty (either empty string or only prefix without field name) + if (!field) { + return { + valid: false, + error: `Parameter "sortby" must specify a field. Allowed fields: ${allowedFields.join(', ')}` + }; + } + + // Check if field is in allowed list + if (!allowedFields.includes(field)) { + return { + valid: false, + error: `Parameter "sortby" field "${field}" is not supported. Allowed fields: ${allowedFields.join(', ')}` + }; + } + + // Map to actual database column name + const dbField = fieldMapping[field]; + + return { valid: true, normalized: { field: dbField, direction } }; +} + +/** + * Validates token parameter (pagination continuation token) + * @param {string|number} token - Pagination token (offset) + * @returns {Object} { valid: boolean, error?: string, normalized?: number } + */ +function validateToken(token) { + if (!token) return { valid: true, normalized: 0 }; // default to start + + const num = parseInt(token, 10); + + if (isNaN(num)) { + return { valid: false, error: 'Parameter "token" must be a valid integer' }; + } + + if (num < 0) { + return { valid: false, error: 'Parameter "token" must be non-negative' }; + } + + return { valid: true, normalized: num }; +} + +module.exports = { + validateQ, + validateBbox, + validateDatetime, + validateLimit, + validateSortby, + validateToken +}; diff --git a/bid.md b/bid.md index a4bcedf..5771fd9 100644 --- a/bid.md +++ b/bid.md @@ -18,7 +18,7 @@ Die Abnahmekriterien definieren die zwingend erforderlichen Funktionalitäten de #### Crawler - Automatisches Crawlen und Indexieren von STAC Collections aus verschiedenen Quellen -- Erfassung von mehr als 95% der Collections vom STAC Index +- Erfassung aller Collections im STAC Index - Rekursive Navigation durch STAC-Kataloge - Wöchentliches Re-Crawling zur Aktualisierung der Daten - Robustes Error-Handling mit Retry-Logic @@ -34,13 +34,17 @@ Die Abnahmekriterien definieren die zwingend erforderlichen Funktionalitäten de #### STAC API - Konforme Implementierung der STAC API Specification - Implementierung der Collection Search Extension -- Bereitstellung von Collections (GET /collections) - Abruf einzelner Collections (GET /collections/{id}) -- Erweiterte Suchfunktion (GET /search) mit Filterung nach: - - Schlüsselwörtern +- Erweiterte Suchfunktion (GET /colllections) mit Filterung nach: + - id + - Titel + - Beschreibung - Räumlicher Ausdehnung - - Zeitraum - - Provider und Lizenz + - Zeitlicher Ausdehnung + - Schlüsselwörtern + - Provider + - Lizenz + - DOIs - CQL2-Filterung für komplexe Abfragen - Parallele Verarbeitung von mindestens 100 Anfragen - Antwortzeiten: einfache Abfragen ≤ 1s, komplexe Abfragen ≤ 5s @@ -52,6 +56,8 @@ Die Abnahmekriterien definieren die zwingend erforderlichen Funktionalitäten de - Bounding Box / räumlicher Ausdehnung - Zeitraum - Thema / Keywords + - Auswahlliste für Lizenzen + - Textsuche mit Vorschlägen für Provider - Responsive Design für verschiedene Bildschirmgrößen - Sprache (Deutsch oder Englisch)) - Barrierefreiheit (farbenblindentauglich) @@ -138,13 +144,14 @@ Datenanbieter und -kuratoren, die ihre STAC-Kataloge bekannter machen und die Nu - Als Infrastrukturbetreiber möchte ich, dass mein STAC-Katalog automatisch gecrawlt wird, um ohne zusätzlichen Aufwand in der Plattform präsent zu sein. ## 3. Produkt-Umgebung - Die Produktumgebung beschreibt die technischen Rahmenbedingungen für Entwicklung, Betrieb und Integration der drei Hauptkomponenten des Projekts – **Crawler**, **STAC API** und **Frontend**. Alle Komponenten werden in einer modernen, containerisierten Umgebung entwickelt und bereitgestellt, um eine einheitliche und reproduzierbare Laufzeitumgebung sicherzustellen. +[![](https://mermaid.ink/img/pako:eNptUsGO0zAQ_RVrTkVkS-K4JckBqWQLVCrQbVqQIBysZJoGErvYjlpo--_YYVOtBPJlZt6bN_OsOUMhS4QEdo08FnuuDFmuc0HIGyWFQVF-zWEIyehThyHZLp7l8M1xZquFhbPNLHUhGX2wUuPvmrwg89NBodYD8f615a2kNpXC7GFJnhOXvF1kj3i6nn1ezteWlCp-bFDdtAaF7ON2nc4zy5ifDCrBG9LPzWSnCtRk1GcLUeLJIyk3vJGV9siw26POU2Pk7u7V5d1ms8rIxVH-wdKHJb0hzp8ruu1_dqhq1Bdr68nyPbw9aLRfmMqmwcLUUvyXlUPhbJLRzvLkkTS1-OE2vAw2wYNK1SUkO95o9KBF1XKXw9lp5WD22GIOiQ3tE1fbcODii5QtJEZ1tkXJrtrfBLpDyQ3e17xSvL1VlbWKKpWdMJAEMWW9CiRnOEES0_HLCQ2jyPd9FkU08uAXJCwMxiGjIZsGccD8KZ1cPfjdz_XHUTil08mEsShgMY1DD7CsjVTv_55Yf2nXP1fHwN0?type=png)](https://mermaid.live/edit#pako:eNptUsGO0zAQ_RVrTkVkS-K4JckBqWQLVCrQbVqQIBysZJoGErvYjlpo--_YYVOtBPJlZt6bN_OsOUMhS4QEdo08FnuuDFmuc0HIGyWFQVF-zWEIyehThyHZLp7l8M1xZquFhbPNLHUhGX2wUuPvmrwg89NBodYD8f615a2kNpXC7GFJnhOXvF1kj3i6nn1ezteWlCp-bFDdtAaF7ON2nc4zy5ifDCrBG9LPzWSnCtRk1GcLUeLJIyk3vJGV9siw26POU2Pk7u7V5d1ms8rIxVH-wdKHJb0hzp8ruu1_dqhq1Bdr68nyPbw9aLRfmMqmwcLUUvyXlUPhbJLRzvLkkTS1-OE2vAw2wYNK1SUkO95o9KBF1XKXw9lp5WD22GIOiQ3tE1fbcODii5QtJEZ1tkXJrtrfBLpDyQ3e17xSvL1VlbWKKpWdMJAEMWW9CiRnOEES0_HLCQ2jyPd9FkU08uAXJCwMxiGjIZsGccD8KZ1cPfjdz_XHUTil08mEsShgMY1DD7CsjVTv_55Yf2nXP1fHwN0) ### 3.1 Crawler -Der Crawler wird in Python implementiert und ist zuständig für das automatische Auffinden und Einlesen von STAC Collections aus dem STAC Index sowie verlinkten Katalogen/APIs. -Er schreibt die Daten in die Datenbank und führt regelmäßige, inkrementelle Aktualisierungen durch, um eine aktuelle Indexierung sicherzustellen. Protokollierung (z. B. Zeitstempel/Status) stellt Nachvollziehbarkeit sicher. +Der Crawler wird in JavaScript implementiert und ist zuständig für das automatische Auffinden und Einlesen von STAC Collections aus dem STAC Index sowie verlinkten Katalogen/APIs (6.1.1.1, 6.1.1.2). (6.2.4.2) +Er schreibt die Daten in die Datenbank (6.1.1.7, 6.1.1.3) und führt regelmäßige, inkrementelle Aktualisierungen durch, um eine aktuelle Indexierung sicherzustellen (6.1.1.6). +Protokollierung (z. B. Zeitstempel/Status) stellt Nachvollziehbarkeit sicher (6.1.1.4, 6.1.1.12). Dokumentation zu Build/Deployment/Testing wird pro Komponente bereitgestellt (6.2.4.3). ### 3.2 Datenbankmanagementsystem PostgreSQL in Kombination mit PostGIS bildet die zentrale Datengrundlage. @@ -155,15 +162,16 @@ CQL2-Filter werden serverseitig in SQL-WHERE-Klauseln übersetzt. Inkrementelle Updates und Soft-Deletes (active = false) sichern Integrität und Revisionsfähigkeit. ### 3.3 STAC API-konforme Schnittstelle -Das Backend stellt eine API bereit, die vollständig mit der STAC API-Spezifikation kompatibel ist und standardisierte Zugriffe auf die gespeicherten STAC Collections ermöglicht, unter anderem die Endpunkte /collections, /collections/{id}, /search und /conformance. -Die API wird primär in JavaScript / Node.js (20) mit Express umgesetzt. -Für die Übersetzung und Auswertung von CQL2-Abfragen wird cql2-rs (Rust) zu WebAssembly kompiliert und in-process im Node-Prozess eingebunden (geringe Latenz, einfache Containerisierung). -Als Fallback bleibt alternativ pycql2; sollten sich gravierende Schwierigkeiten mit cql2-rs ergeben, kann optional ein Python-Backend (z. B. FastAPI) implementiert werden, das die Anfrageverarbeitung und CQL2-Übersetzung übernimmt. +Das Backend stellt eine API bereit, die vollständig mit der STAC API-Spezifikation kompatibel ist und standardisierte Zugriffe auf die gespeicherten STAC Collections ermöglicht, unter anderem die Endpunkte `/` (Landing), `/conformance`, `/collections`, `/collections/{id}` und `/queryables` (global und/oder pro Collection) (6.1.2.1, 6.1.2.2, 6.1.2.3). +Die API wird primär in JavaScript / Node.js (22) mit Express umgesetzt (6.2.4.2). +Für die Übersetzung und Auswertung von CQL2-Abfragen wird cql2-rs (Rust) zu WebAssembly kompiliert und in-process im Node-Prozess eingebunden (geringe Latenz, einfache Containerisierung) (6.1.2.4, 6.1.2.5, 6.1.2.6). +Als Fallback bleibt alternativ pycql2; sollten sich gravierende Schwierigkeiten mit cql2-rs ergeben, kann optional ein Python-Backend (z. B. FastAPI) implementiert werden, das die Anfrageverarbeitung und CQL2-Übersetzung übernimmt (6.1.2.7). Die API ist klar vom Crawler getrennt und fokussiert auf Abfrage und Filterung der gespeicherten Collections. ### 3.4 UI (Web-Frontend) -Das Web-Frontend wird mit Vue.js (Version 3) entwickelt und bietet eine benutzerfreundliche Oberfläche zur Suche, Filterung und Visualisierung der STAC Collections, inklusive Kartenansicht. -Die Kommunikation zwischen Frontend und Backend erfolgt ausschließlich über die STAC API. +Das Web-Frontend wird mit Vue.js (Version 3) entwickelt (6.1.3.2) und bietet eine benutzerfreundliche Oberfläche zur Suche, Filterung und Visualisierung der STAC Collections, inklusive Kartenansicht (6.1.3.1, 6.1.3.3, 6.1.3.4, 6.1.3.5, 6.1.3.7, 6.1.3.8, 6.1.3.9). +Die Kommunikation zwischen Frontend und Backend erfolgt ausschließlich über die STAC API. Zusätzlich stellt die UI Links zur Originalquelle (STAC Catalog / API) bereit und kann optional Verweise zur Item-Search einer Collection zeigen (6.1.3.6). +Die UI und die zugehörigen Dokumentationen/Demos sollen so gestaltet sein, dass Schulungs- und Abnahmezwecke unterstützt werden (6.2.1.1, 6.2.1.2) und die Benutzererfahrung folgende Anforderungen erfüllt: intuitive/responsive UI, Accessibility, aussagekräftige Fehlerbehandlung und Sprachunterstützung (6.2.2.1, 6.2.2.2, 6.2.2.3, 6.2.2.4). ### 3.5 Containerisierung Alle Komponenten (Crawler, Datenbank, STAC-API, UI) werden einzeln mittels Docker containerisiert und als Komplett-Paket miteinander verknüpft, zum Beispiel via Docker Compose, um sowohl die getrennte Verwendung einzelner Komponenten als auch den Betrieb des vollständigen Systems zu ermöglichen. @@ -174,38 +182,44 @@ Docker gewährleistet eine konsistente Laufzeitumgebung und erleichtert die Inte Im folgenden werden die Produktfunktionen nach den einzelnen Komponenten unterteilt, nummeriert und beschrieben. Zusätzlich wird eine Priorität zur Orientierung in der Implementierung angegeben inkl. einer kurzen Beschreibung und einem groben Akzeptanzkriterium. Auf Basis der optionalen Elemente des Lastenhefts wurde auch eine Spalte "Optional" gefüllt, welche Features markiert, welche mit nachrangiger Priorität nach der Entwicklung der Hauptfunktionalitäten entwickelt werden, sollte dafür noch Zeit sein. -| ID | Komponente | Funktion (Kurzbeschreibung) | Optional | Akzeptanzkriterium | Prio | +Komponente | Funktion (Kurzbeschreibung) | Optionale Umsetzung | Akzeptanzkriterium | Prio | Lastenheft-Referenz | |---|---|---|---|---|---| -| PF-CR-01 | Crawler | Alle im STAC Index gelisteten statischen Kataloge und STAC-APIs nach Collections crawlen | – | Mind. 87 Quellen gecrawlt; Trefferquote ≥ 95 % | M | -| PF-CR-02 | Crawler | Collections in nahezu beliebiger Verschachtelungstiefe erfassen (nested catalogs) | – | Maximale Tiefe < 1024; keine Duplikate | M | -| PF-CR-03 | Crawler | Metadaten extrahieren: id, title, description, spatial/temporal extent, keywords, provider, license, DOI, summaries(platform/constellation/gsd/processing:level) | – | ≥ 95 % Felder gefüllt bei Stichprobe n=50 | H | -| PF-CR-04 | Crawler | Quell-URL, Quell-Titel, „zuletzt gecrawlt“ speichern | – | Felder in DB vorhanden und befüllt | M | -| PF-CR-05 | Crawler | Alle stabilen STAC-Versionen unterstützen (alte Ressourcen werden automatisch auf 1.1 migriert) | – | Collections unterschiedl. Versionen werden gespeichert und ggf. migriert | M | -| PF-CR-06 | Crawler | Inkrementelle Updates und periodisches Re-Crawling | – | Änderungen können ohne vollständige Neuindexierung hinzugefügt werden | H | -| PF-CR-07 | Crawler | Vollständige STAC-Collection + extrahierte Suchfelder persistent ablegen | – | ≥ 95 % Felder identisch zwischen Quelle und Datenbank bei Stichprobe n=50 | H | -| PF-CR-08 | Crawler | Erweiterbares DB-Design für zusätzliche Felder vorschlagen (siehe 5. Produktdaten) | – | Schema-Entwurf dokumentiert & abgenommen | M | -| PF-CR-09 | Crawler | Rate-Limiting einhalten (Quellen nicht überlasten) | – | Keine 429-Antworten/Blockings in Testlauf über 12 h | M | -| PF-CR-10 | Crawler | Konfigurierbare Crawl-Zeitpläne/Frequenzen | ✔ | CRON/Intervall vom Anwender frei konfigurierbar | L | -| PF-CR-11 | Crawler | Fehlerbehandlung + Retry; problematische Quellen überspringen | ✔ | Backoff/Retry-Logik; Fehlerbericht vorhanden | M | -| PF-CR-12 | Crawler | Logging & Monitoring der Crawl-Aktivitäten | ✔ | Dashboards/Metriken (Rate, Fehler, Status) | M | -| PF-CR-13 | Crawler | Version-agnostische STAC-Extensions erkennen und als Tags speichern (EO, SAR, Point Cloud) | ✔ | Extensions-Tags in DB & Queryables sichtbar | L | -| PF-API-01 | STAC-API | API gemäß relevanten Spezifikationen gültig (STAC API und Collection Search Extension) | – | `/conformance` enthält zutreffende URIs | H | -| PF-API-02 | STAC-API | Erweiterung der bestehenden STAC Index API; bleibt selbst gültige STAC-API | – | Root/Collections gültig - Getestet durch `STAC Validator` und `STAC API Validator` | H | -| PF-API-03 | STAC-API | Collection Search: Freitext `q`, Filter, Sortierung | – | Beispiel-Queries liefern erwartete Treffer | H | -| PF-API-04 | STAC-API | CQL2-Filtering (Basic CQL2 (`AND`, `OR`, `NOT`, `=`, `<>`, `<`, `<=`, `>`, `>=`, `IS NULL`)) für Collection-Eigenschaften | – | Gültige Filter → 200 Antworten; ungültige → 400 Antworten mit Fehlerbeschreibung | H | -| PF-API-05 | STAC-API | Zusätzliche CQL2-Fähigkeiten (Advanced Comparison Operators (`LIKE/BETWEEN/IN`, `casei/accenti`, `Spatial/Temporal`, `Arrays`)) | ✔ | Conformance-URIs ergänzt; Tests grün | M | -| PF-API-06 | STAC-API | CQL2 als Standalone-Library bereitstellen | ✔ | Lib mit Parser/Validation + README | L | -| PF-API-07 | STAC-API | Integration der neuen Funktionen in bestehende STAC Index API | ✔ | End-to-End-Tests (Crawler→API→UI) grün | M | -| PF-UI-01 | Web-UI | Intuitive Suchoberfläche für Collections | – | Usability-Test: Kernflows bestehen | H | -| PF-UI-02 | Web-UI | Implementierung in Vue (v3) zur Einbindung in STAC Index | – | Build integriert; Routing/State funktionsfähig | M | -| PF-UI-03 | Web-UI | Interaktive Auswahl von Bounding Box und Zeitintervall | – | BBox/Datetime erzeugen korrekte Parameter | H | -| PF-UI-04 | Web-UI | Composable Queryables in der UI → generiert CQL2-Ausdruck | – | UI-Builder erzeugt valide CQL2 (Server-OK) | H | -| PF-UI-05 | Web-UI | Kartenansicht mit Visualisierung räumlicher Extents | – | Extents werden auf interaktiver Karte dargestellt | M | -| PF-UI-06 | Web-UI | Links zur Originalquelle (Katalog/API) und optional zur Item Search | – | Links korrekt & erreichbar | M | -| PF-UI-07 | Web-UI | Inspection-Ansicht für Collections (Details) | – | Detailseite zeigt alle Kernfelder | M | -| PF-UI-08 | Web-UI | Items der Collections inspizieren können | ✔ | Item-Liste/Detail aufrufbar | L | -| PF-UI-09 | Web-UI | Collections vergleichen (Mehrfachauswahl & Vergleich) | ✔ | Vergleichsansicht mit minimum 2 Collections | L | - +| Crawler | Alle im STAC Index gelisteten statischen Kataloge und STAC-APIs nach Collections crawlen | – | Mind. 87 Quellen gecrawlt; Trefferquote ≥ 95 % | M | 6.1.1 1. | +| Crawler | Collections in nahezu beliebiger Verschachtelungstiefe erfassen (nested catalogs) | – | Maximale Tiefe < 1024 | M | 6.1.1 2. | +| Crawler | Metadaten extrahieren: id, title, description, spatial/temporal extent, keywords, provider, license, DOI, summaries(platform/constellation/gsd/processing:level) | – | ≥ 95 % Felder gefüllt bei Stichprobe n=50 | H | 6.1.1 3. | +| Crawler | Quell-URL, Quell-Titel, „zuletzt gecrawlt“ speichern | – | Felder in DB vorhanden und befüllt | M | 6.1.1 4. | +| Crawler | Alle stabilen STAC-Versionen unterstützen (alte Ressourcen werden automatisch auf 1.1 migriert) | – | Collections unterschiedl. Versionen werden gespeichert und ggf. migriert | M | 6.1.1 5. | +| Crawler | Inkrementelle Updates und periodisches Re-Crawling | – | Änderungen können ohne vollständige Neuindexierung hinzugefügt werden | H | 6.1.1 6. | +| Crawler | Vollständige STAC-Collection + extrahierte Suchfelder persistent ablegen | – | ≥ 95 % Felder identisch zwischen Quelle und Datenbank bei Stichprobe n=50 | H | 6.1.1 7. | +| Crawler | Erweiterbares DB-Design für zusätzliche Felder vorschlagen (siehe 5. Produktdaten) | – | Schema-Entwurf dokumentiert & abgenommen | M | 6.1.1 8. | +| Crawler | Rate-Limiting einhalten (Quellen nicht überlasten) | – | Keine 429-Antworten/Blockings in Testlauf über 12 h | M | 6.1.1 9. | +| Crawler | Konfigurierbare Crawl-Zeitpläne/Frequenzen | % | CRON/Intervall vom Anwender frei konfigurierbar | L | 6.1.1 10. | +| Crawler | Fehlerbehandlung + Retry; problematische Quellen überspringen | % | Backoff/Retry-Logik; Fehlerbericht vorhanden | M | 6.1.1 11. | +| Crawler | Logging & Monitoring der Crawl-Aktivitäten | % | Dashboards/Metriken (Rate, Fehler, Status) | M | 6.1.1 12. | +| Crawler | Version-agnostische STAC-Extensions erkennen und als Tags speichern (EO, SAR, Point Cloud) | ✔ | Extensions-Tags in DB & Queryables sichtbar | $ | 6.1.1 13. | +| STAC-API | API gemäß relevanten Spezifikationen gültig (STAC API und Collection Search Extension) | – | GET `/` und `/conformance` enthält zutreffende URIs | H | 6.1.2 1. | +| STAC-API | Erweiterung der bestehenden STAC Index API; bleibt selbst gültige STAC-API | – | Root/Collections gültig - Getestet durch `STAC Validator` und `STAC API Validator` und Jest-Tests für die Collection Search Extension | H | 6.1.2 2. | +| STAC-API | Collection Search: Freitext `q`, Filter, Sortierung | – | Beispiel-Queries liefern erwartete Treffer | H | 6.1.2 3. | +| STAC-API | CQL2-Filtering (Basic CQL2 (`AND`, `OR`, `NOT`, `=`, `<>`, `<`, `<=`, `>`, `>=`, `IS NULL`)) für Collection-Eigenschaften | – | Gültige Filter → 200 Antworten; ungültige → 400 Antworten mit Fehlerbeschreibung | H | 6.1.2 4. | +| STAC-API | Zusätzliche CQL2-Fähigkeiten (Advanced Comparison Operators (`LIKE/BETWEEN/IN`, `casei/accenti`, `Spatial/Temporal`, `Arrays`)) | % | Conformance-URIs ergänzt; Tests erfolgreich | M | 6.1.2 5. (optional) | +| STAC-API | CQL2 als Standalone-Library bereitstellen | $ | Lib mit Parser/Validation + README | L | 6.1.2 6. (optional) | +| STAC-API | Integration der neuen Funktionen in bestehende STAC Index API | $ | End-to-End-Tests (Crawler→API→UI) grün | M | 6.1.2 7. | +| Web-UI | Intuitive Suchoberfläche für Collections | – | Usability-Test: Kernflows bestehen | H | 6.1.3 1. | +| Web-UI | Implementierung in Vue (v3) zur Einbindung in STAC Index | – | Build integriert; Routing/State funktionsfähig | M | 6.1.3 2. | +| Web-UI | Interaktive Auswahl von Bounding Box und Zeitintervall | – | BBox/Datetime erzeugen korrekte Parameter | H | 6.1.3 3. | +| Web-UI | Composable Queryables in der UI → generiert CQL2-Ausdruck | – | UI-Builder erzeugt valide CQL2 (Server-OK) | H | 6.1.3 4. | +| Web-UI | Kartenansicht mit Visualisierung räumlicher Extents | – | Extents werden auf interaktiver Karte dargestellt | M | 6.1.3 5. | +| Web-UI | Links zur Originalquelle (Katalog/API) und optional zur Item Search | – | Links korrekt & erreichbar | M | 6.1.3 6. | +| Web-UI | Inspection-Ansicht für Collections (Details) | – | Detailseite zeigt alle Kernfelder | M | 6.1.3 7. | +| Web-UI | Items der Collections inspizieren können | % | Item-Liste/Detail aufrufbar | L | 6.1.3 8. (optional) | +| Web-UI | Collections vergleichen (Mehrfachauswahl & Vergleich) | % | Vergleichsansicht mit minimum 2 Collections | L | 6.1.3 9. (optional) | + +Legende Spalte "Umsetzung" +| Zeichen | Bedeutung | +| ------- | --------- | +| - | Diese Funtion ist vom Lastenheft verpflichtend vorgegeben und wird umgesetzt | +| % | Dieses Funktion ist vom Lastenheft optinal vorgegeben und wird umgesetzt | +| $ | Diese Funktion ist vom Lastenheft optional vorgegeben und es ist nicht geplant dieses umzusetzen| ## 5. Produktdaten @@ -219,7 +233,7 @@ Die Tabellen enthalten jeweils Primärschlüssel zur eindeutigen Identifikation ### Tabellenbereich „Catalogs“ -Der Bereich **Catalogs** bildet die hierarchische Struktur der STAC-Kataloge ab. Jeder Katalog speichert seine Metadaten inklusive Versionierung, Typ, Beschreibung und zugehöriger Links. Über Zwischentabellen werden Keywords sowie Erweiterungen (STAC Extensions) referenziert. +Der Bereich **Catalogs** bildet die hierarchische Struktur der STAC-Kataloge ab. Jeder Katalog speichert seine Metadaten inklusive Versionierung, Typ, Beschreibung und zugehöriger Links. Über Zwischentabellen werden Erweiterungen (STAC Extensions) referenziert. #### Tabelle: `catalog` @@ -233,7 +247,7 @@ Der Bereich **Catalogs** bildet die hierarchische Struktur der STAC-Kataloge ab. | created_at | Zeitpunkt der Erstellung | timestamp | | updated_at | Zeitpunkt der letzten Änderung | timestamp | -Die Haupttabelle `catalog` bildet den zentralen Einstiegspunkt der Kataloghierarchie. Sie speichert allgemeine Metadaten und dient als Ankerpunkt für die zugehörigen Relationen. +Die Haupttabelle `catalog` bildet den zentralen Einstiegspunkt der Kataloghierarchie. Sie speichert allgemeine Metadaten und dient als Ankerpunkt für die zugehörigen Relationen. Diese Tabelle ist notwenidg um zu schauen, ob der crawler die aktueller Version des `catalogs` gespeichert hat, oder diesen `catalog` erneut crawlen muss. Somit kann der crawler bei einem erneuten crawl effektiver und schneller arbeiten. #### Tabelle: `catalog_links` @@ -248,14 +262,6 @@ Die Haupttabelle `catalog` bildet den zentralen Einstiegspunkt der Kataloghierar Die Tabelle `catalog_links` beschreibt die Verknüpfungen zwischen einzelnen Katalogen oder externen Referenzen und implementiert damit die STAC-Link-Struktur. -#### Tabelle: catalog:keywords -| Spalte | Beschreibung / Inhalt | Datentyp / Format | -|----------------|---------------------------------------------|-------------------| -| **catalog_id** | Referenz auf `catalog.id` | integer (FK) | -| **keyword_id** | Referenz auf `keyword.id` | integer (FK) | - -Relationstabelle zur Mehrfachzuordnung von Keywords an Catalogs. Dadurch können Sammlungen gezielt über Schlagwörter gefiltert werden. Diese Tabelle wird benötigt, da hier eine (n:n)-Beziehung vorliegt. - #### Tabelle: catalog:stac_extension | Spalte | Beschreibung / Inhalt | Datentyp / Format | @@ -283,9 +289,9 @@ Der Bereich **Collections** bildet die Sammlungen von Collections innerhalb eine | license | Lizenzinformation | text | | created_at | Zeitpunkt der Erstellung | timestamp | | updated_at | Zeitpunkt der letzten Änderung | timestamp | -| spatial_extend | Räumliche Ausdehnung (Bounding Box) | bbox (geometry) | -| temporal_extend_start | Startzeitpunkt des zeitlichen Gültigkeitsbereichs | timestamp | -| temporal_extend_end | Endzeitpunkt des zeitlichen Gültigkeitsbereichs | timestamp | +| spatial_extent | Räumliche Ausdehnung (Bounding Box) | bbox (geometry) | +| temporal_extent_start | Startzeitpunkt des zeitlichen Gültigkeitsbereichs | timestamp | +| temporal_extent_end | Endzeitpunkt des zeitlichen Gültigkeitsbereichs | timestamp | | is_api? | ist die collection in einem catalog oder api | boolean | | is_active? | ist die collection noch aktuell, oder wurde diese gelöscht | boolean | | full_json | Hier werden alle JSON-Daten einer collection gespeichert | JSON | @@ -412,10 +418,10 @@ Indizes auf allen relevanten Attributen (IDs, Zeitstempel, Textfelder und Geomet Die Crawler-Komponente soll eine hohe Effizienz, Stabilität und Skalierbar sein, um große Mengen an STAC-Katalogen und -APIs regelmäißg und zuverlässig zu erfassen. #### 6.1.1 Crawling Leistung -Der Crawler soll in der Lage sein aus dem STAC-Index Quellen innerhalb einer Woche zu analysieren. In folge dessen soll auch die Aktualisierung aller bekannter und neuer Quellen maximal eine Woche betragen. Die einzelnen STAC-Collections sollen jeweils innerhalb von < 5 Sekunden abgerufen und verarbeitet werden. Zudem soll der Crawler alle vorgegebenen Rate-Limits einhalten, um die externen Dienste nicht zu überlasten (z.B. max. 5 Request/Sekunde pro Quelle). +Der Crawler soll in der Lage sein aus dem STAC-Index Quellen innerhalb einer Woche zu analysieren. In folge dessen soll auch die Aktualisierung aller bekannter und neuer Quellen maximal eine Woche betragen. Die einzelnen STAC-Collections sollen jeweils innerhalb von < 5 Sekunden abgerufen und verarbeitet werden. Zudem soll der Crawler alle vorgegebenen Rate-Limits einhalten, um die externen Dienste nicht zu überlasten (z.B. max. 20 Request/Minute pro Quelle). #### 6.1.2 Crawling Parallelität und Skalierbarkeit -Die Implementierung soll asynchrones und paralleles Crawling unterstützten. Es wird nur ein einzelene Crawler-Instanz sein, um die Komplexität mit Datenbankkonflikten zu vermeiden. Es wird darauf geachtet eine Modulare weise zu programmieren um in Zukunft horizontale Skalierung mit mehren Cralwern möglich zu machen. +Die Implementierung soll asynchrones und paralleles Crawling unterstützten. Es wird nur ein einzelene Crawler-Instanz sein, um die Komplexität mit Datenbankkonflikten zu vermeiden. Es wird darauf geachtet so zu programmieren, um in Zukunft horizontale Skalierung mit mehren Crawlern möglich zu machen. #### 6.1.3 Crawling Zuverlässigkeit unf Fehlertoleranz Der Crawler darf bei fehlerhaften oder inaktiven Quellen nicht vollständig abbrechen. Die Quellen, die dreimal hintereinander fehlschlagen, sollen als inaktiv bis zum Crawling Event behandelt werden. Fehler und Wiederholungen müssen protokolliert werden. Alle ursprünglich erreichbaren STAC collections und catalogs sollen in der Datenbank dann als inaktiv gekennzeichnet werden. @@ -438,9 +444,9 @@ Die Crawling-Durchläufe sollen über Logging und Metriken wie der Anzahl gecraw #### 6.2.1 Funktionale Leistungsanforderungen 1. **Antwortzeiten der Datenbankabfragen** - - Standardabfragen (z. B. Abruf einer Collection nach ID) müssen innerhalb von **< 5 Sekunde** beantwortet werden. - - Komplexe Suchanfragen mit Filtern (z. B. Freitextsuche, räumliche und zeitliche Filterung) müssen innerhalb von **≤ 60 Sekunden** abgeschlossen sein. - - Langlaufende Abfragen dürfen eine maximale Bearbeitungszeit von **≤ 90 Sekunden** nicht überschreiten. + - Standardabfragen (z. B. Abruf einer Collection nach ID) müssen innerhalb von **< 1 Sekunde** beantwortet werden. + - Komplexe Suchanfragen mit Filtern (z. B. Freitextsuche, räumliche und zeitliche Filterung) müssen innerhalb von **≤ 30 Sekunden** abgeschlossen sein. + - Langlaufende Abfragen dürfen eine maximale Bearbeitungszeit von **≤ 60 Sekunden** nicht überschreiten. 2. **Gleichzeitige Zugriffe (Concurrency)** - Das System muss mindestens **50 gleichzeitige Leseanfragen** und **10 gleichzeitige Schreibanfragen** ohne merkliche Leistungseinbußen (< 10 % längere Antwortzeit) verarbeiten können. @@ -516,7 +522,8 @@ Sie visualisiert Metadaten und räumliche Extents der Collections und ermöglich - Die API-Kommunikation erfolgt in Englisch. - Das Frontend wird zweisprachig (Englisch / Deutsch) bereitgestellt. - **Reaktionszeiten**: - - Benutzerinteraktionen (außer Suchanfragen) sollen innerhalb von 1 Sekunde eine sichtbare Rückmeldung liefern. + - Benutzerinteraktionen (außer Suchanfragen) sollen innerhalb von 1 Sekunde eine sichtbare Rückmeldung liefern. + - Dies kann z. B. durch einen Ladeindikator, Fortschrittsbalken oder ähnliches visuelles Feedback erfolgen. - Einfache Suchanfragen (z. B. Freitextsuche nach Keywords) müssen in unter 5 Sekunden abgeschlossen sein. - Komplexe geometrische oder kombinierte CQL2-Filter dürfen maximal 1 Minute dauern. - **Pagination**: Bei umfangreichen Ergebnismengen erfolgt eine seitenweise Darstellung, um Performance und Übersichtlichkeit zu gewährleisten. @@ -528,7 +535,7 @@ Die nachfolgenden Maßnahmen gewährleisten die Korrektheit, Wartbarkeit, Standa ### 7.1 Code-Qualität und Tests #### 7.1.1 Unit-Tests - - Für alle zentralen Backend-Module (insbesondere STAC-API-Routen, CQL2-Parser, Datenbank-Abfrage-Logik und Crawler-Importfunktionen) werden Unit-Tests mit einem geeigneten Framework (jest and pytest) erstellt. + - Für alle zentralen Backend-Module (insbesondere STAC-API-Routen, CQL2-Parser, Datenbank-Abfrage-Logik und Crawler-Importfunktionen) werden Unit-Tests mit einem geeigneten Framework (jest) erstellt. - Für das Frontend werden Unit-Tests mit einem geeigneten Framework (Jest) erstellt. - Zielabdeckung: mindestens 80 % Branch- und Statement-Coverage. - Tests werden automatisiert bei jedem Commit und Merge-Request in der GitHub-Pipeline ausgeführt. @@ -560,7 +567,8 @@ Die nachfolgenden Maßnahmen gewährleisten die Korrektheit, Wartbarkeit, Standa - beim erstmaligen Import (Crawler-Phase), - bei Änderungen oder Re-Crawls, - zusätzlich regelmäßig in der CI-Pipeline anhand von Stichproben. -- Collections, die nicht vollständig dem aktuellen STAC-Standard entsprechen, werden automatisch angepasst oder konvertiert, bevor sie in den Index übernommen werden. +- Collections, die nicht vollständig dem aktuellen STAC-Standard entsprechen, werden automatisch strukturell angepasst oder konvertiert, bevor sie in den Index übernommen werden. +- Kann eine automatische Anpassung nicht durchgeführt werden, wird die betreffende Collection als inkompatibel markiert, nicht in den Index aufgenommen und in einem separaten Fehlerprotokoll dokumentiert. - Die Validierungsergebnisse, sowie alle Anpassungen werden im Crawler-Log und in den CI-Reports dokumentiert. ### 7.4 STAC-API-Validator @@ -569,11 +577,12 @@ Die nachfolgenden Maßnahmen gewährleisten die Korrektheit, Wartbarkeit, Standa - Geprüfte Aspekte: - Gültigkeit der API-Antworten nach STAC API-Spezifikation (v1.x). - Unterstützung der Collection Search Extension und der CQL2-Query Language (Basic). - - Korrekte Implementierung der Endpoints (/collections, /search, /conformance, /queryables). + - Korrekte Implementierung der Endpoints (`/`, `/conformance`, `/collections`, `/collections/{id}`). +-Die Collection Search Extension wird zusätzlich durch eigene Integrationstests validiert, da der offizielle Validator derzeit keine automatisierte Prüfung dieser Erweiterung unterstützt. - Der Validator wird: - nach jedem erfolgreichen Build in der CI-Pipeline ausgeführt, - manuell vor der Endabgabe für einen vollständigen Compliance-Report verwendet. -- Ziel: 100 % bestehende STAC-Validator-Tests. +- Ziel: 100 % bestehende STAC-Validator-Tests, sowie erfolgreiche interne Validierung der Collection Search Extension. ### 7.5 Dokumentations- und Wartungsqualität - Alle Module werden mit aussagekräftigen Kommentaren dokumentiert, entsprechend der jeweils verwendeten Programmiersprache (z. B. PyDoc für Python-Module oder JSDoc für JavaScript/Vue-Komponenten). @@ -609,6 +618,7 @@ Die nachfolgenden Maßnahmen gewährleisten die Korrektheit, Wartbarkeit, Standa - Sichere Datenbankverbindungen - Eingabevalidierung (SQL-Injection-Schutz) - Sanitization von Nutzereingaben +- Gegen XSS abgesichert - Keine Exposition sensibler Daten in Logs - Protokollierung der Crawl-Aktivitäten - Strukturierte Error-Logs mit konfigurierbaren Log-Levels @@ -630,13 +640,12 @@ Die nachfolgenden Maßnahmen gewährleisten die Korrektheit, Wartbarkeit, Standa ## 9. Gliederung in Teilprodukte ### 9.1 Crawler-Komponente -Der Crawler durchsucht STAC Index nach STAC Kataloge und STAC APIs. Dabei sollen mehr als 95% der Collections erfolgreich erfasst werden. +Der Crawler bekommt über die STAC Index API, alle STAC Kataloge und STAC APIs die gecrawled werden müssen. Dabei sollen alle Collections erfolgreich erfasst werden. Das Crawling erfolgt rekursiv, sodass Collections in nahezu beliebiger Tiefe (<1024 Ebenen) innerhalb verschachtelter Kataloge erkannt werden. Es werden ausschließlich Catalogs und Collections und keine Items erfasst. Die Crawling Vorgänge extrahieren die relevanten Metadaten jeder Collection (6.1.1.3), das vollständige JSON-Objekt jeder Kollektion und speichern sie zusammen mit der Quell-URL, dem Katalognamen und dem Zeitstempel des letzten Crawls. Zusätzlich wird auch ein Parameter, über die aktuelle Verfügbarkeit der Collection hinzugefügt. Es werden alle stabilen STAC-Versionen, durch Migration unterstützt. Eine Crawling-Plan (Schedule) ermöglicht die zeitliche Steuerung einzelner Crawl-Vorgänge. Es soll eine wöchentliche Aktualisierungen des Indexes durchgeführt werden. - -Für die Umsetzung werden PySTAC und asyncio zur Verarbeitung genutzt. Die Ergebnisse werden mittels PypgSTAC in einer PostgreSQL-Datenbank gespeichert. +Die Ergebnisse werden in einer PostgreSQL-Datenbank gespeichert. ### 9.2 Datenbank-Komponente Die Datenbankkomponente stellt die zentrale Grundlage für die Speicherung, Verwaltung und Abfrage aller vom Crawler erfassten Metadaten dar. Sie dient der persistenten Ablage sämtlicher Inhalte, einschließlich der vollständigen STAC-JSON-Strukturen, und ermöglicht deren effiziente Weiterverarbeitung innerhalb der Gesamtarchitektur. Als Datenbanksystem wird **PostgreSQL** in Kombination mit der Erweiterung **PostGIS** eingesetzt, um sowohl relationale als auch geographische Abfragen performant unterstützen zu können. @@ -688,9 +697,9 @@ und ermöglicht einen einheitlichen, standardkonformen Zugriff auf alle gespeich - `GET /collections/{id}` -> Liefert die vollständigen Metadaten einer einzelnen Collection 3. Collection Search -- `GET /search` +- `GET /collections` und -- `POST /search` +- `POST /collections` - Ermöglicht die gezielte Filterung und Suche nach Collections innerhalb des Index. - Unterstützt wird sowohl die einfache Query-Parameter-Variante (GET) als auch komplexe CQL2-Abfragen (POST). @@ -755,7 +764,7 @@ Ziel des Crawler‑Moduls ist die automatische Erfassung, Validierung und Speich #### 10.1.1 Technologien Der Crawler wird als Node.js‑Anwendung konzipiert werden. Es wird JavaScript genutzt, um bessere Wartbarkeit und Weiterentwicklung innerhalb der Gruppe zu erreichen und die Probleme mit bestimmten Versionen von z.B. Python zu unterbinden. -Für das STAC‑Handling kommen [stac-js](https://github.com/moregeo-it/stac-js) und [stac-migrate](https://github.com/stac-utils/stac-migrate) zum Migrieren älterer STAC‑Versionen zum Einsatz. Für HTTP‑Zugriffe eignen sich `axios` oder `got`, da beide Timeouts und Retries unterstützen. Alternativ kann `node‑fetch` verwendet werden. +Für das STAC‑Handling kommen [stac-js](https://github.com/moregeo-it/stac-js) und [stac-migrate](https://github.com/stac-utils/stac-migrate) zum Migrieren älterer STAC‑Versionen zum Einsatz. Für HTTP‑Zugriffe eignen sich `axios`, da es Timeouts und Retries unterstützt. Alternativ kann `node‑fetch` verwendet werden. Beim Crawling und Queueing sind für komplexe Szenarien, Frameworks wie `Crawlee (Apify)` oder vergleichbare Lösungen mit integrierter Queue/Retry‑Logik empfehlenswert, für leichtere Implementierungen bieten sich `p‑queue` oder `Bottleneck` zur Steuerung von Parallelität und Rate‑Limits an. Zur zeitgesteuerten Ausführung kann lokal `node‑cron` genutzt werden. Die Validierung erfolgt via JSON‑Schema Validator (z. B. `ajv`) unter Verwendung der offiziellen STAC‑Schemas. Die Anbindung an die Datenbank kann mit `node‑postgres (pg)` erfolgen. @@ -781,7 +790,7 @@ Optional existiert eine Admin UI / API zur Anzeige von Quellen, Fehlerlogs und f 3. Rekursives Crawling und Pagination: Die Engine folgt Link‑Rela‑Typen wie child, catalog und collection sowie paginiert bei STAC APIs; neue URLs/Tasks werden in die Queue aufgenommen und asynchron abgearbeitet, wobei Rate‑Limits und Parallelität berücksichtigt werden. -4. Migration & Modeling: Gefundene STAC‑Objekte werden mit stac‑migrate in eine einheitliche STAC‑Version überführt und anschließend in ein internes Datenmodell (z. B. stac‑js‑Objekt oder DTO) umgewandelt. +4. Migration: Gefundene STAC‑Objekte werden mit stac‑migrate in eine einheitliche STAC‑Version überführt und anschließend in ein internes Datenmodell für die Suche umgewandelt. 5. Extraktion & Normalisierung: Aus den STAC‑Objekten werden Schlüsselattribute extrahiert (z. B. id, title, description, extent – bbox und temporal, providers, license, assets, HREFs). Die BBOX‑Angaben werden in eine PostGIS‑Geometrie konvertiert (z. B. Envelope/Polygon), zeitliche Angaben als TIMESTAMP abgelegt. @@ -830,7 +839,7 @@ Die Implementierung folgt einem klar strukturierten Vorgehen in mehreren Phasen, Anschließend werden die Such- und Filtermechanismen implementiert. Dazu gehört die Integration einer **Volltextsuche** auf Basis von PostgreSQL-TSVector, die Anbindung von **PostGIS** für Bounding-Box- und Distanzabfragen sowie die Umsetzung einer Übersetzungsschicht für **CQL2-Filterausdrücke**. Ergebnis: performante Such- und Filterfunktionen mit optimierten Indizes. -5. **Deployment und Dokumentation (M6)** +5. **Deployment und Dokumentation (M5)** Die produktive Bereitstellung erfolgt über **Docker Compose** Das Prisma-Schema, die Migrationsdateien und die API-Routen werden versioniert und dokumentiert. Eine technische Dokumentation beschreibt die Struktur, Indexierung und Updateprozesse der Datenbank. Ergebnis: einsatzbereite, dokumentierte Datenbankkomponente. @@ -839,10 +848,10 @@ Die Implementierung folgt einem klar strukturierten Vorgehen in mehreren Phasen, ### 10.3 STAC API | ID | Arbeitspaket | Ziel/Output | Schritte (Stichpunkte) | Reuse/Technologien | |----|--------------|-------------|-------------------------|--------------------| -| AP-01 | Projekt-Skeleton & Infrastruktur | Lauffähiges API-Grundgerüst mit Konfiguration & Logging | Repo-Struktur (`/api`, `/docs`, `/ops`); Apache-2.0 LICENSE; ENV-Konfig (Port, DB-URL vom DB-Team); strukturierte Logs; einfache Health-Route `GET /` | Python+FastAPI *oder* Node+Fastify/Express; uvicorn/node pm2; dotenv | +| AP-01 | Projekt-Skeleton & Infrastruktur | Lauffähiges API-Grundgerüst mit Konfiguration & Logging | Repo-Struktur (`/api`, `/docs`); Apache-2.0 LICENSE; ENV-Konfig (Port, DB-URL vom DB-Team); strukturierte Logs; einfache Health-Route `GET /` | Python+FastAPI *oder* Node+Fastify/Express; uvicorn/node pm2; dotenv | | AP-02 | Daten-Vertrag & Queryables (API-Seite) | Konsistentes Feld-Set & ` /queryables` für die UI | Such-/Filterfelder festlegen (id, title, description, extent, keywords, providers.name, license, doi, `summaries.platform/constellation/gsd/processing:level`); Datentypen (CQL2-kompatibel) definieren; `GET /queryables` (global/optional pro Collection); Dokumentation für UI | STAC Collections/Queryables Best Practices; CQL2 Typen | | AP-03 | STAC-Core Endpunkte | STAC-konforme Basisrouten bereitstellen | `GET /` (Landing + Links), `GET /conformance` (Core+Collections vorerst), `GET /collections`, `GET /collections/{id}`; Link-Relationen & Service-Doku referenzieren | OpenAPI/Swagger-UI; STAC API Core/Collections | -| AP-04 | Collection Search – Routen & Parameter | Collection-Search-Schnittstelle mit `q`, `filter`, `sort`, Paging | Route definieren (`/collections/search` oder Parametrisierung von `/collections`); Request-Validierung; Paging-Links | STAC Collection Search Extension; API Framework Middleware | +| AP-04 | Collection Search – Routen & Parameter | Collection-Search-Schnittstelle mit `q`, `filter`, `sort`, Paging | Route definieren (Parametrisierung von `/collections`); Request-Validierung; Paging-Links | STAC Collection Search Extension; API Framework Middleware | | AP-05 | CQL2 Basic – Parsing & Validierung | Gültige CQL2-Basic-Filter erkennen & valide/klare Fehlermeldungen liefern | Bestehende Parser/Validator-Lib einbinden; Request-Modelle (JSON/Text); Fehlermeldungen standardisieren | *cql2-rs* oder *pycql2* | | AP-06 | CQL2-Ausführung – AST → SQL | CQL2-AST in effiziente SQL-Where-Klauseln übersetzen | Visitor/Mapper je Knotentyp (Vergleich, Logik, `IS NULL`, optional `LIKE/IN/BETWEEN`); Parametrisiertes SQL; Schutz vor teuren Scans (Zeit/Seite begrenzen) | — | | AP-07 | Freitext `q` & Sortierung | Relevanzbasierte Freitextsuche + stabile Sortierung | Felder für `q` bestimmen (title, description, keywords, providers); Whitelist für `sortby`; Validierung bei nicht unterstützten Feldern → 400 | API-seitige Param-Validierung | @@ -936,7 +945,7 @@ Filterparameter werden in den Anfragen nach dem CQL2-Standard übergeben. **Hinweis:** Kalenderwochen (KW) entsprechen ISO-Wochen, angegeben mit **Montag–Sonntag**. **Roadmap nach Kalenderwochen** | Zeitraum (Mo–So) | KW | Phase/Meilenstein | Hauptaktivitäten | Artefakte/Outputs | -|---|:--:|---|---|---|---| +|---|:--:|---|---|---| | **12.11.2025** (Mi) | 46 | **Kick-off Implementierung** | Projektstart, Scope & Schnittstellen finalisieren, Tooling/Standards festlegen | Kick-off-Protokoll, To-Do-Übersicht | | 10.11–16.11.2025 | 46 | **Struktur Repo** | Monorepo/Repos anlegen (`/api`, `/crawler`, `/frontend`, `/ops`, `/docs`), CI/Lint/Format einrichten | Repo-Struktur, CI-Baseline, CONTRIBUTING.md | | 10.11–16.11.2025 | 46 | **API-Endpoints (Core)** | API-Skeleton, Routen `/`, `/conformance`, `/collections`, Stubs/Swagger | Laufende API-Instanz, OpenAPI-Draft | diff --git a/crawler/utils/db.js b/crawler/utils/db.js index 46c1139..1bd2e85 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -167,12 +167,12 @@ async function insertOrUpdateCollection(collection) { await client.query('BEGIN'); // Parse spatial extent (bbox) - let spatialExtend = null; + let spatialExtent = null; if (collection.extent?.spatial?.bbox && collection.extent.spatial.bbox[0]) { const bbox = collection.extent.spatial.bbox[0]; if (bbox.length === 4) { // Create polygon from bbox [west, south, east, north] - spatialExtend = `EPSG:4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; + spatialExtent = `EPSG:4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; } } @@ -204,9 +204,9 @@ async function insertOrUpdateCollection(collection) { type = $2, description = $3, license = $4, - spatial_extend = ST_GeomFromEWKT($5), - temporal_extend_start = $6, - temporal_extend_end = $7, + spatial_extent = ST_GeomFromEWKT($5), + temporal_extent_start = $6, + temporal_extent_end = $7, is_api = $8, is_active = $9, full_json = $10, @@ -217,7 +217,7 @@ async function insertOrUpdateCollection(collection) { collection.type || 'Collection', collection.description || null, collection.license || null, - spatialExtend, + spatialExtent, temporalStart, temporalEnd, false, // is_api - will be determined by crawler @@ -231,7 +231,7 @@ async function insertOrUpdateCollection(collection) { const collectionResult = await client.query( `INSERT INTO collection ( stac_version, type, title, description, license, - spatial_extend, temporal_extend_start, temporal_extend_end, + spatial_extent, temporal_extent_start, temporal_extent_end, is_api, is_active, full_json, updated_at ) VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, now()) @@ -242,7 +242,7 @@ async function insertOrUpdateCollection(collection) { collectionTitle, collection.description || null, collection.license || null, - spatialExtend, + spatialExtent, temporalStart, temporalEnd, false, // is_api - will be determined by crawler diff --git a/db/.gitignore b/db/.gitignore index e69de29..2eea525 100644 --- a/db/.gitignore +++ b/db/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/db/README.md b/db/README.md index da1c603..05e1542 100644 --- a/db/README.md +++ b/db/README.md @@ -59,11 +59,12 @@ Comprehensive indexing for optimal query performance: ```bash cd ./db/ docker-compose up +``` ### Connection Details - **Host**: `atlas.stacindex.org` -- **Port**: `5432` +- **Port**: `5432` and `5433` ## Port Configuration @@ -71,15 +72,19 @@ This project exposes the database service on a port that can be changed. Update The database uses port mapping in the format `HOST:CONTAINER`: - **`5432:5432`** means: - - Left side (`5432`): Port on your local machine (host) + - Left side (`5432`): Port on your local machine (host) (must be changed in the `.env`) - Right side (`5432`): Port inside the Docker container -What to change in the Docker Compose file +How to change the environment parameters in the Docker Compose file - Open the `docker-compose.yml`. -- Locate the `ports:` and change the host side: +- Locate e.g. `ports:` and change the host side: - Format: `":"` -- Example: change `5432:5432` to `15432:5432` to expose the container's 5432 on host port 15432. -- TODO: If the compose file references environment variables (e.g. `${DB_PORT}`), change the value in the corresponding `.env` file. +- Example: change `5432:5432` to `5433:5432` to expose the container's 5432 on host port 5433. +- If the compose file references environment variables (e.g. `${DB_PORT}`), change the value in the corresponding `.env` file. + +**Important**: Do not modify the `docker-compose.yml` file directly. Instead, update the port configuration in the `.env` file by changing the `${DB_PORT}`, `${POSTGRES_DB}`, `${POSTGRES_USER}` and `${POSTGRES_PASSWORD}` variable, then restart the service with `docker-compose up`. +- The change in the `.env` does not count for the ``, you can change that directly in the `docker-compose.yml` if needed. +- There is an `example.env` provided that can be renamed into `.env` and then modified. ## Initialization Scripts diff --git a/db/docker-compose.yml b/db/docker-compose.yml index 68a3f35..d45a248 100644 --- a/db/docker-compose.yml +++ b/db/docker-compose.yml @@ -3,14 +3,16 @@ services: image: postgis/postgis:16-3.4 container_name: stac_db restart: always + env_file: + - .env environment: - POSTGRES_DB: stac_db - POSTGRES_USER: stac_user - POSTGRES_PASSWORD: stac_password + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} ports: - - "5432:5432" + - "${DB_PORT}:5432" volumes: - stac_data:/var/lib/postgresql/data diff --git a/db/example.env b/db/example.env new file mode 100644 index 0000000..e175b0b --- /dev/null +++ b/db/example.env @@ -0,0 +1,7 @@ +# PostgreSQL Database Configuration +POSTGRES_DB= # stac_db is the database we are running on +POSTGRES_USER= # add postgres_user here +POSTGRES_PASSWORD= # add postgres_password here + +# Database Port (host:container) +DB_PORT= # 5432 / 5433 (at the moment both are available) \ No newline at end of file diff --git a/db/init/02_tables_catalog.sql b/db/init/02_tables_catalog.sql index 9078d24..a44355f 100644 --- a/db/init/02_tables_catalog.sql +++ b/db/init/02_tables_catalog.sql @@ -3,7 +3,7 @@ -- Main catalog table: Stores STAC catalog metadata including version, type, title, and description -- Each catalog represents a STAC catalog endpoint that has been discovered and indexed CREATE TABLE catalog ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, stac_version TEXT, type TEXT, title TEXT, @@ -15,7 +15,7 @@ CREATE TABLE catalog ( -- Catalog links table: Stores related links for catalogs (e.g., self, root, child, item links) -- Links define the navigation structure between STAC resources CREATE TABLE catalog_links ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, rel TEXT, href TEXT, @@ -26,21 +26,21 @@ CREATE TABLE catalog_links ( -- Keywords lookup table: Stores unique searchable keywords -- Used by both catalogs and collections for categorization and search CREATE TABLE keywords ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, keyword TEXT UNIQUE ); -- STAC extensions lookup table: Stores unique STAC extension identifiers -- Extensions provide additional standardized fields beyond core STAC spec CREATE TABLE stac_extensions ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, stac_extension TEXT UNIQUE ); -- Crawl log for catalogs: Tracks when each catalog was last crawled for updates -- Used to schedule re-crawling and maintain freshness of catalog data CREATE TABLE crawllog_catalog ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, last_crawled TIMESTAMP ); diff --git a/db/init/03_tables_collections.sql b/db/init/03_tables_collections.sql index 0fd6197..733bb08 100644 --- a/db/init/03_tables_collections.sql +++ b/db/init/03_tables_collections.sql @@ -4,8 +4,9 @@ -- Collections group related STAC items and define their common properties -- full_json: Complete JSONB representation the whole collection CREATE TABLE collection ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, stac_version TEXT, + stac_id INTEGER, type TEXT, title TEXT, description TEXT, @@ -13,9 +14,9 @@ CREATE TABLE collection ( created_at TIMESTAMP DEFAULT now(), updated_at TIMESTAMP DEFAULT now(), - spatial_extend GEOMETRY(POLYGON, 4326), - temporal_extend_start TIMESTAMP, - temporal_extend_end TIMESTAMP, + spatial_extent GEOMETRY(POLYGON, 4326), + temporal_extent_start TIMESTAMP, + temporal_extent_end TIMESTAMP, is_api BOOLEAN DEFAULT FALSE, is_active BOOLEAN DEFAULT TRUE, @@ -27,7 +28,7 @@ CREATE TABLE collection ( -- represent ranges (min/max), sets of values, or JSON schemas -- Used to describe the range of values found in collection items CREATE TABLE collection_summaries ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, collection_id INTEGER REFERENCES collection(id) ON DELETE CASCADE, name TEXT, kind TEXT, @@ -40,14 +41,14 @@ CREATE TABLE collection_summaries ( -- Providers lookup table: Stores unique data provider names -- Providers are organizations or entities that produce, host, or process the data CREATE TABLE providers ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, provider TEXT UNIQUE ); -- Assets table: Stores downloadable assets (data files, thumbnails, metadata files, etc.) -- Assets are the actual data products or resources associated with collections CREATE TABLE assets ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, name TEXT, href TEXT, type TEXT, @@ -59,7 +60,7 @@ CREATE TABLE assets ( -- Used to schedule re-crawling and maintain freshness of collection data -- (same usecase as the crawllog for catalogs) CREATE TABLE crawllog_collection ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, collection_id INTEGER REFERENCES collection(id) ON DELETE CASCADE, last_crawled TIMESTAMP ); diff --git a/db/init/05_indexes.sql b/db/init/05_indexes.sql index 76a750f..9349b88 100644 --- a/db/init/05_indexes.sql +++ b/db/init/05_indexes.sql @@ -25,10 +25,10 @@ CREATE INDEX idx_crawllog_catalog_last ON crawllog_catalog (last_crawled); -- Basic collection lookups CREATE INDEX idx_collection_title ON collection (title); -CREATE INDEX idx_collection_temp ON collection (temporal_extend_start, temporal_extend_end); +CREATE INDEX idx_collection_temp ON collection (temporal_extent_start, temporal_extent_end); CREATE INDEX idx_collection_active ON collection (is_active); -CREATE INDEX idx_collection_spatial ON collection USING GIST (spatial_extend); +CREATE INDEX idx_collection_spatial ON collection USING GIST (spatial_extent); CREATE INDEX idx_collection_fulltext ON collection USING GIN (to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))); From cc60a7c551e9dbb5f9cdb09af9159a229339db3d Mon Sep 17 00:00:00 2001 From: mammutor Date: Thu, 8 Jan 2026 12:09:17 +0100 Subject: [PATCH 39/68] refactor(api): simplify collections link handling in STAC API root function - Streamlined the logic for retrieving the collections link by directly using the href property if available, improving code clarity and reducing unnecessary checks. --- crawler/apis/api.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index ff02ca5..27a0615 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -181,14 +181,10 @@ async function handleApiRoot({ request, json, crawler, log, indent, results }) { if (typeof stacObj.getApiCollectionsLink === 'function') { const collectionsLink = stacObj.getApiCollectionsLink(); - if (collectionsLink) { - const linkUrl = typeof collectionsLink.getAbsoluteUrl === 'function' - ? collectionsLink.getAbsoluteUrl() - : collectionsLink.href; - if (linkUrl) { - collectionsEndpoint = linkUrl; - log.info(`${indent}Found collections link via stac-js: ${collectionsEndpoint}`); - } + if (collectionsLink && collectionsLink.href) { + // Direkt die href verwenden - diese ist bereits absolut im JSON + collectionsEndpoint = collectionsLink.href; + log.info(`${indent}Found collections link via stac-js: ${collectionsEndpoint}`); } } From 08b398df389175c511cd140ad76fe98f5596b0a5 Mon Sep 17 00:00:00 2001 From: mammutor Date: Thu, 8 Jan 2026 12:11:04 +0100 Subject: [PATCH 40/68] chore(crawler): comment out restart policy in docker-compose.yml --- crawler/docker-compose.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crawler/docker-compose.yml b/crawler/docker-compose.yml index 37ffa14..f154ad5 100644 --- a/crawler/docker-compose.yml +++ b/crawler/docker-compose.yml @@ -4,14 +4,11 @@ services: context: . dockerfile: Dockerfile container_name: stac-crawler - restart: unless-stopped + # restart: unless-stopped environment: - NODE_ENV=production networks: - stac-network - # Optional: Add volume to persist data - # volumes: - # - ./data:/app/data networks: stac-network: From e095d0adec4c726cca30eec753cc0731a15dce6b Mon Sep 17 00:00:00 2001 From: mammutor Date: Thu, 8 Jan 2026 14:26:15 +0100 Subject: [PATCH 41/68] fix(crawler): update .env.example and normalize base URL handling - Added new environment variables for Postgres/PostGis configuration and crawler settings in .env.example. - Improved URL handling in tryCollectionEndpoints by normalizing the base URL to prevent double slashes. --- crawler/.env.example | 14 ++++++++++++++ crawler/utils/endpoints.js | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crawler/.env.example b/crawler/.env.example index 7c52159..d257021 100644 --- a/crawler/.env.example +++ b/crawler/.env.example @@ -1,8 +1,22 @@ # STAC Crawler Configuration # Copy this file to .env and adjust values as needed +# URI of the Postgres/PostGis Host PGHOST=example_db_URL +# Postgres/PostGis Port PGPORT=5432 +# Postgres/PostGis user PGUSER=example_user +# Postgres/PostGis password PGPASSWORD=example_password +# Postgres/PostGis databse PGDATABASE=example_db + +# 'catalogs', 'apis', or 'both' +CRAWL_MODE=both +# Maximum Static Catalogs that are crawled (0 ist unlimited) +MAX_CATALOGS=0 +# Maximum API Catalogs that are crawled (0 ist unlimited) +MAX_APIS=0 +# Timeout of connection of crawler +TIMEOUT_MS=30000 \ No newline at end of file diff --git a/crawler/utils/endpoints.js b/crawler/utils/endpoints.js index 09ebe6f..786d68b 100644 --- a/crawler/utils/endpoints.js +++ b/crawler/utils/endpoints.js @@ -14,6 +14,9 @@ * @param {string} indent - Indentation for logging */ export async function tryCollectionEndpoints(baseUrl, catalogId, depth, crawler, log, indent) { + // Trailing Slash remove, because it causes double slashes + const normalizedBaseUrl = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl; + const collectionEndpoints = [ '/collections', '/collections/', @@ -21,7 +24,7 @@ export async function tryCollectionEndpoints(baseUrl, catalogId, depth, crawler, ]; const requests = collectionEndpoints.map(endpoint => ({ - url: `${baseUrl}${endpoint}`, + url: `${normalizedBaseUrl}${endpoint}`, label: 'COLLECTIONS', userData: { catalogUrl: baseUrl, From e0b254349f3eddf5ccc0f3fc963687c03b951fe6 Mon Sep 17 00:00:00 2001 From: mammutor Date: Sun, 11 Jan 2026 13:25:45 +0100 Subject: [PATCH 42/68] chore(db): update docker-compose.yml for network configuration - Changed the networks section to use array syntax for consistency. - Added a driver specification for the stac-network to use bridge mode. --- db/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/docker-compose.yml b/db/docker-compose.yml index d45a248..2e342c0 100644 --- a/db/docker-compose.yml +++ b/db/docker-compose.yml @@ -17,12 +17,12 @@ services: volumes: - stac_data:/var/lib/postgresql/data - ./init:/docker-entrypoint-initdb.d - networks: - - stac-network + networks: [stac-network] volumes: stac_data: networks: stac-network: - name: stac-network \ No newline at end of file + name: stac-network + driver: bridge \ No newline at end of file From d140405100c6e222d0689cc4936fa0cb00ea6f34 Mon Sep 17 00:00:00 2001 From: mammutor Date: Sun, 11 Jan 2026 14:15:04 +0100 Subject: [PATCH 43/68] feat(crawler): implement collection flushing to database - Added a new utility function `flushCollectionsToDb` to handle batch saving of collections to the database. - Integrated flushing logic into the crawling process to save collections periodically and at the end of the crawl. - Updated statistics to track saved and failed collections during the flush operation. - Enhanced logging for better visibility of the flushing process. --- crawler/catalogs/catalog.js | 19 +++++++++-- crawler/utils/handlers.js | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index cbb825a..fb8bde2 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -3,8 +3,8 @@ * @module catalogs/catalog */ -import { HttpCrawler } from 'crawlee'; -import { handleCatalog, handleCollections } from '../utils/handlers.js'; +import { HttpCrawler, log as crawleeLog } from 'crawlee'; +import { handleCatalog, handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; /** * Creates and runs a Crawlee HttpCrawler to crawl STAC catalogs @@ -27,6 +27,8 @@ async function crawlCatalogs(initialCatalogs, config = {}) { successfulRequests: 0, failedRequests: 0, collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0, catalogsProcessed: 0, stacCompliant: 0, nonCompliant: 0 @@ -96,6 +98,15 @@ async function crawlCatalogs(initialCatalogs, config = {}) { console.log(`\nStarting Crawlee crawler with ${initialRequests.length} initial catalogs...\n`); await crawler.run(); + // Flush any remaining collections to database + console.log('\nFlushing remaining collections to database...'); + const finalFlush = await flushCollectionsToDb(results, crawleeLog, true); + results.stats.collectionsSaved += finalFlush.saved; + results.stats.collectionsFailed += finalFlush.failed; + + // Clear catalogs array to free memory (we don't need them after crawl) + results.catalogs.length = 0; + console.log('\nCrawl Statistics:'); console.log(` Total Requests: ${results.stats.totalRequests}`); console.log(` Successful: ${results.stats.successfulRequests}`); @@ -103,7 +114,9 @@ async function crawlCatalogs(initialCatalogs, config = {}) { console.log(` STAC Compliant: ${results.stats.stacCompliant}`); console.log(` Non-Compliant: ${results.stats.nonCompliant}`); console.log(` Catalogs Processed: ${results.stats.catalogsProcessed}`); - console.log(` Collections Found: ${results.stats.collectionsFound}\n`); + console.log(` Collections Found: ${results.stats.collectionsFound}`); + console.log(` Collections Saved to DB: ${results.stats.collectionsSaved}`); + console.log(` Collections Failed: ${results.stats.collectionsFailed}\n`); return results; } diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index f60facc..5a7570b 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -6,6 +6,68 @@ import create from 'stac-js'; import { normalizeCollection } from './normalization.js'; import { tryCollectionEndpoints } from './endpoints.js'; +import db from './db.js'; + +/** + * Batch size for saving collections to database + * After this many collections are collected, they will be flushed to DB + * @type {number} + */ +const BATCH_SIZE = 1000; + +/** + * Flushes collected collections to the database and clears the array + * @async + * @param {Object} results - Results object containing collections array + * @param {Object} log - Logger instance + * @param {boolean} force - If true, flush even if below batch size (used at end of crawl) + * @returns {Promise<{saved: number, failed: number}>} Count of saved and failed collections + */ +export async function flushCollectionsToDb(results, log, force = false) { + if (!force && results.collections.length < BATCH_SIZE) { + return { saved: 0, failed: 0 }; + } + + if (results.collections.length === 0) { + return { saved: 0, failed: 0 }; + } + + const collectionsToSave = [...results.collections]; + results.collections.length = 0; // Clear the array to free memory + + let saved = 0; + let failed = 0; + + log.info(`[BATCH] Flushing ${collectionsToSave.length} collections to database...`); + + for (const collection of collectionsToSave) { + try { + await db.insertOrUpdateCollection(collection); + saved++; + } catch (err) { + log.warning(`[BATCH] Failed to save collection ${collection.id}: ${err.message}`); + failed++; + } + } + + log.info(`[BATCH] Saved ${saved} collections, ${failed} failed`); + + return { saved, failed }; +} + +/** + * Checks if batch size is reached and flushes if necessary + * @async + * @param {Object} results - Results object containing collections array + * @param {Object} log - Logger instance + */ +async function checkAndFlush(results, log) { + if (results.collections.length >= BATCH_SIZE) { + const { saved, failed } = await flushCollectionsToDb(results, log, false); + results.stats.collectionsSaved += saved; + results.stats.collectionsFailed += failed; + } +} /** * Handles catalog requests - validates STAC, extracts child catalogs and collections @@ -57,6 +119,9 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul results.collections.push(collection); results.stats.collectionsFound++; log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + + // Check if we should flush to database + await checkAndFlush(results, log); } // Try to get collections from this catalog @@ -203,5 +268,8 @@ export async function handleCollections({ request, json, crawler, log, indent, r if (collections.length > 0) { log.info(`${indent} Sample: ${collections[0].id} - ${collections[0].title}`); } + + // Check if we should flush to database + await checkAndFlush(results, log); } } From 81bd2460a01eb32a7aafac7c211f2725dbfc372f Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 13 Jan 2026 15:18:46 +0100 Subject: [PATCH 44/68] feat(crawler): enhance concurrency and storage configuration for crawling - Updated `crawlApis` and `crawlCatalogs` functions to use in-memory storage, preventing file lock race conditions under high concurrency. - Set `maxConcurrency` to 20 and `maxRequestsPerMinute` to 200 to limit request rates and improve stability during high-load scenarios. - Imported `Configuration` from `crawlee` to manage global settings effectively. --- crawler/apis/api.js | 7 ++++++- crawler/catalogs/catalog.js | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 27a0615..2b6c45c 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -3,7 +3,7 @@ * @module apis/api */ -import { HttpCrawler } from 'crawlee'; +import { HttpCrawler, Configuration } from 'crawlee'; import create from 'stac-js'; import { normalizeCollection } from '../utils/normalization.js'; import { handleCollections } from '../utils/handlers.js'; @@ -17,6 +17,9 @@ import { handleCollections } from '../utils/handlers.js'; * @returns {Promise} Results object with collections array and statistics */ async function crawlApis(urls, isApi, config = {}) { + // Use in-memory storage to avoid file lock race conditions under high concurrency + Configuration.getGlobalConfig().set('persistStorage', false); + if (!isApi || !Array.isArray(urls) || urls.length === 0) { return { collections: [], @@ -53,6 +56,8 @@ async function crawlApis(urls, isApi, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, + maxConcurrency: 20, // Limit concurrency to prevent lock file race conditions + maxRequestsPerMinute: 200, // Rate limit to avoid overwhelming targets async requestHandler({ request, json, crawler, log }) { results.stats.totalRequests++; diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index fb8bde2..1bdd918 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -3,7 +3,7 @@ * @module catalogs/catalog */ -import { HttpCrawler, log as crawleeLog } from 'crawlee'; +import { HttpCrawler, log as crawleeLog, Configuration } from 'crawlee'; import { handleCatalog, handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; /** @@ -14,6 +14,9 @@ import { handleCatalog, handleCollections, flushCollectionsToDb } from '../utils * @returns {Promise} Crawl results with collections and statistics */ async function crawlCatalogs(initialCatalogs, config = {}) { + // Use in-memory storage to avoid file lock race conditions under high concurrency + Configuration.getGlobalConfig().set('persistStorage', false); + const timeoutSecs = config.timeout && config.timeout !== Infinity ? Math.ceil(config.timeout / 1000) : 60; @@ -37,6 +40,8 @@ async function crawlCatalogs(initialCatalogs, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, + maxConcurrency: 20, // Limit concurrency to prevent lock file race conditions + maxRequestsPerMinute: 200, // Rate limit to avoid overwhelming targets async requestHandler({ request, json, crawler, log }) { results.stats.totalRequests++; From 3ba13a93b5e247176a06e2aada01c23c11d67d78 Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 14 Jan 2026 11:08:54 +0100 Subject: [PATCH 45/68] feat(crawler): implement max depth configuration and batch flushing for collections - Updated the crawler to support a maximum depth for nested catalogs, preventing excessive recursion. - Enhanced the collection handling by implementing batch flushing to the database during the crawl process. - Improved logging to provide detailed statistics on collections found, saved, and failed during the crawl. - Added CLI options for configuring maximum depth and updated the configuration defaults accordingly. --- crawler/apis/api.js | 72 +++++++++++++++++++++++++++++++++++------ crawler/index.js | 34 ++++--------------- crawler/utils/cli.js | 6 ++++ crawler/utils/config.js | 7 ++-- 4 files changed, 80 insertions(+), 39 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 2b6c45c..b10dc13 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -3,10 +3,30 @@ * @module apis/api */ -import { HttpCrawler, Configuration } from 'crawlee'; +import { HttpCrawler, Configuration, log as crawleeLog } from 'crawlee'; import create from 'stac-js'; import { normalizeCollection } from '../utils/normalization.js'; -import { handleCollections } from '../utils/handlers.js'; +import { handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; + +/** + * Batch size for saving collections to database during API crawling + * @type {number} + */ +const BATCH_SIZE = 500; + +/** + * Checks if batch size is reached and flushes if necessary + * @async + * @param {Object} results - Results object containing collections array + * @param {Object} log - Logger instance + */ +async function checkAndFlushApi(results, log) { + if (results.collections.length >= BATCH_SIZE) { + const { saved, failed } = await flushCollectionsToDb(results, log, false); + results.stats.collectionsSaved = (results.stats.collectionsSaved || 0) + saved; + results.stats.collectionsFailed = (results.stats.collectionsFailed || 0) + failed; + } +} /** * Crawls STAC APIs to retrieve collection information without fetching items. @@ -48,12 +68,17 @@ async function crawlApis(urls, isApi, config = {}) { successfulRequests: 0, failedRequests: 0, collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0, apisProcessed: 0, stacCompliant: 0, nonCompliant: 0 } }; + // Maximum depth for nested catalog crawling (0 = unlimited) + const maxDepth = config.maxDepth || 10; + const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, maxConcurrency: 20, // Limit concurrency to prevent lock file race conditions @@ -61,12 +86,13 @@ async function crawlApis(urls, isApi, config = {}) { async requestHandler({ request, json, crawler, log }) { results.stats.totalRequests++; - const indent = ' '; + const depth = request.userData?.depth || 0; + const indent = ' '.repeat(Math.min(depth, 5)); // Cap indent at 5 levels try { // Route based on request label if (request.label === 'API_ROOT') { - await handleApiRoot({ request, json, crawler, log, indent, results }); + await handleApiRoot({ request, json, crawler, log, indent, results, maxDepth }); } else if (request.label === 'API_COLLECTIONS') { await handleCollections({ request, json, crawler, log, indent, results }); } else if (request.label === 'API_COLLECTION') { @@ -109,7 +135,8 @@ async function crawlApis(urls, isApi, config = {}) { label: 'API_ROOT', userData: { apiId: `api-${index}`, - apiUrl: url + apiUrl: url, + depth: 0 } })); @@ -118,6 +145,15 @@ async function crawlApis(urls, isApi, config = {}) { console.log(`\nStarting Crawlee crawler with ${initialRequests.length} APIs...\n`); await crawler.run(); + // Flush any remaining collections to database + console.log('\nFlushing remaining API collections to database...'); + const finalFlush = await flushCollectionsToDb(results, crawleeLog, true); + results.stats.collectionsSaved += finalFlush.saved; + results.stats.collectionsFailed += finalFlush.failed; + + // Clear apis array to free memory (we don't need them after crawl) + results.apis.length = 0; + console.log('\nAPI Crawl Statistics:'); console.log(` Total Requests: ${results.stats.totalRequests}`); console.log(` Successful: ${results.stats.successfulRequests}`); @@ -125,7 +161,9 @@ async function crawlApis(urls, isApi, config = {}) { console.log(` STAC Compliant: ${results.stats.stacCompliant}`); console.log(` Non-Compliant: ${results.stats.nonCompliant}`); console.log(` APIs Processed: ${results.stats.apisProcessed}`); - console.log(` Collections Found: ${results.stats.collectionsFound}\n`); + console.log(` Collections Found: ${results.stats.collectionsFound}`); + console.log(` Collections Saved to DB: ${results.stats.collectionsSaved}`); + console.log(` Collections Failed: ${results.stats.collectionsFailed}\n`); return results; } @@ -141,12 +179,14 @@ async function crawlApis(urls, isApi, config = {}) { * @param {Object} context.log - Logger instance * @param {string} context.indent - Indentation for logging * @param {Object} context.results - Results object to store data + * @param {number} context.maxDepth - Maximum recursion depth (0 = unlimited) */ -async function handleApiRoot({ request, json, crawler, log, indent, results }) { +async function handleApiRoot({ request, json, crawler, log, indent, results, maxDepth = 10 }) { const apiId = request.userData?.apiId || 'unknown'; const apiUrl = request.userData?.apiUrl || request.url; + const depth = request.userData?.depth || 0; - log.info(`${indent}Processing API: ${apiId} at ${apiUrl}`); + log.info(`${indent}Processing API: ${apiId} at ${apiUrl} (depth: ${depth})`); // Validate with stac-js let stacObj; @@ -178,6 +218,9 @@ async function handleApiRoot({ request, json, crawler, log, indent, results }) { results.collections.push(collection); results.stats.collectionsFound++; log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + + // Check if we should flush to database + await checkAndFlushApi(results, log); return; // Single collection, no need to look for more } @@ -233,6 +276,13 @@ async function handleApiRoot({ request, json, crawler, log, indent, results }) { if (childLinks.length > 0) { log.info(`${indent}Found ${childLinks.length} child catalog links`); + // Check if we've reached maximum depth + const nextDepth = depth + 1; + if (maxDepth > 0 && nextDepth > maxDepth) { + log.warning(`${indent}Skipping ${childLinks.length} child catalogs - max depth (${maxDepth}) reached`); + return; + } + const childRequests = childLinks .map((link, idx) => { let childUrl; @@ -260,7 +310,8 @@ async function handleApiRoot({ request, json, crawler, log, indent, results }) { userData: { apiId: `${apiId}-child-${idx}`, apiUrl: apiUrl, - parentId: apiId + parentId: apiId, + depth: nextDepth } }; }) @@ -298,6 +349,9 @@ async function handleApiCollection({ request, json, crawler, log, indent, result results.collections.push(collection); results.stats.collectionsFound++; log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); + + // Check if we should flush to database + await checkAndFlushApi(results, log); } else { log.warning(`${indent}Expected collection but got: ${json.type || 'unknown type'}`); } diff --git a/crawler/index.js b/crawler/index.js index 32cbabd..9b722e3 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -48,7 +48,7 @@ export const crawler = async () => { console.log(`Max Catalogs: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs} (debugging limit)`); console.log(`Max APIs: ${config.maxApis === 0 ? 'unlimited' : config.maxApis} (debugging limit)`); console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); - console.log(`Crawl Depth: unlimited`); + console.log(`Max Depth: ${config.maxDepth === 0 ? 'unlimited' : config.maxDepth} levels`); console.log('==================================\n'); const response = await axios.get(targetUrl); @@ -109,33 +109,11 @@ export const crawler = async () => { try { const apiResults = await crawlApis(apisToProcess, true, config); - const collections = apiResults.collections || []; - console.log(`\nFetched ${collections.length} collections from APIs.`); - - if (collections.length > 0) { - console.log('First 3 collections found:'); - collections.slice(0, 3).forEach(c => { - console.log(` - ${c.id}: ${c.title}`); - }); - - // Save API collections to database - console.log('\n=== Saving API collections to database ==='); - let apiCollectionsSaved = 0; - let apiCollectionsFailed = 0; - - for (const collection of collections) { - try { - const collectionId = await db.insertOrUpdateCollection(collection); - console.log(` Saved: ${collection.title || collection.id} (DB ID: ${collectionId})`); - apiCollectionsSaved++; - } catch (err) { - console.error(` Failed: ${collection.title || collection.id} - ${err.message}`); - apiCollectionsFailed++; - } - } - - console.log(`\nAPI Collections: ${apiCollectionsSaved} saved, ${apiCollectionsFailed} failed`); - } + // Collections are now saved to DB during the crawl (batch flushing) + console.log(`\nAPI Crawl Complete:`); + console.log(` Collections Found: ${apiResults.stats.collectionsFound}`); + console.log(` Collections Saved: ${apiResults.stats.collectionsSaved}`); + console.log(` Collections Failed: ${apiResults.stats.collectionsFailed}`); } catch (error) { console.error(`Failed to crawl APIs: ${error.message}`); } diff --git a/crawler/utils/cli.js b/crawler/utils/cli.js index 035a46f..ac01f5b 100644 --- a/crawler/utils/cli.js +++ b/crawler/utils/cli.js @@ -23,6 +23,8 @@ export function parseCliArgs() { config.maxApis = parseInt(args[++i], 10); } else if (arg === '--timeout' || arg === '-t') { config.timeout = parseInt(args[++i], 10); + } else if (arg === '--max-depth' || arg === '-d') { + config.maxDepth = parseInt(args[++i], 10); } else if (arg === '--help' || arg === '-h') { printHelp(); process.exit(0); @@ -45,6 +47,8 @@ STAC Crawler Configuration Options: -a, --max-apis Maximum number of APIs to crawl (default: 5, use 0 for unlimited) Note: Limits are for debugging purposes only -t, --timeout Timeout for each crawl operation in ms (default: 30000) + -d, --max-depth Maximum recursion depth for nested catalogs (default: 10, use 0 for unlimited) + Prevents memory issues from deeply nested catalog hierarchies -h, --help Show this help message Environment Variables: @@ -52,11 +56,13 @@ Environment Variables: MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) MAX_APIS Same as --max-apis (use 0 for unlimited) TIMEOUT_MS Same as --timeout + MAX_DEPTH Same as --max-depth (use 0 for unlimited) Examples: node index.js --mode catalogs --max-catalogs 20 node index.js -m apis -a 10 -t 60000 node index.js -m both -c 0 -a 0 # Unlimited mode (no debugging limits) + node index.js -m apis -d 5 # Limit nesting depth to 5 levels CRAWL_MODE=both MAX_CATALOGS=50 node index.js `); } diff --git a/crawler/utils/config.js b/crawler/utils/config.js index 6fa883b..39efed5 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -22,7 +22,8 @@ function getConfig() { mode: 'both', // 'catalogs', 'apis', or 'both' maxCatalogs: 10, // Maximum number of catalogs to crawl maxApis: 5, // Maximum number of APIs to crawl - timeout: 30000 // Timeout in milliseconds (30 seconds) + timeout: 30000, // Timeout in milliseconds (30 seconds) + maxDepth: 10 // Maximum recursion depth for nested catalogs (0 = unlimited) }; // Build configuration with precedence: CLI > ENV > Defaults @@ -33,7 +34,9 @@ function getConfig() { maxApis: cliArgs.maxApis !== undefined ? cliArgs.maxApis : (process.env.MAX_APIS ? parseInt(process.env.MAX_APIS, 10) : defaults.maxApis), timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : - (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout) + (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), + maxDepth: cliArgs.maxDepth !== undefined ? cliArgs.maxDepth : + (process.env.MAX_DEPTH ? parseInt(process.env.MAX_DEPTH, 10) : defaults.maxDepth) }; // Validate mode From 44b986a49f988e8f09158bc12f337ecc72bb55c9 Mon Sep 17 00:00:00 2001 From: Jakob <70800454+vertrox78@users.noreply.github.com> Date: Wed, 14 Jan 2026 11:18:53 +0100 Subject: [PATCH 46/68] Crawler: implemented rate limiting, and implements some bug fixes (#211) * added crawling time statistics * #189, added 7 day scheduling, created time util for easier managment/readability * #202, feat(crawler): add rate limiting and STAC link discovery, fixed bug with wndpoint logic and additional MIME types * #227, fixed bug catalogs that where listed as catlogs were treated as apis, no will be handled by the catalog crawler --- crawler/apis/api.js | 57 +++++++++++++++-------------- crawler/catalogs/catalog.js | 12 +++++++ crawler/index.js | 30 +++++++++++----- crawler/utils/cli.js | 27 +++++++++++--- crawler/utils/config.js | 37 +++++++++++++++++-- crawler/utils/endpoints.js | 71 +++++++++++++++++++++++++++---------- crawler/utils/handlers.js | 4 +-- 7 files changed, 174 insertions(+), 64 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 2b6c45c..61ff9fa 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -9,6 +9,9 @@ import { normalizeCollection } from '../utils/normalization.js'; import { handleCollections } from '../utils/handlers.js'; /** + * Note: This crawler is for REAL APIs only. Static catalogs (*.json files) + * are now routed to the catalog crawler in index.js + * * Crawls STAC APIs to retrieve collection information without fetching items. * * @param {string[]} urls - Array of API URLs to crawl @@ -59,6 +62,15 @@ async function crawlApis(urls, isApi, config = {}) { maxConcurrency: 20, // Limit concurrency to prevent lock file race conditions maxRequestsPerMinute: 200, // Rate limit to avoid overwhelming targets + // Rate limiting options + maxConcurrency: config.maxConcurrency || 5, + maxRequestsPerMinute: config.maxRequestsPerMinute || 60, + sameDomainDelaySecs: config.sameDomainDelaySecs || 1, + maxRequestRetries: config.maxRequestRetries || 3, + + // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) + additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], + async requestHandler({ request, json, crawler, log }) { results.stats.totalRequests++; const indent = ' '; @@ -94,6 +106,9 @@ async function crawlApis(urls, isApi, config = {}) { log.warning(`${indent}[TIMEOUT] ${apiId} at ${request.url}`); } else if (error.message.includes('ENOTFOUND') || error.message.includes('ECONNREFUSED')) { log.warning(`${indent}[CONNECTION FAILED] ${apiId} at ${request.url}`); + } else if (error.statusCode === 429) { + const retryAfter = error.response?.headers?.['retry-after'] || 'unknown'; + log.warning(`${indent}[RATE LIMITED] ${apiId} at ${request.url} - Retry-After: ${retryAfter}s`); } else if (error.code === 'ERR_NON_2XX_3XX_RESPONSE') { log.warning(`${indent}[HTTP ERROR] ${apiId} at ${request.url} - Status: ${error.statusCode}`); } else { @@ -193,39 +208,23 @@ async function handleApiRoot({ request, json, crawler, log, indent, results }) { } } - // Fallback: try common STAC API endpoints + // Fallback: use standard /collections endpoint if (!collectionsEndpoint) { const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; - const endpoints = [ - `${baseUrl}/collections`, - `${apiUrl.endsWith('/') ? apiUrl.slice(0, -1) : apiUrl}/collections` - ]; - - for (const endpoint of endpoints) { - if (endpoint && endpoint !== collectionsEndpoint) { - log.info(`${indent}Trying collections endpoint: ${endpoint}`); - await crawler.addRequests([{ - url: endpoint, - label: 'API_COLLECTIONS', - userData: { - apiId: apiId, - apiUrl: apiUrl - } - }]); - } - } - } else { - // Use the discovered collections endpoint - await crawler.addRequests([{ - url: collectionsEndpoint, - label: 'API_COLLECTIONS', - userData: { - apiId: apiId, - apiUrl: apiUrl - } - }]); + collectionsEndpoint = `${baseUrl}/collections`; + log.debug(`${indent}No collections link found, using fallback: ${collectionsEndpoint}`); } + // Enqueue single collections request + await crawler.addRequests([{ + url: collectionsEndpoint, + label: 'API_COLLECTIONS', + userData: { + apiId: apiId, + apiUrl: apiUrl + } + }]); + // Also check for child links (nested catalogs) if (typeof stacObj.getChildLinks === 'function') { const childLinks = stacObj.getChildLinks(); diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 1bdd918..9883081 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -43,6 +43,15 @@ async function crawlCatalogs(initialCatalogs, config = {}) { maxConcurrency: 20, // Limit concurrency to prevent lock file race conditions maxRequestsPerMinute: 200, // Rate limit to avoid overwhelming targets + // Rate limiting options + maxConcurrency: config.maxConcurrency || 5, + maxRequestsPerMinute: config.maxRequestsPerMinute || 60, + sameDomainDelaySecs: config.sameDomainDelaySecs || 1, + maxRequestRetries: config.maxRequestRetries || 3, + + // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) + additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], + async requestHandler({ request, json, crawler, log }) { results.stats.totalRequests++; const depth = request.userData?.depth || 0; @@ -78,6 +87,9 @@ async function crawlCatalogs(initialCatalogs, config = {}) { log.warning(`${indent}[TIMEOUT] ${catalogId} at ${request.url}`); } else if (error.message.includes('ENOTFOUND') || error.message.includes('ECONNREFUSED')) { log.warning(`${indent}[CONNECTION FAILED] ${catalogId} at ${request.url}`); + } else if (error.statusCode === 429) { + const retryAfter = error.response?.headers?.['retry-after'] || 'unknown'; + log.warning(`${indent}[RATE LIMITED] ${catalogId} at ${request.url} - Retry-After: ${retryAfter}s`); } else if (error.code === 'ERR_NON_2XX_3XX_RESPONSE') { log.warning(`${indent}[HTTP ERROR] ${catalogId} at ${request.url} - Status: ${error.statusCode}`); } else { diff --git a/crawler/index.js b/crawler/index.js index 32cbabd..b9bcbb6 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -7,9 +7,9 @@ import axios from 'axios'; import { processCatalogs } from './utils/normalization.js'; import { crawlCatalogs } from './catalogs/catalog.js'; import { crawlApis } from './apis/api.js'; -import { getConfig } from './utils/config.js'; -import db from './utils/db.js'; +import { getConfig, isStaticCatalogUrl } from './utils/config.js'; import { formatDuration } from './utils/time.js'; +import db from './utils/db.js'; /** * URL of the STAC Index API endpoint @@ -72,17 +72,31 @@ export const crawler = async () => { console.log(`\nCatalogs: ${catalogsSaved} saved, ${catalogsFailed} failed\n`); + // Separate static catalogs from real APIs + const staticCatalogs = catalogs.filter(cat => cat.isApi === true && isStaticCatalogUrl(cat.url)); + const regularCatalogs = catalogs.filter(cat => cat.isApi !== true); + const realApis = catalogs.filter(cat => cat.isApi === true && !isStaticCatalogUrl(cat.url)); + + console.log(`\nCatalog Classification:`); + console.log(` Regular Catalogs: ${regularCatalogs.length}`); + console.log(` Static Catalogs (mismarked as APIs): ${staticCatalogs.length}`); + console.log(` Real APIs: ${realApis.length}\n`); + // Crawl catalogs if mode is 'catalogs' or 'both' if (config.mode === 'catalogs' || config.mode === 'both') { console.log('\nCrawling collections and nested catalogs with Crawlee...\n'); + // Combine regular catalogs with static catalogs (mismarked as APIs) + const allCatalogsToProcess = [...regularCatalogs, ...staticCatalogs]; + // Note: MAX_CATALOGS limit is for debugging purposes only // Set maxCatalogs to 0 or use --max-catalogs 0 for unlimited catalog crawling const catalogsToProcess = config.maxCatalogs === 0 - ? catalogs - : catalogs.slice(0, config.maxCatalogs); + ? allCatalogsToProcess + : allCatalogsToProcess.slice(0, config.maxCatalogs); - console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs})\n`); + console.log(`Processing ${catalogsToProcess.length} catalogs (max: ${config.maxCatalogs === 0 ? 'unlimited' : config.maxCatalogs})`); + console.log(` (includes ${staticCatalogs.length} static catalogs mismarked as APIs)\n`); try { const results = await crawlCatalogs(catalogsToProcess, config); @@ -97,15 +111,13 @@ export const crawler = async () => { // Crawl APIs if mode is 'apis' or 'both' if (config.mode === 'apis' || config.mode === 'both') { console.log('\nCrawling APIs...'); - const apiUrls = catalogs - .filter(cat => cat.isApi === true) - .map(cat => cat.url); + const apiUrls = realApis.map(cat => cat.url); if (apiUrls.length > 0) { // Note: MAX_APIS limit is for debugging purposes only // Set maxApis to 0 or use --max-apis 0 for unlimited API crawling const apisToProcess = config.maxApis === 0 ? apiUrls : apiUrls.slice(0, config.maxApis); - console.log(`Found ${apiUrls.length} APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === 0 ? 'unlimited' : config.maxApis})...`); + console.log(`Found ${realApis.length} real APIs. Processing ${apisToProcess.length} (max: ${config.maxApis === 0 ? 'unlimited' : config.maxApis})...`); try { const apiResults = await crawlApis(apisToProcess, true, config); diff --git a/crawler/utils/cli.js b/crawler/utils/cli.js index 035a46f..71408f9 100644 --- a/crawler/utils/cli.js +++ b/crawler/utils/cli.js @@ -23,6 +23,14 @@ export function parseCliArgs() { config.maxApis = parseInt(args[++i], 10); } else if (arg === '--timeout' || arg === '-t') { config.timeout = parseInt(args[++i], 10); + } else if (arg === '--max-concurrency') { + config.maxConcurrency = parseInt(args[++i], 10); + } else if (arg === '--requests-per-minute' || arg === '--rpm') { + config.maxRequestsPerMinute = parseInt(args[++i], 10); + } else if (arg === '--domain-delay') { + config.sameDomainDelaySecs = parseFloat(args[++i]); + } else if (arg === '--max-retries') { + config.maxRequestRetries = parseInt(args[++i], 10); } else if (arg === '--help' || arg === '-h') { printHelp(); process.exit(0); @@ -45,13 +53,24 @@ STAC Crawler Configuration Options: -a, --max-apis Maximum number of APIs to crawl (default: 5, use 0 for unlimited) Note: Limits are for debugging purposes only -t, --timeout Timeout for each crawl operation in ms (default: 30000) + + Rate Limiting Options: + --max-concurrency Maximum concurrent requests (default: 5) + --requests-per-minute, --rpm Maximum requests per minute (default: 60) + --domain-delay Delay between requests to same domain (default: 1) + --max-retries Maximum retries for failed requests (default: 3) + -h, --help Show this help message Environment Variables: - CRAWL_MODE Same as --mode - MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) - MAX_APIS Same as --max-apis (use 0 for unlimited) - TIMEOUT_MS Same as --timeout + CRAWL_MODE Same as --mode + MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) + MAX_APIS Same as --max-apis (use 0 for unlimited) + TIMEOUT_MS Same as --timeout + MAX_CONCURRENCY Same as --max-concurrency + MAX_REQUESTS_PER_MINUTE Same as --requests-per-minute + SAME_DOMAIN_DELAY_SECS Same as --domain-delay + MAX_REQUEST_RETRIES Same as --max-retries Examples: node index.js --mode catalogs --max-catalogs 20 diff --git a/crawler/utils/config.js b/crawler/utils/config.js index 6fa883b..148efd4 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -6,6 +6,25 @@ import dotenv from 'dotenv'; import { parseCliArgs } from './cli.js'; +/** + * Checks if a URL points to a static catalog file rather than an API endpoint + * @param {string} url - URL to check + * @returns {boolean} True if URL appears to be a static file + */ +export function isStaticCatalogUrl(url) { + if (!url || typeof url !== 'string') return false; + + // Check if URL ends with common static catalog file patterns + const staticPatterns = [ + /\.json$/i, // ends with .json + /\/collection\.json/i, // collection.json file + /\/catalog\.json/i, // catalog.json file + /\/stac\.json/i // stac.json file + ]; + + return staticPatterns.some(pattern => pattern.test(url)); +} + // Load environment variables from .env file dotenv.config(); @@ -22,7 +41,12 @@ function getConfig() { mode: 'both', // 'catalogs', 'apis', or 'both' maxCatalogs: 10, // Maximum number of catalogs to crawl maxApis: 5, // Maximum number of APIs to crawl - timeout: 30000 // Timeout in milliseconds (30 seconds) + timeout: 30000, // Timeout in milliseconds (30 seconds) + // Rate limiting options (Crawlee) + maxConcurrency: 5, // Maximum number of concurrent requests + maxRequestsPerMinute: 60, // Maximum requests per minute + sameDomainDelaySecs: 1, // Delay between requests to the same domain + maxRequestRetries: 3 // Maximum number of retries for failed requests }; // Build configuration with precedence: CLI > ENV > Defaults @@ -33,7 +57,16 @@ function getConfig() { maxApis: cliArgs.maxApis !== undefined ? cliArgs.maxApis : (process.env.MAX_APIS ? parseInt(process.env.MAX_APIS, 10) : defaults.maxApis), timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : - (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout) + (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), + // Rate limiting options + maxConcurrency: cliArgs.maxConcurrency !== undefined ? cliArgs.maxConcurrency : + (process.env.MAX_CONCURRENCY ? parseInt(process.env.MAX_CONCURRENCY, 10) : defaults.maxConcurrency), + maxRequestsPerMinute: cliArgs.maxRequestsPerMinute !== undefined ? cliArgs.maxRequestsPerMinute : + (process.env.MAX_REQUESTS_PER_MINUTE ? parseInt(process.env.MAX_REQUESTS_PER_MINUTE, 10) : defaults.maxRequestsPerMinute), + sameDomainDelaySecs: cliArgs.sameDomainDelaySecs !== undefined ? cliArgs.sameDomainDelaySecs : + (process.env.SAME_DOMAIN_DELAY_SECS ? parseFloat(process.env.SAME_DOMAIN_DELAY_SECS) : defaults.sameDomainDelaySecs), + maxRequestRetries: cliArgs.maxRequestRetries !== undefined ? cliArgs.maxRequestRetries : + (process.env.MAX_REQUEST_RETRIES ? parseInt(process.env.MAX_REQUEST_RETRIES, 10) : defaults.maxRequestRetries) }; // Validate mode diff --git a/crawler/utils/endpoints.js b/crawler/utils/endpoints.js index 786d68b..c595933 100644 --- a/crawler/utils/endpoints.js +++ b/crawler/utils/endpoints.js @@ -4,8 +4,11 @@ */ /** - * Tries common STAC collection endpoints and enqueues successful ones + * Finds the collection endpoint from STAC catalog links + * STAC catalogs should advertise their collection endpoint via rel="data" or rel="collections" + * Falls back to a single /collections endpoint if no link is found * @async + * @param {Object} stacCatalog - Parsed STAC catalog object from stac-js * @param {string} baseUrl - Base catalog URL * @param {string} catalogId - Catalog ID for logging * @param {number} depth - Current depth @@ -13,27 +16,59 @@ * @param {Object} log - Logger * @param {string} indent - Indentation for logging */ -export async function tryCollectionEndpoints(baseUrl, catalogId, depth, crawler, log, indent) { - // Trailing Slash remove, because it causes double slashes - const normalizedBaseUrl = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl; - - const collectionEndpoints = [ - '/collections', - '/collections/', - '/api/v1/collections' - ]; +export async function tryCollectionEndpoints(stacCatalog, baseUrl, catalogId, depth, crawler, log, indent) { + let collectionUrl = null; - const requests = collectionEndpoints.map(endpoint => ({ - url: `${normalizedBaseUrl}${endpoint}`, + // Try to find collection endpoint from STAC links (proper STAC discovery) + if (stacCatalog && typeof stacCatalog.getLinks === 'function') { + const links = stacCatalog.getLinks(); + + // Look for rel="data" (STAC API) or rel="collections" link + const collectionLink = links.find(link => + link.rel === 'data' || link.rel === 'collections' + ); + + if (collectionLink) { + try { + collectionUrl = typeof collectionLink.getAbsoluteUrl === 'function' + ? collectionLink.getAbsoluteUrl() + : collectionLink.href; + + // Handle relative URLs + if (collectionUrl && !collectionUrl.startsWith('http')) { + const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); + collectionUrl = `${basePath}/${collectionUrl}`; + } + + log.info(`${indent}Found collection endpoint via STAC link (rel="${collectionLink.rel}"): ${collectionUrl}`); + } catch (err) { + log.warning(`${indent}Error resolving collection link: ${err.message}`); + } + } + } + + // Fallback: if no link found, try the standard /collections endpoint + if (!collectionUrl) { + // Remove trailing filename (like catalog.json) from base URL + const urlParts = baseUrl.split('/'); + const lastPart = urlParts[urlParts.length - 1]; + + if (lastPart.includes('.json') || lastPart.includes('.')) { + urlParts.pop(); + } + + collectionUrl = urlParts.join('/') + '/collections'; + log.debug(`${indent}No collection link found, using fallback: ${collectionUrl}`); + } + + // Enqueue single collection request + await crawler.addRequests([{ + url: collectionUrl, label: 'COLLECTIONS', userData: { catalogUrl: baseUrl, catalogId, depth - }, - // Don't retry failed endpoint attempts - just try the next one - maxRetries: 0 - })); - - await crawler.addRequests(requests); + } + }]); } diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index 5a7570b..f835389 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -124,8 +124,8 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul await checkAndFlush(results, log); } - // Try to get collections from this catalog - await tryCollectionEndpoints(request.url, catalogId, depth, crawler, log, indent); + // Try to get collections from this catalog (using STAC link discovery) + await tryCollectionEndpoints(stacCatalog, request.url, catalogId, depth, crawler, log, indent); // Extract and enqueue child catalog links using stac-js if (stacCatalog && typeof stacCatalog.getChildLinks === 'function') { From df81112e6762d4ca54ff0f7ac3968fe51ceadcbb Mon Sep 17 00:00:00 2001 From: Humam <44206081+Mammutor@users.noreply.github.com> Date: Thu, 15 Jan 2026 18:00:12 +0100 Subject: [PATCH 47/68] merge from dev to dev-crawler-humam and now to dev-crawler (#243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(api): initialize STAC Atlas API with collections, conformance, and queryables routes - Added package.json for project dependencies and scripts. - Implemented GET endpoint for collections. - Created conformance endpoint to list supported conformance classes. - Developed landing page for the API with links to collections and documentation. - Added queryables endpoint to return queryable properties for collections. (If i'm correct this can be removed) * Added all Remarks to the bid and finished it (#74) * changed Texts 1,2 and 8 according to the remarks of the customer * Did my fixes to 4. and 10.3 * added remark why we want to save every catalog * deleted keywords for catalog * changed everything related to the database component * Updated 3.1, 3.3, 3,4 (References to Lastenheft/small other changes) * Added small Graph to 3. Produktumgebung * Update bid.md 7.3STAC-Validator added the handling of collections that cannot be validated automatically. * Update bid.md 7.3STAC-API-Validator changed the way we validate the collection search extension. * Update bid.md 9.3.2Endpunkte small fix collection search extension. * added Skizze for 3, and updated 6.1, 10.1 * Updated 3. Produktumgebung * Update bid.md 7.Qualitätsanforderungen minor fixes * Update 6.4.2 added remark about loading feedback * Minor change to Table in "11 Zeitplan". A collum was missing in the head, therefore the table wasn't rendering correctly.. --------- Co-authored-by: Jakob Co-authored-by: Sönke Hoffmann Co-authored-by: jklaer Co-authored-by: VincentKuehn Co-authored-by: mammutor Co-authored-by: Humam <44206081+Mammutor@users.noreply.github.com> Co-authored-by: Justin K * fix(api): enhance STAC API landing page and conformance links - Overhaul of first idea landing page - Added some more tests for the required elements in the landingpage-Catalog * feat(api): implement shared conformance URIs and add tests for conformance endpoint - implemented condormance endpoint * Implemented 2.3 and 2.4 (#111) * Temporary mock data for testing and frontend development * Added API middleware layer for error handling and validation * TODOs ready? pls review * Added API utilities for query parsing, validation, and response formatting * Added swagger and openapi.yaml * Update queryables.js Refactor queryables endpoint into /collections/queryables * Renamed the collections.js file to mocks-collections.js to better reflect its purpose and improve project clarity * changed README "Projektstruktur" * restart from dev-api 22.11..2025 * API: 2.3 Implement Collections List Endpoint done (added explanations as comments in the code) * API: 2.4 Implement Single Collection Endpoint (added explanations as comments in the code) * Update api/routes/collections.js Co-authored-by: Robin Tammo Gummels * Changed some of the code with the comments on Github (i will finish it tomorrow morning) * Implement most of the feedback and comments (need to talk about some other changes) * Update api/routes/index.js Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/README.md Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/README.md Removed missing folder * Update api/routes/collections.js Removed TODOs from wrong lines * Update api/routes/collections.js Added TODOs * Update api/routes/queryables.js Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/routes/queryables.js Changed wording from `/collections/queryables` to `/collections-queryables` --------- Co-authored-by: VincentKuehn Co-authored-by: Robin Tammo Gummels * feat(api): add collection search parameters and validation middleware (#159) * feat(api): add collection search parameters and validation middleware * Added unit-test for validator-functions and integration-tests for `GET /collections`-Querys. - Also minor bugfix, because the validator accepted deecimals as tokens. * API: 3 Database Integration first version (#161) * database connection in implementated. The parameters for the connection have to added in the .env-file. Also there is test-file for testing and console messages (installed `pg`) * support for spatial queries via postgis + error handling for datatbase operations changed language to english * error handling * added DATABASE_URL There is an issue with the distance query. Changed the error handling and testing, the console messages are now way better structured * found the Problem with the distance query. The layer are so big, that they reach over the 180° long (PostgGIS can't handel that). Now the calc is done by degree and not meters. * The two files `test-data-retrieval.js` and `verify-schema.js` have been added. `test-data-retrieval` (theoretical, checks against the spezification): ``` Discovers all tables and columns and validates against expected schema. ``` The second files `verify-schema.js` (practical, checks against the real data): ``` Discovers all tables and columns, validates against expected schema ``` * pooling error hanling and log imporoved. renamed tests files to actual test-files * standalone node tests were convertad into JEST * write file `validateRequest.js`. Validates every incoming API request, whether the request is valid and logical. * commented `stac_id` from the tests, it is not in both databases, so the tests for `stac_id` will always fail Added explanation to the `.env.example`, which port is which database * added example pattern for API - database connection. * deleted `validateRequest` cause it's already implemented by @robinGummels --------- Co-authored-by: Sönke Hoffmann * Added environment variables and a `.env` for `docker-compose.yml` (#164) * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * Added a CI/CD Pipeline to prevent pull-requests without functioning tests and proper linting. * fixed errors suggested by the linter. - Some lines used tab and spaces... * Implemented Collection Search extension including a DB-Connection (#165) * added SQLQuery-builder with these parameters: q,bbox,datetime,sortby,limit and token * finalised bbox and datetime * adapted to DB, QueryBuilder and added helperfunction runQuery * added question-TODOs * added bbox+datetime to the Query-Builder from Jonas * added tests for Query-Builder from Jonas * added tests from George * added falsely deleted TODOs again * fixed collumn names to match our DB and adjusted full text search to match 05_indexes.sql correctly * Used a formatter and linter on `buildCollectionSearchQuery.js * Did some major and minor fixes to the collection search. - Updated `buildCollectionSearchQuery` to support pagination and improved text search with English language settings. - Modified tests in `buildCollectionsSearchQuery.basic.test.js`, `collections-pagination.test.js`, and `collections-sort.test.js` to reflect new query behavior and validation logic. - Enhanced sort validation in `validators.test.js` and `collectionSearchParams.js` to map API fields to database column names. - Implemented total count retrieval for matched results in `collections.js`. * Added a internal .env creation in the CI/CD Pipeline. It utilzes GitHub Repository Secrets to not publish any private Logins and stuff. * Forgot that the second Job of the CI/CD pipeline runs seperatly and needs a internal .env file too. * Enhance documentation for buildCollectionSearchQuery Updated the documentation for: - the buildCollectionSearchQuery function - the fulltextsearch * Refactor buildCollectionSearchQuery and updated SELECT part Changed the SELECT part to match our bid and the database shema. Updated comments and for clarity. Changed full-text search to use 'simple' configuration instead of 'english'. * Update api/routes/collections.js small typo Co-authored-by: Robin Tammo Gummels * Remove sorting TODO from collections route Removed TODO comment about sorting based on sortby parameter. * Explicitly return undefined for normalized in validateSortby Update validateSortby function to explicitly return undefined for normalized when sortby is not provided. * small fix in buildCollectionSearch.fulltext.test.js Change plainto_tsquery language from 'english' to 'simple' * Fix duplicate SELECT keyword in query Remove duplicate 'SELECT' keyword in SQL query. * Fix missing newline at end of collectionSearchParams.js * Fixed missing bracket in collectionSearchParams.js * Refactor validateSortby for optional parameter handling Refactor validateSortby function to handle optional sortby parameter and improve validation logic. * Stabilize API test pipeline by running Jest in-band with extended timeout Run Jest in CI with --runInBand and a higher default --testTimeout to stabilize database-backed integration tests. Multiple Jest workers were competing for the same PostgreSQL connection pool and some long-running /collections queries exceeded the default 5s timeout, causing failures in existing test suites (e.g. collectionSearch and DBconnection). * Fixed leaking tests that blocked CI/CD-Pipeline. - Added a global Teardown for jest and force-exited the tests to prevent leaking. - Made a change to db_APIconnection to only log the pool-(dis)connection if it isn't run in a test enviroment. * Did a minimum amount of Formatting to the discription * Used `npm audit fix --force` to fix all vulnerabilties in our used packages. * Fixed curious doublechecking for empty Strings for the sortby-Parameter. - Now we only check once for a empty sortby - And added a test which distinguish between `sortby=""` and `sortby="+"` * Update api/routes/collections.js Removed the TODO about switching from mock-data to the real db * Removed globalTeardown as i brought up some problems corresponding to long db-queries (for example BBOX). Instead i increased the maximal testTimeout. --------- Co-authored-by: Robin Tammo Gummels * Update api/.env.example * latest database Version (#187) with `stac_id` and changed definition of `primary Keys` * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md * added `stac_id` for collections * all IDs are now written in the newer PostgrSQL standart: ```SQL id SERIAL PRIMARY KEY, ``` changed to ```SQL id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, ``` * changed `extend` to `extent`. * Changed language used in `./api/README.md` from german to english. I wanted to thsi anyway at some point, but this is now more like a Test-commit to see if the CI/CD Pipeline triggers... --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * Added `GET /collections/{id}`-Endpoint, more Fields in the responses and more Queryables-Parameters (#204) * Updated Query-Builder to get all necessary fields from all db_tables for collections. Added some tests and fixed some already existing tests, becuase now the tablenames start with the alias `c.`. * Updated Query-Builder to get all necessary fields from all db_tables for collections. Added some tests and fixed some already existing tests, becuase now the tablenames start with the alias `c.`. * Added `openapi.yaml` (now http://localhost:3000/api-docs/ is working). - needed to do some modifying to the app.js * Added discription on how to use `stac-api-validator`. Currently we are onyl valid to `core`. * Changed API-Version name to 1.1.0 instead of 1.0.0 * Revert "API is now responding with all necessary fields for each collection" (#195) Reverts #185 @SonkeHoffmann accidentally didn't squash correctly. * Revert "Revert "API is now responding with all necessary fields for each collection"" (#185) (#195) (#196) dev-api: prepare v1.1.0 + API docs + query builder fixes - Change API version to 1.1.0 - Add OpenAPI spec so /api-docs works locally - Document stac-api-validator usage - Update api/.env.example - Query builder: select required fields for collections across db_tables; adjust tests (alias `c.`) Commits included: - 34bf962 Changed API-Version name to 1.1.0 instead of 1.0.0 - b047389 Added description on how to use `stac-api-validator` (currently only valid for `core`) - b811288 Added `openapi.yaml` (so http://localhost:3000/api-docs/ works); modified app.js accordingly - 6e5ab3e Merge branch 'dev-api-robin' of github.com:SpatioCore/STAC-Atlas into dev-api-robin - 70dc043 Updated Query-Builder to get all necessary fields from all db_tables for collections. Added tests and fixed existing tests (table names now start with alias `c.`) - d83eeb4 Update api/.env.example - 5a7af5b Updated Query-Builder to get all necessary fields from all db_tables for collections. Added tests and fixed existing tests (table names now start with alias `c.`) Co-authored-by: Robin Tammo Gummels * Implement more Queryable-Fields and add the keywords-field to q fulltext search (#200) * Add provider and license filters to collection search API - Updated buildCollectionSearchQuery to include provider and license parameters for filtering collections. - Enhanced validateCollectionSearchParams middleware to validate provider and license query parameters. - Modified collections route to handle new provider and license filters in search queries. - Implemented validation functions for provider and license parameters in collectionSearchParams. * Add validation tests for provider and license * Enhance full-text search by including keywords in the tsvector expression and update related tests * Add provider and license to query parameter extraction in collection search validation * Revert "Enhance full-text search by including keywords in the tsvector expression and update related tests" This reverts commit 872443d8e83c55834ef5f0d275c54fefb4b74e2d. * Implement GET /collections/{id} endpoint with validation and QueryBuilder integration (#186) * added SQLQuery-builder with these parameters: q,bbox,datetime,sortby,limit and token * finalised bbox and datetime * adapted to DB, QueryBuilder and added helperfunction runQuery * added question-TODOs * added bbox+datetime to the Query-Builder from Jonas * added tests for Query-Builder from Jonas * added tests from George * added falsely deleted TODOs again * fixed collumn names to match our DB and adjusted full text search to match 05_indexes.sql correctly * Used a formatter and linter on `buildCollectionSearchQuery.js * Did some major and minor fixes to the collection search. - Updated `buildCollectionSearchQuery` to support pagination and improved text search with English language settings. - Modified tests in `buildCollectionsSearchQuery.basic.test.js`, `collections-pagination.test.js`, and `collections-sort.test.js` to reflect new query behavior and validation logic. - Enhanced sort validation in `validators.test.js` and `collectionSearchParams.js` to map API fields to database column names. - Implemented total count retrieval for matched results in `collections.js`. * Added a internal .env creation in the CI/CD Pipeline. It utilzes GitHub Repository Secrets to not publish any private Logins and stuff. * Forgot that the second Job of the CI/CD pipeline runs seperatly and needs a internal .env file too. * Enhance documentation for buildCollectionSearchQuery Updated the documentation for: - the buildCollectionSearchQuery function - the fulltextsearch * Refactor buildCollectionSearchQuery and updated SELECT part Changed the SELECT part to match our bid and the database shema. Updated comments and for clarity. Changed full-text search to use 'simple' configuration instead of 'english'. * Update api/routes/collections.js small typo Co-authored-by: Robin Tammo Gummels * Remove sorting TODO from collections route Removed TODO comment about sorting based on sortby parameter. * Explicitly return undefined for normalized in validateSortby Update validateSortby function to explicitly return undefined for normalized when sortby is not provided. * small fix in buildCollectionSearch.fulltext.test.js Change plainto_tsquery language from 'english' to 'simple' * Fix duplicate SELECT keyword in query Remove duplicate 'SELECT' keyword in SQL query. * Fix missing newline at end of collectionSearchParams.js * Fixed missing bracket in collectionSearchParams.js * Refactor validateSortby for optional parameter handling Refactor validateSortby function to handle optional sortby parameter and improve validation logic. * Stabilize API test pipeline by running Jest in-band with extended timeout Run Jest in CI with --runInBand and a higher default --testTimeout to stabilize database-backed integration tests. Multiple Jest workers were competing for the same PostgreSQL connection pool and some long-running /collections queries exceeded the default 5s timeout, causing failures in existing test suites (e.g. collectionSearch and DBconnection). * Fixed leaking tests that blocked CI/CD-Pipeline. - Added a global Teardown for jest and force-exited the tests to prevent leaking. - Made a change to db_APIconnection to only log the pool-(dis)connection if it isn't run in a test enviroment. * Did a minimum amount of Formatting to the discription * Used `npm audit fix --force` to fix all vulnerabilties in our used packages. * Fixed curious doublechecking for empty Strings for the sortby-Parameter. - Now we only check once for a empty sortby - And added a test which distinguish between `sortby=""` and `sortby="+"` * Update api/routes/collections.js Removed the TODO about switching from mock-data to the real db * Removed globalTeardown as i brought up some problems corresponding to long db-queries (for example BBOX). Instead i increased the maximal testTimeout. * added validator for collections{id} and correctly implemented collections{id} * added test for collections{id} * removed unnecessary parameter * added id parameter to the Query (temporary fix) * test-fixes to match our current tests and a fix to the baseURL for collection{id} * test fix * fixed problem with tests in api.test.js and adjusted the "invalid-id-test" in the validator. * Update api/routes/collections.js - Renamed `collection.id` to `c.collection.id` * added test for negative ids * deleted the whole "existing links" part and build base Links * fixed bug in validateCollectionId.js * Refactor negative ID test i encoded the "-1" value in the negative ID test instead of directly putting it into the path. * Removed a german comment in `api/routes/collections.js` --------- Co-authored-by: Robin Tammo Gummels --------- Co-authored-by: Sönke Hoffmann Co-authored-by: JonasK <156602337+BrokeJ@users.noreply.github.com> Co-authored-by: Vincent Kühn * Dev database: trigger function for better keyword handling (#209) * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md * added `stac_id` for collections * all IDs are now written in the newer PostgrSQL standart: ```SQL id SERIAL PRIMARY KEY, ``` changed to ```SQL id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, ``` * changed `extend` to `extent`. * Changed language used in `./api/README.md` from german to english. I wanted to thsi anyway at some point, but this is now more like a Test-commit to see if the CI/CD Pipeline triggers... * added triggering function for an auto-update search_vector, both for collections and catalogs. The search_vector includes title, description and keywords * changed the CI-Pipeline. Now also Changes in the /db will be acceped by the Pipeline --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * Dev database: added different users (for api and crawler) (#214) * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md * added `stac_id` for collections * all IDs are now written in the newer PostgrSQL standart: ```SQL id SERIAL PRIMARY KEY, ``` changed to ```SQL id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, ``` * changed `extend` to `extent`. * Changed language used in `./api/README.md` from german to english. I wanted to thsi anyway at some point, but this is now more like a Test-commit to see if the CI/CD Pipeline triggers... * added triggering function for an auto-update search_vector, both for collections and catalogs. The search_vector includes title, description and keywords * changed the CI-Pipeline. Now also Changes in the /db will be acceped by the Pipeline * added different users for the api and crawler groups. The api has read-only acces and the crawler user full acces to the database. The acces to the database is now possible by using the users `stac_api`or `stac_crawler`. The admin user (`postgres_user`) is still available but shouldn't be used --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * Dev database: source_url and filter trigger (#220) * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md * added `stac_id` for collections * all IDs are now written in the newer PostgrSQL standart: ```SQL id SERIAL PRIMARY KEY, ``` changed to ```SQL id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, ``` * changed `extend` to `extent`. * Changed language used in `./api/README.md` from german to english. I wanted to thsi anyway at some point, but this is now more like a Test-commit to see if the CI/CD Pipeline triggers... * added triggering function for an auto-update search_vector, both for collections and catalogs. The search_vector includes title, description and keywords * changed the CI-Pipeline. Now also Changes in the /db will be acceped by the Pipeline * added different users for the api and crawler groups. The api has read-only acces and the crawler user full acces to the database. The acces to the database is now possible by using the users `stac_api`or `stac_crawler`. The admin user (`postgres_user`) is still available but shouldn't be used * Refactor(database): SQL trigger definitions for catalog and collection keywords. Moved triggers to 06_triggers.sql for better organization, as they depend on the respective tables created in earlier scripts. * added source_url for collections and catalogs. Now the full_json doesn`t has to be used for getting the url * resolved a Problem I had with git by hand cause I didn't found the function --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels Co-authored-by: mammutor --------- Co-authored-by: RobinGummels Co-authored-by: Jakob Co-authored-by: Sönke Hoffmann Co-authored-by: jklaer Co-authored-by: VincentKuehn Co-authored-by: Justin K Co-authored-by: Georgios Voulgaris Co-authored-by: Sönke Hoffmann Co-authored-by: JonasK <156602337+BrokeJ@users.noreply.github.com> --- .github/workflows/api-ci.yml | 6 +- api/.env.example | 10 +- api/README.md | 48 ++- api/__tests__/api.test.js | 15 +- ...ldCollectionSearchQuery.aggregates.test.js | 221 +++++++++++ ...uildCollectionSearchQuery.fulltext.test.js | 4 +- ...dCollectionSearchQuery.integration.test.js | 322 ++++++++++++++++ .../buildCollectionsSearchQuery.basic.test.js | 8 +- api/__tests__/collections-id.test.js | 87 +++++ api/__tests__/validators.test.js | 80 +++- api/app.js | 30 +- api/db/buildCollectionSearchQuery.js | 163 ++++++-- api/docs/openapi.yaml | 351 ++++++++++++++++++ api/middleware/validateCollectionId.js | 25 ++ api/middleware/validateCollectionSearch.js | 24 +- api/routes/collections.js | 109 +++--- api/routes/index.js | 2 +- api/validators/collectionSearchParams.js | 53 ++- db/docker-compose.yml | 4 + db/example.env | 14 +- db/init/00_users.sh | 34 ++ db/init/02_tables_catalog.sql | 63 +++- db/init/03_tables_collections.sql | 63 +++- db/init/05_indexes.sql | 8 +- db/init/06_triggers.sql | 16 + 25 files changed, 1644 insertions(+), 116 deletions(-) create mode 100644 api/__tests__/buildCollectionSearchQuery.aggregates.test.js create mode 100644 api/__tests__/buildCollectionSearchQuery.integration.test.js create mode 100644 api/__tests__/collections-id.test.js create mode 100644 api/docs/openapi.yaml create mode 100644 api/middleware/validateCollectionId.js create mode 100644 db/init/00_users.sh create mode 100644 db/init/06_triggers.sql diff --git a/.github/workflows/api-ci.yml b/.github/workflows/api-ci.yml index 68c87c0..1b49ad6 100644 --- a/.github/workflows/api-ci.yml +++ b/.github/workflows/api-ci.yml @@ -9,6 +9,7 @@ on: - main paths: - 'api/**' + - 'db/**' - '.github/workflows/api-ci.yml' pull_request: branches: @@ -17,6 +18,7 @@ on: - main paths: - 'api/**' + - 'db/**' - '.github/workflows/api-ci.yml' jobs: @@ -71,7 +73,7 @@ jobs: # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata - API_VERSION=1.0.0 + API_VERSION=1.1.0 EOF # Step 4: Install dependencies @@ -164,7 +166,7 @@ jobs: # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata - API_VERSION=1.0.0 + API_VERSION=1.1.0 EOF - name: Install dependencies diff --git a/api/.env.example b/api/.env.example index 039ac70..c6b2632 100644 --- a/api/.env.example +++ b/api/.env.example @@ -5,15 +5,15 @@ NODE_ENV=development # Database Configuration (Debian Server) # Option 1: Use DATABASE_URL (PostgreSQL connection string) -# add DB_USER and DB_PASSWORD values -DATABASE_URL= postgresql://[**DB_USER**]:[**DB_PASSWORD**]@atlas.stacindex.org:5432/stac_db +# The api-user is stac_api (read-only) (stac_crawler for crawler-group) +DATABASE_URL=postgresql://stac_api:[PASSWORD]@atlas.stacindex.org:5432/stac_db # Option 2: Use individual variables (currently active) DB_HOST=atlas.stacindex.org DB_PORT=5433 # 5432 for production DB_NAME=stac_db -DB_USER= -DB_PASSWORD= +DB_USER=stac_api +DB_PASSWORD= # Add stac_api password here (api_password) DB_SSL=false # Connection Pool Configuration @@ -28,4 +28,4 @@ CORS_ORIGIN=* # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata -API_VERSION=1.0.0 +API_VERSION=1.1.0 diff --git a/api/README.md b/api/README.md index 96d89e2..c337643 100644 --- a/api/README.md +++ b/api/README.md @@ -156,13 +156,59 @@ CORS_ORIGIN=* This API implements: -- ✅ STAC API Core (v1.0.0) +- ✅ STAC API Core (v1.1.0) - ✅ OGC API Features Core - ✅ STAC Collections - ✅ Collection Search Extension - 🚧 CQL2 Basic Filtering (in development) - 🚧 CQL2 Advanced Operators (in development) +### STAC API Validator + +The API can be tested using the official [STAC API Validator](https://github.com/stac-utils/stac-api-validator): + +#### Installation + +```bash +# Python 3.11 required +pip install stac-api-validator +``` + +#### Usage + +```bash +# Validate Core Conformance Class +python -m stac_api_validator --root-url http://localhost:3000 --conformance core + +# Validate Collections Extension (requires collection ID) +python -m stac_api_validator \ + --root-url http://localhost:3000 \ + --conformance core \ + --conformance collections \ + --collection + +# With spatial filtering (requires geometry in dataset) +python -m stac_api_validator \ + --root-url http://localhost:3000 \ + --conformance core \ + --conformance collections \ + --collection \ + --geometry '{"type": "Polygon", "coordinates": [[[7.0, 51.0], [8.0, 51.0], [8.0, 52.0], [7.0, 52.0], [7.0, 51.0]]]}' +``` + +#### Validation Status + +| Conformance Class | Status | Date | Errors | Warnings | +|-------------------|--------|------|--------|----------| +| **STAC API - Core** | ✅ Passed | 2025-12-10 | 0 | 0 | +| STAC API - Collections | ⏳ Pending | - | - | - | +| STAC API - Features | ⏳ Pending | - | - | - | +| STAC API - Item Search | ⏳ Pending | - | - | - | +| CQL2 - Basic | ⏳ Pending | - | - | - | +| CQL2 - Advanced | ⏳ Pending | - | - | - | + +**Note:** The Collection Search Extension is not currently validated automatically by the validator and is instead validated through custom Jest integration tests (see `__tests__/`). + ## 📦 Next Steps ### TODO diff --git a/api/__tests__/api.test.js b/api/__tests__/api.test.js index 6c77a1f..ad4b867 100644 --- a/api/__tests__/api.test.js +++ b/api/__tests__/api.test.js @@ -120,13 +120,16 @@ describe('STAC API Core Endpoints', () => { }); }); - describe('GET /collections/:id', () => { - it('should return 404 for non-existent collection', async () => { - const response = await request(app).get('/collections/non-existent-id').expect(404); + describe('GET /collections/:id', () => { + it('should return 404 for non-existent collection', async () => { + const nonExistingId = 999999999; - expect(response.body).toHaveProperty('code', 'NotFound'); - expect(response.body).toHaveProperty('description'); - expect(response.body).toHaveProperty('id', 'non-existent-id'); + const response = await request(app) + .get(`/collections/${nonExistingId}`) + .expect(404); + + expect(response.body).toHaveProperty('code', 'NotFound'); + expect(response.body).toHaveProperty('description'); }); }); }); diff --git a/api/__tests__/buildCollectionSearchQuery.aggregates.test.js b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js new file mode 100644 index 0000000..069ea46 --- /dev/null +++ b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js @@ -0,0 +1,221 @@ +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +describe('buildCollectionSearchQuery - aggregated fields', () => { + test('SELECT includes all collection base columns with alias c', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Core collection fields should be prefixed with 'c.' + expect(sql).toMatch(/c\.id/); + expect(sql).toMatch(/c\.stac_version/); + expect(sql).toMatch(/c\.type/); + expect(sql).toMatch(/c\.title/); + expect(sql).toMatch(/c\.description/); + expect(sql).toMatch(/c\.license/); + expect(sql).toMatch(/c\.spatial_extend/); + expect(sql).toMatch(/c\.temporal_extend_start/); + expect(sql).toMatch(/c\.temporal_extend_end/); + expect(sql).toMatch(/c\.created_at/); + expect(sql).toMatch(/c\.updated_at/); + expect(sql).toMatch(/c\.is_api/); + expect(sql).toMatch(/c\.is_active/); + expect(sql).toMatch(/c\.full_json/); + }); + + test('SELECT includes aggregated relation fields', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Aggregated fields from LATERAL JOINs + expect(sql).toMatch(/kw\.keywords/); + expect(sql).toMatch(/ext\.stac_extensions/); + expect(sql).toMatch(/prov\.providers/); + expect(sql).toMatch(/a\.assets/); + expect(sql).toMatch(/s\.summaries/); + expect(sql).toMatch(/cl\.last_crawled/); + }); + + test('FROM clause uses collection alias c', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/FROM collection c/); + }); + + describe('LATERAL JOINs for normalized data', () => { + test('includes LATERAL JOIN for keywords', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/LEFT JOIN LATERAL/); + expect(sql).toMatch(/jsonb_agg\(k\.keyword ORDER BY k\.keyword\) AS keywords/); + expect(sql).toMatch(/FROM collection_keywords ck/); + expect(sql).toMatch(/JOIN keywords k ON k\.id = ck\.keyword_id/); + expect(sql).toMatch(/WHERE ck\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for stac_extensions', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_agg\(se\.stac_extension ORDER BY se\.stac_extension\) AS stac_extensions/); + expect(sql).toMatch(/FROM collection_stac_extension cse/); + expect(sql).toMatch(/JOIN stac_extensions se ON se\.id = cse\.stac_extension_id/); + expect(sql).toMatch(/WHERE cse\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for providers with roles', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_agg\(jsonb_build_object\(/); + expect(sql).toMatch(/'name', p\.provider/); + expect(sql).toMatch(/'roles', cpr\.collection_provider_roles/); + expect(sql).toMatch(/FROM collection_providers cpr/); + expect(sql).toMatch(/JOIN providers p ON p\.id = cpr\.provider_id/); + expect(sql).toMatch(/WHERE cpr\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for assets with metadata', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/'name', a\.name/); + expect(sql).toMatch(/'href', a\.href/); + expect(sql).toMatch(/'type', a\.type/); + expect(sql).toMatch(/'roles', a\.roles/); + expect(sql).toMatch(/'metadata', a\.metadata/); + expect(sql).toMatch(/'collection_roles', ca\.collection_asset_roles/); + expect(sql).toMatch(/FROM collection_assets ca/); + expect(sql).toMatch(/JOIN assets a ON a\.id = ca\.asset_id/); + expect(sql).toMatch(/WHERE ca\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for summaries with CASE logic', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_object_agg\(s\.name, s\.s_summary\) AS summaries/); + expect(sql).toMatch(/WHEN cs\.kind = 'range' THEN jsonb_build_object\('min', cs\.range_min, 'max', cs\.range_max\)/); + expect(sql).toMatch(/WHEN cs\.kind = 'set' THEN to_jsonb\(cs\.set_value\)/); + expect(sql).toMatch(/FROM collection_summaries cs/); + expect(sql).toMatch(/WHERE cs\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for last_crawled timestamp', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/MAX\(clc\.last_crawled\) AS last_crawled/); + expect(sql).toMatch(/FROM crawllog_collection clc/); + expect(sql).toMatch(/WHERE clc\.collection_id = c\.id/); + }); + }); + + describe('WHERE clauses use collection alias c', () => { + test('bbox filter uses c.spatial_extend', () => { + const bbox = [-10, 40, 10, 50]; + const { sql } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + expect(sql).toMatch(/c\.spatial_extend/); + expect(sql).toMatch(/ST_Intersects\(\s*c\.spatial_extend/); + }); + + test('datetime filter uses c.temporal_extend_start and c.temporal_extend_end', () => { + const datetime = '2020-01-01/2021-12-31'; + const { sql } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + expect(sql).toMatch(/c\.temporal_extend_end >= \$/); + expect(sql).toMatch(/c\.temporal_extend_start <= \$/); + }); + + test('fulltext search uses c.title and c.description', () => { + const q = 'satellite'; + const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(sql).toMatch(/coalesce\(c\.title,''\)/); + expect(sql).toMatch(/coalesce\(c\.description,''\)/); + expect(sql).toMatch(/to_tsvector\('simple', coalesce\(c\.title,''\) \|\| ' ' \|\| coalesce\(c\.description,''\)\)/); + }); + }); + + describe('ORDER BY uses collection alias c', () => { + test('default ORDER BY uses c.id', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY c\.id ASC/); + }); + + test('sortby parameter uses c. prefix', () => { + const sortby = { field: 'title', direction: 'DESC' }; + const { sql } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY c\.title DESC/); + }); + + test('fulltext search with rank orders by rank DESC, c.id ASC', () => { + const q = 'satellite'; + const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); + }); + }); + + describe('Parameterized values remain correct', () => { + test('bbox parameters are in correct order', () => { + const bbox = [-10, 40, 10, 50]; + const { values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + expect(values.slice(0, 4)).toEqual(bbox); + expect(values[4]).toBe(10); // limit + expect(values[5]).toBe(0); // token + }); + + test('datetime interval parameters are in correct order', () => { + const datetime = '2020-01-01/2021-12-31'; + const { values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + expect(values[0]).toBe('2020-01-01'); + expect(values[1]).toBe('2021-12-31'); + expect(values[2]).toBe(10); // limit + expect(values[3]).toBe(0); // token + }); + + test('fulltext query parameter is bound correctly', () => { + const q = 'satellite'; + const { values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(values[0]).toBe('satellite'); + expect(values[1]).toBe(10); // limit + expect(values[2]).toBe(0); // token + }); + + test('combined filters maintain parameter order', () => { + const bbox = [-10, 40, 10, 50]; + const datetime = '2020-01-01/2021-12-31'; + const q = 'satellite'; + const { values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 10, token: 0 }); + + // Order: q (1), bbox (4), datetime start (1), datetime end (1), limit (1), token (1) = 9 values + expect(values[0]).toBe('satellite'); + expect(values.slice(1, 5)).toEqual(bbox); + expect(values[5]).toBe('2020-01-01'); + expect(values[6]).toBe('2021-12-31'); + expect(values[7]).toBe(10); + expect(values[8]).toBe(0); + }); + }); + + describe('SQL structure validation', () => { + test('no DISTINCT in jsonb_agg to avoid ORDER BY conflict', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // DISTINCT should NOT appear in any jsonb_agg calls + // (PostgreSQL requires ORDER BY expressions to appear in DISTINCT argument list) + const distinctPattern = /jsonb_agg\(DISTINCT/gi; + const matches = sql.match(distinctPattern); + + expect(matches).toBeNull(); + }); + + test('all LATERAL JOINs are LEFT JOIN', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Count LEFT JOIN LATERAL occurrences (should be 6: kw, ext, prov, a, s, cl) + const leftJoinLateralCount = (sql.match(/LEFT JOIN LATERAL/gi) || []).length; + + expect(leftJoinLateralCount).toBe(6); + }); + }); +}); diff --git a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js index 5c8100d..99a9f07 100644 --- a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js +++ b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js @@ -13,7 +13,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { expect(sql).toMatch(/AS rank/); // Ordering defaults to rank DESC when q present and no sortby - expect(sql).toMatch(/ORDER BY rank DESC, id ASC/); + expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); // values: [q, limit, token] expect(values[0]).toBe('forest'); @@ -24,7 +24,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { test('explicit sortby overrides rank ordering', () => { const { sql } = buildCollectionSearchQuery({ q: 'lake', sortby: { field: 'title', direction: 'ASC' }, limit: 5, token: 0 }); - expect(sql).toMatch(/ORDER BY title ASC/); + expect(sql).toMatch(/ORDER BY c\.title ASC/); // rank still present in select expect(sql).toMatch(/AS rank/); }); diff --git a/api/__tests__/buildCollectionSearchQuery.integration.test.js b/api/__tests__/buildCollectionSearchQuery.integration.test.js new file mode 100644 index 0000000..2885fa0 --- /dev/null +++ b/api/__tests__/buildCollectionSearchQuery.integration.test.js @@ -0,0 +1,322 @@ +const { query, closePool } = require('../db/db_APIconnection'); +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +/** + * Integration Tests: Aggregated Fields in Collection Search Query + * + * These tests verify that the LATERAL JOINs correctly aggregate data from + * normalized tables (keywords, providers, assets, stac_extensions, summaries, crawllog). + * + * Prerequisites: + * - Database must be initialized with schema (01-05_*.sql) + * - Test data should include collections with related entities + */ + +describe('Integration: Collection Search with Aggregated Fields', () => { + afterAll(async () => { + await closePool(); + }); + + describe('Query Execution', () => { + test('should execute query successfully without errors', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); + + await expect(query(sql, values)).resolves.not.toThrow(); + }); + + test('should return rows with aggregated fields', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + expect(Array.isArray(result.rows)).toBe(true); + + // If there are collections in DB, verify structure + if (result.rows.length > 0) { + const firstRow = result.rows[0]; + + // Core collection fields + expect(firstRow).toHaveProperty('id'); + expect(firstRow).toHaveProperty('title'); + expect(firstRow).toHaveProperty('description'); + expect(firstRow).toHaveProperty('license'); + expect(firstRow).toHaveProperty('full_json'); + + // Aggregated fields (may be null if no related data) + expect(firstRow).toHaveProperty('keywords'); + expect(firstRow).toHaveProperty('stac_extensions'); + expect(firstRow).toHaveProperty('providers'); + expect(firstRow).toHaveProperty('assets'); + expect(firstRow).toHaveProperty('summaries'); + expect(firstRow).toHaveProperty('last_crawled'); + } + }); + }); + + describe('Aggregated Field Types', () => { + test('keywords should be JSONB array or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.keywords !== null) { + expect(Array.isArray(row.keywords)).toBe(true); + // Each keyword should be a string + row.keywords.forEach(kw => { + expect(typeof kw).toBe('string'); + }); + } + }); + }); + + test('stac_extensions should be JSONB array or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.stac_extensions !== null) { + expect(Array.isArray(row.stac_extensions)).toBe(true); + row.stac_extensions.forEach(ext => { + expect(typeof ext).toBe('string'); + }); + } + }); + }); + + test('providers should be JSONB array of objects or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.providers !== null) { + expect(Array.isArray(row.providers)).toBe(true); + row.providers.forEach(provider => { + expect(provider).toHaveProperty('name'); + expect(provider).toHaveProperty('roles'); + expect(typeof provider.name).toBe('string'); + }); + } + }); + }); + + test('assets should be JSONB array of objects or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.assets !== null) { + expect(Array.isArray(row.assets)).toBe(true); + row.assets.forEach(asset => { + expect(asset).toHaveProperty('name'); + expect(asset).toHaveProperty('href'); + expect(asset).toHaveProperty('type'); + expect(asset).toHaveProperty('roles'); + expect(asset).toHaveProperty('metadata'); + expect(asset).toHaveProperty('collection_roles'); + }); + } + }); + }); + + test('summaries should be JSONB object or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.summaries !== null) { + expect(typeof row.summaries).toBe('object'); + expect(Array.isArray(row.summaries)).toBe(false); + + // Each summary should be a range, set, or schema object + Object.values(row.summaries).forEach(summary => { + const hasRange = summary.min !== undefined && summary.max !== undefined; + const isSet = Array.isArray(summary) || typeof summary === 'string'; + const isSchema = typeof summary === 'object'; + + expect(hasRange || isSet || isSchema).toBe(true); + }); + } + }); + }); + + test('last_crawled should be timestamp or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.last_crawled !== null) { + // Should be a valid Date or parseable timestamp + const date = new Date(row.last_crawled); + expect(date.toString()).not.toBe('Invalid Date'); + } + }); + }); + }); + + describe('Filter Compatibility with Aggregated Fields', () => { + test('bbox filter works with aggregated fields', async () => { + const bbox = [-180, -90, 180, 90]; // World bbox + const { sql, values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + // All returned rows should have the aggregated structure + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + }); + + test('datetime filter works with aggregated fields', async () => { + const datetime = '2000-01-01/2030-12-31'; + const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('stac_extensions'); + expect(row).toHaveProperty('summaries'); + expect(row).toHaveProperty('last_crawled'); + }); + }); + + test('fulltext search works with aggregated fields', async () => { + const q = 'test'; + const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + + test('combined filters work with aggregated fields', async () => { + const bbox = [-180, -90, 180, 90]; + const datetime = '2000-01-01/2030-12-31'; + const q = 'satellite'; + const { sql, values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 5, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + // All aggregated fields should be present + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('stac_extensions'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + expect(row).toHaveProperty('summaries'); + expect(row).toHaveProperty('last_crawled'); + }); + }); + }); + + describe('Sorting with Aggregated Fields', () => { + test('default sort by c.id works with aggregated fields', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + if (result.rows.length > 1) { + // IDs should be in ascending order + for (let i = 1; i < result.rows.length; i++) { + expect(result.rows[i].id).toBeGreaterThanOrEqual(result.rows[i - 1].id); + } + } + }); + + test('sort by title works with aggregated fields', async () => { + const sortby = { field: 'title', direction: 'ASC' }; + const { sql, values } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); + const result = await query(sql, values); + + // Verify SQL contains ORDER BY c.title ASC + expect(sql).toMatch(/ORDER BY c\.title ASC/); + + // Verify all aggregated fields are present + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + }); + + test('fulltext rank sort works with aggregated fields', async () => { + const q = 'satellite'; + const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + const result = await query(sql, values); + + // Should execute without error; rank ordering is implicit in SQL + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + }); + + describe('Pagination with Aggregated Fields', () => { + test('first page returns correct structure', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 3, token: 0 }); + const result = await query(sql, values); + + expect(result.rows.length).toBeLessThanOrEqual(3); + result.rows.forEach(row => { + expect(row).toHaveProperty('id'); + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + + test('second page returns different rows with same structure', async () => { + const page1 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 0 }))); + const page2 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 3 }))); + + if (page1.rows.length > 0 && page2.rows.length > 0) { + // IDs should be different + const page1Ids = page1.rows.map(r => r.id); + const page2Ids = page2.rows.map(r => r.id); + + const overlap = page1Ids.filter(id => page2Ids.includes(id)); + expect(overlap.length).toBe(0); + + // Both pages should have same structure + page2.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + } + }); + }); + + describe('Performance and Cardinality', () => { + test('LATERAL JOINs do not duplicate collection rows', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 100, token: 0 }); + const result = await query(sql, values); + + // Collect all IDs + const ids = result.rows.map(r => r.id); + const uniqueIds = [...new Set(ids)]; + + // No duplicates: each collection should appear exactly once + expect(ids.length).toBe(uniqueIds.length); + }); + + test('query executes in reasonable time (<5s for small dataset)', async () => { + const start = Date.now(); + const { sql, values } = buildCollectionSearchQuery({ limit: 50, token: 0 }); + await query(sql, values); + const duration = Date.now() - start; + + // Should complete within 5 seconds for typical test datasets + expect(duration).toBeLessThan(5000); + }, 10000); // 10s timeout for Jest + }); +}); diff --git a/api/__tests__/buildCollectionsSearchQuery.basic.test.js b/api/__tests__/buildCollectionsSearchQuery.basic.test.js index 8756a5f..4acd7b9 100644 --- a/api/__tests__/buildCollectionsSearchQuery.basic.test.js +++ b/api/__tests__/buildCollectionsSearchQuery.basic.test.js @@ -4,8 +4,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { test('no params returns base SQL with LIMIT/OFFSET placeholders', () => { const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - expect(sql).toMatch(/FROM collection/); - expect(sql).toMatch(/ORDER BY id ASC/); + expect(sql).toMatch(/FROM collection c/); + expect(sql).toMatch(/ORDER BY c\.id ASC/); // there should be LIMIT and OFFSET placeholders expect(sql).toMatch(/LIMIT \$1 OFFSET \$2/); expect(Array.isArray(values)).toBe(true); @@ -30,8 +30,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { const datetime = '2020-01-01/2021-12-31'; const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); - expect(sql).toMatch(/temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated - expect(sql).toMatch(/temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated + expect(sql).toMatch(/c\.temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated + expect(sql).toMatch(/c\.temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated // values order: start, end, limit, token expect(values[0]).toBe('2020-01-01'); expect(values[1]).toBe('2021-12-31'); diff --git a/api/__tests__/collections-id.test.js b/api/__tests__/collections-id.test.js new file mode 100644 index 0000000..63f89c8 --- /dev/null +++ b/api/__tests__/collections-id.test.js @@ -0,0 +1,87 @@ +const request = require('supertest'); +const app = require('../app'); + +describe('GET /collections/:id - Single collection retrieval', () => { + + /** + * Helper: fetch a valid collection id via the public /collections endpoint. + * This avoids hard-coding any specific id from the database. + */ + async function getAnyExistingCollectionId() { + const res = await request(app) + .get('/collections?limit=1&token=0') + .expect(200); + + expect(Array.isArray(res.body.collections)).toBe(true); + expect(res.body.collections.length).toBeGreaterThan(0); + + return res.body.collections[0].id; + } + + test('should return a single collection with matching id and STAC-style links', async () => { + const existingId = await getAnyExistingCollectionId(); + + const res = await request(app) + .get(`/collections/${existingId}`) + .expect(200); + + const collection = res.body; + + // id should match + expect(collection).toBeDefined(); + expect(collection.id).toBe(existingId); + + // basic structure + expect(collection).toHaveProperty('title'); + expect(collection).toHaveProperty('license'); + + + // links should be an array with self, root and parent + expect(Array.isArray(collection.links)).toBe(true); + + const rels = collection.links.map(l => l.rel); + + expect(rels).toContain('self'); + expect(rels).toContain('root'); + expect(rels).toContain('parent'); + + // self link should point to this resource + const selfLink = collection.links.find(l => l.rel === 'self'); + expect(selfLink).toBeDefined(); + expect(selfLink.href).toContain(`/collections/${existingId}`); + }); + + test('should return 400 for an invalid (non-numeric) id', async () => { + const res = await request(app) + .get('/collections/not-a-number') + .expect(400); + + expect(res.body).toHaveProperty('code', 'InvalidParameter'); + expect(res.body.description).toMatch(/id/i); +}); + + test('should return 400 for a negative id', async () => { + const negativeId = '-1'; + + const res = await request(app) + .get(`/collections/${encodeURIComponent(negativeId)}`) + .expect(400); + + expect(res.body).toHaveProperty('code', 'InvalidParameter'); + expect(res.body.description).toMatch(/id/i); +}) + + test('should return 404 for a non-existing numeric id', async () => { + // use a very large id that is unlikely to exist + const nonExistingId = 999999999; + + const res = await request(app) + .get(`/collections/${nonExistingId}`) + .expect(404); + + expect(res.body).toHaveProperty('code', 'NotFound'); + expect(res.body).toHaveProperty('description'); + expect(res.body.description).toMatch(/not found/i); + expect(res.body).toHaveProperty('id', String(nonExistingId)); + }); +}); diff --git a/api/__tests__/validators.test.js b/api/__tests__/validators.test.js index c0e50cd..f6a9cb5 100644 --- a/api/__tests__/validators.test.js +++ b/api/__tests__/validators.test.js @@ -6,7 +6,9 @@ const { validateDatetime, validateLimit, validateSortby, - validateToken + validateToken, + validateProvider, + validateLicense } = require('../validators/collectionSearchParams'); describe('Collection Search Parameter Validators', () => { @@ -404,4 +406,80 @@ describe('Collection Search Parameter Validators', () => { expect(result.normalized).toBe(0); }); }); + + describe('validateProvider - Provider name', () => { + it('should accept valid provider string', () => { + const result = validateProvider('Copernicus'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('Copernicus'); + }); + + it('should trim whitespace from provider', () => { + const result = validateProvider(' Test Provider '); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('Test Provider'); + }); + + it('should accept undefined provider', () => { + const result = validateProvider(undefined); + expect(result.valid).toBe(true); + }); + + it('should reject non-string provider', () => { + const result = validateProvider(123); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be a string'); + }); + + it('should reject empty provider', () => { + const result = validateProvider(' '); + expect(result.valid).toBe(false); + expect(result.error).toContain('must not be empty'); + }); + + it('should reject provider exceeding max length', () => { + const long = 'a'.repeat(256); + const result = validateProvider(long); + expect(result.valid).toBe(false); + expect(result.error).toContain('exceeds maximum length'); + }); + }); + + describe('validateLicense - License identifier', () => { + it('should accept valid license', () => { + const result = validateLicense('CC-BY-4.0'); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('CC-BY-4.0'); + }); + + it('should trim whitespace from license', () => { + const result = validateLicense(' CC0 '); + expect(result.valid).toBe(true); + expect(result.normalized).toBe('CC0'); + }); + + it('should accept undefined license', () => { + const result = validateLicense(undefined); + expect(result.valid).toBe(true); + }); + + it('should reject non-string license', () => { + const result = validateLicense(123); + expect(result.valid).toBe(false); + expect(result.error).toContain('must be a string'); + }); + + it('should reject empty license', () => { + const result = validateLicense(' '); + expect(result.valid).toBe(false); + expect(result.error).toContain('must not be empty'); + }); + + it('should reject license exceeding max length', () => { + const long = 'a'.repeat(256); + const result = validateLicense(long); + expect(result.valid).toBe(false); + expect(result.error).toContain('exceeds maximum length'); + }); + }); }); diff --git a/api/app.js b/api/app.js index bf5f4f5..a5b0393 100644 --- a/api/app.js +++ b/api/app.js @@ -26,7 +26,27 @@ app.use(cors({ allowedHeaders: ['Content-Type', 'Authorization'] })); -// Content-Type header for all JSON responses +// OpenAPI spec endpoint (YAML file with correct content-type) - MUST be before Content-Type middleware +app.get('/openapi.yaml', (req, res, next) => { + try { + const openapiPath = path.join(__dirname, 'docs', 'openapi.yaml'); + res.setHeader('Content-Type', 'application/vnd.oai.openapi+json;version=3.0'); + res.sendFile(openapiPath); + } catch (err) { + next(err); + } +}); + +// Swagger/OpenAPI documentation (if openapi.yaml exists) - MUST be before Content-Type middleware +try { + const swaggerDocument = YAML.load(path.join(__dirname, 'docs', 'openapi.yaml')); + app.use('/api-docs', swaggerUi.serve); + app.get('/api-docs', swaggerUi.setup(swaggerDocument)); +} catch (err) { + console.log('OpenAPI documentation not found. Create docs/openapi.yaml to enable Swagger UI.'); +} + +// Content-Type header for JSON responses (set AFTER special endpoints) app.use((req, res, next) => { res.setHeader('Content-Type', 'application/json'); next(); @@ -38,14 +58,6 @@ app.use('/conformance', conformanceRouter); app.use('/collections', collectionsRouter); app.use('/queryables', queryablesRouter); -// Swagger/OpenAPI documentation (if openapi.yaml exists) -try { - const swaggerDocument = YAML.load(path.join(__dirname, 'docs', 'openapi.yaml')); - app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument)); -} catch (err) { - console.log('OpenAPI documentation not found. Create docs/openapi.yaml to enable Swagger UI.'); -} - // 404 handler app.use((req, res, next) => { res.status(404).json({ diff --git a/api/db/buildCollectionSearchQuery.js b/api/db/buildCollectionSearchQuery.js index cc1d435..064a7d1 100644 --- a/api/db/buildCollectionSearchQuery.js +++ b/api/db/buildCollectionSearchQuery.js @@ -61,6 +61,12 @@ * @param {number} params.token * Offset for pagination (0-based). Translated to OFFSET $n. * + * @param {string|undefined} params.provider + * Provider name to filter collections by their provider (case-insensitive match). + * + * @param {string|undefined} params.license + * License identifier to filter collections by `collection.license`. + * * @returns {{ sql: string, values: any[] }} * sql – complete parameterized SQL string * values – array of bind parameters in the correct order @@ -68,9 +74,12 @@ function buildCollectionSearchQuery(params) { const { + id, q, bbox, datetime, + provider, + license, sortby, limit, token @@ -83,28 +92,43 @@ function buildCollectionSearchQuery(params) { // full-text search) *before* the `FROM` clause. Keeping `sql` fixed with a // `FROM` already included would make inserting additional selected columns // harder and error-prone when building the query dynamically. + // + // We use alias 'c' for the collection table to simplify JOIN expressions and + // distinguish collection columns from aggregated relation data (keywords, providers, etc.). let selectPart = ` SELECT - id, - stac_version, - type, - title, - description, - license, - spatial_extend, - temporal_extend_start, - temporal_extend_end, - created_at, - updated_at, - is_api, - is_active, - full_json + c.id, + c.stac_version, + c.type, + c.title, + c.description, + c.license, + c.spatial_extend, + c.temporal_extend_start, + c.temporal_extend_end, + c.created_at, + c.updated_at, + c.is_api, + c.is_active, + c.full_json, + kw.keywords, + ext.stac_extensions, + prov.providers, + a.assets, + s.summaries, + cl.last_crawled `; const where = []; const values = []; let i = 1; + if (id !== undefined && id !== null) { + where.push(`id = $${i}`); + values.push(id); + i++; + } + // Full-text search using weighted tsvector across title (weight A) and description (weight B). // // Notes: @@ -123,8 +147,8 @@ function buildCollectionSearchQuery(params) { if (q) { const queryIndex = i; // remember index to reuse for rank and condition - // Weighted combined tsvector expression - const vectorExpr = `to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))`; + // Weighted combined tsvector expression (using alias 'c' for collection table) + const vectorExpr = `to_tsvector('simple', coalesce(c.title,'') || ' ' || coalesce(c.description,''))`; // Add rank to selected columns (ts_rank_cd => constant-duration ranking function) // The computed `rank` is available in the result rows and used for ordering @@ -144,7 +168,7 @@ function buildCollectionSearchQuery(params) { where.push(` ST_Intersects( - spatial_extend, + c.spatial_extend, ST_MakeEnvelope($${i}, $${i + 1}, $${i + 2}, $${i + 3}, 4326) ) `); @@ -161,33 +185,111 @@ function buildCollectionSearchQuery(params) { if (start !== '..') { // Collection should run after start - where.push(`temporal_extend_end >= $${i}`); + where.push(`c.temporal_extend_end >= $${i}`); values.push(start); i++; } if (end !== '..') { // Collection should run before end - where.push(`temporal_extend_start <= $${i}`); + where.push(`c.temporal_extend_start <= $${i}`); values.push(end); i++; } } else { // single datetime: collections active at that time where.push(` - temporal_extend_start <= $${i} - AND temporal_extend_end >= $${i} + c.temporal_extend_start <= $${i} + AND c.temporal_extend_end >= $${i} `); values.push(datetime); i++; } } - // Build final SQL from selectPart and add FROM clause. + // Provider filter: match collections that have a provider with the given name (case-insensitive) + if (provider) { + where.push(`EXISTS ( + SELECT 1 FROM collection_providers cp + JOIN providers p ON cp.provider_id = p.id + WHERE cp.collection_id = c.id + AND lower(p.provider) = lower($${i}) + )`); + values.push(provider); + i++; + } + + // License filter: direct match on collection.license + if (license) { + where.push(`c.license = $${i}`); + values.push(license); + i++; + } + + // Build final SQL from selectPart and add FROM clause with LATERAL JOINs. // We delayed adding `FROM collection` to allow conditional additions to the // selected columns above (notably `rank`). The final `sql` string includes the // selected columns, the source table and any WHERE conditions constructed earlier. - let sql = selectPart + `\n FROM collection\n `; + // + // LATERAL JOINs aggregate related data (keywords, extensions, providers, assets, summaries, + // and crawl timestamps) from normalized tables without duplicating collection rows. + // Each LEFT JOIN LATERAL subquery returns a single aggregated row per collection. + let sql = selectPart + ` + FROM collection c + LEFT JOIN LATERAL ( + SELECT jsonb_agg(k.keyword ORDER BY k.keyword) AS keywords + FROM collection_keywords ck + JOIN keywords k ON k.id = ck.keyword_id + WHERE ck.collection_id = c.id + ) kw ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(se.stac_extension ORDER BY se.stac_extension) AS stac_extensions + FROM collection_stac_extension cse + JOIN stac_extensions se ON se.id = cse.stac_extension_id + WHERE cse.collection_id = c.id + ) ext ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object( + 'name', p.provider, + 'roles', cpr.collection_provider_roles + ) ORDER BY p.provider) AS providers + FROM collection_providers cpr + JOIN providers p ON p.id = cpr.provider_id + WHERE cpr.collection_id = c.id + ) prov ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object( + 'name', a.name, + 'href', a.href, + 'type', a.type, + 'roles', a.roles, + 'metadata', a.metadata, + 'collection_roles', ca.collection_asset_roles + ) ORDER BY a.name) AS assets + FROM collection_assets ca + JOIN assets a ON a.id = ca.asset_id + WHERE ca.collection_id = c.id + ) a ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_object_agg(s.name, s.s_summary) AS summaries + FROM ( + SELECT + cs.name, + CASE + WHEN cs.kind = 'range' THEN jsonb_build_object('min', cs.range_min, 'max', cs.range_max) + WHEN cs.kind = 'set' THEN to_jsonb(cs.set_value) + ELSE cs.json_schema + END AS s_summary + FROM collection_summaries cs + WHERE cs.collection_id = c.id + ) s + ) s ON TRUE + LEFT JOIN LATERAL ( + SELECT MAX(clc.last_crawled) AS last_crawled + FROM crawllog_collection clc + WHERE clc.collection_id = c.id + ) cl ON TRUE + `; if (where.length > 0) { sql += ` WHERE ` + where.join(' AND '); @@ -197,15 +299,18 @@ function buildCollectionSearchQuery(params) { // a text query was provided (descending), falling back to id ascending. // // Behaviour summary: - // - `sortby` provided → use that (same as before) - // - no `sortby` & `q` present → order by `rank DESC, id ASC` so higher relevance comes first - // - no `sortby` & no `q` → order by `id ASC` (legacy default) + // - `sortby` provided → use that (with 'c.' prefix for collection columns) + // - no `sortby` & `q` present → order by `rank DESC, c.id ASC` so higher relevance comes first + // - no `sortby` & no `q` → order by `c.id ASC` (legacy default) + // + // Note: sortby.field is validated against a whitelist in the calling code; only collection + // table columns are allowed for sorting (not aggregated fields like keywords/providers). if (sortby) { - sql += ` ORDER BY ${sortby.field} ${sortby.direction}`; + sql += ` ORDER BY c.${sortby.field} ${sortby.direction}`; } else if (q) { - sql += ` ORDER BY rank DESC, id ASC`; + sql += ` ORDER BY rank DESC, c.id ASC`; } else { - sql += ` ORDER BY id ASC`; + sql += ` ORDER BY c.id ASC`; } // Pagination (only add if limit is provided) diff --git a/api/docs/openapi.yaml b/api/docs/openapi.yaml new file mode 100644 index 0000000..abcce52 --- /dev/null +++ b/api/docs/openapi.yaml @@ -0,0 +1,351 @@ +openapi: 3.0.3 +info: + title: STAC Atlas API + description: A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs. + version: 1.1.0 + contact: + name: SpatioCore + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + +servers: + - url: http://localhost:3000 + description: Local development server + +paths: + /: + get: + summary: Landing Page + description: Returns the STAC API landing page with links to available resources + operationId: getLandingPage + tags: + - STAC Core + responses: + '200': + description: STAC API landing page + content: + application/json: + schema: + $ref: '#/components/schemas/LandingPage' + + /conformance: + get: + summary: Conformance Classes + description: Returns the conformance classes that this API implements + operationId: getConformance + tags: + - STAC Core + responses: + '200': + description: Conformance classes + content: + application/json: + schema: + $ref: '#/components/schemas/Conformance' + + /collections: + get: + summary: List Collections + description: Returns a list of STAC Collections with optional filtering + operationId: getCollections + tags: + - Collections + parameters: + - name: limit + in: query + description: Maximum number of collections to return + required: false + schema: + type: integer + minimum: 1 + maximum: 10000 + default: 10 + - name: offset + in: query + description: Number of collections to skip + required: false + schema: + type: integer + minimum: 0 + default: 0 + - name: bbox + in: query + description: Bounding box to filter collections [minLon,minLat,maxLon,maxLat] + required: false + schema: + type: array + items: + type: number + minItems: 4 + maxItems: 6 + - name: datetime + in: query + description: Temporal filter (single datetime or interval) + required: false + schema: + type: string + - name: q + in: query + description: Full-text search query + required: false + schema: + type: string + - name: filter + in: query + description: CQL2 filter expression + required: false + schema: + type: string + - name: filter-lang + in: query + description: Filter language (cql2-text or cql2-json) + required: false + schema: + type: string + enum: + - cql2-text + - cql2-json + default: cql2-text + - name: sortby + in: query + description: Sort order for results + required: false + schema: + type: string + responses: + '200': + description: List of collections + content: + application/json: + schema: + $ref: '#/components/schemas/Collections' + '400': + description: Bad request (invalid parameters) + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /collections/{collectionId}: + get: + summary: Get Collection + description: Returns a single STAC Collection by ID + operationId: getCollection + tags: + - Collections + parameters: + - name: collectionId + in: path + description: Collection identifier + required: true + schema: + type: string + responses: + '200': + description: A STAC Collection + content: + application/json: + schema: + $ref: '#/components/schemas/Collection' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /queryables: + get: + summary: Global Queryables + description: Returns queryable properties for collection search + operationId: getQueryables + tags: + - Queryables + responses: + '200': + description: Queryables schema + content: + application/schema+json: + schema: + type: object + +components: + schemas: + LandingPage: + type: object + required: + - type + - id + - description + - links + - conformsTo + properties: + type: + type: string + enum: + - Catalog + id: + type: string + title: + type: string + description: + type: string + stac_version: + type: string + conformsTo: + type: array + items: + type: string + links: + type: array + items: + $ref: '#/components/schemas/Link' + + Conformance: + type: object + required: + - conformsTo + properties: + conformsTo: + type: array + items: + type: string + + Collections: + type: object + required: + - collections + - links + properties: + collections: + type: array + items: + $ref: '#/components/schemas/Collection' + links: + type: array + items: + $ref: '#/components/schemas/Link' + context: + $ref: '#/components/schemas/Context' + + Collection: + type: object + required: + - type + - id + - description + - license + - extent + - links + properties: + type: + type: string + enum: + - Collection + stac_version: + type: string + stac_extensions: + type: array + items: + type: string + id: + type: string + title: + type: string + description: + type: string + keywords: + type: array + items: + type: string + license: + type: string + providers: + type: array + items: + type: object + extent: + type: object + required: + - spatial + - temporal + properties: + spatial: + type: object + required: + - bbox + properties: + bbox: + type: array + items: + type: array + items: + type: number + temporal: + type: object + required: + - interval + properties: + interval: + type: array + items: + type: array + items: + type: string + nullable: true + links: + type: array + items: + $ref: '#/components/schemas/Link' + summaries: + type: object + assets: + type: object + + Link: + type: object + required: + - rel + - href + properties: + rel: + type: string + href: + type: string + type: + type: string + title: + type: string + + Context: + type: object + properties: + returned: + type: integer + minimum: 0 + limit: + type: integer + minimum: 1 + matched: + type: integer + minimum: 0 + + Error: + type: object + required: + - code + - description + properties: + code: + type: string + description: + type: string + +tags: + - name: STAC Core + description: STAC API Core endpoints + - name: Collections + description: Collection search and retrieval + - name: Queryables + description: Queryable properties diff --git a/api/middleware/validateCollectionId.js b/api/middleware/validateCollectionId.js new file mode 100644 index 0000000..2cb1bb7 --- /dev/null +++ b/api/middleware/validateCollectionId.js @@ -0,0 +1,25 @@ +/** + * Middleware to validate the :id route parameter for /collections/:id. + * + * - Ensures the id looks like a positive integer (all digits). + * - Prevents obviously malformed input reaching the database layer. + * - On error, responds with a 404 JSON body that matches the "NotFound" error + * format used elsewhere in the API tests. + */ +function validateCollectionId(req, res, next) { + const { id } = req.params; + + // id must be present and must be a sequence of digits (no minus, no spaces, no letters) + if (!id || !/^\d+$/u.test(id)) { + return res.status(400).json({ + code: 'InvalidParameter', + description: 'The "id" parameter must be a non-negative integer (digits only).', + parameter: 'id', + value: id + }); + } + + next(); +} + +module.exports = { validateCollectionId }; \ No newline at end of file diff --git a/api/middleware/validateCollectionSearch.js b/api/middleware/validateCollectionSearch.js index f19aa5a..0aa2c74 100644 --- a/api/middleware/validateCollectionSearch.js +++ b/api/middleware/validateCollectionSearch.js @@ -6,7 +6,9 @@ const { validateDatetime, validateLimit, validateSortby, - validateToken + validateToken, + validateProvider, + validateLicense } = require('../validators/collectionSearchParams'); /** @@ -23,6 +25,8 @@ const { * - limit: Result limit (default 10, max 10000) * - sortby: Sort specification (+/-field) * - token: Pagination continuation token + * - provider: Provider name — filter by data provider + * - license: License identifier — filter by collection license * * @param {Request} req - Express request object * @param {Response} res - Express response object @@ -33,7 +37,7 @@ function validateCollectionSearchParams(req, res, next) { const normalized = {}; // Extract query parameters - const { q, bbox, datetime, limit, sortby, token } = req.query; + const { q, bbox, datetime, limit, sortby, token, provider, license } = req.query; // Validate q (free-text search) const qResult = validateQ(q); @@ -82,6 +86,22 @@ function validateCollectionSearchParams(req, res, next) { } else { normalized.token = tokenResult.normalized; } + + // Validate provider (filter by data provider) + const providerResult = validateProvider(provider); + if (!providerResult.valid) { + errors.push(providerResult.error); + } else if (providerResult.normalized !== undefined) { + normalized.provider = providerResult.normalized; + } + + // Validate license (filter by collection license) + const licenseResult = validateLicense(license); + if (!licenseResult.valid) { + errors.push(licenseResult.error); + } else if (licenseResult.normalized !== undefined) { + normalized.license = licenseResult.normalized; + } // If any validation errors occurred, return 400 with details if (errors.length > 0) { diff --git a/api/routes/collections.js b/api/routes/collections.js index 6f83a2f..5e81101 100644 --- a/api/routes/collections.js +++ b/api/routes/collections.js @@ -1,6 +1,6 @@ const express = require('express'); const router = express.Router(); -const collectionsStore = require('../data/collections'); // change with the real collections when we have them +const { validateCollectionId } = require('../middleware/validateCollectionId'); const { validateCollectionSearchParams } = require('../middleware/validateCollectionSearch'); const { query } = require('../db/db_APIconnection'); const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); @@ -27,6 +27,8 @@ async function runQuery(sql, params = []) { * - limit: Number of results (default 10, max 10000) * - sortby: Sort by field (+field for ASC, -field for DESC) * - token: Pagination continuation token (offset) + * - provider: Provider name — filter by data provider + * - license: License identifier — filter by collection license * * All parameters are validated by validateCollectionSearchParams middleware. * Validated/normalized values are available in req.validatedParams. @@ -36,13 +38,15 @@ router.get('/', validateCollectionSearchParams, async (req, res, next) => { // TODO: Implement CQL2 filtering (GET endpoint) and add validator for `filter`, filter-lang` parameters try { // validated parameters from middleware - const { q, bbox, datetime, limit, sortby, token } = req.validatedParams; + const { q, bbox, datetime, limit, sortby, token, provider, license } = req.validatedParams; // build SQL querry and parameters const { sql, values } = buildCollectionSearchQuery({ q, bbox, datetime, + provider, + license, limit, sortby, token @@ -58,6 +62,8 @@ router.get('/', validateCollectionSearchParams, async (req, res, next) => { q, bbox, datetime, + provider, + license, limit: null, // No limit for count sortby: null, // No sorting for count token: null // No offset for count @@ -119,56 +125,71 @@ router.get('/', validateCollectionSearchParams, async (req, res, next) => { /** * GET /collections/:id - * Returns a single collection by ID. Includes all STAC Collection fields - * (stac_version, type, title, description, license, extent, links, etc). - * - * Returns: - * - 200 OK with full Collection object if found - * - 404 NotFound with proper error format if collection does not exist + * Returns a single collection by ID. + * + * Behaviour: + * - Uses the shared buildCollectionSearchQuery helper with an `id` filter + * so that GET /collections and GET /collections/:id stay aligned. + * - Returns: + * - 200 OK with a single Collection object if found + * - 404 NotFound with standardized error body if the collection does not exist + * + * Note: + * - The exact shape / fields of the returned collection are controlled by the + * SELECT part in buildCollectionSearchQuery. This allows the query builder + * (and later a mapping layer) to evolve without touching this route. */ -router.get('/:id', (req, res) => { - // TODO: Create a proper validator middleware for :id parameter to avoid SQL injection, etc. - const { id } = req.params; - - // Look up the collection in the data store by ID - // When connected to a DB, replace this with a SQL query (SELECT * FROM collections WHERE id = ?) - const collection = collectionsStore.find(c => c.id === id); - - if (!collection) { - // Return 404 with standardized error format - return res.status(404).json({ - code: 'NotFound', - description: `Collection with id '${id}' not found`, - id: id +router.get('/:id', validateCollectionId, async (req, res, next) => { + try { + const { id } = req.params; + + // id is already syntactically validated by validateCollectionId. + // For the database we use a numeric id, matching the c.collection.id column type. + const numericId = parseInt(id, 10); + + // Reuse the shared query builder with an exact id filter. + // We request a single row (LIMIT 1) and no offset. + const { sql, values } = buildCollectionSearchQuery({ + id: numericId, + limit: 1, + token: 0, }); - } - - // Return the full STAC Collection object - // Ensure the response includes at least self, root and parent links. - // Start from any links the collection already provides and add missing ones. - const baseHost = `${req.protocol}://${req.get('host')}`; - const selfHref = `${baseHost}/collections/${id}`; - const rootHref = baseHost; - const existingLinks = Array.isArray(collection.links) ? collection.links.slice() : []; + const rows = await runQuery(sql, values); - const hasRel = (rel) => existingLinks.some(l => l && l.rel === rel); + if (!rows || rows.length === 0) { + // Return 404 with standardized error format + return res.status(404).json({ + code: 'NotFound', + description: `Collection with id '${id}' not found`, + id: id + }); + } - if (!hasRel('self')) { - existingLinks.push({ rel: 'self', href: selfHref, type: 'application/json' }); - } + const collection = rows[0]; - if (!hasRel('root')) { - existingLinks.push({ rel: 'root', href: rootHref, type: 'application/json' }); - } + const baseHost = `${req.protocol}://${req.get('host')}`; + const selfHref = `${baseHost}${req.originalUrl}`; + const rootHref = baseHost; + + // TODO: + // Currently we always construct a minimal set of STAC-style links here. + // The crawler already stores the upstream links in full_json, but we do + // not extract or persist them as a separate links column yet. + // In the future we might want to parse those links and merge them here. + const links = [ + { rel: 'self', href: selfHref, type: 'application/json' }, + { rel: 'root', href: rootHref, type: 'application/json' }, + { rel: 'parent', href: rootHref, type: 'application/json' } + ]; - // Prefer an existing parent link if present, otherwise fall back to root - if (!hasRel('parent')) { - existingLinks.push({ rel: 'parent', href: rootHref, type: 'application/json' }); + // Return the collection with a normalized `links` array. + // The rest of the attributes (id, title, extent, full_json, …) come directly + // from the query builder / database. + res.json(Object.assign({}, collection, { links })); + } catch (error) { + next(error); } - - // Return the collection with a normalized `links` array - res.json(Object.assign({}, collection, { links: existingLinks })); }); module.exports = router; diff --git a/api/routes/index.js b/api/routes/index.js index b389488..14a7aca 100644 --- a/api/routes/index.js +++ b/api/routes/index.js @@ -16,7 +16,7 @@ router.get('/', (req, res) => { id: 'stac-atlas', title: 'STAC Atlas', description: 'A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs.', - stac_version: '1.0.0', + stac_version: '1.1.0', conformsTo: CONFORMANCE_URIS, links: [ { diff --git a/api/validators/collectionSearchParams.js b/api/validators/collectionSearchParams.js index 7f24faf..b15c263 100644 --- a/api/validators/collectionSearchParams.js +++ b/api/validators/collectionSearchParams.js @@ -264,11 +264,62 @@ function validateToken(token) { return { valid: true, normalized: num }; } +/** + * Validates provider parameter + * @param {string} provider - Provider name + * @returns {Object} { valid: boolean, error?: string, normalized?: string } + */ +function validateProvider(provider) { + if (!provider) return { valid: true }; + + if (typeof provider !== 'string') { + return { valid: false, error: 'Parameter "provider" must be a string' }; + } + + const trimmed = provider.trim(); + if (trimmed.length === 0) { + return { valid: false, error: 'Parameter "provider" must not be empty' }; + } + + if (trimmed.length > 255) { + return { valid: false, error: 'Parameter "provider" exceeds maximum length of 255 characters' }; + } + + return { valid: true, normalized: trimmed }; +} + +/** + * Validates license parameter + * @param {string} license - License identifier or name + * @returns {Object} { valid: boolean, error?: string, normalized?: string } + */ +function validateLicense(license) { + if (!license) return { valid: true }; + + if (typeof license !== 'string') { + return { valid: false, error: 'Parameter "license" must be a string' }; + } + + const trimmed = license.trim(); + if (trimmed.length === 0) { + return { valid: false, error: 'Parameter "license" must not be empty' }; + } + + if (trimmed.length > 255) { + return { valid: false, error: 'Parameter "license" exceeds maximum length of 255 characters' }; + } + + return { valid: true, normalized: trimmed }; +} + module.exports = { validateQ, validateBbox, validateDatetime, validateLimit, validateSortby, - validateToken + validateToken, + validateProvider, + validateLicense }; + diff --git a/db/docker-compose.yml b/db/docker-compose.yml index 2e342c0..94848fb 100644 --- a/db/docker-compose.yml +++ b/db/docker-compose.yml @@ -7,9 +7,13 @@ services: - .env environment: + # Admin user (required for initial database setup) POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + # Application user passwords + STAC_API_PASSWORD: ${STAC_API_PASSWORD} + STAC_CRAWLER_PASSWORD: ${STAC_CRAWLER_PASSWORD} ports: - "${DB_PORT}:5432" diff --git a/db/example.env b/db/example.env index e175b0b..ad3b4b9 100644 --- a/db/example.env +++ b/db/example.env @@ -1,7 +1,15 @@ # PostgreSQL Database Configuration +# Admin user with superuser privileges (required for initial setup) POSTGRES_DB= # stac_db is the database we are running on -POSTGRES_USER= # add postgres_user here -POSTGRES_PASSWORD= # add postgres_password here +POSTGRES_USER= # add postgres_user here (admin user) +POSTGRES_PASSWORD= # add postgres_password here (admin password) # Database Port (host:container) -DB_PORT= # 5432 / 5433 (at the moment both are available) \ No newline at end of file +DB_PORT= # 5432 / 5433 (at the moment both are available) + +# Application Users (created via init scripts) +# stac_api: read-only access for API +STAC_API_PASSWORD= # Password for api user (read-only); add api_password here + +# stac_crawler: full read-write access for crawler +STAC_CRAWLER_PASSWORD= # Password for crawler user (read-write); add crawler_password here diff --git a/db/init/00_users.sh b/db/init/00_users.sh new file mode 100644 index 0000000..03a5885 --- /dev/null +++ b/db/init/00_users.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +# This script creates users for teh api and crawler group with different permissions. + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + + -- Create read-only user for API access + CREATE USER stac_api WITH PASSWORD '$STAC_API_PASSWORD'; + + -- Create read-write user for crawler + CREATE USER stac_crawler WITH PASSWORD '$STAC_CRAWLER_PASSWORD'; + + GRANT CONNECT ON DATABASE stac_db TO stac_api; + GRANT CONNECT ON DATABASE stac_db TO stac_crawler; + GRANT USAGE ON SCHEMA public TO stac_api; + GRANT USAGE ON SCHEMA public TO stac_crawler; + + -- For stac_api: Grant SELECT (read-only) on all existing tables in public + GRANT SELECT ON ALL TABLES IN SCHEMA public TO stac_api; + + -- For stac_crawler: Grant all privileges on all existing tables in public + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO stac_crawler; + + -- For stac_api: Auto-grant SELECT on future tables + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO stac_api; + + -- For stac_crawler: Auto-grant all privileges on future tables + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO stac_crawler; + + -- For stac_crawler: Grant USAGE on all sequences (needed for SERIAL/IDENTITY columns) + GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO stac_crawler; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE ON SEQUENCES TO stac_crawler; +EOSQL diff --git a/db/init/02_tables_catalog.sql b/db/init/02_tables_catalog.sql index a44355f..dca17c4 100644 --- a/db/init/02_tables_catalog.sql +++ b/db/init/02_tables_catalog.sql @@ -9,7 +9,8 @@ CREATE TABLE catalog ( title TEXT, description TEXT, created_at TIMESTAMP DEFAULT now(), - updated_at TIMESTAMP DEFAULT now() + updated_at TIMESTAMP DEFAULT now(), + search_vector tsvector ); -- Catalog links table: Stores related links for catalogs (e.g., self, root, child, item links) @@ -17,6 +18,7 @@ CREATE TABLE catalog ( CREATE TABLE catalog_links ( id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, + source_url TEXT, rel TEXT, href TEXT, type TEXT, @@ -44,3 +46,62 @@ CREATE TABLE crawllog_catalog ( catalog_id INTEGER REFERENCES catalog(id) ON DELETE CASCADE, last_crawled TIMESTAMP ); + +-- ======================================== +-- FULL-TEXT SEARCH TRIGGERS +-- ======================================== + +-- Trigger function to auto-update search_vector when catalog is inserted or updated +-- Includes title, description, and all associated keywords for comprehensive search +CREATE OR REPLACE FUNCTION update_catalog_search_vector() +RETURNS TRIGGER AS $$ +BEGIN + NEW.search_vector := to_tsvector('simple', + coalesce(NEW.title, '') || ' ' || + coalesce(NEW.description, '') || ' ' || + coalesce( + ( + SELECT string_agg(k.keyword, ' ') + FROM catalog_keywords ck + JOIN keywords k ON k.id = ck.keyword_id + WHERE ck.catalog_id = NEW.id + ), + '' + ) + ); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER catalog_search_vector_update +BEFORE INSERT OR UPDATE ON catalog +FOR EACH ROW +EXECUTE FUNCTION update_catalog_search_vector(); + +-- Trigger function to update search_vector when keywords are added/removed +-- Ensures search index stays in sync with keyword changes +CREATE OR REPLACE FUNCTION update_catalog_search_vector_on_keyword_change() +RETURNS TRIGGER AS $$ +BEGIN + UPDATE catalog + SET search_vector = to_tsvector('simple', + coalesce(title, '') || ' ' || + coalesce(description, '') || ' ' || + coalesce( + ( + SELECT string_agg(k.keyword, ' ') + FROM catalog_keywords ck + JOIN keywords k ON k.id = ck.keyword_id + WHERE ck.catalog_id = catalog.id + ), + '' + ) + ) + WHERE id = COALESCE(NEW.catalog_id, OLD.catalog_id); + + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql; + +-- NOTE: The trigger for catalog_keywords is defined in 06_triggers.sql +-- because it depends on the catalog_keywords table which is created there diff --git a/db/init/03_tables_collections.sql b/db/init/03_tables_collections.sql index 733bb08..a443aa8 100644 --- a/db/init/03_tables_collections.sql +++ b/db/init/03_tables_collections.sql @@ -21,7 +21,8 @@ CREATE TABLE collection ( is_api BOOLEAN DEFAULT FALSE, is_active BOOLEAN DEFAULT TRUE, - full_json JSONB + full_json JSONB, + search_vector tsvector ); -- Collection summaries: Stores summaries for collection properties @@ -32,6 +33,7 @@ CREATE TABLE collection_summaries ( collection_id INTEGER REFERENCES collection(id) ON DELETE CASCADE, name TEXT, kind TEXT, + source_url TEXT, range_min NUMERIC, range_max NUMERIC, set_value TEXT, @@ -64,3 +66,62 @@ CREATE TABLE crawllog_collection ( collection_id INTEGER REFERENCES collection(id) ON DELETE CASCADE, last_crawled TIMESTAMP ); + +-- ======================================== +-- FULL-TEXT SEARCH TRIGGERS +-- ======================================== + +-- Trigger function to auto-update search_vector when collection is inserted or updated +-- Includes title, description, and all associated keywords for comprehensive search +CREATE OR REPLACE FUNCTION update_collection_search_vector() +RETURNS TRIGGER AS $$ +BEGIN + NEW.search_vector := to_tsvector('simple', + coalesce(NEW.title, '') || ' ' || + coalesce(NEW.description, '') || ' ' || + coalesce( + ( + SELECT string_agg(k.keyword, ' ') + FROM collection_keywords ck + JOIN keywords k ON k.id = ck.keyword_id + WHERE ck.collection_id = NEW.id + ), + '' + ) + ); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER collection_search_vector_update +BEFORE INSERT OR UPDATE ON collection +FOR EACH ROW +EXECUTE FUNCTION update_collection_search_vector(); + +-- Trigger function to update search_vector when keywords are added/removed +-- Ensures search index stays in sync with keyword changes +CREATE OR REPLACE FUNCTION update_collection_search_vector_on_keyword_change() +RETURNS TRIGGER AS $$ +BEGIN + UPDATE collection + SET search_vector = to_tsvector('simple', + coalesce(title, '') || ' ' || + coalesce(description, '') || ' ' || + coalesce( + ( + SELECT string_agg(k.keyword, ' ') + FROM collection_keywords ck + JOIN keywords k ON k.id = ck.keyword_id + WHERE ck.collection_id = collection.id + ), + '' + ) + ) + WHERE id = COALESCE(NEW.collection_id, OLD.collection_id); + + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql; + +-- NOTE: The trigger for collection_keywords is defined in 06_triggers.sql +-- because it depends on the catalog_keywords table which is created there \ No newline at end of file diff --git a/db/init/05_indexes.sql b/db/init/05_indexes.sql index 9349b88..7c0d74a 100644 --- a/db/init/05_indexes.sql +++ b/db/init/05_indexes.sql @@ -9,8 +9,8 @@ CREATE INDEX idx_catalog_title ON catalog (title); CREATE INDEX idx_catalog_updated_at ON catalog (updated_at); -CREATE INDEX idx_catalog_fulltext ON catalog -USING GIN (to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))); +-- Full-text search index on computed search_vector column (includes title, description, and keywords) +CREATE INDEX idx_catalog_search_vector ON catalog USING GIN (search_vector); CREATE INDEX idx_catalog_links_catalog_id ON catalog_links (catalog_id); CREATE INDEX idx_catalog_keywords_catalog ON catalog_keywords (catalog_id); @@ -30,8 +30,8 @@ CREATE INDEX idx_collection_active ON collection (is_active); CREATE INDEX idx_collection_spatial ON collection USING GIST (spatial_extent); -CREATE INDEX idx_collection_fulltext ON collection -USING GIN (to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))); +-- Full-text search index on computed search_vector column (includes title, description, and keywords) +CREATE INDEX idx_collection_search_vector ON collection USING GIN (search_vector); CREATE INDEX idx_collection_jsonb ON collection USING GIN (full_json); diff --git a/db/init/06_triggers.sql b/db/init/06_triggers.sql new file mode 100644 index 0000000..5e167a0 --- /dev/null +++ b/db/init/06_triggers.sql @@ -0,0 +1,16 @@ +-- ======================================== +-- FULL-TEXT SEARCH TRIGGERS FOR KEYWORDS +-- ======================================== +-- These triggers must be created here (after junction tables exist) + +-- Trigger to update catalog search_vector when keywords change +CREATE TRIGGER catalog_keywords_update_vector +AFTER INSERT OR DELETE ON catalog_keywords +FOR EACH ROW +EXECUTE FUNCTION update_catalog_search_vector_on_keyword_change(); + +-- Trigger to update collection search_vector when keywords change +CREATE TRIGGER collection_keywords_update_vector +AFTER INSERT OR DELETE ON collection_keywords +FOR EACH ROW +EXECUTE FUNCTION update_collection_search_vector_on_keyword_change(); \ No newline at end of file From 27507ed5275b0ee7348a5bfbe4831c8d6951ee48 Mon Sep 17 00:00:00 2001 From: mammutor Date: Thu, 15 Jan 2026 23:30:37 +0100 Subject: [PATCH 48/68] Refactor: crawler configuration and memory management - Updated rate limiting options in `HttpCrawler` to prevent memory buildup from queue overflow. - Introduced a new constant `CATALOG_CLEAR_BATCH_SIZE` to periodically clear the catalogs array, reducing memory usage. - Enhanced `handleCatalog` function to accept a configuration object, allowing for dynamic max depth checks during catalog processing. - Improved logging to provide clearer insights into catalog processing and memory management. --- crawler/catalogs/catalog.js | 6 ++---- crawler/utils/config.js | 4 ++-- crawler/utils/handlers.js | 36 ++++++++++++++++++++++++++++++++---- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 9883081..bef8f82 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -40,10 +40,8 @@ async function crawlCatalogs(initialCatalogs, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - maxConcurrency: 20, // Limit concurrency to prevent lock file race conditions - maxRequestsPerMinute: 200, // Rate limit to avoid overwhelming targets - // Rate limiting options + // Rate limiting options - prevents memory buildup from queue overflow maxConcurrency: config.maxConcurrency || 5, maxRequestsPerMinute: config.maxRequestsPerMinute || 60, sameDomainDelaySecs: config.sameDomainDelaySecs || 1, @@ -60,7 +58,7 @@ async function crawlCatalogs(initialCatalogs, config = {}) { try { // Route based on request label if (request.label === 'CATALOG') { - await handleCatalog({ request, json, crawler, log, indent, results }); + await handleCatalog({ request, json, crawler, log, indent, results, config }); } else if (request.label === 'COLLECTIONS') { await handleCollections({ request, json, crawler, log, indent, results }); } diff --git a/crawler/utils/config.js b/crawler/utils/config.js index 99c6f55..e0ce5ca 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -42,7 +42,7 @@ function getConfig() { maxCatalogs: 10, // Maximum number of catalogs to crawl maxApis: 5, // Maximum number of APIs to crawl timeout: 30000, // Timeout in milliseconds (30 seconds) - maxDepth: 10 // Maximum recursion depth for nested catalogs (0 = unlimited) + maxDepth: 10, // Maximum recursion depth for nested catalogs (0 = unlimited) // Rate limiting options (Crawlee) maxConcurrency: 5, // Maximum number of concurrent requests maxRequestsPerMinute: 60, // Maximum requests per minute @@ -60,7 +60,7 @@ function getConfig() { timeout: cliArgs.timeout !== undefined ? cliArgs.timeout : (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), maxDepth: cliArgs.maxDepth !== undefined ? cliArgs.maxDepth : - (process.env.MAX_DEPTH ? parseInt(process.env.MAX_DEPTH, 10) : defaults.maxDepth) + (process.env.MAX_DEPTH ? parseInt(process.env.MAX_DEPTH, 10) : defaults.maxDepth), // Rate limiting options maxConcurrency: cliArgs.maxConcurrency !== undefined ? cliArgs.maxConcurrency : (process.env.MAX_CONCURRENCY ? parseInt(process.env.MAX_CONCURRENCY, 10) : defaults.maxConcurrency), diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index f835389..9209b1b 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -13,7 +13,14 @@ import db from './db.js'; * After this many collections are collected, they will be flushed to DB * @type {number} */ -const BATCH_SIZE = 1000; +const BATCH_SIZE = 500; + +/** + * Batch size for clearing catalogs array to free memory + * The catalogs array is only used for statistics, so we clear it periodically + * @type {number} + */ +const CATALOG_CLEAR_BATCH_SIZE = 500; /** * Flushes collected collections to the database and clears the array @@ -57,6 +64,7 @@ export async function flushCollectionsToDb(results, log, force = false) { /** * Checks if batch size is reached and flushes if necessary + * Also clears the catalogs array periodically to free memory * @async * @param {Object} results - Results object containing collections array * @param {Object} log - Logger instance @@ -67,6 +75,13 @@ async function checkAndFlush(results, log) { results.stats.collectionsSaved += saved; results.stats.collectionsFailed += failed; } + + // Clear catalogs array periodically to free memory + // The catalogs array is only used for end statistics, which we track in stats object + if (results.catalogs.length >= CATALOG_CLEAR_BATCH_SIZE) { + log.info(`[MEMORY] Clearing ${results.catalogs.length} catalogs from memory`); + results.catalogs.length = 0; + } } /** @@ -79,12 +94,14 @@ async function checkAndFlush(results, log) { * @param {Object} context.log - Logger instance * @param {string} context.indent - Indentation for logging * @param {Object} context.results - Results object to store data + * @param {Object} context.config - Configuration object with maxDepth */ -export async function handleCatalog({ request, json, crawler, log, indent, results }) { +export async function handleCatalog({ request, json, crawler, log, indent, results, config = {} }) { const depth = request.userData?.depth || 0; const catalogId = request.userData?.catalogId || 'unknown'; + const maxDepth = config.maxDepth || 0; // 0 = unlimited - log.info(`${indent}Processing catalog: ${catalogId} (depth: ${depth})`); + log.info(`${indent}Processing catalog: ${catalogId} (depth: ${depth}${maxDepth > 0 ? `/${maxDepth}` : ''})`); // Validate with stac-js let stacCatalog; @@ -134,6 +151,14 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul if (childLinks.length > 0) { log.info(`${indent}Found ${childLinks.length} child catalog links`); + // Check maxDepth before enqueueing children + if (maxDepth > 0 && depth >= maxDepth) { + log.info(`${indent}Max depth (${maxDepth}) reached, skipping ${childLinks.length} child catalogs`); + // Clear memory and return early - don't enqueue children + await checkAndFlush(results, log); + return; + } + // Log first child link structure for debugging if (childLinks[0]) { log.debug(`${indent}Sample child link structure:`, { @@ -203,6 +228,9 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul } } } + + // Ensure memory is cleared periodically even if no collections were found + await checkAndFlush(results, log); } /** @@ -272,4 +300,4 @@ export async function handleCollections({ request, json, crawler, log, indent, r // Check if we should flush to database await checkAndFlush(results, log); } -} +} \ No newline at end of file From e5f6e33a42223c0866259a8602cd70d49a69945a Mon Sep 17 00:00:00 2001 From: Lenn Date: Fri, 16 Jan 2026 00:23:13 +0100 Subject: [PATCH 49/68] Crawler added source_url to collection_summaries and changed stac_id, just write collections (#245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(api): initialize STAC Atlas API with collections, conformance, and queryables routes - Added package.json for project dependencies and scripts. - Implemented GET endpoint for collections. - Created conformance endpoint to list supported conformance classes. - Developed landing page for the API with links to collections and documentation. - Added queryables endpoint to return queryable properties for collections. (If i'm correct this can be removed) * Added all Remarks to the bid and finished it (#74) * changed Texts 1,2 and 8 according to the remarks of the customer * Did my fixes to 4. and 10.3 * added remark why we want to save every catalog * deleted keywords for catalog * changed everything related to the database component * Updated 3.1, 3.3, 3,4 (References to Lastenheft/small other changes) * Added small Graph to 3. Produktumgebung * Update bid.md 7.3STAC-Validator added the handling of collections that cannot be validated automatically. * Update bid.md 7.3STAC-API-Validator changed the way we validate the collection search extension. * Update bid.md 9.3.2Endpunkte small fix collection search extension. * added Skizze for 3, and updated 6.1, 10.1 * Updated 3. Produktumgebung * Update bid.md 7.Qualitätsanforderungen minor fixes * Update 6.4.2 added remark about loading feedback * Minor change to Table in "11 Zeitplan". A collum was missing in the head, therefore the table wasn't rendering correctly.. --------- Co-authored-by: Jakob Co-authored-by: Sönke Hoffmann Co-authored-by: jklaer Co-authored-by: VincentKuehn Co-authored-by: mammutor Co-authored-by: Humam <44206081+Mammutor@users.noreply.github.com> Co-authored-by: Justin K * fix(api): enhance STAC API landing page and conformance links - Overhaul of first idea landing page - Added some more tests for the required elements in the landingpage-Catalog * feat(api): implement shared conformance URIs and add tests for conformance endpoint - implemented condormance endpoint * Implemented 2.3 and 2.4 (#111) * Temporary mock data for testing and frontend development * Added API middleware layer for error handling and validation * TODOs ready? pls review * Added API utilities for query parsing, validation, and response formatting * Added swagger and openapi.yaml * Update queryables.js Refactor queryables endpoint into /collections/queryables * Renamed the collections.js file to mocks-collections.js to better reflect its purpose and improve project clarity * changed README "Projektstruktur" * restart from dev-api 22.11..2025 * API: 2.3 Implement Collections List Endpoint done (added explanations as comments in the code) * API: 2.4 Implement Single Collection Endpoint (added explanations as comments in the code) * Update api/routes/collections.js Co-authored-by: Robin Tammo Gummels * Changed some of the code with the comments on Github (i will finish it tomorrow morning) * Implement most of the feedback and comments (need to talk about some other changes) * Update api/routes/index.js Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/README.md Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/README.md Removed missing folder * Update api/routes/collections.js Removed TODOs from wrong lines * Update api/routes/collections.js Added TODOs * Update api/routes/queryables.js Changed wording from `/collections/queryables` to `/collections-queryables` * Update api/routes/queryables.js Changed wording from `/collections/queryables` to `/collections-queryables` --------- Co-authored-by: VincentKuehn Co-authored-by: Robin Tammo Gummels * feat(api): add collection search parameters and validation middleware (#159) * feat(api): add collection search parameters and validation middleware * Added unit-test for validator-functions and integration-tests for `GET /collections`-Querys. - Also minor bugfix, because the validator accepted deecimals as tokens. * API: 3 Database Integration first version (#161) * database connection in implementated. The parameters for the connection have to added in the .env-file. Also there is test-file for testing and console messages (installed `pg`) * support for spatial queries via postgis + error handling for datatbase operations changed language to english * error handling * added DATABASE_URL There is an issue with the distance query. Changed the error handling and testing, the console messages are now way better structured * found the Problem with the distance query. The layer are so big, that they reach over the 180° long (PostgGIS can't handel that). Now the calc is done by degree and not meters. * The two files `test-data-retrieval.js` and `verify-schema.js` have been added. `test-data-retrieval` (theoretical, checks against the spezification): ``` Discovers all tables and columns and validates against expected schema. ``` The second files `verify-schema.js` (practical, checks against the real data): ``` Discovers all tables and columns, validates against expected schema ``` * pooling error hanling and log imporoved. renamed tests files to actual test-files * standalone node tests were convertad into JEST * write file `validateRequest.js`. Validates every incoming API request, whether the request is valid and logical. * commented `stac_id` from the tests, it is not in both databases, so the tests for `stac_id` will always fail Added explanation to the `.env.example`, which port is which database * added example pattern for API - database connection. * deleted `validateRequest` cause it's already implemented by @robinGummels --------- Co-authored-by: Sönke Hoffmann * Added environment variables and a `.env` for `docker-compose.yml` (#164) * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * Added a CI/CD Pipeline to prevent pull-requests without functioning tests and proper linting. * fixed errors suggested by the linter. - Some lines used tab and spaces... * Implemented Collection Search extension including a DB-Connection (#165) * added SQLQuery-builder with these parameters: q,bbox,datetime,sortby,limit and token * finalised bbox and datetime * adapted to DB, QueryBuilder and added helperfunction runQuery * added question-TODOs * added bbox+datetime to the Query-Builder from Jonas * added tests for Query-Builder from Jonas * added tests from George * added falsely deleted TODOs again * fixed collumn names to match our DB and adjusted full text search to match 05_indexes.sql correctly * Used a formatter and linter on `buildCollectionSearchQuery.js * Did some major and minor fixes to the collection search. - Updated `buildCollectionSearchQuery` to support pagination and improved text search with English language settings. - Modified tests in `buildCollectionsSearchQuery.basic.test.js`, `collections-pagination.test.js`, and `collections-sort.test.js` to reflect new query behavior and validation logic. - Enhanced sort validation in `validators.test.js` and `collectionSearchParams.js` to map API fields to database column names. - Implemented total count retrieval for matched results in `collections.js`. * Added a internal .env creation in the CI/CD Pipeline. It utilzes GitHub Repository Secrets to not publish any private Logins and stuff. * Forgot that the second Job of the CI/CD pipeline runs seperatly and needs a internal .env file too. * Enhance documentation for buildCollectionSearchQuery Updated the documentation for: - the buildCollectionSearchQuery function - the fulltextsearch * Refactor buildCollectionSearchQuery and updated SELECT part Changed the SELECT part to match our bid and the database shema. Updated comments and for clarity. Changed full-text search to use 'simple' configuration instead of 'english'. * Update api/routes/collections.js small typo Co-authored-by: Robin Tammo Gummels * Remove sorting TODO from collections route Removed TODO comment about sorting based on sortby parameter. * Explicitly return undefined for normalized in validateSortby Update validateSortby function to explicitly return undefined for normalized when sortby is not provided. * small fix in buildCollectionSearch.fulltext.test.js Change plainto_tsquery language from 'english' to 'simple' * Fix duplicate SELECT keyword in query Remove duplicate 'SELECT' keyword in SQL query. * Fix missing newline at end of collectionSearchParams.js * Fixed missing bracket in collectionSearchParams.js * Refactor validateSortby for optional parameter handling Refactor validateSortby function to handle optional sortby parameter and improve validation logic. * Stabilize API test pipeline by running Jest in-band with extended timeout Run Jest in CI with --runInBand and a higher default --testTimeout to stabilize database-backed integration tests. Multiple Jest workers were competing for the same PostgreSQL connection pool and some long-running /collections queries exceeded the default 5s timeout, causing failures in existing test suites (e.g. collectionSearch and DBconnection). * Fixed leaking tests that blocked CI/CD-Pipeline. - Added a global Teardown for jest and force-exited the tests to prevent leaking. - Made a change to db_APIconnection to only log the pool-(dis)connection if it isn't run in a test enviroment. * Did a minimum amount of Formatting to the discription * Used `npm audit fix --force` to fix all vulnerabilties in our used packages. * Fixed curious doublechecking for empty Strings for the sortby-Parameter. - Now we only check once for a empty sortby - And added a test which distinguish between `sortby=""` and `sortby="+"` * Update api/routes/collections.js Removed the TODO about switching from mock-data to the real db * Removed globalTeardown as i brought up some problems corresponding to long db-queries (for example BBOX). Instead i increased the maximal testTimeout. --------- Co-authored-by: Robin Tammo Gummels * Update api/.env.example * latest database Version (#187) with `stac_id` and changed definition of `primary Keys` * added `.env` * added environment for docker-compose.yml now every connection-details are inside an `.env`. There is an `example.env` for better understanding which need to be set as connection details * added description of how to use the `.env` and `example.env` in the `README.md` * changed a few things e.g. DB_PORT --> ${DB_PORT} * now, everthing should be done. my god, help. sorry * layout issues fixed * Fixed Typo/incomplete Sentence in README.md * added `stac_id` for collections * all IDs are now written in the newer PostgrSQL standart: ```SQL id SERIAL PRIMARY KEY, ``` changed to ```SQL id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, ``` * changed `extend` to `extent`. * Changed language used in `./api/README.md` from german to english. I wanted to thsi anyway at some point, but this is now more like a Test-commit to see if the CI/CD Pipeline triggers... --------- Co-authored-by: Sönke Hoffmann Co-authored-by: Robin Tammo Gummels * bug fix: extent was written wrong * feat: add source URL extraction for catalogs and collections * feat: update insertOrUpdateCatalog to only process catalogs for traversal, no longer saving to database --------- Co-authored-by: RobinGummels Co-authored-by: Jakob Co-authored-by: Sönke Hoffmann Co-authored-by: jklaer Co-authored-by: VincentKuehn Co-authored-by: mammutor Co-authored-by: Humam <44206081+Mammutor@users.noreply.github.com> Co-authored-by: Justin K Co-authored-by: Georgios Voulgaris Co-authored-by: Sönke Hoffmann --- crawler/utils/db.js | 153 ++++++++++++-------------------------------- 1 file changed, 40 insertions(+), 113 deletions(-) diff --git a/crawler/utils/db.js b/crawler/utils/db.js index 1bd2e85..df12a44 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -51,107 +51,19 @@ async function initDb() { } /** - * Insert or update a catalog in the database + * Process a catalog for traversal only - catalogs are not saved to database + * Only collections are saved. Catalogs are only used to traverse deeper into the tree. * @param {Object} catalog - STAC catalog object - * @returns {Promise} catalog ID + * @returns {Promise} always returns null (no catalog saved) */ async function insertOrUpdateCatalog(catalog) { if (!catalog || typeof catalog !== 'object') return null; - - const client = await pool.connect(); - try { - await client.query('BEGIN'); - - // Insert or update catalog - const catalogTitle = catalog.title || catalog.id || 'Unnamed Catalog'; - - // Handle both STAC format and STAC Index API format - const description = catalog.description || null; - const stacVersion = catalog.stac_version || catalog.stacVersion || null; - - // Check if catalog with this title already exists - const existingCatalog = await client.query( - 'SELECT id FROM catalog WHERE title = $1', - [catalogTitle] - ); - - let catalogId; - if (existingCatalog.rows.length > 0) { - // Update existing catalog - catalogId = existingCatalog.rows[0].id; - await client.query( - `UPDATE catalog SET - stac_version = $1, - type = $2, - description = $3, - updated_at = now() - WHERE id = $4`, - [ - stacVersion, - catalog.type || 'Catalog', - description, - catalogId - ] - ); - } else { - // Insert new catalog - const catalogResult = await client.query( - `INSERT INTO catalog (stac_version, type, title, description, updated_at) - VALUES ($1, $2, $3, $4, now()) - RETURNING id`, - [ - stacVersion, - catalog.type || 'Catalog', - catalogTitle, - description - ] - ); - catalogId = catalogResult.rows[0].id; - } - - // Insert links - if (catalog.links && Array.isArray(catalog.links)) { - await client.query('DELETE FROM catalog_links WHERE catalog_id = $1', [catalogId]); - for (const link of catalog.links) { - await client.query( - 'INSERT INTO catalog_links (catalog_id, rel, href, type, title) VALUES ($1, $2, $3, $4, $5)', - [catalogId, link.rel, link.href, link.type || null, link.title || null] - ); - } - } - - // Insert keywords - // STAC Index API uses 'categories' instead of 'keywords' - const keywords = catalog.keywords || catalog.categories || []; - if (Array.isArray(keywords) && keywords.length > 0) { - await insertKeywords(client, catalogId, keywords, 'catalog'); - } - - // Insert STAC extensions - const extensions = catalog.stac_extensions || catalog.stacExtensions || []; - if (Array.isArray(extensions) && extensions.length > 0) { - await insertStacExtensions(client, catalogId, extensions, 'catalog'); - } - - // Update crawl log - await client.query( - 'DELETE FROM crawllog_catalog WHERE catalog_id = $1', - [catalogId] - ); - await client.query( - 'INSERT INTO crawllog_catalog (catalog_id, last_crawled) VALUES ($1, now())', - [catalogId] - ); - - await client.query('COMMIT'); - return catalogId; - } catch (error) { - await client.query('ROLLBACK'); - console.error('Error inserting catalog:', error.message); - throw error; - } finally { - client.release(); - } + + // Catalogs are not saved to database - only used for tree traversal + // Only collections will be saved + console.log(`Skipping catalog save (used for traversal only): ${catalog.title || catalog.id}`); + + return null; } /** @@ -160,7 +72,7 @@ async function insertOrUpdateCatalog(catalog) { * @returns {Promise} collection ID */ async function insertOrUpdateCollection(collection) { - if (!collection || typeof collection !== 'object') return null; + const client = await pool.connect(); try { @@ -187,6 +99,15 @@ async function insertOrUpdateCollection(collection) { // Insert or update collection const collectionTitle = collection.title || collection.id || 'Unnamed Collection'; + const stacId = collection.id || null; + + // Extract source URL from links (prefer 'self', fallback to 'root') + let sourceUrl = null; + if (collection.links && Array.isArray(collection.links)) { + const selfLink = collection.links.find(link => link.rel === 'self'); + const rootLink = collection.links.find(link => link.rel === 'root'); + sourceUrl = selfLink?.href || rootLink?.href || null; + } // Check if collection with this title already exists const existingCollection = await client.query( @@ -200,19 +121,22 @@ async function insertOrUpdateCollection(collection) { collectionId = existingCollection.rows[0].id; await client.query( `UPDATE collection SET - stac_version = $1, - type = $2, - description = $3, - license = $4, - spatial_extent = ST_GeomFromEWKT($5), - temporal_extent_start = $6, - temporal_extent_end = $7, - is_api = $8, - is_active = $9, - full_json = $10, + stac_id = $1, + stac_version = $2, + type = $3, + description = $4, + license = $5, + spatial_extent = ST_GeomFromEWKT($6), + temporal_extent_start = $7, + temporal_extent_end = $8, + is_api = $9, + is_active = $10, + full_json = $11, + source_url = $12, updated_at = now() - WHERE id = $11`, + WHERE id = $13`, [ + stacId, collection.stac_version || null, collection.type || 'Collection', collection.description || null, @@ -223,6 +147,7 @@ async function insertOrUpdateCollection(collection) { false, // is_api - will be determined by crawler true, // is_active JSON.stringify(collection), + sourceUrl, collectionId ] ); @@ -230,13 +155,14 @@ async function insertOrUpdateCollection(collection) { // Insert new collection const collectionResult = await client.query( `INSERT INTO collection ( - stac_version, type, title, description, license, + stac_id, stac_version, type, title, description, license, spatial_extent, temporal_extent_start, temporal_extent_end, - is_api, is_active, full_json, updated_at + is_api, is_active, full_json, source_url, updated_at ) - VALUES ($1, $2, $3, $4, $5, ST_GeomFromEWKT($6), $7, $8, $9, $10, $11, now()) + VALUES ($1, $2, $3, $4, $5, $6, ST_GeomFromEWKT($7), $8, $9, $10, $11, $12, $13, now()) RETURNING id`, [ + stacId, collection.stac_version || null, collection.type || 'Collection', collectionTitle, @@ -247,7 +173,8 @@ async function insertOrUpdateCollection(collection) { temporalEnd, false, // is_api - will be determined by crawler true, // is_active - JSON.stringify(collection) + JSON.stringify(collection), + sourceUrl ] ); collectionId = collectionResult.rows[0].id; From c5b2e8f6e65c71c53c155111b23022b24f558b81 Mon Sep 17 00:00:00 2001 From: mammutor Date: Fri, 16 Jan 2026 15:59:59 +0100 Subject: [PATCH 50/68] Enhance memory management in API and handlers - Introduced periodic clearing of the `apis` array in `checkAndFlushApi` to free memory, with a defined batch size. - Updated `checkAndFlush` in `handlers.js` to ensure the `catalogs` array is cleared only if it exists, preventing potential errors. - Improved logging for memory management actions to provide better insights during API operations. --- crawler/apis/api.js | 15 +++++++++++++++ crawler/utils/handlers.js | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index a439a07..df09c6e 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -14,8 +14,16 @@ import { handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; */ const BATCH_SIZE = 500; +/** + * Batch size for clearing apis array to free memory + * The apis array is only used for statistics, so we clear it periodically + * @type {number} + */ +const API_CLEAR_BATCH_SIZE = 500; + /** * Checks if batch size is reached and flushes if necessary + * Also clears the apis array periodically to free memory * @async * @param {Object} results - Results object containing collections array * @param {Object} log - Logger instance @@ -26,6 +34,13 @@ async function checkAndFlushApi(results, log) { results.stats.collectionsSaved = (results.stats.collectionsSaved || 0) + saved; results.stats.collectionsFailed = (results.stats.collectionsFailed || 0) + failed; } + + // Clear apis array periodically to free memory + // The apis array is only used for end statistics, which we track in stats object + if (results.apis && results.apis.length >= API_CLEAR_BATCH_SIZE) { + log.info(`[MEMORY] Clearing ${results.apis.length} APIs from memory`); + results.apis.length = 0; + } } /** diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index 9209b1b..c9ee0df 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -78,7 +78,8 @@ async function checkAndFlush(results, log) { // Clear catalogs array periodically to free memory // The catalogs array is only used for end statistics, which we track in stats object - if (results.catalogs.length >= CATALOG_CLEAR_BATCH_SIZE) { + // Note: catalogs may not exist when called from API crawler (which uses apis instead) + if (results.catalogs && results.catalogs.length >= CATALOG_CLEAR_BATCH_SIZE) { log.info(`[MEMORY] Clearing ${results.catalogs.length} catalogs from memory`); results.catalogs.length = 0; } From 9a26ae08d0294c88eebcee77b22be2c32611eb68 Mon Sep 17 00:00:00 2001 From: mammutor Date: Sun, 18 Jan 2026 14:34:12 +0100 Subject: [PATCH 51/68] Refactor database operations in insertOrUpdateCollection and insertSummary functions - Removed the source_url parameter from the insertOrUpdateCollection function, simplifying the SQL query. - Updated the insertSummary function to include source_url in the database insert statement. - Adjusted related calls to insertSummary to pass the new source_url parameter. - Enhanced the handling of collection summaries to ensure proper data insertion. --- crawler/utils/db.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/crawler/utils/db.js b/crawler/utils/db.js index df12a44..ffc1463 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -132,9 +132,8 @@ async function insertOrUpdateCollection(collection) { is_api = $9, is_active = $10, full_json = $11, - source_url = $12, updated_at = now() - WHERE id = $13`, + WHERE id = $12`, [ stacId, collection.stac_version || null, @@ -147,7 +146,6 @@ async function insertOrUpdateCollection(collection) { false, // is_api - will be determined by crawler true, // is_active JSON.stringify(collection), - sourceUrl, collectionId ] ); @@ -157,9 +155,9 @@ async function insertOrUpdateCollection(collection) { `INSERT INTO collection ( stac_id, stac_version, type, title, description, license, spatial_extent, temporal_extent_start, temporal_extent_end, - is_api, is_active, full_json, source_url, updated_at + is_api, is_active, full_json, updated_at ) - VALUES ($1, $2, $3, $4, $5, $6, ST_GeomFromEWKT($7), $8, $9, $10, $11, $12, $13, now()) + VALUES ($1, $2, $3, $4, $5, $6, ST_GeomFromEWKT($7), $8, $9, $10, $11, $12, now()) RETURNING id`, [ stacId, @@ -173,8 +171,7 @@ async function insertOrUpdateCollection(collection) { temporalEnd, false, // is_api - will be determined by crawler true, // is_active - JSON.stringify(collection), - sourceUrl + JSON.stringify(collection) ] ); collectionId = collectionResult.rows[0].id; @@ -184,7 +181,7 @@ async function insertOrUpdateCollection(collection) { if (collection.summaries && typeof collection.summaries === 'object') { await client.query('DELETE FROM collection_summaries WHERE collection_id = $1', [collectionId]); for (const [name, value] of Object.entries(collection.summaries)) { - await insertSummary(client, collectionId, name, value); + await insertSummary(client, collectionId, name, value, sourceUrl); } } @@ -289,7 +286,7 @@ async function insertStacExtensions(client, parentId, extensions, type) { /** * Helper function to insert collection summaries */ -async function insertSummary(client, collectionId, name, value) { +async function insertSummary(client, collectionId, name, value, sourceUrl) { let kind = 'unknown'; let rangeMin = null; let rangeMax = null; @@ -314,8 +311,8 @@ async function insertSummary(client, collectionId, name, value) { } await client.query( - 'INSERT INTO collection_summaries (collection_id, name, kind, range_min, range_max, set_value, json_schema) VALUES ($1, $2, $3, $4, $5, $6, $7)', - [collectionId, name, kind, rangeMin, rangeMax, setValue, jsonSchema] + 'INSERT INTO collection_summaries (collection_id, name, kind, source_url, range_min, range_max, set_value, json_schema) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)', + [collectionId, name, kind, sourceUrl, rangeMin, rangeMax, setValue, jsonSchema] ); } From 8e9d504064c1829f21c41b3edfd641d363bb5e1d Mon Sep 17 00:00:00 2001 From: Humam <44206081+Mammutor@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:41:57 +0100 Subject: [PATCH 52/68] Delete .env.example in root we have to decide later what exactly to do --- .env.example | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .env.example diff --git a/.env.example b/.env.example deleted file mode 100644 index e0d6ba7..0000000 --- a/.env.example +++ /dev/null @@ -1,23 +0,0 @@ -# Database Configuration -# PostgreSQL connection parameters -PGHOST=atlas.example.org -PGPORT=5432 -PGUSER=example_user -PGPASSWORD=example_password -PGDATABASE=example_db - -# Crawler Configuration -# Crawl mode: 'catalogs', 'apis', or 'both' -CRAWL_MODE=both - -# Maximum number of catalogs to crawl (default: 10) -MAX_CATALOGS=10 - -# Maximum number of APIs to crawl (default: 5) -MAX_APIS=5 - -# Timeout for each crawl operation in milliseconds (default: 30000) -TIMEOUT_MS=30000 - -# Node Environment -NODE_ENV=development From 70004cb6ac02bf632ff2f46dcac97a9b173cc194 Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 20 Jan 2026 12:35:19 +0100 Subject: [PATCH 53/68] fix(crawler-db): Enhance collection extent handling in insertOrUpdateCollection function - Updated the function to support both normalized (bbox) and original STAC format (extent.spatial.bbox) for spatial extent. - Improved temporal extent parsing to accommodate both normalized (temporal) and original STAC format (extent.temporal.interval). - Refactored code for better clarity and maintainability. --- crawler/utils/db.js | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/crawler/utils/db.js b/crawler/utils/db.js index ffc1463..e93434e 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -79,20 +79,40 @@ async function insertOrUpdateCollection(collection) { await client.query('BEGIN'); // Parse spatial extent (bbox) + // Support both normalized format (bbox) and original STAC format (extent.spatial.bbox) let spatialExtent = null; - if (collection.extent?.spatial?.bbox && collection.extent.spatial.bbox[0]) { - const bbox = collection.extent.spatial.bbox[0]; - if (bbox.length === 4) { - // Create polygon from bbox [west, south, east, north] - spatialExtent = `EPSG:4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; - } + let bbox = null; + + // Try normalized format first (from normalizeCollection) + if (collection.bbox && Array.isArray(collection.bbox)) { + bbox = collection.bbox; + } + // Fallback to original STAC format + else if (collection.extent?.spatial?.bbox && collection.extent.spatial.bbox[0]) { + bbox = collection.extent.spatial.bbox[0]; + } + + if (bbox && bbox.length === 4) { + // Create polygon from bbox [west, south, east, north] + spatialExtent = `EPSG:4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; } // Parse temporal extent + // Support both normalized format (temporal) and original STAC format (extent.temporal.interval) let temporalStart = null; let temporalEnd = null; - if (collection.extent?.temporal?.interval && collection.extent.temporal.interval[0]) { - const interval = collection.extent.temporal.interval[0]; + let interval = null; + + // Try normalized format first (from normalizeCollection) + if (collection.temporal && Array.isArray(collection.temporal)) { + interval = collection.temporal; + } + // Fallback to original STAC format + else if (collection.extent?.temporal?.interval && collection.extent.temporal.interval[0]) { + interval = collection.extent.temporal.interval[0]; + } + + if (interval) { temporalStart = interval[0] ? new Date(interval[0]) : null; temporalEnd = interval[1] ? new Date(interval[1]) : null; } From f53d2e783604242261fbcca07597387ae3194bad Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 20 Jan 2026 12:43:02 +0100 Subject: [PATCH 54/68] fix(crawler-api): update STAC object creation to disable URL migration - Modified the `create` function calls in `handleApiRoot`, `handleApiCollection`, and `handleCatalog` to set the second parameter to `false`, preventing URL migration. - Added comments to clarify the change and its implications for STAC compliance validation. --- crawler/apis/api.js | 6 ++++-- crawler/utils/handlers.js | 11 +++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index df09c6e..cd05f12 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -219,9 +219,10 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max log.info(`${indent}Processing API: ${apiId} at ${apiUrl} (depth: ${depth})`); // Validate with stac-js + // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL let stacObj; try { - stacObj = create(json, request.url); + stacObj = create(json, false); results.stats.stacCompliant++; if (typeof stacObj.isCatalog === 'function' && stacObj.isCatalog()) { @@ -354,9 +355,10 @@ async function handleApiCollection({ request, json, crawler, log, indent, result const apiId = request.userData?.apiId || 'unknown'; // Validate with stac-js + // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL let stacObj; try { - stacObj = create(json, request.url); + stacObj = create(json, false); if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { const collection = normalizeCollection(stacObj, results.collections.length); diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index c9ee0df..d61a931 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -105,9 +105,11 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul log.info(`${indent}Processing catalog: ${catalogId} (depth: ${depth}${maxDepth > 0 ? `/${maxDepth}` : ''})`); // Validate with stac-js + // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL + // Disable migration as it can cause issues with newer STAC versions (1.1.0+) let stacCatalog; try { - stacCatalog = create(json, request.url); + stacCatalog = create(json, false); results.stats.stacCompliant++; // Log STAC object type @@ -249,9 +251,10 @@ export async function handleCollections({ request, json, crawler, log, indent, r const catalogId = request.userData?.catalogId || 'unknown'; // Parse response with stac-js + // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL let stacObj; try { - stacObj = create(json, request.url); + stacObj = create(json, false); } catch (parseError) { log.warning(`${indent}Skipping non-compliant STAC collections at ${request.url}`); return; @@ -266,7 +269,7 @@ export async function handleCollections({ request, json, crawler, log, indent, r // Handle array of collections collectionsData = json.map(col => { try { - return create(col, request.url); + return create(col, false); } catch { return null; } @@ -275,7 +278,7 @@ export async function handleCollections({ request, json, crawler, log, indent, r // Handle nested collections property collectionsData = json.collections.map(col => { try { - return create(col, request.url); + return create(col, false); } catch { return null; } From 23be843d04ba1be7f2d38d64d5a3a32894449d34 Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 20 Jan 2026 12:48:51 +0100 Subject: [PATCH 55/68] fix(crawler-api): add defensive JSON validation in API and catalog handlers - Implemented checks in `handleApiRoot` and `handleCatalog` to ensure the JSON response is valid before processing. - Added logging for invalid JSON responses to improve error tracking and handling. - Updated comments to clarify the purpose of disabling URL migration in STAC object creation. --- crawler/apis/api.js | 7 +++++++ crawler/utils/handlers.js | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index cd05f12..6b70c00 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -218,8 +218,15 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max log.info(`${indent}Processing API: ${apiId} at ${apiUrl} (depth: ${depth})`); + // Defensive check: ensure json is valid before passing to stac-js + if (!json || typeof json !== 'object') { + log.warning(`${indent}Invalid JSON response for ${apiId} at ${request.url}`); + throw new Error('Invalid JSON response: null or not an object'); + } + // Validate with stac-js // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL + // Using migrate=false to avoid issues with stac-migrate and newer STAC versions let stacObj; try { stacObj = create(json, false); diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index d61a931..becc5bb 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -104,9 +104,15 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul log.info(`${indent}Processing catalog: ${catalogId} (depth: ${depth}${maxDepth > 0 ? `/${maxDepth}` : ''})`); + // Defensive check: ensure json is valid before passing to stac-js + if (!json || typeof json !== 'object') { + log.warning(`${indent}Invalid JSON response for catalog ${catalogId} at ${request.url}`); + throw new Error('Invalid JSON response: null or not an object'); + } + // Validate with stac-js // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL - // Disable migration as it can cause issues with newer STAC versions (1.1.0+) + // Using migrate=false to avoid issues with stac-migrate and newer STAC versions let stacCatalog; try { stacCatalog = create(json, false); From 3ab484c4773a6481c0edade5b05c83dcc0a18526 Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 20 Jan 2026 12:49:54 +0100 Subject: [PATCH 56/68] chore(crawler): update Node.js version in Dockerfile to 20-alpine --- crawler/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler/Dockerfile b/crawler/Dockerfile index 6c77666..651de07 100644 --- a/crawler/Dockerfile +++ b/crawler/Dockerfile @@ -1,5 +1,5 @@ # Use official Node.js LTS image -FROM node:18-alpine +FROM node:20-alpine # Set working directory WORKDIR /app From c8aec56e6d5b4a48503cdf28ce44995c18ce99ce Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 20 Jan 2026 12:57:26 +0100 Subject: [PATCH 57/68] fix(crawler): enhance JSON parsing in API and catalog request handlers - Updated request handlers in `crawlApis` and `crawlCatalogs` to include a fallback mechanism for manually parsing JSON responses when automatic parsing fails, because some servers like the DLR usese 14 seconds. - Added logging for successful and failed manual parsing attempts to improve debugging and error tracking. - Adjusted comments to clarify the purpose of the new parsing logic. --- crawler/apis/api.js | 14 +++++++++++++- crawler/catalogs/catalog.js | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 6b70c00..f1aa5a7 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -111,11 +111,23 @@ async function crawlApis(urls, isApi, config = {}) { // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], - async requestHandler({ request, json, crawler, log }) { + async requestHandler({ request, json, body, crawler, log }) { results.stats.totalRequests++; const depth = request.userData?.depth || 0; const indent = ' '.repeat(Math.min(depth, 5)); // Cap indent at 5 levels + // Fallback: manually parse JSON if Crawlee's automatic parsing failed + // This can happen with slow endpoints or unusual response streaming + if (!json && body) { + try { + const bodyStr = typeof body === 'string' ? body : body.toString('utf8'); + json = JSON.parse(bodyStr); + log.debug(`${indent}Manually parsed JSON for ${request.url} (${bodyStr.length} bytes)`); + } catch (parseError) { + log.warning(`${indent}Failed to parse response body as JSON: ${parseError.message}`); + } + } + try { // Route based on request label if (request.label === 'API_ROOT') { diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index bef8f82..e9002ea 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -50,11 +50,23 @@ async function crawlCatalogs(initialCatalogs, config = {}) { // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], - async requestHandler({ request, json, crawler, log }) { + async requestHandler({ request, json, body, crawler, log }) { results.stats.totalRequests++; const depth = request.userData?.depth || 0; const indent = ' '.repeat(depth); + // Fallback: manually parse JSON if Crawlee's automatic parsing failed + // This can happen with slow endpoints or unusual response streaming + if (!json && body) { + try { + const bodyStr = typeof body === 'string' ? body : body.toString('utf8'); + json = JSON.parse(bodyStr); + log.debug(`${indent}Manually parsed JSON for ${request.url} (${bodyStr.length} bytes)`); + } catch (parseError) { + log.warning(`${indent}Failed to parse response body as JSON: ${parseError.message}`); + } + } + try { // Route based on request label if (request.label === 'CATALOG') { From 392b0e9bb9e95c8c511c23cc687c28cc6e68a7a9 Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 20 Jan 2026 13:02:04 +0100 Subject: [PATCH 58/68] fix(crawler): add S3 URL handling and relative URL conversion in API and handlers - Implemented conversion of S3 protocol URLs to HTTPS format in `handleApiRoot`, `tryCollectionEndpoints`, and `handleCatalog` functions. - Added logging for successful conversions and warnings for malformed S3 URLs. - Enhanced handling of relative URLs to ensure they are converted to absolute URLs based on the request context. - Improved validation checks for URLs to skip invalid entries with appropriate logging. --- crawler/apis/api.js | 22 ++++++++++++++++++++++ crawler/utils/endpoints.js | 10 ++++++++++ crawler/utils/handlers.js | 14 ++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index f1aa5a7..7205675 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -329,8 +329,30 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max return null; } + // Handle S3 protocol URLs - convert to HTTPS + if (childUrl && typeof childUrl === 'string' && childUrl.startsWith('s3://')) { + // s3://bucket-name/path -> https://bucket-name.s3.amazonaws.com/path + const s3Match = childUrl.match(/^s3:\/\/([^/]+)\/(.*)$/); + if (s3Match) { + const [, bucket, path] = s3Match; + childUrl = `https://${bucket}.s3.amazonaws.com/${path}`; + log.debug(`${indent}Converted S3 URL: ${link.href} -> ${childUrl}`); + } else { + log.warning(`${indent}Skipping malformed S3 URL at index ${idx}: ${childUrl}`); + return null; + } + } + + // If URL is relative, make it absolute using the API URL + if (childUrl && typeof childUrl === 'string' && !childUrl.startsWith('http')) { + const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; + const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); + childUrl = `${basePath}/${childUrl}`; + } + // Validate URL if (!childUrl || typeof childUrl !== 'string' || !childUrl.startsWith('http')) { + log.warning(`${indent}Skipping invalid URL at index ${idx}: ${childUrl}`); return null; } diff --git a/crawler/utils/endpoints.js b/crawler/utils/endpoints.js index c595933..2feb27c 100644 --- a/crawler/utils/endpoints.js +++ b/crawler/utils/endpoints.js @@ -34,6 +34,16 @@ export async function tryCollectionEndpoints(stacCatalog, baseUrl, catalogId, de ? collectionLink.getAbsoluteUrl() : collectionLink.href; + // Handle S3 protocol URLs - convert to HTTPS + if (collectionUrl && collectionUrl.startsWith('s3://')) { + const s3Match = collectionUrl.match(/^s3:\/\/([^/]+)\/(.*)$/); + if (s3Match) { + const [, bucket, path] = s3Match; + collectionUrl = `https://${bucket}.s3.amazonaws.com/${path}`; + log.debug(`${indent}Converted S3 URL: ${collectionLink.href} -> ${collectionUrl}`); + } + } + // Handle relative URLs if (collectionUrl && !collectionUrl.startsWith('http')) { const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/')); diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index becc5bb..922ff2e 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -190,6 +190,20 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul return null; } + // Handle S3 protocol URLs - convert to HTTPS + if (childUrl && typeof childUrl === 'string' && childUrl.startsWith('s3://')) { + // s3://bucket-name/path -> https://bucket-name.s3.amazonaws.com/path + const s3Match = childUrl.match(/^s3:\/\/([^/]+)\/(.*)$/); + if (s3Match) { + const [, bucket, path] = s3Match; + childUrl = `https://${bucket}.s3.amazonaws.com/${path}`; + log.debug(`${indent}Converted S3 URL: ${link.href} -> ${childUrl}`); + } else { + log.warning(`${indent}Skipping malformed S3 URL at index ${idx}: ${childUrl}`); + return null; + } + } + // If URL is relative, make it absolute using the catalog URL if (childUrl && typeof childUrl === 'string' && !childUrl.startsWith('http')) { const baseUrl = request.url.endsWith('/') ? request.url.slice(0, -1) : request.url; From 5ee15b61c160824e853690b15fa07efcb9cf4b25 Mon Sep 17 00:00:00 2001 From: mammutor Date: Tue, 20 Jan 2026 13:09:51 +0100 Subject: [PATCH 59/68] fix(crawler): update Dockerfile to omit development dependencies during npm install - Changed the npm install command to use the --omit=dev flag, ensuring that only production dependencies are installed in the Docker image. --- crawler/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler/Dockerfile b/crawler/Dockerfile index 651de07..3a15606 100644 --- a/crawler/Dockerfile +++ b/crawler/Dockerfile @@ -8,7 +8,7 @@ WORKDIR /app COPY package*.json ./ # Install dependencies -RUN npm install --production +RUN npm install --omit=dev # Copy application files COPY . . From ef3c498bfc5d80cb2e5e8f86445e8d21aba138ce Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 01:38:10 +0100 Subject: [PATCH 60/68] feat(crawler): implement parallel crawling for APIs and catalogs - Added support for parallel crawling of multiple domains in both API and catalog crawlers. - Introduced new configuration options for parallel domains, max requests per minute per domain, and max concurrency per domain. - Enhanced logging to provide detailed statistics on parallel crawling performance and domain distribution. - Refactored existing crawling functions to accommodate the new parallel execution model, improving overall efficiency and throughput. - Updated CLI arguments and configuration to support the new parallel crawling features. --- crawler/apis/api.js | 200 +++++++++++++++++++++--------------- crawler/catalogs/catalog.js | 118 ++++++++++++++++----- crawler/index.js | 5 + crawler/utils/cli.js | 51 ++++++--- crawler/utils/config.js | 35 +++++-- crawler/utils/parallel.js | 186 +++++++++++++++++++++++++++++++++ 6 files changed, 461 insertions(+), 134 deletions(-) create mode 100644 crawler/utils/parallel.js diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 7205675..3d49198 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -1,5 +1,6 @@ /** * @fileoverview API crawling functionality for STAC Index using Crawlee + * Supports parallel crawling of multiple domains simultaneously * @module apis/api */ @@ -7,6 +8,13 @@ import { HttpCrawler, Configuration, log as crawleeLog } from 'crawlee'; import create from 'stac-js'; import { normalizeCollection } from '../utils/normalization.js'; import { handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; +import { + groupByDomain, + executeWithConcurrency, + aggregateStats, + calculateRateLimits, + logDomainStats +} from '../utils/parallel.js'; /** * Batch size for saving collections to database during API crawling @@ -16,14 +24,12 @@ const BATCH_SIZE = 500; /** * Batch size for clearing apis array to free memory - * The apis array is only used for statistics, so we clear it periodically * @type {number} */ const API_CLEAR_BATCH_SIZE = 500; /** * Checks if batch size is reached and flushes if necessary - * Also clears the apis array periodically to free memory * @async * @param {Object} results - Results object containing collections array * @param {Object} log - Logger instance @@ -35,8 +41,6 @@ async function checkAndFlushApi(results, log) { results.stats.collectionsFailed = (results.stats.collectionsFailed || 0) + failed; } - // Clear apis array periodically to free memory - // The apis array is only used for end statistics, which we track in stats object if (results.apis && results.apis.length >= API_CLEAR_BATCH_SIZE) { log.info(`[MEMORY] Clearing ${results.apis.length} APIs from memory`); results.apis.length = 0; @@ -44,39 +48,24 @@ async function checkAndFlushApi(results, log) { } /** - * Note: This crawler is for REAL APIs only. Static catalogs (*.json files) - * are now routed to the catalog crawler in index.js - * - * Crawls STAC APIs to retrieve collection information without fetching items. - * - * @param {string[]} urls - Array of API URLs to crawl - * @param {boolean} isApi - Boolean flag indicating if the URLs are APIs - * @param {Object} config - Configuration object with timeout settings - * @returns {Promise} Results object with collections array and statistics + * Creates and runs a single Crawlee HttpCrawler for a specific domain + * @async + * @param {Array} urls - Array of API URLs for this domain + * @param {string} domain - The domain being crawled + * @param {Object} config - Configuration object + * @returns {Promise} Crawl results with collections and statistics */ -async function crawlApis(urls, isApi, config = {}) { +async function crawlSingleApiDomain(urls, domain, config = {}) { // Use in-memory storage to avoid file lock race conditions under high concurrency Configuration.getGlobalConfig().set('persistStorage', false); - if (!isApi || !Array.isArray(urls) || urls.length === 0) { - return { - collections: [], - stats: { - totalRequests: 0, - successfulRequests: 0, - failedRequests: 0, - collectionsFound: 0, - apisProcessed: 0, - stacCompliant: 0, - nonCompliant: 0 - } - }; - } - const timeoutSecs = config.timeout && config.timeout !== Infinity ? Math.ceil(config.timeout / 1000) : 60; + // Calculate rate limits for this domain + const rateLimits = calculateRateLimits(config.maxRequestsPerMinutePerDomain || 120); + // Store results const results = { collections: [], @@ -99,25 +88,22 @@ async function crawlApis(urls, isApi, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - maxConcurrency: 20, // Limit concurrency to prevent lock file race conditions - maxRequestsPerMinute: 200, // Rate limit to avoid overwhelming targets - // Rate limiting options - maxConcurrency: config.maxConcurrency || 5, - maxRequestsPerMinute: config.maxRequestsPerMinute || 60, - sameDomainDelaySecs: config.sameDomainDelaySecs || 1, + // Per-domain rate limiting + maxConcurrency: config.maxConcurrencyPerDomain || 10, + maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, + sameDomainDelaySecs: rateLimits.sameDomainDelaySecs, maxRequestRetries: config.maxRequestRetries || 3, - // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) + // Accept additional MIME types additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], async requestHandler({ request, json, body, crawler, log }) { results.stats.totalRequests++; const depth = request.userData?.depth || 0; - const indent = ' '.repeat(Math.min(depth, 5)); // Cap indent at 5 levels + const indent = ' '.repeat(Math.min(depth, 5)); // Fallback: manually parse JSON if Crawlee's automatic parsing failed - // This can happen with slow endpoints or unusual response streaming if (!json && body) { try { const bodyStr = typeof body === 'string' ? body : body.toString('utf8'); @@ -129,7 +115,6 @@ async function crawlApis(urls, isApi, config = {}) { } try { - // Route based on request label if (request.label === 'API_ROOT') { await handleApiRoot({ request, json, crawler, log, indent, results, maxDepth }); } else if (request.label === 'API_COLLECTIONS') { @@ -141,7 +126,7 @@ async function crawlApis(urls, isApi, config = {}) { results.stats.successfulRequests++; } catch (error) { log.error(`${indent}Error handling ${request.label} at ${request.url}: ${error.message}`); - throw error; // Re-throw to trigger failedRequestHandler + throw error; } }, @@ -150,7 +135,6 @@ async function crawlApis(urls, isApi, config = {}) { const indent = ' '; const apiId = request.userData?.apiId || 'unknown'; - // Log detailed failure information if (error.message.includes('STAC validation')) { log.info(`${indent}[STAC VALIDATION FAILED] ${apiId} at ${request.url}`); log.info(`${indent} Reason: ${error.message}`); @@ -176,7 +160,7 @@ async function crawlApis(urls, isApi, config = {}) { url: url, label: 'API_ROOT', userData: { - apiId: `api-${index}`, + apiId: `${domain}-api-${index}`, apiUrl: url, depth: 0 } @@ -184,44 +168,113 @@ async function crawlApis(urls, isApi, config = {}) { await crawler.addRequests(initialRequests); - console.log(`\nStarting Crawlee crawler with ${initialRequests.length} APIs...\n`); + console.log(` [${domain}] Starting API crawl with ${initialRequests.length} APIs (max ${rateLimits.maxRequestsPerMinute} req/min)`); await crawler.run(); // Flush any remaining collections to database - console.log('\nFlushing remaining API collections to database...'); const finalFlush = await flushCollectionsToDb(results, crawleeLog, true); results.stats.collectionsSaved += finalFlush.saved; results.stats.collectionsFailed += finalFlush.failed; - // Clear apis array to free memory (we don't need them after crawl) + // Clear apis array to free memory results.apis.length = 0; - console.log('\nAPI Crawl Statistics:'); - console.log(` Total Requests: ${results.stats.totalRequests}`); - console.log(` Successful: ${results.stats.successfulRequests}`); - console.log(` Failed: ${results.stats.failedRequests}`); - console.log(` STAC Compliant: ${results.stats.stacCompliant}`); - console.log(` Non-Compliant: ${results.stats.nonCompliant}`); - console.log(` APIs Processed: ${results.stats.apisProcessed}`); - console.log(` Collections Found: ${results.stats.collectionsFound}`); - console.log(` Collections Saved to DB: ${results.stats.collectionsSaved}`); - console.log(` Collections Failed: ${results.stats.collectionsFailed}\n`); + console.log(` [${domain}] Finished: ${results.stats.collectionsFound} collections, ${results.stats.successfulRequests}/${results.stats.totalRequests} requests`); return results; } +/** + * Crawls STAC APIs to retrieve collection information without fetching items. + * Groups APIs by domain and crawls multiple domains simultaneously. + * + * @param {string[]} urls - Array of API URLs to crawl + * @param {boolean} isApi - Boolean flag indicating if the URLs are APIs + * @param {Object} config - Configuration object with timeout settings + * @returns {Promise} Results object with collections array and statistics + */ +async function crawlApis(urls, isApi, config = {}) { + if (!isApi || !Array.isArray(urls) || urls.length === 0) { + return { + collections: [], + stats: { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + } + }; + } + + // Convert URLs to objects for groupByDomain + const urlObjects = urls.map(url => ({ url })); + const domainMap = groupByDomain(urlObjects); + + // Convert back to URL arrays per domain + const domainUrlMap = new Map(); + for (const [domain, objs] of domainMap.entries()) { + domainUrlMap.set(domain, objs.map(o => o.url)); + } + + // Log domain distribution + logDomainStats(domainMap, 'APIs'); + + // Number of domains to crawl in parallel (default: 5) + const parallelDomains = config.parallelDomains || 5; + const maxRequestsPerMinutePerDomain = config.maxRequestsPerMinutePerDomain || 120; + + console.log(`\n=== Parallel API Crawling Configuration ===`); + console.log(`Parallel domains: ${parallelDomains}`); + console.log(`Max requests/min per domain: ${maxRequestsPerMinutePerDomain}`); + console.log(`Theoretical max throughput: ${parallelDomains * maxRequestsPerMinutePerDomain} req/min across all domains`); + console.log(`============================================\n`); + + // Create tasks for each domain + const domainTasks = Array.from(domainUrlMap.entries()).map(([domain, domainUrls]) => { + return () => crawlSingleApiDomain(domainUrls, domain, config); + }); + + console.log(`Starting parallel API crawl of ${domainUrlMap.size} domains (${parallelDomains} at a time)...\n`); + + // Execute with concurrency limit + const allResults = await executeWithConcurrency( + domainTasks, + parallelDomains, + (completed, total) => { + console.log(`\n>>> Domain progress: ${completed}/${total} domains completed <<<\n`); + } + ); + + // Aggregate all statistics + const aggregatedStats = aggregateStats(allResults); + + console.log('\n=== Parallel API Crawl Statistics ==='); + console.log(` Domains Processed: ${domainUrlMap.size}`); + console.log(` Total Requests: ${aggregatedStats.totalRequests}`); + console.log(` Successful: ${aggregatedStats.successfulRequests}`); + console.log(` Failed: ${aggregatedStats.failedRequests}`); + console.log(` STAC Compliant: ${aggregatedStats.stacCompliant}`); + console.log(` Non-Compliant: ${aggregatedStats.nonCompliant}`); + console.log(` APIs Processed: ${aggregatedStats.apisProcessed}`); + console.log(` Collections Found: ${aggregatedStats.collectionsFound}`); + console.log(` Collections Saved to DB: ${aggregatedStats.collectionsSaved}`); + console.log(` Collections Failed: ${aggregatedStats.collectionsFailed}`); + console.log('=====================================\n'); + + return { + collections: [], + apis: [], + stats: aggregatedStats + }; +} + /** * Handles API root endpoint - validates STAC, discovers collections endpoints * @async - * @param {Object} context - Request handler context - * @param {Object} context.request - Crawlee request object - * @param {Object} context.json - Parsed JSON response - * @param {Object} context.crawler - Crawlee crawler instance - * @param {Object} context.log - Logger instance - * @param {string} context.indent - Indentation for logging - * @param {Object} context.results - Results object to store data - * @param {number} context.maxDepth - Maximum recursion depth (0 = unlimited) */ async function handleApiRoot({ request, json, crawler, log, indent, results, maxDepth = 10 }) { const apiId = request.userData?.apiId || 'unknown'; @@ -230,15 +283,11 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max log.info(`${indent}Processing API: ${apiId} at ${apiUrl} (depth: ${depth})`); - // Defensive check: ensure json is valid before passing to stac-js if (!json || typeof json !== 'object') { log.warning(`${indent}Invalid JSON response for ${apiId} at ${request.url}`); throw new Error('Invalid JSON response: null or not an object'); } - // Validate with stac-js - // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL - // Using migrate=false to avoid issues with stac-migrate and newer STAC versions let stacObj; try { stacObj = create(json, false); @@ -269,9 +318,8 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max results.stats.collectionsFound++; log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); - // Check if we should flush to database await checkAndFlushApi(results, log); - return; // Single collection, no need to look for more + return; } // Try to find collections endpoint using stac-js @@ -280,7 +328,6 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max if (typeof stacObj.getApiCollectionsLink === 'function') { const collectionsLink = stacObj.getApiCollectionsLink(); if (collectionsLink && collectionsLink.href) { - // Direkt die href verwenden - diese ist bereits absolut im JSON collectionsEndpoint = collectionsLink.href; log.info(`${indent}Found collections link via stac-js: ${collectionsEndpoint}`); } @@ -310,7 +357,6 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max if (childLinks.length > 0) { log.info(`${indent}Found ${childLinks.length} child catalog links`); - // Check if we've reached maximum depth const nextDepth = depth + 1; if (maxDepth > 0 && nextDepth > maxDepth) { log.warning(`${indent}Skipping ${childLinks.length} child catalogs - max depth (${maxDepth}) reached`); @@ -331,7 +377,6 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max // Handle S3 protocol URLs - convert to HTTPS if (childUrl && typeof childUrl === 'string' && childUrl.startsWith('s3://')) { - // s3://bucket-name/path -> https://bucket-name.s3.amazonaws.com/path const s3Match = childUrl.match(/^s3:\/\/([^/]+)\/(.*)$/); if (s3Match) { const [, bucket, path] = s3Match; @@ -356,7 +401,6 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max return null; } - // Check if this is a collection link or child catalog const rel = link.rel || ''; const label = rel === 'child' || rel === 'item' ? 'API_ROOT' : 'API_COLLECTION'; @@ -384,19 +428,10 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max /** * Handles individual API collection endpoint * @async - * @param {Object} context - Request handler context - * @param {Object} context.request - Crawlee request object - * @param {Object} context.json - Parsed JSON response - * @param {Object} context.crawler - Crawlee crawler instance - * @param {Object} context.log - Logger instance - * @param {string} context.indent - Indentation for logging - * @param {Object} context.results - Results object to store data */ async function handleApiCollection({ request, json, crawler, log, indent, results }) { const apiId = request.userData?.apiId || 'unknown'; - // Validate with stac-js - // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL let stacObj; try { stacObj = create(json, false); @@ -407,7 +442,6 @@ async function handleApiCollection({ request, json, crawler, log, indent, result results.stats.collectionsFound++; log.info(`${indent}Extracted collection: ${collection.id} - ${collection.title}`); - // Check if we should flush to database await checkAndFlushApi(results, log); } else { log.warning(`${indent}Expected collection but got: ${json.type || 'unknown type'}`); diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index e9002ea..09359a2 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -1,19 +1,28 @@ /** * @fileoverview Catalog crawling functionality for STAC Index using Crawlee + * Supports parallel crawling of multiple domains simultaneously * @module catalogs/catalog */ import { HttpCrawler, log as crawleeLog, Configuration } from 'crawlee'; import { handleCatalog, handleCollections, flushCollectionsToDb } from '../utils/handlers.js'; +import { + groupByDomain, + executeWithConcurrency, + aggregateStats, + calculateRateLimits, + logDomainStats +} from '../utils/parallel.js'; /** - * Creates and runs a Crawlee HttpCrawler to crawl STAC catalogs + * Creates and runs a single Crawlee HttpCrawler for a specific domain * @async - * @param {Array} initialCatalogs - Array of catalog objects to start crawling from - * @param {Object} config - Configuration object with timeout and depth settings + * @param {Array} catalogs - Array of catalog objects for this domain + * @param {string} domain - The domain being crawled + * @param {Object} config - Configuration object * @returns {Promise} Crawl results with collections and statistics */ -async function crawlCatalogs(initialCatalogs, config = {}) { +async function crawlSingleDomain(catalogs, domain, config = {}) { // Use in-memory storage to avoid file lock race conditions under high concurrency Configuration.getGlobalConfig().set('persistStorage', false); @@ -21,6 +30,9 @@ async function crawlCatalogs(initialCatalogs, config = {}) { ? Math.ceil(config.timeout / 1000) : 60; + // Calculate rate limits for this domain + const rateLimits = calculateRateLimits(config.maxRequestsPerMinutePerDomain || 120); + // Store results const results = { collections: [], @@ -41,10 +53,10 @@ async function crawlCatalogs(initialCatalogs, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - // Rate limiting options - prevents memory buildup from queue overflow - maxConcurrency: config.maxConcurrency || 5, - maxRequestsPerMinute: config.maxRequestsPerMinute || 60, - sameDomainDelaySecs: config.sameDomainDelaySecs || 1, + // Per-domain rate limiting + maxConcurrency: config.maxConcurrencyPerDomain || 10, + maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, + sameDomainDelaySecs: rateLimits.sameDomainDelaySecs, maxRequestRetries: config.maxRequestRetries || 3, // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) @@ -56,7 +68,6 @@ async function crawlCatalogs(initialCatalogs, config = {}) { const indent = ' '.repeat(depth); // Fallback: manually parse JSON if Crawlee's automatic parsing failed - // This can happen with slow endpoints or unusual response streaming if (!json && body) { try { const bodyStr = typeof body === 'string' ? body : body.toString('utf8'); @@ -78,7 +89,7 @@ async function crawlCatalogs(initialCatalogs, config = {}) { results.stats.successfulRequests++; } catch (error) { log.error(`${indent}Error handling ${request.label} at ${request.url}: ${error.message}`); - throw error; // Re-throw to trigger failedRequestHandler + throw error; } }, @@ -88,7 +99,6 @@ async function crawlCatalogs(initialCatalogs, config = {}) { const indent = ' '.repeat(depth); const catalogId = request.userData?.catalogId || 'unknown'; - // Log detailed failure information if (error.message.includes('STAC validation')) { log.info(`${indent}[STAC VALIDATION FAILED] ${catalogId} at ${request.url}`); log.info(`${indent} Reason: ${error.message}`); @@ -109,8 +119,8 @@ async function crawlCatalogs(initialCatalogs, config = {}) { } }); - // Seed the crawler with initial catalog requests - const initialRequests = initialCatalogs.map(catalog => ({ + // Seed the crawler with catalog requests for this domain + const initialRequests = catalogs.map(catalog => ({ url: catalog.url, label: 'CATALOG', userData: { @@ -122,30 +132,84 @@ async function crawlCatalogs(initialCatalogs, config = {}) { await crawler.addRequests(initialRequests); - console.log(`\nStarting Crawlee crawler with ${initialRequests.length} initial catalogs...\n`); + console.log(` [${domain}] Starting crawl with ${initialRequests.length} catalogs (max ${rateLimits.maxRequestsPerMinute} req/min)`); await crawler.run(); // Flush any remaining collections to database - console.log('\nFlushing remaining collections to database...'); const finalFlush = await flushCollectionsToDb(results, crawleeLog, true); results.stats.collectionsSaved += finalFlush.saved; results.stats.collectionsFailed += finalFlush.failed; - // Clear catalogs array to free memory (we don't need them after crawl) + // Clear catalogs array to free memory results.catalogs.length = 0; - console.log('\nCrawl Statistics:'); - console.log(` Total Requests: ${results.stats.totalRequests}`); - console.log(` Successful: ${results.stats.successfulRequests}`); - console.log(` Failed: ${results.stats.failedRequests}`); - console.log(` STAC Compliant: ${results.stats.stacCompliant}`); - console.log(` Non-Compliant: ${results.stats.nonCompliant}`); - console.log(` Catalogs Processed: ${results.stats.catalogsProcessed}`); - console.log(` Collections Found: ${results.stats.collectionsFound}`); - console.log(` Collections Saved to DB: ${results.stats.collectionsSaved}`); - console.log(` Collections Failed: ${results.stats.collectionsFailed}\n`); + console.log(` [${domain}] Finished: ${results.stats.collectionsFound} collections, ${results.stats.successfulRequests}/${results.stats.totalRequests} requests`); return results; } -export { crawlCatalogs }; \ No newline at end of file +/** + * Creates and runs parallel Crawlee HttpCrawlers to crawl STAC catalogs + * Groups catalogs by domain and crawls multiple domains simultaneously + * @async + * @param {Array} initialCatalogs - Array of catalog objects to start crawling from + * @param {Object} config - Configuration object with timeout, depth, and parallel settings + * @returns {Promise} Crawl results with collections and statistics + */ +async function crawlCatalogs(initialCatalogs, config = {}) { + // Group catalogs by domain + const domainMap = groupByDomain(initialCatalogs); + + // Log domain distribution + logDomainStats(domainMap, 'catalogs'); + + // Number of domains to crawl in parallel (default: 5) + const parallelDomains = config.parallelDomains || 5; + const maxRequestsPerMinutePerDomain = config.maxRequestsPerMinutePerDomain || 120; + + console.log(`\n=== Parallel Catalog Crawling Configuration ===`); + console.log(`Parallel domains: ${parallelDomains}`); + console.log(`Max requests/min per domain: ${maxRequestsPerMinutePerDomain}`); + console.log(`Theoretical max throughput: ${parallelDomains * maxRequestsPerMinutePerDomain} req/min across all domains`); + console.log(`===============================================\n`); + + // Create tasks for each domain + const domainTasks = Array.from(domainMap.entries()).map(([domain, catalogs]) => { + return () => crawlSingleDomain(catalogs, domain, config); + }); + + console.log(`Starting parallel crawl of ${domainMap.size} domains (${parallelDomains} at a time)...\n`); + + // Execute with concurrency limit + const allResults = await executeWithConcurrency( + domainTasks, + parallelDomains, + (completed, total) => { + console.log(`\n>>> Domain progress: ${completed}/${total} domains completed <<<\n`); + } + ); + + // Aggregate all statistics + const aggregatedStats = aggregateStats(allResults); + + console.log('\n=== Parallel Crawl Statistics ==='); + console.log(` Domains Processed: ${domainMap.size}`); + console.log(` Total Requests: ${aggregatedStats.totalRequests}`); + console.log(` Successful: ${aggregatedStats.successfulRequests}`); + console.log(` Failed: ${aggregatedStats.failedRequests}`); + console.log(` STAC Compliant: ${aggregatedStats.stacCompliant}`); + console.log(` Non-Compliant: ${aggregatedStats.nonCompliant}`); + console.log(` Catalogs Processed: ${aggregatedStats.catalogsProcessed}`); + console.log(` Collections Found: ${aggregatedStats.collectionsFound}`); + console.log(` Collections Saved to DB: ${aggregatedStats.collectionsSaved}`); + console.log(` Collections Failed: ${aggregatedStats.collectionsFailed}`); + console.log('=================================\n'); + + return { + collections: [], + catalogs: [], + stats: aggregatedStats + }; +} + +export { crawlCatalogs }; diff --git a/crawler/index.js b/crawler/index.js index d7d6b1e..31f4840 100644 --- a/crawler/index.js +++ b/crawler/index.js @@ -49,6 +49,11 @@ export const crawler = async () => { console.log(`Max APIs: ${config.maxApis === 0 ? 'unlimited' : config.maxApis} (debugging limit)`); console.log(`Timeout: ${config.timeout === Infinity ? 'unlimited' : config.timeout + 'ms'}`); console.log(`Max Depth: ${config.maxDepth === 0 ? 'unlimited' : config.maxDepth} levels`); + console.log('--- Parallel Crawling ---'); + console.log(`Parallel Domains: ${config.parallelDomains}`); + console.log(`Max Requests/Min per Domain: ${config.maxRequestsPerMinutePerDomain}`); + console.log(`Max Concurrency per Domain: ${config.maxConcurrencyPerDomain}`); + console.log(`Theoretical Max Throughput: ${config.parallelDomains * config.maxRequestsPerMinutePerDomain} req/min`); console.log('==================================\n'); const response = await axios.get(targetUrl); diff --git a/crawler/utils/cli.js b/crawler/utils/cli.js index c7246cf..2cf0d2c 100644 --- a/crawler/utils/cli.js +++ b/crawler/utils/cli.js @@ -33,6 +33,13 @@ export function parseCliArgs() { config.sameDomainDelaySecs = parseFloat(args[++i]); } else if (arg === '--max-retries') { config.maxRequestRetries = parseInt(args[++i], 10); + // New parallel crawling options + } else if (arg === '--parallel-domains' || arg === '-p') { + config.parallelDomains = parseInt(args[++i], 10); + } else if (arg === '--rpm-per-domain') { + config.maxRequestsPerMinutePerDomain = parseInt(args[++i], 10); + } else if (arg === '--concurrency-per-domain') { + config.maxConcurrencyPerDomain = parseInt(args[++i], 10); } else if (arg === '--help' || arg === '-h') { printHelp(); process.exit(0); @@ -56,7 +63,14 @@ STAC Crawler Configuration Options: Note: Limits are for debugging purposes only -t, --timeout Timeout for each crawl operation in ms (default: 30000) - Rate Limiting Options: + Parallel Crawling Options (NEW): + -p, --parallel-domains Number of domains to crawl in parallel (default: 5) + Each domain gets its own crawler instance + --rpm-per-domain Max requests per minute PER DOMAIN (default: 120) + Total throughput = parallel-domains × rpm-per-domain + --concurrency-per-domain Max concurrent requests per domain (default: 10) + + Legacy Rate Limiting Options (still supported): --max-concurrency Maximum concurrent requests (default: 5) --requests-per-minute, --rpm Maximum requests per minute (default: 60) --domain-delay Delay between requests to same domain (default: 1) @@ -67,21 +81,32 @@ STAC Crawler Configuration Options: -h, --help Show this help message Environment Variables: - CRAWL_MODE Same as --mode - MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) - MAX_APIS Same as --max-apis (use 0 for unlimited) - TIMEOUT_MS Same as --timeout - MAX_CONCURRENCY Same as --max-concurrency - MAX_REQUESTS_PER_MINUTE Same as --requests-per-minute - SAME_DOMAIN_DELAY_SECS Same as --domain-delay - MAX_REQUEST_RETRIES Same as --max-retries - MAX_DEPTH Same as --max-depth (use 0 for unlimited) + CRAWL_MODE Same as --mode + MAX_CATALOGS Same as --max-catalogs (use 0 for unlimited) + MAX_APIS Same as --max-apis (use 0 for unlimited) + TIMEOUT_MS Same as --timeout + PARALLEL_DOMAINS Same as --parallel-domains + MAX_REQUESTS_PER_MINUTE_PER_DOMAIN Same as --rpm-per-domain + MAX_CONCURRENCY_PER_DOMAIN Same as --concurrency-per-domain + MAX_CONCURRENCY Same as --max-concurrency + MAX_REQUESTS_PER_MINUTE Same as --requests-per-minute + SAME_DOMAIN_DELAY_SECS Same as --domain-delay + MAX_REQUEST_RETRIES Same as --max-retries + MAX_DEPTH Same as --max-depth (use 0 for unlimited) Examples: + # Basic usage node index.js --mode catalogs --max-catalogs 20 node index.js -m apis -a 10 -t 60000 - node index.js -m both -c 0 -a 0 # Unlimited mode (no debugging limits) - node index.js -m apis -d 5 # Limit nesting depth to 5 levels - CRAWL_MODE=both MAX_CATALOGS=50 node index.js + + # Parallel crawling (recommended for performance) + node index.js -p 5 --rpm-per-domain 120 # 5 domains × 120 req/min = 600 req/min max + node index.js -p 10 --rpm-per-domain 60 # 10 domains × 60 req/min = 600 req/min max + + # Unlimited mode (no debugging limits) + node index.js -m both -c 0 -a 0 + + # With environment variables + PARALLEL_DOMAINS=5 MAX_REQUESTS_PER_MINUTE_PER_DOMAIN=120 node index.js `); } diff --git a/crawler/utils/config.js b/crawler/utils/config.js index e0ce5ca..3858b7c 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -43,9 +43,15 @@ function getConfig() { maxApis: 5, // Maximum number of APIs to crawl timeout: 30000, // Timeout in milliseconds (30 seconds) maxDepth: 10, // Maximum recursion depth for nested catalogs (0 = unlimited) - // Rate limiting options (Crawlee) - maxConcurrency: 5, // Maximum number of concurrent requests - maxRequestsPerMinute: 60, // Maximum requests per minute + + // NEW: Parallel crawling options (recommended for performance) + parallelDomains: 5, // Number of domains to crawl in parallel + maxRequestsPerMinutePerDomain: 120, // Max requests per minute PER domain + maxConcurrencyPerDomain: 10, // Max concurrent requests per domain + + // Legacy rate limiting options (still supported but parallel options are preferred) + maxConcurrency: 5, // Maximum number of concurrent requests (global) + maxRequestsPerMinute: 60, // Maximum requests per minute (global) sameDomainDelaySecs: 1, // Delay between requests to the same domain maxRequestRetries: 3 // Maximum number of retries for failed requests }; @@ -61,7 +67,16 @@ function getConfig() { (process.env.TIMEOUT_MS ? parseInt(process.env.TIMEOUT_MS, 10) : defaults.timeout), maxDepth: cliArgs.maxDepth !== undefined ? cliArgs.maxDepth : (process.env.MAX_DEPTH ? parseInt(process.env.MAX_DEPTH, 10) : defaults.maxDepth), - // Rate limiting options + + // NEW: Parallel crawling options + parallelDomains: cliArgs.parallelDomains !== undefined ? cliArgs.parallelDomains : + (process.env.PARALLEL_DOMAINS ? parseInt(process.env.PARALLEL_DOMAINS, 10) : defaults.parallelDomains), + maxRequestsPerMinutePerDomain: cliArgs.maxRequestsPerMinutePerDomain !== undefined ? cliArgs.maxRequestsPerMinutePerDomain : + (process.env.MAX_REQUESTS_PER_MINUTE_PER_DOMAIN ? parseInt(process.env.MAX_REQUESTS_PER_MINUTE_PER_DOMAIN, 10) : defaults.maxRequestsPerMinutePerDomain), + maxConcurrencyPerDomain: cliArgs.maxConcurrencyPerDomain !== undefined ? cliArgs.maxConcurrencyPerDomain : + (process.env.MAX_CONCURRENCY_PER_DOMAIN ? parseInt(process.env.MAX_CONCURRENCY_PER_DOMAIN, 10) : defaults.maxConcurrencyPerDomain), + + // Legacy rate limiting options maxConcurrency: cliArgs.maxConcurrency !== undefined ? cliArgs.maxConcurrency : (process.env.MAX_CONCURRENCY ? parseInt(process.env.MAX_CONCURRENCY, 10) : defaults.maxConcurrency), maxRequestsPerMinute: cliArgs.maxRequestsPerMinute !== undefined ? cliArgs.maxRequestsPerMinute : @@ -79,15 +94,13 @@ function getConfig() { process.exit(1); } - // NOTE: Removed automatic Infinity override for 'catalogs' mode - // Users can now control limits via CLI args even when mode is 'catalogs' - // Set maxCatalogs or maxApis to 0 for unlimited crawling (for debugging purposes) - - // Validate numeric values (0 means unlimited) + // Validate numeric values (0 means unlimited for some options) if ((config.maxCatalogs < 0) || (config.maxApis < 0) || - (config.timeout !== Infinity && config.timeout < 0)) { - console.error('Numeric configuration values must be non-negative (use 0 for unlimited)'); + (config.timeout !== Infinity && config.timeout < 0) || + (config.parallelDomains < 1) || + (config.maxRequestsPerMinutePerDomain < 1)) { + console.error('Invalid configuration: parallelDomains and maxRequestsPerMinutePerDomain must be >= 1'); process.exit(1); } diff --git a/crawler/utils/parallel.js b/crawler/utils/parallel.js new file mode 100644 index 0000000..c8711e6 --- /dev/null +++ b/crawler/utils/parallel.js @@ -0,0 +1,186 @@ +/** + * @fileoverview Parallel execution utilities for domain-based crawling + * Allows crawling multiple domains simultaneously while respecting per-domain rate limits + * @module utils/parallel + */ + +/** + * Extracts the domain from a URL + * @param {string} url - URL to extract domain from + * @returns {string} The domain (hostname) of the URL + */ +export function getDomain(url) { + try { + const urlObj = new URL(url); + return urlObj.hostname; + } catch { + return 'unknown'; + } +} + +/** + * Groups items by their URL domain + * @param {Array} items - Array of objects with url property + * @returns {Map>} Map of domain -> items + */ +export function groupByDomain(items) { + const domainMap = new Map(); + + for (const item of items) { + const domain = getDomain(item.url); + if (!domainMap.has(domain)) { + domainMap.set(domain, []); + } + domainMap.get(domain).push(item); + } + + return domainMap; +} + +/** + * Creates batches of domains for parallel processing + * @param {Map} domainMap - Map of domain -> items + * @param {number} batchSize - Number of domains to process in parallel + * @returns {Array>} Array of batches, each containing [domain, items] pairs + */ +export function createDomainBatches(domainMap, batchSize = 5) { + const entries = Array.from(domainMap.entries()); + const batches = []; + + for (let i = 0; i < entries.length; i += batchSize) { + batches.push(entries.slice(i, i + batchSize)); + } + + return batches; +} + +/** + * Aggregates statistics from multiple crawler results + * @param {Array} results - Array of result objects with stats + * @returns {Object} Aggregated statistics + */ +export function aggregateStats(results) { + const aggregated = { + totalRequests: 0, + successfulRequests: 0, + failedRequests: 0, + collectionsFound: 0, + collectionsSaved: 0, + collectionsFailed: 0, + catalogsProcessed: 0, + apisProcessed: 0, + stacCompliant: 0, + nonCompliant: 0 + }; + + for (const result of results) { + if (!result || !result.stats) continue; + + for (const key of Object.keys(aggregated)) { + if (typeof result.stats[key] === 'number') { + aggregated[key] += result.stats[key]; + } + } + } + + return aggregated; +} + +/** + * Executes async functions in parallel with a concurrency limit + * @param {Array} tasks - Array of async functions to execute + * @param {number} concurrency - Maximum number of tasks to run in parallel + * @param {Function} onProgress - Optional callback for progress updates + * @returns {Promise} Array of results from all tasks + */ +export async function executeWithConcurrency(tasks, concurrency, onProgress = null) { + const results = []; + let completed = 0; + let running = 0; + let index = 0; + + return new Promise((resolve) => { + const runNext = async () => { + if (index >= tasks.length) { + if (running === 0) { + resolve(results); + } + return; + } + + const currentIndex = index++; + running++; + + try { + const result = await tasks[currentIndex](); + results[currentIndex] = result; + } catch (error) { + results[currentIndex] = { error: error.message, stats: {} }; + } + + running--; + completed++; + + if (onProgress) { + onProgress(completed, tasks.length); + } + + runNext(); + }; + + // Start initial batch + const initialBatch = Math.min(concurrency, tasks.length); + for (let i = 0; i < initialBatch; i++) { + runNext(); + } + + // Handle empty tasks array + if (tasks.length === 0) { + resolve(results); + } + }); +} + +/** + * Calculates optimal rate limiting based on max requests per minute + * @param {number} maxRequestsPerMinute - Maximum requests per minute (per domain) + * @returns {Object} Rate limiting configuration + */ +export function calculateRateLimits(maxRequestsPerMinute = 120) { + // Calculate delay between requests to stay under the limit + // Add 10% safety margin + const safeLimit = Math.floor(maxRequestsPerMinute * 0.9); + const delayMs = Math.ceil(60000 / safeLimit); + const delaySecs = delayMs / 1000; + + return { + maxRequestsPerMinute: safeLimit, + sameDomainDelaySecs: Math.max(delaySecs, 0.5), // Minimum 0.5s delay + // With 120 req/min target, we get ~108 effective req/min with safety margin + // That's ~1.8 req/sec or ~0.55s between requests + }; +} + +/** + * Logs domain statistics for debugging + * @param {Map} domainMap - Map of domain -> items + * @param {string} itemType - Type of items (e.g., 'catalogs', 'APIs') + */ +export function logDomainStats(domainMap, itemType = 'items') { + console.log(`\n=== Domain Distribution for ${itemType} ===`); + console.log(`Total domains: ${domainMap.size}`); + + const sorted = Array.from(domainMap.entries()) + .sort((a, b) => b[1].length - a[1].length); + + // Show top 10 domains + const top = sorted.slice(0, 10); + for (const [domain, items] of top) { + console.log(` ${domain}: ${items.length} ${itemType}`); + } + + if (sorted.length > 10) { + console.log(` ... and ${sorted.length - 10} more domains`); + } + console.log(''); +} From 14ff676dd879fb443e9ffb3391b706a6544fdee0 Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 01:42:59 +0100 Subject: [PATCH 61/68] fix(crawler): enhance normalization of collection metadata extraction - Improved the `normalizeCollection` function to utilize raw data from stac-js objects for more robust metadata extraction. - Added fallback mechanisms for bounding box, temporal extent, self URL, and other properties to ensure reliable data retrieval from both stac-js methods and raw data. - Updated the handling of collection properties to prioritize raw data when available, enhancing overall data integrity. --- crawler/utils/normalization.js | 52 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/crawler/utils/normalization.js b/crawler/utils/normalization.js index b58f5b0..34cff2a 100644 --- a/crawler/utils/normalization.js +++ b/crawler/utils/normalization.js @@ -70,14 +70,37 @@ export function normalizeCatalog(catalog, index) { * @returns {Object} Normalized collection object */ export function normalizeCollection(colObj, index) { - // Use stac-js methods for robust metadata extraction - const bbox = typeof colObj.getBoundingBox === 'function' - ? colObj.getBoundingBox() - : (colObj.extent?.spatial?.bbox?.[0] || null); + // Get raw data from stac-js object if available + // stac-js stores the original data in toJSON() or we can access it directly + const rawData = typeof colObj.toJSON === 'function' ? colObj.toJSON() : colObj; - const temporal = typeof colObj.getTemporalExtent === 'function' - ? colObj.getTemporalExtent() - : (colObj.extent?.temporal?.interval?.[0] || null); + // Extract bbox: try stac-js method first, then fallback to raw data + let bbox = null; + if (typeof colObj.getBoundingBox === 'function') { + bbox = colObj.getBoundingBox(); + } + // Fallback to raw data if stac-js method returned null/undefined + if (!bbox && rawData?.extent?.spatial?.bbox?.[0]) { + bbox = rawData.extent.spatial.bbox[0]; + } + // Final fallback: direct access on colObj + if (!bbox && colObj?.extent?.spatial?.bbox?.[0]) { + bbox = colObj.extent.spatial.bbox[0]; + } + + // Extract temporal: try stac-js method first, then fallback to raw data + let temporal = null; + if (typeof colObj.getTemporalExtent === 'function') { + temporal = colObj.getTemporalExtent(); + } + // Fallback to raw data if stac-js method returned null/undefined + if (!temporal && rawData?.extent?.temporal?.interval?.[0]) { + temporal = rawData.extent.temporal.interval[0]; + } + // Final fallback: direct access on colObj + if (!temporal && colObj?.extent?.temporal?.interval?.[0]) { + temporal = colObj.extent.temporal.interval[0]; + } // Get self URL using stac-js link navigation let selfUrl = null; @@ -87,17 +110,22 @@ export function normalizeCollection(colObj, index) { const selfLink = colObj.links.find(l => l.rel === 'self'); selfUrl = selfLink?.href || null; } + // Fallback to raw data for URL + if (!selfUrl && rawData?.links) { + const selfLink = rawData.links.find(l => l.rel === 'self'); + selfUrl = selfLink?.href || null; + } return { index, - id: colObj.id || 'Unknown', + id: colObj.id || rawData?.id || 'Unknown', url: selfUrl, - title: colObj.title || null, - description: colObj.description || colObj.summary || null, + title: colObj.title || rawData?.title || null, + description: colObj.description || colObj.summary || rawData?.description || rawData?.summary || null, bbox, temporal, - license: colObj.license || null, - keywords: colObj.keywords || [] + license: colObj.license || rawData?.license || null, + keywords: colObj.keywords || rawData?.keywords || [] }; } From 735708962465859ff2a1b2c51a51f4c9ac86461b Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 01:47:22 +0100 Subject: [PATCH 62/68] fix(crawler): enhance normalization of collection to include additional fields for database insertion - Updated the `normalizeCollection` function to extract and preserve additional fields such as links, summaries, and extensions from collection objects. - Ensured compatibility with both stac-js and raw data formats for comprehensive metadata extraction. - Improved overall data integrity by including all necessary fields for database insertion. --- crawler/utils/normalization.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/crawler/utils/normalization.js b/crawler/utils/normalization.js index 34cff2a..444c938 100644 --- a/crawler/utils/normalization.js +++ b/crawler/utils/normalization.js @@ -65,9 +65,10 @@ export function normalizeCatalog(catalog, index) { /** * Normalizes a collection object using stac-js methods for metadata extraction + * Preserves all fields needed for database insertion including summaries, extensions, etc. * @param {Object} colObj - Collection object (stac-js or plain object) * @param {number} index - Index position - * @returns {Object} Normalized collection object + * @returns {Object} Normalized collection object with all fields for db.js */ export function normalizeCollection(colObj, index) { // Get raw data from stac-js object if available @@ -116,6 +117,20 @@ export function normalizeCollection(colObj, index) { selfUrl = selfLink?.href || null; } + // Extract links array (needed for source_url extraction in db.js) + let links = null; + if (Array.isArray(colObj.links)) { + // Convert stac-js link objects to plain objects if needed + links = colObj.links.map(l => ({ + rel: l.rel, + href: l.href, + type: l.type, + title: l.title + })); + } else if (Array.isArray(rawData?.links)) { + links = rawData.links; + } + return { index, id: colObj.id || rawData?.id || 'Unknown', @@ -125,7 +140,16 @@ export function normalizeCollection(colObj, index) { bbox, temporal, license: colObj.license || rawData?.license || null, - keywords: colObj.keywords || rawData?.keywords || [] + keywords: colObj.keywords || rawData?.keywords || [], + + // Additional fields needed for db.js - pass through from raw data + links, + stac_version: colObj.stac_version || rawData?.stac_version || null, + type: colObj.type || rawData?.type || 'Collection', + summaries: colObj.summaries || rawData?.summaries || null, + stac_extensions: colObj.stac_extensions || rawData?.stac_extensions || [], + providers: colObj.providers || rawData?.providers || [], + assets: colObj.assets || rawData?.assets || null }; } From c48840c64bbd1002657c51371069a4e6c315da78 Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 02:55:54 +0100 Subject: [PATCH 63/68] fix(crawler): increase max concurrency for improved throughput in API and catalog crawlers - Updated the maxConcurrencyPerDomain setting from 10 to 20 to enhance performance and avoid bottlenecks during crawling. - Adjusted logging to provide detailed information on the number of APIs and catalogs being crawled, including concurrency and rate limits. - Modified rate limiting calculations to ensure minimal delay between requests, allowing for better handling of slow responses. --- crawler/apis/api.js | 10 +++++++--- crawler/catalogs/catalog.js | 10 +++++++--- crawler/utils/config.js | 2 +- crawler/utils/parallel.js | 18 +++++++++--------- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 3d49198..51d5364 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -89,8 +89,11 @@ async function crawlSingleApiDomain(urls, domain, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - // Per-domain rate limiting - maxConcurrency: config.maxConcurrencyPerDomain || 10, + // Per-domain rate limiting - HIGH concurrency for throughput + // maxConcurrency should be higher than req/sec to avoid bottlenecks + // 120 req/min = 2 req/sec, so we need at least 2-3 concurrent requests + // Using 20 to handle slow responses and maintain throughput + maxConcurrency: config.maxConcurrencyPerDomain || 20, maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, sameDomainDelaySecs: rateLimits.sameDomainDelaySecs, maxRequestRetries: config.maxRequestRetries || 3, @@ -168,7 +171,8 @@ async function crawlSingleApiDomain(urls, domain, config = {}) { await crawler.addRequests(initialRequests); - console.log(` [${domain}] Starting API crawl with ${initialRequests.length} APIs (max ${rateLimits.maxRequestsPerMinute} req/min)`); + const concurrency = config.maxConcurrencyPerDomain || 20; + console.log(` [${domain}] Starting: ${initialRequests.length} APIs, ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent, ${rateLimits.sameDomainDelaySecs}s delay`); await crawler.run(); // Flush any remaining collections to database diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 09359a2..197bbe7 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -53,8 +53,11 @@ async function crawlSingleDomain(catalogs, domain, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - // Per-domain rate limiting - maxConcurrency: config.maxConcurrencyPerDomain || 10, + // Per-domain rate limiting - HIGH concurrency for throughput + // maxConcurrency should be higher than req/sec to avoid bottlenecks + // 120 req/min = 2 req/sec, so we need at least 2-3 concurrent requests + // Using 20 to handle slow responses and maintain throughput + maxConcurrency: config.maxConcurrencyPerDomain || 20, maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, sameDomainDelaySecs: rateLimits.sameDomainDelaySecs, maxRequestRetries: config.maxRequestRetries || 3, @@ -132,7 +135,8 @@ async function crawlSingleDomain(catalogs, domain, config = {}) { await crawler.addRequests(initialRequests); - console.log(` [${domain}] Starting crawl with ${initialRequests.length} catalogs (max ${rateLimits.maxRequestsPerMinute} req/min)`); + const concurrency = config.maxConcurrencyPerDomain || 20; + console.log(` [${domain}] Starting: ${initialRequests.length} catalogs, ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent, ${rateLimits.sameDomainDelaySecs}s delay`); await crawler.run(); // Flush any remaining collections to database diff --git a/crawler/utils/config.js b/crawler/utils/config.js index 3858b7c..c7e76ed 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -47,7 +47,7 @@ function getConfig() { // NEW: Parallel crawling options (recommended for performance) parallelDomains: 5, // Number of domains to crawl in parallel maxRequestsPerMinutePerDomain: 120, // Max requests per minute PER domain - maxConcurrencyPerDomain: 10, // Max concurrent requests per domain + maxConcurrencyPerDomain: 20, // Max concurrent requests per domain (must be > req/sec) // Legacy rate limiting options (still supported but parallel options are preferred) maxConcurrency: 5, // Maximum number of concurrent requests (global) diff --git a/crawler/utils/parallel.js b/crawler/utils/parallel.js index c8711e6..efab13c 100644 --- a/crawler/utils/parallel.js +++ b/crawler/utils/parallel.js @@ -147,17 +147,17 @@ export async function executeWithConcurrency(tasks, concurrency, onProgress = nu * @returns {Object} Rate limiting configuration */ export function calculateRateLimits(maxRequestsPerMinute = 120) { - // Calculate delay between requests to stay under the limit - // Add 10% safety margin - const safeLimit = Math.floor(maxRequestsPerMinute * 0.9); - const delayMs = Math.ceil(60000 / safeLimit); - const delaySecs = delayMs / 1000; + // IMPORTANT: In Crawlee, maxRequestsPerMinute and sameDomainDelaySecs work independently + // Using BOTH causes double-throttling. We rely primarily on maxRequestsPerMinute. + // + // sameDomainDelaySecs is set to a minimal value (0.05s = 50ms) just to prevent + // overwhelming a single server with truly simultaneous requests. return { - maxRequestsPerMinute: safeLimit, - sameDomainDelaySecs: Math.max(delaySecs, 0.5), // Minimum 0.5s delay - // With 120 req/min target, we get ~108 effective req/min with safety margin - // That's ~1.8 req/sec or ~0.55s between requests + maxRequestsPerMinute: maxRequestsPerMinute, // Use full limit, no safety margin needed + sameDomainDelaySecs: 0.05, // Minimal delay (50ms) - just prevent request flooding + // With 120 req/min, that's 2 req/sec + // With concurrency 20+, we can achieve this easily }; } From 657bd0a1a4c30c3eefa40ca59b9143a3922c8e1e Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 03:01:20 +0100 Subject: [PATCH 64/68] fix(crawler): optimize concurrency settings for API and catalog crawlers - Introduced configurable concurrency settings to enhance crawling performance, allowing for immediate scaling and minimal delays between requests. - Updated logging to reflect the new concurrency model and removed unnecessary delay parameters for improved throughput. - Adjusted rate limiting calculations to rely solely on maxRequestsPerMinute, ensuring maximum efficiency during crawling operations. --- crawler/apis/api.js | 28 ++++++++++++++++++++-------- crawler/catalogs/catalog.js | 28 ++++++++++++++++++++-------- crawler/utils/parallel.js | 13 ++++--------- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index 51d5364..e27741f 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -86,18 +86,31 @@ async function crawlSingleApiDomain(urls, domain, config = {}) { // Maximum depth for nested catalog crawling (0 = unlimited) const maxDepth = config.maxDepth || 10; + const concurrency = config.maxConcurrencyPerDomain || 20; + const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - // Per-domain rate limiting - HIGH concurrency for throughput - // maxConcurrency should be higher than req/sec to avoid bottlenecks - // 120 req/min = 2 req/sec, so we need at least 2-3 concurrent requests - // Using 20 to handle slow responses and maintain throughput - maxConcurrency: config.maxConcurrencyPerDomain || 20, + // Rate limiting - use maxRequestsPerMinute as primary control maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, - sameDomainDelaySecs: rateLimits.sameDomainDelaySecs, maxRequestRetries: config.maxRequestRetries || 3, + // Concurrency settings - disable autoscaling for consistent speed + maxConcurrency: concurrency, + minConcurrency: Math.min(concurrency, 5), // Start with at least 5 concurrent + + // Disable autoscaling delays - we want full speed immediately + autoscaledPoolOptions: { + desiredConcurrency: concurrency, + minConcurrency: Math.min(concurrency, 5), + maxConcurrency: concurrency, + scaleUpStepRatio: 1, // Scale up immediately + scaleDownStepRatio: 0.5, + }, + + // Minimal delays - let maxRequestsPerMinute handle rate limiting + sameDomainDelaySecs: 0, // No delay between requests + // Accept additional MIME types additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], @@ -171,8 +184,7 @@ async function crawlSingleApiDomain(urls, domain, config = {}) { await crawler.addRequests(initialRequests); - const concurrency = config.maxConcurrencyPerDomain || 20; - console.log(` [${domain}] Starting: ${initialRequests.length} APIs, ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent, ${rateLimits.sameDomainDelaySecs}s delay`); + console.log(` [${domain}] Starting: ${initialRequests.length} APIs, max ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent`); await crawler.run(); // Flush any remaining collections to database diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index 197bbe7..f146572 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -50,18 +50,31 @@ async function crawlSingleDomain(catalogs, domain, config = {}) { } }; + const concurrency = config.maxConcurrencyPerDomain || 20; + const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - // Per-domain rate limiting - HIGH concurrency for throughput - // maxConcurrency should be higher than req/sec to avoid bottlenecks - // 120 req/min = 2 req/sec, so we need at least 2-3 concurrent requests - // Using 20 to handle slow responses and maintain throughput - maxConcurrency: config.maxConcurrencyPerDomain || 20, + // Rate limiting - use maxRequestsPerMinute as primary control maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, - sameDomainDelaySecs: rateLimits.sameDomainDelaySecs, maxRequestRetries: config.maxRequestRetries || 3, + // Concurrency settings - disable autoscaling for consistent speed + maxConcurrency: concurrency, + minConcurrency: Math.min(concurrency, 5), // Start with at least 5 concurrent + + // Disable autoscaling delays - we want full speed immediately + autoscaledPoolOptions: { + desiredConcurrency: concurrency, + minConcurrency: Math.min(concurrency, 5), + maxConcurrency: concurrency, + scaleUpStepRatio: 1, // Scale up immediately + scaleDownStepRatio: 0.5, + }, + + // Minimal delays - let maxRequestsPerMinute handle rate limiting + sameDomainDelaySecs: 0, // No delay between requests + // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], @@ -135,8 +148,7 @@ async function crawlSingleDomain(catalogs, domain, config = {}) { await crawler.addRequests(initialRequests); - const concurrency = config.maxConcurrencyPerDomain || 20; - console.log(` [${domain}] Starting: ${initialRequests.length} catalogs, ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent, ${rateLimits.sameDomainDelaySecs}s delay`); + console.log(` [${domain}] Starting: ${initialRequests.length} catalogs, max ${rateLimits.maxRequestsPerMinute} req/min, ${concurrency} concurrent`); await crawler.run(); // Flush any remaining collections to database diff --git a/crawler/utils/parallel.js b/crawler/utils/parallel.js index efab13c..bf3247f 100644 --- a/crawler/utils/parallel.js +++ b/crawler/utils/parallel.js @@ -147,17 +147,12 @@ export async function executeWithConcurrency(tasks, concurrency, onProgress = nu * @returns {Object} Rate limiting configuration */ export function calculateRateLimits(maxRequestsPerMinute = 120) { - // IMPORTANT: In Crawlee, maxRequestsPerMinute and sameDomainDelaySecs work independently - // Using BOTH causes double-throttling. We rely primarily on maxRequestsPerMinute. - // - // sameDomainDelaySecs is set to a minimal value (0.05s = 50ms) just to prevent - // overwhelming a single server with truly simultaneous requests. + // Use only maxRequestsPerMinute for rate limiting + // sameDomainDelaySecs is set to 0 - we rely solely on the rate limiter + // This gives us maximum throughput while respecting the rate limit return { - maxRequestsPerMinute: maxRequestsPerMinute, // Use full limit, no safety margin needed - sameDomainDelaySecs: 0.05, // Minimal delay (50ms) - just prevent request flooding - // With 120 req/min, that's 2 req/sec - // With concurrency 20+, we can achieve this easily + maxRequestsPerMinute: maxRequestsPerMinute, }; } From edbf6144bd04a1dd7b8a4f338e5af3a3ea0f49de Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 03:04:35 +0100 Subject: [PATCH 65/68] refactor(crawler): streamline concurrency settings in API and catalog crawlers - Simplified concurrency configuration by removing autoscaling options and unnecessary delay parameters, focusing on high throughput. - Updated comments for clarity on rate limiting and concurrency settings, ensuring better understanding of the crawling process. - Maintained additional MIME type support for diverse content handling during crawls. --- crawler/apis/api.js | 17 ++--------------- crawler/catalogs/catalog.js | 17 ++--------------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index e27741f..ec7688c 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -91,25 +91,12 @@ async function crawlSingleApiDomain(urls, domain, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - // Rate limiting - use maxRequestsPerMinute as primary control + // Rate limiting maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, maxRequestRetries: config.maxRequestRetries || 3, - // Concurrency settings - disable autoscaling for consistent speed + // High concurrency for throughput maxConcurrency: concurrency, - minConcurrency: Math.min(concurrency, 5), // Start with at least 5 concurrent - - // Disable autoscaling delays - we want full speed immediately - autoscaledPoolOptions: { - desiredConcurrency: concurrency, - minConcurrency: Math.min(concurrency, 5), - maxConcurrency: concurrency, - scaleUpStepRatio: 1, // Scale up immediately - scaleDownStepRatio: 0.5, - }, - - // Minimal delays - let maxRequestsPerMinute handle rate limiting - sameDomainDelaySecs: 0, // No delay between requests // Accept additional MIME types additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], diff --git a/crawler/catalogs/catalog.js b/crawler/catalogs/catalog.js index f146572..3a6fa09 100644 --- a/crawler/catalogs/catalog.js +++ b/crawler/catalogs/catalog.js @@ -55,25 +55,12 @@ async function crawlSingleDomain(catalogs, domain, config = {}) { const crawler = new HttpCrawler({ requestHandlerTimeoutSecs: timeoutSecs, - // Rate limiting - use maxRequestsPerMinute as primary control + // Rate limiting maxRequestsPerMinute: rateLimits.maxRequestsPerMinute, maxRequestRetries: config.maxRequestRetries || 3, - // Concurrency settings - disable autoscaling for consistent speed + // High concurrency for throughput maxConcurrency: concurrency, - minConcurrency: Math.min(concurrency, 5), // Start with at least 5 concurrent - - // Disable autoscaling delays - we want full speed immediately - autoscaledPoolOptions: { - desiredConcurrency: concurrency, - minConcurrency: Math.min(concurrency, 5), - maxConcurrency: concurrency, - scaleUpStepRatio: 1, // Scale up immediately - scaleDownStepRatio: 0.5, - }, - - // Minimal delays - let maxRequestsPerMinute handle rate limiting - sameDomainDelaySecs: 0, // No delay between requests // Accept additional MIME types (some STAC endpoints return JSON with incorrect Content-Type) additionalMimeTypes: ['application/geo+json', 'text/plain', 'binary/octet-stream', 'application/octet-stream'], From 9cd68c6ed8450e06443e705df3fa22f69a1c94c6 Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 11:16:00 +0100 Subject: [PATCH 66/68] fix(crawler): correct spatial extent format in database insertion - Updated the spatial extent string format from EPSG:4326 to SRID=4326 to comply with EWKT requirements. - Ensured proper handling of bounding box coordinates for polygon creation in the database insertion process. --- crawler/utils/db.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crawler/utils/db.js b/crawler/utils/db.js index e93434e..07dc46e 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -94,7 +94,8 @@ async function insertOrUpdateCollection(collection) { if (bbox && bbox.length === 4) { // Create polygon from bbox [west, south, east, north] - spatialExtent = `EPSG:4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; + // EWKT format requires SRID=4326, not EPSG:4326 + spatialExtent = `SRID=4326;POLYGON((${bbox[0]} ${bbox[1]}, ${bbox[2]} ${bbox[1]}, ${bbox[2]} ${bbox[3]}, ${bbox[0]} ${bbox[3]}, ${bbox[0]} ${bbox[1]}))`; } // Parse temporal extent From e140246b4fd96a38d028199cf0d81a6965e89bed Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 11:44:48 +0100 Subject: [PATCH 67/68] feat(crawler): add deadlock handling and retry logic for database collection updates - Implemented a deadlock detection mechanism to identify PostgreSQL deadlock errors during collection insertion or update. - Introduced a retry logic with exponential backoff for handling deadlocks, allowing for more robust database operations. - Enhanced the internal collection insertion function to support improved querying by stac_id and source_url, ensuring better handling of collections with the same title from different sources. - Updated error logging to differentiate between deadlock errors and other types of errors during database operations. --- crawler/utils/db.js | 92 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 9 deletions(-) diff --git a/crawler/utils/db.js b/crawler/utils/db.js index 07dc46e..dc1d428 100644 --- a/crawler/utils/db.js +++ b/crawler/utils/db.js @@ -67,13 +67,62 @@ async function insertOrUpdateCatalog(catalog) { } /** - * Insert or update a collection in the database + * Check if an error is a PostgreSQL deadlock error + * @param {Error} error - The error to check + * @returns {boolean} true if it's a deadlock error + */ +function isDeadlockError(error) { + // PostgreSQL deadlock error code is '40P01' + return error.code === '40P01' || error.message?.includes('deadlock detected'); +} + +/** + * Sleep for a given number of milliseconds + * @param {number} ms - Milliseconds to sleep + * @returns {Promise} + */ +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +/** + * Insert or update a collection in the database with deadlock retry logic * @param {Object} collection - STAC collection object + * @param {number} maxRetries - Maximum number of retry attempts for deadlocks (default: 3) * @returns {Promise} collection ID */ -async function insertOrUpdateCollection(collection) { - +async function insertOrUpdateCollection(collection, maxRetries = 3) { + let lastError; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + return await _insertOrUpdateCollectionInternal(collection); + } catch (error) { + lastError = error; + + if (isDeadlockError(error) && attempt < maxRetries) { + // Exponential backoff: 100ms, 200ms, 400ms, ... + const delayMs = 100 * Math.pow(2, attempt - 1) + Math.random() * 50; + console.warn(`WARN [DB] Deadlock detected for collection "${collection.title || collection.id}", retrying in ${Math.round(delayMs)}ms (attempt ${attempt}/${maxRetries})`); + await sleep(delayMs); + continue; + } + + // Not a deadlock or max retries reached, throw the error + throw error; + } + } + + // Should not reach here, but just in case + throw lastError; +} +/** + * Internal implementation of insertOrUpdateCollection + * @param {Object} collection - STAC collection object + * @returns {Promise} collection ID + */ +async function _insertOrUpdateCollectionInternal(collection) { const client = await pool.connect(); try { await client.query('BEGIN'); @@ -130,11 +179,33 @@ async function insertOrUpdateCollection(collection) { sourceUrl = selfLink?.href || rootLink?.href || null; } - // Check if collection with this title already exists - const existingCollection = await client.query( - 'SELECT id FROM collection WHERE title = $1', - [collectionTitle] - ); + // Check if collection with same stac_id from same source already exists + // This allows collections with the same title from different sources to coexist + let existingCollection; + if (stacId && sourceUrl) { + // Best case: match by stac_id + source_url (most precise) + existingCollection = await client.query( + 'SELECT id FROM collection WHERE stac_id = $1 AND full_json->>\'links\' LIKE $2', + [stacId, `%${sourceUrl}%`] + ); + } + + // Fallback: if no stac_id or source_url, check by title only (legacy behavior) + if (!existingCollection || existingCollection.rows.length === 0) { + if (stacId) { + // Try matching by stac_id + title combination + existingCollection = await client.query( + 'SELECT id FROM collection WHERE stac_id = $1 AND title = $2', + [stacId, collectionTitle] + ); + } else { + // Last resort: match by title only (for collections without stac_id) + existingCollection = await client.query( + 'SELECT id FROM collection WHERE title = $1 AND stac_id IS NULL', + [collectionTitle] + ); + } + } let collectionId; if (existingCollection.rows.length > 0) { @@ -240,7 +311,10 @@ async function insertOrUpdateCollection(collection) { return collectionId; } catch (error) { await client.query('ROLLBACK'); - console.error('Error inserting collection:', error.message); + // Only log non-deadlock errors here, deadlocks are handled by retry wrapper + if (!isDeadlockError(error)) { + console.error('Error inserting collection:', error.message); + } throw error; } finally { client.release(); From 9fe7298ac2da627d22cb0226459bc9df324e9587 Mon Sep 17 00:00:00 2001 From: mammutor Date: Wed, 21 Jan 2026 14:29:58 +0100 Subject: [PATCH 68/68] refactor(crawler): optimize batch sizes and memory management for low RAM servers - Reduced batch sizes for saving collections and clearing arrays to 25 to accommodate servers with limited RAM (2GB). - Updated API and catalog handling to create STAC objects with memory efficiency in mind, minimizing stored data. - Enhanced garbage collection by dereferencing large objects after processing to improve memory usage. - stac migrate true --- crawler/apis/api.js | 21 ++++++++++++++------- crawler/utils/config.js | 10 +++++----- crawler/utils/handlers.js | 39 ++++++++++++++++++++++++++------------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/crawler/apis/api.js b/crawler/apis/api.js index ec7688c..3365c17 100644 --- a/crawler/apis/api.js +++ b/crawler/apis/api.js @@ -18,15 +18,17 @@ import { /** * Batch size for saving collections to database during API crawling + * Set low (25) for servers with limited RAM (2GB) * @type {number} */ -const BATCH_SIZE = 500; +const BATCH_SIZE = 25; /** * Batch size for clearing apis array to free memory + * Set low (25) for servers with limited RAM (2GB) * @type {number} */ -const API_CLEAR_BATCH_SIZE = 500; +const API_CLEAR_BATCH_SIZE = 25; /** * Checks if batch size is reached and flushes if necessary @@ -293,7 +295,7 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max let stacObj; try { - stacObj = create(json, false); + stacObj = create(json, true); results.stats.stacCompliant++; if (typeof stacObj.isCatalog === 'function' && stacObj.isCatalog()) { @@ -308,10 +310,9 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max } results.stats.apisProcessed++; + // Only track minimal info to reduce memory results.apis.push({ - id: apiId, - url: request.url, - stacType: stacObj.isCollection() ? 'collection' : 'catalog' + id: apiId }); // If this is a STAC Collection directly, extract and store it @@ -426,6 +427,9 @@ async function handleApiRoot({ request, json, crawler, log, indent, results, max } } } + + // Help garbage collector by dereferencing large objects + stacObj = null; } /** @@ -437,7 +441,7 @@ async function handleApiCollection({ request, json, crawler, log, indent, result let stacObj; try { - stacObj = create(json, false); + stacObj = create(json, true); if (typeof stacObj.isCollection === 'function' && stacObj.isCollection()) { const collection = normalizeCollection(stacObj, results.collections.length); @@ -452,6 +456,9 @@ async function handleApiCollection({ request, json, crawler, log, indent, result } catch (parseError) { log.warning(`${indent}Skipping non-compliant STAC collection at ${request.url}`); } + + // Help garbage collector + stacObj = null; } export { diff --git a/crawler/utils/config.js b/crawler/utils/config.js index c7e76ed..e0ebdcf 100644 --- a/crawler/utils/config.js +++ b/crawler/utils/config.js @@ -36,7 +36,7 @@ dotenv.config(); function getConfig() { const cliArgs = parseCliArgs(); - // Default configuration + // Default configuration (optimized for 2GB RAM servers) const defaults = { mode: 'both', // 'catalogs', 'apis', or 'both' maxCatalogs: 10, // Maximum number of catalogs to crawl @@ -44,10 +44,10 @@ function getConfig() { timeout: 30000, // Timeout in milliseconds (30 seconds) maxDepth: 10, // Maximum recursion depth for nested catalogs (0 = unlimited) - // NEW: Parallel crawling options (recommended for performance) - parallelDomains: 5, // Number of domains to crawl in parallel - maxRequestsPerMinutePerDomain: 120, // Max requests per minute PER domain - maxConcurrencyPerDomain: 20, // Max concurrent requests per domain (must be > req/sec) + // Parallel crawling options (reduced for 2GB RAM servers) + parallelDomains: 2, // Number of domains to crawl in parallel (reduced from 5) + maxRequestsPerMinutePerDomain: 60, // Max requests per minute PER domain (reduced from 120) + maxConcurrencyPerDomain: 5, // Max concurrent requests per domain (reduced from 20) // Legacy rate limiting options (still supported but parallel options are preferred) maxConcurrency: 5, // Maximum number of concurrent requests (global) diff --git a/crawler/utils/handlers.js b/crawler/utils/handlers.js index 922ff2e..907bae1 100644 --- a/crawler/utils/handlers.js +++ b/crawler/utils/handlers.js @@ -11,16 +11,18 @@ import db from './db.js'; /** * Batch size for saving collections to database * After this many collections are collected, they will be flushed to DB + * Set low (25) for servers with limited RAM (2GB) * @type {number} */ -const BATCH_SIZE = 500; +const BATCH_SIZE = 25; /** * Batch size for clearing catalogs array to free memory * The catalogs array is only used for statistics, so we clear it periodically + * Set low (25) for servers with limited RAM (2GB) * @type {number} */ -const CATALOG_CLEAR_BATCH_SIZE = 500; +const CATALOG_CLEAR_BATCH_SIZE = 25; /** * Flushes collected collections to the database and clears the array @@ -115,7 +117,7 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul // Using migrate=false to avoid issues with stac-migrate and newer STAC versions let stacCatalog; try { - stacCatalog = create(json, false); + stacCatalog = create(json, true); results.stats.stacCompliant++; // Log STAC object type @@ -132,15 +134,17 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul } results.stats.catalogsProcessed++; + // Only track minimal info to reduce memory - don't store full catalog data results.catalogs.push({ id: catalogId, - url: request.url, - depth, - stacType: stacCatalog.isCatalog() ? 'catalog' : 'collection' + depth }); // If this is a STAC Collection (not a catalog), extract and store it - if (typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection()) { + // Collections don't have /collections endpoints, so we skip tryCollectionEndpoints for them + const isCollection = typeof stacCatalog.isCollection === 'function' && stacCatalog.isCollection(); + + if (isCollection) { const collection = normalizeCollection(stacCatalog, results.collections.length); results.collections.push(collection); results.stats.collectionsFound++; @@ -148,11 +152,13 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul // Check if we should flush to database await checkAndFlush(results, log); + } else { + // Only try /collections endpoint for Catalogs, not Collections + // Static STAC catalogs don't have /collections endpoints - they use rel="child" links + // STAC APIs have /collections endpoints and advertise them via rel="data" or rel="collections" + await tryCollectionEndpoints(stacCatalog, request.url, catalogId, depth, crawler, log, indent); } - // Try to get collections from this catalog (using STAC link discovery) - await tryCollectionEndpoints(stacCatalog, request.url, catalogId, depth, crawler, log, indent); - // Extract and enqueue child catalog links using stac-js if (stacCatalog && typeof stacCatalog.getChildLinks === 'function') { const childLinks = stacCatalog.getChildLinks(); @@ -254,6 +260,9 @@ export async function handleCatalog({ request, json, crawler, log, indent, resul // Ensure memory is cleared periodically even if no collections were found await checkAndFlush(results, log); + + // Help garbage collector by dereferencing large objects + stacCatalog = null; } /** @@ -274,7 +283,7 @@ export async function handleCollections({ request, json, crawler, log, indent, r // Note: create(data, migrate, updateVersionNumber) - second param is boolean, not URL let stacObj; try { - stacObj = create(json, false); + stacObj = create(json, true); } catch (parseError) { log.warning(`${indent}Skipping non-compliant STAC collections at ${request.url}`); return; @@ -289,7 +298,7 @@ export async function handleCollections({ request, json, crawler, log, indent, r // Handle array of collections collectionsData = json.map(col => { try { - return create(col, false); + return create(col, true); } catch { return null; } @@ -298,7 +307,7 @@ export async function handleCollections({ request, json, crawler, log, indent, r // Handle nested collections property collectionsData = json.collections.map(col => { try { - return create(col, false); + return create(col, true); } catch { return null; } @@ -324,4 +333,8 @@ export async function handleCollections({ request, json, crawler, log, indent, r // Check if we should flush to database await checkAndFlush(results, log); } + + // Help garbage collector by dereferencing large objects + stacObj = null; + collectionsData = null; } \ No newline at end of file