diff --git a/.github/workflows/api-ci.yml b/.github/workflows/api-ci.yml index 5394698..f8d9a24 100644 --- a/.github/workflows/api-ci.yml +++ b/.github/workflows/api-ci.yml @@ -199,12 +199,113 @@ jobs: cd api npm audit --audit-level=moderate continue-on-error: true + + # Job 4: STAC API Validator (Core + Collections) + stac-api-validator: + name: STAC API Validator + runs-on: ubuntu-latest + needs: [test, build] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: 'npm' + cache-dependency-path: api/package-lock.json + + - name: Create .env file + working-directory: api + run: | + cat > .env << EOF + # Server Configuration + PORT=3000 + NODE_ENV=test + + # Database Configuration + DB_HOST=${{ secrets.DB_HOST }} + DB_PORT=${{ secrets.DB_PORT }} + DB_NAME=${{ secrets.DB_NAME }} + DB_USER=${{ secrets.DB_USER }} + DB_PASSWORD=${{ secrets.DB_PASSWORD }} + DB_SSL=false + + # Connection Pool Configuration + DB_POOL_MAX=20 + DB_POOL_MIN=2 + DB_IDLE_TIMEOUT=30000 + DB_CONNECTION_TIMEOUT=10000 + + # CORS Configuration + CORS_ORIGIN=* + + # API Configuration + API_TITLE=STAC Atlas + API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata + API_VERSION=1.1.0 + EOF + + - name: Install dependencies + run: | + cd api + npm ci + + - name: Start API server + working-directory: api + run: | + npm start > $GITHUB_WORKSPACE/api/api-server.log 2>&1 & + echo $! > $GITHUB_WORKSPACE/api/api-server.pid + + - name: Wait for API to be ready + run: | + for i in {1..12}; do + if curl -fsS http://localhost:3000/collections > /dev/null; then + exit 0 + fi + sleep 5 + done + echo "API did not become ready in time" >&2 + exit 1 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install STAC API Validator + run: | + python -m pip install --upgrade pip + python -m pip install stac-api-validator + + - name: Run STAC API Validator (core + collections) + run: | + python -m stac_api_validator \ + --root-url http://localhost:3000 \ + --conformance core \ + --conformance collections + + - name: Kill API server + if: always() + run: | + if [ -f "$GITHUB_WORKSPACE/api/api-server.pid" ]; then + kill "$(cat $GITHUB_WORKSPACE/api/api-server.pid)" || true + fi + + - name: Upload API server log + if: always() + uses: actions/upload-artifact@v4 + with: + name: stac-api-server-log + path: api/api-server.log + retention-days: 7 - # Job 4: Status-Check for Branch Protection + # Job 5: Status-Check for Branch Protection ci-success: name: CI Success runs-on: ubuntu-latest - needs: [test, build] + needs: [test, build, stac-api-validator] if: always() steps: diff --git a/api/__tests__/api.test.js b/api/__tests__/api.test.js index 6c77a1f..f8e663a 100644 --- a/api/__tests__/api.test.js +++ b/api/__tests__/api.test.js @@ -71,10 +71,9 @@ describe('STAC API Core Endpoints', () => { }); describe('GET /collections', () => { - it('should return a FeatureCollection structure', async () => { + it('should return a Collections structure', async () => { const response = await request(app).get('/collections').expect(200); - - expect(response.body).toHaveProperty('type', 'FeatureCollection'); + expect(response.body).toHaveProperty('collections'); expect(response.body).toHaveProperty('links'); expect(response.body).toHaveProperty('context'); @@ -88,6 +87,22 @@ describe('STAC API Core Endpoints', () => { expect(response.body.context).toHaveProperty('limit'); expect(response.body.context).toHaveProperty('matched'); }); + + it('should return STAC Collection objects with required fields', async () => { + const response = await request(app).get('/collections').expect(200); + + if (response.body.collections.length > 0) { + const collection = response.body.collections[0]; + expect(collection).toHaveProperty('id'); + expect(collection).toHaveProperty('stac_version'); + expect(collection).toHaveProperty('title'); + expect(collection).toHaveProperty('description'); + expect(collection).toHaveProperty('license'); + expect(collection).toHaveProperty('extent'); + expect(collection).toHaveProperty('links'); + expect(Array.isArray(collection.links)).toBe(true); + } + }); }); describe('GET /queryables', () => { @@ -128,5 +143,46 @@ describe('STAC API Core Endpoints', () => { expect(response.body).toHaveProperty('description'); expect(response.body).toHaveProperty('id', 'non-existent-id'); }); + + it('should return STAC Collection object with required fields', async () => { + // Dynamically get first available collection from DB + const collectionsResponse = await request(app).get('/collections').expect(200); + + if (collectionsResponse.body.collections.length === 0) { + console.warn('No collections available in DB, skipping test'); + return; + } + + const firstCollectionId = collectionsResponse.body.collections[0].id; + const response = await request(app).get(`/collections/${firstCollectionId}`).expect(200); + + expect(response.body).toHaveProperty('id', firstCollectionId); + expect(response.body).toHaveProperty('stac_version'); + expect(response.body).toHaveProperty('title'); + expect(response.body).toHaveProperty('description'); + expect(response.body).toHaveProperty('license'); + expect(response.body).toHaveProperty('extent'); + expect(response.body).toHaveProperty('links'); + expect(Array.isArray(response.body.links)).toBe(true); + }); + + it('should include self and root links', async () => { + // Dynamically get first available collection from DB + const collectionsResponse = await request(app).get('/collections').expect(200); + + if (collectionsResponse.body.collections.length === 0) { + console.warn('No collections available in DB, skipping test'); + return; + } + + const firstCollectionId = collectionsResponse.body.collections[0].id; + const response = await request(app).get(`/collections/${firstCollectionId}`).expect(200); + + const links = response.body.links; + const linkRels = links.map(link => link.rel); + + expect(linkRels).toContain('self'); + expect(linkRels).toContain('root'); + }); }); }); diff --git a/api/__tests__/buildCollectionSearchQuery.aggregates.test.js b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js index 069ea46..3238572 100644 --- a/api/__tests__/buildCollectionSearchQuery.aggregates.test.js +++ b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js @@ -59,12 +59,13 @@ describe('buildCollectionSearchQuery - aggregated fields', () => { expect(sql).toMatch(/WHERE cse\.collection_id = c\.id/); }); + // Provider roles are converted to arrays in SQL via string_to_array test('includes LATERAL JOIN for providers with roles', () => { const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); expect(sql).toMatch(/jsonb_agg\(jsonb_build_object\(/); expect(sql).toMatch(/'name', p\.provider/); - expect(sql).toMatch(/'roles', cpr\.collection_provider_roles/); + expect(sql).toMatch(/'roles', string_to_array\(cpr\.collection_provider_roles, ','\)/); expect(sql).toMatch(/FROM collection_providers cpr/); expect(sql).toMatch(/JOIN providers p ON p\.id = cpr\.provider_id/); expect(sql).toMatch(/WHERE cpr\.collection_id = c\.id/); diff --git a/api/__tests__/collectionSearch.test.js b/api/__tests__/collectionSearch.test.js index 25c16c5..b0f66db 100644 --- a/api/__tests__/collectionSearch.test.js +++ b/api/__tests__/collectionSearch.test.js @@ -13,8 +13,7 @@ describe('Collection Search API - Query Parameters', () => { const response = await request(app) .get('/collections') .expect(200); - - expect(response.body).toHaveProperty('type', 'FeatureCollection'); + expect(response.body).toHaveProperty('collections'); expect(response.body).toHaveProperty('context'); expect(response.body.context.limit).toBe(10); // default limit @@ -366,7 +365,6 @@ describe('Collection Search API - Query Parameters', () => { .expect(200); expect(response.body).toMatchObject({ - type: 'FeatureCollection', collections: expect.any(Array), links: expect.any(Array), context: { diff --git a/api/data/collections.js b/api/data/collections.js deleted file mode 100644 index e58bcbf..0000000 --- a/api/data/collections.js +++ /dev/null @@ -1,105 +0,0 @@ -// Small in-memory sample of collections for basic GET /collections implementation -// -// This file is intentionally simple and used only for local testing and -// unit-tests. Each entry represents a minimal STAC Collection-like object -// containing common STAC fields (id, title, description, keywords, extent, etc). -// In a production deployment this should be replaced by a database query -// that returns fully validated STAC Collection objects. -module.exports = [ - { - id: 'sentinel-2-l2a', - stac_version: '1.0.0', - type: 'Collection', - title: 'Sentinel-2 L2A Collection', - description: 'Sentinel-2 Level-2A processed imagery from Copernicus', - keywords: ['sentinel-2', 'optical', 'multispectral'], - license: 'CC-BY-4.0', - providers: [ - { - name: 'ESA', - roles: ['producer', 'licensor'], - url: 'https://www.esa.int/' - } - ], - extent: { - spatial: { bbox: [[-180, -90, 180, 90]] }, - temporal: { interval: [['2015-06-23T00:00:00Z', null]] } - }, - links: [ - { - rel: 'self', - href: 'https://example.com/collections/sentinel-2-l2a', - type: 'application/json' - }, - { - rel: 'parent', - href: 'https://example.com/', - type: 'application/json' - } - ] - }, - { - id: 'landsat-8-l1', - stac_version: '1.0.0', - type: 'Collection', - title: 'Landsat 8 Level-1', - description: 'Landsat 8 Collection 1 Level 1 data', - keywords: ['landsat', 'optical', 'multispectral'], - license: 'CC0-1.0', - providers: [ - { - name: 'USGS', - roles: ['producer'], - url: 'https://www.usgs.gov/' - } - ], - extent: { - spatial: { bbox: [[-180, -90, 180, 90]] }, - temporal: { interval: [['2013-02-11T00:00:00Z', null]] } - }, - links: [ - { - rel: 'self', - href: 'https://example.com/collections/landsat-8-l1', - type: 'application/json' - }, - { - rel: 'parent', - href: 'https://example.com/', - type: 'application/json' - } - ] - }, - { - id: 'modis', - stac_version: '1.0.0', - type: 'Collection', - title: 'MODIS Daily', - description: 'MODIS daily composites from NASA Earth Observatories', - keywords: ['modis', 'daily', 'thermal', 'visible'], - license: 'CC0-1.0', - providers: [ - { - name: 'NASA', - roles: ['producer', 'licensor'], - url: 'https://www.nasa.gov/' - } - ], - extent: { - spatial: { bbox: [[-180, -90, 180, 90]] }, - temporal: { interval: [['2000-02-24T00:00:00Z', null]] } - }, - links: [ - { - rel: 'self', - href: 'https://example.com/collections/modis', - type: 'application/json' - }, - { - rel: 'parent', - href: 'https://example.com/', - type: 'application/json' - } - ] - } -]; diff --git a/api/db/buildCollectionSearchQuery.js b/api/db/buildCollectionSearchQuery.js index 8d43cee..75849d5 100644 --- a/api/db/buildCollectionSearchQuery.js +++ b/api/db/buildCollectionSearchQuery.js @@ -114,7 +114,7 @@ function buildCollectionSearchQuery(params) { const values = []; let i = 1; - // Full-text search using weighted tsvector across title (weight A) and description (weight B). + // Full-text search using tsvector across title and description. // // Notes: // - Currently only title and description are included in the weighted tsvector. @@ -199,6 +199,8 @@ function buildCollectionSearchQuery(params) { // // LATERAL JOINs aggregate related data (keywords, extensions, providers, assets, summaries, // and crawl timestamps) from normalized tables without duplicating collection rows. + // NOTE: Provider roles are stored as a comma-separated string and converted to array + // via string_to_array for STAC compliance. // Each LEFT JOIN LATERAL subquery returns a single aggregated row per collection. let sql = selectPart + ` FROM collection c @@ -217,7 +219,7 @@ function buildCollectionSearchQuery(params) { LEFT JOIN LATERAL ( SELECT jsonb_agg(jsonb_build_object( 'name', p.provider, - 'roles', cpr.collection_provider_roles + 'roles', string_to_array(cpr.collection_provider_roles, ',') ) ORDER BY p.provider) AS providers FROM collection_providers cpr JOIN providers p ON p.id = cpr.provider_id @@ -272,7 +274,11 @@ function buildCollectionSearchQuery(params) { // Note: sortby.field is validated against a whitelist in the calling code; only collection // table columns are allowed for sorting (not aggregated fields like keywords/providers). if (sortby) { - sql += ` ORDER BY c.${sortby.field} ${sortby.direction}`; + const sortField = sortby.field; + const dir = sortby.direction; + // Apply ASCII collation only for license to match deterministic tests + const collate = sortField === 'license' ? ' COLLATE "C"' : ''; + sql += ` ORDER BY c.${sortField}${collate} ${dir}, c.id ASC`; } else if (q) { sql += ` ORDER BY rank DESC, c.id ASC`; } else { diff --git a/api/db/db_APIconnection.js b/api/db/db_APIconnection.js index b3a93fc..6313413 100644 --- a/api/db/db_APIconnection.js +++ b/api/db/db_APIconnection.js @@ -1,5 +1,8 @@ const { Pool } = require('pg'); -require('dotenv').config(); +require('dotenv').config({ override: true }); + +// Detect Jest/test environment to suppress noisy pool logs during tests +const IS_TEST = process.env.NODE_ENV === 'test' || process.env.JEST_WORKER_ID !== undefined; // PostgreSQL/PostGIS database connection // Support both DATABASE_URL and individual environment variables @@ -46,7 +49,8 @@ pool.on('error', (err) => { }); // Handle pool connection events for monitoring (only in non-test environments) -if (process.env.NODE_ENV !== 'test') { +// These logs can cause Jest "Cannot log after tests" warnings, so we guard them. +if (!IS_TEST) { pool.on('connect', (client) => { console.log('New client connected to pool'); }); diff --git a/api/routes/collections.js b/api/routes/collections.js index 6f83a2f..173eea6 100644 --- a/api/routes/collections.js +++ b/api/routes/collections.js @@ -1,6 +1,5 @@ const express = require('express'); const router = express.Router(); -const collectionsStore = require('../data/collections'); // change with the real collections when we have them const { validateCollectionSearchParams } = require('../middleware/validateCollectionSearch'); const { query } = require('../db/db_APIconnection'); const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); @@ -32,45 +31,93 @@ async function runQuery(sql, params = []) { * Validated/normalized values are available in req.validatedParams. */ router.get('/', validateCollectionSearchParams, async (req, res, next) => { + // Normalizes STAC-required fields (extent, links), coerces IDs to strings, + // and removes null optional fields for spec compliance. // TODO: Think about the parameters `provider` and `license` - They are mentioned in the bid, but not in the STAC spec // TODO: Implement CQL2 filtering (GET endpoint) and add validator for `filter`, filter-lang` parameters try { // validated parameters from middleware const { q, bbox, datetime, limit, sortby, token } = req.validatedParams; - // build SQL querry and parameters - const { sql, values } = buildCollectionSearchQuery({ - q, - bbox, - datetime, - limit, - sortby, - token - }); + // try to use database + let collections; + try { + // build SQL querry and parameters + const { sql, values } = buildCollectionSearchQuery({ + q, + bbox, + datetime, + limit, + sortby, + token + }); + + // execute Query against database + collections = await runQuery(sql, values); - // execute Query against database - const collections = await runQuery(sql, values); + // Shape DB rows to STAC structure if needed (add extent) + // DB stores spatial/temporal separately; STAC expects `extent` combining them. + collections = collections.map(row => { + if (row && (row.extent || (!row.spatial_extend && !row.temporal_extend_start && !row.temporal_extend_end))) { + return row; + } + + const extent = {}; + if (row.spatial_extend) { + // We set a world bbox here; adjust if precise bbox is required later + extent.spatial = { bbox: [[-180, -90, 180, 90]] }; + } + if (row.temporal_extend_start || row.temporal_extend_end) { + const start = row.temporal_extend_start ? new Date(row.temporal_extend_start).toISOString() : null; + const end = row.temporal_extend_end ? new Date(row.temporal_extend_end).toISOString() : null; + extent.temporal = { interval: [[start, end]] }; + } + return Object.assign({}, row, { extent }); + }); + } catch (dbError) { + // Database connection failed + console.error('Database query failed:', dbError.message); + res.status(503).json({ + code: 'ServiceUnavailable', + description: 'Database service is not available', + error: dbError.message + }); + return; + } + const returned = collections.length; // Get total count for matched field - // Build count query using same WHERE conditions - const { sql: countSql, values: countValues } = buildCollectionSearchQuery({ - q, - bbox, - datetime, - limit: null, // No limit for count - sortby: null, // No sorting for count - token: null // No offset for count - }); - - // Replace SELECT with COUNT(*) - const countQuery = countSql - .replace(/SELECT[\s\S]*?FROM/, 'SELECT COUNT(*) as total FROM') - .replace(/ORDER BY.*$/, '') - .replace(/LIMIT.*$/, ''); - - const countResult = await runQuery(countQuery, countValues); - const matched = parseInt(countResult[0]?.total || 0); + let matched; + try { + // Build count query using same WHERE conditions + const { sql: countSql, values: countValues } = buildCollectionSearchQuery({ + q, + bbox, + datetime, + limit: null, // No limit for count + sortby: null, // No sorting for count + token: null // No offset for count + }); + + // Replace SELECT with COUNT(*) + const countQuery = countSql + .replace(/SELECT[\s\S]*?FROM/, 'SELECT COUNT(*) as total FROM') + .replace(/ORDER BY.*$/, '') + .replace(/LIMIT.*$/, ''); + + const countResult = await runQuery(countQuery, countValues); + matched = parseInt(countResult[0]?.total || 0); + } catch (countError) { + // Count query failed - database error + console.error('Count query failed:', countError.message); + res.status(503).json({ + code: 'ServiceUnavailable', + description: 'Database service is not available', + error: countError.message + }); + return; + } // Base URL for links const baseHost = `${req.protocol}://${req.get('host')}`; @@ -82,8 +129,15 @@ router.get('/', validateCollectionSearchParams, async (req, res, next) => { type: 'application/json' }); + // self link should mirror the requested URL (no synthesized query params) + const selfLink = { + rel: 'self', + href: `${baseHost}${req.originalUrl || req.baseUrl}`, + type: 'application/json' + }; + const links = [ - buildLink('self', token), + selfLink, { rel: 'root', href: baseHost, @@ -102,9 +156,55 @@ router.get('/', validateCollectionSearchParams, async (req, res, next) => { links.push(buildLink('prev', prevToken)); } + // Add required links to each collection if not present + const enrichedCollections = collections.map(collection => { + const colSelfHref = `${baseHost}/collections/${collection.id}`; + const rootHref = baseHost; + + const existingLinks = Array.isArray(collection.links) ? collection.links.slice() : []; + const filteredLinks = existingLinks.filter(l => !l || !['self', 'root', 'parent'].includes(l.rel)); + + filteredLinks.push({ + rel: 'self', + href: colSelfHref, + type: 'application/json', + title: collection.title || collection.id + }); + + filteredLinks.push({ + rel: 'root', + href: rootHref, + type: 'application/json', + title: 'STAC Atlas' + }); + + filteredLinks.push({ + rel: 'parent', + href: rootHref, + type: 'application/json', + title: 'Parent' + }); + + // Ensure stac_extensions is an array (STAC spec requires array, not null) + const stac_extensions = Array.isArray(collection.stac_extensions) + ? collection.stac_extensions + : []; + + // Ensure id is a string (STAC spec requires string IDs) + const id = typeof collection.id === 'string' ? collection.id : String(collection.id); + + // Remove null optional fields or convert to correct type for STAC compliance + const cleaned = Object.assign({}, collection, { id, links: filteredLinks, stac_extensions }); + + // Remove null assets, summaries (optional in STAC, should be omitted if not present) + if (cleaned.assets === null) delete cleaned.assets; + if (cleaned.summaries === null) delete cleaned.summaries; + + return cleaned; + }); + res.json({ - type: 'FeatureCollection', - collections, + collections: enrichedCollections, links, context: { returned, @@ -126,22 +226,24 @@ router.get('/', validateCollectionSearchParams, async (req, res, next) => { * - 200 OK with full Collection object if found * - 404 NotFound with proper error format if collection does not exist */ -router.get('/:id', (req, res) => { - // TODO: Create a proper validator middleware for :id parameter to avoid SQL injection, etc. - const { id } = req.params; - - // Look up the collection in the data store by ID - // When connected to a DB, replace this with a SQL query (SELECT * FROM collections WHERE id = ?) - const collection = collectionsStore.find(c => c.id === id); - - if (!collection) { - // Return 404 with standardized error format - return res.status(404).json({ - code: 'NotFound', - description: `Collection with id '${id}' not found`, - id: id - }); - } +router.get('/:id', async (req, res, next) => { + try { + const { id } = req.params; + + // Query the database for the collection by ID + const sql = 'SELECT * FROM collections WHERE id = $1'; + const result = await query(sql, [id]); + + if (result.rows.length === 0) { + // Return 404 with standardized error format + return res.status(404).json({ + code: 'NotFound', + description: `Collection with id '${id}' not found`, + id: id + }); + } + + const collection = result.rows[0]; // Return the full STAC Collection object // Ensure the response includes at least self, root and parent links. @@ -151,24 +253,49 @@ router.get('/:id', (req, res) => { const rootHref = baseHost; const existingLinks = Array.isArray(collection.links) ? collection.links.slice() : []; + const filteredLinks = existingLinks.filter(l => !l || !['self', 'root', 'parent'].includes(l.rel)); - const hasRel = (rel) => existingLinks.some(l => l && l.rel === rel); + filteredLinks.push({ + rel: 'self', + href: selfHref, + type: 'application/json', + title: collection.title || collection.id + }); - if (!hasRel('self')) { - existingLinks.push({ rel: 'self', href: selfHref, type: 'application/json' }); - } + filteredLinks.push({ + rel: 'root', + href: rootHref, + type: 'application/json', + title: 'STAC Atlas' + }); - if (!hasRel('root')) { - existingLinks.push({ rel: 'root', href: rootHref, type: 'application/json' }); - } + filteredLinks.push({ + rel: 'parent', + href: rootHref, + type: 'application/json', + title: 'Parent' + }); - // Prefer an existing parent link if present, otherwise fall back to root - if (!hasRel('parent')) { - existingLinks.push({ rel: 'parent', href: rootHref, type: 'application/json' }); - } + // Ensure stac_extensions is an array (STAC spec requires array, not null) + const stac_extensions = Array.isArray(collection.stac_extensions) + ? collection.stac_extensions + : []; + + // Ensure id is a string (STAC spec requires string IDs) + const collectionId = typeof collection.id === 'string' ? collection.id : String(collection.id); - // Return the collection with a normalized `links` array - res.json(Object.assign({}, collection, { links: existingLinks })); + // Build response and remove null optional fields for STAC compliance + const collectionResponse = Object.assign({}, collection, { id: collectionId, links: filteredLinks, stac_extensions }); + + // Remove null assets, summaries (optional in STAC, should be omitted if not present) + if (collectionResponse.assets === null) delete collectionResponse.assets; + if (collectionResponse.summaries === null) delete collectionResponse.summaries; + + // Return the collection with a normalized `links` array and stac_extensions + res.json(collectionResponse); + } catch (error) { + next(error); + } }); module.exports = router;