diff --git a/.github/workflows/api-ci.yml b/.github/workflows/api-ci.yml index 5394698..68c87c0 100644 --- a/.github/workflows/api-ci.yml +++ b/.github/workflows/api-ci.yml @@ -71,7 +71,7 @@ jobs: # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata - API_VERSION=1.1.0 + API_VERSION=1.0.0 EOF # Step 4: Install dependencies @@ -164,7 +164,7 @@ jobs: # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata - API_VERSION=1.1.0 + API_VERSION=1.0.0 EOF - name: Install dependencies diff --git a/api/.env.example b/api/.env.example index 5906869..039ac70 100644 --- a/api/.env.example +++ b/api/.env.example @@ -28,4 +28,4 @@ CORS_ORIGIN=* # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata -API_VERSION=1.1.0 +API_VERSION=1.0.0 diff --git a/api/README.md b/api/README.md index 00cc72e..aff0ae1 100644 --- a/api/README.md +++ b/api/README.md @@ -156,59 +156,13 @@ CORS_ORIGIN=* Diese API implementiert: -- ✅ STAC API Core (v1.1.0) +- ✅ STAC API Core (v1.0.0) - ✅ OGC API Features Core - ✅ STAC Collections - ✅ Collection Search Extension - 🚧 CQL2 Basic Filtering (in Entwicklung) - 🚧 CQL2 Advanced Operators (in Entwicklung) -### STAC API Validator - -The API can be tested using the official [STAC API Validator](https://github.com/stac-utils/stac-api-validator): - -#### Installation - -```bash -# Python 3.11 required -pip install stac-api-validator -``` - -#### Usage - -```bash -# Validate Core Conformance Class -python -m stac_api_validator --root-url http://localhost:3000 --conformance core - -# Validate Collections Extension (requires collection ID) -python -m stac_api_validator \ - --root-url http://localhost:3000 \ - --conformance core \ - --conformance collections \ - --collection - -# With spatial filtering (requires geometry in dataset) -python -m stac_api_validator \ - --root-url http://localhost:3000 \ - --conformance core \ - --conformance collections \ - --collection \ - --geometry '{"type": "Polygon", "coordinates": [[[7.0, 51.0], [8.0, 51.0], [8.0, 52.0], [7.0, 52.0], [7.0, 51.0]]]}' -``` - -#### Validation Status - -| Conformance Class | Status | Date | Errors | Warnings | -|-------------------|--------|------|--------|----------| -| **STAC API - Core** | ✅ Passed | 2025-12-10 | 0 | 0 | -| STAC API - Collections | ⏳ Pending | - | - | - | -| STAC API - Features | ⏳ Pending | - | - | - | -| STAC API - Item Search | ⏳ Pending | - | - | - | -| CQL2 - Basic | ⏳ Pending | - | - | - | -| CQL2 - Advanced | ⏳ Pending | - | - | - | - -**Note:** The Collection Search Extension is not currently validated automatically by the validator and is instead validated through custom Jest integration tests (see `__tests__/`). - ## 📦 Nächste Schritte ### TODO diff --git a/api/__tests__/buildCollectionSearchQuery.aggregates.test.js b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js deleted file mode 100644 index 069ea46..0000000 --- a/api/__tests__/buildCollectionSearchQuery.aggregates.test.js +++ /dev/null @@ -1,221 +0,0 @@ -const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); - -describe('buildCollectionSearchQuery - aggregated fields', () => { - test('SELECT includes all collection base columns with alias c', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - // Core collection fields should be prefixed with 'c.' - expect(sql).toMatch(/c\.id/); - expect(sql).toMatch(/c\.stac_version/); - expect(sql).toMatch(/c\.type/); - expect(sql).toMatch(/c\.title/); - expect(sql).toMatch(/c\.description/); - expect(sql).toMatch(/c\.license/); - expect(sql).toMatch(/c\.spatial_extend/); - expect(sql).toMatch(/c\.temporal_extend_start/); - expect(sql).toMatch(/c\.temporal_extend_end/); - expect(sql).toMatch(/c\.created_at/); - expect(sql).toMatch(/c\.updated_at/); - expect(sql).toMatch(/c\.is_api/); - expect(sql).toMatch(/c\.is_active/); - expect(sql).toMatch(/c\.full_json/); - }); - - test('SELECT includes aggregated relation fields', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - // Aggregated fields from LATERAL JOINs - expect(sql).toMatch(/kw\.keywords/); - expect(sql).toMatch(/ext\.stac_extensions/); - expect(sql).toMatch(/prov\.providers/); - expect(sql).toMatch(/a\.assets/); - expect(sql).toMatch(/s\.summaries/); - expect(sql).toMatch(/cl\.last_crawled/); - }); - - test('FROM clause uses collection alias c', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/FROM collection c/); - }); - - describe('LATERAL JOINs for normalized data', () => { - test('includes LATERAL JOIN for keywords', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/LEFT JOIN LATERAL/); - expect(sql).toMatch(/jsonb_agg\(k\.keyword ORDER BY k\.keyword\) AS keywords/); - expect(sql).toMatch(/FROM collection_keywords ck/); - expect(sql).toMatch(/JOIN keywords k ON k\.id = ck\.keyword_id/); - expect(sql).toMatch(/WHERE ck\.collection_id = c\.id/); - }); - - test('includes LATERAL JOIN for stac_extensions', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/jsonb_agg\(se\.stac_extension ORDER BY se\.stac_extension\) AS stac_extensions/); - expect(sql).toMatch(/FROM collection_stac_extension cse/); - expect(sql).toMatch(/JOIN stac_extensions se ON se\.id = cse\.stac_extension_id/); - expect(sql).toMatch(/WHERE cse\.collection_id = c\.id/); - }); - - test('includes LATERAL JOIN for providers with roles', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/jsonb_agg\(jsonb_build_object\(/); - expect(sql).toMatch(/'name', p\.provider/); - expect(sql).toMatch(/'roles', cpr\.collection_provider_roles/); - expect(sql).toMatch(/FROM collection_providers cpr/); - expect(sql).toMatch(/JOIN providers p ON p\.id = cpr\.provider_id/); - expect(sql).toMatch(/WHERE cpr\.collection_id = c\.id/); - }); - - test('includes LATERAL JOIN for assets with metadata', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/'name', a\.name/); - expect(sql).toMatch(/'href', a\.href/); - expect(sql).toMatch(/'type', a\.type/); - expect(sql).toMatch(/'roles', a\.roles/); - expect(sql).toMatch(/'metadata', a\.metadata/); - expect(sql).toMatch(/'collection_roles', ca\.collection_asset_roles/); - expect(sql).toMatch(/FROM collection_assets ca/); - expect(sql).toMatch(/JOIN assets a ON a\.id = ca\.asset_id/); - expect(sql).toMatch(/WHERE ca\.collection_id = c\.id/); - }); - - test('includes LATERAL JOIN for summaries with CASE logic', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/jsonb_object_agg\(s\.name, s\.s_summary\) AS summaries/); - expect(sql).toMatch(/WHEN cs\.kind = 'range' THEN jsonb_build_object\('min', cs\.range_min, 'max', cs\.range_max\)/); - expect(sql).toMatch(/WHEN cs\.kind = 'set' THEN to_jsonb\(cs\.set_value\)/); - expect(sql).toMatch(/FROM collection_summaries cs/); - expect(sql).toMatch(/WHERE cs\.collection_id = c\.id/); - }); - - test('includes LATERAL JOIN for last_crawled timestamp', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/MAX\(clc\.last_crawled\) AS last_crawled/); - expect(sql).toMatch(/FROM crawllog_collection clc/); - expect(sql).toMatch(/WHERE clc\.collection_id = c\.id/); - }); - }); - - describe('WHERE clauses use collection alias c', () => { - test('bbox filter uses c.spatial_extend', () => { - const bbox = [-10, 40, 10, 50]; - const { sql } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); - - expect(sql).toMatch(/c\.spatial_extend/); - expect(sql).toMatch(/ST_Intersects\(\s*c\.spatial_extend/); - }); - - test('datetime filter uses c.temporal_extend_start and c.temporal_extend_end', () => { - const datetime = '2020-01-01/2021-12-31'; - const { sql } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); - - expect(sql).toMatch(/c\.temporal_extend_end >= \$/); - expect(sql).toMatch(/c\.temporal_extend_start <= \$/); - }); - - test('fulltext search uses c.title and c.description', () => { - const q = 'satellite'; - const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); - - expect(sql).toMatch(/coalesce\(c\.title,''\)/); - expect(sql).toMatch(/coalesce\(c\.description,''\)/); - expect(sql).toMatch(/to_tsvector\('simple', coalesce\(c\.title,''\) \|\| ' ' \|\| coalesce\(c\.description,''\)\)/); - }); - }); - - describe('ORDER BY uses collection alias c', () => { - test('default ORDER BY uses c.id', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - expect(sql).toMatch(/ORDER BY c\.id ASC/); - }); - - test('sortby parameter uses c. prefix', () => { - const sortby = { field: 'title', direction: 'DESC' }; - const { sql } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); - - expect(sql).toMatch(/ORDER BY c\.title DESC/); - }); - - test('fulltext search with rank orders by rank DESC, c.id ASC', () => { - const q = 'satellite'; - const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); - - expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); - }); - }); - - describe('Parameterized values remain correct', () => { - test('bbox parameters are in correct order', () => { - const bbox = [-10, 40, 10, 50]; - const { values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); - - expect(values.slice(0, 4)).toEqual(bbox); - expect(values[4]).toBe(10); // limit - expect(values[5]).toBe(0); // token - }); - - test('datetime interval parameters are in correct order', () => { - const datetime = '2020-01-01/2021-12-31'; - const { values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); - - expect(values[0]).toBe('2020-01-01'); - expect(values[1]).toBe('2021-12-31'); - expect(values[2]).toBe(10); // limit - expect(values[3]).toBe(0); // token - }); - - test('fulltext query parameter is bound correctly', () => { - const q = 'satellite'; - const { values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); - - expect(values[0]).toBe('satellite'); - expect(values[1]).toBe(10); // limit - expect(values[2]).toBe(0); // token - }); - - test('combined filters maintain parameter order', () => { - const bbox = [-10, 40, 10, 50]; - const datetime = '2020-01-01/2021-12-31'; - const q = 'satellite'; - const { values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 10, token: 0 }); - - // Order: q (1), bbox (4), datetime start (1), datetime end (1), limit (1), token (1) = 9 values - expect(values[0]).toBe('satellite'); - expect(values.slice(1, 5)).toEqual(bbox); - expect(values[5]).toBe('2020-01-01'); - expect(values[6]).toBe('2021-12-31'); - expect(values[7]).toBe(10); - expect(values[8]).toBe(0); - }); - }); - - describe('SQL structure validation', () => { - test('no DISTINCT in jsonb_agg to avoid ORDER BY conflict', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - // DISTINCT should NOT appear in any jsonb_agg calls - // (PostgreSQL requires ORDER BY expressions to appear in DISTINCT argument list) - const distinctPattern = /jsonb_agg\(DISTINCT/gi; - const matches = sql.match(distinctPattern); - - expect(matches).toBeNull(); - }); - - test('all LATERAL JOINs are LEFT JOIN', () => { - const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - - // Count LEFT JOIN LATERAL occurrences (should be 6: kw, ext, prov, a, s, cl) - const leftJoinLateralCount = (sql.match(/LEFT JOIN LATERAL/gi) || []).length; - - expect(leftJoinLateralCount).toBe(6); - }); - }); -}); diff --git a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js index 99a9f07..5c8100d 100644 --- a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js +++ b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js @@ -13,7 +13,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { expect(sql).toMatch(/AS rank/); // Ordering defaults to rank DESC when q present and no sortby - expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); + expect(sql).toMatch(/ORDER BY rank DESC, id ASC/); // values: [q, limit, token] expect(values[0]).toBe('forest'); @@ -24,7 +24,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { test('explicit sortby overrides rank ordering', () => { const { sql } = buildCollectionSearchQuery({ q: 'lake', sortby: { field: 'title', direction: 'ASC' }, limit: 5, token: 0 }); - expect(sql).toMatch(/ORDER BY c\.title ASC/); + expect(sql).toMatch(/ORDER BY title ASC/); // rank still present in select expect(sql).toMatch(/AS rank/); }); diff --git a/api/__tests__/buildCollectionSearchQuery.integration.test.js b/api/__tests__/buildCollectionSearchQuery.integration.test.js deleted file mode 100644 index 2885fa0..0000000 --- a/api/__tests__/buildCollectionSearchQuery.integration.test.js +++ /dev/null @@ -1,322 +0,0 @@ -const { query, closePool } = require('../db/db_APIconnection'); -const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); - -/** - * Integration Tests: Aggregated Fields in Collection Search Query - * - * These tests verify that the LATERAL JOINs correctly aggregate data from - * normalized tables (keywords, providers, assets, stac_extensions, summaries, crawllog). - * - * Prerequisites: - * - Database must be initialized with schema (01-05_*.sql) - * - Test data should include collections with related entities - */ - -describe('Integration: Collection Search with Aggregated Fields', () => { - afterAll(async () => { - await closePool(); - }); - - describe('Query Execution', () => { - test('should execute query successfully without errors', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); - - await expect(query(sql, values)).resolves.not.toThrow(); - }); - - test('should return rows with aggregated fields', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); - const result = await query(sql, values); - - expect(result.rows).toBeDefined(); - expect(Array.isArray(result.rows)).toBe(true); - - // If there are collections in DB, verify structure - if (result.rows.length > 0) { - const firstRow = result.rows[0]; - - // Core collection fields - expect(firstRow).toHaveProperty('id'); - expect(firstRow).toHaveProperty('title'); - expect(firstRow).toHaveProperty('description'); - expect(firstRow).toHaveProperty('license'); - expect(firstRow).toHaveProperty('full_json'); - - // Aggregated fields (may be null if no related data) - expect(firstRow).toHaveProperty('keywords'); - expect(firstRow).toHaveProperty('stac_extensions'); - expect(firstRow).toHaveProperty('providers'); - expect(firstRow).toHaveProperty('assets'); - expect(firstRow).toHaveProperty('summaries'); - expect(firstRow).toHaveProperty('last_crawled'); - } - }); - }); - - describe('Aggregated Field Types', () => { - test('keywords should be JSONB array or null', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - const result = await query(sql, values); - - result.rows.forEach(row => { - if (row.keywords !== null) { - expect(Array.isArray(row.keywords)).toBe(true); - // Each keyword should be a string - row.keywords.forEach(kw => { - expect(typeof kw).toBe('string'); - }); - } - }); - }); - - test('stac_extensions should be JSONB array or null', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - const result = await query(sql, values); - - result.rows.forEach(row => { - if (row.stac_extensions !== null) { - expect(Array.isArray(row.stac_extensions)).toBe(true); - row.stac_extensions.forEach(ext => { - expect(typeof ext).toBe('string'); - }); - } - }); - }); - - test('providers should be JSONB array of objects or null', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - const result = await query(sql, values); - - result.rows.forEach(row => { - if (row.providers !== null) { - expect(Array.isArray(row.providers)).toBe(true); - row.providers.forEach(provider => { - expect(provider).toHaveProperty('name'); - expect(provider).toHaveProperty('roles'); - expect(typeof provider.name).toBe('string'); - }); - } - }); - }); - - test('assets should be JSONB array of objects or null', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - const result = await query(sql, values); - - result.rows.forEach(row => { - if (row.assets !== null) { - expect(Array.isArray(row.assets)).toBe(true); - row.assets.forEach(asset => { - expect(asset).toHaveProperty('name'); - expect(asset).toHaveProperty('href'); - expect(asset).toHaveProperty('type'); - expect(asset).toHaveProperty('roles'); - expect(asset).toHaveProperty('metadata'); - expect(asset).toHaveProperty('collection_roles'); - }); - } - }); - }); - - test('summaries should be JSONB object or null', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - const result = await query(sql, values); - - result.rows.forEach(row => { - if (row.summaries !== null) { - expect(typeof row.summaries).toBe('object'); - expect(Array.isArray(row.summaries)).toBe(false); - - // Each summary should be a range, set, or schema object - Object.values(row.summaries).forEach(summary => { - const hasRange = summary.min !== undefined && summary.max !== undefined; - const isSet = Array.isArray(summary) || typeof summary === 'string'; - const isSchema = typeof summary === 'object'; - - expect(hasRange || isSet || isSchema).toBe(true); - }); - } - }); - }); - - test('last_crawled should be timestamp or null', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - const result = await query(sql, values); - - result.rows.forEach(row => { - if (row.last_crawled !== null) { - // Should be a valid Date or parseable timestamp - const date = new Date(row.last_crawled); - expect(date.toString()).not.toBe('Invalid Date'); - } - }); - }); - }); - - describe('Filter Compatibility with Aggregated Fields', () => { - test('bbox filter works with aggregated fields', async () => { - const bbox = [-180, -90, 180, 90]; // World bbox - const { sql, values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); - - const result = await query(sql, values); - - expect(result.rows).toBeDefined(); - // All returned rows should have the aggregated structure - result.rows.forEach(row => { - expect(row).toHaveProperty('keywords'); - expect(row).toHaveProperty('providers'); - expect(row).toHaveProperty('assets'); - }); - }); - - test('datetime filter works with aggregated fields', async () => { - const datetime = '2000-01-01/2030-12-31'; - const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); - - const result = await query(sql, values); - - expect(result.rows).toBeDefined(); - result.rows.forEach(row => { - expect(row).toHaveProperty('stac_extensions'); - expect(row).toHaveProperty('summaries'); - expect(row).toHaveProperty('last_crawled'); - }); - }); - - test('fulltext search works with aggregated fields', async () => { - const q = 'test'; - const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); - - const result = await query(sql, values); - - expect(result.rows).toBeDefined(); - result.rows.forEach(row => { - expect(row).toHaveProperty('keywords'); - expect(row).toHaveProperty('providers'); - }); - }); - - test('combined filters work with aggregated fields', async () => { - const bbox = [-180, -90, 180, 90]; - const datetime = '2000-01-01/2030-12-31'; - const q = 'satellite'; - const { sql, values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 5, token: 0 }); - - const result = await query(sql, values); - - expect(result.rows).toBeDefined(); - result.rows.forEach(row => { - // All aggregated fields should be present - expect(row).toHaveProperty('keywords'); - expect(row).toHaveProperty('stac_extensions'); - expect(row).toHaveProperty('providers'); - expect(row).toHaveProperty('assets'); - expect(row).toHaveProperty('summaries'); - expect(row).toHaveProperty('last_crawled'); - }); - }); - }); - - describe('Sorting with Aggregated Fields', () => { - test('default sort by c.id works with aggregated fields', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - const result = await query(sql, values); - - if (result.rows.length > 1) { - // IDs should be in ascending order - for (let i = 1; i < result.rows.length; i++) { - expect(result.rows[i].id).toBeGreaterThanOrEqual(result.rows[i - 1].id); - } - } - }); - - test('sort by title works with aggregated fields', async () => { - const sortby = { field: 'title', direction: 'ASC' }; - const { sql, values } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); - const result = await query(sql, values); - - // Verify SQL contains ORDER BY c.title ASC - expect(sql).toMatch(/ORDER BY c\.title ASC/); - - // Verify all aggregated fields are present - expect(result.rows).toBeDefined(); - result.rows.forEach(row => { - expect(row).toHaveProperty('keywords'); - expect(row).toHaveProperty('providers'); - expect(row).toHaveProperty('assets'); - }); - }); - - test('fulltext rank sort works with aggregated fields', async () => { - const q = 'satellite'; - const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); - const result = await query(sql, values); - - // Should execute without error; rank ordering is implicit in SQL - expect(result.rows).toBeDefined(); - result.rows.forEach(row => { - expect(row).toHaveProperty('keywords'); - expect(row).toHaveProperty('providers'); - }); - }); - }); - - describe('Pagination with Aggregated Fields', () => { - test('first page returns correct structure', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 3, token: 0 }); - const result = await query(sql, values); - - expect(result.rows.length).toBeLessThanOrEqual(3); - result.rows.forEach(row => { - expect(row).toHaveProperty('id'); - expect(row).toHaveProperty('keywords'); - expect(row).toHaveProperty('providers'); - }); - }); - - test('second page returns different rows with same structure', async () => { - const page1 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 0 }))); - const page2 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 3 }))); - - if (page1.rows.length > 0 && page2.rows.length > 0) { - // IDs should be different - const page1Ids = page1.rows.map(r => r.id); - const page2Ids = page2.rows.map(r => r.id); - - const overlap = page1Ids.filter(id => page2Ids.includes(id)); - expect(overlap.length).toBe(0); - - // Both pages should have same structure - page2.rows.forEach(row => { - expect(row).toHaveProperty('keywords'); - expect(row).toHaveProperty('providers'); - expect(row).toHaveProperty('assets'); - }); - } - }); - }); - - describe('Performance and Cardinality', () => { - test('LATERAL JOINs do not duplicate collection rows', async () => { - const { sql, values } = buildCollectionSearchQuery({ limit: 100, token: 0 }); - const result = await query(sql, values); - - // Collect all IDs - const ids = result.rows.map(r => r.id); - const uniqueIds = [...new Set(ids)]; - - // No duplicates: each collection should appear exactly once - expect(ids.length).toBe(uniqueIds.length); - }); - - test('query executes in reasonable time (<5s for small dataset)', async () => { - const start = Date.now(); - const { sql, values } = buildCollectionSearchQuery({ limit: 50, token: 0 }); - await query(sql, values); - const duration = Date.now() - start; - - // Should complete within 5 seconds for typical test datasets - expect(duration).toBeLessThan(5000); - }, 10000); // 10s timeout for Jest - }); -}); diff --git a/api/__tests__/buildCollectionsSearchQuery.basic.test.js b/api/__tests__/buildCollectionsSearchQuery.basic.test.js index 4acd7b9..8756a5f 100644 --- a/api/__tests__/buildCollectionsSearchQuery.basic.test.js +++ b/api/__tests__/buildCollectionsSearchQuery.basic.test.js @@ -4,8 +4,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { test('no params returns base SQL with LIMIT/OFFSET placeholders', () => { const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - expect(sql).toMatch(/FROM collection c/); - expect(sql).toMatch(/ORDER BY c\.id ASC/); + expect(sql).toMatch(/FROM collection/); + expect(sql).toMatch(/ORDER BY id ASC/); // there should be LIMIT and OFFSET placeholders expect(sql).toMatch(/LIMIT \$1 OFFSET \$2/); expect(Array.isArray(values)).toBe(true); @@ -30,8 +30,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { const datetime = '2020-01-01/2021-12-31'; const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); - expect(sql).toMatch(/c\.temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated - expect(sql).toMatch(/c\.temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated + expect(sql).toMatch(/temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated + expect(sql).toMatch(/temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated // values order: start, end, limit, token expect(values[0]).toBe('2020-01-01'); expect(values[1]).toBe('2021-12-31'); diff --git a/api/app.js b/api/app.js index a5b0393..bf5f4f5 100644 --- a/api/app.js +++ b/api/app.js @@ -26,27 +26,7 @@ app.use(cors({ allowedHeaders: ['Content-Type', 'Authorization'] })); -// OpenAPI spec endpoint (YAML file with correct content-type) - MUST be before Content-Type middleware -app.get('/openapi.yaml', (req, res, next) => { - try { - const openapiPath = path.join(__dirname, 'docs', 'openapi.yaml'); - res.setHeader('Content-Type', 'application/vnd.oai.openapi+json;version=3.0'); - res.sendFile(openapiPath); - } catch (err) { - next(err); - } -}); - -// Swagger/OpenAPI documentation (if openapi.yaml exists) - MUST be before Content-Type middleware -try { - const swaggerDocument = YAML.load(path.join(__dirname, 'docs', 'openapi.yaml')); - app.use('/api-docs', swaggerUi.serve); - app.get('/api-docs', swaggerUi.setup(swaggerDocument)); -} catch (err) { - console.log('OpenAPI documentation not found. Create docs/openapi.yaml to enable Swagger UI.'); -} - -// Content-Type header for JSON responses (set AFTER special endpoints) +// Content-Type header for all JSON responses app.use((req, res, next) => { res.setHeader('Content-Type', 'application/json'); next(); @@ -58,6 +38,14 @@ app.use('/conformance', conformanceRouter); app.use('/collections', collectionsRouter); app.use('/queryables', queryablesRouter); +// Swagger/OpenAPI documentation (if openapi.yaml exists) +try { + const swaggerDocument = YAML.load(path.join(__dirname, 'docs', 'openapi.yaml')); + app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument)); +} catch (err) { + console.log('OpenAPI documentation not found. Create docs/openapi.yaml to enable Swagger UI.'); +} + // 404 handler app.use((req, res, next) => { res.status(404).json({ diff --git a/api/db/buildCollectionSearchQuery.js b/api/db/buildCollectionSearchQuery.js index 8d43cee..cc1d435 100644 --- a/api/db/buildCollectionSearchQuery.js +++ b/api/db/buildCollectionSearchQuery.js @@ -83,31 +83,22 @@ function buildCollectionSearchQuery(params) { // full-text search) *before* the `FROM` clause. Keeping `sql` fixed with a // `FROM` already included would make inserting additional selected columns // harder and error-prone when building the query dynamically. - // - // We use alias 'c' for the collection table to simplify JOIN expressions and - // distinguish collection columns from aggregated relation data (keywords, providers, etc.). let selectPart = ` SELECT - c.id, - c.stac_version, - c.type, - c.title, - c.description, - c.license, - c.spatial_extend, - c.temporal_extend_start, - c.temporal_extend_end, - c.created_at, - c.updated_at, - c.is_api, - c.is_active, - c.full_json, - kw.keywords, - ext.stac_extensions, - prov.providers, - a.assets, - s.summaries, - cl.last_crawled + id, + stac_version, + type, + title, + description, + license, + spatial_extend, + temporal_extend_start, + temporal_extend_end, + created_at, + updated_at, + is_api, + is_active, + full_json `; const where = []; @@ -132,8 +123,8 @@ function buildCollectionSearchQuery(params) { if (q) { const queryIndex = i; // remember index to reuse for rank and condition - // Weighted combined tsvector expression (using alias 'c' for collection table) - const vectorExpr = `to_tsvector('simple', coalesce(c.title,'') || ' ' || coalesce(c.description,''))`; + // Weighted combined tsvector expression + const vectorExpr = `to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))`; // Add rank to selected columns (ts_rank_cd => constant-duration ranking function) // The computed `rank` is available in the result rows and used for ordering @@ -153,7 +144,7 @@ function buildCollectionSearchQuery(params) { where.push(` ST_Intersects( - c.spatial_extend, + spatial_extend, ST_MakeEnvelope($${i}, $${i + 1}, $${i + 2}, $${i + 3}, 4326) ) `); @@ -170,92 +161,33 @@ function buildCollectionSearchQuery(params) { if (start !== '..') { // Collection should run after start - where.push(`c.temporal_extend_end >= $${i}`); + where.push(`temporal_extend_end >= $${i}`); values.push(start); i++; } if (end !== '..') { // Collection should run before end - where.push(`c.temporal_extend_start <= $${i}`); + where.push(`temporal_extend_start <= $${i}`); values.push(end); i++; } } else { // single datetime: collections active at that time where.push(` - c.temporal_extend_start <= $${i} - AND c.temporal_extend_end >= $${i} + temporal_extend_start <= $${i} + AND temporal_extend_end >= $${i} `); values.push(datetime); i++; } } - // Build final SQL from selectPart and add FROM clause with LATERAL JOINs. + // Build final SQL from selectPart and add FROM clause. // We delayed adding `FROM collection` to allow conditional additions to the // selected columns above (notably `rank`). The final `sql` string includes the // selected columns, the source table and any WHERE conditions constructed earlier. - // - // LATERAL JOINs aggregate related data (keywords, extensions, providers, assets, summaries, - // and crawl timestamps) from normalized tables without duplicating collection rows. - // Each LEFT JOIN LATERAL subquery returns a single aggregated row per collection. - let sql = selectPart + ` - FROM collection c - LEFT JOIN LATERAL ( - SELECT jsonb_agg(k.keyword ORDER BY k.keyword) AS keywords - FROM collection_keywords ck - JOIN keywords k ON k.id = ck.keyword_id - WHERE ck.collection_id = c.id - ) kw ON TRUE - LEFT JOIN LATERAL ( - SELECT jsonb_agg(se.stac_extension ORDER BY se.stac_extension) AS stac_extensions - FROM collection_stac_extension cse - JOIN stac_extensions se ON se.id = cse.stac_extension_id - WHERE cse.collection_id = c.id - ) ext ON TRUE - LEFT JOIN LATERAL ( - SELECT jsonb_agg(jsonb_build_object( - 'name', p.provider, - 'roles', cpr.collection_provider_roles - ) ORDER BY p.provider) AS providers - FROM collection_providers cpr - JOIN providers p ON p.id = cpr.provider_id - WHERE cpr.collection_id = c.id - ) prov ON TRUE - LEFT JOIN LATERAL ( - SELECT jsonb_agg(jsonb_build_object( - 'name', a.name, - 'href', a.href, - 'type', a.type, - 'roles', a.roles, - 'metadata', a.metadata, - 'collection_roles', ca.collection_asset_roles - ) ORDER BY a.name) AS assets - FROM collection_assets ca - JOIN assets a ON a.id = ca.asset_id - WHERE ca.collection_id = c.id - ) a ON TRUE - LEFT JOIN LATERAL ( - SELECT jsonb_object_agg(s.name, s.s_summary) AS summaries - FROM ( - SELECT - cs.name, - CASE - WHEN cs.kind = 'range' THEN jsonb_build_object('min', cs.range_min, 'max', cs.range_max) - WHEN cs.kind = 'set' THEN to_jsonb(cs.set_value) - ELSE cs.json_schema - END AS s_summary - FROM collection_summaries cs - WHERE cs.collection_id = c.id - ) s - ) s ON TRUE - LEFT JOIN LATERAL ( - SELECT MAX(clc.last_crawled) AS last_crawled - FROM crawllog_collection clc - WHERE clc.collection_id = c.id - ) cl ON TRUE - `; + let sql = selectPart + `\n FROM collection\n `; if (where.length > 0) { sql += ` WHERE ` + where.join(' AND '); @@ -265,18 +197,15 @@ function buildCollectionSearchQuery(params) { // a text query was provided (descending), falling back to id ascending. // // Behaviour summary: - // - `sortby` provided → use that (with 'c.' prefix for collection columns) - // - no `sortby` & `q` present → order by `rank DESC, c.id ASC` so higher relevance comes first - // - no `sortby` & no `q` → order by `c.id ASC` (legacy default) - // - // Note: sortby.field is validated against a whitelist in the calling code; only collection - // table columns are allowed for sorting (not aggregated fields like keywords/providers). + // - `sortby` provided → use that (same as before) + // - no `sortby` & `q` present → order by `rank DESC, id ASC` so higher relevance comes first + // - no `sortby` & no `q` → order by `id ASC` (legacy default) if (sortby) { - sql += ` ORDER BY c.${sortby.field} ${sortby.direction}`; + sql += ` ORDER BY ${sortby.field} ${sortby.direction}`; } else if (q) { - sql += ` ORDER BY rank DESC, c.id ASC`; + sql += ` ORDER BY rank DESC, id ASC`; } else { - sql += ` ORDER BY c.id ASC`; + sql += ` ORDER BY id ASC`; } // Pagination (only add if limit is provided) diff --git a/api/docs/openapi.yaml b/api/docs/openapi.yaml deleted file mode 100644 index abcce52..0000000 --- a/api/docs/openapi.yaml +++ /dev/null @@ -1,351 +0,0 @@ -openapi: 3.0.3 -info: - title: STAC Atlas API - description: A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs. - version: 1.1.0 - contact: - name: SpatioCore - license: - name: Apache 2.0 - url: https://www.apache.org/licenses/LICENSE-2.0.html - -servers: - - url: http://localhost:3000 - description: Local development server - -paths: - /: - get: - summary: Landing Page - description: Returns the STAC API landing page with links to available resources - operationId: getLandingPage - tags: - - STAC Core - responses: - '200': - description: STAC API landing page - content: - application/json: - schema: - $ref: '#/components/schemas/LandingPage' - - /conformance: - get: - summary: Conformance Classes - description: Returns the conformance classes that this API implements - operationId: getConformance - tags: - - STAC Core - responses: - '200': - description: Conformance classes - content: - application/json: - schema: - $ref: '#/components/schemas/Conformance' - - /collections: - get: - summary: List Collections - description: Returns a list of STAC Collections with optional filtering - operationId: getCollections - tags: - - Collections - parameters: - - name: limit - in: query - description: Maximum number of collections to return - required: false - schema: - type: integer - minimum: 1 - maximum: 10000 - default: 10 - - name: offset - in: query - description: Number of collections to skip - required: false - schema: - type: integer - minimum: 0 - default: 0 - - name: bbox - in: query - description: Bounding box to filter collections [minLon,minLat,maxLon,maxLat] - required: false - schema: - type: array - items: - type: number - minItems: 4 - maxItems: 6 - - name: datetime - in: query - description: Temporal filter (single datetime or interval) - required: false - schema: - type: string - - name: q - in: query - description: Full-text search query - required: false - schema: - type: string - - name: filter - in: query - description: CQL2 filter expression - required: false - schema: - type: string - - name: filter-lang - in: query - description: Filter language (cql2-text or cql2-json) - required: false - schema: - type: string - enum: - - cql2-text - - cql2-json - default: cql2-text - - name: sortby - in: query - description: Sort order for results - required: false - schema: - type: string - responses: - '200': - description: List of collections - content: - application/json: - schema: - $ref: '#/components/schemas/Collections' - '400': - description: Bad request (invalid parameters) - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /collections/{collectionId}: - get: - summary: Get Collection - description: Returns a single STAC Collection by ID - operationId: getCollection - tags: - - Collections - parameters: - - name: collectionId - in: path - description: Collection identifier - required: true - schema: - type: string - responses: - '200': - description: A STAC Collection - content: - application/json: - schema: - $ref: '#/components/schemas/Collection' - '404': - description: Collection not found - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - - /queryables: - get: - summary: Global Queryables - description: Returns queryable properties for collection search - operationId: getQueryables - tags: - - Queryables - responses: - '200': - description: Queryables schema - content: - application/schema+json: - schema: - type: object - -components: - schemas: - LandingPage: - type: object - required: - - type - - id - - description - - links - - conformsTo - properties: - type: - type: string - enum: - - Catalog - id: - type: string - title: - type: string - description: - type: string - stac_version: - type: string - conformsTo: - type: array - items: - type: string - links: - type: array - items: - $ref: '#/components/schemas/Link' - - Conformance: - type: object - required: - - conformsTo - properties: - conformsTo: - type: array - items: - type: string - - Collections: - type: object - required: - - collections - - links - properties: - collections: - type: array - items: - $ref: '#/components/schemas/Collection' - links: - type: array - items: - $ref: '#/components/schemas/Link' - context: - $ref: '#/components/schemas/Context' - - Collection: - type: object - required: - - type - - id - - description - - license - - extent - - links - properties: - type: - type: string - enum: - - Collection - stac_version: - type: string - stac_extensions: - type: array - items: - type: string - id: - type: string - title: - type: string - description: - type: string - keywords: - type: array - items: - type: string - license: - type: string - providers: - type: array - items: - type: object - extent: - type: object - required: - - spatial - - temporal - properties: - spatial: - type: object - required: - - bbox - properties: - bbox: - type: array - items: - type: array - items: - type: number - temporal: - type: object - required: - - interval - properties: - interval: - type: array - items: - type: array - items: - type: string - nullable: true - links: - type: array - items: - $ref: '#/components/schemas/Link' - summaries: - type: object - assets: - type: object - - Link: - type: object - required: - - rel - - href - properties: - rel: - type: string - href: - type: string - type: - type: string - title: - type: string - - Context: - type: object - properties: - returned: - type: integer - minimum: 0 - limit: - type: integer - minimum: 1 - matched: - type: integer - minimum: 0 - - Error: - type: object - required: - - code - - description - properties: - code: - type: string - description: - type: string - -tags: - - name: STAC Core - description: STAC API Core endpoints - - name: Collections - description: Collection search and retrieval - - name: Queryables - description: Queryable properties diff --git a/api/routes/index.js b/api/routes/index.js index 14a7aca..b389488 100644 --- a/api/routes/index.js +++ b/api/routes/index.js @@ -16,7 +16,7 @@ router.get('/', (req, res) => { id: 'stac-atlas', title: 'STAC Atlas', description: 'A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs.', - stac_version: '1.1.0', + stac_version: '1.0.0', conformsTo: CONFORMANCE_URIS, links: [ {