From 5a7af5b7baf4f2e3a1141dfdc22bb4fca41597ba Mon Sep 17 00:00:00 2001 From: RobinGummels Date: Tue, 9 Dec 2025 11:20:41 +0100 Subject: [PATCH 1/6] Updated Query-Builder to get all necessary fields from all db_tables for collections. Added some tests and fixed some already existing tests, becuase now the tablenames start with the alias `c.`. --- ...ldCollectionSearchQuery.aggregates.test.js | 221 ++++++++++++ ...uildCollectionSearchQuery.fulltext.test.js | 4 +- ...dCollectionSearchQuery.integration.test.js | 322 ++++++++++++++++++ .../buildCollectionsSearchQuery.basic.test.js | 8 +- api/db/buildCollectionSearchQuery.js | 129 +++++-- 5 files changed, 649 insertions(+), 35 deletions(-) create mode 100644 api/__tests__/buildCollectionSearchQuery.aggregates.test.js create mode 100644 api/__tests__/buildCollectionSearchQuery.integration.test.js diff --git a/api/__tests__/buildCollectionSearchQuery.aggregates.test.js b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js new file mode 100644 index 0000000..069ea46 --- /dev/null +++ b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js @@ -0,0 +1,221 @@ +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +describe('buildCollectionSearchQuery - aggregated fields', () => { + test('SELECT includes all collection base columns with alias c', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Core collection fields should be prefixed with 'c.' + expect(sql).toMatch(/c\.id/); + expect(sql).toMatch(/c\.stac_version/); + expect(sql).toMatch(/c\.type/); + expect(sql).toMatch(/c\.title/); + expect(sql).toMatch(/c\.description/); + expect(sql).toMatch(/c\.license/); + expect(sql).toMatch(/c\.spatial_extend/); + expect(sql).toMatch(/c\.temporal_extend_start/); + expect(sql).toMatch(/c\.temporal_extend_end/); + expect(sql).toMatch(/c\.created_at/); + expect(sql).toMatch(/c\.updated_at/); + expect(sql).toMatch(/c\.is_api/); + expect(sql).toMatch(/c\.is_active/); + expect(sql).toMatch(/c\.full_json/); + }); + + test('SELECT includes aggregated relation fields', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Aggregated fields from LATERAL JOINs + expect(sql).toMatch(/kw\.keywords/); + expect(sql).toMatch(/ext\.stac_extensions/); + expect(sql).toMatch(/prov\.providers/); + expect(sql).toMatch(/a\.assets/); + expect(sql).toMatch(/s\.summaries/); + expect(sql).toMatch(/cl\.last_crawled/); + }); + + test('FROM clause uses collection alias c', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/FROM collection c/); + }); + + describe('LATERAL JOINs for normalized data', () => { + test('includes LATERAL JOIN for keywords', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/LEFT JOIN LATERAL/); + expect(sql).toMatch(/jsonb_agg\(k\.keyword ORDER BY k\.keyword\) AS keywords/); + expect(sql).toMatch(/FROM collection_keywords ck/); + expect(sql).toMatch(/JOIN keywords k ON k\.id = ck\.keyword_id/); + expect(sql).toMatch(/WHERE ck\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for stac_extensions', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_agg\(se\.stac_extension ORDER BY se\.stac_extension\) AS stac_extensions/); + expect(sql).toMatch(/FROM collection_stac_extension cse/); + expect(sql).toMatch(/JOIN stac_extensions se ON se\.id = cse\.stac_extension_id/); + expect(sql).toMatch(/WHERE cse\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for providers with roles', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_agg\(jsonb_build_object\(/); + expect(sql).toMatch(/'name', p\.provider/); + expect(sql).toMatch(/'roles', cpr\.collection_provider_roles/); + expect(sql).toMatch(/FROM collection_providers cpr/); + expect(sql).toMatch(/JOIN providers p ON p\.id = cpr\.provider_id/); + expect(sql).toMatch(/WHERE cpr\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for assets with metadata', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/'name', a\.name/); + expect(sql).toMatch(/'href', a\.href/); + expect(sql).toMatch(/'type', a\.type/); + expect(sql).toMatch(/'roles', a\.roles/); + expect(sql).toMatch(/'metadata', a\.metadata/); + expect(sql).toMatch(/'collection_roles', ca\.collection_asset_roles/); + expect(sql).toMatch(/FROM collection_assets ca/); + expect(sql).toMatch(/JOIN assets a ON a\.id = ca\.asset_id/); + expect(sql).toMatch(/WHERE ca\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for summaries with CASE logic', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_object_agg\(s\.name, s\.s_summary\) AS summaries/); + expect(sql).toMatch(/WHEN cs\.kind = 'range' THEN jsonb_build_object\('min', cs\.range_min, 'max', cs\.range_max\)/); + expect(sql).toMatch(/WHEN cs\.kind = 'set' THEN to_jsonb\(cs\.set_value\)/); + expect(sql).toMatch(/FROM collection_summaries cs/); + expect(sql).toMatch(/WHERE cs\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for last_crawled timestamp', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/MAX\(clc\.last_crawled\) AS last_crawled/); + expect(sql).toMatch(/FROM crawllog_collection clc/); + expect(sql).toMatch(/WHERE clc\.collection_id = c\.id/); + }); + }); + + describe('WHERE clauses use collection alias c', () => { + test('bbox filter uses c.spatial_extend', () => { + const bbox = [-10, 40, 10, 50]; + const { sql } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + expect(sql).toMatch(/c\.spatial_extend/); + expect(sql).toMatch(/ST_Intersects\(\s*c\.spatial_extend/); + }); + + test('datetime filter uses c.temporal_extend_start and c.temporal_extend_end', () => { + const datetime = '2020-01-01/2021-12-31'; + const { sql } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + expect(sql).toMatch(/c\.temporal_extend_end >= \$/); + expect(sql).toMatch(/c\.temporal_extend_start <= \$/); + }); + + test('fulltext search uses c.title and c.description', () => { + const q = 'satellite'; + const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(sql).toMatch(/coalesce\(c\.title,''\)/); + expect(sql).toMatch(/coalesce\(c\.description,''\)/); + expect(sql).toMatch(/to_tsvector\('simple', coalesce\(c\.title,''\) \|\| ' ' \|\| coalesce\(c\.description,''\)\)/); + }); + }); + + describe('ORDER BY uses collection alias c', () => { + test('default ORDER BY uses c.id', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY c\.id ASC/); + }); + + test('sortby parameter uses c. prefix', () => { + const sortby = { field: 'title', direction: 'DESC' }; + const { sql } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY c\.title DESC/); + }); + + test('fulltext search with rank orders by rank DESC, c.id ASC', () => { + const q = 'satellite'; + const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); + }); + }); + + describe('Parameterized values remain correct', () => { + test('bbox parameters are in correct order', () => { + const bbox = [-10, 40, 10, 50]; + const { values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + expect(values.slice(0, 4)).toEqual(bbox); + expect(values[4]).toBe(10); // limit + expect(values[5]).toBe(0); // token + }); + + test('datetime interval parameters are in correct order', () => { + const datetime = '2020-01-01/2021-12-31'; + const { values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + expect(values[0]).toBe('2020-01-01'); + expect(values[1]).toBe('2021-12-31'); + expect(values[2]).toBe(10); // limit + expect(values[3]).toBe(0); // token + }); + + test('fulltext query parameter is bound correctly', () => { + const q = 'satellite'; + const { values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(values[0]).toBe('satellite'); + expect(values[1]).toBe(10); // limit + expect(values[2]).toBe(0); // token + }); + + test('combined filters maintain parameter order', () => { + const bbox = [-10, 40, 10, 50]; + const datetime = '2020-01-01/2021-12-31'; + const q = 'satellite'; + const { values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 10, token: 0 }); + + // Order: q (1), bbox (4), datetime start (1), datetime end (1), limit (1), token (1) = 9 values + expect(values[0]).toBe('satellite'); + expect(values.slice(1, 5)).toEqual(bbox); + expect(values[5]).toBe('2020-01-01'); + expect(values[6]).toBe('2021-12-31'); + expect(values[7]).toBe(10); + expect(values[8]).toBe(0); + }); + }); + + describe('SQL structure validation', () => { + test('no DISTINCT in jsonb_agg to avoid ORDER BY conflict', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // DISTINCT should NOT appear in any jsonb_agg calls + // (PostgreSQL requires ORDER BY expressions to appear in DISTINCT argument list) + const distinctPattern = /jsonb_agg\(DISTINCT/gi; + const matches = sql.match(distinctPattern); + + expect(matches).toBeNull(); + }); + + test('all LATERAL JOINs are LEFT JOIN', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Count LEFT JOIN LATERAL occurrences (should be 6: kw, ext, prov, a, s, cl) + const leftJoinLateralCount = (sql.match(/LEFT JOIN LATERAL/gi) || []).length; + + expect(leftJoinLateralCount).toBe(6); + }); + }); +}); diff --git a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js index 5c8100d..99a9f07 100644 --- a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js +++ b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js @@ -13,7 +13,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { expect(sql).toMatch(/AS rank/); // Ordering defaults to rank DESC when q present and no sortby - expect(sql).toMatch(/ORDER BY rank DESC, id ASC/); + expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); // values: [q, limit, token] expect(values[0]).toBe('forest'); @@ -24,7 +24,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { test('explicit sortby overrides rank ordering', () => { const { sql } = buildCollectionSearchQuery({ q: 'lake', sortby: { field: 'title', direction: 'ASC' }, limit: 5, token: 0 }); - expect(sql).toMatch(/ORDER BY title ASC/); + expect(sql).toMatch(/ORDER BY c\.title ASC/); // rank still present in select expect(sql).toMatch(/AS rank/); }); diff --git a/api/__tests__/buildCollectionSearchQuery.integration.test.js b/api/__tests__/buildCollectionSearchQuery.integration.test.js new file mode 100644 index 0000000..2885fa0 --- /dev/null +++ b/api/__tests__/buildCollectionSearchQuery.integration.test.js @@ -0,0 +1,322 @@ +const { query, closePool } = require('../db/db_APIconnection'); +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +/** + * Integration Tests: Aggregated Fields in Collection Search Query + * + * These tests verify that the LATERAL JOINs correctly aggregate data from + * normalized tables (keywords, providers, assets, stac_extensions, summaries, crawllog). + * + * Prerequisites: + * - Database must be initialized with schema (01-05_*.sql) + * - Test data should include collections with related entities + */ + +describe('Integration: Collection Search with Aggregated Fields', () => { + afterAll(async () => { + await closePool(); + }); + + describe('Query Execution', () => { + test('should execute query successfully without errors', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); + + await expect(query(sql, values)).resolves.not.toThrow(); + }); + + test('should return rows with aggregated fields', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + expect(Array.isArray(result.rows)).toBe(true); + + // If there are collections in DB, verify structure + if (result.rows.length > 0) { + const firstRow = result.rows[0]; + + // Core collection fields + expect(firstRow).toHaveProperty('id'); + expect(firstRow).toHaveProperty('title'); + expect(firstRow).toHaveProperty('description'); + expect(firstRow).toHaveProperty('license'); + expect(firstRow).toHaveProperty('full_json'); + + // Aggregated fields (may be null if no related data) + expect(firstRow).toHaveProperty('keywords'); + expect(firstRow).toHaveProperty('stac_extensions'); + expect(firstRow).toHaveProperty('providers'); + expect(firstRow).toHaveProperty('assets'); + expect(firstRow).toHaveProperty('summaries'); + expect(firstRow).toHaveProperty('last_crawled'); + } + }); + }); + + describe('Aggregated Field Types', () => { + test('keywords should be JSONB array or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.keywords !== null) { + expect(Array.isArray(row.keywords)).toBe(true); + // Each keyword should be a string + row.keywords.forEach(kw => { + expect(typeof kw).toBe('string'); + }); + } + }); + }); + + test('stac_extensions should be JSONB array or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.stac_extensions !== null) { + expect(Array.isArray(row.stac_extensions)).toBe(true); + row.stac_extensions.forEach(ext => { + expect(typeof ext).toBe('string'); + }); + } + }); + }); + + test('providers should be JSONB array of objects or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.providers !== null) { + expect(Array.isArray(row.providers)).toBe(true); + row.providers.forEach(provider => { + expect(provider).toHaveProperty('name'); + expect(provider).toHaveProperty('roles'); + expect(typeof provider.name).toBe('string'); + }); + } + }); + }); + + test('assets should be JSONB array of objects or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.assets !== null) { + expect(Array.isArray(row.assets)).toBe(true); + row.assets.forEach(asset => { + expect(asset).toHaveProperty('name'); + expect(asset).toHaveProperty('href'); + expect(asset).toHaveProperty('type'); + expect(asset).toHaveProperty('roles'); + expect(asset).toHaveProperty('metadata'); + expect(asset).toHaveProperty('collection_roles'); + }); + } + }); + }); + + test('summaries should be JSONB object or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.summaries !== null) { + expect(typeof row.summaries).toBe('object'); + expect(Array.isArray(row.summaries)).toBe(false); + + // Each summary should be a range, set, or schema object + Object.values(row.summaries).forEach(summary => { + const hasRange = summary.min !== undefined && summary.max !== undefined; + const isSet = Array.isArray(summary) || typeof summary === 'string'; + const isSchema = typeof summary === 'object'; + + expect(hasRange || isSet || isSchema).toBe(true); + }); + } + }); + }); + + test('last_crawled should be timestamp or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.last_crawled !== null) { + // Should be a valid Date or parseable timestamp + const date = new Date(row.last_crawled); + expect(date.toString()).not.toBe('Invalid Date'); + } + }); + }); + }); + + describe('Filter Compatibility with Aggregated Fields', () => { + test('bbox filter works with aggregated fields', async () => { + const bbox = [-180, -90, 180, 90]; // World bbox + const { sql, values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + // All returned rows should have the aggregated structure + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + }); + + test('datetime filter works with aggregated fields', async () => { + const datetime = '2000-01-01/2030-12-31'; + const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('stac_extensions'); + expect(row).toHaveProperty('summaries'); + expect(row).toHaveProperty('last_crawled'); + }); + }); + + test('fulltext search works with aggregated fields', async () => { + const q = 'test'; + const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + + test('combined filters work with aggregated fields', async () => { + const bbox = [-180, -90, 180, 90]; + const datetime = '2000-01-01/2030-12-31'; + const q = 'satellite'; + const { sql, values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 5, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + // All aggregated fields should be present + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('stac_extensions'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + expect(row).toHaveProperty('summaries'); + expect(row).toHaveProperty('last_crawled'); + }); + }); + }); + + describe('Sorting with Aggregated Fields', () => { + test('default sort by c.id works with aggregated fields', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + if (result.rows.length > 1) { + // IDs should be in ascending order + for (let i = 1; i < result.rows.length; i++) { + expect(result.rows[i].id).toBeGreaterThanOrEqual(result.rows[i - 1].id); + } + } + }); + + test('sort by title works with aggregated fields', async () => { + const sortby = { field: 'title', direction: 'ASC' }; + const { sql, values } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); + const result = await query(sql, values); + + // Verify SQL contains ORDER BY c.title ASC + expect(sql).toMatch(/ORDER BY c\.title ASC/); + + // Verify all aggregated fields are present + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + }); + + test('fulltext rank sort works with aggregated fields', async () => { + const q = 'satellite'; + const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + const result = await query(sql, values); + + // Should execute without error; rank ordering is implicit in SQL + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + }); + + describe('Pagination with Aggregated Fields', () => { + test('first page returns correct structure', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 3, token: 0 }); + const result = await query(sql, values); + + expect(result.rows.length).toBeLessThanOrEqual(3); + result.rows.forEach(row => { + expect(row).toHaveProperty('id'); + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + + test('second page returns different rows with same structure', async () => { + const page1 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 0 }))); + const page2 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 3 }))); + + if (page1.rows.length > 0 && page2.rows.length > 0) { + // IDs should be different + const page1Ids = page1.rows.map(r => r.id); + const page2Ids = page2.rows.map(r => r.id); + + const overlap = page1Ids.filter(id => page2Ids.includes(id)); + expect(overlap.length).toBe(0); + + // Both pages should have same structure + page2.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + } + }); + }); + + describe('Performance and Cardinality', () => { + test('LATERAL JOINs do not duplicate collection rows', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 100, token: 0 }); + const result = await query(sql, values); + + // Collect all IDs + const ids = result.rows.map(r => r.id); + const uniqueIds = [...new Set(ids)]; + + // No duplicates: each collection should appear exactly once + expect(ids.length).toBe(uniqueIds.length); + }); + + test('query executes in reasonable time (<5s for small dataset)', async () => { + const start = Date.now(); + const { sql, values } = buildCollectionSearchQuery({ limit: 50, token: 0 }); + await query(sql, values); + const duration = Date.now() - start; + + // Should complete within 5 seconds for typical test datasets + expect(duration).toBeLessThan(5000); + }, 10000); // 10s timeout for Jest + }); +}); diff --git a/api/__tests__/buildCollectionsSearchQuery.basic.test.js b/api/__tests__/buildCollectionsSearchQuery.basic.test.js index 8756a5f..4acd7b9 100644 --- a/api/__tests__/buildCollectionsSearchQuery.basic.test.js +++ b/api/__tests__/buildCollectionsSearchQuery.basic.test.js @@ -4,8 +4,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { test('no params returns base SQL with LIMIT/OFFSET placeholders', () => { const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - expect(sql).toMatch(/FROM collection/); - expect(sql).toMatch(/ORDER BY id ASC/); + expect(sql).toMatch(/FROM collection c/); + expect(sql).toMatch(/ORDER BY c\.id ASC/); // there should be LIMIT and OFFSET placeholders expect(sql).toMatch(/LIMIT \$1 OFFSET \$2/); expect(Array.isArray(values)).toBe(true); @@ -30,8 +30,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { const datetime = '2020-01-01/2021-12-31'; const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); - expect(sql).toMatch(/temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated - expect(sql).toMatch(/temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated + expect(sql).toMatch(/c\.temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated + expect(sql).toMatch(/c\.temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated // values order: start, end, limit, token expect(values[0]).toBe('2020-01-01'); expect(values[1]).toBe('2021-12-31'); diff --git a/api/db/buildCollectionSearchQuery.js b/api/db/buildCollectionSearchQuery.js index cc1d435..8d43cee 100644 --- a/api/db/buildCollectionSearchQuery.js +++ b/api/db/buildCollectionSearchQuery.js @@ -83,22 +83,31 @@ function buildCollectionSearchQuery(params) { // full-text search) *before* the `FROM` clause. Keeping `sql` fixed with a // `FROM` already included would make inserting additional selected columns // harder and error-prone when building the query dynamically. + // + // We use alias 'c' for the collection table to simplify JOIN expressions and + // distinguish collection columns from aggregated relation data (keywords, providers, etc.). let selectPart = ` SELECT - id, - stac_version, - type, - title, - description, - license, - spatial_extend, - temporal_extend_start, - temporal_extend_end, - created_at, - updated_at, - is_api, - is_active, - full_json + c.id, + c.stac_version, + c.type, + c.title, + c.description, + c.license, + c.spatial_extend, + c.temporal_extend_start, + c.temporal_extend_end, + c.created_at, + c.updated_at, + c.is_api, + c.is_active, + c.full_json, + kw.keywords, + ext.stac_extensions, + prov.providers, + a.assets, + s.summaries, + cl.last_crawled `; const where = []; @@ -123,8 +132,8 @@ function buildCollectionSearchQuery(params) { if (q) { const queryIndex = i; // remember index to reuse for rank and condition - // Weighted combined tsvector expression - const vectorExpr = `to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))`; + // Weighted combined tsvector expression (using alias 'c' for collection table) + const vectorExpr = `to_tsvector('simple', coalesce(c.title,'') || ' ' || coalesce(c.description,''))`; // Add rank to selected columns (ts_rank_cd => constant-duration ranking function) // The computed `rank` is available in the result rows and used for ordering @@ -144,7 +153,7 @@ function buildCollectionSearchQuery(params) { where.push(` ST_Intersects( - spatial_extend, + c.spatial_extend, ST_MakeEnvelope($${i}, $${i + 1}, $${i + 2}, $${i + 3}, 4326) ) `); @@ -161,33 +170,92 @@ function buildCollectionSearchQuery(params) { if (start !== '..') { // Collection should run after start - where.push(`temporal_extend_end >= $${i}`); + where.push(`c.temporal_extend_end >= $${i}`); values.push(start); i++; } if (end !== '..') { // Collection should run before end - where.push(`temporal_extend_start <= $${i}`); + where.push(`c.temporal_extend_start <= $${i}`); values.push(end); i++; } } else { // single datetime: collections active at that time where.push(` - temporal_extend_start <= $${i} - AND temporal_extend_end >= $${i} + c.temporal_extend_start <= $${i} + AND c.temporal_extend_end >= $${i} `); values.push(datetime); i++; } } - // Build final SQL from selectPart and add FROM clause. + // Build final SQL from selectPart and add FROM clause with LATERAL JOINs. // We delayed adding `FROM collection` to allow conditional additions to the // selected columns above (notably `rank`). The final `sql` string includes the // selected columns, the source table and any WHERE conditions constructed earlier. - let sql = selectPart + `\n FROM collection\n `; + // + // LATERAL JOINs aggregate related data (keywords, extensions, providers, assets, summaries, + // and crawl timestamps) from normalized tables without duplicating collection rows. + // Each LEFT JOIN LATERAL subquery returns a single aggregated row per collection. + let sql = selectPart + ` + FROM collection c + LEFT JOIN LATERAL ( + SELECT jsonb_agg(k.keyword ORDER BY k.keyword) AS keywords + FROM collection_keywords ck + JOIN keywords k ON k.id = ck.keyword_id + WHERE ck.collection_id = c.id + ) kw ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(se.stac_extension ORDER BY se.stac_extension) AS stac_extensions + FROM collection_stac_extension cse + JOIN stac_extensions se ON se.id = cse.stac_extension_id + WHERE cse.collection_id = c.id + ) ext ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object( + 'name', p.provider, + 'roles', cpr.collection_provider_roles + ) ORDER BY p.provider) AS providers + FROM collection_providers cpr + JOIN providers p ON p.id = cpr.provider_id + WHERE cpr.collection_id = c.id + ) prov ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object( + 'name', a.name, + 'href', a.href, + 'type', a.type, + 'roles', a.roles, + 'metadata', a.metadata, + 'collection_roles', ca.collection_asset_roles + ) ORDER BY a.name) AS assets + FROM collection_assets ca + JOIN assets a ON a.id = ca.asset_id + WHERE ca.collection_id = c.id + ) a ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_object_agg(s.name, s.s_summary) AS summaries + FROM ( + SELECT + cs.name, + CASE + WHEN cs.kind = 'range' THEN jsonb_build_object('min', cs.range_min, 'max', cs.range_max) + WHEN cs.kind = 'set' THEN to_jsonb(cs.set_value) + ELSE cs.json_schema + END AS s_summary + FROM collection_summaries cs + WHERE cs.collection_id = c.id + ) s + ) s ON TRUE + LEFT JOIN LATERAL ( + SELECT MAX(clc.last_crawled) AS last_crawled + FROM crawllog_collection clc + WHERE clc.collection_id = c.id + ) cl ON TRUE + `; if (where.length > 0) { sql += ` WHERE ` + where.join(' AND '); @@ -197,15 +265,18 @@ function buildCollectionSearchQuery(params) { // a text query was provided (descending), falling back to id ascending. // // Behaviour summary: - // - `sortby` provided → use that (same as before) - // - no `sortby` & `q` present → order by `rank DESC, id ASC` so higher relevance comes first - // - no `sortby` & no `q` → order by `id ASC` (legacy default) + // - `sortby` provided → use that (with 'c.' prefix for collection columns) + // - no `sortby` & `q` present → order by `rank DESC, c.id ASC` so higher relevance comes first + // - no `sortby` & no `q` → order by `c.id ASC` (legacy default) + // + // Note: sortby.field is validated against a whitelist in the calling code; only collection + // table columns are allowed for sorting (not aggregated fields like keywords/providers). if (sortby) { - sql += ` ORDER BY ${sortby.field} ${sortby.direction}`; + sql += ` ORDER BY c.${sortby.field} ${sortby.direction}`; } else if (q) { - sql += ` ORDER BY rank DESC, id ASC`; + sql += ` ORDER BY rank DESC, c.id ASC`; } else { - sql += ` ORDER BY id ASC`; + sql += ` ORDER BY c.id ASC`; } // Pagination (only add if limit is provided) From d83eeb483e263786e8f46ff0fbc2cd975a792de0 Mon Sep 17 00:00:00 2001 From: Robin Tammo Gummels Date: Tue, 9 Dec 2025 16:43:53 +0100 Subject: [PATCH 2/6] Update api/.env.example --- api/.env.example | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/.env.example b/api/.env.example index cb715aa..039ac70 100644 --- a/api/.env.example +++ b/api/.env.example @@ -10,8 +10,7 @@ DATABASE_URL= postgresql://[**DB_USER**]:[**DB_PASSWORD**]@atlas.stacindex.org:5 # Option 2: Use individual variables (currently active) DB_HOST=atlas.stacindex.org -DB_PORT=5432 # 5432 for old database -# 5433 for new database (change it in the URL as well if needed!!!) +DB_PORT=5433 # 5432 for production DB_NAME=stac_db DB_USER= DB_PASSWORD= From 70dc0434e3b92efc2f698d85718d89db630113ae Mon Sep 17 00:00:00 2001 From: RobinGummels Date: Tue, 9 Dec 2025 11:20:41 +0100 Subject: [PATCH 3/6] Updated Query-Builder to get all necessary fields from all db_tables for collections. Added some tests and fixed some already existing tests, becuase now the tablenames start with the alias `c.`. --- ...ldCollectionSearchQuery.aggregates.test.js | 221 ++++++++++++ ...uildCollectionSearchQuery.fulltext.test.js | 4 +- ...dCollectionSearchQuery.integration.test.js | 322 ++++++++++++++++++ .../buildCollectionsSearchQuery.basic.test.js | 8 +- api/db/buildCollectionSearchQuery.js | 129 +++++-- 5 files changed, 649 insertions(+), 35 deletions(-) create mode 100644 api/__tests__/buildCollectionSearchQuery.aggregates.test.js create mode 100644 api/__tests__/buildCollectionSearchQuery.integration.test.js diff --git a/api/__tests__/buildCollectionSearchQuery.aggregates.test.js b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js new file mode 100644 index 0000000..069ea46 --- /dev/null +++ b/api/__tests__/buildCollectionSearchQuery.aggregates.test.js @@ -0,0 +1,221 @@ +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +describe('buildCollectionSearchQuery - aggregated fields', () => { + test('SELECT includes all collection base columns with alias c', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Core collection fields should be prefixed with 'c.' + expect(sql).toMatch(/c\.id/); + expect(sql).toMatch(/c\.stac_version/); + expect(sql).toMatch(/c\.type/); + expect(sql).toMatch(/c\.title/); + expect(sql).toMatch(/c\.description/); + expect(sql).toMatch(/c\.license/); + expect(sql).toMatch(/c\.spatial_extend/); + expect(sql).toMatch(/c\.temporal_extend_start/); + expect(sql).toMatch(/c\.temporal_extend_end/); + expect(sql).toMatch(/c\.created_at/); + expect(sql).toMatch(/c\.updated_at/); + expect(sql).toMatch(/c\.is_api/); + expect(sql).toMatch(/c\.is_active/); + expect(sql).toMatch(/c\.full_json/); + }); + + test('SELECT includes aggregated relation fields', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Aggregated fields from LATERAL JOINs + expect(sql).toMatch(/kw\.keywords/); + expect(sql).toMatch(/ext\.stac_extensions/); + expect(sql).toMatch(/prov\.providers/); + expect(sql).toMatch(/a\.assets/); + expect(sql).toMatch(/s\.summaries/); + expect(sql).toMatch(/cl\.last_crawled/); + }); + + test('FROM clause uses collection alias c', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/FROM collection c/); + }); + + describe('LATERAL JOINs for normalized data', () => { + test('includes LATERAL JOIN for keywords', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/LEFT JOIN LATERAL/); + expect(sql).toMatch(/jsonb_agg\(k\.keyword ORDER BY k\.keyword\) AS keywords/); + expect(sql).toMatch(/FROM collection_keywords ck/); + expect(sql).toMatch(/JOIN keywords k ON k\.id = ck\.keyword_id/); + expect(sql).toMatch(/WHERE ck\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for stac_extensions', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_agg\(se\.stac_extension ORDER BY se\.stac_extension\) AS stac_extensions/); + expect(sql).toMatch(/FROM collection_stac_extension cse/); + expect(sql).toMatch(/JOIN stac_extensions se ON se\.id = cse\.stac_extension_id/); + expect(sql).toMatch(/WHERE cse\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for providers with roles', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_agg\(jsonb_build_object\(/); + expect(sql).toMatch(/'name', p\.provider/); + expect(sql).toMatch(/'roles', cpr\.collection_provider_roles/); + expect(sql).toMatch(/FROM collection_providers cpr/); + expect(sql).toMatch(/JOIN providers p ON p\.id = cpr\.provider_id/); + expect(sql).toMatch(/WHERE cpr\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for assets with metadata', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/'name', a\.name/); + expect(sql).toMatch(/'href', a\.href/); + expect(sql).toMatch(/'type', a\.type/); + expect(sql).toMatch(/'roles', a\.roles/); + expect(sql).toMatch(/'metadata', a\.metadata/); + expect(sql).toMatch(/'collection_roles', ca\.collection_asset_roles/); + expect(sql).toMatch(/FROM collection_assets ca/); + expect(sql).toMatch(/JOIN assets a ON a\.id = ca\.asset_id/); + expect(sql).toMatch(/WHERE ca\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for summaries with CASE logic', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/jsonb_object_agg\(s\.name, s\.s_summary\) AS summaries/); + expect(sql).toMatch(/WHEN cs\.kind = 'range' THEN jsonb_build_object\('min', cs\.range_min, 'max', cs\.range_max\)/); + expect(sql).toMatch(/WHEN cs\.kind = 'set' THEN to_jsonb\(cs\.set_value\)/); + expect(sql).toMatch(/FROM collection_summaries cs/); + expect(sql).toMatch(/WHERE cs\.collection_id = c\.id/); + }); + + test('includes LATERAL JOIN for last_crawled timestamp', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/MAX\(clc\.last_crawled\) AS last_crawled/); + expect(sql).toMatch(/FROM crawllog_collection clc/); + expect(sql).toMatch(/WHERE clc\.collection_id = c\.id/); + }); + }); + + describe('WHERE clauses use collection alias c', () => { + test('bbox filter uses c.spatial_extend', () => { + const bbox = [-10, 40, 10, 50]; + const { sql } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + expect(sql).toMatch(/c\.spatial_extend/); + expect(sql).toMatch(/ST_Intersects\(\s*c\.spatial_extend/); + }); + + test('datetime filter uses c.temporal_extend_start and c.temporal_extend_end', () => { + const datetime = '2020-01-01/2021-12-31'; + const { sql } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + expect(sql).toMatch(/c\.temporal_extend_end >= \$/); + expect(sql).toMatch(/c\.temporal_extend_start <= \$/); + }); + + test('fulltext search uses c.title and c.description', () => { + const q = 'satellite'; + const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(sql).toMatch(/coalesce\(c\.title,''\)/); + expect(sql).toMatch(/coalesce\(c\.description,''\)/); + expect(sql).toMatch(/to_tsvector\('simple', coalesce\(c\.title,''\) \|\| ' ' \|\| coalesce\(c\.description,''\)\)/); + }); + }); + + describe('ORDER BY uses collection alias c', () => { + test('default ORDER BY uses c.id', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY c\.id ASC/); + }); + + test('sortby parameter uses c. prefix', () => { + const sortby = { field: 'title', direction: 'DESC' }; + const { sql } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY c\.title DESC/); + }); + + test('fulltext search with rank orders by rank DESC, c.id ASC', () => { + const q = 'satellite'; + const { sql } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); + }); + }); + + describe('Parameterized values remain correct', () => { + test('bbox parameters are in correct order', () => { + const bbox = [-10, 40, 10, 50]; + const { values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + expect(values.slice(0, 4)).toEqual(bbox); + expect(values[4]).toBe(10); // limit + expect(values[5]).toBe(0); // token + }); + + test('datetime interval parameters are in correct order', () => { + const datetime = '2020-01-01/2021-12-31'; + const { values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + expect(values[0]).toBe('2020-01-01'); + expect(values[1]).toBe('2021-12-31'); + expect(values[2]).toBe(10); // limit + expect(values[3]).toBe(0); // token + }); + + test('fulltext query parameter is bound correctly', () => { + const q = 'satellite'; + const { values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + expect(values[0]).toBe('satellite'); + expect(values[1]).toBe(10); // limit + expect(values[2]).toBe(0); // token + }); + + test('combined filters maintain parameter order', () => { + const bbox = [-10, 40, 10, 50]; + const datetime = '2020-01-01/2021-12-31'; + const q = 'satellite'; + const { values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 10, token: 0 }); + + // Order: q (1), bbox (4), datetime start (1), datetime end (1), limit (1), token (1) = 9 values + expect(values[0]).toBe('satellite'); + expect(values.slice(1, 5)).toEqual(bbox); + expect(values[5]).toBe('2020-01-01'); + expect(values[6]).toBe('2021-12-31'); + expect(values[7]).toBe(10); + expect(values[8]).toBe(0); + }); + }); + + describe('SQL structure validation', () => { + test('no DISTINCT in jsonb_agg to avoid ORDER BY conflict', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // DISTINCT should NOT appear in any jsonb_agg calls + // (PostgreSQL requires ORDER BY expressions to appear in DISTINCT argument list) + const distinctPattern = /jsonb_agg\(DISTINCT/gi; + const matches = sql.match(distinctPattern); + + expect(matches).toBeNull(); + }); + + test('all LATERAL JOINs are LEFT JOIN', () => { + const { sql } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + + // Count LEFT JOIN LATERAL occurrences (should be 6: kw, ext, prov, a, s, cl) + const leftJoinLateralCount = (sql.match(/LEFT JOIN LATERAL/gi) || []).length; + + expect(leftJoinLateralCount).toBe(6); + }); + }); +}); diff --git a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js index 5c8100d..99a9f07 100644 --- a/api/__tests__/buildCollectionSearchQuery.fulltext.test.js +++ b/api/__tests__/buildCollectionSearchQuery.fulltext.test.js @@ -13,7 +13,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { expect(sql).toMatch(/AS rank/); // Ordering defaults to rank DESC when q present and no sortby - expect(sql).toMatch(/ORDER BY rank DESC, id ASC/); + expect(sql).toMatch(/ORDER BY rank DESC, c\.id ASC/); // values: [q, limit, token] expect(values[0]).toBe('forest'); @@ -24,7 +24,7 @@ describe('buildCollectionSearchQuery - full-text search and ranking', () => { test('explicit sortby overrides rank ordering', () => { const { sql } = buildCollectionSearchQuery({ q: 'lake', sortby: { field: 'title', direction: 'ASC' }, limit: 5, token: 0 }); - expect(sql).toMatch(/ORDER BY title ASC/); + expect(sql).toMatch(/ORDER BY c\.title ASC/); // rank still present in select expect(sql).toMatch(/AS rank/); }); diff --git a/api/__tests__/buildCollectionSearchQuery.integration.test.js b/api/__tests__/buildCollectionSearchQuery.integration.test.js new file mode 100644 index 0000000..2885fa0 --- /dev/null +++ b/api/__tests__/buildCollectionSearchQuery.integration.test.js @@ -0,0 +1,322 @@ +const { query, closePool } = require('../db/db_APIconnection'); +const { buildCollectionSearchQuery } = require('../db/buildCollectionSearchQuery'); + +/** + * Integration Tests: Aggregated Fields in Collection Search Query + * + * These tests verify that the LATERAL JOINs correctly aggregate data from + * normalized tables (keywords, providers, assets, stac_extensions, summaries, crawllog). + * + * Prerequisites: + * - Database must be initialized with schema (01-05_*.sql) + * - Test data should include collections with related entities + */ + +describe('Integration: Collection Search with Aggregated Fields', () => { + afterAll(async () => { + await closePool(); + }); + + describe('Query Execution', () => { + test('should execute query successfully without errors', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); + + await expect(query(sql, values)).resolves.not.toThrow(); + }); + + test('should return rows with aggregated fields', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 5, token: 0 }); + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + expect(Array.isArray(result.rows)).toBe(true); + + // If there are collections in DB, verify structure + if (result.rows.length > 0) { + const firstRow = result.rows[0]; + + // Core collection fields + expect(firstRow).toHaveProperty('id'); + expect(firstRow).toHaveProperty('title'); + expect(firstRow).toHaveProperty('description'); + expect(firstRow).toHaveProperty('license'); + expect(firstRow).toHaveProperty('full_json'); + + // Aggregated fields (may be null if no related data) + expect(firstRow).toHaveProperty('keywords'); + expect(firstRow).toHaveProperty('stac_extensions'); + expect(firstRow).toHaveProperty('providers'); + expect(firstRow).toHaveProperty('assets'); + expect(firstRow).toHaveProperty('summaries'); + expect(firstRow).toHaveProperty('last_crawled'); + } + }); + }); + + describe('Aggregated Field Types', () => { + test('keywords should be JSONB array or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.keywords !== null) { + expect(Array.isArray(row.keywords)).toBe(true); + // Each keyword should be a string + row.keywords.forEach(kw => { + expect(typeof kw).toBe('string'); + }); + } + }); + }); + + test('stac_extensions should be JSONB array or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.stac_extensions !== null) { + expect(Array.isArray(row.stac_extensions)).toBe(true); + row.stac_extensions.forEach(ext => { + expect(typeof ext).toBe('string'); + }); + } + }); + }); + + test('providers should be JSONB array of objects or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.providers !== null) { + expect(Array.isArray(row.providers)).toBe(true); + row.providers.forEach(provider => { + expect(provider).toHaveProperty('name'); + expect(provider).toHaveProperty('roles'); + expect(typeof provider.name).toBe('string'); + }); + } + }); + }); + + test('assets should be JSONB array of objects or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.assets !== null) { + expect(Array.isArray(row.assets)).toBe(true); + row.assets.forEach(asset => { + expect(asset).toHaveProperty('name'); + expect(asset).toHaveProperty('href'); + expect(asset).toHaveProperty('type'); + expect(asset).toHaveProperty('roles'); + expect(asset).toHaveProperty('metadata'); + expect(asset).toHaveProperty('collection_roles'); + }); + } + }); + }); + + test('summaries should be JSONB object or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.summaries !== null) { + expect(typeof row.summaries).toBe('object'); + expect(Array.isArray(row.summaries)).toBe(false); + + // Each summary should be a range, set, or schema object + Object.values(row.summaries).forEach(summary => { + const hasRange = summary.min !== undefined && summary.max !== undefined; + const isSet = Array.isArray(summary) || typeof summary === 'string'; + const isSchema = typeof summary === 'object'; + + expect(hasRange || isSet || isSchema).toBe(true); + }); + } + }); + }); + + test('last_crawled should be timestamp or null', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + result.rows.forEach(row => { + if (row.last_crawled !== null) { + // Should be a valid Date or parseable timestamp + const date = new Date(row.last_crawled); + expect(date.toString()).not.toBe('Invalid Date'); + } + }); + }); + }); + + describe('Filter Compatibility with Aggregated Fields', () => { + test('bbox filter works with aggregated fields', async () => { + const bbox = [-180, -90, 180, 90]; // World bbox + const { sql, values } = buildCollectionSearchQuery({ bbox, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + // All returned rows should have the aggregated structure + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + }); + + test('datetime filter works with aggregated fields', async () => { + const datetime = '2000-01-01/2030-12-31'; + const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('stac_extensions'); + expect(row).toHaveProperty('summaries'); + expect(row).toHaveProperty('last_crawled'); + }); + }); + + test('fulltext search works with aggregated fields', async () => { + const q = 'test'; + const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + + test('combined filters work with aggregated fields', async () => { + const bbox = [-180, -90, 180, 90]; + const datetime = '2000-01-01/2030-12-31'; + const q = 'satellite'; + const { sql, values } = buildCollectionSearchQuery({ q, bbox, datetime, limit: 5, token: 0 }); + + const result = await query(sql, values); + + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + // All aggregated fields should be present + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('stac_extensions'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + expect(row).toHaveProperty('summaries'); + expect(row).toHaveProperty('last_crawled'); + }); + }); + }); + + describe('Sorting with Aggregated Fields', () => { + test('default sort by c.id works with aggregated fields', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); + const result = await query(sql, values); + + if (result.rows.length > 1) { + // IDs should be in ascending order + for (let i = 1; i < result.rows.length; i++) { + expect(result.rows[i].id).toBeGreaterThanOrEqual(result.rows[i - 1].id); + } + } + }); + + test('sort by title works with aggregated fields', async () => { + const sortby = { field: 'title', direction: 'ASC' }; + const { sql, values } = buildCollectionSearchQuery({ sortby, limit: 10, token: 0 }); + const result = await query(sql, values); + + // Verify SQL contains ORDER BY c.title ASC + expect(sql).toMatch(/ORDER BY c\.title ASC/); + + // Verify all aggregated fields are present + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + }); + + test('fulltext rank sort works with aggregated fields', async () => { + const q = 'satellite'; + const { sql, values } = buildCollectionSearchQuery({ q, limit: 10, token: 0 }); + const result = await query(sql, values); + + // Should execute without error; rank ordering is implicit in SQL + expect(result.rows).toBeDefined(); + result.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + }); + + describe('Pagination with Aggregated Fields', () => { + test('first page returns correct structure', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 3, token: 0 }); + const result = await query(sql, values); + + expect(result.rows.length).toBeLessThanOrEqual(3); + result.rows.forEach(row => { + expect(row).toHaveProperty('id'); + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + }); + }); + + test('second page returns different rows with same structure', async () => { + const page1 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 0 }))); + const page2 = await query(...Object.values(buildCollectionSearchQuery({ limit: 3, token: 3 }))); + + if (page1.rows.length > 0 && page2.rows.length > 0) { + // IDs should be different + const page1Ids = page1.rows.map(r => r.id); + const page2Ids = page2.rows.map(r => r.id); + + const overlap = page1Ids.filter(id => page2Ids.includes(id)); + expect(overlap.length).toBe(0); + + // Both pages should have same structure + page2.rows.forEach(row => { + expect(row).toHaveProperty('keywords'); + expect(row).toHaveProperty('providers'); + expect(row).toHaveProperty('assets'); + }); + } + }); + }); + + describe('Performance and Cardinality', () => { + test('LATERAL JOINs do not duplicate collection rows', async () => { + const { sql, values } = buildCollectionSearchQuery({ limit: 100, token: 0 }); + const result = await query(sql, values); + + // Collect all IDs + const ids = result.rows.map(r => r.id); + const uniqueIds = [...new Set(ids)]; + + // No duplicates: each collection should appear exactly once + expect(ids.length).toBe(uniqueIds.length); + }); + + test('query executes in reasonable time (<5s for small dataset)', async () => { + const start = Date.now(); + const { sql, values } = buildCollectionSearchQuery({ limit: 50, token: 0 }); + await query(sql, values); + const duration = Date.now() - start; + + // Should complete within 5 seconds for typical test datasets + expect(duration).toBeLessThan(5000); + }, 10000); // 10s timeout for Jest + }); +}); diff --git a/api/__tests__/buildCollectionsSearchQuery.basic.test.js b/api/__tests__/buildCollectionsSearchQuery.basic.test.js index 8756a5f..4acd7b9 100644 --- a/api/__tests__/buildCollectionsSearchQuery.basic.test.js +++ b/api/__tests__/buildCollectionsSearchQuery.basic.test.js @@ -4,8 +4,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { test('no params returns base SQL with LIMIT/OFFSET placeholders', () => { const { sql, values } = buildCollectionSearchQuery({ limit: 10, token: 0 }); - expect(sql).toMatch(/FROM collection/); - expect(sql).toMatch(/ORDER BY id ASC/); + expect(sql).toMatch(/FROM collection c/); + expect(sql).toMatch(/ORDER BY c\.id ASC/); // there should be LIMIT and OFFSET placeholders expect(sql).toMatch(/LIMIT \$1 OFFSET \$2/); expect(Array.isArray(values)).toBe(true); @@ -30,8 +30,8 @@ describe('buildCollectionSearchQuery - basic cases', () => { const datetime = '2020-01-01/2021-12-31'; const { sql, values } = buildCollectionSearchQuery({ datetime, limit: 10, token: 0 }); - expect(sql).toMatch(/temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated - expect(sql).toMatch(/temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated + expect(sql).toMatch(/c\.temporal_extend_end >= \$1/); // TODO: Adjust naming to temporal_extent_end, when DB names are updated + expect(sql).toMatch(/c\.temporal_extend_start <= \$2/); // TODO: Adjust naming to temporal_extent_start, when DB names are updated // values order: start, end, limit, token expect(values[0]).toBe('2020-01-01'); expect(values[1]).toBe('2021-12-31'); diff --git a/api/db/buildCollectionSearchQuery.js b/api/db/buildCollectionSearchQuery.js index cc1d435..8d43cee 100644 --- a/api/db/buildCollectionSearchQuery.js +++ b/api/db/buildCollectionSearchQuery.js @@ -83,22 +83,31 @@ function buildCollectionSearchQuery(params) { // full-text search) *before* the `FROM` clause. Keeping `sql` fixed with a // `FROM` already included would make inserting additional selected columns // harder and error-prone when building the query dynamically. + // + // We use alias 'c' for the collection table to simplify JOIN expressions and + // distinguish collection columns from aggregated relation data (keywords, providers, etc.). let selectPart = ` SELECT - id, - stac_version, - type, - title, - description, - license, - spatial_extend, - temporal_extend_start, - temporal_extend_end, - created_at, - updated_at, - is_api, - is_active, - full_json + c.id, + c.stac_version, + c.type, + c.title, + c.description, + c.license, + c.spatial_extend, + c.temporal_extend_start, + c.temporal_extend_end, + c.created_at, + c.updated_at, + c.is_api, + c.is_active, + c.full_json, + kw.keywords, + ext.stac_extensions, + prov.providers, + a.assets, + s.summaries, + cl.last_crawled `; const where = []; @@ -123,8 +132,8 @@ function buildCollectionSearchQuery(params) { if (q) { const queryIndex = i; // remember index to reuse for rank and condition - // Weighted combined tsvector expression - const vectorExpr = `to_tsvector('simple', coalesce(title,'') || ' ' || coalesce(description,''))`; + // Weighted combined tsvector expression (using alias 'c' for collection table) + const vectorExpr = `to_tsvector('simple', coalesce(c.title,'') || ' ' || coalesce(c.description,''))`; // Add rank to selected columns (ts_rank_cd => constant-duration ranking function) // The computed `rank` is available in the result rows and used for ordering @@ -144,7 +153,7 @@ function buildCollectionSearchQuery(params) { where.push(` ST_Intersects( - spatial_extend, + c.spatial_extend, ST_MakeEnvelope($${i}, $${i + 1}, $${i + 2}, $${i + 3}, 4326) ) `); @@ -161,33 +170,92 @@ function buildCollectionSearchQuery(params) { if (start !== '..') { // Collection should run after start - where.push(`temporal_extend_end >= $${i}`); + where.push(`c.temporal_extend_end >= $${i}`); values.push(start); i++; } if (end !== '..') { // Collection should run before end - where.push(`temporal_extend_start <= $${i}`); + where.push(`c.temporal_extend_start <= $${i}`); values.push(end); i++; } } else { // single datetime: collections active at that time where.push(` - temporal_extend_start <= $${i} - AND temporal_extend_end >= $${i} + c.temporal_extend_start <= $${i} + AND c.temporal_extend_end >= $${i} `); values.push(datetime); i++; } } - // Build final SQL from selectPart and add FROM clause. + // Build final SQL from selectPart and add FROM clause with LATERAL JOINs. // We delayed adding `FROM collection` to allow conditional additions to the // selected columns above (notably `rank`). The final `sql` string includes the // selected columns, the source table and any WHERE conditions constructed earlier. - let sql = selectPart + `\n FROM collection\n `; + // + // LATERAL JOINs aggregate related data (keywords, extensions, providers, assets, summaries, + // and crawl timestamps) from normalized tables without duplicating collection rows. + // Each LEFT JOIN LATERAL subquery returns a single aggregated row per collection. + let sql = selectPart + ` + FROM collection c + LEFT JOIN LATERAL ( + SELECT jsonb_agg(k.keyword ORDER BY k.keyword) AS keywords + FROM collection_keywords ck + JOIN keywords k ON k.id = ck.keyword_id + WHERE ck.collection_id = c.id + ) kw ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(se.stac_extension ORDER BY se.stac_extension) AS stac_extensions + FROM collection_stac_extension cse + JOIN stac_extensions se ON se.id = cse.stac_extension_id + WHERE cse.collection_id = c.id + ) ext ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object( + 'name', p.provider, + 'roles', cpr.collection_provider_roles + ) ORDER BY p.provider) AS providers + FROM collection_providers cpr + JOIN providers p ON p.id = cpr.provider_id + WHERE cpr.collection_id = c.id + ) prov ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object( + 'name', a.name, + 'href', a.href, + 'type', a.type, + 'roles', a.roles, + 'metadata', a.metadata, + 'collection_roles', ca.collection_asset_roles + ) ORDER BY a.name) AS assets + FROM collection_assets ca + JOIN assets a ON a.id = ca.asset_id + WHERE ca.collection_id = c.id + ) a ON TRUE + LEFT JOIN LATERAL ( + SELECT jsonb_object_agg(s.name, s.s_summary) AS summaries + FROM ( + SELECT + cs.name, + CASE + WHEN cs.kind = 'range' THEN jsonb_build_object('min', cs.range_min, 'max', cs.range_max) + WHEN cs.kind = 'set' THEN to_jsonb(cs.set_value) + ELSE cs.json_schema + END AS s_summary + FROM collection_summaries cs + WHERE cs.collection_id = c.id + ) s + ) s ON TRUE + LEFT JOIN LATERAL ( + SELECT MAX(clc.last_crawled) AS last_crawled + FROM crawllog_collection clc + WHERE clc.collection_id = c.id + ) cl ON TRUE + `; if (where.length > 0) { sql += ` WHERE ` + where.join(' AND '); @@ -197,15 +265,18 @@ function buildCollectionSearchQuery(params) { // a text query was provided (descending), falling back to id ascending. // // Behaviour summary: - // - `sortby` provided → use that (same as before) - // - no `sortby` & `q` present → order by `rank DESC, id ASC` so higher relevance comes first - // - no `sortby` & no `q` → order by `id ASC` (legacy default) + // - `sortby` provided → use that (with 'c.' prefix for collection columns) + // - no `sortby` & `q` present → order by `rank DESC, c.id ASC` so higher relevance comes first + // - no `sortby` & no `q` → order by `c.id ASC` (legacy default) + // + // Note: sortby.field is validated against a whitelist in the calling code; only collection + // table columns are allowed for sorting (not aggregated fields like keywords/providers). if (sortby) { - sql += ` ORDER BY ${sortby.field} ${sortby.direction}`; + sql += ` ORDER BY c.${sortby.field} ${sortby.direction}`; } else if (q) { - sql += ` ORDER BY rank DESC, id ASC`; + sql += ` ORDER BY rank DESC, c.id ASC`; } else { - sql += ` ORDER BY id ASC`; + sql += ` ORDER BY c.id ASC`; } // Pagination (only add if limit is provided) From b8112880e279a98198fadca3c85616a05a6d1627 Mon Sep 17 00:00:00 2001 From: RobinGummels Date: Wed, 10 Dec 2025 16:33:51 +0100 Subject: [PATCH 4/6] Added `openapi.yaml` (now http://localhost:3000/api-docs/ is working). - needed to do some modifying to the app.js --- api/app.js | 30 ++-- api/docs/openapi.yaml | 351 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 372 insertions(+), 9 deletions(-) create mode 100644 api/docs/openapi.yaml diff --git a/api/app.js b/api/app.js index bf5f4f5..a5b0393 100644 --- a/api/app.js +++ b/api/app.js @@ -26,7 +26,27 @@ app.use(cors({ allowedHeaders: ['Content-Type', 'Authorization'] })); -// Content-Type header for all JSON responses +// OpenAPI spec endpoint (YAML file with correct content-type) - MUST be before Content-Type middleware +app.get('/openapi.yaml', (req, res, next) => { + try { + const openapiPath = path.join(__dirname, 'docs', 'openapi.yaml'); + res.setHeader('Content-Type', 'application/vnd.oai.openapi+json;version=3.0'); + res.sendFile(openapiPath); + } catch (err) { + next(err); + } +}); + +// Swagger/OpenAPI documentation (if openapi.yaml exists) - MUST be before Content-Type middleware +try { + const swaggerDocument = YAML.load(path.join(__dirname, 'docs', 'openapi.yaml')); + app.use('/api-docs', swaggerUi.serve); + app.get('/api-docs', swaggerUi.setup(swaggerDocument)); +} catch (err) { + console.log('OpenAPI documentation not found. Create docs/openapi.yaml to enable Swagger UI.'); +} + +// Content-Type header for JSON responses (set AFTER special endpoints) app.use((req, res, next) => { res.setHeader('Content-Type', 'application/json'); next(); @@ -38,14 +58,6 @@ app.use('/conformance', conformanceRouter); app.use('/collections', collectionsRouter); app.use('/queryables', queryablesRouter); -// Swagger/OpenAPI documentation (if openapi.yaml exists) -try { - const swaggerDocument = YAML.load(path.join(__dirname, 'docs', 'openapi.yaml')); - app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument)); -} catch (err) { - console.log('OpenAPI documentation not found. Create docs/openapi.yaml to enable Swagger UI.'); -} - // 404 handler app.use((req, res, next) => { res.status(404).json({ diff --git a/api/docs/openapi.yaml b/api/docs/openapi.yaml new file mode 100644 index 0000000..8502f3a --- /dev/null +++ b/api/docs/openapi.yaml @@ -0,0 +1,351 @@ +openapi: 3.0.3 +info: + title: STAC Atlas API + description: A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs. + version: 1.0.0 + contact: + name: SpatioCore + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + +servers: + - url: http://localhost:3000 + description: Local development server + +paths: + /: + get: + summary: Landing Page + description: Returns the STAC API landing page with links to available resources + operationId: getLandingPage + tags: + - STAC Core + responses: + '200': + description: STAC API landing page + content: + application/json: + schema: + $ref: '#/components/schemas/LandingPage' + + /conformance: + get: + summary: Conformance Classes + description: Returns the conformance classes that this API implements + operationId: getConformance + tags: + - STAC Core + responses: + '200': + description: Conformance classes + content: + application/json: + schema: + $ref: '#/components/schemas/Conformance' + + /collections: + get: + summary: List Collections + description: Returns a list of STAC Collections with optional filtering + operationId: getCollections + tags: + - Collections + parameters: + - name: limit + in: query + description: Maximum number of collections to return + required: false + schema: + type: integer + minimum: 1 + maximum: 10000 + default: 10 + - name: offset + in: query + description: Number of collections to skip + required: false + schema: + type: integer + minimum: 0 + default: 0 + - name: bbox + in: query + description: Bounding box to filter collections [minLon,minLat,maxLon,maxLat] + required: false + schema: + type: array + items: + type: number + minItems: 4 + maxItems: 6 + - name: datetime + in: query + description: Temporal filter (single datetime or interval) + required: false + schema: + type: string + - name: q + in: query + description: Full-text search query + required: false + schema: + type: string + - name: filter + in: query + description: CQL2 filter expression + required: false + schema: + type: string + - name: filter-lang + in: query + description: Filter language (cql2-text or cql2-json) + required: false + schema: + type: string + enum: + - cql2-text + - cql2-json + default: cql2-text + - name: sortby + in: query + description: Sort order for results + required: false + schema: + type: string + responses: + '200': + description: List of collections + content: + application/json: + schema: + $ref: '#/components/schemas/Collections' + '400': + description: Bad request (invalid parameters) + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /collections/{collectionId}: + get: + summary: Get Collection + description: Returns a single STAC Collection by ID + operationId: getCollection + tags: + - Collections + parameters: + - name: collectionId + in: path + description: Collection identifier + required: true + schema: + type: string + responses: + '200': + description: A STAC Collection + content: + application/json: + schema: + $ref: '#/components/schemas/Collection' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /queryables: + get: + summary: Global Queryables + description: Returns queryable properties for collection search + operationId: getQueryables + tags: + - Queryables + responses: + '200': + description: Queryables schema + content: + application/schema+json: + schema: + type: object + +components: + schemas: + LandingPage: + type: object + required: + - type + - id + - description + - links + - conformsTo + properties: + type: + type: string + enum: + - Catalog + id: + type: string + title: + type: string + description: + type: string + stac_version: + type: string + conformsTo: + type: array + items: + type: string + links: + type: array + items: + $ref: '#/components/schemas/Link' + + Conformance: + type: object + required: + - conformsTo + properties: + conformsTo: + type: array + items: + type: string + + Collections: + type: object + required: + - collections + - links + properties: + collections: + type: array + items: + $ref: '#/components/schemas/Collection' + links: + type: array + items: + $ref: '#/components/schemas/Link' + context: + $ref: '#/components/schemas/Context' + + Collection: + type: object + required: + - type + - id + - description + - license + - extent + - links + properties: + type: + type: string + enum: + - Collection + stac_version: + type: string + stac_extensions: + type: array + items: + type: string + id: + type: string + title: + type: string + description: + type: string + keywords: + type: array + items: + type: string + license: + type: string + providers: + type: array + items: + type: object + extent: + type: object + required: + - spatial + - temporal + properties: + spatial: + type: object + required: + - bbox + properties: + bbox: + type: array + items: + type: array + items: + type: number + temporal: + type: object + required: + - interval + properties: + interval: + type: array + items: + type: array + items: + type: string + nullable: true + links: + type: array + items: + $ref: '#/components/schemas/Link' + summaries: + type: object + assets: + type: object + + Link: + type: object + required: + - rel + - href + properties: + rel: + type: string + href: + type: string + type: + type: string + title: + type: string + + Context: + type: object + properties: + returned: + type: integer + minimum: 0 + limit: + type: integer + minimum: 1 + matched: + type: integer + minimum: 0 + + Error: + type: object + required: + - code + - description + properties: + code: + type: string + description: + type: string + +tags: + - name: STAC Core + description: STAC API Core endpoints + - name: Collections + description: Collection search and retrieval + - name: Queryables + description: Queryable properties From b0473892199e0963f1d3e4e4387c104e5f4a06fb Mon Sep 17 00:00:00 2001 From: RobinGummels Date: Wed, 10 Dec 2025 16:43:05 +0100 Subject: [PATCH 5/6] Added discription on how to use `stac-api-validator`. Currently we are onyl valid to `core`. --- api/README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/api/README.md b/api/README.md index aff0ae1..32cd01f 100644 --- a/api/README.md +++ b/api/README.md @@ -163,6 +163,52 @@ Diese API implementiert: - 🚧 CQL2 Basic Filtering (in Entwicklung) - 🚧 CQL2 Advanced Operators (in Entwicklung) +### STAC API Validator + +The API can be tested using the official [STAC API Validator](https://github.com/stac-utils/stac-api-validator): + +#### Installation + +```bash +# Python 3.11 required +pip install stac-api-validator +``` + +#### Usage + +```bash +# Validate Core Conformance Class +python -m stac_api_validator --root-url http://localhost:3000 --conformance core + +# Validate Collections Extension (requires collection ID) +python -m stac_api_validator \ + --root-url http://localhost:3000 \ + --conformance core \ + --conformance collections \ + --collection + +# With spatial filtering (requires geometry in dataset) +python -m stac_api_validator \ + --root-url http://localhost:3000 \ + --conformance core \ + --conformance collections \ + --collection \ + --geometry '{"type": "Polygon", "coordinates": [[[7.0, 51.0], [8.0, 51.0], [8.0, 52.0], [7.0, 52.0], [7.0, 51.0]]]}' +``` + +#### Validation Status + +| Conformance Class | Status | Date | Errors | Warnings | +|-------------------|--------|------|--------|----------| +| **STAC API - Core** | ✅ Passed | 2025-12-10 | 0 | 0 | +| STAC API - Collections | ⏳ Pending | - | - | - | +| STAC API - Features | ⏳ Pending | - | - | - | +| STAC API - Item Search | ⏳ Pending | - | - | - | +| CQL2 - Basic | ⏳ Pending | - | - | - | +| CQL2 - Advanced | ⏳ Pending | - | - | - | + +**Note:** The Collection Search Extension is not currently validated automatically by the validator and is instead validated through custom Jest integration tests (see `__tests__/`). + ## 📦 Nächste Schritte ### TODO From 34bf9620709caa302f7ecf62d6bb12fa3cfc6cec Mon Sep 17 00:00:00 2001 From: RobinGummels Date: Sun, 14 Dec 2025 11:26:41 +0100 Subject: [PATCH 6/6] Changed API-Version name to 1.1.0 instead of 1.0.0 --- .github/workflows/api-ci.yml | 4 ++-- api/.env.example | 2 +- api/README.md | 2 +- api/docs/openapi.yaml | 2 +- api/routes/index.js | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/api-ci.yml b/.github/workflows/api-ci.yml index 68c87c0..5394698 100644 --- a/.github/workflows/api-ci.yml +++ b/.github/workflows/api-ci.yml @@ -71,7 +71,7 @@ jobs: # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata - API_VERSION=1.0.0 + API_VERSION=1.1.0 EOF # Step 4: Install dependencies @@ -164,7 +164,7 @@ jobs: # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata - API_VERSION=1.0.0 + API_VERSION=1.1.0 EOF - name: Install dependencies diff --git a/api/.env.example b/api/.env.example index 039ac70..5906869 100644 --- a/api/.env.example +++ b/api/.env.example @@ -28,4 +28,4 @@ CORS_ORIGIN=* # API Configuration API_TITLE=STAC Atlas API_DESCRIPTION=A centralized platform for managing, indexing, and providing STAC Collection metadata -API_VERSION=1.0.0 +API_VERSION=1.1.0 diff --git a/api/README.md b/api/README.md index 32cd01f..00cc72e 100644 --- a/api/README.md +++ b/api/README.md @@ -156,7 +156,7 @@ CORS_ORIGIN=* Diese API implementiert: -- ✅ STAC API Core (v1.0.0) +- ✅ STAC API Core (v1.1.0) - ✅ OGC API Features Core - ✅ STAC Collections - ✅ Collection Search Extension diff --git a/api/docs/openapi.yaml b/api/docs/openapi.yaml index 8502f3a..abcce52 100644 --- a/api/docs/openapi.yaml +++ b/api/docs/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.0.3 info: title: STAC Atlas API description: A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs. - version: 1.0.0 + version: 1.1.0 contact: name: SpatioCore license: diff --git a/api/routes/index.js b/api/routes/index.js index b389488..14a7aca 100644 --- a/api/routes/index.js +++ b/api/routes/index.js @@ -16,7 +16,7 @@ router.get('/', (req, res) => { id: 'stac-atlas', title: 'STAC Atlas', description: 'A centralized platform for managing, indexing, and providing STAC Collection metadata from distributed catalogs and APIs.', - stac_version: '1.0.0', + stac_version: '1.1.0', conformsTo: CONFORMANCE_URIS, links: [ {