diff --git a/src/batch/data-rules.js b/src/batch/data-rules.js index 694f56b58..639b6e8ec 100644 --- a/src/batch/data-rules.js +++ b/src/batch/data-rules.js @@ -418,6 +418,7 @@ const metadata = { sync: [ 'ref', 'name', + 'specVersion', 'title', 'version', 'spec', @@ -443,6 +444,7 @@ const metadata = { }, example: { ref: 'my-api-spec', + specVersion: 'openapi-3.1.0', title: 'My API Spec', description: 'Description of my API Spec', version: '1.0.0', diff --git a/src/controllers/sdx/v1/CatalogController.ts b/src/controllers/sdx/v1/CatalogController.ts index cbd6b010e..dcae35ac6 100644 --- a/src/controllers/sdx/v1/CatalogController.ts +++ b/src/controllers/sdx/v1/CatalogController.ts @@ -88,6 +88,7 @@ export class CatalogController extends Controller { @SuccessResponse('200', 'OK') @Example([ { + specVersion: 'openapi-3.1.0', name: 'LAB.MIN.CITZ.SAMPLE-API.v1', title: 'Sample OAS Service', version: '1.0.0', diff --git a/src/controllers/sdx/v1/OrgGatewaysController.ts b/src/controllers/sdx/v1/OrgGatewaysController.ts index c6b91cad5..8643c2336 100644 --- a/src/controllers/sdx/v1/OrgGatewaysController.ts +++ b/src/controllers/sdx/v1/OrgGatewaysController.ts @@ -21,6 +21,7 @@ import { GWAService } from '../../../services/gwaapi'; import YAML from 'js-yaml'; import getSubjectToken from '../../../auth/auth-token'; import { Logger } from '../../../logger'; +import { publishAPEConfig } from '../../../services/ape/publish-config'; const logger = Logger('OrgGatewaysController'); @@ -137,6 +138,12 @@ export class OrgGatewaysController extends Controller { const config = await GetConfigUsingPattern(ctx, body); + if (action === 'preview') { + request.res?.header('Content-Type', 'application/yaml; charset=utf-8'); + request.res?.send(YAML.dump(config.documents, { noRefs: true })); + return ''; + } + const gwaService = new GWAService(process.env.GWA_API_URL); const payload: any = { @@ -162,23 +169,42 @@ export class OrgGatewaysController extends Controller { const artifact = YAML.dump(payload, { noRefs: true }); - if (action === 'preview') { - request.res?.header('Content-Type', 'application/yaml; charset=utf-8'); - request.res?.send(artifact); - return ''; + let result; + if ( + payload.services.length > 0 || + payload.keys.length > 0 || + payload.key_sets.length > 0 + ) { + // Validate the generated config to ensure it only contains allowed configurations for the organization + result = await gwaService.publishGatewayConfiguration( + action === 'remove' ? 'DELETE' : 'PUT', + getSubjectToken(request), + config._gateway_id, + dryRun, + artifact + ); } - // Validate the generated config to ensure it only contains allowed configurations for the organization - const result = await gwaService.publishGatewayConfiguration( - action === 'remove' ? 'DELETE' : 'PUT', - getSubjectToken(request), - config._gateway_id, - dryRun, - artifact - ); + // Handle the processing of these (dryRun not supported atm) + // - Webhook + // - RegoPolicy + // - PolicyDataSource + const apeResult = dryRun + ? [{ message: 'Dry run not supported for APE' }] + : await publishAPEConfig(action, config.documents); request.res?.header('Content-Type', 'application/yaml; charset=utf-8'); - request.res?.send(YAML.dump(result, { noRefs: true })); + request.res?.send( + YAML.dump( + [ + ...(result + ? [{ resource: 'GatewayResources', response: result }] + : []), + ...apeResult, + ], + { noRefs: true } + ) + ); return ''; } } diff --git a/src/controllers/v3/types.ts b/src/controllers/v3/types.ts index b0cce69c6..3cba85890 100644 --- a/src/controllers/v3/types.ts +++ b/src/controllers/v3/types.ts @@ -273,6 +273,7 @@ export interface Application { * @tsoaModel * @example { * "ref": "my-api-spec", + * "specVersion": "openapi-3.1.0", * "title": "My API Spec", * "description": "Description of my API Spec", * "version": "1.0.0", @@ -283,6 +284,7 @@ export interface Application { export interface OpenAPISpec { name?: string; // Primary Key ref?: string; + specVersion?: string; title?: string; version?: string; spec?: string; diff --git a/src/lists/OpenAPISpec.js b/src/lists/OpenAPISpec.js index 0368c3a11..ec111aee4 100644 --- a/src/lists/OpenAPISpec.js +++ b/src/lists/OpenAPISpec.js @@ -9,6 +9,11 @@ module.exports = { isUnique: true, access: { update: false }, }, + specVersion: { + type: Text, + isRequired: true, + access: { update: false }, + }, name: { type: Text, isRequired: true, diff --git a/src/services/ape/config.ts b/src/services/ape/config.ts new file mode 100644 index 000000000..ce33fcc16 --- /dev/null +++ b/src/services/ape/config.ts @@ -0,0 +1,22 @@ +export const APEConfig = { + // used by the webhook for sending messages to the RS via SDX + pubsub_dispatch_url: 'http://sdx-edge-share0', + + // SDX exchange + pubsub_forward_url: 'http://share0.servers.sdx', + pubsub_dispatch_ip: '142.34.229.4', + + // publish destination + events_publisher_url: 'http://pubsub-kafka', + + // These are for administration, not for runtime routes/plugins + + webhook_admin_url: 'http://pubsub-webhook', + + opal_policy_url: 'https://opal-policies-api-gov-bc-ca.dev.api.gov.bc.ca', + + opal_pip_catalog_url: + 'https://opal-pip-catalog-api-gov-bc-ca.dev.api.gov.bc.ca', + + opal_client_url: 'https://opal-client-api-gov-bc-ca.dev.api.gov.bc.ca', +}; diff --git a/src/services/ape/events-webhooks-service.ts b/src/services/ape/events-webhooks-service.ts new file mode 100644 index 000000000..7b3e261aa --- /dev/null +++ b/src/services/ape/events-webhooks-service.ts @@ -0,0 +1,42 @@ +/** + * Manage webhooks + */ +import { checkStatus } from '../checkStatus'; +import { Logger } from '../../logger'; + +const logger = Logger('ape.EventsWebhooksService'); + +export interface WebhookRequest { + conn_id: string; + topic: string; + webhook_url: string; +} + +export interface WebhookResponse { + conn_id: string; + topic: string; + webhook_url: string; +} + +export class EventsWebhooksService { + private webhookAdminUrl: string; + + constructor(webhookAdminUrl: string) { + this.webhookAdminUrl = webhookAdminUrl; + } + + public async upsertWebhook( + webhook: WebhookRequest + ): Promise { + const url = `${this.webhookAdminUrl}/webhooks`; + return await fetch(url, { + method: 'put', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(webhook), + }) + .then(checkStatus) + .then((res) => res.json()); + } +} diff --git a/src/services/ape/opal-pip-catalog-service.ts b/src/services/ape/opal-pip-catalog-service.ts new file mode 100644 index 000000000..0681c87c0 --- /dev/null +++ b/src/services/ape/opal-pip-catalog-service.ts @@ -0,0 +1,47 @@ +/** + * Manage OPAL data sources + */ +import { checkStatus } from '../checkStatus'; +import { Logger } from '../../logger'; + +const logger = Logger('ape.OpalPIPCatalogService'); + +export interface DataSourceRequest { + name: string; + url: string; + topics: string[]; + dst_path: string; +} + +export interface CatalogEntry { + id: string; + name: string; + url: string; + topics: string[]; + dst_path: string; +} + +export class OpalPIPCatalogService { + private OpalPIPCatalogUrl: string; + + constructor(OpalPIPCatalogUrl: string) { + this.OpalPIPCatalogUrl = OpalPIPCatalogUrl; + } + + public async upsertDataSource( + dataSource: DataSourceRequest + ): Promise { + const url = `${this.OpalPIPCatalogUrl}/entries`; + logger.debug(`Upserting data source at ${url}`); + + return await fetch(url, { + method: 'put', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(dataSource), + }) + .then(checkStatus) + .then((res) => res.json()); + } +} diff --git a/src/services/ape/opal-policies-service.ts b/src/services/ape/opal-policies-service.ts new file mode 100644 index 000000000..78cd3e75f --- /dev/null +++ b/src/services/ape/opal-policies-service.ts @@ -0,0 +1,40 @@ +/** + * Manage OPAL policies + */ +import { checkStatus } from '../checkStatus'; +import { Logger } from '../../logger'; + +const logger = Logger('ape.OpalPoliciesService'); + +export interface PolicyRequest { + package: string; + policy: string; +} + +export interface PolicyResponse { + package: string; + policy: string; +} + +export class OpalPoliciesService { + private opalPoliciesUrl: string; + + constructor(opalPoliciesUrl: string) { + this.opalPoliciesUrl = opalPoliciesUrl; + } + + public async upsertPolicy(policy: PolicyRequest): Promise { + const url = `${this.opalPoliciesUrl}/policies/${policy.package}`; + logger.debug(`Upserting policy at ${url}`); + + return await fetch(url, { + method: 'put', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(policy), + }) + .then(checkStatus) + .then((res) => res.json()); + } +} diff --git a/src/services/ape/publish-config.ts b/src/services/ape/publish-config.ts new file mode 100644 index 000000000..40df8b9e1 --- /dev/null +++ b/src/services/ape/publish-config.ts @@ -0,0 +1,92 @@ +/** + * This module is responsible for generating the configuration for the publish component of the APE pattern. + * It will generate the necessary configuration for the publish component based on the inputs provided to the pattern. + * This includes generating the necessary policies, routes, and plugins for the publish component. + * It will also handle the generation of the necessary configuration for the events component of the APE pattern. + * This includes generating the necessary policies, routes, and plugins for the events component. + * + * The publish component is responsible for receiving events from the gateway and forwarding them to the APE events component. + * The events component is responsible for receiving events from the publish component and forwarding them to the appropriate destination (e.g. Event Grid, Service Bus, etc.). + * + */ + +import { APEConfig } from './config'; +import { EventsWebhooksService } from './events-webhooks-service'; +import { OpalPIPCatalogService } from './opal-pip-catalog-service'; +import { OpalPoliciesService } from './opal-policies-service'; + +export async function publishAPEConfig( + action: 'preview' | 'apply' | 'remove', + documents: any[] +) { + if (action === 'apply') { + return await applyAPEConfig(documents); + } else if (action === 'preview') { + return { message: 'Preview not implemented yet' }; + } else if (action === 'remove') { + return { message: 'Remove not implemented yet' }; + } else { + throw new Error(`Unsupported action: ${action}`); + } +} + +async function applyAPEConfig(documents: any[]): Promise { + const results: any[] = []; + const tasks = documents + .filter((doc) => doc.kind) + .map(async (doc) => { + switch (doc.kind) { + case 'Webhook': + // handle publish config for Webhook + const webhookService = new EventsWebhooksService( + APEConfig.webhook_admin_url + ); + results.push({ + resource: 'Webhook', + result: 'success', + response: await webhookService.upsertWebhook({ + conn_id: doc.conn_id, + topic: doc.topic, + webhook_url: doc.url, + }), + }); + break; + + case 'RegoPolicy': + // handle publish config for RegoPolicy + const policyService = new OpalPoliciesService( + APEConfig.opal_policy_url + ); + results.push({ + resource: 'RegoPolicy', + result: 'success', + response: await policyService.upsertPolicy({ + package: doc.package, + policy: doc.policy, + }), + }); + break; + + case 'PolicyDataSource': + const dataSourceService = new OpalPIPCatalogService( + APEConfig.opal_pip_catalog_url + ); + results.push({ + resource: 'PolicyDataSource', + result: 'success', + response: await dataSourceService.upsertDataSource({ + name: doc.name, + url: doc.url, + topics: doc.topics, + dst_path: doc.dst_path, + }), + }); + break; + + default: + throw new Error(`Unsupported document kind: ${doc.kind}`); + } + }); + await Promise.all(tasks); + return results; +} diff --git a/src/services/batch/types.ts b/src/services/batch/types.ts index b0cce69c6..3cba85890 100644 --- a/src/services/batch/types.ts +++ b/src/services/batch/types.ts @@ -273,6 +273,7 @@ export interface Application { * @tsoaModel * @example { * "ref": "my-api-spec", + * "specVersion": "openapi-3.1.0", * "title": "My API Spec", * "description": "Description of my API Spec", * "version": "1.0.0", @@ -283,6 +284,7 @@ export interface Application { export interface OpenAPISpec { name?: string; // Primary Key ref?: string; + specVersion?: string; title?: string; version?: string; spec?: string; diff --git a/src/services/gateway-patterns/catalog.ts b/src/services/gateway-patterns/catalog.ts index 7701303f3..0aed49fad 100644 --- a/src/services/gateway-patterns/catalog.ts +++ b/src/services/gateway-patterns/catalog.ts @@ -60,6 +60,7 @@ export interface ServiceClient { export interface ServiceCatalogEntry { name: string; title: string; + specVersion: string; version: string; summary?: string; description: string; @@ -110,6 +111,7 @@ export async function GetCatalog( ); return { + specVersion: c.specVersion, name: c.name, title: c.title, version: c.version, diff --git a/src/services/gateway-patterns/evaluator.ts b/src/services/gateway-patterns/evaluator.ts index 5d6506f37..015c8c341 100644 --- a/src/services/gateway-patterns/evaluator.ts +++ b/src/services/gateway-patterns/evaluator.ts @@ -4,6 +4,10 @@ import { SDXP2PConsumerPattern } from './patterns/sdx-p2p-consumer'; import { SDXP2PProviderPattern } from './patterns/sdx-p2p-provider'; import { SDXRuntimeGroupPattern } from './patterns/sdx-runtime-group'; import { SDXKeysPattern } from './patterns/sdx-keys'; +import { EventsPublisherPattern } from './patterns/ape/events-publisher'; +import { OPALPolicyPattern } from './patterns/ape/opal-policy'; +import { EventsWebhookPattern } from './patterns/ape/events-webhook'; +import { OPALDataPattern } from './patterns/ape/opal-data-source'; interface PatternProcessor { id: string; @@ -18,6 +22,10 @@ const PATTERNS: Record = { [SDXP2PProviderPattern.id]: SDXP2PProviderPattern, [SDXRuntimeGroupPattern.id]: SDXRuntimeGroupPattern, [SDXKeysPattern.id]: SDXKeysPattern, + [EventsPublisherPattern.id]: EventsPublisherPattern, + [EventsWebhookPattern.id]: EventsWebhookPattern, + [OPALDataPattern.id]: OPALDataPattern, + [OPALPolicyPattern.id]: OPALPolicyPattern, }; export interface GatewayPatternConfig { diff --git a/src/services/gateway-patterns/patterns/ape/events-publisher.ts b/src/services/gateway-patterns/patterns/ape/events-publisher.ts new file mode 100644 index 000000000..af4ab693b --- /dev/null +++ b/src/services/gateway-patterns/patterns/ape/events-publisher.ts @@ -0,0 +1,148 @@ +/** + * sdx-events-publisher + * + * This pattern will establish a secure endpoint for the publisher to send events to + * via the https://sdx-events.api.gov.bc.ca endpoint. + * + */ + +import assert from '../../../user-assert'; +import { SubsystemService } from '../../../batch/subsystem'; +import { + EnrichWithRuntimeGroup, + GetCatalogByName, + GetSubsystemEntryForSubsystem, + ServiceCatalogEntry, + ServiceClient, + SubsystemEntry, +} from '../../catalog'; +import { getRoutePathPrefix } from '../../../utils'; +import { APEConfig } from '../../../ape/config'; +import cyrpto from 'crypto'; + +export interface EventsPublisherPatternConfig extends Record { + organization: string; + service_id: string; +} + +export interface EventsPublisherPatternData { + gateway_id: string; + service: ServiceCatalogEntry; +} + +/** + * This pattern will provision the default route policies for a consumer of an SDX service + * + */ +export const EventsPublisherPattern = { + id: 'events-publisher.r1', + requiredParams: ['organization', 'service_id'], + + inject: async (ctx: any, inputs: EventsPublisherPatternConfig) => { + const service = await GetCatalogByName(ctx, inputs.service_id); + await EnrichWithRuntimeGroup(ctx, service.subsystem); + + return { + gateway_id: service.subsystem.gateway.id, + service, + }; + }, + + eval: (inputs: Record, data: EventsPublisherPatternData) => { + const producerLocator = data.service.name; + + const tags = [`ns.${data.gateway_id}.${producerLocator}.pub`, 'sdx']; + + const routeHostUrl = new URL( + data.service.subsystem.runtimeGroup.consumerEndpoint + ); + + const nameWH = `sdx.evt.pub.c.${producerLocator}`; + + // sha256 hash of clientServiceLocator to ensure the path is not too long for the gateway + // output has hex string format + const clientServiceShaHash = cyrpto + .createHash('sha256') + .update(nameWH) + .digest('hex') + .substring(0, 24); + + const nameC = `sdx.evt.pub.c.${producerLocator}`; + const nameP = `sdx.evt.pub.p.${producerLocator}`; + + const routePathPrefix = `/sdx/1/${clientServiceShaHash}`; + + const clientServiceLocator = `${producerLocator}.pub`; + + const routeC = { + kind: 'GatewayService', + name: nameC, + retries: 0, + routes: [ + { + hosts: [routeHostUrl.hostname], + paths: [routePathPrefix], + methods: ['POST'], + name: nameC, + strip_path: false, + protocols: ['https', 'http'], + tags, + }, + ], + tags: [...tags, `service:${producerLocator}`], + host: APEConfig.pubsub_dispatch_ip, + port: 443, + protocol: 'https', + tls_verify: true, + plugins: [...[transformer(tags, data, clientServiceLocator)]], + } as any; + + const clientServiceHost = new URL(APEConfig.pubsub_forward_url).hostname; + + const routeP = { + kind: 'GatewayService', + name: nameP, + retries: 0, + routes: [ + { + hosts: [clientServiceHost], + snis: [clientServiceHost], + paths: [routePathPrefix], + methods: ['POST'], + headers: { + 'X-Client-Id': [`${clientServiceLocator}`], + }, + protocols: ['https'], + name: `${nameP}.UPSTREAM`, + strip_path: true, + tags, + }, + ], + tags: [...tags, `service:${producerLocator}`], + url: `${APEConfig.events_publisher_url}/Event-${producerLocator}`, + plugins: [], + } as any; + + return [routeC, routeP] as any[]; + }, +}; + +function transformer( + tags: string[], + data: EventsPublisherPatternData, + clientServiceLocator: string +) { + const serviceHost = data.service.subsystem.runtimeGroup.host; + return { + name: 'request-transformer', + tags, + config: { + add: { + headers: [`X-Client-Id:${clientServiceLocator}`], + }, + replace: { + headers: [`Host:${serviceHost}`], + }, + }, + }; +} diff --git a/src/services/gateway-patterns/patterns/ape/events-webhook.ts b/src/services/gateway-patterns/patterns/ape/events-webhook.ts new file mode 100644 index 000000000..d3e3ee105 --- /dev/null +++ b/src/services/gateway-patterns/patterns/ape/events-webhook.ts @@ -0,0 +1,223 @@ +/** + * sdx-events-EventsWebhook + * + * This pattern will create a EventsWebhook + * + */ + +import assert from '../../../user-assert'; +import { SubsystemService } from '../../../batch/subsystem'; +import { + EnrichWithRuntimeGroup, + GetCatalogByName, + GetSubsystemEntryForSubsystem, + ServiceCatalogEntry, + ServiceClient, + SubsystemEntry, +} from '../../catalog'; +import { getRoutePathPrefix } from '../../../utils'; +import { ConnectionService } from '../../../batch/connection-service'; +import cyrpto from 'crypto'; +import { randomBytes } from 'crypto'; +import { APEConfig } from '../../../ape/config'; + +export interface EventsWebhookPatternConfig extends Record { + organization: string; + conn_id: string; + client_id: string; + service_id: string; + webhook_url: string; +} + +export interface EventsWebhookPatternData { + gateway_id: string; + client: SubsystemEntry; + service: ServiceCatalogEntry; +} + +/** + * This pattern will provision the default route policies for a consumer of an SDX service + * + */ +export const EventsWebhookPattern = { + id: 'events-webhook.r1', + requiredParams: [ + 'organization', + 'webhook_url', + 'conn_id', + 'client_id', + 'service_id', + ], + + inject: async (ctx: any, inputs: EventsWebhookPatternConfig) => { + const connService = new ConnectionService(); + + const conn = await connService.getConnectionById(ctx, inputs.conn_id); // validate the connection request exists + + assert.strictEqual( + conn.clientId === inputs.client_id, + true, + 'Connection request clientId does not match the specified client_id' + ); + + assert.strictEqual( + conn.serviceId === inputs.service_id, + true, + 'Connection request serviceId does not match the specified service_id' + ); + + assert.strictEqual(conn.isActive, true, 'Connection request is not active'); + assert.strictEqual( + conn.isApproved, + true, + 'Connection request is not approved' + ); + + // retrieve the catalog items for + const subsysService = new SubsystemService(); + const subsystem = await subsysService.findSubsystemByClientId( + ctx, + inputs.client_id + ); + + assert.strictEqual( + subsystem.organization.name === inputs.organization, + true, + 'Client subsystem does not belong to the specified organization' + ); + + const client = GetSubsystemEntryForSubsystem(subsystem); + await EnrichWithRuntimeGroup(ctx, client); + + const service = await GetCatalogByName(ctx, inputs.service_id); + await EnrichWithRuntimeGroup(ctx, service.subsystem); + + return { + gateway_id: client.gateway.id, + client, + service, + }; + }, + + eval: (inputs: Record, data: EventsWebhookPatternData) => { + const serviceLocator = data.service.name; + + const clientLocator = data.client.clientId; + + const consumerGateway = data.client.gateway.id; + + const nameWH = `sdx.evt.webhook.${inputs.conn_id}.c.${clientLocator}`; + + // sha256 hash of clientServiceLocator to ensure the path is not too long for the gateway + // output has hex string format + const clientServiceShaHash = cyrpto + .createHash('sha256') + .update(nameWH) + .digest('hex') + .substring(0, 24); + + const tags = [`ns.${consumerGateway}.${inputs.conn_id}.c`, 'sdx']; + const nameC = `sdx.evt.webhook.${inputs.conn_id}.c.${clientLocator}`; + const nameP = `sdx.evt.webhook.${inputs.conn_id}.p.${clientLocator}`; + + // webhook is the consumer, but as far as data flow it is: + // pubsub-webhook -> pubsub runtime group -> client runtime group -> webhook_url + // This means that the webhook_url that is passed in, is not the same webhook url + // that is registered with the Webhook service. + // + // Consumer gateway needs to be able to configure routes on the pubsub edge server + // so that the full route can be established. + // + // webhook url: https://internal.pubsub.servers.sdx/${clientServiceLocator} + // route on pubsub edge: /${clientServiceLocator} -> ${client.runtimeGroup.host} + // route on client runtime group: /${serviceLocator} -> ${service.subsystem.runtimeGroup.host} + const routeHostUrl = new URL(APEConfig.pubsub_dispatch_url); + const routePathPrefix = `/sdx/1/${clientServiceShaHash}`; + + const clientServiceLocator = `${clientLocator}.webhook`; + + // client is always the pubsub-webhook system + // so will be routing through the internal endpoint for its edge server + // TODO: Gateway needs permission to use this events_url endpoint! + // + const webhookRouteC = { + kind: 'GatewayService', + name: nameC, + retries: 0, + routes: [ + { + hosts: [routeHostUrl.hostname], + paths: [routePathPrefix], + methods: ['POST'], + name: nameC, + strip_path: false, + protocols: + routeHostUrl.protocol === 'https:' ? ['https', 'http'] : ['http'], + tags, + }, + ], + tags: [...tags, `service:${serviceLocator}`, `client:${clientLocator}`], + url: data.service.subsystem.runtimeGroup.sdxEndpoint, + plugins: [...[transformer(tags, data, clientServiceLocator)]], + } as any; + + const clientServiceHost = data.client.runtimeGroup.host; + + const webhookRouteP = { + kind: 'GatewayService', + name: nameP, + retries: 0, + routes: [ + { + hosts: [clientServiceHost], + snis: [clientServiceHost], + paths: [routePathPrefix], + methods: ['POST'], + headers: { + 'X-Client-Id': [`${clientServiceLocator}`], + }, + protocols: ['https'], + name: `${nameP}.UPSTREAM`, + strip_path: true, + tags, + }, + ], + tags: [...tags, `service:${serviceLocator}`, `client:${clientLocator}`], + url: inputs.webhook_url, + plugins: [] as any[], + }; + + const newWebhookUrl = `${APEConfig.pubsub_dispatch_url}/sdx/1/${clientServiceShaHash}`; + + const config = { + kind: 'Webhook', + name: nameWH, + conn_id: inputs.conn_id, + topic: `Event-${serviceLocator}`, + url: newWebhookUrl, + tags: [...tags, `service:${serviceLocator}`, `client:${clientLocator}`], + } as any; + + return [webhookRouteC, webhookRouteP, config] as any[]; + }, +}; + +function transformer( + tags: string[], + data: EventsWebhookPatternData, + clientServiceLocator: string +) { + const serviceHost = data.client.runtimeGroup.host; + return { + name: 'request-transformer', + tags, + config: { + add: { + headers: [`X-Client-Id:${clientServiceLocator}`], + }, + replace: { + headers: [`Host:${serviceHost}`], + }, + }, + }; +} diff --git a/src/services/gateway-patterns/patterns/ape/opal-data-source.ts b/src/services/gateway-patterns/patterns/ape/opal-data-source.ts new file mode 100644 index 000000000..c258430d8 --- /dev/null +++ b/src/services/gateway-patterns/patterns/ape/opal-data-source.ts @@ -0,0 +1,83 @@ +/** + * sdx-opal-data + * + * This pattern will create a data source entry + * + */ + +import assert from '../../../user-assert'; +import { SubsystemService } from '../../../batch/subsystem'; +import { + EnrichWithRuntimeGroup, + GetCatalogByName, + GetSubsystemEntryForSubsystem, + ServiceCatalogEntry, + ServiceClient, + SubsystemEntry, +} from '../../catalog'; +import { getRoutePathPrefix } from '../../../utils'; + +export interface OPALDataPatternConfig extends Record { + organization: string; + subsystem_id: string; + name: string; + upstream_url: string; +} + +export interface OPALDataPatternData { + gateway_id: string; + client: SubsystemEntry; +} + +/** + * This pattern will provision the default route policies for a consumer of an SDX service + * + */ +export const OPALDataPattern = { + id: 'opal-data-source.r1', + requiredParams: ['organization', 'subsystem_id', 'upstream_url', 'name'], + + inject: async (ctx: any, inputs: OPALDataPatternConfig) => { + // retrieve the catalog items for + const subsysService = new SubsystemService(); + const subsystem = await subsysService.findSubsystemByClientId( + ctx, + inputs.subsystem_id + ); + + assert.strictEqual( + subsystem.organization.name === inputs.organization, + true, + 'Client subsystem does not belong to the specified organization' + ); + + const client = GetSubsystemEntryForSubsystem(subsystem); + await EnrichWithRuntimeGroup(ctx, client); + + return { + gateway_id: client.gateway.id, + client, + }; + }, + + eval: (inputs: Record, data: OPALDataPatternData) => { + const subsystemLocator = data.client.clientId; + + const tags = [ + `ns.${data.gateway_id}.${subsystemLocator}.${inputs.name}.ds`, + 'sdx', + ]; + const name = `sdx.opal.ds.${subsystemLocator}.${inputs.name}`; + + const config = { + kind: 'PolicyDataSource', + name, + url: inputs.upstream_url, + topics: ['tenant_data'], + dst_path: `/tenant/${subsystemLocator}/${inputs.name}`, + tags: [...tags, `client:${subsystemLocator}`], + } as any; + + return [config] as any[]; + }, +}; diff --git a/src/services/gateway-patterns/patterns/ape/opal-policy.ts b/src/services/gateway-patterns/patterns/ape/opal-policy.ts new file mode 100644 index 000000000..aaaca78c2 --- /dev/null +++ b/src/services/gateway-patterns/patterns/ape/opal-policy.ts @@ -0,0 +1,100 @@ +/** + * sdx-opal-policy + * + * This pattern will create a policy + * + */ + +import assert from '../../../user-assert'; +import { SubsystemService } from '../../../batch/subsystem'; +import { + EnrichWithRuntimeGroup, + GetCatalogByName, + GetSubsystemEntryForSubsystem, + ServiceCatalogEntry, + ServiceClient, + SubsystemEntry, +} from '../../catalog'; +import { getRoutePathPrefix } from '../../../utils'; + +export interface OPALPolicyPatternConfig extends Record { + organization: string; + subsystem_id: string; + name: string; + policy: string; +} + +export interface OPALPolicyPatternData { + gateway_id: string; + client: SubsystemEntry; + packageName: string; +} + +/** + * This pattern will provision the default route policies for a consumer of an SDX service + * + */ +export const OPALPolicyPattern = { + id: 'opal-policy.r1', + requiredParams: ['organization', 'subsystem_id', 'policy'], + + inject: async (ctx: any, inputs: OPALPolicyPatternConfig) => { + // retrieve the catalog items for + const subsysService = new SubsystemService(); + const subsystem = await subsysService.findSubsystemByClientId( + ctx, + inputs.subsystem_id + ); + + assert.strictEqual( + subsystem.organization.name === inputs.organization, + true, + 'Client subsystem does not belong to the specified organization' + ); + + const client = GetSubsystemEntryForSubsystem(subsystem); + await EnrichWithRuntimeGroup(ctx, client); + + // make sure the policy package in first line matches + const policyLines = inputs.policy.split('\n'); + assert.strictEqual( + policyLines.length > 0, + true, + 'Policy must be a non-empty string' + ); + + const packageName = `${client.clientId.replace(/\.|-/g, '_')}.${ + inputs.name + }`; + + const packageLine = policyLines[0].trim(); + + assert.strictEqual( + packageLine === `package ${packageName}`, + true, + `Unexpected package name - expecting \`package ${packageName}\`, found \'${packageLine}\'` + ); + return { + gateway_id: client.gateway.id, + client, + packageName, + }; + }, + + eval: (inputs: Record, data: OPALPolicyPatternData) => { + const subsystemLocator = data.client.clientId; + + const tags = [`ns.${data.gateway_id}.${subsystemLocator}.pol`, 'sdx']; + const name = `sdx.opal.pol.${subsystemLocator}.${inputs.name}`; + + const config = { + kind: 'RegoPolicy', + name, + package: data.packageName, + policy: inputs.policy, + tags: [...tags, `client:${subsystemLocator}`], + } as any; + + return [config] as any[]; + }, +}; diff --git a/src/services/gateway-patterns/patterns/sdx-p2p-provider.ts b/src/services/gateway-patterns/patterns/sdx-p2p-provider.ts index 6d6c2f4a8..d78cde2f1 100644 --- a/src/services/gateway-patterns/patterns/sdx-p2p-provider.ts +++ b/src/services/gateway-patterns/patterns/sdx-p2p-provider.ts @@ -10,6 +10,7 @@ import { } from '../catalog'; import { getRoutePathPrefix } from '../../utils'; import { ConnectionService } from '../../batch/connection-service'; +import { APEConfig } from '../../ape/config'; // TODO: clean this up a bit! const SDX_PUBLIC_URL = process.env.SDX_PUBLIC_URL || 'https://sdx.gov.bc.ca'; @@ -19,6 +20,14 @@ interface ProviderUpgrades { mtls_acl: {}; sign: {}; verify: {}; + pep: { + policy_name: string; + json_locator: string[]; + }; + policy_list: { + policy_name: string; + json_locator: string[]; + }; token: { allowed_aud: string; allowed_iss: string[]; @@ -50,7 +59,6 @@ export interface SDXP2PProviderPatternConfig extends Record { export interface SDXP2PProviderPatternData { service: ServiceCatalogEntry; client: SubsystemEntry; - key: any; } /** @@ -188,6 +196,15 @@ export const SDXP2PProviderPattern = { ...(upgrades.hasOwnProperty('verify') ? [upgradeToTrustVerify(tags, data)] : []), + ...(upgrades.hasOwnProperty('pep') + ? [ + upgradeToPolicyEnforcement( + tags, + data, + inputs as SDXP2PProviderPatternConfig + ), + ] + : []), ...(upgrades.hasOwnProperty('token') ? [ upgradeToJWTKeycloak( @@ -197,6 +214,15 @@ export const SDXP2PProviderPattern = { ), ] : []), + ...(upgrades.hasOwnProperty('policy_list') + ? [ + upgradeToPolicyList( + tags, + data, + inputs as SDXP2PProviderPatternConfig + ), + ] + : []), ...(upgrades.hasOwnProperty('counter_sign') ? [upgradeToTrustKMS(tags, data)] : []), @@ -323,6 +349,56 @@ function upgradeToTokenExchange( }; } +function upgradeToPolicyEnforcement( + tags: string[], + data: SDXP2PProviderPatternData, + inputs: SDXP2PProviderPatternConfig +) { + const service = data.service; + + const policyName = inputs.upgrades.pep.policy_name; + + const packageName = `${service.subsystem.clientId.replace( + /\.|-/g, + '_' + )}/${policyName}`; + + return { + name: 'openid-authzen', + tags: tags, + config: { + target_url: `${APEConfig.opal_client_url}/v1/data/${packageName}`, + json_locator: inputs.upgrades.pep.json_locator, + result_type: 'decision', + }, + }; +} + +function upgradeToPolicyList( + tags: string[], + data: SDXP2PProviderPatternData, + inputs: SDXP2PProviderPatternConfig +) { + const service = data.service; + + const policyName = inputs.upgrades.policy_list.policy_name; + + const packageName = `${service.subsystem.clientId.replace( + /\.|-/g, + '_' + )}/${policyName}`; + + return { + name: 'openid-authzen', + tags: tags, + config: { + target_url: `${APEConfig.opal_client_url}/v1/data/${packageName}`, + json_locator: inputs.upgrades.policy_list.json_locator, + result_type: 'table', + }, + }; +} + function upgradeToTrustKMS(tags: string[], data: SDXP2PProviderPatternData) { const member = data.service.subsystem.member; const memberText = `${member.memberClass}.${member.memberId}`.toLowerCase(); diff --git a/src/services/keystone/batch-service.ts b/src/services/keystone/batch-service.ts index 20463b18c..7b1ded8e8 100644 --- a/src/services/keystone/batch-service.ts +++ b/src/services/keystone/batch-service.ts @@ -57,6 +57,7 @@ export class BatchService { return null; } + logger.debug('[listAll] RESULT %j', result); logger.debug('[listAll] RESULT COUNT %d', result['data'][query].length); return result['data'][query].length == 0 ? [] : result['data'][query]; } diff --git a/src/services/keystone/types.ts b/src/services/keystone/types.ts index 5d6ff7e08..b9b5bd607 100644 --- a/src/services/keystone/types.ts +++ b/src/services/keystone/types.ts @@ -5839,6 +5839,7 @@ export type OpenApiSpec = { _label_?: Maybe; id: Scalars['ID']; ref?: Maybe; + specVersion?: Maybe; name?: Maybe; namespace?: Maybe; organization?: Maybe; @@ -5853,6 +5854,7 @@ export type OpenApiSpec = { export type OpenApiSpecCreateInput = { ref?: Maybe; + specVersion?: Maybe; name?: Maybe; namespace?: Maybe; organization?: Maybe; @@ -5902,6 +5904,24 @@ export type OpenApiSpecWhereInput = { ref_not_ends_with_i?: Maybe; ref_in?: Maybe>>; ref_not_in?: Maybe>>; + specVersion?: Maybe; + specVersion_not?: Maybe; + specVersion_contains?: Maybe; + specVersion_not_contains?: Maybe; + specVersion_starts_with?: Maybe; + specVersion_not_starts_with?: Maybe; + specVersion_ends_with?: Maybe; + specVersion_not_ends_with?: Maybe; + specVersion_i?: Maybe; + specVersion_not_i?: Maybe; + specVersion_contains_i?: Maybe; + specVersion_not_contains_i?: Maybe; + specVersion_starts_with_i?: Maybe; + specVersion_not_starts_with_i?: Maybe; + specVersion_ends_with_i?: Maybe; + specVersion_not_ends_with_i?: Maybe; + specVersion_in?: Maybe>>; + specVersion_not_in?: Maybe>>; name?: Maybe; name_not?: Maybe; name_contains?: Maybe; @@ -8891,6 +8911,8 @@ export enum SortOpenApiSpecsBy { IdDesc = 'id_DESC', RefAsc = 'ref_ASC', RefDesc = 'ref_DESC', + SpecVersionAsc = 'specVersion_ASC', + SpecVersionDesc = 'specVersion_DESC', NameAsc = 'name_ASC', NameDesc = 'name_DESC', NamespaceAsc = 'namespace_ASC', diff --git a/src/services/workflow/openapi-spec-loader.ts b/src/services/workflow/openapi-spec-loader.ts index c37eb75dd..a77c5cbc8 100644 --- a/src/services/workflow/openapi-spec-loader.ts +++ b/src/services/workflow/openapi-spec-loader.ts @@ -4,6 +4,7 @@ import YAML from 'yaml'; import { Subsystem } from '../keystone/types'; import { SubsystemService } from '../batch/subsystem'; import { BuildServiceName } from '../gateway-patterns/catalog'; +import { strict as assert } from 'assert'; const logger = Logger('wf.OASLoader'); @@ -51,6 +52,14 @@ export const LoadOpenAPISpec = async ( const serviceName = BuildServiceName(subsystemRecord, oas); + assert.strictEqual( + Boolean(oas.openapi) || Boolean(oas.asyncapi), + true, + 'Invalid OpenAPI specification: must contain either openapi or asyncapi field' + ); + + outSpec.specVersion = + 'openapi' in oas ? `openapi=${oas.openapi}` : `asyncapi=${oas.asyncapi}`; outSpec.spec = spec.spec; outSpec.name = serviceName; (outSpec as any).namespace = subsystemRecord.namespace; @@ -61,12 +70,60 @@ export const LoadOpenAPISpec = async ( outSpec.version = oas.info?.version; outSpec.description = oas.info?.description; outSpec.ref = outSpec.name; - outSpec.operations = JSON.stringify(parseSpecOperations(oas)); + outSpec.operations = JSON.stringify( + outSpec.specVersion?.startsWith('openapi') + ? parseOpenapiSpecOperations(oas) + : parseAsyncapiSpecOperations(oas) + ); return outSpec; }; -function parseSpecOperations(spec: any) { +function parseAsyncapiSpecOperations(spec: any) { + const flattenedOperations: { + operationId: string; + summary: string; + method: string; + path?: string; + scopes?: string[]; + }[] = []; + + const majorVersion = parseInt(spec?.asyncapi?.split('.')?.[0] ?? '2', 10); + + assert.strictEqual( + majorVersion >= 3, + true, + 'Unsupported AsyncAPI version: only versions 3.x and above are supported' + ); + + // AsyncAPI 3.x: operations are a top-level object; channels have an address field + if (!spec?.operations) { + return flattenedOperations; + } + + Object.entries(spec.operations).forEach( + ([operationId, operation]: [string, any]) => { + const channelRef: string | undefined = operation.channel?.['$ref']; + let channelAddress: string | undefined; + if (channelRef) { + const channelId = channelRef.replace('#/channels/', ''); + channelAddress = spec.channels?.[channelId]?.address ?? channelId; + } + + flattenedOperations.push({ + method: operation.action === 'send' ? 'SEND' : 'RECEIVE', + path: channelAddress, + operationId, + summary: operation.summary || '', + scopes: operation.security?.[0]?.['bearer_auth'] || [], + }); + } + ); + + return flattenedOperations; +} + +function parseOpenapiSpecOperations(spec: any) { const operations = spec?.paths && Object.keys(spec.paths).map((path) => {