diff --git a/src/services/keycloak/client-service.ts b/src/services/keycloak/client-service.ts index 353b44d0e..d1da437f3 100644 --- a/src/services/keycloak/client-service.ts +++ b/src/services/keycloak/client-service.ts @@ -9,6 +9,7 @@ import ClientScopeRepresentation from '@keycloak/keycloak-admin-client/lib/defs/ import CertificateRepresentation from '@keycloak/keycloak-admin-client/lib/defs/certificateRepresentation'; import RoleRepresentation from '@keycloak/keycloak-admin-client/lib/defs/roleRepresentation'; import ClientRepresentation from '@keycloak/keycloak-admin-client/lib/defs/clientRepresentation'; +import ProtocolMapperRepresentation from '@keycloak/keycloak-admin-client/lib/defs/protocolMapperRepresentation'; const logger = Logger('kc.client'); @@ -207,4 +208,8 @@ export class KeycloakClientService { roles ); } + + public async updateClient (id: string, mapperId: string, payload: ProtocolMapperRepresentation) { + await this.kcAdminClient.clients.updateProtocolMapper({id, mapperId}, payload) + } } diff --git a/src/services/keystone/access-request.ts b/src/services/keystone/access-request.ts index 289bbd2c0..605f346de 100644 --- a/src/services/keystone/access-request.ts +++ b/src/services/keystone/access-request.ts @@ -22,6 +22,7 @@ export async function getAccessRequestsByNamespace( requestor { name username + providerUsername } application { name diff --git a/src/services/keystone/gateway-service.ts b/src/services/keystone/gateway-service.ts index 1b0ebf528..e12d8ec28 100644 --- a/src/services/keystone/gateway-service.ts +++ b/src/services/keystone/gateway-service.ts @@ -122,12 +122,73 @@ export async function lookupServicesByNamespace( }); logger.debug('Query result %j', result); result.data.allGatewayServices.forEach((svc: GatewayService) => { - svc.plugins?.map((plugin) => (plugin.config = JSON.parse(plugin.config))); - svc.routes?.map((route) => + svc.plugins?.forEach( + (plugin) => (plugin.config = JSON.parse(plugin.config)) + ); + svc.routes?.forEach((route) => { route.plugins?.map( (plugin) => (plugin.config = JSON.parse(plugin.config)) - ) + ); + }); + }); + return result.data.allGatewayServices; +} + +export async function lookupServicesByNamespaceForReporting( + context: any, + ns: string +): Promise { + const result = await context.executeGraphQL({ + query: `query GetServicesForReporting($ns: String!) { + allGatewayServices(where: {namespace: $ns}) { + name + host + plugins { + name + config + } + routes { + name + methods + hosts + paths + plugins { + name + config + } + } + environment { + name + active + appId + flow + credentialIssuer { + inheritFrom { + name + } + } + product { + name + } + } + } + }`, + variables: { ns: ns }, + }); + logger.debug( + '[lookupServicesByNamespaceForReporting] Query result %j', + result + ); + result.data.allGatewayServices.forEach((svc: GatewayService) => { + svc.plugins?.forEach( + (plugin) => (plugin.config = JSON.parse(plugin.config)) ); + svc.routes?.forEach((route) => { + route.hosts = JSON.parse(route.hosts); + route.plugins?.map( + (plugin) => (plugin.config = JSON.parse(plugin.config)) + ); + }); }); return result.data.allGatewayServices; } diff --git a/src/services/keystone/metrics.ts b/src/services/keystone/metrics.ts index 15299cfcb..8b97b227a 100644 --- a/src/services/keystone/metrics.ts +++ b/src/services/keystone/metrics.ts @@ -7,6 +7,7 @@ import format from 'date-fns/format'; import times from 'lodash/times'; import sum from 'lodash/sum'; import formatISO from 'date-fns/formatISO'; +import { strictEqual } from 'assert'; interface DailyDatum { day: string; @@ -21,13 +22,13 @@ interface DailyDatum { const logger = Logger('keystone.metrics'); const getServiceMetricsQuery = gql` - query GetServiceMetrics($service: String!, $days: [String!]) { + query GetServiceMetrics($services: [String]!, $days: [String!]) { allMetrics( sortBy: day_ASC where: { query: "kong_http_requests_hourly_service" day_in: $days - service: { name_contains: $service } + service: { name_in: $services } } ) { query @@ -82,17 +83,23 @@ const getAllConsumerDailyMetricsQuery = gql` export async function getServiceMetrics( context: any, - service: string, + services: string[], days: string[] ): Promise { + strictEqual(services.length > 0, true); + const result = await context.executeGraphQL({ query: getServiceMetricsQuery, - variables: { service, days }, + variables: { services, days }, }); + + if (result.errors) { + logger.error('[getServiceMetrics] %j', result.errors); + } logger.debug( - '[getServiceMetrics] (%s) result row count %d', - service, - result.data.allMetrics.length + '[getServiceMetrics] (%j) result row count %d', + services, + result.data.allMetrics?.length ?? 0 ); return result.data.allMetrics; } diff --git a/src/services/keystone/product-environment.ts b/src/services/keystone/product-environment.ts index e710d0744..d3490d244 100644 --- a/src/services/keystone/product-environment.ts +++ b/src/services/keystone/product-environment.ts @@ -180,11 +180,20 @@ export async function lookupEnvironmentsByNS( appId name flow + active additionalDetailsToRequest approval + services { + name + } product { id name + dataset { + name + title + isInCatalog + } } legal { reference diff --git a/src/services/report/data/consumer-requests.ts b/src/services/report/data/consumer-requests.ts index d0332e442..ba589a8ce 100644 --- a/src/services/report/data/consumer-requests.ts +++ b/src/services/report/data/consumer-requests.ts @@ -5,7 +5,7 @@ import { ReportOfNamespaces } from './namespaces'; import { ReportOfGatewayMetrics } from './gateway-metrics'; import { getAccessRequestsByNamespace } from '../../keystone'; -interface ReportOfConsumerRequest { +export interface ReportOfConsumerRequest { namespace: string; displayName?: string; prod_name: string; @@ -44,7 +44,7 @@ export async function getConsumerRequests( prod_env_flow: req.productEnvironment?.flow, app_name: req.application.name, app_id: req.application.appId, - requestor: req.requestor.name, + requestor: req.requestor.name ?? req.requestor.providerUsername, req_created: req.createdAt, req_reviewer: '', req_result: req.isComplete diff --git a/src/services/report/data/features/api_directory.ts b/src/services/report/data/features/api_directory.ts new file mode 100644 index 000000000..7d9a41a92 --- /dev/null +++ b/src/services/report/data/features/api_directory.ts @@ -0,0 +1,18 @@ +import { GatewayService } from '../../../../services/keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; +import { ReportOfProducts } from '../products'; + +export function has_feature_api_directory_for_product( + ns: ReportOfNamespaces, + product: ReportOfProducts +): Boolean { + return product.prod_env_active === 'Y'; +} + +export function has_feature_api_directory_for_service( + ns: ReportOfNamespaces, + service: GatewayService, + routeName: string +): Boolean { + return Boolean(service.environment?.active); +} diff --git a/src/services/report/data/features/consumer_mgmt.ts b/src/services/report/data/features/consumer_mgmt.ts new file mode 100644 index 000000000..9c3a4e4cb --- /dev/null +++ b/src/services/report/data/features/consumer_mgmt.ts @@ -0,0 +1,13 @@ +import { GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; +import { ReportOfProducts } from '../products'; + +export function has_feature_consumer_mgmt( + ns: ReportOfNamespaces, + product: ReportOfProducts +): Boolean { + return Boolean( + ['client-credentials', 'kong-api-key-acl'].indexOf(product.prod_env_flow) >= + 0 && product.prod_env_active === 'Y' + ); +} diff --git a/src/services/report/data/features/dataset_in_catalog.ts b/src/services/report/data/features/dataset_in_catalog.ts new file mode 100644 index 000000000..346fb23c2 --- /dev/null +++ b/src/services/report/data/features/dataset_in_catalog.ts @@ -0,0 +1,10 @@ +import { GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; +import { ReportOfProducts } from '../products'; + +export function has_feature_dataset_in_catalog( + ns: ReportOfNamespaces, + product: ReportOfProducts +): Boolean { + return product.dataset_in_catalog; +} diff --git a/src/services/report/data/features/has_gateway_mgmt.ts b/src/services/report/data/features/has_gateway_mgmt.ts new file mode 100644 index 000000000..6a0a706bf --- /dev/null +++ b/src/services/report/data/features/has_gateway_mgmt.ts @@ -0,0 +1,12 @@ +import { GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; + +// If there is a Route, then can consider gateway +// management being used +export function has_gateway_mgmt( + ns: ReportOfNamespaces, + service: GatewayService, + routeName: string +): Boolean { + return Boolean(true); +} diff --git a/src/services/report/data/features/index.ts b/src/services/report/data/features/index.ts new file mode 100644 index 000000000..c980c338b --- /dev/null +++ b/src/services/report/data/features/index.ts @@ -0,0 +1,79 @@ +import { + GatewayPlugin, + GatewayService, +} from '../../../../services/keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; +import { + has_feature_api_directory_for_product, + has_feature_api_directory_for_service, +} from './api_directory'; +import { has_feature_consumer_mgmt } from './consumer_mgmt'; +import { has_feature_dataset_in_catalog } from './dataset_in_catalog'; +import { has_gateway_mgmt } from './has_gateway_mgmt'; +import { is_production } from './production'; +import { has_feature_protected } from './protected'; +import { has_feature_protected_externally } from './protected_exterrnally'; +import { has_feature_shared_idp } from './shared_idp'; +import { has_feature_two_tiered_access } from './two_tiered_access'; + +export const ProductFeatureList: { [key: string]: Function } = { + consumer_mgmt: has_feature_consumer_mgmt, + protected_externally: has_feature_protected_externally, + dataset_in_catalog: has_feature_dataset_in_catalog, + api_directory: has_feature_api_directory_for_product, +}; + +export const FeatureList: { [key: string]: Function } = { + api_directory: has_feature_api_directory_for_service, // evaluated at a Gateway level + shared_idp: has_feature_shared_idp, + gateway_mgmt: has_gateway_mgmt, + consumer_mgmt: undefined, // evaluated at a Gateway level + protected: has_feature_protected, + two_tiered_access: has_feature_two_tiered_access, + production: is_production, + protected_externally: undefined, // evaluated at a Gateway level + dataset_in_catalog: undefined, // evaluated at a Gateway level +}; + +export function getFeatures( + ns: ReportOfNamespaces, + services: GatewayService[], + routeName: string +): string[] { + const service = findService(services, routeName); + const features: string[] = []; + Object.entries(FeatureList).forEach((func) => { + if (func[1] && func[1](ns, service, routeName)) { + features.push(func[0]); + } + }); + return features; +} + +export function getPlugins( + ns: ReportOfNamespaces, + services: GatewayService[], + routeName: string +): string[] { + const plugins: string[] = []; + const service = findService(services, routeName); + + plugins.push.apply(plugins, getPluginNames(service.plugins)); + service.routes.forEach((route) => { + plugins.push.apply(plugins, getPluginNames(route.plugins)); + }); + return [...new Set(plugins)].sort(); +} + +function findService( + services: GatewayService[], + routeName: string +): GatewayService { + return services + .filter((s) => s.routes.filter((r) => r.name == routeName).length > 0) + .pop(); +} + +function getPluginNames(plugins: GatewayPlugin[]): string[] { + return plugins?.map((p) => p.name); +} diff --git a/src/services/report/data/features/production.ts b/src/services/report/data/features/production.ts new file mode 100644 index 000000000..ba82590e4 --- /dev/null +++ b/src/services/report/data/features/production.ts @@ -0,0 +1,23 @@ +import { GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; + +const re = /(dev.|test.|tst.|dlv.|delivery.|-dev|-test|-d.|-t.).*$/; + +export function is_production( + ns: ReportOfNamespaces, + service: GatewayService, + routeName: string +): Boolean { + return ( + service.routes.filter( + (r) => + r.name == routeName && + (r.hosts as any).filter((h: string) => checkNonProd(h) == false) + .length > 0 + ).length > 0 + ); +} + +function checkNonProd(host: string) { + return re.test(host); +} diff --git a/src/services/report/data/features/protected.ts b/src/services/report/data/features/protected.ts new file mode 100644 index 000000000..edf1e03b3 --- /dev/null +++ b/src/services/report/data/features/protected.ts @@ -0,0 +1,25 @@ +import { GatewayPlugin, GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; + +export function has_feature_protected( + ns: ReportOfNamespaces, + service: GatewayService, + routeName: string +): Boolean { + return ( + // check either a `jwt-keycloak`, `oidc` or `acl` + // plugins exists and is active + check(service.plugins) || + service.routes.filter((r) => r.name == routeName && check(r.plugins)) + .length > 0 + ); +} + +function check(plugins: GatewayPlugin[]): boolean { + return ( + plugins + // .filter((p: any) => p.enabled) + .filter((p: any) => ['jwt-keycloak', 'oidc', 'acl'].indexOf(p.name) >= 0) + .length > 0 + ); +} diff --git a/src/services/report/data/features/protected_exterrnally.ts b/src/services/report/data/features/protected_exterrnally.ts new file mode 100644 index 000000000..25240a679 --- /dev/null +++ b/src/services/report/data/features/protected_exterrnally.ts @@ -0,0 +1,10 @@ +import { GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; +import { ReportOfProducts } from '../products'; + +export function has_feature_protected_externally( + ns: ReportOfNamespaces, + product: ReportOfProducts +): Boolean { + return 'protected-externally' === product.prod_env_flow; +} diff --git a/src/services/report/data/features/shared_idp.ts b/src/services/report/data/features/shared_idp.ts new file mode 100644 index 000000000..89316ea6e --- /dev/null +++ b/src/services/report/data/features/shared_idp.ts @@ -0,0 +1,10 @@ +import { GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; + +export function has_feature_shared_idp( + ns: ReportOfNamespaces, + service: GatewayService, + routeName: string +): Boolean { + return Boolean(service.environment?.credentialIssuer?.inheritFrom?.name); +} diff --git a/src/services/report/data/features/two_tiered_access.ts b/src/services/report/data/features/two_tiered_access.ts new file mode 100644 index 000000000..ad4b499ac --- /dev/null +++ b/src/services/report/data/features/two_tiered_access.ts @@ -0,0 +1,20 @@ +import { GatewayPlugin, GatewayService } from '../../../keystone/types'; +import { ReportOfNamespaces } from '../namespaces'; + +export function has_feature_two_tiered_access( + ns: ReportOfNamespaces, + service: GatewayService, + routeName: string +): Boolean { + return ( + // check either service or route plugin + // has the "config.anonymous" + check(service.plugins) || + service.routes.filter((r) => r.name == routeName && check(r.plugins)) + .length > 0 + ); +} + +function check(plugins: GatewayPlugin[]): boolean { + return plugins.filter((p) => (p.config as any).anonymous).length > 0; +} diff --git a/src/services/report/data/gateway-metrics.ts b/src/services/report/data/gateway-metrics.ts index 42128f0b1..7dd4fa4a8 100644 --- a/src/services/report/data/gateway-metrics.ts +++ b/src/services/report/data/gateway-metrics.ts @@ -1,17 +1,24 @@ -import { lookupServicesByNamespace } from '../../keystone/gateway-service'; +import { lookupServicesByNamespaceForReporting } from '../../keystone/gateway-service'; import { Keystone } from '@keystonejs/keystone'; import { ReportOfNamespaces } from './namespaces'; import { getServiceMetrics, calculateStats } from '../../keystone'; import { dateRange } from '../../utils'; +import { getFeatures, getPlugins } from './features'; export interface ReportOfGatewayMetrics { namespace: string; - displayName?: string; + display_name?: string; + data_plane: string; + service_name: string; + request_uri_host: string; prod_name?: string; prod_env_name?: string; prod_env_app_id?: string; - service_name: string; day_30_total: number; + route_names: string[]; + plugins: string[]; + features: string[]; + upstream: string; } /* @@ -20,26 +27,77 @@ export async function getGatewayMetrics( ksCtx: Keystone, namespaces: ReportOfNamespaces[] ): Promise { + const days = dateRange(30); + const dataPromises = namespaces.map( async (ns): Promise => { - const services = await lookupServicesByNamespace(ksCtx, ns.name); + const services = await lookupServicesByNamespaceForReporting( + ksCtx, + ns.name + ); + + const metrics = + services.length == 0 + ? [] + : await getServiceMetrics( + ksCtx, + services.map((s) => s.name), + days + ); let data: ReportOfGatewayMetrics[] = []; const subPromises = services.map(async (svc) => { - const days = dateRange(30); - const metrics = await getServiceMetrics(ksCtx, svc.name, days); - - const { totalRequests } = calculateStats(metrics); - - data.push({ - namespace: ns.name, - displayName: ns.displayName, - service_name: svc.name, - prod_name: svc.environment?.product?.name, - prod_env_name: svc.environment?.name, - prod_env_app_id: svc.environment?.appId, - day_30_total: totalRequests, - }); + const { totalRequests } = metrics + ? calculateStats( + metrics.filter((metric) => metric.service?.name == svc.name) + ) + : { totalRequests: -1 }; + + const prelimRouteList: any[] = []; + for (const route of svc.routes) { + for (const host of route.hosts) { + prelimRouteList.push({ + route_name: route.name, + request_uri_host: host, + plugins: getPlugins(ns, services, route.name), + features: getFeatures(ns, services, route.name), + }); + } + } + + // we want to have a row for each request_uri_host + // and merge/dedup route_name, plugins and features + const requestUriHosts = [ + ...new Set(prelimRouteList.map((route) => route.request_uri_host)), + ]; + + for (const host of requestUriHosts) { + const prelimListForRoute = prelimRouteList.filter( + (route) => route.request_uri_host == host + ); + + data.push({ + namespace: ns.name, + display_name: ns.displayName, + data_plane: ns.permDataPlane ?? 'default', + service_name: svc.name, + request_uri_host: host, + upstream: svc.host, + prod_name: svc.environment?.product?.name, + prod_env_name: svc.environment?.name, + prod_env_app_id: svc.environment?.appId, + day_30_total: totalRequests, + route_names: prelimListForRoute + .map((route) => route.route_name) + .sort(), + plugins: mergeAndDedup( + prelimListForRoute.map((route) => route.plugins) + ), + features: mergeAndDedup( + prelimListForRoute.map((route) => route.features) + ), + }); + } }); await Promise.all(subPromises); @@ -50,3 +108,9 @@ export async function getGatewayMetrics( const reportOfReports = await Promise.all(dataPromises); return [].concat.apply([], reportOfReports); } + +function mergeAndDedup(items: string[][]): string[] { + const newList: string[] = []; + items.forEach((values) => newList.push.apply(newList, values)); + return [...new Set(newList)].sort(); +} diff --git a/src/services/report/data/namespaces.ts b/src/services/report/data/namespaces.ts index c348d5c0a..cdb034422 100644 --- a/src/services/report/data/namespaces.ts +++ b/src/services/report/data/namespaces.ts @@ -1,6 +1,7 @@ import { transformSingleValueAttributes, camelCaseAttributes, + dedup, } from '../../utils'; import { KeycloakGroupService } from '../../keycloak'; import { getMyNamespaces } from '../../workflow'; @@ -9,6 +10,10 @@ import { NamespaceSummary, } from '../../workflow/get-namespaces'; import { GWAService } from '../../gwaapi'; +import { ReportOfGatewayMetrics } from './gateway-metrics'; +import { ReportOfProducts } from './products'; +import { ReportOfConsumerRequest } from './consumer-requests'; +import { lookupUsersByNamespace } from '@/services/keystone'; export interface ReportOfNamespaces { resource_id: string; @@ -20,6 +25,9 @@ export interface ReportOfNamespaces { org?: string; orgUnit?: string; decommissioned?: string; + consumers?: number; + day_30_total?: number; + features?: { [feature: string]: string }; } /* @@ -76,3 +84,36 @@ export async function getNamespaces( return Promise.all(dataPromises); } + +export function rollupFeatures( + namespaces: ReportOfNamespaces[], + gatewayMetrics: ReportOfGatewayMetrics[], + products: ReportOfProducts[] +) { + namespaces.forEach((ns) => { + ns.features = {}; + ns.day_30_total = 0; + gatewayMetrics + .filter((gw) => gw.namespace == ns.name) + .forEach((gw) => { + gw.features.forEach((feat) => (ns.features[feat] = 'Y')); + ns.day_30_total = ns.day_30_total + gw.day_30_total; + }); + products + .filter((gw) => gw.namespace == ns.name) + .forEach((gw) => { + gw.features.forEach((feat) => (ns.features[feat] = 'Y')); + }); + }); +} + +export function rollupConsumers( + namespaces: ReportOfNamespaces[], + requests: ReportOfConsumerRequest[] +) { + namespaces.forEach((ns) => { + ns.consumers = requests.filter( + (req) => req.namespace === ns.name && req.req_result === 'Approved' + ).length; + }); +} diff --git a/src/services/report/data/products.ts b/src/services/report/data/products.ts new file mode 100644 index 000000000..10b7b173c --- /dev/null +++ b/src/services/report/data/products.ts @@ -0,0 +1,64 @@ +import { lookupServicesByNamespaceForReporting } from '../../keystone/gateway-service'; +import { Keystone } from '@keystonejs/keystone'; +import { ReportOfNamespaces } from './namespaces'; +import { getServiceMetrics, calculateStats } from '../../keystone'; +import { dateRange } from '../../utils'; +import { getFeatures, getPlugins, ProductFeatureList } from './features'; +import { lookupEnvironmentsByNS } from '../../../services/keystone/product-environment'; +import { Environment } from '../../../services/keystone/types'; +import { has_feature_protected_externally } from './features/protected_exterrnally'; +import { has_feature_dataset_in_catalog } from './features/dataset_in_catalog'; + +export interface ReportOfProducts { + namespace: string; + display_name?: string; + prod_name?: string; + prod_env_active: 'Y' | 'N'; + prod_env_approval: 'Y' | 'N'; + prod_env_name?: string; + prod_env_flow?: string; + prod_env_app_id?: string; + dataset_name?: string; + dataset_title?: string; + dataset_in_catalog: boolean; + service_names?: string[]; + features: string[]; +} + +export async function getProducts( + ksCtx: Keystone, + namespaces: ReportOfNamespaces[] +): Promise { + const dataPromises = namespaces.map( + async (ns): Promise => { + const environments = await lookupEnvironmentsByNS(ksCtx, ns.name); + return environments.map((env) => { + const product = { + namespace: ns.name, + display_name: ns.displayName, + prod_name: env.product?.name, + prod_env_name: env.name, + prod_env_app_id: env.appId, + prod_env_flow: env.flow, + prod_env_active: env.active ? 'Y' : 'N', + prod_env_approval: env.approval ? 'Y' : 'N', + dataset_name: env.product?.dataset?.name, + dataset_title: env.product?.dataset?.title, + dataset_in_catalog: env.product?.dataset?.isInCatalog, + service_names: env.services.map((s) => s.name), + features: [], + } as ReportOfProducts; + + Object.entries(ProductFeatureList).forEach((func) => { + if (func[1] && func[1](ns, product)) { + product.features.push(func[0]); + } + }); + return product; + }); + } + ); + + const reportOfReports = await Promise.all(dataPromises); + return [].concat.apply([], reportOfReports); +} diff --git a/src/services/report/output/structure.ts b/src/services/report/output/structure.ts index 6c0d77b22..0aefb4d45 100644 --- a/src/services/report/output/structure.ts +++ b/src/services/report/output/structure.ts @@ -1,6 +1,9 @@ +import { FeatureList } from '../data/features'; + export const reportOrder = [ 'namespaces', 'ns_access', + 'products', 'gateway_metrics', 'gateway_controls', 'consumer_requests', @@ -9,7 +12,7 @@ export const reportOrder = [ 'consumer_controls', ]; -export const reportStructure: any = { +const reportStructure: any = { namespaces: { label: 'Gateways', fields: [ @@ -23,6 +26,16 @@ export const reportStructure: any = { key: 'displayName', width: 26, }, + { + header: 'Consumers', + key: 'consumers', + width: 25, + }, + { + header: '30 Day Total', + key: 'day_30_total', + width: 25, + }, { header: 'Privileged', key: 'permProtectedNs', @@ -38,20 +51,21 @@ export const reportStructure: any = { key: 'permDataPlane', width: 25, }, + { header: 'Decommissioned', key: 'decommissioned', width: 20 }, { header: 'Org', - key: 'org', - width: 40, + key: 'org.title', + width: 50, }, { header: 'Org Unit', - key: 'orgUnit', - width: 25, + key: 'orgUnit.title', + width: 50, }, ], }, ns_access: { - label: 'Gateway Access', + label: 'Gateway Admin Access', fields: [ { header: 'Gateway ID', @@ -64,13 +78,13 @@ export const reportStructure: any = { width: 26, }, { - header: 'Subject', - key: 'subject', + header: 'Subject Name', + key: 'subjectName', width: 40, }, { - header: 'Subject Name', - key: 'subjectName', + header: 'Subject Email', + key: 'subjectEmail', width: 40, }, { @@ -80,8 +94,8 @@ export const reportStructure: any = { }, ], }, - gateway_metrics: { - label: 'Gateway Metrics', + products: { + label: 'Gateway Products', fields: [ { header: 'Gateway ID', @@ -90,9 +104,19 @@ export const reportStructure: any = { }, { header: 'Gateway Display Name', - key: 'displayName', + key: 'display_name', width: 26, }, + { + header: 'Dataset', + key: 'dataset_title', + width: 45, + }, + { + header: 'Dataset Name', + key: 'dataset_name', + width: 45, + }, { header: 'Product', key: 'prod_name', @@ -103,12 +127,97 @@ export const reportStructure: any = { key: 'prod_env_name', width: 15, }, + { + header: 'Environment App ID', + key: 'prod_env_app_id', + width: 30, + }, + { + header: 'Active', + key: 'prod_env_active', + width: 15, + }, + { + header: 'Approval', + key: 'prod_env_approval', + width: 15, + }, + { + header: 'Flow', + key: 'prod_env_flow', + width: 32, + }, + { + header: 'Services', + key: 'service_names', + width: 32, + }, + { + header: 'Features', + key: 'features', + width: 32, + }, + ], + }, + gateway_metrics: { + label: 'Gateway Service Metrics', + fields: [ + { + header: 'Gateway ID', + key: 'namespace', + width: 20, + }, + { + header: 'Gateway Display Name', + key: 'display_name', + width: 26, + }, + { + header: 'Route Host', + key: 'request_uri_host', + width: 50, + }, + { header: '30 Day Total', key: 'day_30_total', width: 20 }, { header: 'Service', key: 'service_name', width: 40, }, - { header: '30 Day Total', key: 'day_30_total', width: 20 }, + { + header: 'Service Upstream', + key: 'upstream', + width: 40, + }, + { + header: 'Features', + key: 'features', + width: 32, + }, + { + header: 'Plugins', + key: 'plugins', + width: 32, + }, + { + header: 'Data Plane', + key: 'data_plane', + width: 20, + }, + { + header: 'Route', + key: 'route_names', + width: 40, + }, + { + header: 'Product', + key: 'prod_name', + width: 32, + }, + { + header: 'Environment', + key: 'prod_env_name', + width: 15, + }, ], }, gateway_controls: { @@ -308,3 +417,16 @@ export const reportStructure: any = { ], }, }; + +reportStructure.namespaces.fields.push.apply( + reportStructure.namespaces.fields, + Object.keys(FeatureList) + .sort() + .map((feat) => ({ + header: feat, + key: `features.${feat}`, + width: 30, + })) +); + +export { reportStructure }; diff --git a/src/services/report/output/xls-generator.ts b/src/services/report/output/xls-generator.ts index fc342565a..d1470e77a 100644 --- a/src/services/report/output/xls-generator.ts +++ b/src/services/report/output/xls-generator.ts @@ -10,25 +10,30 @@ function toText(field: any, value: any) { } } +function getValueByPath(raw: any, path: string) { + return path.split('.').reduce((acc, key) => acc && acc[key], raw); +} + export function generateExcelWorkbook(data: any) { const workbook = new ExcelJS.Workbook(); reportOrder.forEach((tab: string) => { const struct = reportStructure[tab]; - const sheet = workbook.addWorksheet(struct.label); - sheet.columns = struct.fields.map((field: any) => - Object.assign(field, { - style: { font: { bold: false, size: 12, name: 'Arial' } }, - }) - ); + if (tab in data) { + const sheet = workbook.addWorksheet(struct.label); - sheet.getRow(1).font = { bold: true, size: 12, name: 'Arial' }; + sheet.columns = struct.fields.map((field: any) => + Object.assign(field, { + style: { font: { bold: false, size: 12, name: 'Arial' } }, + }) + ); - if (tab in data) { - data[tab].forEach((raw: any) => { - const row = struct.fields.map((field: any) => - field.key in raw ? toText(field, raw[field.key]) : '' + sheet.getRow(1).font = { bold: true, size: 12, name: 'Arial' }; + + data[tab]?.forEach((raw: any) => { + const row = struct.fields.map( + (field: any) => toText(field, getValueByPath(raw, field.key)) ?? '' ); sheet.addRow(row); }); diff --git a/src/services/report/workbook.service.ts b/src/services/report/workbook.service.ts index c187cf09a..da8d97d7a 100644 --- a/src/services/report/workbook.service.ts +++ b/src/services/report/workbook.service.ts @@ -17,6 +17,8 @@ import { getGatewayMetrics, ReportOfGatewayMetrics, } from './data'; +import { rollupFeatures } from './data/namespaces'; +import { getProducts } from './data/products'; export class WorkbookService { keystone: Keystone; @@ -43,6 +45,10 @@ export class WorkbookService { gateway_metrics ); + const products = await getProducts(this.keystone, namespaces); + + rollupFeatures(namespaces, gateway_metrics, products); + const gateway_controls = await getGatewayControls( this.keystone, namespaces, @@ -79,6 +85,7 @@ export class WorkbookService { const data = { namespaces, ns_access, + products, gateway_metrics, gateway_controls, service_access, diff --git a/src/services/utils.ts b/src/services/utils.ts index 7bcfb87d4..307f46ab8 100644 --- a/src/services/utils.ts +++ b/src/services/utils.ts @@ -79,3 +79,7 @@ export async function fetchWithTimeout(resource: string, options: any = {}) { export function alphanumericNoSpaces(str: string) { return str.replace(/[^A-Za-z0-9 :-]/gim, '').replace(/[ :]/gim, '-'); } + +export function dedup(ls: string[]) { + return [...new Set(ls)]; +} diff --git a/src/test/integrated/keycloak/client.ts b/src/test/integrated/keycloak/client.ts index 22852ed26..5e07dfb37 100644 --- a/src/test/integrated/keycloak/client.ts +++ b/src/test/integrated/keycloak/client.ts @@ -19,43 +19,72 @@ import { o } from '../util'; import { KeycloakClientService } from '../../../services/keycloak'; import { UMAResourceRegistrationService } from '../../../services/uma2'; +import ProtocolMapperRepresentation from '@keycloak/keycloak-admin-client/lib/defs/protocolMapperRepresentation'; (async () => { - if (false) { + if (true) { + // Cleanup of a Mapper for ClientId that has changed from "clientId" + // to "client_id" + const kc = new KeycloakClientService(process.env.ISSUER); await kc.login(process.env.CID, process.env.CSC); - await kc.regenerateSecret('6af832cb-6178-438f-a4fc-8c5e1d14d5d2'); - const res = await kc.list('42da4f15'); - o(res); + const cids: string[] = []; + + for (const cid of cids) { + try { + const res = await kc.findByClientId(cid); + if (res.enabled) { + const mapper: ProtocolMapperRepresentation = res.protocolMappers + .filter((m) => m.name == 'Client ID') + .pop(); + if (mapper) { + console.log(`${cid} : ${mapper.config['claim.name']}`); + if (mapper.config['claim.name'] == 'clientId') { + console.log('Updating!'); + mapper.config['claim.name'] = 'client_id'; + await kc.updateClient(res.id, mapper.id, mapper); + console.log('Updated!'); + } + } else { + console.log(`${cid} : MISSING`); + } + } else { + console.log(`${cid} : DISABLED`); + } + } catch (e) { + console.log(e); + console.log(`${cid} : SKIPPED`); + } + } } - const resourceRegistrationEndpoint = - process.env.ISSUER + '/authz/protection/resource_set'; - const accessToken = process.env.TOK; - const clientUuid = '250398da-a174-404d-ae18-6a6ebc9de06d'; - - const kcprotectApi = new UMAResourceRegistrationService( - resourceRegistrationEndpoint, - accessToken - ); - const resOwnerResourceIds = await kcprotectApi.listResources({ - owner: clientUuid, - type: 'namespace', - }); - - const namespaces = await kcprotectApi.listResourcesByIdList( - resOwnerResourceIds - ); - - const matched = namespaces - .filter((ns) => ns.name == 'testelson') - .map((ns) => ({ - id: ns.id, - name: ns.name, - scopes: ns.resource_scopes, - })); - - o(matched); + // const resourceRegistrationEndpoint = + // process.env.ISSUER + '/authz/protection/resource_set'; + // const accessToken = process.env.TOK; + // const clientUuid = '250398da-a174-404d-ae18-6a6ebc9de06d'; + + // const kcprotectApi = new UMAResourceRegistrationService( + // resourceRegistrationEndpoint, + // accessToken + // ); + // const resOwnerResourceIds = await kcprotectApi.listResources({ + // owner: clientUuid, + // type: 'namespace', + // }); + + // const namespaces = await kcprotectApi.listResourcesByIdList( + // resOwnerResourceIds + // ); + + // const matched = namespaces + // .filter((ns) => ns.name == 'testelson') + // .map((ns) => ({ + // id: ns.id, + // name: ns.name, + // scopes: ns.resource_scopes, + // })); + + // o(matched); })(); diff --git a/src/test/integrated/reports/gatewayMetrics.ts b/src/test/integrated/reports/gatewayMetrics.ts new file mode 100644 index 000000000..c41a09dd9 --- /dev/null +++ b/src/test/integrated/reports/gatewayMetrics.ts @@ -0,0 +1,114 @@ +/* +Wire up directly with Keycloak and use the Services +export TOK="" +To run: +npm run ts-build +npm run ts-watch +node dist/test/integrated/reports/gatewayMetrics.js +*/ +import { createWriteStream } from 'fs'; +import InitKeystone from '../keystonejs/init'; +import { o } from '../util'; +import { Logger } from '../../../logger'; +import { + getConsumerRequests, + getGatewayMetrics, + getNamespaceAccess, +} from '../../../services/report/data'; +import { + getGwaProductEnvironment, + injectResSvrAccessTokenToContext, +} from '../../../services/workflow'; +import { lookupProductEnvironmentServicesBySlug } from '../../../services/keystone'; +import { getNamespaces } from '../../../services/report/ops-metrics'; +import { generateExcelWorkbook } from '../../../services/report/output/xls-generator'; +import { + rollupConsumers, + rollupFeatures, +} from '../../../services/report/data/namespaces'; +import { getProducts } from '../../../services/report/data/products'; + +const logger = Logger('test.reports'); + +(async () => { + const keystone = await InitKeystone(); + + const ns = 'refactortime'; + const skipAccessControl = true; + + const identity = { + id: null, + name: 'Sample User', + username: 'sample_username', + namespace: ns, + roles: JSON.stringify(['access-manager']), + scopes: [], + //userId: '60c9124f3518951bb519084d', + userId: '60c9124f3518951bb519084d', // acope@idir + } as any; + + const ctx = keystone.createContext({ + skipAccessControl, + authentication: { item: identity }, + }); + ctx.req = { + headers: { + 'x-forwarded-access-token': process.env.TOK, + }, + }; + + ctx.req.user = { sub: '15a3cbbe-95b5-49f0-84ee-434a9b92d04a' }; + + // const envCtx = await getGwaProductEnvironment(ctx, true); + + //await injectResSvrAccessTokenToContext(envCtx); + + const nslist = await getNamespaces(ctx); + nslist.sort((a, b) => a.name.localeCompare(b.name)); + o(nslist); + + const filteredNS = nslist + // .filter( + // (ns) => ['dss-loc-gold', 'dss-loc', 'social-gold'].indexOf(ns.name) >= 0 + // ) + .map((ns) => ({ + resource_id: ns.id, + name: ns.name, + displayName: ns.displayName, + permDataPlane: ns.permDataPlane, + })); + + const envCtx = await getGwaProductEnvironment(ctx, true); + await injectResSvrAccessTokenToContext(envCtx); + // const nsAccess = await getNamespaceAccess(ctx, envCtx, filteredNS); + + // const consumerRequests = await getConsumerRequests(ctx, filteredNS); + + const products = await getProducts(ctx, filteredNS); + + // const gatewayMetrics = await getGatewayMetrics(ctx, filteredNS); + // gatewayMetrics.sort((a, b) => + // a.request_uri_host.localeCompare(b.request_uri_host) + // ); + + // rollupFeatures(nslist as any, gatewayMetrics, products); + + // rollupConsumers(nslist as any, consumerRequests); + + const workbook = generateExcelWorkbook({ + namespaces: nslist, + products, + // ns_access: nsAccess, + // consumer_requests: consumerRequests, + // gateway_metrics: gatewayMetrics, + }); + const buffer = await workbook.xlsx.writeBuffer(); + + var stream = createWriteStream('gatewaymetrics.xlsx'); + stream.once('open', function (fd) { + stream.write(buffer); + stream.end(); + }); + + await keystone.disconnect(); +})(); diff --git a/src/test/services/keystone/metrics.test.ts b/src/test/services/keystone/metrics.test.ts index 0348fe636..2d63a3f2f 100644 --- a/src/test/services/keystone/metrics.test.ts +++ b/src/test/services/keystone/metrics.test.ts @@ -15,9 +15,9 @@ describe('KeystoneJS', function () { '2021-10-18', '2021-10-17', ]; - const service = 'aps-authz'; + const services = ['aps-authz']; - const metrics = await getServiceMetrics(context, service, days); + const metrics = await getServiceMetrics(context, services, days); expect(metrics.length).toBe(5); const { totalRequests } = calculateStats(metrics);