diff --git a/apps/api-server/package.json b/apps/api-server/package.json index cc05597da3..70457a5079 100644 --- a/apps/api-server/package.json +++ b/apps/api-server/package.json @@ -18,7 +18,8 @@ "test:e2e:debug": "jest --no-cache --config jest-e2e.config.js --verbose --forceExit", "init-database": "node ./scripts/init-database.js", "migrate-database": "node ./scripts/migrate-database.js", - "build-packages": "node ./scripts/build-packages.js" + "build-packages": "node ./scripts/build-packages.js", + "run-crons": "node run-crons.js" }, "dependencies": { "@azure/identity": "^4.6.0", diff --git a/apps/api-server/run-crons.js b/apps/api-server/run-crons.js new file mode 100644 index 0000000000..775b6da243 --- /dev/null +++ b/apps/api-server/run-crons.js @@ -0,0 +1,103 @@ +#!/usr/bin/env node + +/** + * Standalone script for running cron jobs via Kubernetes CronJob. + * + * Loads all cron definitions from src/cron/ and executes any job whose + * cronTime schedule has at least one match in the past CRON_WINDOW_MINUTES + * (default: 15). This allows a single K8s CronJob running every X minutes + * to cover all the different schedules defined in the application. + * + * Usage: + * CRON_WINDOW_MINUTES=15 node run-crons.js + */ + +require('dotenv').config(); + +const config = require('config'); + +// Env variable used by npm's `debug` package. +process.env.DEBUG = config.logging; + +require('./config/promises'); +require('./config/moment'); + +const { CronTime } = require('cron'); +const moment = require('moment-timezone'); +const path = require('path'); +const fs = require('fs'); + +const windowMinutes = parseInt(process.env.CRON_WINDOW_MINUTES, 10) || 15; +const timeZone = config.get('timeZone'); +const cronDir = path.join(__dirname, 'src', 'cron'); + +/** + * Check if a cron schedule had at least one tick in the window [now - windowMs, now]. + * + * Uses CronTime._getNextDateFrom(windowStart) — if that next fire time falls + * before "now", the job should have fired within the window. + */ +function shouldRunInWindow(cronTime, windowMs) { + try { + const ct = new CronTime(cronTime, timeZone); + const now = moment(); + const windowStart = moment().subtract(windowMs, 'milliseconds'); + const nextFromWindow = ct._getNextDateFrom(windowStart); + return nextFromWindow.isSameOrBefore(now); + } catch (e) { + console.warn(`[run-crons] Invalid cron expression: ${cronTime}`, e.message); + return false; + } +} + +async function main() { + const windowMs = windowMinutes * 60 * 1000; + console.log( + `[run-crons] Checking cron jobs with a ${windowMinutes}-minute window` + ); + + const files = fs + .readdirSync(cronDir) + .filter((f) => f.endsWith('.js') && f !== 'index.js'); + + const jobs = []; + + for (const file of files) { + const jobDef = require(path.join(cronDir, file)); + const name = file.replace(/\.js$/, ''); + + if (!jobDef.cronTime || !jobDef.onTick) { + console.log(`[run-crons] Skipping ${name}: missing cronTime or onTick`); + continue; + } + + if (shouldRunInWindow(jobDef.cronTime, windowMs)) { + console.log( + `[run-crons] Running: ${name} (schedule: ${jobDef.cronTime})` + ); + jobs.push( + Promise.resolve() + .then(() => jobDef.onTick()) + .then(() => console.log(`[run-crons] Completed: ${name}`)) + .catch((err) => console.error(`[run-crons] Failed: ${name}`, err)) + ); + } else { + console.log(`[run-crons] Skipping ${name}: not scheduled in window`); + } + } + + if (jobs.length === 0) { + console.log('[run-crons] No jobs to run in this window'); + } else { + await Promise.allSettled(jobs); + } + + console.log('[run-crons] Done'); +} + +main() + .then(() => process.exit(0)) + .catch((err) => { + console.error('[run-crons] Fatal error:', err); + process.exit(1); + }); diff --git a/apps/api-server/src/cron-calendar.js b/apps/api-server/src/cron-calendar.js index dfda8fa711..3a72f873bb 100755 --- a/apps/api-server/src/cron-calendar.js +++ b/apps/api-server/src/cron-calendar.js @@ -7,6 +7,11 @@ module.exports = { jobs: new Map(), start: function () { + if (process.env.USE_KUBERNETES_CRONJOBS === 'true') { + console.log('Cron jobs disabled: USE_KUBERNETES_CRONJOBS is enabled'); + return this; + } + var jobs = this.jobs; util.invokeDir('./cron', function (jobDef, fileName) { try { diff --git a/apps/auth-server/app.js b/apps/auth-server/app.js index c458f83bee..4930a2372d 100644 --- a/apps/auth-server/app.js +++ b/apps/auth-server/app.js @@ -1,13 +1,13 @@ const initializeApp = require('./app-init'); const config = require('./config'); -const memoryStorage = require('./memoryStorage'); +const databaseStorage = require('./databaseStorage'); /** * From time to time we need to clean up any expired tokens * in the database */ setInterval(() => { - memoryStorage.accessTokens + databaseStorage.accessTokens .removeExpired() .catch((err) => console.error('Error trying to remove expired tokens:', err.stack) diff --git a/apps/auth-server/auth.js b/apps/auth-server/auth.js index 8227a4a8e0..1fb7fce335 100644 --- a/apps/auth-server/auth.js +++ b/apps/auth-server/auth.js @@ -1,6 +1,6 @@ 'use strict'; -const memoryStorage = require('./memoryStorage'); +const databaseStorage = require('./databaseStorage'); const passport = require('passport'); const { Strategy: LocalStrategy } = require('passport-local'); const { BasicStrategy } = require('passport-http'); @@ -237,7 +237,7 @@ passport.use( */ passport.use( new BearerStrategy((accessToken, done) => { - memoryStorage.accessTokens + databaseStorage.accessTokens .find(accessToken) .then((token) => validate.token(token, accessToken)) .then((token) => { diff --git a/apps/auth-server/controllers/admin/api/uniqueCode.js b/apps/auth-server/controllers/admin/api/uniqueCode.js index 5ea382467c..fa4514c683 100644 --- a/apps/auth-server/controllers/admin/api/uniqueCode.js +++ b/apps/auth-server/controllers/admin/api/uniqueCode.js @@ -1,4 +1,4 @@ -const Tasks = require('../../../memoryStorage/tasks'); +const Tasks = require('../../../databaseStorage/tasks'); const outputUniqueCode = (req, res, next) => { res.json(req.code); diff --git a/apps/auth-server/controllers/oauth/oauth2.js b/apps/auth-server/controllers/oauth/oauth2.js index 48df0e1872..1185a78ab3 100644 --- a/apps/auth-server/controllers/oauth/oauth2.js +++ b/apps/auth-server/controllers/oauth/oauth2.js @@ -5,7 +5,7 @@ const passport = require('passport'); const URL = require('url').URL; const db = require('../../db'); const config = require('../../config'); -const memoryStorage = require('../../memoryStorage'); +const databaseStorage = require('../../databaseStorage'); const utils = require('../../utils'); const validate = require('../../validate'); @@ -83,7 +83,7 @@ server.grant( sub: user.id, exp: config.codeToken.expiresIn, }); - memoryStorage.authorizationCodes + databaseStorage.authorizationCodes .save(code, client.id, redirectURI, user.id, client.scope) .then(() => done(null, code)) .catch((err) => done(err)); @@ -106,7 +106,7 @@ server.grant( }); const expiration = config.token.calculateExpirationDate(); - memoryStorage.accessTokens + databaseStorage.accessTokens .save(token, expiration, user.id, client.id, client.scope) .then(() => done(null, token, expiresIn)) .catch((err) => done(err)); @@ -123,7 +123,7 @@ server.grant( */ server.exchange( oauth2orize.exchange.code((client, code, redirectURI, done) => { - memoryStorage.authorizationCodes + databaseStorage.authorizationCodes .delete(code) .then((authCode) => validate.authCode(code, authCode, client, redirectURI) @@ -191,7 +191,7 @@ server.exchange( const expiration = config.token.calculateExpirationDate(); // Pass in a null for user id since there is no user when using this grant type - memoryStorage.accessTokens + databaseStorage.accessTokens .save(token, expiration, null, client.id, scope) .then(() => done(null, token, null, expiresIn)) .catch((err) => done(err)); @@ -207,7 +207,7 @@ server.exchange( */ server.exchange( oauth2orize.exchange.refreshToken((client, refreshToken, scope, done) => { - memoryStorage.refreshTokens + databaseStorage.refreshTokens .find(refreshToken) .then((foundRefreshToken) => validate.refreshToken(foundRefreshToken, refreshToken, client) diff --git a/apps/auth-server/controllers/oauth/token.js b/apps/auth-server/controllers/oauth/token.js index 894ca9a7ed..56a13eb954 100644 --- a/apps/auth-server/controllers/oauth/token.js +++ b/apps/auth-server/controllers/oauth/token.js @@ -1,6 +1,7 @@ 'use strict'; -const memoryStorage = require('../../memoryStorage'); +const databaseStorage = require('../../databaseStorage'); +const db = require('../../db'); const validate = require('../../validate'); /** @@ -27,11 +28,10 @@ const validate = require('../../validate'); exports.info = (req, res) => validate .tokenForHttp(req.query.access_token) - .then(() => memoryStorage.accessTokens.find(req.query.access_token)) + .then(() => databaseStorage.accessTokens.find(req.query.access_token)) .then((token) => validate.tokenExistsForHttp(token)) .then((token) => - memoryStorage.clients - .find(token.clientID) + db.Client.findOne({ where: { id: token.clientID } }) .then((client) => validate.clientExistsForHttp(client)) .then((client) => ({ client, token })) ) @@ -70,10 +70,10 @@ exports.info = (req, res) => exports.revoke = (req, res) => validate .tokenForHttp(req.query.token) - .then(() => memoryStorage.accessTokens.delete(req.query.token)) + .then(() => databaseStorage.accessTokens.delete(req.query.token)) .then((token) => { if (token == null) { - return memoryStorage.refreshTokens.delete(req.query.token); + return databaseStorage.refreshTokens.delete(req.query.token); } return token; }) diff --git a/apps/auth-server/databaseStorage/accesstokens.js b/apps/auth-server/databaseStorage/accesstokens.js new file mode 100644 index 0000000000..718d4cbb31 --- /dev/null +++ b/apps/auth-server/databaseStorage/accesstokens.js @@ -0,0 +1,110 @@ +'use strict'; + +const db = require('../db'); +const utils = require('../utils'); + +/** + * Returns an access token if it finds one, otherwise returns undefined. + * @param {String} token - The token to decode to get the id of the access token to find. + * @returns {Promise} resolved with the token if found, otherwise resolved with undefined + */ +exports.find = (token) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + return Promise.resolve(undefined); + } + + const id = decoded.jti; + + return db.AccessToken.findOne({ where: { tokenId: id } }) + .then((token) => token || undefined) + .catch((e) => { + console.warn('Error finding accesstoken: ', e); + return undefined; + }); +}; + +/** + * Saves an access token, expiration date, user id, client id, and scope. + * @param {Object} token - The access token (required) + * @param {Date} expirationDate - The expiration of the access token (required) + * @param {String} userID - The user ID (required) + * @param {String} clientID - The client ID (required) + * @param {String} scope - The scope (optional) + * @returns {Promise} resolved with the saved token + */ +exports.save = (token, expirationDate, userID, clientID, scope) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + return Promise.resolve(undefined); + } + + const id = decoded.jti; + + return db.AccessToken.create({ + tokenId: id, + userID, + expirationDate, + clientID, + scope, + }) + .then((token) => token || undefined) + .catch((e) => { + console.warn('Error creating accesstoken: ', e); + return undefined; + }); +}; + +/** + * Deletes/Revokes an access token by getting the ID and removing it from the storage. + * @param {String} token - The token to decode to get the id of the access token to delete. + * @returns {Promise} resolved with the deleted token + */ +exports.delete = (token) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + return Promise.resolve(undefined); + } + + const id = decoded.jti; + + return db.AccessToken.findOne({ where: { tokenId: id } }) + .then((token) => { + if (!token) return undefined; + return token.destroy().then(() => token); + }) + .catch((e) => { + console.warn('Error deleting accesstoken: ', e); + return undefined; + }); +}; + +/** + * Removes expired access tokens. + * @returns {Promise} resolved when expired tokens are removed + */ +exports.removeExpired = () => { + return db.AccessToken.destroy({ + where: { + expirationDate: { + [db.Sequelize.Op.lt]: new Date(), + }, + }, + }).catch((e) => { + console.warn('Error removing expired access tokens: ', e); + }); +}; + +/** + * Removes all access tokens. + * @returns {Promise} resolved when all tokens are removed + */ +exports.removeAll = () => { + return db.AccessToken.destroy({ where: {} }); +}; diff --git a/apps/auth-server/databaseStorage/authorizationcodes.js b/apps/auth-server/databaseStorage/authorizationcodes.js new file mode 100644 index 0000000000..bfc105ed7f --- /dev/null +++ b/apps/auth-server/databaseStorage/authorizationcodes.js @@ -0,0 +1,113 @@ +'use strict'; + +const db = require('../db'); +const utils = require('../utils'); + +/** + * Returns an authorization code if it finds one, otherwise returns undefined. + * @param {String} token - The token to decode to get the id of the authorization token to find. + * @returns {Promise} resolved with the authorization code if found, otherwise undefined + */ +exports.find = (token) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + return Promise.resolve(undefined); + } + + const id = decoded.jti; + + return db.AuthorizationCode.findOne({ where: { tokenId: id } }) + .then((code) => { + if (!code) return undefined; + return { + clientID: code.clientID, + redirectURI: code.redirectURI, + userID: code.userID, + scope: code.scope, + }; + }) + .catch((e) => { + console.warn('Error finding authorization code: ', e); + return undefined; + }); +}; + +/** + * Saves an authorization code, client id, redirect uri, user id, and scope. + * @param {String} code - The authorization code (required) + * @param {String} clientID - The client ID (required) + * @param {String} redirectURI - The redirect URI of where to send access tokens once exchanged + * @param {String} userID - The user ID (required) + * @param {String} scope - The scope (optional) + * @returns {Promise} resolved with the saved token + */ +exports.save = (code, clientID, redirectURI, userID, scope) => { + let decoded; + try { + decoded = utils.verifyToken(code); + } catch (e) { + return Promise.reject(new Error('Invalid authorization code')); + } + + const id = decoded.jti; + + return db.AuthorizationCode.create({ + tokenId: id, + clientID, + redirectURI, + userID, + scope, + }) + .then((record) => ({ + clientID: record.clientID, + redirectURI: record.redirectURI, + userID: record.userID, + scope: record.scope, + })) + .catch((e) => { + console.warn('Error saving authorization code: ', e); + return undefined; + }); +}; + +/** + * Deletes an authorization code + * @param {String} token - The authorization code to delete + * @returns {Promise} resolved with the deleted value + */ +exports.delete = (token) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + return Promise.resolve(undefined); + } + + const id = decoded.jti; + + return db.AuthorizationCode.findOne({ where: { tokenId: id } }) + .then((code) => { + if (!code) return undefined; + const data = { + clientID: code.clientID, + redirectURI: code.redirectURI, + userID: code.userID, + scope: code.scope, + }; + return code.destroy().then(() => data); + }) + .catch((e) => { + console.warn('Error deleting authorization code: ', e); + return undefined; + }); +}; + +/** + * Removes all authorization codes. + * @returns {Promise} resolved when all codes are removed + */ +exports.removeAll = () => { + return db.AuthorizationCode.destroy({ where: {} }); +}; diff --git a/apps/auth-server/memoryStorage/index.js b/apps/auth-server/databaseStorage/index.js similarity index 67% rename from apps/auth-server/memoryStorage/index.js rename to apps/auth-server/databaseStorage/index.js index 0800469f3e..3f03d8893b 100644 --- a/apps/auth-server/memoryStorage/index.js +++ b/apps/auth-server/databaseStorage/index.js @@ -2,7 +2,5 @@ exports.accessTokens = require('./accesstokens'); exports.authorizationCodes = require('./authorizationcodes'); -//exports.clients = require('./clients'); exports.refreshTokens = require('./refreshtokens'); exports.tasks = require('./tasks'); -//exports.users = require('./users'); diff --git a/apps/auth-server/databaseStorage/refreshtokens.js b/apps/auth-server/databaseStorage/refreshtokens.js new file mode 100644 index 0000000000..107b669763 --- /dev/null +++ b/apps/auth-server/databaseStorage/refreshtokens.js @@ -0,0 +1,109 @@ +'use strict'; + +const db = require('../db'); +const utils = require('../utils'); + +/** + * Returns a refresh token if it finds one, otherwise returns undefined. + * @param {String} token - The token to decode to get the id of the refresh token to find. + * @returns {Promise} resolved with the token + */ +exports.find = (token) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + return Promise.resolve(undefined); + } + + const id = decoded.jti; + + return db.RefreshToken.findOne({ where: { tokenId: id } }) + .then((record) => { + if (!record) return undefined; + return { + userID: record.userID, + clientID: record.clientID, + scope: record.scope, + }; + }) + .catch((e) => { + console.warn('Error finding refresh token: ', e); + return undefined; + }); +}; + +/** + * Saves a refresh token, user id, client id, and scope. + * @param {Object} token - The refresh token (required) + * @param {String} userID - The user ID (required) + * @param {String} clientID - The client ID (required) + * @param {String} scope - The scope (optional) + * @returns {Promise} resolved with the saved token + */ +exports.save = (token, userID, clientID, scope) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + console.warn('Error verifying JWT: ', e); + return Promise.resolve(new Error('Invalid refresh token')); + } + + const id = decoded.jti; + + return db.RefreshToken.create({ + tokenId: id, + userID, + clientID, + scope, + }) + .then((record) => ({ + userID: record.userID, + clientID: record.clientID, + scope: record.scope, + })) + .catch((e) => { + console.warn('Error saving refresh token: ', e); + return undefined; + }); +}; + +/** + * Deletes a refresh token + * @param {String} token - The token to decode to get the id of the refresh token to delete. + * @returns {Promise} resolved with the deleted token + */ +exports.delete = (token) => { + let decoded; + try { + decoded = utils.verifyToken(token); + } catch (e) { + return Promise.resolve(undefined); + } + + const id = decoded.jti; + + return db.RefreshToken.findOne({ where: { tokenId: id } }) + .then((record) => { + if (!record) return undefined; + const data = { + userID: record.userID, + clientID: record.clientID, + scope: record.scope, + }; + return record.destroy().then(() => data); + }) + .catch((e) => { + console.warn('Error deleting refresh token: ', e); + return undefined; + }); +}; + +/** + * Removes all refresh tokens. + * @returns {Promise} resolved when all tokens are removed + */ +exports.removeAll = () => { + return db.RefreshToken.destroy({ where: {} }); +}; diff --git a/apps/auth-server/databaseStorage/tasks.js b/apps/auth-server/databaseStorage/tasks.js new file mode 100644 index 0000000000..2b4b32c0ad --- /dev/null +++ b/apps/auth-server/databaseStorage/tasks.js @@ -0,0 +1,120 @@ +'use strict'; + +const merge = require('merge'); +const db = require('../db'); + +let Tasks = {}; + +const MAX_TASK_AGE = 60 * 60 * 1000; // milliseconds + +/** + * Returns a task if it finds one, otherwise returns undefined. + * @param {String} taskId - The task to find. + * @returns {Promise} resolved with the task + */ +Tasks.find = async (taskId) => { + await Tasks.cleanUp(); + + return db.Task.findOne({ where: { taskId } }) + .then((record) => { + if (!record) return undefined; + return { + ...record.data, + taskId: record.taskId, + lastUpdateDate: record.lastUpdateDate, + }; + }) + .catch((e) => { + console.warn('Error finding task: ', e); + return undefined; + }); +}; + +/** + * Update or create a task + * @param {String} taskId - The taskId (required) + * @param {Object} newData - The data to be stored + * @returns {Promise} resolved with the task + */ +Tasks.save = async (taskId, newData) => { + taskId = taskId || String(parseInt(Math.random() * 10000000) + 10000000); + + const existing = await db.Task.findOne({ where: { taskId } }).catch( + () => null + ); + + if (existing) { + const oldData = existing.data || {}; + const merged = merge.recursive(oldData, newData, { + taskId, + lastUpdateDate: Date.now(), + }); + await existing.update({ data: merged, lastUpdateDate: Date.now() }); + return { ...merged, taskId, lastUpdateDate: Date.now() }; + } + + const data = merge.recursive({}, newData, { + taskId, + lastUpdateDate: Date.now(), + }); + + const record = await db.Task.create({ + taskId, + data, + lastUpdateDate: Date.now(), + }); + + return { + ...record.data, + taskId: record.taskId, + lastUpdateDate: record.lastUpdateDate, + }; +}; + +/** + * Deletes a task + * @param {String} taskId - The id of the task to delete. + * @returns {Promise} resolved with the deleted task + */ +Tasks.delete = async (taskId) => { + const record = await db.Task.findOne({ where: { taskId } }).catch(() => null); + if (!record) return undefined; + + const data = { + ...record.data, + taskId: record.taskId, + lastUpdateDate: record.lastUpdateDate, + }; + await record.destroy(); + return data; +}; + +/** + * Removes all tasks. + * @returns {Promise} resolved when all tasks are removed + */ +Tasks.removeAll = () => { + return db.Task.destroy({ where: {} }); +}; + +/** + * Removes old tasks. + * @returns {Promise} resolved with undefined + */ +Tasks.cleanUp = async () => { + const cutoff = Date.now() - MAX_TASK_AGE; + + await db.Task.destroy({ + where: { + lastUpdateDate: { + [db.Sequelize.Op.lt]: cutoff, + }, + }, + }).catch((e) => { + console.warn('Error cleaning up tasks: ', e); + }); + + return undefined; +}; + +module.exports = exports = Tasks; diff --git a/apps/auth-server/db.js b/apps/auth-server/db.js index 14a5ea7793..fc371b366d 100755 --- a/apps/auth-server/db.js +++ b/apps/auth-server/db.js @@ -77,6 +77,13 @@ db.UniqueCode = require('./model/unique~code')(db, sequelize, Sequelize); db.User = require('./model/user')(db, sequelize, Sequelize); db.UserRole = require('./model/user-role')(db, sequelize, Sequelize); db.AccessCode = require('./model/access-code')(db, sequelize, Sequelize); +db.AuthorizationCode = require('./model/authorization-code')( + db, + sequelize, + Sequelize +); +db.RefreshToken = require('./model/refresh-token')(db, sequelize, Sequelize); +db.Task = require('./model/task')(db, sequelize, Sequelize); // invoke associations and scopes for (let modelName in sequelize.models) { diff --git a/apps/auth-server/memoryStorage/accesstokens.js b/apps/auth-server/memoryStorage/accesstokens.js deleted file mode 100644 index c4ef191538..0000000000 --- a/apps/auth-server/memoryStorage/accesstokens.js +++ /dev/null @@ -1,182 +0,0 @@ -'use strict'; - -const jwt = require('jsonwebtoken'); -const db = require('../db'); -const utils = require('../utils'); - -// The access tokens. -// You will use these to access your end point data through the means outlined -// in the RFC The OAuth 2.0 Authorization Framework: Bearer Token Usage -// (http://tools.ietf.org/html/rfc6750) - -/** - * Tokens in-memory data structure which stores all of the access tokens - */ -let tokens = Object.create(null); - -/** - * Returns an access token if it finds one, otherwise returns null if one is not found. - * @param {String} token - The token to decode to get the id of the access token to find. - * @returns {Promise} resolved with the token if found, otherwise resolved with undefined - */ -exports.find = (token) => { - const findAction = new Promise((resolve, reject) => { - let decoded; - try { - decoded = utils.verifyToken(token); - } catch (e) { - console.warn('Error verifying JWT: ', e); - return resolve(undefined); - } - - const id = decoded.jti; - - db.AccessToken.findOne({ where: { tokenId: id } }) - .then((token) => { - if (!token) { - resolve(undefined); - } - return resolve(token); - }) - .catch((e) => { - console.warn('Error finding accesstoken: ', e); - return resolve(undefined); - }); - }); - - return findAction; -}; - -/** - * Saves a access token, expiration date, user id, client id, and scope. Note: The actual full - * access token is never saved. Instead just the ID of the token is saved. In case of a database - * breach this prevents anyone from stealing the live tokens. - * @param {Object} token - The access token (required) - * @param {Date} expirationDate - The expiration of the access token (required) - * @param {String} userID - The user ID (required) - * @param {String} clientID - The client ID (required) - * @param {String} scope - The scope (optional) - * @returns {Promise} resolved with the saved token - */ -exports.save = (token, expirationDate, userID, clientID, scope) => { - const saveAction = new Promise((resolve, reject) => { - let decoded; - try { - decoded = utils.verifyToken(token); - } catch (e) { - console.warn('Error verifying JWT: ', e); - return resolve(undefined); - } - - const id = decoded.jti; - - db.AccessToken.create({ - tokenId: id, - userID, - expirationDate, - clientID, - scope, - }) - .then((token) => { - console.log('Savedddd access token'); - if (!token) { - resolve(undefined); - } - return resolve(token); - }) - .catch((e) => { - console.warn('Error creating accesstoken: ', e); - return resolve(undefined); - }); - }); - - return saveAction; -}; - -/** - * Deletes/Revokes an access token by getting the ID and removing it from the storage. - * @param {String} token - The token to decode to get the id of the access token to delete. - * @returns {Promise} resolved with the deleted token - */ -exports.delete = (token) => { - const deleteAction = new Promise((resolve, reject) => { - let decoded; - try { - decoded = utils.verifyToken(token); - } catch (e) { - console.warn('Error verifying JWT: ', e); - return resolve(undefined); - } - - const id = decoded.jti; - - db.AccessToken.findOne({ where: { tokenId: id } }) - .then((token) => { - return token - .destroy() - .then(() => { - return token ? resolve(token) : resolve(undefined); - }) - .catch(() => { - console.warn('Error delete accesstoken: ', e); - return resolve(undefined); - }); - }) - .catch((e) => { - console.warn('Error delete accesstoken: ', e); - return resolve(undefined); - }); - }); - - return deleteAction; -}; - -/** - * Removes expired access tokens. It does this by looping through them all and then removing the - * expired ones it finds. - * @returns {Promise} resolved with an associative of tokens that were expired - */ -exports.removeExpired = () => { - const removeExpiredAction = new Promise((resolve, reject) => { - db.AccessToken.findAll() - .then(async (tokens) => { - const deleteActions = []; - - tokens.forEach((accessToken) => { - const expirationDate = accessToken.get('expirationDate'); - - if (new Date() > expirationDate) { - deleteActions.push(accessToken.destroy()); - } - }); - - Promise.all(deleteActions) - .then((success) => { - console.log('success', success); - resolve(); - }) - .catch((e) => { - resolve(); - console.log('e', e); - }); - }) - .catch((e) => { - console.warn('Error delete accesstoken: ', e); - return Promise.resolve(undefined); - }); - - resolve(undefined); - }); - - return removeExpiredAction; -}; - -/** - * Removes all access tokens. - * @returns {Promise} resolved with all removed tokens returned - */ -exports.removeAll = () => { - //const deletedTokens = tokens; - //tokens = Object.create(null); - // return Promise.resolve(deletedTokens); -}; diff --git a/apps/auth-server/memoryStorage/authorizationcodes.js b/apps/auth-server/memoryStorage/authorizationcodes.js deleted file mode 100644 index 8c262d7bcf..0000000000 --- a/apps/auth-server/memoryStorage/authorizationcodes.js +++ /dev/null @@ -1,79 +0,0 @@ -'use strict'; - -const jwt = require('jsonwebtoken'); -const utils = require('../utils'); - -// The authorization codes. -// You will use these to get the access codes to get to the data in your endpoints as outlined -// in the RFC The OAuth 2.0 Authorization Framework: Bearer Token Usage -// (http://tools.ietf.org/html/rfc6750) - -/** - * Authorization codes in-memory data structure which stores all of the authorization codes - */ -let codes = Object.create(null); - -/** - * Returns an authorization code if it finds one, otherwise returns null if one is not found. - * @param {String} token - The token to decode to get the id of the authorization token to find. - * @returns {Promise} resolved with the authorization code if found, otherwise undefined - */ -exports.find = (token) => { - try { - const decoded = utils.verifyToken(token); - const id = decoded.jti; - return Promise.resolve(codes[id]); - } catch (error) { - return Promise.resolve(undefined); - } -}; - -/** - * Saves a authorization code, client id, redirect uri, user id, and scope. Note: The actual full - * authorization token is never saved. Instead just the ID of the token is saved. In case of a - * database breach this prevents anyone from stealing the live tokens. - * @param {String} code - The authorization code (required) - * @param {String} clientID - The client ID (required) - * @param {String} redirectURI - The redirect URI of where to send access tokens once exchanged - * @param {String} userID - The user ID (required) - * @param {String} scope - The scope (optional) - * @returns {Promise} resolved with the saved token - */ -exports.save = (code, clientID, redirectURI, userID, scope) => { - let decoded; - try { - decoded = utils.verifyToken(code); - } catch (error) { - return Promise.reject(new Error('Invalid authorization code')); - } - const id = decoded.jti; - codes[id] = { clientID, redirectURI, userID, scope }; - return Promise.resolve(codes[id]); -}; - -/** - * Deletes an authorization code - * @param {String} token - The authorization code to delete - * @returns {Promise} resolved with the deleted value - */ -exports.delete = (token) => { - try { - const decoded = utils.verifyToken(token); - const id = decoded.jti; - const deletedToken = codes[id]; - delete codes[id]; - return Promise.resolve(deletedToken); - } catch (error) { - return Promise.resolve(undefined); - } -}; - -/** - * Removes all authorization codes. - * @returns {Promise} resolved with all removed authorization codes returned - */ -exports.removeAll = () => { - const deletedTokens = codes; - codes = Object.create(null); - return Promise.resolve(deletedTokens); -}; diff --git a/apps/auth-server/memoryStorage/refreshtokens.js b/apps/auth-server/memoryStorage/refreshtokens.js deleted file mode 100644 index 58ec60ead4..0000000000 --- a/apps/auth-server/memoryStorage/refreshtokens.js +++ /dev/null @@ -1,80 +0,0 @@ -'use strict'; - -const jwt = require('jsonwebtoken'); -const path = require('node:path'); -const utils = require('../utils'); - -// The refresh tokens. -// You will use these to get access tokens to access your end point data through the means outlined -// in the RFC The OAuth 2.0 Authorization Framework: Bearer Token Usage -// (http://tools.ietf.org/html/rfc6750) - -/** - * Tokens in-memory data structure which stores all of the refresh tokens - */ -let tokens = Object.create(null); - -/** - * Returns a refresh token if it finds one, otherwise returns null if one is not found. - * @param {String} token - The token to decode to get the id of the refresh token to find. - * @returns {Promise} resolved with the token - */ -exports.find = (token) => { - try { - const decoded = utils.verifyToken(token); - const id = decoded.jti; - return Promise.resolve(tokens[id]); - } catch (error) { - return Promise.resolve(undefined); - } -}; - -/** - * Saves a refresh token, user id, client id, and scope. Note: The actual full refresh token is - * never saved. Instead just the ID of the token is saved. In case of a database breach this - * prevents anyone from stealing the live tokens. - * @param {Object} token - The refresh token (required) - * @param {String} userID - The user ID (required) - * @param {String} clientID - The client ID (required) - * @param {String} scope - The scope (optional) - * @returns {Promise} resolved with the saved token - */ -exports.save = (token, userID, clientID, scope) => { - let decoded; - try { - decoded = utils.verifyToken(token); - } catch (e) { - console.warn('Error verifying JWT: ', e); - return Promise.resolve(new Error('Invalid refresh token')); - } - const id = decoded.jti; - tokens[id] = { userID, clientID, scope }; - return Promise.resolve(tokens[id]); -}; - -/** - * Deletes a refresh token - * @param {String} token - The token to decode to get the id of the refresh token to delete. - * @returns {Promise} resolved with the deleted token - */ -exports.delete = (token) => { - try { - const decoded = utils.verifyToken(token); - const id = decoded.jti; - const deletedToken = tokens[id]; - delete tokens[id]; - return Promise.resolve(deletedToken); - } catch (error) { - return Promise.resolve(undefined); - } -}; - -/** - * Removes all refresh tokens. - * @returns {Promise} resolved with all removed tokens returned - */ -exports.removeAll = () => { - const deletedTokens = tokens; - tokens = Object.create(null); - return Promise.resolve(deletedTokens); -}; diff --git a/apps/auth-server/memoryStorage/tasks.js b/apps/auth-server/memoryStorage/tasks.js deleted file mode 100755 index 2c79c1b29f..0000000000 --- a/apps/auth-server/memoryStorage/tasks.js +++ /dev/null @@ -1,90 +0,0 @@ -'use strict'; - -const jwt = require('jsonwebtoken'); -const merge = require('merge'); - -let Tasks = {}; - -/** - * Task data is stored in memory only - * Purpose is only to be able to send information about background tasks. - * Currently the only available task is the creation of unique tokens - * - * The structure of this file is similar to tat of other files here; I think it should work as an object instead of this but then maybe so should the rest - */ - -/** - * Tasks in-memory data structure - */ -let tasks = Object.create(null); - -/** - * Returns a task if it finds one, otherwise returns null if one is not found. - * @param {String} taskId - The task to find. - * @returns {Promise} resolved with the task - */ -Tasks.find = (taskId) => { - try { - Tasks.cleanUp(); - return Promise.resolve(tasks[taskId]); - } catch (error) { - return Promise.resolve(undefined); - } -}; - -/** - * Update or create a task - * @param {Object} taskId - The taskId (required) - * @param {String} data - The data to be stored - */ -Tasks.save = (taskId, newData) => { - taskId = taskId || parseInt(Math.random() * 10000000) + 10000000; - let oldData = tasks[taskId] || {}; - tasks[taskId] = merge.recursive(oldData, newData, { - taskId, - lastUpdateDate: Date.now(), - }); - return Promise.resolve(tasks[taskId]); -}; - -/** - * Deletes a task - * @param {String} taskId - The id of the task to delete. - * @returns {Promise} resolved with the deleted task - */ -Tasks.delete = (taskId) => { - try { - const deletedTask = tasks[taskId]; - delete tasks[taskId]; - return Promise.resolve(deletedTask); - } catch (error) { - return Promise.resolve(undefined); - } -}; - -/** - * Removes all tasks. - * @returns {Promise} resolved with all removed tasks returned - */ -Tasks.removeAll = () => { - const deletedTasks = tasks; - tasks = Object.create(null); - return Promise.resolve(deletedTasks); -}; - -/** - * Removes old tasks. - * @returns {Promise} resolved with undef - */ -Tasks.cleanUp = async () => { - let maxTaskAge = 60 * 60 * 1000; // milliseconds // TODO: configurable - for (let taskId of Object.keys(tasks)) { - let task = tasks[taskId]; - if (Date.now() - task.lastUpdateDate > maxTaskAge) { - await Tasks.delete(taskId); - } - } - return Promise.resolve(undefined); -}; - -module.exports = exports = Tasks; diff --git a/apps/auth-server/middleware/access-code.js b/apps/auth-server/middleware/access-code.js index 83afd73f91..27a428a79d 100644 --- a/apps/auth-server/middleware/access-code.js +++ b/apps/auth-server/middleware/access-code.js @@ -1,6 +1,6 @@ const db = require('../db'); const generateCode = require('../utils/generateCode'); -const Tasks = require('../memoryStorage/tasks'); +const Tasks = require('../databaseStorage/tasks'); exports.withAll = (req, res, next) => { const limit = req.query.limit ? parseInt(req.query.limit, 10) : 1000; diff --git a/apps/auth-server/middleware/code.js b/apps/auth-server/middleware/code.js index a6180a0382..8f75a0441f 100644 --- a/apps/auth-server/middleware/code.js +++ b/apps/auth-server/middleware/code.js @@ -1,6 +1,6 @@ const db = require('../db'); const generateCode = require('../utils/generateCode'); -const Tasks = require('../memoryStorage/tasks'); +const Tasks = require('../databaseStorage/tasks'); exports.withAll = (req, res, next) => { const isExport = req.query.export === 'true'; diff --git a/apps/auth-server/migrations/009-add-oauth-storage-tables.js b/apps/auth-server/migrations/009-add-oauth-storage-tables.js new file mode 100644 index 0000000000..22d0280f09 --- /dev/null +++ b/apps/auth-server/migrations/009-add-oauth-storage-tables.js @@ -0,0 +1,113 @@ +const { Sequelize } = require('sequelize'); + +module.exports = { + async up({ context: queryInterface }) { + await queryInterface.createTable('authorization_codes', { + id: { + allowNull: false, + autoIncrement: true, + primaryKey: true, + type: Sequelize.INTEGER, + }, + tokenId: { + type: Sequelize.STRING, + allowNull: false, + unique: true, + }, + clientID: { + type: Sequelize.INTEGER, + allowNull: false, + }, + redirectURI: { + type: Sequelize.STRING(2048), + allowNull: true, + }, + userID: { + type: Sequelize.INTEGER, + allowNull: false, + }, + scope: { + type: Sequelize.STRING, + allowNull: true, + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE, + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE, + }, + }); + + await queryInterface.createTable('refresh_tokens', { + id: { + allowNull: false, + autoIncrement: true, + primaryKey: true, + type: Sequelize.INTEGER, + }, + tokenId: { + type: Sequelize.STRING, + allowNull: false, + unique: true, + }, + userID: { + type: Sequelize.INTEGER, + allowNull: false, + }, + clientID: { + type: Sequelize.INTEGER, + allowNull: false, + }, + scope: { + type: Sequelize.STRING, + allowNull: true, + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE, + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE, + }, + }); + + await queryInterface.createTable('tasks', { + id: { + allowNull: false, + autoIncrement: true, + primaryKey: true, + type: Sequelize.INTEGER, + }, + taskId: { + type: Sequelize.STRING, + allowNull: false, + unique: true, + }, + data: { + type: Sequelize.JSON, + allowNull: true, + }, + lastUpdateDate: { + type: Sequelize.BIGINT, + allowNull: false, + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE, + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE, + }, + }); + }, + + async down({ context: queryInterface }) { + await queryInterface.dropTable('authorization_codes'); + await queryInterface.dropTable('refresh_tokens'); + await queryInterface.dropTable('tasks'); + }, +}; diff --git a/apps/auth-server/model/authorization-code.js b/apps/auth-server/model/authorization-code.js new file mode 100644 index 0000000000..34691e40e0 --- /dev/null +++ b/apps/auth-server/model/authorization-code.js @@ -0,0 +1,41 @@ +'use strict'; + +const { DataTypes } = require('sequelize'); + +module.exports = (db, sequelize, Sequelize) => { + let AuthorizationCode = sequelize.define( + 'authorization_code', + { + tokenId: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + }, + + clientID: { + type: DataTypes.INTEGER, + allowNull: false, + }, + + redirectURI: { + type: DataTypes.STRING(2048), + allowNull: true, + }, + + userID: { + type: DataTypes.INTEGER, + allowNull: false, + }, + + scope: { + type: DataTypes.STRING, + allowNull: true, + }, + }, + { + tableName: 'authorization_codes', + } + ); + + return AuthorizationCode; +}; diff --git a/apps/auth-server/model/refresh-token.js b/apps/auth-server/model/refresh-token.js new file mode 100644 index 0000000000..bbeacbea3e --- /dev/null +++ b/apps/auth-server/model/refresh-token.js @@ -0,0 +1,36 @@ +'use strict'; + +const { DataTypes } = require('sequelize'); + +module.exports = (db, sequelize, Sequelize) => { + let RefreshToken = sequelize.define( + 'refresh_token', + { + tokenId: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + }, + + userID: { + type: DataTypes.INTEGER, + allowNull: false, + }, + + clientID: { + type: DataTypes.INTEGER, + allowNull: false, + }, + + scope: { + type: DataTypes.STRING, + allowNull: true, + }, + }, + { + tableName: 'refresh_tokens', + } + ); + + return RefreshToken; +}; diff --git a/apps/auth-server/model/task.js b/apps/auth-server/model/task.js new file mode 100644 index 0000000000..3b0d050224 --- /dev/null +++ b/apps/auth-server/model/task.js @@ -0,0 +1,31 @@ +'use strict'; + +const { DataTypes } = require('sequelize'); + +module.exports = (db, sequelize, Sequelize) => { + let Task = sequelize.define( + 'task', + { + taskId: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + }, + + data: { + type: DataTypes.JSON, + allowNull: true, + }, + + lastUpdateDate: { + type: DataTypes.BIGINT, + allowNull: false, + }, + }, + { + tableName: 'tasks', + } + ); + + return Task; +}; diff --git a/apps/auth-server/validate.js b/apps/auth-server/validate.js index 9f9e9be7a6..938761aa89 100644 --- a/apps/auth-server/validate.js +++ b/apps/auth-server/validate.js @@ -2,7 +2,7 @@ const process = require('process'); const bcrypt = require('bcrypt'); const config = require('./config'); -const memoryStorage = require('./memoryStorage'); +const databaseStorage = require('./databaseStorage'); const utils = require('./utils'); const db = require('./db'); const saltRounds = 10; @@ -183,7 +183,7 @@ validate.generateRefreshToken = ({ userId, clientID, scope }) => { sub: userId, exp: config.refreshToken.expiresIn, }); - return memoryStorage.refreshTokens + return databaseStorage.refreshTokens .save(refreshToken, userId, clientID, scope) .then(() => refreshToken); }; @@ -198,7 +198,7 @@ validate.generateRefreshToken = ({ userId, clientID, scope }) => { validate.generateToken = ({ userID, clientID, scope }) => { const token = utils.createToken({ sub: userID, exp: config.token.expiresIn }); const expiration = config.token.calculateExpirationDate(); - return memoryStorage.accessTokens + return databaseStorage.accessTokens .save(token, expiration, userID, clientID, scope) .then(() => token); }; diff --git a/apps/cms-server/package.json b/apps/cms-server/package.json old mode 100755 new mode 100644 index 9196ba99cd..a856e759db --- a/apps/cms-server/package.json +++ b/apps/cms-server/package.json @@ -45,7 +45,7 @@ "@openstad-headless/lib": "*", "@apostrophecms/anchors": "~1.1.0", "@apostrophecms/blog": "~1.0.6", - "@apostrophecms/import-export": "~3.5.2", + "@apostrophecms/import-export": "~3.5.3", "@apostrophecms/redirect": "~1.5.0", "@apostrophecms/seo": "~1.4.0", "@apostrophecms/sitemap": "~1.2.0", diff --git a/charts/openstad-headless/templates/_helpers.tpl b/charts/openstad-headless/templates/_helpers.tpl index f498525ae3..b29df890f3 100644 --- a/charts/openstad-headless/templates/_helpers.tpl +++ b/charts/openstad-headless/templates/_helpers.tpl @@ -359,6 +359,105 @@ nginx When traefik, returns a dict with the traefik middleware annotation. When nginx, returns the user-provided annotations from values. */}} +{{/* + Pod anti-affinity block. Pass the app label as .appLabel. + Usage: {{ include "openstad.podAntiAffinity" (dict "Values" .Values "Release" .Release "appLabel" .Values.api.label) }} +*/}} +{{- define "openstad.podAntiAffinity" -}} +{{- if .Values.global.podAntiAffinity.enabled }} +affinity: + podAntiAffinity: + {{- if eq .Values.global.podAntiAffinity.mode "required" }} + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: {{ .appLabel }} + topologyKey: kubernetes.io/hostname + namespaceSelector: {} + namespaces: + - {{ .Release.Namespace }} + {{- else }} + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: {{ .appLabel }} + topologyKey: kubernetes.io/hostname + namespaceSelector: {} + namespaces: + - {{ .Release.Namespace }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* + Merge global autoscaling defaults with per-service overrides. + Usage: {{ include "openstad.autoscaling.mergedValues" (dict "Values" .Values "serviceAutoscaling" .Values.api.autoscaling) }} + Returns a YAML dict with the merged autoscaling config. +*/}} +{{- define "openstad.autoscaling.mergedValues" -}} +{{- $global := .Values.global.autoscaling | default dict -}} +{{- $service := .serviceAutoscaling | default dict -}} +{{- $merged := mustMergeOverwrite (deepCopy $global) $service -}} +{{- toYaml $merged -}} +{{- end -}} + +{{/* + HPA template. Pass a dict with Values, Release, fullname, and serviceAutoscaling. + Usage: {{ include "openstad.hpa" (dict "Values" .Values "Release" .Release "fullname" (include "openstad.api.fullname" .) "serviceAutoscaling" .Values.api.autoscaling) }} +*/}} +{{- define "openstad.hpa" -}} +{{- $merged := include "openstad.autoscaling.mergedValues" . | fromYaml -}} +{{- if $merged.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .fullname }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .fullname }} + minReplicas: {{ $merged.minReplicas | default 1 }} + maxReplicas: {{ $merged.maxReplicas | default 5 }} + metrics: + {{- if $merged.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ $merged.targetCPUUtilizationPercentage }} + {{- end }} + {{- if $merged.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ $merged.targetMemoryUtilizationPercentage }} + {{- end }} + {{- if $merged.behavior }} + behavior: + {{- toYaml $merged.behavior | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* + Replicas helper. Returns the replicas field only when autoscaling is NOT enabled, + so the HPA can manage replicas instead. + Usage: {{ include "openstad.replicas" (dict "Values" .Values "serviceAutoscaling" .Values.api.autoscaling "replicas" (.Values.api.replicas | default 1)) }} +*/}} +{{- define "openstad.replicas" -}} +{{- $merged := include "openstad.autoscaling.mergedValues" . | fromYaml -}} +{{- if not $merged.enabled }} +replicas: {{ .replicas }} +{{- end }} +{{- end -}} + {{- define "openstad.createdIngresses.annotations" -}} {{- if eq (include "openstad.ingress.type" .) "traefik" -}} {{- $user := .Values.api.createdIngresses.annotations | default dict -}} diff --git a/charts/openstad-headless/templates/admin/deployment.yaml b/charts/openstad-headless/templates/admin/deployment.yaml index f5ad812c55..07259f508c 100644 --- a/charts/openstad-headless/templates/admin/deployment.yaml +++ b/charts/openstad-headless/templates/admin/deployment.yaml @@ -8,7 +8,7 @@ metadata: {{- include "openstad.labels" . | nindent 4 }} app.kubernetes.io/component: {{ template "openstad.admin.fullname" . }}-deployment spec: - replicas: {{ .Values.admin.replicas }} + {{- include "openstad.replicas" (dict "Values" .Values "serviceAutoscaling" .Values.admin.autoscaling "replicas" (.Values.admin.replicas | default 1)) | nindent 2 }} selector: matchLabels: app: {{ .Values.admin.label }} @@ -22,6 +22,7 @@ spec: runAsUser: 1000 runAsGroup: 1000 serviceAccountName: openstad-headless-ingress-sa + {{- include "openstad.podAntiAffinity" (dict "Values" .Values "Release" .Release "appLabel" .Values.admin.label) | nindent 6 }} containers: - name: {{ template "openstad.admin.fullname" . }} securityContext: diff --git a/charts/openstad-headless/templates/admin/hpa.yaml b/charts/openstad-headless/templates/admin/hpa.yaml new file mode 100644 index 0000000000..55adb81b60 --- /dev/null +++ b/charts/openstad-headless/templates/admin/hpa.yaml @@ -0,0 +1 @@ +{{- include "openstad.hpa" (dict "Values" .Values "Release" .Release "fullname" (include "openstad.admin.fullname" .) "serviceAutoscaling" .Values.admin.autoscaling) -}} diff --git a/charts/openstad-headless/templates/api/cronjob.yaml b/charts/openstad-headless/templates/api/cronjob.yaml new file mode 100644 index 0000000000..b4bbf5a556 --- /dev/null +++ b/charts/openstad-headless/templates/api/cronjob.yaml @@ -0,0 +1,246 @@ +{{- if eq (toString .Values.api.useKubernetesCronjobs) "true" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ template "openstad.api.fullname" . }}-cron + namespace: {{ .Release.Namespace }} + labels: + {{- include "openstad.labels" . | nindent 4 }} + app.kubernetes.io/component: {{ template "openstad.api.fullname" . }}-cron +spec: + schedule: "*/{{ .Values.api.cronWindowMinutes }} * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: {{ mul (int .Values.api.cronWindowMinutes) 60 }} + template: + metadata: + labels: + app: {{ .Values.api.label }}-cron + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + serviceAccountName: openstad-headless-ingress-sa + restartPolicy: OnFailure + containers: + - name: {{ template "openstad.api.fullname" . }}-cron + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + image: {{ .Values.api.deploymentContainer.image }} + imagePullPolicy: IfNotPresent + command: ["node", "run-crons.js"] + volumeMounts: + - mountPath: /tmp + name: tmp-dir + resources: +{{ toYaml .Values.api.resources | indent 16 }} + env: + - name: CRON_WINDOW_MINUTES + value: "{{ .Values.api.cronWindowMinutes }}" + + - name: URL + value: https://{{ template "openstad.api.url" . }} + + - name: HOSTNAME + value: www.{{ .Values.host.base}} + + # MySQL DATABASE + - name: DB_USERNAME + valueFrom: + secretKeyRef: + name: {{ template "openstad.database.credentialsSecret.fullname" . }} + key: username + + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "openstad.passwordSecret.fullname" . }} + key: mysql-password + + - name: DB_HOST + valueFrom: + secretKeyRef: + name: {{ template "openstad.database.credentialsSecret.fullname" . }} + key: hostname + + - name: DB_PORT + valueFrom: + secretKeyRef: + name: {{ template "openstad.database.credentialsSecret.fullname" . }} + key: hostport + + - name: DB_MAX_POOL_SIZE + valueFrom: + secretKeyRef: + name: {{ template "openstad.database.credentialsSecret.fullname" . }} + key: maxPoolSize + + - name: DB_NAME + valueFrom: + secretKeyRef: + name: {{ template "openstad.api.secret.fullname" . }} + key: {{ .Values.secrets.api.dbNameKey | default "database" }} + + - name: USER_ID_SALT + valueFrom: + secretKeyRef: + key: userIdSalt + name: {{ template "openstad.api.secret.fullname" . }} + + - name: EMAILADDRESS + value: {{ .Values.api.emailAddress }} + + - name: PORT + value: "{{ .Values.api.service.httpPort }}" + + - name: MAIL_FROM + value: {{ .Values.secrets.mail.api.emailFrom | default "Openstad" }} + + - name: MAIL_TRANSPORTER_TYPE + valueFrom: + secretKeyRef: + key: transporterType + name: {{ template "openstad.email.secret.fullname" . }} + + - name: MAIL_MICROSOFT_GRAPH_CLIENT_ID + valueFrom: + secretKeyRef: + key: msClientId + name: {{ template "openstad.email.secret.fullname" . }} + + - name: MAIL_MICROSOFT_GRAPH_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: msClientSecret + name: {{ template "openstad.email.secret.fullname" . }} + + - name: MAIL_MICROSOFT_GRAPH_TENANT_ID + valueFrom: + secretKeyRef: + key: msTenantId + name: {{ template "openstad.email.secret.fullname" . }} + + - name: HASH_IP_ADDRESSES + valueFrom: + secretKeyRef: + key: hashIpAddresses + name: {{ template "openstad.ipaddress.secret.fullname" . }} + + - name: HASH_IP_SALT + valueFrom: + secretKeyRef: + key: hashIpSalt + name: {{ template "openstad.ipaddress.secret.fullname" . }} + + - name: MAIL_TRANSPORT_SMTP_PORT + valueFrom: + secretKeyRef: + key: {{ .Values.secrets.mail.portKey | default "port" }} + name: {{ template "openstad.email.secret.fullname" . }} + + - name: MAIL_TRANSPORT_SMTP_HOST + valueFrom: + secretKeyRef: + key: {{ .Values.secrets.mail.hostKey | default "host" }} + name: {{ template "openstad.email.secret.fullname" . }} + + - name: MAIL_TRANSPORT_SMTP_REQUIRESSL + value: "true" + + - name: MAIL_TRANSPORT_SMTP_AUTH_USER + valueFrom: + secretKeyRef: + key: {{ .Values.secrets.mail.usernameKey | default "username" }} + name: {{ template "openstad.email.secret.fullname" . }} + + - name: MAIL_TRANSPORT_SMTP_AUTH_PASS + valueFrom: + secretKeyRef: + key: {{ .Values.secrets.mail.passwordKey | default "password" }} + name: {{ template "openstad.email.secret.fullname" . }} + + - name: NOTIFICATIONS_ADMIN_EMAILADDRESS + value: {{ .Values.api.emailAddress }} + + - name: DISABLE_PROJECT_ISSUE_WARNINGS + value: "{{ .Values.api.disableProjectIssueWarnings }}" + + - name: AUTH_JWTSECRET + valueFrom: + secretKeyRef: + key: jwtSecret + name: {{ template "openstad.api.secret.fullname" . }} + + - name: AUTH_FIXEDAUTHTOKENS + valueFrom: + secretKeyRef: + key: fixedAuthTokens + name: {{ template "openstad.api.secret.fullname" . }} + + - name: AUTH_ADAPTER_OPENSTAD_SERVERURL + value: https://{{ template "openstad.auth.url" . }} + + - name: AUTH_ADAPTER_OPENSTAD_SERVERURL_INTERNAL + value: http://{{ template "openstad.auth.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.auth.service.httpPort }} + + - name: IMAGE_APP_URL + value: https://{{ template "openstad.image.url" . }} + + - name: IMAGE_APP_URL_INTERNAL + value: http://{{ printf "%s.%s.svc.cluster.local:%d" (include "openstad.image.fullname" .) .Release.Namespace (.Values.image.service.httpPort | int) }} + + - name: IMAGE_VERIFICATION_TOKEN + valueFrom: + secretKeyRef: + key: verificationToken + name: {{ template "openstad.image.secret.fullname" . }} + + - name: NODE_ENV + value: production + + - name: CMS_URL + value: https://{{ template "openstad.cms.url" . }} + + - name: BASE_DOMAIN + value: {{ .Values.host.base }} + + - name: ADMIN_DOMAIN + value: {{ template "openstad.admin.url" . }} + + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + + {{- if .Values.api.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.api.extraEnvVars "context" $) | nindent 14 }} + {{- end }} + volumes: + - name: tmp-dir + emptyDir: {} + initContainers: + - command: ["/bin/bash", "-c"] + args: + - nc $DB_HOST $DB_PORT -z -w1 + env: + - name: DB_HOST + valueFrom: + secretKeyRef: + name: {{ template "openstad.database.credentialsSecret.fullname" . }} + key: hostname + - name: DB_PORT + valueFrom: + secretKeyRef: + name: {{ template "openstad.database.credentialsSecret.fullname" . }} + key: hostport + image: {{ .Values.api.deploymentContainer.image }} + imagePullPolicy: IfNotPresent + name: wait-for-db +{{- end }} diff --git a/charts/openstad-headless/templates/api/deployment.yaml b/charts/openstad-headless/templates/api/deployment.yaml index a31f6e30f5..61433226a1 100644 --- a/charts/openstad-headless/templates/api/deployment.yaml +++ b/charts/openstad-headless/templates/api/deployment.yaml @@ -8,7 +8,7 @@ metadata: {{- include "openstad.labels" . | nindent 4 }} app.kubernetes.io/component: {{ template "openstad.api.fullname" . }}-deployment spec: - replicas: {{ .Values.api.replicas | default 1 }} + {{- include "openstad.replicas" (dict "Values" .Values "serviceAutoscaling" .Values.api.autoscaling "replicas" (.Values.api.replicas | default 1)) | nindent 2 }} revisionHistoryLimit: 10 selector: matchLabels: @@ -23,6 +23,7 @@ spec: runAsUser: 1000 runAsGroup: 1000 serviceAccountName: openstad-headless-ingress-sa + {{- include "openstad.podAntiAffinity" (dict "Values" .Values "Release" .Release "appLabel" .Values.api.label) | nindent 6 }} containers: - securityContext: allowPrivilegeEscalation: false @@ -155,6 +156,9 @@ spec: - name: DISABLE_PROJECT_ISSUE_WARNINGS value: "{{ .Values.api.disableProjectIssueWarnings }}" + - name: USE_KUBERNETES_CRONJOBS + value: "{{ .Values.api.useKubernetesCronjobs }}" + - name: AUTH_JWTSECRET valueFrom: secretKeyRef: diff --git a/charts/openstad-headless/templates/api/hpa.yaml b/charts/openstad-headless/templates/api/hpa.yaml new file mode 100644 index 0000000000..0c8e4fbf22 --- /dev/null +++ b/charts/openstad-headless/templates/api/hpa.yaml @@ -0,0 +1 @@ +{{- include "openstad.hpa" (dict "Values" .Values "Release" .Release "fullname" (include "openstad.api.fullname" .) "serviceAutoscaling" .Values.api.autoscaling) -}} diff --git a/charts/openstad-headless/templates/auth/deployment.yaml b/charts/openstad-headless/templates/auth/deployment.yaml index 4122796e40..0064c4303f 100644 --- a/charts/openstad-headless/templates/auth/deployment.yaml +++ b/charts/openstad-headless/templates/auth/deployment.yaml @@ -8,7 +8,7 @@ metadata: {{- include "openstad.labels" . | nindent 4 }} app.kubernetes.io/component: {{ template "openstad.auth.fullname" . }}-deployment spec: - replicas: {{ .Values.auth.replicas | default 1 }} + {{- include "openstad.replicas" (dict "Values" .Values "serviceAutoscaling" .Values.auth.autoscaling "replicas" (.Values.auth.replicas | default 1)) | nindent 2 }} selector: matchLabels: app: {{ .Values.auth.label }} @@ -21,6 +21,7 @@ spec: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 + {{- include "openstad.podAntiAffinity" (dict "Values" .Values "Release" .Release "appLabel" .Values.auth.label) | nindent 6 }} volumes: - name: certs secret: diff --git a/charts/openstad-headless/templates/auth/hpa.yaml b/charts/openstad-headless/templates/auth/hpa.yaml new file mode 100644 index 0000000000..6d217dab13 --- /dev/null +++ b/charts/openstad-headless/templates/auth/hpa.yaml @@ -0,0 +1 @@ +{{- include "openstad.hpa" (dict "Values" .Values "Release" .Release "fullname" (include "openstad.auth.fullname" .) "serviceAutoscaling" .Values.auth.autoscaling) -}} diff --git a/charts/openstad-headless/templates/cms/deployment.yaml b/charts/openstad-headless/templates/cms/deployment.yaml index 4a2315f467..dcdb084e7e 100644 --- a/charts/openstad-headless/templates/cms/deployment.yaml +++ b/charts/openstad-headless/templates/cms/deployment.yaml @@ -8,7 +8,7 @@ metadata: {{- include "openstad.labels" . | nindent 4 }} app.kubernetes.io/component: {{ template "openstad.cms.fullname" . }}-deployment spec: - replicas: {{ .Values.cms.replicas | default 1 }} + {{- include "openstad.replicas" (dict "Values" .Values "serviceAutoscaling" .Values.cms.autoscaling "replicas" (.Values.cms.replicas | default 1)) | nindent 2 }} selector: matchLabels: app: {{ .Values.cms.label }} @@ -21,6 +21,7 @@ spec: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 + {{- include "openstad.podAntiAffinity" (dict "Values" .Values "Release" .Release "appLabel" .Values.cms.label) | nindent 6 }} containers: - name: {{ template "openstad.cms.fullname" . }} image: {{ .Values.cms.deploymentContainer.image }} diff --git a/charts/openstad-headless/templates/cms/hpa.yaml b/charts/openstad-headless/templates/cms/hpa.yaml new file mode 100644 index 0000000000..ce208947ce --- /dev/null +++ b/charts/openstad-headless/templates/cms/hpa.yaml @@ -0,0 +1 @@ +{{- include "openstad.hpa" (dict "Values" .Values "Release" .Release "fullname" (include "openstad.cms.fullname" .) "serviceAutoscaling" .Values.cms.autoscaling) -}} diff --git a/charts/openstad-headless/templates/image/deployment.yaml b/charts/openstad-headless/templates/image/deployment.yaml index 8de2a8d0b5..a75d34281c 100644 --- a/charts/openstad-headless/templates/image/deployment.yaml +++ b/charts/openstad-headless/templates/image/deployment.yaml @@ -8,7 +8,7 @@ metadata: {{- include "openstad.labels" . | nindent 4 }} app.kubernetes.io/component: {{ template "openstad.image.fullname" . }}-deployment spec: - replicas: {{ .Values.image.replicas | default 1 }} + {{- include "openstad.replicas" (dict "Values" .Values "serviceAutoscaling" .Values.image.autoscaling "replicas" (.Values.image.replicas | default 1)) | nindent 2 }} selector: matchLabels: app: {{ .Values.image.label }} @@ -21,6 +21,7 @@ spec: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 + {{- include "openstad.podAntiAffinity" (dict "Values" .Values "Release" .Release "appLabel" .Values.image.label) | nindent 6 }} containers: - name: {{ template "openstad.image.fullname" . }} image: {{ .Values.image.deploymentContainer.image }} diff --git a/charts/openstad-headless/templates/image/hpa.yaml b/charts/openstad-headless/templates/image/hpa.yaml new file mode 100644 index 0000000000..50fc60a202 --- /dev/null +++ b/charts/openstad-headless/templates/image/hpa.yaml @@ -0,0 +1 @@ +{{- include "openstad.hpa" (dict "Values" .Values "Release" .Release "fullname" (include "openstad.image.fullname" .) "serviceAutoscaling" .Values.image.autoscaling) -}} diff --git a/charts/openstad-headless/values.yaml b/charts/openstad-headless/values.yaml index fd33d84609..6be0178b3f 100644 --- a/charts/openstad-headless/values.yaml +++ b/charts/openstad-headless/values.yaml @@ -2,6 +2,32 @@ global: ingress: type: nginx + # Pod anti-affinity to spread replicas of each deployment across nodes. + # mode: "preferred" (best-effort) or "required" (hard constraint, use with cluster autoscaler) + podAntiAffinity: + enabled: false + mode: preferred + # Horizontal Pod Autoscaler defaults for all services. + # Per-service overrides can be set under .autoscaling. + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + # targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + behavior: + scaleUp: + stabilizationWindowSeconds: 60 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 10 + periodSeconds: 60 # Optional Dependencies @@ -306,6 +332,12 @@ api: # Inject extra environment variables extraEnvVars: [] disableProjectIssueWarnings: "false" + # When running multiple replicas, offload cron jobs to a Kubernetes CronJob + # to prevent duplicate execution across pods. + useKubernetesCronjobs: false + # How often the K8s CronJob runs (in minutes). Each run picks up all + # application-level cron jobs that were scheduled within this window. + cronWindowMinutes: 15 # Resources: resources: # Max resources diff --git a/package-lock.json b/package-lock.json index 02ddfcde2a..97eaebadef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -640,7 +640,7 @@ "dependencies": { "@apostrophecms/anchors": "~1.1.0", "@apostrophecms/blog": "~1.0.6", - "@apostrophecms/import-export": "~3.5.2", + "@apostrophecms/import-export": "^3.5.3", "@apostrophecms/redirect": "~1.5.0", "@apostrophecms/seo": "~1.4.0", "@apostrophecms/sitemap": "~1.2.0", @@ -852,9 +852,9 @@ } }, "node_modules/@apostrophecms/import-export": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@apostrophecms/import-export/-/import-export-3.5.2.tgz", - "integrity": "sha512-yYBuLOIVkfObMuxU7qSiMmzmiyv67aPbEtltupvo6GKVPxxW122P/1haf6AOHMcpziZZ1P8ebVcuHosFP9dcJA==", + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/@apostrophecms/import-export/-/import-export-3.5.3.tgz", + "integrity": "sha512-CjmVHc8Yv8DQy2bzuFMg2+OEaP3vu0DMYBLlDR0cTCuuR8mO9DvYH2JI3ptePYofyDE/VUnP8m4nxQCB4oEiiw==", "license": "UNLICENSED", "dependencies": { "bluebird": "^3.7.2",