support per extension log level

Issue: BB-807

Author: SylvainSenechal

bin/ingestion.js

@@ -33,8 +33,9 @@ const { connectionString, autoCreateNamespace, retries } = zkConfig;
 const RESUME_NODE = 'scheduledResume';
 
 const log = new werelogs.Logger('Backbeat:IngestionPopulator');
-werelogs.configure({ level: config.log.logLevel,
-    dump: config.log.dumpLevel });
+const ingestionLogConfig = ingestionExtConfigs.log ?? config.log;
+werelogs.configure({ level: ingestionLogConfig.logLevel,
+    dump: ingestionLogConfig.dumpLevel });
 
 let scheduler;
 let ingestionPopulator;

extensions/ingestion/IngestionConfigValidator.js

@@ -1,5 +1,5 @@
 const joi = require('joi');
-const { probeServerJoi } = require('../../lib/config/configItems.joi');
+const { probeServerJoi, logJoiOptional } = require('../../lib/config/configItems.joi');
 
 const joiSchema = joi.object({
     auth: joi.object({
@@ -18,6 +18,7 @@ const joiSchema = joi.object({
         circuitBreaker: joi.object().optional(),
     }).optional(),
     producerParams: joi.object().unknown(true).default({}),
+    log: logJoiOptional,
 });
 
 function configValidator(backbeatConfig, extConfig) {

extensions/mongoProcessor/MongoProcessorConfigValidator.js

@@ -1,5 +1,5 @@
 const joi = require('joi');
-const { retryParamsJoi, probeServerJoi } = require('../../lib/config/configItems.joi');
+const { retryParamsJoi, probeServerJoi, logJoiOptional } = require('../../lib/config/configItems.joi');
 
 const { MAX_QUEUED_DEFAULT }  = require('../../lib/constants').backbeatConsumer;
 
@@ -11,6 +11,7 @@ const joiSchema = joi.object({
     maxQueued: joi.number().greater(0).default(MAX_QUEUED_DEFAULT),
     probeServer: probeServerJoi.default(),
     circuitBreaker: joi.object().optional(),
+    log: logJoiOptional,
 });
 
 function configValidator(backbeatConfig, extConfig) {

extensions/mongoProcessor/mongoProcessorTask.js

@@ -23,8 +23,9 @@ const mongoProcessorConfig = config.extensions.mongoProcessor;
 const mongoClientConfig = config.queuePopulator.mongo;
 
 const log = new werelogs.Logger('Backbeat:MongoProcessor:task');
-werelogs.configure({ level: config.log.logLevel,
-    dump: config.log.dumpLevel });
+const mongoProcessorLogConfig = mongoProcessorConfig.log ?? config.log;
+werelogs.configure({ level: mongoProcessorLogConfig.logLevel,
+    dump: mongoProcessorLogConfig.dumpLevel });
 
 const mqp = new MongoQueueProcessor(kafkaConfig, mongoProcessorConfig,
     mongoClientConfig, mConfig);

lib/config/configItems.joi.js

@@ -51,6 +51,16 @@ const logJoi =
               dumpLevel: 'error',
           });
 
+// logJoi with no default :
+// Callers fall back to the global log config when this one is not configured
+const logJoiOptional =
+          joi.object({
+              logLevel: joi.alternatives()
+                  .try('error', 'warn', 'info', 'debug', 'trace'),
+              dumpLevel: joi.alternatives()
+                  .try('error', 'warn', 'info', 'debug', 'trace'),
+          }).optional();
+
 const adminCredsJoi = joi.object()
           .min(1)
           .pattern(/^[A-Za-z0-9]{20}$/, joi.string());
@@ -168,6 +178,7 @@ module.exports = {
     transportJoi,
     bootstrapListJoi,
     logJoi,
+    logJoiOptional,
     adminCredsJoi,
     authJoi,
     inheritedAuthJoi,

tests/unit/ingestion/IngestionConfigValidator.spec.js

@@ -14,6 +14,21 @@ const baseExtConfig = {
 
 const qpBatchMaxRead = config.queuePopulator.batchMaxRead;
 
+describe('IngestionConfigValidator log override', () => {
+    it('should pass through log config when set', () => {
+        const validated = configValidator({}, {
+            ...baseExtConfig,
+            log: { logLevel: 'debug', dumpLevel: 'error' },
+        });
+        assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'error' });
+    });
+
+    it('should leave log undefined when not set, deferring to global config.log', () => {
+        const validated = configValidator({}, baseExtConfig);
+        assert.strictEqual(validated.log, undefined);
+    });
+});
+
 describe('IngestionConfigValidator batchMaxRead fallback', () => {
     it('should override queuePopulator.batchMaxRead when set in extension config', () => {
         const validated = configValidator({}, { ...baseExtConfig, batchMaxRead: 500 });

tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const assert = require('assert');
+const config = require('../../../lib/Config');
+const configValidator = require('../../../extensions/mongoProcessor/MongoProcessorConfigValidator');
+
+const baseConfig = {
+    topic: 'backbeat-ingestion',
+    groupId: 'backbeat-ingestion-group',
+    probeServer: { port: 4000 },
+};
+
+describe('MongoProcessorConfigValidator log override', () => {
+    it('should pass through log config when set', () => {
+        const validated = configValidator({}, {
+            ...baseConfig,
+            log: { logLevel: 'warn', dumpLevel: 'error' },
+        });
+        assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'error' });
+    });
+
+    it('should leave log undefined when not set, deferring to global config.log', () => {
+        const validated = configValidator({}, baseConfig);
+        assert.strictEqual(validated.log, undefined);
+        // mongoProcessorTask.js: const logConfig = mongoProcessorConfig.log ?? config.log
+        const logConfig = validated.log ?? config.log;
+        assert.strictEqual(logConfig.logLevel, config.log.logLevel);
+        assert.strictEqual(logConfig.dumpLevel, config.log.dumpLevel);
+    });
+});