diff --git a/src/utils.js b/src/utils.js index 28d856d..28fed08 100644 --- a/src/utils.js +++ b/src/utils.js @@ -337,7 +337,30 @@ export const median = (array) => { * @param {number} responseTimeHighThreshold - The maximum acceptable response time threshold in MS. * @param {number} accuracyThreshold - The minimum acceptable accuracy threshold. * @param {array} includedReliabilityFlags - An array of flags that should be included - * when evaluating reliability. + * when evaluating reliability. **Note:** Custom flags will not be evaluated unless they are included in this array. + * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags. + * - Condition arguments: responseTimes, responses, correct, completed, existingFlags + * - Custom conditions requires conditions array (functions) and flag name. logicalOperation ('and', 'or', 'xor') is required when conditions has more than one entry; ignored when conditions has only one entry + * - logicalOperation semantics: 'and' = all conditions true, 'or' = any condition true, 'xor' = exactly one condition true + * - Can return existing and/or custom flags depending on includedReliabilityFlags + * - Custom validations are only evaluated when the run has at least minResponsesRequired responses (i.e. it is not flagged 'notEnoughResponses') + * - **Note:** Custom validation flags cannot depend on each other. Conditions may only reference built-in flags via existingFlags. + * @example + * createEvaluateValidity({ + * customValidations: [{ + * logicalOperation: 'and', + * conditions: [ + * (data) => data.existingFlags.includes('responseTimeTooFast'), + * (data) => { + * const mean = data.responseTimes.reduce((a, b) => a + b, 0) / data.responseTimes.length; + * const variance = data.responseTimes.reduce((sum, rt) => sum + Math.pow(rt - mean, 2), 0) / data.responseTimes.length; + * return variance > 1000000; + * } + * ], + * flag: 'inconsistentTiming' + * }], + * includedReliabilityFlags: ['inconsistentTiming'] // If only responseTimeTooFast is true, run will not be marked as unreliable + * }); * @returns {function} baseValidityEvaluator - A function that evaluates the reliability of a run. */ export function createEvaluateValidity({ @@ -346,7 +369,29 @@ export function createEvaluateValidity({ accuracyThreshold = 0.2, minResponsesRequired = 0, includedReliabilityFlags = ['responseTimeTooFast'], + customValidations = [], }) { + const validLogicalOperations = ['and', 'or', 'xor']; + const normalizedCustomValidations = customValidations.map((v) => ({ + ...v, + logicalOperation: v.logicalOperation?.toLowerCase(), + })); + + for (const { flag, conditions, logicalOperation } of normalizedCustomValidations) { + if (conditions.length > 1) { + if (logicalOperation === undefined) { + throw new Error( + `logicalOperation is required for custom validation flag "${flag}" when multiple conditions are provided. Must be one of: ${validLogicalOperations.join(', ')}.`, + ); + } + if (!validLogicalOperations.includes(logicalOperation)) { + throw new Error( + `Invalid logicalOperation "${logicalOperation}" for custom validation flag "${flag}". Must be one of: ${validLogicalOperations.join(', ')}.`, + ); + } + } + } + return function baseEvaluateValidity({ responseTimes, responses, correct, completed }) { let flags = []; let isReliable = false; @@ -373,10 +418,36 @@ export function createEvaluateValidity({ flags.push('accuracyTooLow'); } } + + // Evaluate custom validations. Only rules whose flag is in includedReliabilityFlags are evaluated, + // and only when there are enough responses (mirroring the built-in checks above). + // Snapshot the built-in flags once, before any custom flag is added, so custom rules stay independent: + // a condition may reference built-in flags via existingFlags but never another custom rule's flag. + if (responseTimes.length >= minResponsesRequired) { + const builtInFlags = [...flags]; + for (const { flag, conditions, logicalOperation } of normalizedCustomValidations) { + if (!includedReliabilityFlags.includes(flag)) continue; + + const data = { responseTimes, responses, correct, completed, existingFlags: [...builtInFlags] }; + + let triggered; + if (conditions.length === 1) { + triggered = conditions[0](data); + } else if (logicalOperation === 'or') { + triggered = conditions.some((c) => c(data)); + } else if (logicalOperation === 'xor') { + triggered = conditions.filter((c) => c(data)).length === 1; + } else { + triggered = conditions.every((c) => c(data)); + } + + if (triggered) flags.push(flag); + } + } isReliable = flags.filter((x) => includedReliabilityFlags.includes(x)).length === 0; flags = flags.filter((x) => includedReliabilityFlags.includes(x)); - + return { flags, isReliable }; }; } diff --git a/test/utils.spec.js b/test/utils.spec.js index cdba13e..5b55231 100644 --- a/test/utils.spec.js +++ b/test/utils.spec.js @@ -233,68 +233,70 @@ test('Creates the correct preload trials from all possible inputs', () => { } }); -const testDate = new Date(); - -const agePossibilities = [ - { - birthMonth: 8, - birthYear: 2009, - age: null, - ageMonths: null, - expectedBirthMonth: 8, - expectedBirthYear: 2009, - }, - { - birthMonth: 5, - birthYear: 2000, - age: 23, - ageMonths: null, - expectedBirthMonth: 5, - expectedBirthYear: 2000, - }, - { - birthMonth: null, - birthYear: 2007, - age: null, - ageMonths: null, - expectedBirthMonth: testDate.getMonth() + 1, - expectedBirthYear: 2007, - }, - { - birthMonth: null, - birthYear: null, - age: null, - ageMonths: 254, - expectedBirthMonth: 12 + ((testDate.getMonth() + 1 - 254) % 12), - expectedBirthYear: testDate.getFullYear() - Math.floor(254 / 12), - }, - { - birthMonth: null, - birthYear: null, - age: 12, - ageMonths: null, - expectedBirthMonth: testDate.getMonth() + 1, - expectedBirthYear: testDate.getFullYear() - 12, - }, - { - birthMonth: 9, - birthYear: null, - age: null, - ageMonths: null, - expectedBirthMonth: null, - expectedBirthYear: null, - }, - { - birthMonth: null, - birthYear: null, - age: null, - ageMonths: null, - expectedBirthMonth: null, - expectedBirthYear: null, - }, -]; - test('Sets the correct age fields for all possible inputs', () => { + // Mock date to avoid timezone-related flakiness + jest.useFakeTimers(); + jest.setSystemTime(new Date('2024-06-15T12:00:00Z')); + const testDate = new Date(); + + const agePossibilities = [ + { + birthMonth: 8, + birthYear: 2009, + age: null, + ageMonths: null, + expectedBirthMonth: 8, + expectedBirthYear: 2009, + }, + { + birthMonth: 5, + birthYear: 2000, + age: 23, + ageMonths: null, + expectedBirthMonth: 5, + expectedBirthYear: 2000, + }, + { + birthMonth: null, + birthYear: 2007, + age: null, + ageMonths: null, + expectedBirthMonth: testDate.getMonth() + 1, + expectedBirthYear: 2007, + }, + { + birthMonth: null, + birthYear: null, + age: null, + ageMonths: 254, + expectedBirthMonth: 12 + ((testDate.getMonth() + 1 - 254) % 12), + expectedBirthYear: testDate.getFullYear() - Math.floor(254 / 12), + }, + { + birthMonth: null, + birthYear: null, + age: 12, + ageMonths: null, + expectedBirthMonth: testDate.getMonth() + 1, + expectedBirthYear: testDate.getFullYear() - 12, + }, + { + birthMonth: 9, + birthYear: null, + age: null, + ageMonths: null, + expectedBirthMonth: null, + expectedBirthYear: null, + }, + { + birthMonth: null, + birthYear: null, + age: null, + ageMonths: null, + expectedBirthMonth: null, + expectedBirthYear: null, + }, + ]; for (const poss of agePossibilities) { const ageData = getAgeData(poss.birthMonth, poss.birthYear, poss.age, poss.ageMonths); @@ -318,6 +320,9 @@ test('Sets the correct age fields for all possible inputs', () => { expect(ageData.age).toBe(expectedAge); expect(ageData.ageMonths).toBe(expectedAgeMonths); } + + // Restore real timers after test + jest.useRealTimers(); }); test('Correctly parses grade', () => { @@ -766,3 +771,525 @@ describe('ValidityEval test for 2 block based assessments (e.g. PA-es)', () => { }); }); }); + +describe('ValidityEvaluator with Custom Rules', () => { + let validityEval; + + test('Throws at setup time when logicalOperation is invalid', () => { + expect(() => { + createEvaluateValidity({ + customValidations: [{ flag: 'myFlag', logicalOperation: 'nand', conditions: [() => true, () => true] }], + includedReliabilityFlags: ['myFlag'], + }); + }).toThrow('Invalid logicalOperation "nand"'); + }); + + test('Single condition rule triggers without logicalOperation', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + minResponsesRequired: 4, + customValidations: [ + { + conditions: [(data) => data.existingFlags.includes('responseTimeTooFast')], + flag: 'fastRun', + }, + ], + includedReliabilityFlags: ['fastRun'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(300, 'left_arrow', 1); + validityEval.addResponseData(350, 'right_arrow', 1); + validityEval.addResponseData(320, 'left_arrow', 1); + validityEval.addResponseData(380, 'right_arrow', 1); + + expect(testAddFlags).toHaveBeenLastCalledWith(['fastRun'], false); + }); + + test('Custom rule with AND logic using existing flag and custom condition', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'and', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => { + const uniqueResponses = new Set(data.responses).size; + return uniqueResponses <= 2; + }, + ], + flag: 'fastAndRepetitive', + }, + ], + includedReliabilityFlags: ['fastAndRepetitive'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(300, 'left_arrow', 1); + validityEval.addResponseData(350, 'left_arrow', 1); + validityEval.addResponseData(320, 'right_arrow', 0); + validityEval.addResponseData(380, 'left_arrow', 1); + validityEval.addResponseData(340, 'right_arrow', 0); + validityEval.addResponseData(360, 'left_arrow', 1); + + expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndRepetitive'], false); + }); + + test('Custom rule returns reliable when custom conditions are not met', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'and', + conditions: [ + (data) => { + const maxRT = Math.max(...data.responseTimes); + return maxRT > 5000; + }, + (data) => { + const counts = {}; + data.responses.forEach((r) => counts[r] = (counts[r] || 0) + 1); + return Math.max(...Object.values(counts)) / data.responses.length > 0.9; + }, + ], + flag: 'slowAndRepetitive', + }, + ], + includedReliabilityFlags: ['slowAndRepetitive'], + }), + handleEngagementFlags: testAddFlags, + }); + + // Good performance: fast responses, varied keys + validityEval.addResponseData(600, 'left_arrow', 1); + validityEval.addResponseData(550, 'right_arrow', 1); + validityEval.addResponseData(620, 'left_arrow', 1); + validityEval.addResponseData(580, 'right_arrow', 1); + validityEval.addResponseData(590, 'left_arrow', 1); + validityEval.addResponseData(610, 'right_arrow', 1); + + // No flags triggered, run is reliable + expect(testAddFlags).toHaveBeenLastCalledWith([], true); + }); + + test('Custom rule with OR logic triggers when any condition is met', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'or', + conditions: [ + (data) => data.existingFlags.includes('accuracyTooLow'), + (data) => { + const maxRT = Math.max(...data.responseTimes); + return maxRT > 8000; + }, + ], + flag: 'poorPerformance', + }, + ], + includedReliabilityFlags: ['poorPerformance'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(600, 'left_arrow', 0); + validityEval.addResponseData(650, 'right_arrow', 0); + validityEval.addResponseData(700, 'left_arrow', 0); + validityEval.addResponseData(620, 'right_arrow', 0); + validityEval.addResponseData(680, 'left_arrow', 0); + validityEval.addResponseData(640, 'right_arrow', 1); + + expect(testAddFlags).toHaveBeenLastCalledWith(['poorPerformance'], false); + }); + + test('Custom rule with OR logic using custom condition only', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'or', + conditions: [ + (data) => { + const mean = data.responseTimes.reduce((a, b) => a + b, 0) / data.responseTimes.length; + const variance = data.responseTimes.reduce((sum, rt) => sum + (rt - mean) ** 2, 0) / data.responseTimes.length; + return variance > 500000; + }, + (data) => Math.max(...data.responseTimes) > 5000, + ], + flag: 'inconsistentTiming', + }, + ], + includedReliabilityFlags: ['inconsistentTiming'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(500, 'left_arrow', 1); + validityEval.addResponseData(6000, 'right_arrow', 1); + validityEval.addResponseData(550, 'left_arrow', 1); + validityEval.addResponseData(600, 'right_arrow', 1); + validityEval.addResponseData(520, 'left_arrow', 1); + + expect(testAddFlags).toHaveBeenLastCalledWith(['inconsistentTiming'], false); + }); + + test('Multiple custom rules can be evaluated together', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'and', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => new Set(data.responses).size === 1, + ], + flag: 'fastAndSameKey', + }, + { + logicalOperation: 'and', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.existingFlags.includes('accuracyTooLow'), + ], + flag: 'fastAndInaccurate', + }, + ], + includedReliabilityFlags: ['fastAndSameKey', 'fastAndInaccurate'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(300, 'left_arrow', 0); + validityEval.addResponseData(350, 'left_arrow', 0); + validityEval.addResponseData(320, 'left_arrow', 0); + validityEval.addResponseData(380, 'left_arrow', 0); + validityEval.addResponseData(340, 'left_arrow', 1); + validityEval.addResponseData(360, 'left_arrow', 0); + + expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndSameKey', 'fastAndInaccurate'], false); + }); + + test('Custom rules work alongside default flags when both are in includedReliabilityFlags', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'and', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.responseTimes.length < 5, + ], + flag: 'fastAndFew', + }, + ], + includedReliabilityFlags: ['responseTimeTooFast', 'fastAndFew'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(300, 'left_arrow', 1); + validityEval.addResponseData(350, 'right_arrow', 0); + validityEval.addResponseData(320, 'left_arrow', 1); + validityEval.addResponseData(380, 'right_arrow', 0); + + expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'fastAndFew'], false); + }); + + test('Base flag used in custom condition is excluded when not in includedReliabilityFlags', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'and', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.existingFlags.includes('accuracyTooLow'), + ], + flag: 'fastAndInaccurate', + }, + ], + // Only include custom flag, not the base flags + includedReliabilityFlags: ['fastAndInaccurate'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(300, 'left_arrow', 0); + validityEval.addResponseData(350, 'right_arrow', 0); + validityEval.addResponseData(320, 'left_arrow', 0); + validityEval.addResponseData(380, 'right_arrow', 0); + validityEval.addResponseData(340, 'left_arrow', 0); + validityEval.addResponseData(360, 'right_arrow', 1); + + // Only custom flag returned, base flags excluded + expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndInaccurate'], false); + }); + + test('Custom validation flag not evaluated when not in includedReliabilityFlags', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'and', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => { + const uniqueResponses = new Set(data.responses).size; + return uniqueResponses <= 2; + }, + ], + flag: 'fastAndRepetitive', + }, + ], + // Custom flag NOT included in includedReliabilityFlags + includedReliabilityFlags: ['responseTimeTooFast'], + }), + handleEngagementFlags: testAddFlags, + }); + + // Trigger both responseTimeTooFast AND fastAndRepetitive conditions + validityEval.addResponseData(300, 'left_arrow', 1); + validityEval.addResponseData(350, 'left_arrow', 1); + validityEval.addResponseData(320, 'right_arrow', 0); + validityEval.addResponseData(380, 'left_arrow', 1); + + // Only responseTimeTooFast returned, fastAndRepetitive ignored + expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast'], false); + }); + + test('Custom rule with XOR logic triggers when exactly one condition is met', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'xor', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.existingFlags.includes('accuracyTooLow'), + ], + flag: 'eitherFastOrInaccurate', + }, + ], + includedReliabilityFlags: ['eitherFastOrInaccurate'], + }), + handleEngagementFlags: testAddFlags, + }); + + // Fast but accurate - only one condition met + validityEval.addResponseData(300, 'left_arrow', 1); + validityEval.addResponseData(350, 'right_arrow', 1); + validityEval.addResponseData(320, 'left_arrow', 1); + validityEval.addResponseData(380, 'right_arrow', 1); + + expect(testAddFlags).toHaveBeenLastCalledWith(['eitherFastOrInaccurate'], false); + }); + + test('Custom rule with XOR logic does not trigger when both conditions are met', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'xor', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.existingFlags.includes('accuracyTooLow'), + ], + flag: 'eitherFastOrInaccurate', + }, + ], + includedReliabilityFlags: ['eitherFastOrInaccurate'], + }), + handleEngagementFlags: testAddFlags, + }); + + // Fast AND inaccurate - both conditions met, XOR should NOT trigger + validityEval.addResponseData(300, 'left_arrow', 0); + validityEval.addResponseData(350, 'right_arrow', 0); + validityEval.addResponseData(320, 'left_arrow', 0); + validityEval.addResponseData(380, 'right_arrow', 0); + validityEval.addResponseData(340, 'left_arrow', 1); + + expect(testAddFlags).toHaveBeenLastCalledWith([], true); + }); + + test('Custom rule with XOR logic does not trigger when no conditions are met', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'xor', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.existingFlags.includes('accuracyTooLow'), + ], + flag: 'eitherFastOrInaccurate', + }, + ], + includedReliabilityFlags: ['eitherFastOrInaccurate'], + }), + handleEngagementFlags: testAddFlags, + }); + + // Good performance - neither condition met + validityEval.addResponseData(600, 'left_arrow', 1); + validityEval.addResponseData(650, 'right_arrow', 1); + validityEval.addResponseData(700, 'left_arrow', 1); + validityEval.addResponseData(750, 'right_arrow', 1); + + expect(testAddFlags).toHaveBeenLastCalledWith([], true); + }); + + test('Throws at setup time when logicalOperation is missing for multiple conditions', () => { + expect(() => { + createEvaluateValidity({ + customValidations: [ + { + conditions: [(data) => data.existingFlags.includes('responseTimeTooFast'), () => true], + flag: 'fastAndRepetitive', + }, + ], + includedReliabilityFlags: ['fastAndRepetitive'], + }); + }).toThrow('logicalOperation is required'); + }); + + test('includedReliabilityFlags order does not affect custom validation - base flag last', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'and', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.existingFlags.includes('accuracyTooLow'), + ], + flag: 'fastAndInaccurate', + }, + ], + // Custom flag listed BEFORE base flags + includedReliabilityFlags: ['fastAndInaccurate', 'responseTimeTooFast', 'accuracyTooLow'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(300, 'left_arrow', 0); + validityEval.addResponseData(350, 'right_arrow', 0); + validityEval.addResponseData(320, 'left_arrow', 0); + validityEval.addResponseData(380, 'right_arrow', 0); + validityEval.addResponseData(340, 'left_arrow', 1); + + // Same flags returned, order in includedReliabilityFlags doesn't matter + expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'accuracyTooLow', 'fastAndInaccurate'], false); + }); + + test('Capitalized logicalOperation is normalized and works correctly', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + responseTimeLowThreshold: 400, + accuracyThreshold: 0.2, + minResponsesRequired: 4, + customValidations: [ + { + logicalOperation: 'AND', + conditions: [ + (data) => data.existingFlags.includes('responseTimeTooFast'), + (data) => data.existingFlags.includes('accuracyTooLow'), + ], + flag: 'fastAndInaccurate', + }, + ], + includedReliabilityFlags: ['fastAndInaccurate'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(300, 'left_arrow', 0); + validityEval.addResponseData(350, 'right_arrow', 0); + validityEval.addResponseData(320, 'left_arrow', 0); + validityEval.addResponseData(380, 'right_arrow', 0); + + expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndInaccurate'], false); + }); + + test('Custom validation flags are independent - a rule cannot observe another custom flag', () => { + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + minResponsesRequired: 4, + customValidations: [ + // Always fires + { conditions: [() => true], flag: 'flagA' }, + // Tries to depend on the earlier custom flag; existingFlags must only contain built-in flags + { conditions: [(data) => data.existingFlags.includes('flagA')], flag: 'flagB' }, + ], + includedReliabilityFlags: ['flagA', 'flagB'], + }), + handleEngagementFlags: testAddFlags, + }); + + // Response times high / all correct, so no built-in flags fire + validityEval.addResponseData(600, 'left_arrow', 1); + validityEval.addResponseData(650, 'right_arrow', 1); + validityEval.addResponseData(620, 'left_arrow', 1); + validityEval.addResponseData(580, 'right_arrow', 1); + + // flagA fires; flagB must NOT, because conditions only see built-in flags + expect(testAddFlags).toHaveBeenLastCalledWith(['flagA'], false); + }); + + test('Custom validations are not evaluated when there are not enough responses', () => { + const condition = jest.fn(() => true); + validityEval = new ValidityEvaluator({ + evaluateValidity: createEvaluateValidity({ + minResponsesRequired: 10, + customValidations: [{ conditions: [condition], flag: 'alwaysFlag' }], + includedReliabilityFlags: ['notEnoughResponses', 'alwaysFlag'], + }), + handleEngagementFlags: testAddFlags, + }); + + validityEval.addResponseData(600, 'left_arrow', 1); + validityEval.addResponseData(650, 'right_arrow', 1); + validityEval.addResponseData(620, 'left_arrow', 1); + + // Too few responses: only notEnoughResponses, the custom rule is skipped entirely + expect(testAddFlags).toHaveBeenLastCalledWith(['notEnoughResponses'], false); + expect(condition).not.toHaveBeenCalled(); + }); + +});