From 07fb1b8264b6e79bf1a760f184ae32590f49055d Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Tue, 2 Jun 2026 20:23:38 -0400
Subject: [PATCH 01/16] Implement custom reliability flags

---
 src/utils.js | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/src/utils.js b/src/utils.js
index 7f3b4f6..e777a21 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -337,6 +337,25 @@ export const median = (array) => {
  * @param {number} accuracyThreshold - The minimum acceptable accuracy threshold.
  * @param {array} includedReliabilityFlags - An array of flags that should be included
  * when evaluating reliability.
+ * @param {array} compositeRules - An array of composite rules to evaluate. Can pass in custom conditions or use pre-existing flags. 
+ *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
+ *  - Custom conditions requires logicalOperation, conditions array (functions), and flag name
+ *  @example
+ * createEvaluateValidity({
+ *   compositeRules: [{
+ *     logicalOperation: 'and',
+ *     conditions: [
+ *       (data) => data.existingFlags.includes('responseTimeTooFast'),
+ *       (data) => {
+ *         const mean = data.responseTimes.reduce((a, b) => a + b) / data.responseTimes.length;
+ *         const variance = data.responseTimes.reduce((sum, rt) => sum + Math.pow(rt - mean, 2), 0) / data.responseTimes.length;
+ *         return variance > 1000000;
+ *       }
+ *     ],
+ *     flag: 'inconsistentTiming'
+ *   }],
+ *   includedReliabilityFlags: ['inconsistentTiming', 'responseTimeTooFast']
+ * });
  * @returns {function} baseValidityEvaluator - A function that evaluates the reliability of a run.
  */
 export function createEvaluateValidity({
@@ -345,6 +364,7 @@ export function createEvaluateValidity({
   accuracyThreshold = 0.2,
   minResponsesRequired = 0,
   includedReliabilityFlags = ['responseTimeTooFast'],
+  compositeRules = [] //[{logicalOperation: 'and', conditions: [(data) => {}], flag: 'compositeFlag'}]
 }) {
   return function baseEvaluateValidity({
     responseTimes, responses, correct, completed,
@@ -373,6 +393,27 @@ export function createEvaluateValidity({
       if (numCorrect / correct.length <= accuracyThreshold) {
         flags.push('accuracyTooLow');
       }
+      
+      // Evaluate composite rules alongside default checks
+      if (compositeRules.length > 0) {
+        const data = { responseTimes, responses, correct, completed, existingFlags: flags };
+        
+        compositeRules.forEach((rule) => {
+          const { logicalOperation, conditions, flag } = rule;
+          let conditionMet = false;
+          
+          if (logicalOperation === 'and') {
+            conditionMet = conditions.every((condition) => condition(data));
+          } else if (logicalOperation === 'or') {
+            conditionMet = conditions.some((condition) => condition(data));
+          }
+          
+          if (conditionMet) {
+            flags.push(flag);
+          }
+        });
+      }
+      
       isReliable = flags.filter((x) => includedReliabilityFlags.includes(x)).length === 0;
       flags = flags.filter((x) => includedReliabilityFlags.includes(x));
     }

From 056b012fd6d741a6b0b7ad2a06bf4d86bcb0426d Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 10:11:55 -0400
Subject: [PATCH 02/16] Add test cases for custom reliability flags

---
 test/utils.spec.js | 205 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 205 insertions(+)

diff --git a/test/utils.spec.js b/test/utils.spec.js
index 5927369..31802be 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -719,3 +719,208 @@ describe('ValidityEval test for 2 block based assessments (e.g. PA-es)', () => {
     });
   });
 });
+
+describe('ValidityEvaluator with Composite Rules', () => {
+  let validityEval;
+
+  test('Composite rule with AND logic using existing flag and custom condition', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        minResponsesRequired: 4,
+        compositeRules: [
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => {
+                const uniqueResponses = new Set(data.responses).size;
+                return uniqueResponses <= 2;
+              },
+            ],
+            flag: 'fastAndRepetitive',
+          },
+        ],
+        includedReliabilityFlags: ['fastAndRepetitive'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(300, 'left_arrow', 1);
+    validityEval.addResponseData(350, 'left_arrow', 1);
+    validityEval.addResponseData(320, 'right_arrow', 0);
+    validityEval.addResponseData(380, 'left_arrow', 1);
+    validityEval.addResponseData(340, 'right_arrow', 0);
+    validityEval.addResponseData(360, 'left_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndRepetitive'], false);
+  });
+
+  test('Composite rule with AND logic does not trigger when only one condition is met', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        minResponsesRequired: 4,
+        compositeRules: [
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => {
+                const counts = {};
+                data.responses.forEach(r => counts[r] = (counts[r] || 0) + 1);
+                return Math.max(...Object.values(counts)) / data.responses.length > 0.8;
+              },
+            ],
+            flag: 'fastAndRepetitive',
+          },
+        ],
+        includedReliabilityFlags: ['fastAndRepetitive'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(300, 'left_arrow', 1);
+    validityEval.addResponseData(350, 'right_arrow', 0);
+    validityEval.addResponseData(320, 'left_arrow', 1);
+    validityEval.addResponseData(380, 'right_arrow', 0);
+    validityEval.addResponseData(340, 'left_arrow', 1);
+    validityEval.addResponseData(360, 'right_arrow', 0);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith([], true);
+  });
+
+  test('Composite rule with OR logic triggers when any condition is met', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
+        minResponsesRequired: 4,
+        compositeRules: [
+          {
+            logicalOperation: 'or',
+            conditions: [
+              (data) => data.existingFlags.includes('accuracyTooLow'),
+              (data) => {
+                const maxRT = Math.max(...data.responseTimes);
+                return maxRT > 8000;
+              },
+            ],
+            flag: 'poorPerformance',
+          },
+        ],
+        includedReliabilityFlags: ['poorPerformance'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(600, 'left_arrow', 0);
+    validityEval.addResponseData(650, 'right_arrow', 0);
+    validityEval.addResponseData(700, 'left_arrow', 0);
+    validityEval.addResponseData(620, 'right_arrow', 0);
+    validityEval.addResponseData(680, 'left_arrow', 0);
+    validityEval.addResponseData(640, 'right_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['poorPerformance'], false);
+  });
+
+  test('Composite rule with OR logic using custom condition only', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        minResponsesRequired: 4,
+        compositeRules: [
+          {
+            logicalOperation: 'or',
+            conditions: [
+              (data) => {
+                const mean = data.responseTimes.reduce((a, b) => a + b, 0) / data.responseTimes.length;
+                const variance = data.responseTimes.reduce((sum, rt) => sum + Math.pow(rt - mean, 2), 0) / data.responseTimes.length;
+                return variance > 500000;
+              },
+              (data) => Math.max(...data.responseTimes) > 5000,
+            ],
+            flag: 'inconsistentTiming',
+          },
+        ],
+        includedReliabilityFlags: ['inconsistentTiming'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(500, 'left_arrow', 1);
+    validityEval.addResponseData(6000, 'right_arrow', 1);
+    validityEval.addResponseData(550, 'left_arrow', 1);
+    validityEval.addResponseData(600, 'right_arrow', 1);
+    validityEval.addResponseData(520, 'left_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['inconsistentTiming'], false);
+  });
+
+  test('Multiple composite rules can be evaluated together', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
+        minResponsesRequired: 4,
+        compositeRules: [
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => new Set(data.responses).size === 1,
+            ],
+            flag: 'fastAndSameKey',
+          },
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => data.existingFlags.includes('accuracyTooLow'),
+            ],
+            flag: 'fastAndInaccurate',
+          },
+        ],
+        includedReliabilityFlags: ['fastAndSameKey', 'fastAndInaccurate'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(300, 'left_arrow', 0);
+    validityEval.addResponseData(350, 'left_arrow', 0);
+    validityEval.addResponseData(320, 'left_arrow', 0);
+    validityEval.addResponseData(380, 'left_arrow', 0);
+    validityEval.addResponseData(340, 'left_arrow', 1);
+    validityEval.addResponseData(360, 'left_arrow', 0);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndSameKey', 'fastAndInaccurate'], false);
+  });
+
+  test('Composite rules work alongside default flags when both are in includedReliabilityFlags', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        minResponsesRequired: 4,
+        compositeRules: [
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => data.responseTimes.length < 5,
+            ],
+            flag: 'fastAndFew',
+          },
+        ],
+        includedReliabilityFlags: ['responseTimeTooFast', 'fastAndFew'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(300, 'left_arrow', 1);
+    validityEval.addResponseData(350, 'right_arrow', 0);
+    validityEval.addResponseData(320, 'left_arrow', 1);
+    validityEval.addResponseData(380, 'right_arrow', 0);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'fastAndFew'], false);
+  });
+});

From dd44ce0dcad084908c337a6287b562c95fcdf535 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 10:15:30 -0400
Subject: [PATCH 03/16] Fix flaky getAgeData test using fixed date value

---
 test/utils.spec.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/utils.spec.js b/test/utils.spec.js
index 31802be..39775ed 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -233,6 +233,9 @@ test('Creates the correct preload trials from all possible inputs', () => {
   }
 });
 
+// Mock date to avoid timezone-related flakiness
+jest.useFakeTimers();
+jest.setSystemTime(new Date('2024-06-15T12:00:00Z'));
 const testDate = new Date();
 
 const agePossibilities = [
@@ -318,6 +321,9 @@ test('Sets the correct age fields for all possible inputs', () => {
     expect(ageData.age).toBe(expectedAge);
     expect(ageData.ageMonths).toBe(expectedAgeMonths);
   }
+  
+  // Restore real timers after test
+  jest.useRealTimers();
 });
 
 test('Correctly parses grade', () => {

From 62324fbe1ace6c4e02e3447398c8527f944b7644 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 10:19:46 -0400
Subject: [PATCH 04/16] Change one of the multi-condition tests to return
 reliable = true

---
 test/utils.spec.js | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/test/utils.spec.js b/test/utils.spec.js
index 39775ed..f269b4e 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -762,7 +762,7 @@ describe('ValidityEvaluator with Composite Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndRepetitive'], false);
   });
 
-  test('Composite rule with AND logic does not trigger when only one condition is met', () => {
+  test('Composite rule returns reliable when custom conditions are not met', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
@@ -771,28 +771,33 @@ describe('ValidityEvaluator with Composite Rules', () => {
           {
             logicalOperation: 'and',
             conditions: [
-              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => {
+                const maxRT = Math.max(...data.responseTimes);
+                return maxRT > 5000;
+              },
               (data) => {
                 const counts = {};
                 data.responses.forEach(r => counts[r] = (counts[r] || 0) + 1);
-                return Math.max(...Object.values(counts)) / data.responses.length > 0.8;
+                return Math.max(...Object.values(counts)) / data.responses.length > 0.9;
               },
             ],
-            flag: 'fastAndRepetitive',
+            flag: 'slowAndRepetitive',
           },
         ],
-        includedReliabilityFlags: ['fastAndRepetitive'],
+        includedReliabilityFlags: ['slowAndRepetitive'],
       }),
       handleEngagementFlags: testAddFlags,
     });
 
-    validityEval.addResponseData(300, 'left_arrow', 1);
-    validityEval.addResponseData(350, 'right_arrow', 0);
-    validityEval.addResponseData(320, 'left_arrow', 1);
-    validityEval.addResponseData(380, 'right_arrow', 0);
-    validityEval.addResponseData(340, 'left_arrow', 1);
-    validityEval.addResponseData(360, 'right_arrow', 0);
+    // Good performance: fast responses, varied keys
+    validityEval.addResponseData(600, 'left_arrow', 1);
+    validityEval.addResponseData(550, 'right_arrow', 1);
+    validityEval.addResponseData(620, 'left_arrow', 1);
+    validityEval.addResponseData(580, 'right_arrow', 1);
+    validityEval.addResponseData(590, 'left_arrow', 1);
+    validityEval.addResponseData(610, 'right_arrow', 1);
 
+    // No flags triggered, run is reliable
     expect(testAddFlags).toHaveBeenLastCalledWith([], true);
   });
 

From 89cbd8edb04783de9be20a6bf555f9b8825d9554 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 10:25:05 -0400
Subject: [PATCH 05/16] Add test case that uses base engagementFlags in their
 conditions but does not return them

---
 test/utils.spec.js | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/test/utils.spec.js b/test/utils.spec.js
index f269b4e..1a3c9c9 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -934,4 +934,37 @@ describe('ValidityEvaluator with Composite Rules', () => {
 
     expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'fastAndFew'], false);
   });
+
+  test('Base flag used in composite condition is excluded when not in includedReliabilityFlags', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
+        minResponsesRequired: 4,
+        compositeRules: [
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => data.existingFlags.includes('accuracyTooLow'),
+            ],
+            flag: 'fastAndInaccurate',
+          },
+        ],
+        // Only include composite flag, not the base flags
+        includedReliabilityFlags: ['fastAndInaccurate'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(300, 'left_arrow', 0);
+    validityEval.addResponseData(350, 'right_arrow', 0);
+    validityEval.addResponseData(320, 'left_arrow', 0);
+    validityEval.addResponseData(380, 'right_arrow', 0);
+    validityEval.addResponseData(340, 'left_arrow', 0);
+    validityEval.addResponseData(360, 'right_arrow', 1);
+
+    // Only composite flag returned, base flags excluded
+    expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndInaccurate'], false);
+  });
 });

From 8db50f96d178f0e31a1da9c5e0491effb756ee0c Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 10:30:33 -0400
Subject: [PATCH 06/16] Delete comment

---
 src/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/utils.js b/src/utils.js
index e777a21..f108f77 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -364,7 +364,7 @@ export function createEvaluateValidity({
   accuracyThreshold = 0.2,
   minResponsesRequired = 0,
   includedReliabilityFlags = ['responseTimeTooFast'],
-  compositeRules = [] //[{logicalOperation: 'and', conditions: [(data) => {}], flag: 'compositeFlag'}]
+  compositeRules = [] 
 }) {
   return function baseEvaluateValidity({
     responseTimes, responses, correct, completed,

From 8f0e60ef98928a81a7e66bf11e4f71526e86ff24 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 10:38:28 -0400
Subject: [PATCH 07/16] Standardize logicalOperators to lowerCase

---
 src/utils.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index f108f77..1a6ecbf 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -402,9 +402,9 @@ export function createEvaluateValidity({
           const { logicalOperation, conditions, flag } = rule;
           let conditionMet = false;
           
-          if (logicalOperation === 'and') {
+          if (logicalOperation.toLowerCase() === 'and') {
             conditionMet = conditions.every((condition) => condition(data));
-          } else if (logicalOperation === 'or') {
+          } else if (logicalOperation.toLowerCase() === 'or') {
             conditionMet = conditions.some((condition) => condition(data));
           }
           

From 00438bb4befdd26c36ce167f91da1abd28f33c45 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 11:25:19 -0400
Subject: [PATCH 08/16] Clarify example in createEvaluateValidity jsdocs

---
 src/utils.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index 1a6ecbf..90cac29 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -340,7 +340,8 @@ export const median = (array) => {
  * @param {array} compositeRules - An array of composite rules to evaluate. Can pass in custom conditions or use pre-existing flags. 
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
  *  - Custom conditions requires logicalOperation, conditions array (functions), and flag name
- *  @example
+ *  - Can return existing and/or custom flags depending on includedReliabilityFlags 
+ * @example
  * createEvaluateValidity({
  *   compositeRules: [{
  *     logicalOperation: 'and',
@@ -354,7 +355,7 @@ export const median = (array) => {
  *     ],
  *     flag: 'inconsistentTiming'
  *   }],
- *   includedReliabilityFlags: ['inconsistentTiming', 'responseTimeTooFast']
+ *   includedReliabilityFlags: ['inconsistentTiming'] // If only responseTimeTooFast is true, run will not be marked as unreliable
  * });
  * @returns {function} baseValidityEvaluator - A function that evaluates the reliability of a run.
  */

From 87e179cf72bde9705c23ee531b4532f2f4b056dc Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Wed, 3 Jun 2026 11:30:25 -0400
Subject: [PATCH 09/16] Rename compositeRules to customValidations

---
 src/utils.js       | 12 ++++++------
 test/utils.spec.js | 34 +++++++++++++++++-----------------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index 90cac29..d035f5e 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -337,13 +337,13 @@ export const median = (array) => {
  * @param {number} accuracyThreshold - The minimum acceptable accuracy threshold.
  * @param {array} includedReliabilityFlags - An array of flags that should be included
  * when evaluating reliability.
- * @param {array} compositeRules - An array of composite rules to evaluate. Can pass in custom conditions or use pre-existing flags. 
+ * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags. 
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
  *  - Custom conditions requires logicalOperation, conditions array (functions), and flag name
  *  - Can return existing and/or custom flags depending on includedReliabilityFlags 
  * @example
  * createEvaluateValidity({
- *   compositeRules: [{
+ *   customValidations: [{
  *     logicalOperation: 'and',
  *     conditions: [
  *       (data) => data.existingFlags.includes('responseTimeTooFast'),
@@ -365,7 +365,7 @@ export function createEvaluateValidity({
   accuracyThreshold = 0.2,
   minResponsesRequired = 0,
   includedReliabilityFlags = ['responseTimeTooFast'],
-  compositeRules = [] 
+  customValidations = [] 
 }) {
   return function baseEvaluateValidity({
     responseTimes, responses, correct, completed,
@@ -395,11 +395,11 @@ export function createEvaluateValidity({
         flags.push('accuracyTooLow');
       }
       
-      // Evaluate composite rules alongside default checks
-      if (compositeRules.length > 0) {
+      // Evaluate custom validations alongside default checks
+      if (customValidations.length > 0) {
         const data = { responseTimes, responses, correct, completed, existingFlags: flags };
         
-        compositeRules.forEach((rule) => {
+        customValidations.forEach((rule) => {
           const { logicalOperation, conditions, flag } = rule;
           let conditionMet = false;
           
diff --git a/test/utils.spec.js b/test/utils.spec.js
index 1a3c9c9..e1c7918 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -726,15 +726,15 @@ describe('ValidityEval test for 2 block based assessments (e.g. PA-es)', () => {
   });
 });
 
-describe('ValidityEvaluator with Composite Rules', () => {
+describe('ValidityEvaluator with Custom Rules', () => {
   let validityEval;
 
-  test('Composite rule with AND logic using existing flag and custom condition', () => {
+  test('Custom rule with AND logic using existing flag and custom condition', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
         minResponsesRequired: 4,
-        compositeRules: [
+        customValidations: [
           {
             logicalOperation: 'and',
             conditions: [
@@ -762,12 +762,12 @@ describe('ValidityEvaluator with Composite Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndRepetitive'], false);
   });
 
-  test('Composite rule returns reliable when custom conditions are not met', () => {
+  test('Custom rule returns reliable when custom conditions are not met', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
         minResponsesRequired: 4,
-        compositeRules: [
+        customValidations: [
           {
             logicalOperation: 'and',
             conditions: [
@@ -801,13 +801,13 @@ describe('ValidityEvaluator with Composite Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith([], true);
   });
 
-  test('Composite rule with OR logic triggers when any condition is met', () => {
+  test('Custom rule with OR logic triggers when any condition is met', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
         accuracyThreshold: 0.2,
         minResponsesRequired: 4,
-        compositeRules: [
+        customValidations: [
           {
             logicalOperation: 'or',
             conditions: [
@@ -835,12 +835,12 @@ describe('ValidityEvaluator with Composite Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith(['poorPerformance'], false);
   });
 
-  test('Composite rule with OR logic using custom condition only', () => {
+  test('Custom rule with OR logic using custom condition only', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
         minResponsesRequired: 4,
-        compositeRules: [
+        customValidations: [
           {
             logicalOperation: 'or',
             conditions: [
@@ -868,13 +868,13 @@ describe('ValidityEvaluator with Composite Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith(['inconsistentTiming'], false);
   });
 
-  test('Multiple composite rules can be evaluated together', () => {
+  test('Multiple custom rules can be evaluated together', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
         accuracyThreshold: 0.2,
         minResponsesRequired: 4,
-        compositeRules: [
+        customValidations: [
           {
             logicalOperation: 'and',
             conditions: [
@@ -907,12 +907,12 @@ describe('ValidityEvaluator with Composite Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndSameKey', 'fastAndInaccurate'], false);
   });
 
-  test('Composite rules work alongside default flags when both are in includedReliabilityFlags', () => {
+  test('Custom rules work alongside default flags when both are in includedReliabilityFlags', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
         minResponsesRequired: 4,
-        compositeRules: [
+        customValidations: [
           {
             logicalOperation: 'and',
             conditions: [
@@ -935,13 +935,13 @@ describe('ValidityEvaluator with Composite Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'fastAndFew'], false);
   });
 
-  test('Base flag used in composite condition is excluded when not in includedReliabilityFlags', () => {
+  test('Base flag used in custom condition is excluded when not in includedReliabilityFlags', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
         accuracyThreshold: 0.2,
         minResponsesRequired: 4,
-        compositeRules: [
+        customValidations: [
           {
             logicalOperation: 'and',
             conditions: [
@@ -951,7 +951,7 @@ describe('ValidityEvaluator with Composite Rules', () => {
             flag: 'fastAndInaccurate',
           },
         ],
-        // Only include composite flag, not the base flags
+        // Only include custom flag, not the base flags
         includedReliabilityFlags: ['fastAndInaccurate'],
       }),
       handleEngagementFlags: testAddFlags,
@@ -964,7 +964,7 @@ describe('ValidityEvaluator with Composite Rules', () => {
     validityEval.addResponseData(340, 'left_arrow', 0);
     validityEval.addResponseData(360, 'right_arrow', 1);
 
-    // Only composite flag returned, base flags excluded
+    // Only custom flag returned, base flags excluded
     expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndInaccurate'], false);
   });
 });

From 214b5a7653525da1d1f75d8c1d5e20fee798ae3f Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Tue, 9 Jun 2026 14:26:25 -0700
Subject: [PATCH 10/16] Move jest.fakeTimers into test case

---
 src/utils.js       |  20 ++++---
 test/utils.spec.js | 133 ++++++++++++++++++++++-----------------------
 2 files changed, 77 insertions(+), 76 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index d035f5e..791eacc 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -337,10 +337,10 @@ export const median = (array) => {
  * @param {number} accuracyThreshold - The minimum acceptable accuracy threshold.
  * @param {array} includedReliabilityFlags - An array of flags that should be included
  * when evaluating reliability.
- * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags. 
+ * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags.
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
  *  - Custom conditions requires logicalOperation, conditions array (functions), and flag name
- *  - Can return existing and/or custom flags depending on includedReliabilityFlags 
+ *  - Can return existing and/or custom flags depending on includedReliabilityFlags
  * @example
  * createEvaluateValidity({
  *   customValidations: [{
@@ -365,7 +365,7 @@ export function createEvaluateValidity({
   accuracyThreshold = 0.2,
   minResponsesRequired = 0,
   includedReliabilityFlags = ['responseTimeTooFast'],
-  customValidations = [] 
+  customValidations = [],
 }) {
   return function baseEvaluateValidity({
     responseTimes, responses, correct, completed,
@@ -394,27 +394,29 @@ export function createEvaluateValidity({
       if (numCorrect / correct.length <= accuracyThreshold) {
         flags.push('accuracyTooLow');
       }
-      
+
       // Evaluate custom validations alongside default checks
       if (customValidations.length > 0) {
-        const data = { responseTimes, responses, correct, completed, existingFlags: flags };
-        
+        const data = {
+          responseTimes, responses, correct, completed, existingFlags: flags,
+        };
+
         customValidations.forEach((rule) => {
           const { logicalOperation, conditions, flag } = rule;
           let conditionMet = false;
-          
+
           if (logicalOperation.toLowerCase() === 'and') {
             conditionMet = conditions.every((condition) => condition(data));
           } else if (logicalOperation.toLowerCase() === 'or') {
             conditionMet = conditions.some((condition) => condition(data));
           }
-          
+
           if (conditionMet) {
             flags.push(flag);
           }
         });
       }
-      
+
       isReliable = flags.filter((x) => includedReliabilityFlags.includes(x)).length === 0;
       flags = flags.filter((x) => includedReliabilityFlags.includes(x));
     }
diff --git a/test/utils.spec.js b/test/utils.spec.js
index e1c7918..7333e5f 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -233,71 +233,70 @@ test('Creates the correct preload trials from all possible inputs', () => {
   }
 });
 
-// Mock date to avoid timezone-related flakiness
-jest.useFakeTimers();
-jest.setSystemTime(new Date('2024-06-15T12:00:00Z'));
-const testDate = new Date();
-
-const agePossibilities = [
-  {
-    birthMonth: 8,
-    birthYear: 2009,
-    age: null,
-    ageMonths: null,
-    expectedBirthMonth: 8,
-    expectedBirthYear: 2009,
-  },
-  {
-    birthMonth: 5,
-    birthYear: 2000,
-    age: 23,
-    ageMonths: null,
-    expectedBirthMonth: 5,
-    expectedBirthYear: 2000,
-  },
-  {
-    birthMonth: null,
-    birthYear: 2007,
-    age: null,
-    ageMonths: null,
-    expectedBirthMonth: testDate.getMonth() + 1,
-    expectedBirthYear: 2007,
-  },
-  {
-    birthMonth: null,
-    birthYear: null,
-    age: null,
-    ageMonths: 254,
-    expectedBirthMonth: 12 + ((testDate.getMonth() + 1 - 254) % 12),
-    expectedBirthYear: testDate.getFullYear() - Math.floor(254 / 12),
-  },
-  {
-    birthMonth: null,
-    birthYear: null,
-    age: 12,
-    ageMonths: null,
-    expectedBirthMonth: testDate.getMonth() + 1,
-    expectedBirthYear: testDate.getFullYear() - 12,
-  },
-  {
-    birthMonth: 9,
-    birthYear: null,
-    age: null,
-    ageMonths: null,
-    expectedBirthMonth: null,
-    expectedBirthYear: null,
-  },
-  {
-    birthMonth: null,
-    birthYear: null,
-    age: null,
-    ageMonths: null,
-    expectedBirthMonth: null,
-    expectedBirthYear: null,
-  },
-];
-
 test('Sets the correct age fields for all possible inputs', () => {
+  // Mock date to avoid timezone-related flakiness
+  jest.useFakeTimers();
+  jest.setSystemTime(new Date('2024-06-15T12:00:00Z'));
+  const testDate = new Date();
+
+  const agePossibilities = [
+    {
+      birthMonth: 8,
+      birthYear: 2009,
+      age: null,
+      ageMonths: null,
+      expectedBirthMonth: 8,
+      expectedBirthYear: 2009,
+    },
+    {
+      birthMonth: 5,
+      birthYear: 2000,
+      age: 23,
+      ageMonths: null,
+      expectedBirthMonth: 5,
+      expectedBirthYear: 2000,
+    },
+    {
+      birthMonth: null,
+      birthYear: 2007,
+      age: null,
+      ageMonths: null,
+      expectedBirthMonth: testDate.getMonth() + 1,
+      expectedBirthYear: 2007,
+    },
+    {
+      birthMonth: null,
+      birthYear: null,
+      age: null,
+      ageMonths: 254,
+      expectedBirthMonth: 12 + ((testDate.getMonth() + 1 - 254) % 12),
+      expectedBirthYear: testDate.getFullYear() - Math.floor(254 / 12),
+    },
+    {
+      birthMonth: null,
+      birthYear: null,
+      age: 12,
+      ageMonths: null,
+      expectedBirthMonth: testDate.getMonth() + 1,
+      expectedBirthYear: testDate.getFullYear() - 12,
+    },
+    {
+      birthMonth: 9,
+      birthYear: null,
+      age: null,
+      ageMonths: null,
+      expectedBirthMonth: null,
+      expectedBirthYear: null,
+    },
+    {
+      birthMonth: null,
+      birthYear: null,
+      age: null,
+      ageMonths: null,
+      expectedBirthMonth: null,
+      expectedBirthYear: null,
+    },
+  ];
   for (const poss of agePossibilities) {
     const ageData = getAgeData(poss.birthMonth, poss.birthYear, poss.age, poss.ageMonths);
 
@@ -321,7 +320,7 @@ test('Sets the correct age fields for all possible inputs', () => {
     expect(ageData.age).toBe(expectedAge);
     expect(ageData.ageMonths).toBe(expectedAgeMonths);
   }
-  
+
   // Restore real timers after test
   jest.useRealTimers();
 });
@@ -777,7 +776,7 @@ describe('ValidityEvaluator with Custom Rules', () => {
               },
               (data) => {
                 const counts = {};
-                data.responses.forEach(r => counts[r] = (counts[r] || 0) + 1);
+                data.responses.forEach((r) => counts[r] = (counts[r] || 0) + 1);
                 return Math.max(...Object.values(counts)) / data.responses.length > 0.9;
               },
             ],
@@ -846,7 +845,7 @@ describe('ValidityEvaluator with Custom Rules', () => {
             conditions: [
               (data) => {
                 const mean = data.responseTimes.reduce((a, b) => a + b, 0) / data.responseTimes.length;
-                const variance = data.responseTimes.reduce((sum, rt) => sum + Math.pow(rt - mean, 2), 0) / data.responseTimes.length;
+                const variance = data.responseTimes.reduce((sum, rt) => sum + (rt - mean) ** 2, 0) / data.responseTimes.length;
                 return variance > 500000;
               },
               (data) => Math.max(...data.responseTimes) > 5000,

From 2d54d3346544815c20f4e7099c0269283113395c Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Tue, 9 Jun 2026 14:44:32 -0700
Subject: [PATCH 11/16] Clarify jsDocs and add test case for missing flag in
 includedReliabilityFlags

---
 src/utils.js       |  2 +-
 test/utils.spec.js | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/utils.js b/src/utils.js
index 791eacc..aec80c7 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -336,7 +336,7 @@ export const median = (array) => {
  * @param {number} responseTimeHighThreshold - The maximum acceptable response time threshold in MS.
  * @param {number} accuracyThreshold - The minimum acceptable accuracy threshold.
  * @param {array} includedReliabilityFlags - An array of flags that should be included
- * when evaluating reliability.
+ * when evaluating reliability. **Note:** Custom flags will not be evaluated unless they are included in this array.
  * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags.
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
  *  - Custom conditions requires logicalOperation, conditions array (functions), and flag name
diff --git a/test/utils.spec.js b/test/utils.spec.js
index 7333e5f..47da4cf 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -966,4 +966,38 @@ describe('ValidityEvaluator with Custom Rules', () => {
     // Only custom flag returned, base flags excluded
     expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndInaccurate'], false);
   });
+
+  test('Custom validation flag not evaluated when not in includedReliabilityFlags', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        minResponsesRequired: 4,
+        customValidations: [
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => {
+                const uniqueResponses = new Set(data.responses).size;
+                return uniqueResponses <= 2;
+              },
+            ],
+            flag: 'fastAndRepetitive',
+          },
+        ],
+        // Custom flag NOT included in includedReliabilityFlags
+        includedReliabilityFlags: ['responseTimeTooFast'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    // Trigger both responseTimeTooFast AND fastAndRepetitive conditions
+    validityEval.addResponseData(300, 'left_arrow', 1);
+    validityEval.addResponseData(350, 'left_arrow', 1);
+    validityEval.addResponseData(320, 'right_arrow', 0);
+    validityEval.addResponseData(380, 'left_arrow', 1);
+
+    // Only responseTimeTooFast returned, fastAndRepetitive ignored
+    expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast'], false);
+  });
 });

From 1657abf3e8e7ecb91021c7ea27ca25ee537bb1e5 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Tue, 9 Jun 2026 15:05:06 -0700
Subject: [PATCH 12/16] Add xor and default logicalOperation value

---
 src/utils.js       |   8 ++-
 test/utils.spec.js | 122 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+), 2 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index aec80c7..d282d4a 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -339,7 +339,7 @@ export const median = (array) => {
  * when evaluating reliability. **Note:** Custom flags will not be evaluated unless they are included in this array.
  * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags.
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
- *  - Custom conditions requires logicalOperation, conditions array (functions), and flag name
+ *  - Custom conditions requires logicalOperation ('and', 'or', 'xor', default: 'and'), conditions array (functions), and flag name
  *  - Can return existing and/or custom flags depending on includedReliabilityFlags
  * @example
  * createEvaluateValidity({
@@ -402,13 +402,17 @@ export function createEvaluateValidity({
         };
 
         customValidations.forEach((rule) => {
-          const { logicalOperation, conditions, flag } = rule;
+          const { logicalOperation = 'and', conditions, flag } = rule;
           let conditionMet = false;
 
           if (logicalOperation.toLowerCase() === 'and') {
             conditionMet = conditions.every((condition) => condition(data));
           } else if (logicalOperation.toLowerCase() === 'or') {
             conditionMet = conditions.some((condition) => condition(data));
+          } else if (logicalOperation.toLowerCase() === 'xor') {
+            // XOR: exactly one condition must be true
+            const trueCount = conditions.filter((condition) => condition(data)).length;
+            conditionMet = trueCount === 1;
           }
 
           if (conditionMet) {
diff --git a/test/utils.spec.js b/test/utils.spec.js
index 47da4cf..702923f 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -1000,4 +1000,126 @@ describe('ValidityEvaluator with Custom Rules', () => {
     // Only responseTimeTooFast returned, fastAndRepetitive ignored
     expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast'], false);
   });
+
+  test('Custom rule with XOR logic triggers when exactly one condition is met', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
+        minResponsesRequired: 4,
+        customValidations: [
+          {
+            logicalOperation: 'xor',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => data.existingFlags.includes('accuracyTooLow'),
+            ],
+            flag: 'eitherFastOrInaccurate',
+          },
+        ],
+        includedReliabilityFlags: ['eitherFastOrInaccurate'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    // Fast but accurate - only one condition met
+    validityEval.addResponseData(300, 'left_arrow', 1);
+    validityEval.addResponseData(350, 'right_arrow', 1);
+    validityEval.addResponseData(320, 'left_arrow', 1);
+    validityEval.addResponseData(380, 'right_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['eitherFastOrInaccurate'], false);
+  });
+
+  test('Custom rule with XOR logic does not trigger when both conditions are met', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
+        minResponsesRequired: 4,
+        customValidations: [
+          {
+            logicalOperation: 'xor',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => data.existingFlags.includes('accuracyTooLow'),
+            ],
+            flag: 'eitherFastOrInaccurate',
+          },
+        ],
+        includedReliabilityFlags: ['eitherFastOrInaccurate'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    // Fast AND inaccurate - both conditions met, XOR should NOT trigger
+    validityEval.addResponseData(300, 'left_arrow', 0);
+    validityEval.addResponseData(350, 'right_arrow', 0);
+    validityEval.addResponseData(320, 'left_arrow', 0);
+    validityEval.addResponseData(380, 'right_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith([], true);
+  });
+
+  test('Custom rule with XOR logic does not trigger when no conditions are met', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
+        minResponsesRequired: 4,
+        customValidations: [
+          {
+            logicalOperation: 'xor',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => data.existingFlags.includes('accuracyTooLow'),
+            ],
+            flag: 'eitherFastOrInaccurate',
+          },
+        ],
+        includedReliabilityFlags: ['eitherFastOrInaccurate'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    // Good performance - neither condition met
+    validityEval.addResponseData(600, 'left_arrow', 1);
+    validityEval.addResponseData(650, 'right_arrow', 1);
+    validityEval.addResponseData(700, 'left_arrow', 1);
+    validityEval.addResponseData(750, 'right_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith([], true);
+  });
+
+  test('Custom rule defaults to AND logic when logicalOperation is not specified', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        minResponsesRequired: 4,
+        customValidations: [
+          {
+            // logicalOperation not specified - should default to 'and'
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => {
+                const uniqueResponses = new Set(data.responses).size;
+                return uniqueResponses <= 2;
+              },
+            ],
+            flag: 'fastAndRepetitive',
+          },
+        ],
+        includedReliabilityFlags: ['fastAndRepetitive'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    // Both conditions met - should trigger with default AND logic
+    validityEval.addResponseData(300, 'left_arrow', 1);
+    validityEval.addResponseData(350, 'left_arrow', 1);
+    validityEval.addResponseData(320, 'right_arrow', 0);
+    validityEval.addResponseData(380, 'left_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndRepetitive'], false);
+  });
 });

From 198307a300c8519092c2c376804f31979582c158 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Tue, 9 Jun 2026 15:26:45 -0700
Subject: [PATCH 13/16] Add tests that verify order of includedReliabilityFlags
 does not affect custom validations

---
 test/utils.spec.js | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/test/utils.spec.js b/test/utils.spec.js
index 702923f..0288ad8 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -1056,7 +1056,8 @@ describe('ValidityEvaluator with Custom Rules', () => {
     validityEval.addResponseData(300, 'left_arrow', 0);
     validityEval.addResponseData(350, 'right_arrow', 0);
     validityEval.addResponseData(320, 'left_arrow', 0);
-    validityEval.addResponseData(380, 'right_arrow', 1);
+    validityEval.addResponseData(380, 'right_arrow', 0);
+    validityEval.addResponseData(340, 'left_arrow', 1);
 
     expect(testAddFlags).toHaveBeenLastCalledWith([], true);
   });
@@ -1122,4 +1123,35 @@ describe('ValidityEvaluator with Custom Rules', () => {
 
     expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndRepetitive'], false);
   });
+
+  test('includedReliabilityFlags order does not affect custom validation - base flag last', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
+        minResponsesRequired: 4,
+        customValidations: [
+          {
+            logicalOperation: 'and',
+            conditions: [
+              (data) => data.existingFlags.includes('responseTimeTooFast'),
+              (data) => data.existingFlags.includes('accuracyTooLow'),
+            ],
+            flag: 'fastAndInaccurate',
+          },
+        ],
+        // Custom flag listed BEFORE base flags
+        includedReliabilityFlags: ['fastAndInaccurate', 'responseTimeTooFast', 'accuracyTooLow'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(300, 'left_arrow', 0);
+    validityEval.addResponseData(350, 'right_arrow', 0);
+    validityEval.addResponseData(320, 'left_arrow', 0);
+    validityEval.addResponseData(380, 'right_arrow', 1);
+
+    // Same flags returned, order in includedReliabilityFlags doesn't matter
+    expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'accuracyTooLow', 'fastAndInaccurate'], false);
+  });
 });

From 2492863945c7e709cdd17f15619f42b381679160 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Tue, 9 Jun 2026 17:13:27 -0700
Subject: [PATCH 14/16] Delete customValidation existingFlags logic

---
 src/utils.js       | 44 +++++++++++++++++++-------------------------
 test/utils.spec.js |  4 +++-
 2 files changed, 22 insertions(+), 26 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index 1d3edcd..d72e2db 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -342,6 +342,7 @@ export const median = (array) => {
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
  *  - Custom conditions requires logicalOperation ('and', 'or', 'xor', default: 'and'), conditions array (functions), and flag name
  *  - Can return existing and/or custom flags depending on includedReliabilityFlags
+ *  - **Note:** Custom validation flags cannot depend on each other. Conditions may only reference built-in flags via existingFlags.
  * @example
  * createEvaluateValidity({
  *   customValidations: [{
@@ -394,36 +395,29 @@ export function createEvaluateValidity({
         flags.push('accuracyTooLow');
       }
     }
+    
+    // Evaluate custom validations. Only rules whose flag is in includedReliabilityFlags are evaluated.
+    // Note: conditions may only reference built-in flags via existingFlags, not other custom flags.
+    for (const { flag, conditions, logicalOperation = 'and' } of customValidations) {
+      if (!includedReliabilityFlags.includes(flag)) continue;
+
+      const data = { responseTimes, responses, correct, completed, existingFlags: [...flags] };
+
+      let triggered;
+      if (logicalOperation === 'or') {
+        triggered = conditions.some((c) => c(data));
+      } else if (logicalOperation === 'xor') {
+        triggered = conditions.filter((c) => c(data)).length === 1;
+      } else {
+        triggered = conditions.every((c) => c(data));
+      }
 
-    // Evaluate custom validations alongside default checks
-    if (customValidations.length > 0) {
-      const data = {
-        responseTimes, responses, correct, completed, existingFlags: [...flags],
-      };
-
-      customValidations.forEach((rule) => {
-        const { logicalOperation = 'and', conditions, flag } = rule;
-        let conditionMet = false;
-
-        if (logicalOperation.toLowerCase() === 'and') {
-          conditionMet = conditions.every((condition) => condition(data));
-        } else if (logicalOperation.toLowerCase() === 'or') {
-          conditionMet = conditions.some((condition) => condition(data));
-        } else if (logicalOperation.toLowerCase() === 'xor') {
-          // XOR: exactly one condition must be true
-          const trueCount = conditions.filter((condition) => condition(data)).length;
-          conditionMet = trueCount === 1;
-        }
-
-        if (conditionMet) {
-          flags.push(flag);
-        }
-      });
+      if (triggered) flags.push(flag);
     }
 
     isReliable = flags.filter((x) => includedReliabilityFlags.includes(x)).length === 0;
     flags = flags.filter((x) => includedReliabilityFlags.includes(x));
-    
+
     return { flags, isReliable };
   };
 }
diff --git a/test/utils.spec.js b/test/utils.spec.js
index e477b4f..25bce65 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -1196,9 +1196,11 @@ describe('ValidityEvaluator with Custom Rules', () => {
     validityEval.addResponseData(300, 'left_arrow', 0);
     validityEval.addResponseData(350, 'right_arrow', 0);
     validityEval.addResponseData(320, 'left_arrow', 0);
-    validityEval.addResponseData(380, 'right_arrow', 1);
+    validityEval.addResponseData(380, 'right_arrow', 0);
+    validityEval.addResponseData(340, 'left_arrow', 1);
 
     // Same flags returned, order in includedReliabilityFlags doesn't matter
     expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'accuracyTooLow', 'fastAndInaccurate'], false);
   });
+
 });

From 26a71f936b0f3bdebf80d256cc3ebc4567eaf573 Mon Sep 17 00:00:00 2001
From: Amy Yeung <ayeung21@stanford.edu>
Date: Tue, 9 Jun 2026 17:41:18 -0700
Subject: [PATCH 15/16] Allow single custom validation rule

---
 src/utils.js       | 29 ++++++++++++++--
 test/utils.spec.js | 86 +++++++++++++++++++++++++++++++++++-----------
 2 files changed, 91 insertions(+), 24 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index d72e2db..ccc291d 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -340,7 +340,7 @@ export const median = (array) => {
  * when evaluating reliability. **Note:** Custom flags will not be evaluated unless they are included in this array.
  * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags.
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
- *  - Custom conditions requires logicalOperation ('and', 'or', 'xor', default: 'and'), conditions array (functions), and flag name
+ *  - Custom conditions requires conditions array (functions) and flag name. logicalOperation ('and', 'or', 'xor') is required when conditions has more than one entry; ignored when conditions has only one entry
  *  - Can return existing and/or custom flags depending on includedReliabilityFlags
  *  - **Note:** Custom validation flags cannot depend on each other. Conditions may only reference built-in flags via existingFlags.
  * @example
@@ -369,6 +369,27 @@ export function createEvaluateValidity({
   includedReliabilityFlags = ['responseTimeTooFast'],
   customValidations = [],
 }) {
+  const validLogicalOperations = ['and', 'or', 'xor'];
+  const normalizedCustomValidations = customValidations.map((v) => ({
+    ...v,
+    logicalOperation: v.logicalOperation?.toLowerCase(),
+  }));
+
+  for (const { flag, conditions, logicalOperation } of normalizedCustomValidations) {
+    if (conditions.length > 1) {
+      if (logicalOperation === undefined) {
+        throw new Error(
+          `logicalOperation is required for custom validation flag "${flag}" when multiple conditions are provided. Must be one of: ${validLogicalOperations.join(', ')}.`,
+        );
+      }
+      if (!validLogicalOperations.includes(logicalOperation)) {
+        throw new Error(
+          `Invalid logicalOperation "${logicalOperation}" for custom validation flag "${flag}". Must be one of: ${validLogicalOperations.join(', ')}.`,
+        );
+      }
+    }
+  }
+
   return function baseEvaluateValidity({ responseTimes, responses, correct, completed }) {
     let flags = [];
     let isReliable = false;
@@ -398,13 +419,15 @@ export function createEvaluateValidity({
     
     // Evaluate custom validations. Only rules whose flag is in includedReliabilityFlags are evaluated.
     // Note: conditions may only reference built-in flags via existingFlags, not other custom flags.
-    for (const { flag, conditions, logicalOperation = 'and' } of customValidations) {
+    for (const { flag, conditions, logicalOperation } of normalizedCustomValidations) {
       if (!includedReliabilityFlags.includes(flag)) continue;
 
       const data = { responseTimes, responses, correct, completed, existingFlags: [...flags] };
 
       let triggered;
-      if (logicalOperation === 'or') {
+      if (conditions.length === 1) {
+        triggered = conditions[0](data);
+      } else if (logicalOperation === 'or') {
         triggered = conditions.some((c) => c(data));
       } else if (logicalOperation === 'xor') {
         triggered = conditions.filter((c) => c(data)).length === 1;
diff --git a/test/utils.spec.js b/test/utils.spec.js
index 25bce65..8732d93 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -775,6 +775,39 @@ describe('ValidityEval test for 2 block based assessments (e.g. PA-es)', () => {
 describe('ValidityEvaluator with Custom Rules', () => {
   let validityEval;
 
+  test('Throws at setup time when logicalOperation is invalid', () => {
+    expect(() => {
+      createEvaluateValidity({
+        customValidations: [{ flag: 'myFlag', logicalOperation: 'nand', conditions: [() => true, () => true] }],
+        includedReliabilityFlags: ['myFlag'],
+      });
+    }).toThrow('Invalid logicalOperation "nand"');
+  });
+
+  test('Single condition rule triggers without logicalOperation', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        responseTimeLowThreshold: 400,
+        minResponsesRequired: 4,
+        customValidations: [
+          {
+            conditions: [(data) => data.existingFlags.includes('responseTimeTooFast')],
+            flag: 'fastRun',
+          },
+        ],
+        includedReliabilityFlags: ['fastRun'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(300, 'left_arrow', 1);
+    validityEval.addResponseData(350, 'right_arrow', 1);
+    validityEval.addResponseData(320, 'left_arrow', 1);
+    validityEval.addResponseData(380, 'right_arrow', 1);
+
+    expect(testAddFlags).toHaveBeenLastCalledWith(['fastRun'], false);
+  });
+
   test('Custom rule with AND logic using existing flag and custom condition', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
@@ -1139,39 +1172,53 @@ describe('ValidityEvaluator with Custom Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith([], true);
   });
 
-  test('Custom rule defaults to AND logic when logicalOperation is not specified', () => {
+  test('Throws at setup time when logicalOperation is missing for multiple conditions', () => {
+    expect(() => {
+      createEvaluateValidity({
+        customValidations: [
+          {
+            conditions: [(data) => data.existingFlags.includes('responseTimeTooFast'), () => true],
+            flag: 'fastAndRepetitive',
+          },
+        ],
+        includedReliabilityFlags: ['fastAndRepetitive'],
+      });
+    }).toThrow('logicalOperation is required');
+  });
+
+  test('includedReliabilityFlags order does not affect custom validation - base flag last', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
+        accuracyThreshold: 0.2,
         minResponsesRequired: 4,
         customValidations: [
           {
-            // logicalOperation not specified - should default to 'and'
+            logicalOperation: 'and',
             conditions: [
               (data) => data.existingFlags.includes('responseTimeTooFast'),
-              (data) => {
-                const uniqueResponses = new Set(data.responses).size;
-                return uniqueResponses <= 2;
-              },
+              (data) => data.existingFlags.includes('accuracyTooLow'),
             ],
-            flag: 'fastAndRepetitive',
+            flag: 'fastAndInaccurate',
           },
         ],
-        includedReliabilityFlags: ['fastAndRepetitive'],
+        // Custom flag listed BEFORE base flags
+        includedReliabilityFlags: ['fastAndInaccurate', 'responseTimeTooFast', 'accuracyTooLow'],
       }),
       handleEngagementFlags: testAddFlags,
     });
 
-    // Both conditions met - should trigger with default AND logic
-    validityEval.addResponseData(300, 'left_arrow', 1);
-    validityEval.addResponseData(350, 'left_arrow', 1);
-    validityEval.addResponseData(320, 'right_arrow', 0);
-    validityEval.addResponseData(380, 'left_arrow', 1);
+    validityEval.addResponseData(300, 'left_arrow', 0);
+    validityEval.addResponseData(350, 'right_arrow', 0);
+    validityEval.addResponseData(320, 'left_arrow', 0);
+    validityEval.addResponseData(380, 'right_arrow', 0);
+    validityEval.addResponseData(340, 'left_arrow', 1);
 
-    expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndRepetitive'], false);
+    // Same flags returned, order in includedReliabilityFlags doesn't matter
+    expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'accuracyTooLow', 'fastAndInaccurate'], false);
   });
 
-  test('includedReliabilityFlags order does not affect custom validation - base flag last', () => {
+  test('Capitalized logicalOperation is normalized and works correctly', () => {
     validityEval = new ValidityEvaluator({
       evaluateValidity: createEvaluateValidity({
         responseTimeLowThreshold: 400,
@@ -1179,7 +1226,7 @@ describe('ValidityEvaluator with Custom Rules', () => {
         minResponsesRequired: 4,
         customValidations: [
           {
-            logicalOperation: 'and',
+            logicalOperation: 'AND',
             conditions: [
               (data) => data.existingFlags.includes('responseTimeTooFast'),
               (data) => data.existingFlags.includes('accuracyTooLow'),
@@ -1187,8 +1234,7 @@ describe('ValidityEvaluator with Custom Rules', () => {
             flag: 'fastAndInaccurate',
           },
         ],
-        // Custom flag listed BEFORE base flags
-        includedReliabilityFlags: ['fastAndInaccurate', 'responseTimeTooFast', 'accuracyTooLow'],
+        includedReliabilityFlags: ['fastAndInaccurate'],
       }),
       handleEngagementFlags: testAddFlags,
     });
@@ -1197,10 +1243,8 @@ describe('ValidityEvaluator with Custom Rules', () => {
     validityEval.addResponseData(350, 'right_arrow', 0);
     validityEval.addResponseData(320, 'left_arrow', 0);
     validityEval.addResponseData(380, 'right_arrow', 0);
-    validityEval.addResponseData(340, 'left_arrow', 1);
 
-    // Same flags returned, order in includedReliabilityFlags doesn't matter
-    expect(testAddFlags).toHaveBeenLastCalledWith(['responseTimeTooFast', 'accuracyTooLow', 'fastAndInaccurate'], false);
+    expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndInaccurate'], false);
   });
 
 });

From a3a957fa49487b0e201dffc72c3d668998f64c6b Mon Sep 17 00:00:00 2001
From: Adam <adamrh@stanford.edu>
Date: Wed, 10 Jun 2026 17:01:30 +0000
Subject: [PATCH 16/16] Address review nits for custom validations

- Snapshot built-in flags once before the custom-validation loop so a custom
  rule cannot observe another custom rule's flag (enforces the documented
  independence contract)
- Only evaluate custom validations when responseTimes.length >=
  minResponsesRequired, so they are skipped on insufficient-response runs
- Clarify JSDoc: logicalOperation semantics (and=all, or=any, xor=exactly one),
  note the min-responses gating, and give the example reduce an initial value
- Add tests for custom-flag independence and skip-on-insufficient-responses
---
 src/utils.js       | 45 ++++++++++++++++++++++++++-------------------
 test/utils.spec.js | 45 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 19 deletions(-)

diff --git a/src/utils.js b/src/utils.js
index ccc291d..28fed08 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -341,7 +341,9 @@ export const median = (array) => {
  * @param {array} customValidations - An array of custom validation rules to evaluate. Can pass in custom conditions or use pre-existing flags.
  *  - Condition arguments: responseTimes, responses, correct, completed, existingFlags
  *  - Custom conditions requires conditions array (functions) and flag name. logicalOperation ('and', 'or', 'xor') is required when conditions has more than one entry; ignored when conditions has only one entry
+ *  - logicalOperation semantics: 'and' = all conditions true, 'or' = any condition true, 'xor' = exactly one condition true
  *  - Can return existing and/or custom flags depending on includedReliabilityFlags
+ *  - Custom validations are only evaluated when the run has at least minResponsesRequired responses (i.e. it is not flagged 'notEnoughResponses')
  *  - **Note:** Custom validation flags cannot depend on each other. Conditions may only reference built-in flags via existingFlags.
  * @example
  * createEvaluateValidity({
@@ -350,7 +352,7 @@ export const median = (array) => {
  *     conditions: [
  *       (data) => data.existingFlags.includes('responseTimeTooFast'),
  *       (data) => {
- *         const mean = data.responseTimes.reduce((a, b) => a + b) / data.responseTimes.length;
+ *         const mean = data.responseTimes.reduce((a, b) => a + b, 0) / data.responseTimes.length;
  *         const variance = data.responseTimes.reduce((sum, rt) => sum + Math.pow(rt - mean, 2), 0) / data.responseTimes.length;
  *         return variance > 1000000;
  *       }
@@ -417,25 +419,30 @@ export function createEvaluateValidity({
       }
     }
     
-    // Evaluate custom validations. Only rules whose flag is in includedReliabilityFlags are evaluated.
-    // Note: conditions may only reference built-in flags via existingFlags, not other custom flags.
-    for (const { flag, conditions, logicalOperation } of normalizedCustomValidations) {
-      if (!includedReliabilityFlags.includes(flag)) continue;
-
-      const data = { responseTimes, responses, correct, completed, existingFlags: [...flags] };
-
-      let triggered;
-      if (conditions.length === 1) {
-        triggered = conditions[0](data);
-      } else if (logicalOperation === 'or') {
-        triggered = conditions.some((c) => c(data));
-      } else if (logicalOperation === 'xor') {
-        triggered = conditions.filter((c) => c(data)).length === 1;
-      } else {
-        triggered = conditions.every((c) => c(data));
+    // Evaluate custom validations. Only rules whose flag is in includedReliabilityFlags are evaluated,
+    // and only when there are enough responses (mirroring the built-in checks above).
+    // Snapshot the built-in flags once, before any custom flag is added, so custom rules stay independent:
+    // a condition may reference built-in flags via existingFlags but never another custom rule's flag.
+    if (responseTimes.length >= minResponsesRequired) {
+      const builtInFlags = [...flags];
+      for (const { flag, conditions, logicalOperation } of normalizedCustomValidations) {
+        if (!includedReliabilityFlags.includes(flag)) continue;
+
+        const data = { responseTimes, responses, correct, completed, existingFlags: [...builtInFlags] };
+
+        let triggered;
+        if (conditions.length === 1) {
+          triggered = conditions[0](data);
+        } else if (logicalOperation === 'or') {
+          triggered = conditions.some((c) => c(data));
+        } else if (logicalOperation === 'xor') {
+          triggered = conditions.filter((c) => c(data)).length === 1;
+        } else {
+          triggered = conditions.every((c) => c(data));
+        }
+
+        if (triggered) flags.push(flag);
       }
-
-      if (triggered) flags.push(flag);
     }
 
     isReliable = flags.filter((x) => includedReliabilityFlags.includes(x)).length === 0;
diff --git a/test/utils.spec.js b/test/utils.spec.js
index 8732d93..5b55231 100644
--- a/test/utils.spec.js
+++ b/test/utils.spec.js
@@ -1247,4 +1247,49 @@ describe('ValidityEvaluator with Custom Rules', () => {
     expect(testAddFlags).toHaveBeenLastCalledWith(['fastAndInaccurate'], false);
   });
 
+  test('Custom validation flags are independent - a rule cannot observe another custom flag', () => {
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        minResponsesRequired: 4,
+        customValidations: [
+          // Always fires
+          { conditions: [() => true], flag: 'flagA' },
+          // Tries to depend on the earlier custom flag; existingFlags must only contain built-in flags
+          { conditions: [(data) => data.existingFlags.includes('flagA')], flag: 'flagB' },
+        ],
+        includedReliabilityFlags: ['flagA', 'flagB'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    // Response times high / all correct, so no built-in flags fire
+    validityEval.addResponseData(600, 'left_arrow', 1);
+    validityEval.addResponseData(650, 'right_arrow', 1);
+    validityEval.addResponseData(620, 'left_arrow', 1);
+    validityEval.addResponseData(580, 'right_arrow', 1);
+
+    // flagA fires; flagB must NOT, because conditions only see built-in flags
+    expect(testAddFlags).toHaveBeenLastCalledWith(['flagA'], false);
+  });
+
+  test('Custom validations are not evaluated when there are not enough responses', () => {
+    const condition = jest.fn(() => true);
+    validityEval = new ValidityEvaluator({
+      evaluateValidity: createEvaluateValidity({
+        minResponsesRequired: 10,
+        customValidations: [{ conditions: [condition], flag: 'alwaysFlag' }],
+        includedReliabilityFlags: ['notEnoughResponses', 'alwaysFlag'],
+      }),
+      handleEngagementFlags: testAddFlags,
+    });
+
+    validityEval.addResponseData(600, 'left_arrow', 1);
+    validityEval.addResponseData(650, 'right_arrow', 1);
+    validityEval.addResponseData(620, 'left_arrow', 1);
+
+    // Too few responses: only notEnoughResponses, the custom rule is skipped entirely
+    expect(testAddFlags).toHaveBeenLastCalledWith(['notEnoughResponses'], false);
+    expect(condition).not.toHaveBeenCalled();
+  });
+
 });