From 753c117f38ed8e5536fb4987420314381e8a7509 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 21:47:59 +0000 Subject: [PATCH 01/11] Refactor ability estimators and item selectors into registry pattern Estimators now implement the AbilityEstimator interface in src/estimators/ and item selectors implement ItemSelector in src/selectors/. Each family is registered in a registry that serves as the single source of truth for the method type unions, runtime validation, and dispatch. Cat delegates to the registries, so adding an algorithm no longer requires editing Cat. Also fixes a latent bug: the private likelihood method seeded its reduce with 1 instead of 0, returning ln L + 1. The shared logLikelihood helper seeds at 0. (No observable behavior change: the offset cancelled in both the MLE argmax and the EAP posterior ratio.) --- src/cat.ts | 209 ++++++++----------------------- src/estimators/eap.ts | 37 ++++++ src/estimators/index.ts | 12 ++ src/estimators/log-likelihood.ts | 23 ++++ src/estimators/mle.ts | 28 +++++ src/estimators/optimize.ts | 28 +++++ src/estimators/registry.ts | 54 ++++++++ src/estimators/types.ts | 43 +++++++ src/index.ts | 25 ++++ src/selectors/closest.ts | 43 +++++++ src/selectors/fixed.ts | 26 ++++ src/selectors/index.ts | 19 +++ src/selectors/mfi.ts | 43 +++++++ src/selectors/middle.ts | 35 ++++++ src/selectors/random.ts | 28 +++++ src/selectors/registry.ts | 93 ++++++++++++++ src/selectors/types.ts | 49 ++++++++ 17 files changed, 641 insertions(+), 154 deletions(-) create mode 100644 src/estimators/eap.ts create mode 100644 src/estimators/index.ts create mode 100644 src/estimators/log-likelihood.ts create mode 100644 src/estimators/mle.ts create mode 100644 src/estimators/optimize.ts create mode 100644 src/estimators/registry.ts create mode 100644 src/estimators/types.ts create mode 100644 src/selectors/closest.ts create mode 100644 src/selectors/fixed.ts create mode 100644 src/selectors/index.ts create mode 100644 src/selectors/mfi.ts create mode 100644 src/selectors/middle.ts create mode 100644 src/selectors/random.ts create mode 100644 src/selectors/registry.ts create mode 100644 src/selectors/types.ts diff --git a/src/cat.ts b/src/cat.ts index 7f2a31c..f4be6a0 100644 --- a/src/cat.ts +++ b/src/cat.ts @@ -1,17 +1,27 @@ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -import { minimize_Powell } from 'optimization-js'; import { Stimulus, Zeta } from './type'; -import { itemResponseFunction, fisherInformation, normal, uniform, findClosest } from './utils'; +import { fisherInformation, normal, uniform } from './utils'; import { validateZetaParams, fillZetaDefaults, ensureZetaNumericValues } from './corpus'; +import { EstimationMethod, EstimationMethodInput, getEstimator, validateEstimationMethod } from './estimators'; +import { + ItemSelectMethod, + ItemSelectMethodInput, + SelectorContext, + SelectorMethod, + StartSelectMethod, + StartSelectMethodInput, + getSelector, + validateItemSelect, + validateStartSelect, +} from './selectors'; import seedrandom from 'seedrandom'; import _clamp from 'lodash/clamp'; import _cloneDeep from 'lodash/cloneDeep'; export interface CatInput { - method?: string; - itemSelect?: string; + method?: EstimationMethodInput; + itemSelect?: ItemSelectMethodInput; nStartItems?: number; - startSelect?: string; + startSelect?: StartSelectMethodInput; theta?: number; minTheta?: number; maxTheta?: number; @@ -21,8 +31,8 @@ export interface CatInput { } export class Cat { - public method: string; - public itemSelect: string; + public method: EstimationMethod; + public itemSelect: ItemSelectMethod; public minTheta: number; public maxTheta: number; public priorDist: string; @@ -32,14 +42,14 @@ export class Cat { private _theta: number; private _seMeasurement: number; public nStartItems: number; - public startSelect: string; + public startSelect: StartSelectMethod; private readonly _rng: ReturnType; private _prior: [number, number][]; /** * Create a Cat object. This expects an single object parameter with the following keys * @param {{method: string, itemSelect: string, nStartItems: number, startSelect:string, theta: number, minTheta: number, maxTheta: number, priorDist: string, priorPar: number[]}=} destructuredParam - * method: ability estimator, e.g. MLE or EAP, default = 'MLE' + * method: ability estimator, e.g. MLE or EAP, default = 'MLE' (see src/estimators/registry.ts for the full list) * itemSelect: the method of item selection, e.g. "MFI", "random", "closest", default method = 'MFI' * nStartItems: first n trials to keep non-adaptive selection * startSelect: rule to select first n trials @@ -52,8 +62,8 @@ export class Cat { */ constructor({ - method = 'MLE', - itemSelect = 'MFI', + method = 'mle', + itemSelect = 'mfi', nStartItems = 0, startSelect = 'middle', theta = 0, @@ -144,41 +154,30 @@ export class Cat { throw new Error(`priorDist must be "unif" or "norm." Received ${priorDist} instead.`); } - private static validateMethod(method: string) { - const lowerMethod = method.toLowerCase(); - const validMethods: Array = ['mle', 'eap']; // TO DO: add staircase - if (!validMethods.includes(lowerMethod)) { - throw new Error('The abilityEstimator you provided is not in the list of valid methods'); - } - return lowerMethod; + private static validateMethod(method: string): EstimationMethod { + return validateEstimationMethod(method); } - private static validateItemSelect(itemSelect: string) { - const lowerItemSelect = itemSelect.toLowerCase(); - const validItemSelect: Array = ['mfi', 'random', 'closest', 'fixed']; - if (!validItemSelect.includes(lowerItemSelect)) { - throw new Error('The itemSelector you provided is not in the list of valid methods'); - } - return lowerItemSelect; + private static validateItemSelect(itemSelect: string): ItemSelectMethod { + return validateItemSelect(itemSelect); } - private static validateStartSelect(startSelect: string) { - const lowerStartSelect = startSelect.toLowerCase(); - const validStartSelect: Array = ['random', 'middle', 'fixed']; // TO DO: add staircase - if (!validStartSelect.includes(lowerStartSelect)) { - throw new Error('The startSelect you provided is not in the list of valid methods'); - } - return lowerStartSelect; + private static validateStartSelect(startSelect: string): StartSelectMethod { + return validateStartSelect(startSelect); } /** * use previous response patterns and item params to calculate the estimate ability based on a defined method * @param zeta - last item param * @param answer - last response pattern - * @param method + * @param method - the estimation method to use; defaults to the Cat's configured method */ - public updateAbilityEstimate(zeta: Zeta | Zeta[], answer: (0 | 1) | (0 | 1)[], method: string = this.method) { - method = Cat.validateMethod(method); + public updateAbilityEstimate( + zeta: Zeta | Zeta[], + answer: (0 | 1) | (0 | 1)[], + method: EstimationMethodInput = this.method, + ) { + const validatedMethod = Cat.validateMethod(method); zeta = Array.isArray(zeta) ? zeta : [zeta]; answer = Array.isArray(answer) ? answer : [answer]; @@ -193,43 +192,18 @@ export class Cat { this._zetas.push(...zeta); this._resps.push(...answer); - if (method === 'eap') { - this._theta = this.estimateAbilityEAP(); - } else if (method === 'mle') { - this._theta = this.estimateAbilityMLE(); - } - this._theta = _clamp(this._theta, this.minTheta, this.maxTheta); - this.calculateSE(); - } - - private estimateAbilityEAP() { - let num = 0; - let nf = 0; - this._prior.forEach(([theta, probability]) => { - const like = Math.exp(this.likelihood(theta)); // Convert back to probability - num += theta * like * probability; - nf += like * probability; + // All estimators are dispatched through the registry. To add a new + // estimator, see src/estimators/registry.ts — no changes are needed here. + this._theta = getEstimator(validatedMethod).estimateAbility({ + zetas: this._zetas, + resps: this._resps, + minTheta: this.minTheta, + maxTheta: this.maxTheta, + prior: this._prior, }); - return num / nf; - } - - private estimateAbilityMLE() { - const theta0 = [0]; - const solution = minimize_Powell(this.negLikelihood.bind(this), theta0); - const theta = solution.argument[0]; - return theta; - } - - private negLikelihood(thetaArray: Array) { - return -this.likelihood(thetaArray[0]); - } - - private likelihood(theta: number) { - return this._zetas.reduce((acc, zeta, i) => { - const irf = itemResponseFunction(theta, zeta); - return this._resps[i] === 1 ? acc + Math.log(irf) : acc + Math.log(1 - irf); - }, 1); + this._theta = _clamp(this._theta, this.minTheta, this.maxTheta); + this.calculateSE(); } /** @@ -249,9 +223,9 @@ export class Cat { * @param deepCopy - default deepCopy = true * @returns {nextStimulus: Stimulus, remainingStimuli: Array} */ - public findNextItem(stimuli: Stimulus[], itemSelect: string = this.itemSelect, deepCopy = true) { + public findNextItem(stimuli: Stimulus[], itemSelect: ItemSelectMethodInput = this.itemSelect, deepCopy = true) { let arr: Array; - let selector = Cat.validateItemSelect(itemSelect); + let selector: SelectorMethod = Cat.validateItemSelect(itemSelect); if (deepCopy) { arr = _cloneDeep(stimuli); } else { @@ -267,91 +241,18 @@ export class Cat { // for mfi, we sort the arr by fisher information in the private function to select the best item, // and then sort by difficulty to return the remainingStimuli // for fixed, we want to keep the corpus order as input + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion arr.sort((a: Stimulus, b: Stimulus) => a.difficulty! - b.difficulty!); } - if (selector === 'middle') { - // middle will only be used in startSelect - return this.selectorMiddle(arr); - } else if (selector === 'closest') { - return this.selectorClosest(arr); - } else if (selector === 'random') { - return this.selectorRandom(arr); - } else if (selector === 'fixed') { - return this.selectorFixed(arr); - } else { - return this.selectorMFI(arr); - } - } - - private selectorMFI(inputStimuli: Stimulus[]) { - const stimuli = inputStimuli.map((stim) => fillZetaDefaults(stim, 'semantic')); - const stimuliAddFisher = stimuli.map((element: Stimulus) => ({ - fisherInformation: fisherInformation(this._theta, fillZetaDefaults(element, 'symbolic')), - ...element, - })); - - stimuliAddFisher.sort((a, b) => b.fisherInformation - a.fisherInformation); - stimuliAddFisher.forEach((stimulus: Stimulus) => { - delete stimulus['fisherInformation']; - }); - return { - nextStimulus: stimuliAddFisher[0], - remainingStimuli: stimuliAddFisher.slice(1).sort((a: Stimulus, b: Stimulus) => a.difficulty! - b.difficulty!), - }; - } - - private selectorMiddle(arr: Stimulus[]) { - let index: number; - index = Math.floor(arr.length / 2); - - if (arr.length >= this.nStartItems) { - index += this.randomInteger(-Math.floor(this.nStartItems / 2), Math.floor(this.nStartItems / 2)); - } - - const nextItem = arr[index]; - arr.splice(index, 1); - return { - nextStimulus: nextItem, - remainingStimuli: arr, - }; - } - - private selectorClosest(arr: Stimulus[]) { - //findClosest requires arr is sorted by difficulty - const index = findClosest(arr, this._theta + 0.481); - const nextItem = arr[index]; - arr.splice(index, 1); - return { - nextStimulus: nextItem, - remainingStimuli: arr, - }; - } - - private selectorRandom(arr: Stimulus[]) { - const index = this.randomInteger(0, arr.length - 1); - const nextItem = arr.splice(index, 1)[0]; - return { - nextStimulus: nextItem, - remainingStimuli: arr, - }; - } - - /** - * Picks the next item in line from the given list of stimuli. - * It grabs the first item from the list, removes it, and then returns it along with the rest of the list. - * - * @param arr - The list of stimuli to choose from. - * @returns {Object} - An object with the next item and the updated list. - * @returns {Stimulus} return.nextStimulus - The item that was picked from the list. - * @returns {Stimulus[]} return.remainingStimuli - The list of what's left after picking the item. - */ - private selectorFixed(arr: Stimulus[]) { - const nextItem = arr.shift(); - return { - nextStimulus: nextItem, - remainingStimuli: arr, + // All selectors are dispatched through the registry. To add a new + // selector, see src/selectors/registry.ts — no changes are needed here. + const context: SelectorContext = { + theta: this._theta, + nStartItems: this.nStartItems, + randomInteger: this.randomInteger.bind(this), }; + return getSelector(selector).select(arr, context); } /** diff --git a/src/estimators/eap.ts b/src/estimators/eap.ts new file mode 100644 index 0000000..3c48923 --- /dev/null +++ b/src/estimators/eap.ts @@ -0,0 +1,37 @@ +import { AbilityEstimator, EstimationContext } from './types'; +import { logLikelihood } from './log-likelihood'; + +/** + * Expected A Posteriori (EAP) estimation of ability. + * + * @remarks + * Computes the mean of the posterior distribution of theta over a quantized + * prior (`context.prior`), where the posterior is proportional to + * L(theta) * prior(theta). The prior is constructed and validated by `Cat` + * from the `priorDist` and `priorPar` constructor options. + * + * Reference: Bock, R. D., & Mislevy, R. J. (1982). Adaptive EAP estimation of + * ability in a microcomputer environment. Applied Psychological Measurement, + * 6(4), 431-444. + */ +export class EAPEstimator implements AbilityEstimator { + /** + * Estimate ability as the posterior mean over the quantized prior. + * + * @param {EstimationContext} context - The accumulated response history and prior + * @returns {number} the EAP theta estimate (unclamped) + */ + estimateAbility(context: EstimationContext): number { + const { zetas, resps, prior } = context; + let numerator = 0; + let normalizingFactor = 0; + + for (const [theta, probability] of prior) { + const likelihood = Math.exp(logLikelihood(theta, zetas, resps)); + numerator += theta * likelihood * probability; + normalizingFactor += likelihood * probability; + } + + return numerator / normalizingFactor; + } +} diff --git a/src/estimators/index.ts b/src/estimators/index.ts new file mode 100644 index 0000000..034ddd2 --- /dev/null +++ b/src/estimators/index.ts @@ -0,0 +1,12 @@ +export { AbilityEstimator, EstimationContext } from './types'; +export { logLikelihood } from './log-likelihood'; +export { maximizeOverTheta } from './optimize'; +export { MLEEstimator } from './mle'; +export { EAPEstimator } from './eap'; +export { + ABILITY_ESTIMATORS, + EstimationMethod, + EstimationMethodInput, + validateEstimationMethod, + getEstimator, +} from './registry'; diff --git a/src/estimators/log-likelihood.ts b/src/estimators/log-likelihood.ts new file mode 100644 index 0000000..e3fd8b3 --- /dev/null +++ b/src/estimators/log-likelihood.ts @@ -0,0 +1,23 @@ +import { Zeta } from '../type'; +import { itemResponseFunction } from '../utils'; + +/** + * Compute the log-likelihood of a response pattern at a given ability level. + * + * @remarks + * This is the shared likelihood used by all likelihood-based estimators + * (MLE, EAP, and future additions such as WLE/MAP). It assumes local + * independence: the log-likelihood is the sum of per-item log probabilities + * under the 4PL item response function. + * + * @param {number} theta - ability estimate at which to evaluate the likelihood + * @param {Zeta[]} zetas - item parameters for the administered items + * @param {(0 | 1)[]} resps - responses aligned with `zetas` (1 = correct, 0 = incorrect) + * @returns {number} the log-likelihood ln L(theta) + */ +export const logLikelihood = (theta: number, zetas: Zeta[], resps: (0 | 1)[]): number => { + return zetas.reduce((acc, zeta, i) => { + const probability = itemResponseFunction(theta, zeta); + return acc + (resps[i] === 1 ? Math.log(probability) : Math.log(1 - probability)); + }, 0); +}; diff --git a/src/estimators/mle.ts b/src/estimators/mle.ts new file mode 100644 index 0000000..b21622a --- /dev/null +++ b/src/estimators/mle.ts @@ -0,0 +1,28 @@ +import { AbilityEstimator, EstimationContext } from './types'; +import { logLikelihood } from './log-likelihood'; +import { maximizeOverTheta } from './optimize'; + +/** + * Maximum Likelihood Estimation (MLE) of ability. + * + * @remarks + * Maximizes the log-likelihood ln L(theta) of the observed response pattern + * under the 4PL model. MLE is unbiased asymptotically but diverges for + * all-correct or all-incorrect response patterns; in those cases the optimizer + * runs toward the bound and `Cat` clamps the result to [minTheta, maxTheta]. + * + * Reference: Lord, F. M. (1980). Applications of item response theory to + * practical testing problems. Erlbaum. + */ +export class MLEEstimator implements AbilityEstimator { + /** + * Estimate ability by maximizing the log-likelihood. + * + * @param {EstimationContext} context - The accumulated response history + * @returns {number} the maximum likelihood theta estimate (unclamped) + */ + estimateAbility(context: EstimationContext): number { + const { zetas, resps } = context; + return maximizeOverTheta((theta) => logLikelihood(theta, zetas, resps)); + } +} diff --git a/src/estimators/optimize.ts b/src/estimators/optimize.ts new file mode 100644 index 0000000..daf1a4a --- /dev/null +++ b/src/estimators/optimize.ts @@ -0,0 +1,28 @@ +import { minimize_Powell } from 'optimization-js'; + +/** + * The starting point for the one-dimensional theta search. + * + * @remarks + * TODO: Consider seeding the search with the current theta estimate instead of + * a fixed start. Powell's method is a local optimizer, and 3PL/4PL likelihoods + * can be multimodal; a fixed start at 0 is the historical behavior. + */ +export const THETA_SEARCH_START = 0; + +/** + * Maximize a one-dimensional objective function over theta. + * + * @remarks + * Shared optimizer scaffolding for estimators that maximize an objective + * (e.g., the log-likelihood for MLE, or a penalized log-likelihood for WLE). + * Internally negates the objective and runs Powell's method, which searches + * unbounded; `Cat` clamps the result to [minTheta, maxTheta] afterward. + * + * @param {(theta: number) => number} objective - the function of theta to maximize + * @returns {number} the theta value that maximizes the objective + */ +export const maximizeOverTheta = (objective: (theta: number) => number): number => { + const solution = minimize_Powell((thetaArray: number[]) => -objective(thetaArray[0]), [THETA_SEARCH_START]); + return solution.argument[0]; +}; diff --git a/src/estimators/registry.ts b/src/estimators/registry.ts new file mode 100644 index 0000000..8a5bf02 --- /dev/null +++ b/src/estimators/registry.ts @@ -0,0 +1,54 @@ +import { AbilityEstimator } from './types'; +import { MLEEstimator } from './mle'; +import { EAPEstimator } from './eap'; + +/** + * The registry of available ability estimators. + * + * @remarks + * This object is the single source of truth for which estimation methods + * exist. The `EstimationMethod` type, runtime validation, and dispatch are all + * derived from it. To add a new estimator: create a class implementing + * `AbilityEstimator` in its own file in `src/estimators/`, then add a single + * entry here. Do not add dispatch logic anywhere else. + */ +export const ABILITY_ESTIMATORS = { + mle: new MLEEstimator(), + eap: new EAPEstimator(), +} as const; + +/** The canonical (lowercase) names of all registered ability estimators. */ +export type EstimationMethod = keyof typeof ABILITY_ESTIMATORS; + +/** + * An estimation method as provided by the user. Estimation methods are + * case-insensitive, so this type offers autocomplete on the canonical names + * while still accepting arbitrary strings (validated at runtime). + */ +export type EstimationMethodInput = EstimationMethod | (string & Record); + +/** + * Validate a user-provided estimation method and normalize it to its canonical + * (lowercase) name. + * + * @param {string} method - the user-provided estimation method (case-insensitive) + * @returns {EstimationMethod} the canonical estimation method name + * @throws {Error} if the method is not in the registry + */ +export const validateEstimationMethod = (method: string): EstimationMethod => { + const lowerMethod = method.toLowerCase(); + if (!Object.keys(ABILITY_ESTIMATORS).includes(lowerMethod)) { + throw new Error('The abilityEstimator you provided is not in the list of valid methods'); + } + return lowerMethod as EstimationMethod; +}; + +/** + * Look up the estimator instance for a validated estimation method. + * + * @param {EstimationMethod} method - a canonical estimation method name + * @returns {AbilityEstimator} the registered estimator + */ +export const getEstimator = (method: EstimationMethod): AbilityEstimator => { + return ABILITY_ESTIMATORS[method]; +}; diff --git a/src/estimators/types.ts b/src/estimators/types.ts new file mode 100644 index 0000000..bdfacef --- /dev/null +++ b/src/estimators/types.ts @@ -0,0 +1,43 @@ +import { Zeta } from '../type'; + +/** + * Everything an ability estimator may consult when producing a theta estimate. + * + * @remarks + * The context is assembled by `Cat.updateAbilityEstimate` from the accumulated + * response history. Estimators must treat it as read-only. + */ +export interface EstimationContext { + /** Item parameters (zetas) for all administered items, in administration order */ + zetas: Zeta[]; + /** Responses to the administered items (1 = correct, 0 = incorrect), aligned with `zetas` */ + resps: (0 | 1)[]; + /** Lower bound of theta. `Cat` clamps the returned estimate to [minTheta, maxTheta]. */ + minTheta: number; + /** Upper bound of theta. `Cat` clamps the returned estimate to [minTheta, maxTheta]. */ + maxTheta: number; + /** + * Quantized prior distribution as [theta, probability] pairs. + * Populated only when the Cat was constructed with a Bayesian estimator (e.g., EAP); + * empty array otherwise. + */ + prior: [number, number][]; +} + +/** + * The interface every ability estimator must implement. + * + * @remarks + * Implementations live in `src/estimators/`, one file per estimator, and are + * registered in `src/estimators/registry.ts`. See CONTRIBUTING.md for a + * step-by-step guide to adding a new estimator. + */ +export interface AbilityEstimator { + /** + * Estimate the examinee's ability (theta) from the response history. + * + * @param {EstimationContext} context - The accumulated response history and configuration + * @returns {number} the theta estimate (unclamped; `Cat` applies the [minTheta, maxTheta] bounds) + */ + estimateAbility(context: EstimationContext): number; +} diff --git a/src/index.ts b/src/index.ts index e3c4db1..b62caef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,3 +7,28 @@ export { StopOnSEMeasurementPlateau, StopIfSEMeasurementBelowThreshold, } from './stopping'; +export { + AbilityEstimator, + EstimationContext, + EstimationMethod, + EstimationMethodInput, + ABILITY_ESTIMATORS, + validateEstimationMethod, + logLikelihood, + maximizeOverTheta, +} from './estimators'; +export { + ItemSelector, + SelectorContext, + SelectorResult, + SelectorMethod, + ItemSelectMethod, + StartSelectMethod, + ItemSelectMethodInput, + StartSelectMethodInput, + SELECTORS, + ITEM_SELECT_METHODS, + START_SELECT_METHODS, + validateItemSelect, + validateStartSelect, +} from './selectors'; diff --git a/src/selectors/closest.ts b/src/selectors/closest.ts new file mode 100644 index 0000000..325ddec --- /dev/null +++ b/src/selectors/closest.ts @@ -0,0 +1,43 @@ +import { Stimulus } from '../type'; +import { findClosest } from '../utils'; +import { ItemSelector, SelectorContext, SelectorResult } from './types'; + +/** + * The offset added to theta when selecting the item with the closest difficulty. + * + * @remarks + * For a 3PL item, Fisher information is maximized when the item difficulty is + * b = theta + ln((1 + sqrt(1 + 8c)) / 2) / a. With guessing c = 0.5 and + * discrimination a = 1, the offset is ln((1 + sqrt(5)) / 2) ≈ 0.481. + * + * Reference: Birnbaum, A. (1968). Some latent trait models. In F. M. Lord & + * M. R. Novick (Eds.), Statistical theories of mental test scores. Addison-Wesley. + */ +export const CLOSEST_SELECTION_OFFSET = 0.481; + +/** + * Closest-difficulty item selection. + * + * @remarks + * Selects the item whose difficulty is closest to theta + 0.481 (see + * `CLOSEST_SELECTION_OFFSET`). Requires the input stimuli to be sorted by + * difficulty; `Cat.findNextItem` guarantees this. + */ +export class ClosestSelector implements ItemSelector { + /** + * Select the stimulus with difficulty closest to the offset theta. + * + * @param {Stimulus[]} stimuli - the available stimuli, sorted by difficulty + * @param {SelectorContext} context - provides the current theta estimate + * @returns {SelectorResult} the closest stimulus and the remaining stimuli + */ + select(stimuli: Stimulus[], context: SelectorContext): SelectorResult { + const index = findClosest(stimuli, context.theta + CLOSEST_SELECTION_OFFSET); + const nextItem = stimuli[index]; + stimuli.splice(index, 1); + return { + nextStimulus: nextItem, + remainingStimuli: stimuli, + }; + } +} diff --git a/src/selectors/fixed.ts b/src/selectors/fixed.ts new file mode 100644 index 0000000..b6f96ab --- /dev/null +++ b/src/selectors/fixed.ts @@ -0,0 +1,26 @@ +import { Stimulus } from '../type'; +import { ItemSelector, SelectorResult } from './types'; + +/** + * Fixed-order item selection. + * + * @remarks + * Picks the next item in line from the given list of stimuli, preserving the + * caller's corpus order. Grabs the first item from the list, removes it, and + * returns it along with the rest of the list. + */ +export class FixedSelector implements ItemSelector { + /** + * Select the first stimulus in the list. + * + * @param {Stimulus[]} stimuli - the available stimuli, in corpus order + * @returns {SelectorResult} the first stimulus and the remaining stimuli + */ + select(stimuli: Stimulus[]): SelectorResult { + const nextItem = stimuli.shift(); + return { + nextStimulus: nextItem, + remainingStimuli: stimuli, + }; + } +} diff --git a/src/selectors/index.ts b/src/selectors/index.ts new file mode 100644 index 0000000..525ec45 --- /dev/null +++ b/src/selectors/index.ts @@ -0,0 +1,19 @@ +export { ItemSelector, SelectorContext, SelectorResult } from './types'; +export { MFISelector } from './mfi'; +export { ClosestSelector, CLOSEST_SELECTION_OFFSET } from './closest'; +export { RandomSelector } from './random'; +export { FixedSelector } from './fixed'; +export { MiddleSelector } from './middle'; +export { + SELECTORS, + ITEM_SELECT_METHODS, + START_SELECT_METHODS, + SelectorMethod, + ItemSelectMethod, + StartSelectMethod, + ItemSelectMethodInput, + StartSelectMethodInput, + validateItemSelect, + validateStartSelect, + getSelector, +} from './registry'; diff --git a/src/selectors/mfi.ts b/src/selectors/mfi.ts new file mode 100644 index 0000000..1cd234c --- /dev/null +++ b/src/selectors/mfi.ts @@ -0,0 +1,43 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { Stimulus } from '../type'; +import { fisherInformation } from '../utils'; +import { fillZetaDefaults } from '../corpus'; +import { ItemSelector, SelectorContext, SelectorResult } from './types'; + +/** + * Maximum Fisher Information (MFI) item selection. + * + * @remarks + * Selects the item with the highest Fisher information at the current theta + * estimate. The remaining stimuli are returned sorted by difficulty so that + * downstream selectors that require difficulty-sorted input keep working. + * + * Reference: Lord, F. M. (1980). Applications of item response theory to + * practical testing problems. Erlbaum. + */ +export class MFISelector implements ItemSelector { + /** + * Select the stimulus with maximum Fisher information at the current theta. + * + * @param {Stimulus[]} stimuli - the available stimuli + * @param {SelectorContext} context - provides the current theta estimate + * @returns {SelectorResult} the most informative stimulus and the rest, sorted by difficulty + */ + select(stimuli: Stimulus[], context: SelectorContext): SelectorResult { + const filledStimuli = stimuli.map((stim) => fillZetaDefaults(stim, 'semantic')); + const stimuliAddFisher = filledStimuli.map((element: Stimulus) => ({ + fisherInformation: fisherInformation(context.theta, fillZetaDefaults(element, 'symbolic')), + ...element, + })); + + stimuliAddFisher.sort((a, b) => b.fisherInformation - a.fisherInformation); + stimuliAddFisher.forEach((stimulus: Stimulus) => { + delete stimulus['fisherInformation']; + }); + + return { + nextStimulus: stimuliAddFisher[0], + remainingStimuli: stimuliAddFisher.slice(1).sort((a: Stimulus, b: Stimulus) => a.difficulty! - b.difficulty!), + }; + } +} diff --git a/src/selectors/middle.ts b/src/selectors/middle.ts new file mode 100644 index 0000000..1599a36 --- /dev/null +++ b/src/selectors/middle.ts @@ -0,0 +1,35 @@ +import { Stimulus } from '../type'; +import { ItemSelector, SelectorContext, SelectorResult } from './types'; + +/** + * Middle-difficulty item selection. + * + * @remarks + * Selects an item near the middle of the difficulty-sorted stimuli, jittered + * by up to ±nStartItems/2 positions using the seeded random number generator. + * This selector is only valid as a `startSelect` rule (for the first + * `nStartItems` trials), not as an adaptive `itemSelect` rule. + */ +export class MiddleSelector implements ItemSelector { + /** + * Select a stimulus near the middle of the difficulty-sorted list. + * + * @param {Stimulus[]} stimuli - the available stimuli, sorted by difficulty + * @param {SelectorContext} context - provides nStartItems and the seeded random integer generator + * @returns {SelectorResult} the selected stimulus and the remaining stimuli + */ + select(stimuli: Stimulus[], context: SelectorContext): SelectorResult { + let index = Math.floor(stimuli.length / 2); + + if (stimuli.length >= context.nStartItems) { + index += context.randomInteger(-Math.floor(context.nStartItems / 2), Math.floor(context.nStartItems / 2)); + } + + const nextItem = stimuli[index]; + stimuli.splice(index, 1); + return { + nextStimulus: nextItem, + remainingStimuli: stimuli, + }; + } +} diff --git a/src/selectors/random.ts b/src/selectors/random.ts new file mode 100644 index 0000000..29c8567 --- /dev/null +++ b/src/selectors/random.ts @@ -0,0 +1,28 @@ +import { Stimulus } from '../type'; +import { ItemSelector, SelectorContext, SelectorResult } from './types'; + +/** + * Random item selection. + * + * @remarks + * Selects a uniformly random item using the Cat's seeded random number + * generator (via `context.randomInteger`), so simulations with a fixed + * `randomSeed` are reproducible. + */ +export class RandomSelector implements ItemSelector { + /** + * Select a uniformly random stimulus. + * + * @param {Stimulus[]} stimuli - the available stimuli + * @param {SelectorContext} context - provides the seeded random integer generator + * @returns {SelectorResult} the selected stimulus and the remaining stimuli + */ + select(stimuli: Stimulus[], context: SelectorContext): SelectorResult { + const index = context.randomInteger(0, stimuli.length - 1); + const nextItem = stimuli.splice(index, 1)[0]; + return { + nextStimulus: nextItem, + remainingStimuli: stimuli, + }; + } +} diff --git a/src/selectors/registry.ts b/src/selectors/registry.ts new file mode 100644 index 0000000..1078fb5 --- /dev/null +++ b/src/selectors/registry.ts @@ -0,0 +1,93 @@ +import { ItemSelector } from './types'; +import { MFISelector } from './mfi'; +import { ClosestSelector } from './closest'; +import { RandomSelector } from './random'; +import { FixedSelector } from './fixed'; +import { MiddleSelector } from './middle'; + +/** + * The registry of available item selectors. + * + * @remarks + * This object is the single source of truth for which selectors exist. The + * selector method types, runtime validation, and dispatch are all derived from + * it (together with the `ITEM_SELECT_METHODS` / `START_SELECT_METHODS` + * allowlists below). To add a new selector: create a class implementing + * `ItemSelector` in its own file in `src/selectors/`, add an entry here, and + * add its name to the appropriate allowlist(s). Do not add dispatch logic + * anywhere else. + */ +export const SELECTORS = { + mfi: new MFISelector(), + closest: new ClosestSelector(), + random: new RandomSelector(), + fixed: new FixedSelector(), + middle: new MiddleSelector(), +} as const; + +/** Selectors that are valid as the adaptive `itemSelect` rule. */ +export const ITEM_SELECT_METHODS = ['mfi', 'random', 'closest', 'fixed'] as const; + +/** Selectors that are valid as the non-adaptive `startSelect` rule for the first `nStartItems` trials. */ +export const START_SELECT_METHODS = ['random', 'middle', 'fixed'] as const; + +/** The canonical (lowercase) names of all registered selectors. */ +export type SelectorMethod = keyof typeof SELECTORS; + +/** The canonical names of selectors valid as the adaptive `itemSelect` rule. */ +export type ItemSelectMethod = typeof ITEM_SELECT_METHODS[number]; + +/** The canonical names of selectors valid as the `startSelect` rule. */ +export type StartSelectMethod = typeof START_SELECT_METHODS[number]; + +/** + * An item selection method as provided by the user. Selector names are + * case-insensitive, so this type offers autocomplete on the canonical names + * while still accepting arbitrary strings (validated at runtime). + */ +export type ItemSelectMethodInput = ItemSelectMethod | (string & Record); + +/** A start selection method as provided by the user. See `ItemSelectMethodInput`. */ +export type StartSelectMethodInput = StartSelectMethod | (string & Record); + +/** + * Validate a user-provided item selection method and normalize it to its + * canonical (lowercase) name. + * + * @param {string} itemSelect - the user-provided item selection method (case-insensitive) + * @returns {ItemSelectMethod} the canonical item selection method name + * @throws {Error} if the method is not a valid adaptive item selector + */ +export const validateItemSelect = (itemSelect: string): ItemSelectMethod => { + const lowerItemSelect = itemSelect.toLowerCase(); + if (!(ITEM_SELECT_METHODS as readonly string[]).includes(lowerItemSelect)) { + throw new Error('The itemSelector you provided is not in the list of valid methods'); + } + return lowerItemSelect as ItemSelectMethod; +}; + +/** + * Validate a user-provided start selection method and normalize it to its + * canonical (lowercase) name. + * + * @param {string} startSelect - the user-provided start selection method (case-insensitive) + * @returns {StartSelectMethod} the canonical start selection method name + * @throws {Error} if the method is not a valid start selector + */ +export const validateStartSelect = (startSelect: string): StartSelectMethod => { + const lowerStartSelect = startSelect.toLowerCase(); + if (!(START_SELECT_METHODS as readonly string[]).includes(lowerStartSelect)) { + throw new Error('The startSelect you provided is not in the list of valid methods'); + } + return lowerStartSelect as StartSelectMethod; +}; + +/** + * Look up the selector instance for a validated selector method. + * + * @param {SelectorMethod} method - a canonical selector method name + * @returns {ItemSelector} the registered selector + */ +export const getSelector = (method: SelectorMethod): ItemSelector => { + return SELECTORS[method]; +}; diff --git a/src/selectors/types.ts b/src/selectors/types.ts new file mode 100644 index 0000000..44481c5 --- /dev/null +++ b/src/selectors/types.ts @@ -0,0 +1,49 @@ +import { Stimulus } from '../type'; + +/** + * Everything an item selector may consult when choosing the next item. + * + * @remarks + * The context is assembled by `Cat.findNextItem`. The `randomInteger` function + * is backed by the Cat's seeded random number generator so that simulations + * remain reproducible; selectors must use it instead of `Math.random()`. + */ +export interface SelectorContext { + /** The current ability estimate */ + theta: number; + /** The number of non-adaptive start items configured on the Cat */ + nStartItems: number; + /** Seeded random integer generator: returns an integer in [min, max] (both inclusive) */ + randomInteger: (min: number, max: number) => number; +} + +/** The result of an item selection. */ +export interface SelectorResult { + /** The selected stimulus, or `undefined` if the input array was empty */ + nextStimulus: Stimulus | undefined; + /** The remaining stimuli after removing the selected one */ + remainingStimuli: Stimulus[]; +} + +/** + * The interface every item selector must implement. + * + * @remarks + * Implementations live in `src/selectors/`, one file per selector, and are + * registered in `src/selectors/registry.ts`. Selectors may mutate the input + * array; `Cat.findNextItem` deep-copies the caller's array by default before + * dispatching. See CONTRIBUTING.md for a step-by-step guide to adding a new + * selector. + */ +export interface ItemSelector { + /** + * Select the next item from the available stimuli. + * + * @param {Stimulus[]} stimuli - the available stimuli, with zeta defaults already filled in. + * Sorted by difficulty for all selectors except `mfi` (which sorts internally by + * Fisher information) and `fixed` (which preserves the caller's order). + * @param {SelectorContext} context - the current Cat state relevant to selection + * @returns {SelectorResult} the selected stimulus and the remaining stimuli + */ + select(stimuli: Stimulus[], context: SelectorContext): SelectorResult; +} From bba3ed1fe806725f5bcb675dbe1312f9468b4810 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 21:47:59 +0000 Subject: [PATCH 02/11] Use seeded RNG for validated/unvalidated item choice in Clowder Clowder._selectNextItem used Math.random() for the validated-vs-unvalidated coin flip while every other draw used the seeded RNG, breaking the reproducibility that the randomSeed parameter promises to simulation users. --- src/clowder.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/clowder.ts b/src/clowder.ts index b46abe7..33833e7 100644 --- a/src/clowder.ts +++ b/src/clowder.ts @@ -452,8 +452,9 @@ export class Clowder { } else if (missing.length === 0 || !randomlySelectUnvalidated) { return returnStimulus; // Return validated item if available } else { - // Randomly decide whether to return a validated or unvalidated item - const random = Math.random(); + // Randomly decide whether to return a validated or unvalidated item. + // Use the seeded RNG (not Math.random()) so simulations are reproducible. + const random = this._rng(); const numRemaining = { available: available.length, missing: missing.length }; return random < numRemaining.missing / (numRemaining.available + numRemaining.missing) ? missing[Math.floor(this._rng() * missing.length)] From bc6c25e23ae8c3c8c1d785361768be3d37cf6fee Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 21:47:59 +0000 Subject: [PATCH 03/11] Add golden fixture test harness for estimator validation Adds deterministic fixture generation (50-item bank x 100 examinees, fixed seed), a characterization baseline of jsCAT's own MLE/EAP estimates, and a Jest suite that exactly reproduces the baseline on every run. A companion R script (validation/generate-r-reference.R) scores the same fixtures with catR; once its output CSV is committed, the suite also asserts agreement with the independent reference within tolerance on every CI run, with no R dependency at test time. See validation/README.md for the workflow. --- package.json | 1 + scripts/generate-golden-fixtures.js | 158 ++++++++++++++++++ .../__fixtures__/golden/expected-jscat.csv | 101 +++++++++++ src/__tests__/__fixtures__/golden/items.csv | 51 ++++++ .../__fixtures__/golden/provenance.json | 13 ++ .../__fixtures__/golden/responses.csv | 101 +++++++++++ .../__fixtures__/golden/true-thetas.csv | 101 +++++++++++ src/__tests__/golden.test.ts | 131 +++++++++++++++ validation/README.md | 43 +++++ validation/generate-r-reference.R | 56 +++++++ 10 files changed, 756 insertions(+) create mode 100644 scripts/generate-golden-fixtures.js create mode 100644 src/__tests__/__fixtures__/golden/expected-jscat.csv create mode 100644 src/__tests__/__fixtures__/golden/items.csv create mode 100644 src/__tests__/__fixtures__/golden/provenance.json create mode 100644 src/__tests__/__fixtures__/golden/responses.csv create mode 100644 src/__tests__/__fixtures__/golden/true-thetas.csv create mode 100644 src/__tests__/golden.test.ts create mode 100644 validation/README.md create mode 100644 validation/generate-r-reference.R diff --git a/package.json b/package.json index 48a747d..d9cd9ff 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "format": "prettier --write \"src/**/*.ts\" \"lib/**/*.js\"", "lint": "eslint . --ext .js,.jsx,.ts,.tsx", "doc": "npx typedoc", + "fixtures:generate": "npm run build && node scripts/generate-golden-fixtures.js", "prepare": "npm run build", "prepublishOnly": "npm test && npm run lint", "preversion": "npm run lint", diff --git a/scripts/generate-golden-fixtures.js b/scripts/generate-golden-fixtures.js new file mode 100644 index 0000000..3e99ef0 --- /dev/null +++ b/scripts/generate-golden-fixtures.js @@ -0,0 +1,158 @@ +'use strict'; +/* eslint-env node */ +/* eslint-disable @typescript-eslint/no-var-requires */ + +/** + * Deterministically generate the golden fixture set used by + * src/__tests__/golden.test.ts. + * + * This script writes four files to src/__tests__/__fixtures__/golden/: + * + * - items.csv : a 50-item bank (a, b, c, d parameters) + * - responses.csv : simulated responses for 100 examinees + * - true-thetas.csv : the generating thetas (documentation/sanity only) + * - expected-jscat.csv : jsCAT's own MLE and EAP estimates (characterization + * baseline — regenerate ONLY when an intentional + * algorithm change is made, and explain why in the PR) + * + * The item bank and responses are generated with a fixed seed, so re-running + * this script always produces identical items.csv / responses.csv / + * true-thetas.csv. The external reference file (expected-r-reference.csv) is + * generated separately from the same inputs by validation/generate-r-reference.R. + * + * Usage: + * npm run build && node scripts/generate-golden-fixtures.js + */ + +const fs = require('fs'); +const path = require('path'); +const seedrandom = require('seedrandom'); +const { Cat } = require('../lib/index.js'); + +const SEED = 'jscat-golden-v1'; +const N_ITEMS = 50; +const N_PEOPLE = 100; +const MIN_THETA = -6; +const MAX_THETA = 6; + +const FIXTURE_DIR = path.join(__dirname, '..', 'src', '__tests__', '__fixtures__', 'golden'); + +const rng = seedrandom(SEED); + +/** Standard normal draw via the Box-Muller transform, using the seeded rng. */ +function randomNormal() { + let u = 0; + let v = 0; + // Avoid log(0) + while (u === 0) u = rng(); + while (v === 0) v = rng(); + return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v); +} + +/** The 4PL item response function (kept inline so this script is standalone). */ +function itemResponseFunction(theta, { a, b, c, d }) { + return c + (d - c) / (1 + Math.exp(-a * (theta - b))); +} + +function round(x, digits) { + const factor = Math.pow(10, digits); + return Math.round(x * factor) / factor; +} + +// --------------------------------------------------------------------------- +// 1. Item bank: difficulties evenly spaced in [-3, 3], discriminations in +// [0.8, 2.2], every fifth item with a nonzero guessing parameter so the +// bank exercises the 3PL code path. +// --------------------------------------------------------------------------- +const items = []; +for (let i = 0; i < N_ITEMS; i++) { + items.push({ + item: i + 1, + a: round(0.8 + 1.4 * rng(), 3), + b: round(-3 + (6 * i) / (N_ITEMS - 1), 3), + c: i % 5 === 0 ? 0.15 : 0, + d: 1, + }); +} + +// --------------------------------------------------------------------------- +// 2. Examinees: thetas drawn from N(0, 1.5), truncated to [-4, 4]. +// --------------------------------------------------------------------------- +const people = []; +for (let p = 0; p < N_PEOPLE; p++) { + const theta = Math.max(-4, Math.min(4, 1.5 * randomNormal())); + people.push({ pid: p + 1, theta: round(theta, 4) }); +} + +// --------------------------------------------------------------------------- +// 3. Responses: Bernoulli draws from the 4PL response probability. +// --------------------------------------------------------------------------- +const responses = people.map(({ pid, theta }) => { + const resps = items.map((item) => (rng() < itemResponseFunction(theta, item) ? 1 : 0)); + return { pid, resps }; +}); + +// --------------------------------------------------------------------------- +// 4. jsCAT estimates (characterization baseline). +// --------------------------------------------------------------------------- +const expected = responses.map(({ pid, resps }) => { + const zetas = items.map(({ a, b, c, d }) => ({ a, b, c, d })); + + const catMLE = new Cat({ method: 'MLE', minTheta: MIN_THETA, maxTheta: MAX_THETA }); + catMLE.updateAbilityEstimate(zetas, resps); + + const catEAP = new Cat({ + method: 'EAP', + minTheta: MIN_THETA, + maxTheta: MAX_THETA, + priorDist: 'norm', + priorPar: [0, 1], + }); + catEAP.updateAbilityEstimate(zetas, resps); + + return { + pid, + theta_mle: catMLE.theta, + se_mle: catMLE.seMeasurement, + theta_eap: catEAP.theta, + se_eap: catEAP.seMeasurement, + }; +}); + +// --------------------------------------------------------------------------- +// 5. Write fixtures. +// --------------------------------------------------------------------------- +fs.mkdirSync(FIXTURE_DIR, { recursive: true }); + +const itemsCsv = ['item,a,b,c,d', ...items.map((i) => `${i.item},${i.a},${i.b},${i.c},${i.d}`)].join('\n'); +fs.writeFileSync(path.join(FIXTURE_DIR, 'items.csv'), itemsCsv + '\n'); + +const respHeader = ['pid', ...items.map((i) => `i${i.item}`)].join(','); +const respCsv = [respHeader, ...responses.map(({ pid, resps }) => [pid, ...resps].join(','))].join('\n'); +fs.writeFileSync(path.join(FIXTURE_DIR, 'responses.csv'), respCsv + '\n'); + +const thetaCsv = ['pid,theta', ...people.map(({ pid, theta }) => `${pid},${theta}`)].join('\n'); +fs.writeFileSync(path.join(FIXTURE_DIR, 'true-thetas.csv'), thetaCsv + '\n'); + +const expectedCsv = [ + 'pid,theta_mle,se_mle,theta_eap,se_eap', + ...expected.map((r) => `${r.pid},${r.theta_mle},${r.se_mle},${r.theta_eap},${r.se_eap}`), +].join('\n'); +fs.writeFileSync(path.join(FIXTURE_DIR, 'expected-jscat.csv'), expectedCsv + '\n'); + +const provenance = { + seed: SEED, + nItems: N_ITEMS, + nPeople: N_PEOPLE, + thetaBounds: [MIN_THETA, MAX_THETA], + generator: 'scripts/generate-golden-fixtures.js', + generatedAt: new Date().toISOString(), + jscatVersion: require('../package.json').version, + note: + 'expected-jscat.csv is a characterization baseline generated by jsCAT itself. ' + + 'expected-r-reference.csv (if present) is the independent reference generated by ' + + 'validation/generate-r-reference.R using the catR package.', +}; +fs.writeFileSync(path.join(FIXTURE_DIR, 'provenance.json'), JSON.stringify(provenance, null, 2) + '\n'); + +console.log(`Wrote golden fixtures for ${N_PEOPLE} examinees x ${N_ITEMS} items to ${FIXTURE_DIR}`); diff --git a/src/__tests__/__fixtures__/golden/expected-jscat.csv b/src/__tests__/__fixtures__/golden/expected-jscat.csv new file mode 100644 index 0000000..372c40e --- /dev/null +++ b/src/__tests__/__fixtures__/golden/expected-jscat.csv @@ -0,0 +1,101 @@ +pid,theta_mle,se_mle,theta_eap,se_eap +1,1.8588998322048391,0.30289389949139117,1.700532203833846,0.302200425112208 +2,0.6312680288637098,0.3081130797205417,0.57057888316549,0.3080813995631253 +3,0.46636579243212584,0.30791005674851046,0.4231617809965266,0.3077947803895712 +4,1.0872239197598106,0.3066069863544136,0.9869934694201481,0.3072271405230544 +5,-1.6295303923940565,0.3124431816244769,-1.491366049939463,0.310034725373963 +6,0.5990534248844676,0.30810246159550875,0.5418242638517438,0.3080489856736773 +7,-0.21893814889697702,0.3038877731371155,-0.2016727179465006,0.30398934613246226 +8,0.5228498499626514,0.308021430444065,0.4762856851698992,0.3079328223834859 +9,-3.304414944190416,0.5047580067910393,-2.8137093532432202,0.41070481525143954 +10,0.7957885561840818,0.3079434184800617,0.7232811569735115,0.308065386566452 +11,0.44651756126425424,0.30786036608336437,0.4044007069963272,0.30773649271669834 +12,-0.688053888136376,0.30374277322973137,-0.6338753548568294,0.30348358907539325 +13,0.9786310316258504,0.3072712161794592,0.890510369898711,0.30766351054235097 +14,0.6840014788102506,0.308100037338158,0.6192394978181645,0.3081107565676052 +15,2.590808405719624,0.33588968330631847,2.357678895225598,0.3187827762334994 +16,-2.051313299398468,0.32973600370537925,-1.8678645876408746,0.3200374295829952 +17,0.5347048803685224,0.3080392232404728,0.49112190404926526,0.30796431139701574 +18,4.081563505967497,0.6999640203393643,3.2584137086469886,0.43448324302628344 +19,1.0485397788700037,0.30686616245576315,0.9523832860777697,0.3074018031281328 +20,1.0648892080469894,0.30675959366434147,0.9704890428366585,0.30731298189371065 +21,-0.19732761899033568,0.30401563188604086,-0.17982710504656868,0.3041242561563092 +22,-2.566260805534066,0.3770290402193326,-2.307139890535378,0.3495059163035916 +23,-2.4933203420392465,0.3685123405788368,-2.2466710698226557,0.3441691586131341 +24,-1.1016359879396131,0.3066538058958921,-1.0042882423916415,0.30594620754812 +25,1.3747191851846685,0.3041357996710781,1.2525858483776584,0.3052614647318543 +26,-0.9951124225212474,0.305877932310807,-0.9145489042055802,0.30527260727830463 +27,-2.5507211147377653,0.3751631309455136,-2.298206267261468,0.348691659655619 +28,-0.5199689274278102,0.30315780436311995,-0.47967282536556666,0.30312229487658693 +29,-1.0908114339664003,0.3065766859301559,-1.0007257579393756,0.3059197270159801 +30,3.187699133559006,0.41985398112218003,2.814302936370721,0.35979332962992094 +31,0.35512085283946054,0.307559482828496,0.3211679422522274,0.30741740277410956 +32,0.7679208798894146,0.30799940604556436,0.7057692974257187,0.3080835005996358 +33,-2.8034513259255944,0.40915203985064014,-2.4622566832041635,0.3650699848468007 +34,-3.129634714898387,0.46634606547290247,-2.721035316658195,0.39718752091128195 +35,1.6547242405565092,0.3022613062323735,1.515389140623105,0.30295829227033316 +36,0.8523154871741293,0.3077934826296074,0.7692467185560894,0.30799700363704696 +37,-1.8332497751457384,0.3186066536197898,-1.6718602405240583,0.3134348511662817 +38,2.290271903034184,0.315127930537629,2.1022864451590384,0.30760951998181113 +39,-6,2.398559964172724,-3.266650290415419,0.4959322916934462 +40,-0.2832333167115184,0.3035553818117071,-0.25992860198797973,0.3036668584758051 +41,0.7419566440783543,0.3080412779680965,0.670574980879014,0.3081069676682578 +42,-0.961199666862612,0.3056239294329479,-0.881540310474357,0.3050253692360071 +43,1.076466254397067,0.30668149110948667,0.9775525891277135,0.3072768133055201 +44,2.3215712314950867,0.31675813989857,2.124295280828844,0.3082995340120932 +45,-0.9605631386369317,0.3056191428301417,-0.8840867153650075,0.30504433944388576 +46,-1.7002550651285326,0.3141786219932775,-1.5595747182633954,0.311081085364047 +47,4.361913485683735,0.8392942549358298,3.33930759240767,0.45261406917909264 +48,0.9350405827032476,0.3074816526332066,0.8546260608226959,0.30778628857937756 +49,-0.8397798562358747,0.30471834770230555,-0.7690583879980197,0.30422772622180955 +50,1.5338816172359118,0.30283049421514785,1.4001697661254717,0.3039046587177168 +51,0.7619042770200629,0.30800998351452535,0.6919261553014854,0.3080947791561971 +52,-1.198055661069566,0.30733220536223366,-1.0995312933420702,0.30663883819431953 +53,1.30021166476189,0.30482556210247047,1.186865617087513,0.30583559959104684 +54,0.02377179354786882,0.30557666134955985,0.024012364452672713,0.3055784388162179 +55,0.6207313164293172,0.3081111510451915,0.5634238142709116,0.30807438266611537 +56,-0.029391051967001966,0.3051819057453777,-0.026533748095300975,0.3052031324854259 +57,0.2570981073775451,0.3071054811400358,0.23530632364101015,0.3069868127677043 +58,0.6840762016835037,0.30809999181579584,0.6261323329713476,0.308112327403867 +59,0.9185707098650185,0.30755277741145093,0.82882932728843,0.30786181073129354 +60,0.1302064391721982,0.3063350724461827,0.1201219618770456,0.30626655496676447 +61,-1.235365382966972,0.3075979048479154,-1.133057909284805,0.3068759242595351 +62,1.116555242540809,0.3063946644231522,1.0144342697855482,0.30707412450163524 +63,1.8909915827133934,0.3032372256069476,1.7243590603580101,0.3022100198023483 +64,-0.13218047766300742,0.30443952575847383,-0.12221580975449908,0.3045084948279562 +65,-0.9619671063031212,0.30562969978509125,-0.8766265891016785,0.30498882748318135 +66,3.255383934383351,0.433833444693727,2.855615448765288,0.365125591561657 +67,-2.349452217608828,0.35348390125369494,-2.1242551883876617,0.33462301235640335 +68,0.08635216328876531,0.3060311413013919,0.08033703308218636,0.30598835150279863 +69,0.5253621638017819,0.30802536170361117,0.47631376638693923,0.3079328848781848 +70,-0.34270254007884676,0.30332366523178717,-0.3158938955107988,0.30341827595200466 +71,0.4158903588247219,0.30777278331456837,0.37695944269411846,0.3076422026080383 +72,0.17462608601823512,0.3066253857825623,0.16102902169378971,0.3065386349137881 +73,1.2931528125808889,0.30489079889841314,1.1763748896122685,0.30592309559680153 +74,2.565120359207661,0.33363880519712397,2.337008240715823,0.31760460791941403 +75,-2.4707796188002784,0.3660035716363793,-2.222362941932229,0.3421394529510895 +76,2.805641455275647,0.35871345205316124,2.538734620173131,0.33142580621877027 +77,0.8925772017668997,0.3076557993721729,0.8141542674566465,0.3079001263997979 +78,1.8063494724755713,0.3024893135545689,1.6561583639126574,0.3022579208233519 +79,-0.35581234886062013,0.3032836026723006,-0.3281615432056812,0.3033729061590717 +80,-0.39291884490434087,0.3031933011359565,-0.36027796376180354,0.3032709123436851 +81,0.08045602042036563,0.30598920019395487,0.07504233310174668,0.3059504976270422 +82,-0.11355876495060353,0.30456912050454193,-0.10528170020141399,0.30462764852904184 +83,-1.8298772892425408,0.3184736924919598,-1.6717876919162233,0.31343303392160454 +84,-1.66802066461559,0.31333924483049386,-1.5220264878437402,0.31047444749856723 +85,-1.2078321591090857,0.30740128170625863,-1.1065007320311044,0.3066883538436063 +86,1.352931858476105,0.3043365066781005,1.2369291205562367,0.305401773967447 +87,2.721067578993394,0.3488407585079084,2.461268210527924,0.3254884436239299 +88,-2.0862992990883606,0.3320055586536638,-1.9017932097602628,0.32155909791752146 +89,3.848642111447811,0.6050144446041955,3.1711609446492712,0.4165921521127591 +90,1.576080131035285,0.30257443009947393,1.43949403397396,0.3035590304024674 +91,0.8301278681791631,0.30785825928622745,0.7521658335152794,0.3080259826740249 +92,-1.9231103774218077,0.32257689554003927,-1.7566789546542119,0.3158590750956975 +93,0.8285698495556992,0.3078625172370411,0.7468579795485826,0.30803412273865616 +94,1.0409396723283348,0.3069141952297097,0.9516888067141759,0.3074050988064524 +95,0.2716636901106792,0.30718131215676325,0.25030204307437304,0.3070691364719882 +96,0.14808850669698664,0.3064542958460234,0.1354655314312086,0.3063704457325791 +97,-0.5403842090069795,0.3031922275374423,-0.4933906174917755,0.30312950820130935 +98,-2.026539343312127,0.32821154691000565,-1.8457920883135064,0.319111101321839 +99,-1.4962704612247058,0.3101019513591121,-1.3699644052156956,0.3086762118715515 +100,1.6777556916578245,0.3022182303282904,1.5317431404702406,0.3028448245418307 diff --git a/src/__tests__/__fixtures__/golden/items.csv b/src/__tests__/__fixtures__/golden/items.csv new file mode 100644 index 0000000..cc393c0 --- /dev/null +++ b/src/__tests__/__fixtures__/golden/items.csv @@ -0,0 +1,51 @@ +item,a,b,c,d +1,1.532,-3,0.15,1 +2,1.003,-2.878,0,1 +3,1.829,-2.755,0,1 +4,1.341,-2.633,0,1 +5,1.1,-2.51,0,1 +6,1.052,-2.388,0.15,1 +7,1.226,-2.265,0,1 +8,1.925,-2.143,0,1 +9,0.867,-2.02,0,1 +10,1.943,-1.898,0,1 +11,2.045,-1.776,0.15,1 +12,2.194,-1.653,0,1 +13,1.466,-1.531,0,1 +14,1.312,-1.408,0,1 +15,1.138,-1.286,0,1 +16,1.492,-1.163,0.15,1 +17,1.017,-1.041,0,1 +18,1.482,-0.918,0,1 +19,0.944,-0.796,0,1 +20,2.116,-0.673,0,1 +21,1.029,-0.551,0.15,1 +22,2.076,-0.429,0,1 +23,1.491,-0.306,0,1 +24,1.794,-0.184,0,1 +25,1.728,-0.061,0,1 +26,1.042,0.061,0.15,1 +27,1.095,0.184,0,1 +28,1.531,0.306,0,1 +29,0.843,0.429,0,1 +30,1.741,0.551,0,1 +31,2.068,0.673,0.15,1 +32,1.548,0.796,0,1 +33,0.912,0.918,0,1 +34,1.563,1.041,0,1 +35,1.242,1.163,0,1 +36,1.44,1.286,0.15,1 +37,1.554,1.408,0,1 +38,1.12,1.531,0,1 +39,2.059,1.653,0,1 +40,1.285,1.776,0,1 +41,1.794,1.898,0.15,1 +42,2.107,2.02,0,1 +43,1.821,2.143,0,1 +44,1.539,2.265,0,1 +45,1.588,2.388,0,1 +46,1.343,2.51,0.15,1 +47,2.091,2.633,0,1 +48,1.693,2.755,0,1 +49,1.074,2.878,0,1 +50,1.493,3,0,1 diff --git a/src/__tests__/__fixtures__/golden/provenance.json b/src/__tests__/__fixtures__/golden/provenance.json new file mode 100644 index 0000000..cf7b291 --- /dev/null +++ b/src/__tests__/__fixtures__/golden/provenance.json @@ -0,0 +1,13 @@ +{ + "seed": "jscat-golden-v1", + "nItems": 50, + "nPeople": 100, + "thetaBounds": [ + -6, + 6 + ], + "generator": "scripts/generate-golden-fixtures.js", + "generatedAt": "2026-06-11T21:37:20.071Z", + "jscatVersion": "5.3.2", + "note": "expected-jscat.csv is a characterization baseline generated by jsCAT itself. expected-r-reference.csv (if present) is the independent reference generated by validation/generate-r-reference.R using the catR package." +} diff --git a/src/__tests__/__fixtures__/golden/responses.csv b/src/__tests__/__fixtures__/golden/responses.csv new file mode 100644 index 0000000..ff0b637 --- /dev/null +++ b/src/__tests__/__fixtures__/golden/responses.csv @@ -0,0 +1,101 @@ +pid,i1,i2,i3,i4,i5,i6,i7,i8,i9,i10,i11,i12,i13,i14,i15,i16,i17,i18,i19,i20,i21,i22,i23,i24,i25,i26,i27,i28,i29,i30,i31,i32,i33,i34,i35,i36,i37,i38,i39,i40,i41,i42,i43,i44,i45,i46,i47,i48,i49,i50 +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,0,1,1,1,0,1,1,0,0,0,1,0,0,0,0,1 +2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0 +3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,0,1,1,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0 +5,1,1,1,1,0,1,1,0,1,0,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +6,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0 +9,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +10,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,0,1,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0 +11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0 +12,1,1,0,1,0,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +13,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +14,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,1,1,1,0,1,1,1,1,1,0,1,1,0,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0 +15,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,0,1,1,0,0,0 +16,1,0,0,0,1,1,1,0,0,1,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +17,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +18,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1 +19,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,0,1,0,0,1,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0 +20,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,0,1,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0 +21,1,1,1,1,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0 +22,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +23,1,1,0,1,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +24,1,0,1,1,1,1,1,1,0,1,1,1,1,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +25,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,0,1,1,1,1,1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,0 +26,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +27,1,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +28,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +29,1,1,1,1,1,1,0,1,0,1,1,1,1,0,0,1,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +30,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,1,1,1,1,0 +31,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,0,0,1,1,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +32,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,0,1,1,0,0,0,1,1,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0 +33,1,1,0,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +34,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +35,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0 +36,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,1,0 +37,1,0,1,1,0,0,0,1,0,0,1,1,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +38,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,1,1,0,1,1,1,0,0,1,0,0,0 +39,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +40,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0 +41,1,1,1,1,1,1,1,1,0,1,0,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0 +42,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +43,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0 +44,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,1,1,1,0,1,1,1,0,0,0,1,0 +45,1,1,0,1,1,1,0,1,0,1,1,0,1,1,0,0,1,1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +46,1,1,1,1,1,1,1,0,1,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +47,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1 +48,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0 +49,1,1,0,1,1,0,1,1,1,1,1,1,1,0,1,1,0,0,0,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +50,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,0,1,1,0,0,0,0,0,0,0,1,1,0,1,1,0 +51,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0 +52,1,1,0,1,0,1,1,1,1,1,1,1,0,1,1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +53,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,1,0,0 +54,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +55,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,0,1,1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0 +56,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1,1,1,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0 +57,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +58,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,1,0,1,0,1,1,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0 +59,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,0,1,0,1,1,0,1,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +60,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +61,1,1,1,0,0,1,1,1,1,1,0,0,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +62,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0 +63,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,0,1,1,1,1,1,1,0,0,0,0,1,0,1,0,0 +64,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0 +65,1,1,1,1,1,1,1,1,0,1,1,1,1,0,1,0,0,1,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +66,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0 +67,1,1,1,1,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +68,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1,0,1,1,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0 +69,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,1,1,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +70,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,1,1,1,1,1,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +71,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,0,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +72,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,0,1,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0 +73,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,0,1,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0 +74,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,0,1,1,0,0,1,1,1 +75,1,0,0,0,1,1,1,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +76,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,0,1,1,0,0 +77,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,0,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0 +78,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,1,1,1,0,0,1,0,0,1,0,0,0,0 +79,1,1,1,0,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +80,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0 +81,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +82,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,1,0,0,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +83,1,1,1,1,1,1,1,0,0,0,1,0,0,1,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +84,1,1,1,0,0,0,1,1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0 +85,1,1,1,1,1,1,1,1,0,0,1,1,0,0,1,1,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +86,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0 +87,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1,1,0,1,0 +88,1,1,0,1,0,1,0,0,1,1,0,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +89,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1 +90,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,0,1,0,0,0,0,0,0,0,0,1 +91,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +92,0,1,1,1,0,1,0,1,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +93,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,1,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +94,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0 +95,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0 +96,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +97,1,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,1,1,0,0,0,1,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +98,1,0,1,0,1,1,1,1,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +99,1,1,1,1,0,1,1,1,0,0,1,1,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +100,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,0,0,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0 diff --git a/src/__tests__/__fixtures__/golden/true-thetas.csv b/src/__tests__/__fixtures__/golden/true-thetas.csv new file mode 100644 index 0000000..7b2ab6e --- /dev/null +++ b/src/__tests__/__fixtures__/golden/true-thetas.csv @@ -0,0 +1,101 @@ +pid,theta +1,1.5493 +2,0.9977 +3,0.2338 +4,0.6327 +5,-1.4211 +6,0.169 +7,-0.218 +8,0.6356 +9,-4 +10,0.3361 +11,0.3097 +12,-0.7065 +13,1.7331 +14,1.4874 +15,2.0524 +16,-2.1662 +17,1.1197 +18,3.6041 +19,1.1138 +20,0.8705 +21,-0.1034 +22,-2.6556 +23,-2.8022 +24,-0.9247 +25,1.4484 +26,-0.4201 +27,-2.6762 +28,-0.8653 +29,-0.9956 +30,2.6603 +31,0.4053 +32,0.977 +33,-2.5488 +34,-2.4594 +35,1.762 +36,0.9479 +37,-1.1426 +38,2.0587 +39,-4 +40,-0.4128 +41,0.6033 +42,-1.3407 +43,1.5728 +44,2.5059 +45,-1.0345 +46,-1.6164 +47,2.8799 +48,0.2852 +49,-0.9185 +50,1.4104 +51,1.1677 +52,-0.983 +53,1.062 +54,-0.2123 +55,0.2151 +56,-0.3377 +57,0.2349 +58,0.4129 +59,0.5499 +60,-0.1651 +61,-0.7659 +62,1.2564 +63,1.7459 +64,0.0958 +65,-1.0144 +66,3.0096 +67,-2.9184 +68,0.3598 +69,1.1969 +70,-0.3418 +71,0.5208 +72,0.3094 +73,1.0901 +74,2.8763 +75,-1.7941 +76,1.9174 +77,0.5807 +78,1.2489 +79,-0.8363 +80,-0.3064 +81,-0.0284 +82,-0.2343 +83,-1.5483 +84,-0.9578 +85,-1.0302 +86,0.9764 +87,3.2884 +88,-1.8712 +89,3.7134 +90,2.0752 +91,0.9696 +92,-1.6929 +93,0.9969 +94,0.8713 +95,0.2128 +96,0.183 +97,-0.8435 +98,-1.9091 +99,-1.5054 +100,1.5331 diff --git a/src/__tests__/golden.test.ts b/src/__tests__/golden.test.ts new file mode 100644 index 0000000..a8a6904 --- /dev/null +++ b/src/__tests__/golden.test.ts @@ -0,0 +1,131 @@ +/** + * Golden fixture tests. + * + * These tests validate the ability estimators against committed fixture data + * at two levels: + * + * 1. Characterization (always runs): jsCAT's estimates must exactly reproduce + * the committed baseline in expected-jscat.csv. This catches unintended + * numerical changes from refactors. If you change an algorithm + * intentionally, regenerate the baseline with + * `npm run fixtures:generate` and justify the change in your PR. + * + * 2. External reference (runs when expected-r-reference.csv is present): + * jsCAT's estimates must agree with the catR (R) reference implementation + * within tolerance. Generate the reference with + * `Rscript validation/generate-r-reference.R` and commit the CSV. + * + * See validation/README.md for the full workflow. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { Cat } from '..'; +import { Zeta } from '../type'; + +const FIXTURE_DIR = path.join(__dirname, '__fixtures__', 'golden'); +const R_REFERENCE_FILE = path.join(FIXTURE_DIR, 'expected-r-reference.csv'); + +// Tolerances for agreement with the external (catR) reference. jsCAT and catR +// use different optimizers and quadrature schemes, so exact equality is not +// expected. These bounds are intentionally provisional and should be tightened +// after the first committed regeneration if observed agreement allows. +const R_REFERENCE_MAX_ABS_DIFF = 0.05; +const R_REFERENCE_MEAN_ABS_DIFF = 0.01; + +// The characterization baseline is jsCAT's own output, so agreement should be +// exact up to floating point noise across platforms. +const CHARACTERIZATION_PRECISION = 6; // decimal places + +interface CsvRow { + [key: string]: string; +} + +function parseCsv(filepath: string): CsvRow[] { + const lines = fs.readFileSync(filepath, 'utf8').trim().split('\n'); + const headers = lines[0].split(',').map((h) => h.trim()); + return lines.slice(1).map((line) => { + const values = line.split(',').map((v) => v.trim()); + return headers.reduce((row, header, i) => { + row[header] = values[i] ?? ''; + return row; + }, {}); + }); +} + +function loadFixtures() { + const items = parseCsv(path.join(FIXTURE_DIR, 'items.csv')); + const responses = parseCsv(path.join(FIXTURE_DIR, 'responses.csv')); + + const zetas: Zeta[] = items.map((item) => ({ + a: parseFloat(item.a), + b: parseFloat(item.b), + c: parseFloat(item.c), + d: parseFloat(item.d), + })); + + const respRows = responses.map((row) => { + const resps = items.map((item) => parseInt(row[`i${item.item}`], 10) as 0 | 1); + return { pid: parseInt(row.pid, 10), resps }; + }); + + return { zetas, respRows }; +} + +function estimateAll(zetas: Zeta[], respRows: { pid: number; resps: (0 | 1)[] }[]) { + return respRows.map(({ pid, resps }) => { + const catMLE = new Cat({ method: 'MLE', minTheta: -6, maxTheta: 6 }); + catMLE.updateAbilityEstimate(zetas, resps); + + const catEAP = new Cat({ method: 'EAP', minTheta: -6, maxTheta: 6, priorDist: 'norm', priorPar: [0, 1] }); + catEAP.updateAbilityEstimate(zetas, resps); + + return { + pid, + theta_mle: catMLE.theta, + se_mle: catMLE.seMeasurement, + theta_eap: catEAP.theta, + se_eap: catEAP.seMeasurement, + }; + }); +} + +describe('Golden fixture tests', () => { + const { zetas, respRows } = loadFixtures(); + const estimates = estimateAll(zetas, respRows); + + describe('characterization baseline (expected-jscat.csv)', () => { + const baseline = parseCsv(path.join(FIXTURE_DIR, 'expected-jscat.csv')); + + it('has one baseline row per examinee', () => { + expect(baseline.length).toBe(respRows.length); + }); + + it.each(['theta_mle', 'se_mle', 'theta_eap', 'se_eap'])('reproduces the committed %s values', (column) => { + baseline.forEach((row, i) => { + const actual = estimates[i][column as keyof typeof estimates[number]] as number; + expect(actual).toBeCloseTo(parseFloat(row[column]), CHARACTERIZATION_PRECISION); + }); + }); + }); + + const describeRReference = fs.existsSync(R_REFERENCE_FILE) ? describe : describe.skip; + + describeRReference('external reference (expected-r-reference.csv, catR)', () => { + // Note: describe bodies run during collection even when skipped, so only + // parse the reference file when it exists. + const reference = fs.existsSync(R_REFERENCE_FILE) ? parseCsv(R_REFERENCE_FILE) : []; + + it.each(['theta_mle', 'theta_eap'])('agrees with catR on %s within tolerance', (column) => { + const diffs = reference.map((row, i) => { + const actual = estimates[i][column as keyof typeof estimates[number]] as number; + return Math.abs(actual - parseFloat(row[column])); + }); + + const maxDiff = Math.max(...diffs); + const meanDiff = diffs.reduce((acc, d) => acc + d, 0) / diffs.length; + + expect(maxDiff).toBeLessThan(R_REFERENCE_MAX_ABS_DIFF); + expect(meanDiff).toBeLessThan(R_REFERENCE_MEAN_ABS_DIFF); + }); + }); +}); diff --git a/validation/README.md b/validation/README.md new file mode 100644 index 0000000..368715f --- /dev/null +++ b/validation/README.md @@ -0,0 +1,43 @@ +# Validation + +jsCAT validates its psychometric algorithms against independent reference implementations in R ([catR](https://cran.r-project.org/package=catR) and [mirt](https://cran.r-project.org/package=mirt)). Validation happens at two levels. + +## 1. Automated golden tests (run on every CI build) + +`src/__tests__/golden.test.ts` scores a committed fixture set (50-item bank × 100 simulated examinees) with every ability estimator and asserts two things: + +1. **Characterization**: estimates exactly reproduce the committed baseline (`expected-jscat.csv`). This catches unintended numerical changes from refactors. +2. **External reference**: estimates agree with catR within tolerance (`expected-r-reference.csv`). This catches algorithmic errors. These tests skip automatically if the reference file has not been generated yet. + +### Regenerating fixtures + +The fixture inputs (item bank, responses) are generated deterministically from a fixed seed: + +```bash +npm run fixtures:generate # writes src/__tests__/__fixtures__/golden/ +Rscript validation/generate-r-reference.R # writes expected-r-reference.csv (requires R + catR) +``` + +Regenerate `expected-jscat.csv` **only** when you intentionally change an algorithm's numerical behavior, and explain why in your PR. Regenerate `expected-r-reference.csv` whenever the fixture inputs change or a new estimator is added. + +### Adding a new estimator to the golden tests + +When you add an estimator (see CONTRIBUTING.md), extend three places: + +1. `scripts/generate-golden-fixtures.js` — add a column for the new estimator to `expected-jscat.csv` +2. `validation/generate-r-reference.R` — add the corresponding catR/mirt scoring call (e.g., `method = "WL"` in `thetaEst` for WLE) +3. `src/__tests__/golden.test.ts` — add the column to the comparison lists + +## 2. Exploratory validation (manual, per release or per new algorithm) + +`analysis/jsCAT-validation.Rmd` is the original simulation-based comparison of jsCAT against mirt and catR, producing the plots in `plots/` that are shown in the README. Run it when adding a new algorithm or preparing a release: + +1. Install R (≥ 4.0) with `mirt`, `catR`, `tidyverse` +2. Build jsCAT: `npm run build` +3. Knit `analysis/jsCAT-validation.Rmd` from the `analysis/` directory + +The `simulation/` directory contains the Node scripts the Rmd shells out to. + +## Why both levels? + +The golden tests are the enforcement mechanism: they run on every PR, require no R at test time, and fail loudly if an estimator drifts from the reference. The Rmd is the exploration mechanism: it covers adaptive item-selection trajectories and produces human-readable plots, but nothing fails automatically if it rots. A plot is documentation; a test is a guarantee. diff --git a/validation/generate-r-reference.R b/validation/generate-r-reference.R new file mode 100644 index 0000000..d63938c --- /dev/null +++ b/validation/generate-r-reference.R @@ -0,0 +1,56 @@ +#!/usr/bin/env Rscript + +# Generate the independent reference estimates for the golden fixture tests. +# +# Reads the item bank and simulated responses produced by +# scripts/generate-golden-fixtures.js and scores every examinee with the catR +# package (Magis et al.), writing the reference thetas to +# src/__tests__/__fixtures__/golden/expected-r-reference.csv. +# +# Once that file is committed, src/__tests__/golden.test.ts automatically +# compares jsCAT's MLE and EAP estimates against it within tolerance on every +# test run — no R required at test time. +# +# Usage (from the repo root): +# Rscript validation/generate-r-reference.R +# +# Requirements: R >= 4.0, catR (install.packages("catR")) + +library(catR) + +fixture_dir <- file.path("src", "__tests__", "__fixtures__", "golden") + +items <- read.csv(file.path(fixture_dir, "items.csv")) +responses <- read.csv(file.path(fixture_dir, "responses.csv")) + +# catR expects an item parameter matrix with columns a, b, c, d +it <- as.matrix(items[, c("a", "b", "c", "d")]) +resp_matrix <- as.matrix(responses[, grepl("^i", names(responses))]) + +# Match jsCAT's settings: theta bounds [-6, 6]; EAP with a standard normal +# prior. jsCAT quantizes the EAP prior on a 0.1-step grid over [-6, 6] +# (121 points), so we use 121 quadrature points here as well. +theta_mle <- apply(resp_matrix, 1, function(x) { + thetaEst(it, x, method = "ML", range = c(-6, 6)) +}) + +theta_eap <- apply(resp_matrix, 1, function(x) { + thetaEst( + it, x, + method = "EAP", + priorDist = "norm", + priorPar = c(0, 1), + lower = -6, + upper = 6, + nqp = 121 + ) +}) + +out <- data.frame(pid = responses$pid, theta_mle = theta_mle, theta_eap = theta_eap) +write.csv(out, file.path(fixture_dir, "expected-r-reference.csv"), row.names = FALSE) + +cat(sprintf( + "Wrote catR reference estimates for %d examinees to %s\n", + nrow(out), + file.path(fixture_dir, "expected-r-reference.csv") +)) From 0cbd7a56556f69fe52e707d359dc7ca1d49efb2c Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 21:47:59 +0000 Subject: [PATCH 04/11] Add contribution guide, PR template, and AI agent rules CONTRIBUTING.md documents the registry architecture and walks through adding a new ability estimator end to end, including the validation requirements (literature reference, analytic tests, golden fixtures vs. catR). The PR template encodes the algorithmic-contribution checklist and an AI-disclosure section. .ai/rules/ provides machine-readable rules for AI coding agents, surfaced via AGENTS.md and a CLAUDE.md symlink. --- .ai/rules/README.md | 32 +++++ .ai/rules/architecture-extension-points.md | 59 ++++++++++ .ai/rules/quality-api-stability.md | 31 +++++ .ai/rules/quality-pr-creation.md | 33 ++++++ .ai/rules/quality-typescript-strictness.md | 38 ++++++ .ai/rules/science-numerical-correctness.md | 51 ++++++++ .ai/rules/science-validation-required.md | 49 ++++++++ .ai/rules/testing-expectations.md | 44 +++++++ .github/CONTRIBUTING.md | 131 +++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 27 +++++ AGENTS.md | 73 ++++++++++++ CLAUDE.md | 1 + README.md | 6 + 13 files changed, 575 insertions(+) create mode 100644 .ai/rules/README.md create mode 100644 .ai/rules/architecture-extension-points.md create mode 100644 .ai/rules/quality-api-stability.md create mode 100644 .ai/rules/quality-pr-creation.md create mode 100644 .ai/rules/quality-typescript-strictness.md create mode 100644 .ai/rules/science-numerical-correctness.md create mode 100644 .ai/rules/science-validation-required.md create mode 100644 .ai/rules/testing-expectations.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 AGENTS.md create mode 120000 CLAUDE.md diff --git a/.ai/rules/README.md b/.ai/rules/README.md new file mode 100644 index 0000000..ed07c56 --- /dev/null +++ b/.ai/rules/README.md @@ -0,0 +1,32 @@ +# jsCAT Engineering Rules + +Modular, enforceable engineering rules for jsCAT. Rules are both human-readable (for contributors to browse) and machine-readable (for AI coding tools to consume automatically). jsCAT is a psychometrics library used in live assessments, so the rules weight scientific correctness above all else. + +## Rules index + +| Rule | Impact | Description | +|------|--------|-------------| +| [architecture-extension-points](architecture-extension-points.md) | CRITICAL | Estimators, selectors, and stopping rules are added via registries/interfaces, never by editing `Cat` dispatch | +| [science-validation-required](science-validation-required.md) | CRITICAL | Every new or changed algorithm ships a literature reference, analytic tests, and golden fixtures vs. catR/mirt | +| [science-numerical-correctness](science-numerical-correctness.md) | HIGH | Log-space likelihoods, seeded RNG only, optimizer conventions, documented model scope | +| [testing-expectations](testing-expectations.md) | HIGH | Three test tiers (analytic, golden, behavioral); explicit tolerances; never snapshot floats | +| [quality-typescript-strictness](quality-typescript-strictness.md) | HIGH | Literal unions over raw strings, no `as any`, strict mode conventions | +| [quality-api-stability](quality-api-stability.md) | MEDIUM | Public API is semver-bound; both zeta formats keep working; case-insensitive method names | +| [quality-pr-creation](quality-pr-creation.md) | MEDIUM | Branch naming, draft PRs, the algorithmic checklist, AI disclosure | + +## Impact levels + +- **CRITICAL**: Violations risk shipping incorrect ability estimates to real assessments. Must always be followed. +- **HIGH**: Violations cause architectural inconsistency or undermine the validation safety net. +- **MEDIUM**: Best practices for consistency. Follow for new code. + +## How AI tools discover these rules + +- **Claude Code / Cowork**: `CLAUDE.md` at the repo root (symlink to `AGENTS.md`). +- **Cursor / Copilot and others**: `AGENTS.md` at the repo root. + +Both point here. Each rule is self-contained with incorrect/correct examples. + +## Contributing a rule + +Add or update rules in the same PR that introduces or changes the pattern they describe. Keep the set small (under ~10 rules); prune rather than accumulate. diff --git a/.ai/rules/architecture-extension-points.md b/.ai/rules/architecture-extension-points.md new file mode 100644 index 0000000..621a8ca --- /dev/null +++ b/.ai/rules/architecture-extension-points.md @@ -0,0 +1,59 @@ +--- +title: Algorithm extension points +description: Estimators, item selectors, and stopping criteria are added through their registries and interfaces — never by adding dispatch logic to Cat or Clowder. +impact: CRITICAL +scope: all +tags: architecture, estimators, selectors, registry +--- + +## Algorithm extension points + +jsCAT's algorithms are pluggable. Each algorithm family has an interface, a directory, and a registry that serves as the single source of truth for the family's type union, runtime validation, and dispatch. Adding an algorithm means adding a file and one registry entry — existing code, especially `Cat`, does not change. + +| Family | Interface | Directory | Registry | +|--------|-----------|-----------|----------| +| Ability estimators | `AbilityEstimator` | `src/estimators/` | `src/estimators/registry.ts` (`ABILITY_ESTIMATORS`) | +| Item selectors | `ItemSelector` | `src/selectors/` | `src/selectors/registry.ts` (`SELECTORS` + allowlists) | +| Stopping criteria | `EarlyStopping` (abstract class) | `src/stopping.ts` | exported subclasses | + +### Incorrect + +```typescript +// Adding an estimator by editing Cat's dispatch — every new algorithm +// makes the most safety-critical file in the library grow +// src/cat.ts +if (method === 'eap') { + this._theta = this.estimateAbilityEAP(); +} else if (method === 'mle') { + this._theta = this.estimateAbilityMLE(); +} else if (method === 'wle') { // ❌ new branch in Cat + this._theta = this.estimateAbilityWLE(); // ❌ new private method in Cat +} + +// ❌ and separately maintaining the validation list by hand +const validMethods: Array = ['mle', 'eap', 'wle']; +``` + +### Correct + +```typescript +// 1. New file: src/estimators/wle.ts +export class WLEEstimator implements AbilityEstimator { + estimateAbility(context: EstimationContext): number { /* ... */ } +} + +// 2. One entry in src/estimators/registry.ts +export const ABILITY_ESTIMATORS = { + mle: new MLEEstimator(), + eap: new EAPEstimator(), + wle: new WLEEstimator(), // ← the only edit to existing code +} as const; + +// 3. Re-export from src/estimators/index.ts +``` + +The `EstimationMethod` type, `validateEstimationMethod`, and `Cat`'s dispatch all derive from the registry object automatically. + +### The principle + +`Cat` orchestrates; it does not implement algorithms. When dispatch lives in one registry, a reviewer of an algorithmic PR reads exactly one new file plus one-line diffs — there is no opportunity to silently alter MLE while adding WLE. This is what makes AI-generated algorithm contributions reviewable: the blast radius of a new estimator is structurally confined. diff --git a/.ai/rules/quality-api-stability.md b/.ai/rules/quality-api-stability.md new file mode 100644 index 0000000..96fb43b --- /dev/null +++ b/.ai/rules/quality-api-stability.md @@ -0,0 +1,31 @@ +--- +title: Public API stability +description: The npm package is semver-bound; preserve both zeta formats, case-insensitive method names, and existing error messages. +impact: MEDIUM +scope: all +tags: api, semver, compatibility +--- + +## Public API stability + +`@bdelab/jscat` is consumed by ROAR assessments in production and by external users. Everything exported from `src/index.ts` is public API and changes to it are semver events. + +### Invariants to preserve + +- **Both zeta formats work everywhere**: symbolic (`a, b, c, d`) and semantic (`discrimination, difficulty, guessing, slipping`) item parameters are interchangeable inputs. New code must handle both (use `fillZetaDefaults` / `convertZeta`); tests loop over both formats. +- **Method names are case-insensitive**: `'MLE'`, `'mle'`, and `'Mle'` are all valid and normalize to lowercase. Constructor inputs stay loosely typed (`EstimationMethodInput`) for this reason. +- **Error messages are contract**: existing throw messages (e.g., `'The abilityEstimator you provided is not in the list of valid methods'`) are asserted by downstream tests. Don't reword them casually. +- **Getters keep returning live state**: `theta`, `seMeasurement`, `nItems`, `resps`, `zetas`, `prior` on `Cat`; the corresponding aggregates on `Clowder`. +- **Mutation semantics of `findNextItem`**: `deepCopy = true` by default; `deepCopy = false` is documented to mutate the caller's array. Don't change either default. + +### Changes that require a major version + +Removing or renaming exports, narrowing accepted input types, changing default parameter values (`minTheta`, `maxTheta`, prior defaults, `startSelect`), or changing numerical behavior of an existing estimator (also see `science-validation-required.md` — baseline regeneration plus justification). + +### Changes that are minor + +Adding estimators/selectors/stopping criteria via the registries, adding optional fields to context interfaces, adding new exports. + +### The principle + +Assessment results must be comparable across time. A silent change to a default prior or an estimator's behavior changes children's scores between sessions of the same study. Additive evolution through the registries is cheap; behavioral change is expensive and must be deliberate, versioned, and announced. diff --git a/.ai/rules/quality-pr-creation.md b/.ai/rules/quality-pr-creation.md new file mode 100644 index 0000000..ef44879 --- /dev/null +++ b/.ai/rules/quality-pr-creation.md @@ -0,0 +1,33 @@ +--- +title: PR creation +description: Branch naming, draft mode, the algorithmic-contribution checklist, and AI disclosure. +impact: MEDIUM +scope: all +tags: process, git, prs +--- + +## PR creation + +### Branch naming + +`/` with prefixes: `enh/` (features), `fix/` (bug fixes), `refactor/`, `maint/`, `dep/` (dependency bumps). Example: `enh/wle-estimation`. + +### Commits and titles + +Imperative verb phrase, sentence-cased, no trailing period: `Add WLE ability estimator`, `Fix seeded RNG usage in Clowder item selection`. PR titles follow the same convention. + +### Draft mode and checks + +Open PRs as drafts. Before marking ready: `npm test && npm run lint` locally, and fill in the PR template checklist — the algorithmic checklist is mandatory for any change under `src/estimators/`, `src/selectors/`, `src/stopping.ts`, or to `itemResponseFunction`/`fisherInformation`/`logLikelihood`. + +### AI disclosure + +State in the PR description whether and how AI tools were used. This is not a gate — AI-assisted PRs are welcome — it tells reviewers where to spend attention: generated structure gets normal review; generated numerics get verified through the golden tests, not by trust. + +### Scope discipline + +One concern per PR. If you notice a small adjacent improvement (a misleading name, a missing JSDoc), fix it in the same PR rather than leaving a TODO — but split genuinely unrelated changes. A new estimator PR should contain: the estimator file, the registry entry, the export, tests, regenerated fixtures, and docs — and nothing else. + +### The principle + +The PR is the unit of scientific review here. A tightly-scoped PR with disclosed provenance and machine-checkable validation can be reviewed in minutes with high confidence; a sprawling one can only be skimmed and trusted. diff --git a/.ai/rules/quality-typescript-strictness.md b/.ai/rules/quality-typescript-strictness.md new file mode 100644 index 0000000..06b2ebb --- /dev/null +++ b/.ai/rules/quality-typescript-strictness.md @@ -0,0 +1,38 @@ +--- +title: TypeScript strictness +description: Strict mode conventions — literal unions derived from registries, no `as any`, narrow escape hatches. +impact: HIGH +scope: all +tags: typescript, types +--- + +## TypeScript strictness + +The package compiles with `strict: true`. Work with the type system, not around it. + +### Literal unions derive from the registries + +Method names are typed as unions derived from the registry objects — never as free-standing string lists that can drift from the implementation: + +```typescript +// ✅ src/estimators/registry.ts — the object is the source of truth +export const ABILITY_ESTIMATORS = { mle: ..., eap: ... } as const; +export type EstimationMethod = keyof typeof ABILITY_ESTIMATORS; + +// ❌ a hand-maintained list that can disagree with the registry +const validMethods: Array = ['mle', 'eap']; +``` + +Public inputs use the `Input` variants (e.g., `EstimationMethodInput`), which add `(string & Record)` so editors autocomplete the canonical names while arbitrary-case strings ('MLE', 'miDdle') keep compiling and are normalized at runtime. Don't narrow public inputs to the bare unions — that's a breaking change for JS consumers and case-variant callers. + +### No `as any` + +Use `as const` for literal types, generics for reusable helpers, and `// eslint-disable-next-line @typescript-eslint/no-non-null-assertion` scoped to a single line where a value is guaranteed by a prior check (the codebase uses `difficulty!` after `fillZetaDefaults`). If a third-party module lacks types, add a minimal `.d.ts` (see `src/optimization-js.d.ts`) rather than casting call sites. + +### Interfaces over structural drift + +New estimators/selectors implement the published interfaces (`AbilityEstimator`, `ItemSelector`). Don't add untyped extra parameters to `estimateAbility`/`select` — extend `EstimationContext`/`SelectorContext` instead, with optional fields and JSDoc, so all implementations stay call-compatible. + +### The principle + +In a library consumed by both TS and JS users, types are API documentation that the compiler enforces. Deriving them from runtime objects means the docs cannot lie; an AI or human contributor who invents a method name gets a compile error instead of a silent fallthrough. diff --git a/.ai/rules/science-numerical-correctness.md b/.ai/rules/science-numerical-correctness.md new file mode 100644 index 0000000..5375fba --- /dev/null +++ b/.ai/rules/science-numerical-correctness.md @@ -0,0 +1,51 @@ +--- +title: Numerical correctness conventions +description: Likelihood, optimization, and randomness conventions that keep estimates correct and simulations reproducible. +impact: HIGH +scope: all +tags: numerics, likelihood, rng, optimization +--- + +## Numerical correctness conventions + +### Likelihoods live in log space, computed once + +All likelihood-based estimators use `logLikelihood` from `src/estimators/log-likelihood.ts`. Do not reimplement it per estimator, and do not work with raw likelihood products (they underflow beyond a few dozen items). + +```typescript +// ❌ Incorrect: private likelihood with a nonzero reduce seed (returns ln L + 1) +private likelihood(theta: number) { + return this._zetas.reduce((acc, zeta, i) => /* ... */, 1); +} + +// ✅ Correct: the shared helper, seeded at 0 +import { logLikelihood } from './log-likelihood'; +``` + +### Optimization goes through the shared scaffold + +Estimators that maximize an objective use `maximizeOverTheta` from `src/estimators/optimize.ts`. It owns the optimizer choice (Powell), the start value, and the negation convention. Powell searches unbounded; `Cat` clamps the result to `[minTheta, maxTheta]` afterward — document, don't duplicate, this behavior. Guard objectives against domain errors (e.g., `Math.log` of a non-positive information sum). + +### Randomness is always seeded + +Every random draw goes through the Cat/Clowder seeded RNG (`this._rng` or the `randomInteger` provided in `SelectorContext`). `Math.random()` is forbidden in `src/` — it silently breaks the reproducibility that `randomSeed` promises to simulation users. + +```typescript +// ❌ Incorrect +const random = Math.random(); + +// ✅ Correct +const random = this._rng(); +``` + +### Model scope is explicit + +jsCAT accepts 4PL parameters (a, b, c, d) everywhere. If an algorithm's theory only covers a subset (e.g., a bias correction exact for 1PL/2PL but approximate under 3PL/4PL), say so in the JSDoc and validate at least the exact scope against the R reference. + +### Magic constants get derivations + +Numerical constants carry a comment deriving them (see `CLOSEST_SELECTION_OFFSET` in `src/selectors/closest.ts`: 0.481 = ln((1+√5)/2), the information-maximizing offset for c = 0.5). An underived constant is unreviewable. + +### The principle + +Numerical bugs don't throw — they return plausible wrong numbers. Conventions that centralize the likelihood, the optimizer, and the RNG mean there is exactly one place for each class of bug to live, and the golden tests watch that place. diff --git a/.ai/rules/science-validation-required.md b/.ai/rules/science-validation-required.md new file mode 100644 index 0000000..8e06cf0 --- /dev/null +++ b/.ai/rules/science-validation-required.md @@ -0,0 +1,49 @@ +--- +title: Scientific validation required +description: Every new or changed psychometric algorithm must ship a literature reference, analytic unit tests, and golden fixture tests against an independent reference implementation (catR/mirt) that run in CI. +impact: CRITICAL +scope: all +tags: validation, golden-tests, psychometrics +--- + +## Scientific validation required + +jsCAT scores real assessments. An algorithm that "looks right" and passes structural tests can still be numerically wrong. Every PR that adds or changes an estimator, selector, information function, or likelihood must carry three kinds of evidence, and the evidence must be executable — a validation plot in the README is documentation, not proof. + +### Required evidence + +1. **Literature reference** in the class JSDoc (`@remarks` + Reference line: author, year, journal), plus documented model scope — which IRT models (1PL/2PL/3PL/4PL) the implementation is exact for and what happens outside that scope. +2. **Analytic unit tests**: at least one closed-form case with the derivation in a comment (e.g., a single Rasch item answered correctly gives WLE θ = ln 3 ≈ 1.0986). +3. **Golden fixture tests**: regenerate `src/__tests__/__fixtures__/golden/` via `npm run fixtures:generate`, regenerate the independent reference via `Rscript validation/generate-r-reference.R` (catR), and extend `src/__tests__/golden.test.ts` with the new columns. The committed CSVs make the validation run in CI on every future PR, with no R at test time. + +### Incorrect + +```typescript +// ❌ New estimator validated only by a plot generated once and committed as a PNG +// ❌ Tests that assert structure, not values: +it('estimates ability', () => { + cat.updateAbilityEstimate(zetas, resps); + expect(typeof cat.theta).toBe('number'); // passes for any wrong answer +}); +// ❌ Changing the characterization baseline (expected-jscat.csv) without +// explaining the numerical change in the PR description +``` + +### Correct + +```typescript +// Analytic test with derivation +it('correctly updates ability estimate through WLE', () => { + // Single Rasch item (a=1, b=0), correct response. + // WLE score equation: (1−σ) + 0.5·(1−2σ) = 0 ⟹ σ = 0.75 ⟹ θ = logit(0.75) = ln 3 + const cat = new Cat({ method: 'WLE' }); + cat.updateAbilityEstimate({ a: 1, b: 0, c: 0, d: 1 }, 1); + expect(cat.theta).toBeCloseTo(Math.log(3), 2); +}); +``` + +Plus regenerated golden fixtures with the catR `method = "WL"` reference column, asserted within the tolerances defined in `golden.test.ts`. + +### The principle + +This is how SciPy-class projects stay correct: reference values from an independent implementation, committed to the repo, asserted on every CI run. A reviewer cannot referee numerical code by reading it — but a passing golden test against catR cannot be faked by plausible-looking code, human- or AI-written. Validation that doesn't run automatically will silently rot the first time someone touches the algorithm. diff --git a/.ai/rules/testing-expectations.md b/.ai/rules/testing-expectations.md new file mode 100644 index 0000000..e01af73 --- /dev/null +++ b/.ai/rules/testing-expectations.md @@ -0,0 +1,44 @@ +--- +title: Testing expectations +description: Three tiers of tests for psychometric code — analytic, golden, and behavioral — with explicit numeric tolerances. +impact: HIGH +scope: all +tags: testing, tolerances, golden-tests +--- + +## Testing expectations + +### The three tiers + +1. **Analytic tests** — closed-form cases with the derivation in a comment. The strongest evidence: no reference software needed, exact expected value known. +2. **Golden tests** — `src/__tests__/golden.test.ts` compares every estimator against (a) a committed characterization baseline (exact reproduction) and (b) an independent catR reference (within tolerance). New estimators must be wired in; see `validation/README.md`. +3. **Behavioral/property tests** — invariants that hold regardless of exact values: estimates respect `[minTheta, maxTheta]`; `seMeasurement` is finite and positive after informative items; symbolic (`a,b,c,d`) and semantic (`discrimination,difficulty,...`) parameter formats produce identical results (the existing test suites loop over both formats — keep that pattern); WLE is less extreme than MLE for extreme response patterns; etc. + +### Numeric assertions use explicit tolerances + +```typescript +// ✅ tolerance is visible and justified +expect(cat.theta).toBeCloseTo(Math.log(3), 2); +expect(maxDiff).toBeLessThan(R_REFERENCE_MAX_ABS_DIFF); // named constant with comment + +// ❌ snapshot of floating point output — breaks on any platform/optimizer noise, +// and a careless `--update` silently rewrites the expected science +expect(cat.theta).toMatchSnapshot(); + +// ❌ structure-only assertion — passes for any wrong number +expect(typeof cat.theta).toBe('number'); +``` + +### Test data conventions + +- Fixtures live in `src/__tests__/__fixtures__/` (excluded from test discovery by jest config). +- Fixture generation is deterministic: fixed seeds via `seedrandom`, generation scripts committed (`scripts/generate-golden-fixtures.js`). +- Don't hand-edit generated fixture CSVs; regenerate them. + +### When behavior intentionally changes + +Regenerating `expected-jscat.csv` is allowed only with an intentional, explained algorithm change. The PR description must state which values moved and why; the catR reference comparison must still pass. + +### The principle + +Tests for scientific code answer two different questions: "is it right?" (analytic + golden) and "did it change?" (characterization). Keep both. A test suite that only checks shapes and types will happily certify an estimator that's off by a sign. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index e69de29..4c401ee 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,131 @@ +# Contributing to jsCAT + +Thank you for contributing! jsCAT is a research software project: it implements psychometric algorithms whose correctness matters for real assessments of real students. This guide explains how to set up a development environment, how the codebase is organized, and — most importantly — what we require before merging a new algorithm. + +AI-assisted contributions are welcome (many of our best PRs are AI-generated). See [AI-assisted contributions](#ai-assisted-contributions) for the disclosure policy, and note that the requirements below are designed so that a reviewer can verify correctness without trusting the author or the tool that produced the code. + +## Development setup + +```bash +git clone https://github.com/yeatmanlab/jsCAT.git +cd jsCAT +nvm use # respects .nvmrc +npm ci +npm test # Jest unit + golden tests +npm run lint # ESLint +npm run build # tsc → lib/ +``` + +Optional, for regenerating validation references: R ≥ 4.0 with the `catR` package (and `mirt` + `tidyverse` for the exploratory Rmd in `validation/`). + +## Architecture overview + +``` +src/ + cat.ts # Cat class: orchestrates estimation + item selection + clowder.ts # Clowder: manages multiple Cats over a shared corpus + corpus.ts # Zeta (item parameter) validation and format conversion + estimators/ # Ability estimators (MLE, EAP, ...) — registry pattern + selectors/ # Item selectors (MFI, closest, random, fixed, middle) — registry pattern + stopping.ts # Early-stopping criteria — abstract base class pattern + utils.ts # IRF, Fisher information, distributions + type.ts # Zeta, Stimulus, and related types +``` + +The key design rule: **algorithms are added by creating new files, not by editing existing ones.** Estimators implement the `AbilityEstimator` interface and are registered in `src/estimators/registry.ts`. Selectors implement `ItemSelector` and are registered in `src/selectors/registry.ts`. Stopping criteria subclass `EarlyStopping` in `src/stopping.ts`. The `Cat` class dispatches through the registries and should not need changes when you add an algorithm. + +## Tutorial: adding a new ability estimator + +Suppose you want to add WLE (Warm's weighted likelihood estimation). The complete change is: + +**1. Create `src/estimators/wle.ts`** implementing the interface: + +```typescript +import { AbilityEstimator, EstimationContext } from './types'; +import { logLikelihood } from './log-likelihood'; +import { maximizeOverTheta } from './optimize'; +import { fisherInformation } from '../utils'; + +/** + * Weighted Likelihood Estimation (WLE) of ability. + * + * @remarks + * Document the math, the IRT models it is exact for, and any approximations. + * + * Reference: Warm, T. A. (1989). Weighted likelihood estimation of ability in + * item response theory. Psychometrika, 54(3), 427-450. + */ +export class WLEEstimator implements AbilityEstimator { + estimateAbility(context: EstimationContext): number { + const { zetas, resps } = context; + return maximizeOverTheta((theta) => { + const totalInfo = zetas.reduce((sum, zeta) => sum + fisherInformation(theta, zeta), 0); + if (totalInfo <= 0) return logLikelihood(theta, zetas, resps); + return logLikelihood(theta, zetas, resps) + 0.5 * Math.log(totalInfo); + }); + } +} +``` + +**2. Register it** in `src/estimators/registry.ts`: + +```typescript +export const ABILITY_ESTIMATORS = { + mle: new MLEEstimator(), + eap: new EAPEstimator(), + wle: new WLEEstimator(), // ← the only edit to existing code +} as const; +``` + +The `EstimationMethod` type, runtime validation, `Cat` dispatch, and error messages all update automatically. Export the class from `src/estimators/index.ts`. + +**3. Add unit tests** in `src/__tests__/cat.test.ts` (or a new file). At minimum: + +- An **analytic test**: a case with a closed-form answer (e.g., one Rasch item answered correctly gives a WLE of ln 3 — show the derivation in a comment). +- A **behavioral test**: a property the estimator must satisfy (e.g., |WLE| < |MLE| for extreme response patterns). +- A **bounds test**: estimates respect `minTheta`/`maxTheta` and produce finite `seMeasurement`. + +**4. Wire it into the golden tests** (see `validation/README.md`): + +- Add the estimator to `scripts/generate-golden-fixtures.js` and run `npm run fixtures:generate`. +- Add the corresponding reference call to `validation/generate-r-reference.R` (catR's `thetaEst` supports `method = "WL"`) and run it with `Rscript`. +- Add the new columns to `src/__tests__/golden.test.ts`. +- Commit the regenerated CSVs. The PR must show the golden test passing against the independent R reference. + +**5. Document it**: update the README's method list and add the validation plot if you ran the exploratory Rmd. + +Adding an item selector or stopping criterion follows the same shape — implement the interface (`src/selectors/types.ts`) or subclass `EarlyStopping` (`src/stopping.ts`), register, test. + +## Requirements for algorithmic PRs + +Every PR that adds or changes a psychometric algorithm must include: + +1. **A literature reference** for the method, cited in the class JSDoc (author, year, journal). +2. **Documented scope**: which IRT models (1PL/2PL/3PL/4PL) the implementation is exact for, and what happens outside that scope. +3. **Analytic unit tests** for closed-form cases, with the derivation in a comment. +4. **Golden test coverage**: committed fixtures showing agreement with an independent reference implementation (catR or mirt), passing in CI. A validation plot alone is not sufficient — plots rot, tests don't. +5. **No new dispatch logic** outside the registries. + +PRs that change existing numerical behavior must regenerate the characterization baseline (`npm run fixtures:generate`) and justify the change in the PR description. + +## AI-assisted contributions + +We welcome PRs drafted with AI coding tools. Two requirements: + +1. **Disclose it** in the PR description (tool and rough extent, e.g., "implementation and tests generated with Claude, validation design by me"). +2. **The validation evidence must be machine-checkable.** Reviewers will not line-by-line trust generated numerical code; they will trust a passing golden test against committed catR/mirt reference values, because that cannot be pattern-matched into existence. + +If you are using an AI agent in this repo, point it at `CLAUDE.md` / `AGENTS.md` and the rules in `.ai/rules/` — they encode everything on this page in machine-readable form. + +## Style and process + +- TypeScript strict mode; no `as any`; prefer the literal-union types exported from the registries over raw strings. +- All public functions get JSDoc with `@param`/`@returns`; algorithm classes additionally get `@remarks` with the math and a reference. +- Prettier and ESLint are enforced in CI: `npm run lint && npm run format`. +- Branch naming: `enh/`, `fix/`, `refactor/`, `maint/`, `dep/` prefixes (e.g., `enh/wle-estimation`). +- Open PRs in draft mode; mark ready for review once CI is green. +- Run `npm test && npm run lint` locally before pushing. + +## Questions + +Open a [GitHub issue](https://github.com/yeatmanlab/jsCAT/issues) — including for questions about whether an algorithmic idea fits the library before you build it. We'd rather discuss the design first. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6ca16c8 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,27 @@ +## Summary + + + +## Checklist (all PRs) + +- [ ] Tests added/updated at the right level for the change +- [ ] `npm test` and `npm run lint` pass locally +- [ ] AI assistance disclosed below (if any) + +## Checklist (PRs that add or change a psychometric algorithm) + +- [ ] Literature reference cited in the class JSDoc +- [ ] Documented model scope (1PL/2PL/3PL/4PL; exact vs. approximate) +- [ ] Analytic unit test(s) for closed-form cases, with derivation in a comment +- [ ] Algorithm registered via the registry (`src/estimators/registry.ts` / `src/selectors/registry.ts`) — no new dispatch logic elsewhere +- [ ] Golden fixtures regenerated (`npm run fixtures:generate`) and R reference regenerated (`Rscript validation/generate-r-reference.R`) +- [ ] Golden tests pass against the independent R reference +- [ ] If existing numerical behavior changed: justification included below + +## AI assistance disclosure + + + +## Validation evidence + + diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a1c9a88 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,73 @@ +# jsCAT Development Guide for AI Agents + +You are working in jsCAT, a TypeScript library for IRT-based computer adaptive testing used in production assessments of real students (the ROAR platform). Scientific correctness outranks everything else: a plausible-but-wrong estimator silently corrupts children's scores. + +## Do + +- Add algorithms through the extension points, never by editing dispatch logic (see [architecture-extension-points](.ai/rules/architecture-extension-points.md)): + - Ability estimators: implement `AbilityEstimator`, register in `src/estimators/registry.ts` + - Item selectors: implement `ItemSelector`, register in `src/selectors/registry.ts` + - Stopping criteria: subclass `EarlyStopping` in `src/stopping.ts` +- Ship validation with every algorithm: literature reference in JSDoc, analytic unit tests with derivations, regenerated golden fixtures vs. the catR reference (see [science-validation-required](.ai/rules/science-validation-required.md)) +- Use the shared numerics: `logLikelihood`, `maximizeOverTheta`, the seeded RNG (see [science-numerical-correctness](.ai/rules/science-numerical-correctness.md)) +- Support both zeta formats (symbolic `a,b,c,d` and semantic `discrimination,difficulty,guessing,slipping`) in any code touching item parameters +- Write numeric assertions with explicit tolerances (see [testing-expectations](.ai/rules/testing-expectations.md)) +- Follow the tutorial in [.github/CONTRIBUTING.md](.github/CONTRIBUTING.md) when adding an estimator — it walks through the complete change + +## Don't + +- Never add `if/else` method dispatch to `Cat` or hand-maintained `validMethods` arrays — the registries are the single source of truth +- Never use `Math.random()` in `src/` — use the seeded RNG (`this._rng` / `SelectorContext.randomInteger`) +- Never reimplement the likelihood, IRF, or Fisher information per estimator — import the shared functions +- Never use `as any` or snapshot-test floating point values +- Never change existing error message strings, default parameter values, or numerical behavior without flagging it as a breaking/behavioral change (see [quality-api-stability](.ai/rules/quality-api-stability.md)) +- Never hand-edit generated fixture CSVs in `src/__tests__/__fixtures__/golden/` — regenerate them +- Never present a validation plot as sufficient evidence — wire the comparison into the golden tests so it runs in CI + +## Commands + +```bash +npm test # Jest unit + golden tests +npm run lint # ESLint +npm run build # tsc → lib/ +npm run fixtures:generate # regenerate golden fixtures (build + script) +Rscript validation/generate-r-reference.R # regenerate catR reference (requires R) +``` + +## Project structure + +``` +src/ + cat.ts # Cat: orchestrates estimation + item selection (do not add algorithms here) + clowder.ts # Clowder: multiple Cats over a shared corpus + corpus.ts # Zeta validation + symbolic/semantic conversion + estimators/ # AbilityEstimator implementations + registry + selectors/ # ItemSelector implementations + registry + stopping.ts # EarlyStopping abstract class + subclasses + utils.ts # itemResponseFunction, fisherInformation, distributions + __tests__/ # Jest tests; __fixtures__/golden/ holds generated fixtures +scripts/ # fixture generation +validation/ # R reference generation + exploratory Rmd (see validation/README.md) +.ai/rules/ # the full rule set — read before nontrivial changes +``` + +## Boundaries + +### Ask first + +- Adding dependencies +- Changing anything exported from `src/index.ts` (public API) +- Changing default parameter values or error message strings +- Regenerating the characterization baseline (`expected-jscat.csv`) + +### Never do + +- Skip the validation requirements for algorithmic changes +- Bypass the registries with new dispatch logic +- Commit edits to generated fixtures without regenerating them from the scripts + +## When stuck + +- The reference implementation for "how to add an estimator" is the tutorial in `.github/CONTRIBUTING.md` +- The golden test workflow is documented in `validation/README.md` +- Open a draft PR with notes rather than guessing about psychometric intent diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index efc5dcf..df78cda 100644 --- a/README.md +++ b/README.md @@ -294,6 +294,12 @@ const nextItem = clowder.updateCatAndGetNextItem({ By integrating `Clowder`, your application can efficiently manage adaptive testing scenarios with robust trial and stimuli handling, multi-CAT configurations, and stopping conditions to ensure optimal performance. +## Contributing + +Contributions are welcome — including AI-assisted ones. See [CONTRIBUTING.md](.github/CONTRIBUTING.md) for the development workflow, the architecture overview, and a step-by-step tutorial on adding a new ability estimator, item selector, or stopping rule. New psychometric algorithms must ship with validation evidence (literature reference, analytic tests, and golden fixture tests against catR/mirt — see [validation/README.md](validation/README.md)). + +If you point an AI coding agent at this repo, it will pick up the rules automatically from `CLAUDE.md` / `AGENTS.md` and `.ai/rules/`. + ## References - Chalmers, R. P. (2012). mirt: A multidimensional item response theory package for the R environment. Journal of Statistical Software. From ec8e6b7e539e17f23ff224fe5c1e1505ceeae0d4 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 21:47:59 +0000 Subject: [PATCH 05/11] Untrack IDE and OS junk files Removes .idea/ and validation/.DS_Store from version control and ignores .idea/, .DS_Store, .Rproj.user/, and .Rhistory going forward. --- .gitignore | 6 ++++++ .idea/.gitignore | 8 -------- .idea/jsCAT.iml | 9 --------- .idea/misc.xml | 6 ------ .idea/modules.xml | 8 -------- .idea/runConfigurations.xml | 10 ---------- .idea/vcs.xml | 6 ------ validation/.DS_Store | Bin 6148 -> 0 bytes 8 files changed, 6 insertions(+), 47 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/jsCAT.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/runConfigurations.xml delete mode 100644 .idea/vcs.xml delete mode 100644 validation/.DS_Store diff --git a/.gitignore b/.gitignore index 2539780..2a83793 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,9 @@ docs validation/data validation/simulation/resp validation/.DS_Store + +# IDE and OS junk +.idea/ +.DS_Store +.Rproj.user/ +.Rhistory diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e0..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/jsCAT.iml b/.idea/jsCAT.iml deleted file mode 100644 index d6ebd48..0000000 --- a/.idea/jsCAT.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 639900d..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 6b4f5f0..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml deleted file mode 100644 index 797acea..0000000 --- a/.idea/runConfigurations.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/validation/.DS_Store b/validation/.DS_Store deleted file mode 100644 index e29f9707ca33938d54e9627c925e2eca98db706d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK&1xGl5FXiXBPSGmXdy>Iu7=odNhrN6uJZ!fkQ~zB?AE4Tx>*J9I;I%Io|_ly zlk|D=jUA~l>(`a~Tf(on`m8^dpe{j3|(2$m*Lxi!91 zNg0*tqP*RR7Q_F@0KYp;2^G+sp1i*_w<&x-)7v~Nt6uMyYBXCfUhe1}y{E6@uXYhv zNi{Er$?TGUomrcw>v@u%XVb;F^ZKJLsw69>GnbG}rU?0TkrflW7}|L;DP3;t9lEXC zJwB#j;;v%E@>RSKbppHL2`~y+dxQld9|DdB8H9m<%D@A$4pim< From 9cc580b481894bea2dadf52e9870b1c1423c0773 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 22:16:10 +0000 Subject: [PATCH 06/11] Stabilize coverage reporting and cover default-parameter branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to make the coverage report trustworthy: 1. Switch coverageProvider from v8 to babel (Istanbul). The v8 provider under Jest 28 merged per-worker coverage nondeterministically — corpus.ts and utils.ts oscillated between 100% and ~94% across identical runs. Istanbul instrumentation is deterministic. 2. Exclude barrel files (src/**/index.ts) from coverage, matching the existing exclusion of the root src/index.ts. Re-export bindings were counted as uncovered functions. Also adds two micro-tests for previously unexercised default-parameter branches (bare uniform() and fillZetaDefaults() without a format), now visible under deterministic reporting. Remaining branch misses are two pre-existing Clowder branches (clowder.ts:86,311). --- jestconfig.json | 4 ++-- src/__tests__/corpus.test.ts | 16 ++++++++++++++++ src/__tests__/utils.test.ts | 9 +++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/jestconfig.json b/jestconfig.json index 1e8697a..c0f1fd0 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -1,7 +1,7 @@ { "collectCoverage": true, "coverageDirectory": "coverage", - "coverageProvider": "v8", + "coverageProvider": "babel", "coveragePathIgnorePatterns": ["/node_modules/", "/__tests__/"], "coverageReporters": ["text", "lcov"], "collectCoverageFrom": [ @@ -9,7 +9,7 @@ "!src/**/*.d.ts", "!src/**/types.ts", "!src/**/type.ts", - "!src/index.ts" + "!src/**/index.ts" ], "transform": { "^.+\\.ts?$": "ts-jest", diff --git a/src/__tests__/corpus.test.ts b/src/__tests__/corpus.test.ts index a78b828..f7e211c 100644 --- a/src/__tests__/corpus.test.ts +++ b/src/__tests__/corpus.test.ts @@ -50,6 +50,22 @@ describe('validateZetaParams', () => { }); describe('fillZetaDefaults', () => { + it('defaults to symbolic format when no format is provided', () => { + const zeta: Zeta = { + b: 1, + c: 0.5, + }; + + const filledZeta = fillZetaDefaults(zeta); + + expect(filledZeta).toEqual({ + a: 1, + b: 1, + c: 0.5, + d: 1, + }); + }); + it('fills in default values for missing keys', () => { const zeta: Zeta = { difficulty: 1, diff --git a/src/__tests__/utils.test.ts b/src/__tests__/utils.test.ts index bd2e463..c4438c3 100644 --- a/src/__tests__/utils.test.ts +++ b/src/__tests__/utils.test.ts @@ -154,6 +154,15 @@ describe('uniform', () => { }); }); + it(`it should use default bounds of [-4, 4] and step size 0.1 when called with no arguments`, () => { + const result = uniform(); + const probs = result.map(([, p]: [number, number]) => p); + const xs = result.map(([x]: [number, number]) => x); + expect(probs.reduce((a: number, b: number) => a + b, 0)).toBeCloseTo(1, 6); + expect(xs[0]).toBeCloseTo(-4, 6); + expect(xs[xs.length - 1]).toBeCloseTo(4, 6); + }); + it(`it should use the first two values as the fullmin and fullmax if they are not provided`, () => { const result = uniform(-1, 1, 0.5); const probs = result.map(([, p]: [number, number]) => p); From a25e138c8c7835b2730e279fdbebf2b14815078d Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 22:18:18 +0000 Subject: [PATCH 07/11] Round characterization baseline to 8 decimals for cross-platform stability Regenerating the baseline on a different OS perturbed the Powell optimizer path at the ~1e-9 level (different libm/V8 builds), churning every line of expected-jscat.csv with meaningless diffs. Rounding to 8 decimals makes regeneration byte-stable across machines while staying three orders of magnitude above the golden test's 1e-6 comparison tolerance. --- scripts/generate-golden-fixtures.js | 15 +- .../__fixtures__/golden/expected-jscat.csv | 200 +++++++++--------- .../__fixtures__/golden/provenance.json | 2 +- 3 files changed, 112 insertions(+), 105 deletions(-) diff --git a/scripts/generate-golden-fixtures.js b/scripts/generate-golden-fixtures.js index 3e99ef0..946462a 100644 --- a/scripts/generate-golden-fixtures.js +++ b/scripts/generate-golden-fixtures.js @@ -35,6 +35,13 @@ const N_PEOPLE = 100; const MIN_THETA = -6; const MAX_THETA = 6; +// Estimates are written rounded to this many decimal places. Cross-platform +// floating point noise (different libm / V8 builds) perturbs the optimizer +// path at the ~1e-9 level; rounding to 8 decimals makes regeneration +// byte-stable across machines while staying far above the golden test's +// comparison tolerance (1e-6). +const BASELINE_DECIMALS = 8; + const FIXTURE_DIR = path.join(__dirname, '..', 'src', '__tests__', '__fixtures__', 'golden'); const rng = seedrandom(SEED); @@ -112,10 +119,10 @@ const expected = responses.map(({ pid, resps }) => { return { pid, - theta_mle: catMLE.theta, - se_mle: catMLE.seMeasurement, - theta_eap: catEAP.theta, - se_eap: catEAP.seMeasurement, + theta_mle: round(catMLE.theta, BASELINE_DECIMALS), + se_mle: round(catMLE.seMeasurement, BASELINE_DECIMALS), + theta_eap: round(catEAP.theta, BASELINE_DECIMALS), + se_eap: round(catEAP.seMeasurement, BASELINE_DECIMALS), }; }); diff --git a/src/__tests__/__fixtures__/golden/expected-jscat.csv b/src/__tests__/__fixtures__/golden/expected-jscat.csv index 372c40e..ab4a27b 100644 --- a/src/__tests__/__fixtures__/golden/expected-jscat.csv +++ b/src/__tests__/__fixtures__/golden/expected-jscat.csv @@ -1,101 +1,101 @@ pid,theta_mle,se_mle,theta_eap,se_eap -1,1.8588998322048391,0.30289389949139117,1.700532203833846,0.302200425112208 -2,0.6312680288637098,0.3081130797205417,0.57057888316549,0.3080813995631253 -3,0.46636579243212584,0.30791005674851046,0.4231617809965266,0.3077947803895712 -4,1.0872239197598106,0.3066069863544136,0.9869934694201481,0.3072271405230544 -5,-1.6295303923940565,0.3124431816244769,-1.491366049939463,0.310034725373963 -6,0.5990534248844676,0.30810246159550875,0.5418242638517438,0.3080489856736773 -7,-0.21893814889697702,0.3038877731371155,-0.2016727179465006,0.30398934613246226 -8,0.5228498499626514,0.308021430444065,0.4762856851698992,0.3079328223834859 -9,-3.304414944190416,0.5047580067910393,-2.8137093532432202,0.41070481525143954 -10,0.7957885561840818,0.3079434184800617,0.7232811569735115,0.308065386566452 -11,0.44651756126425424,0.30786036608336437,0.4044007069963272,0.30773649271669834 -12,-0.688053888136376,0.30374277322973137,-0.6338753548568294,0.30348358907539325 -13,0.9786310316258504,0.3072712161794592,0.890510369898711,0.30766351054235097 -14,0.6840014788102506,0.308100037338158,0.6192394978181645,0.3081107565676052 -15,2.590808405719624,0.33588968330631847,2.357678895225598,0.3187827762334994 -16,-2.051313299398468,0.32973600370537925,-1.8678645876408746,0.3200374295829952 -17,0.5347048803685224,0.3080392232404728,0.49112190404926526,0.30796431139701574 -18,4.081563505967497,0.6999640203393643,3.2584137086469886,0.43448324302628344 -19,1.0485397788700037,0.30686616245576315,0.9523832860777697,0.3074018031281328 -20,1.0648892080469894,0.30675959366434147,0.9704890428366585,0.30731298189371065 -21,-0.19732761899033568,0.30401563188604086,-0.17982710504656868,0.3041242561563092 -22,-2.566260805534066,0.3770290402193326,-2.307139890535378,0.3495059163035916 -23,-2.4933203420392465,0.3685123405788368,-2.2466710698226557,0.3441691586131341 -24,-1.1016359879396131,0.3066538058958921,-1.0042882423916415,0.30594620754812 -25,1.3747191851846685,0.3041357996710781,1.2525858483776584,0.3052614647318543 -26,-0.9951124225212474,0.305877932310807,-0.9145489042055802,0.30527260727830463 -27,-2.5507211147377653,0.3751631309455136,-2.298206267261468,0.348691659655619 -28,-0.5199689274278102,0.30315780436311995,-0.47967282536556666,0.30312229487658693 -29,-1.0908114339664003,0.3065766859301559,-1.0007257579393756,0.3059197270159801 -30,3.187699133559006,0.41985398112218003,2.814302936370721,0.35979332962992094 -31,0.35512085283946054,0.307559482828496,0.3211679422522274,0.30741740277410956 -32,0.7679208798894146,0.30799940604556436,0.7057692974257187,0.3080835005996358 -33,-2.8034513259255944,0.40915203985064014,-2.4622566832041635,0.3650699848468007 -34,-3.129634714898387,0.46634606547290247,-2.721035316658195,0.39718752091128195 -35,1.6547242405565092,0.3022613062323735,1.515389140623105,0.30295829227033316 -36,0.8523154871741293,0.3077934826296074,0.7692467185560894,0.30799700363704696 -37,-1.8332497751457384,0.3186066536197898,-1.6718602405240583,0.3134348511662817 -38,2.290271903034184,0.315127930537629,2.1022864451590384,0.30760951998181113 -39,-6,2.398559964172724,-3.266650290415419,0.4959322916934462 -40,-0.2832333167115184,0.3035553818117071,-0.25992860198797973,0.3036668584758051 -41,0.7419566440783543,0.3080412779680965,0.670574980879014,0.3081069676682578 -42,-0.961199666862612,0.3056239294329479,-0.881540310474357,0.3050253692360071 -43,1.076466254397067,0.30668149110948667,0.9775525891277135,0.3072768133055201 -44,2.3215712314950867,0.31675813989857,2.124295280828844,0.3082995340120932 -45,-0.9605631386369317,0.3056191428301417,-0.8840867153650075,0.30504433944388576 -46,-1.7002550651285326,0.3141786219932775,-1.5595747182633954,0.311081085364047 -47,4.361913485683735,0.8392942549358298,3.33930759240767,0.45261406917909264 -48,0.9350405827032476,0.3074816526332066,0.8546260608226959,0.30778628857937756 -49,-0.8397798562358747,0.30471834770230555,-0.7690583879980197,0.30422772622180955 -50,1.5338816172359118,0.30283049421514785,1.4001697661254717,0.3039046587177168 -51,0.7619042770200629,0.30800998351452535,0.6919261553014854,0.3080947791561971 -52,-1.198055661069566,0.30733220536223366,-1.0995312933420702,0.30663883819431953 -53,1.30021166476189,0.30482556210247047,1.186865617087513,0.30583559959104684 -54,0.02377179354786882,0.30557666134955985,0.024012364452672713,0.3055784388162179 -55,0.6207313164293172,0.3081111510451915,0.5634238142709116,0.30807438266611537 -56,-0.029391051967001966,0.3051819057453777,-0.026533748095300975,0.3052031324854259 -57,0.2570981073775451,0.3071054811400358,0.23530632364101015,0.3069868127677043 -58,0.6840762016835037,0.30809999181579584,0.6261323329713476,0.308112327403867 -59,0.9185707098650185,0.30755277741145093,0.82882932728843,0.30786181073129354 -60,0.1302064391721982,0.3063350724461827,0.1201219618770456,0.30626655496676447 -61,-1.235365382966972,0.3075979048479154,-1.133057909284805,0.3068759242595351 -62,1.116555242540809,0.3063946644231522,1.0144342697855482,0.30707412450163524 -63,1.8909915827133934,0.3032372256069476,1.7243590603580101,0.3022100198023483 -64,-0.13218047766300742,0.30443952575847383,-0.12221580975449908,0.3045084948279562 -65,-0.9619671063031212,0.30562969978509125,-0.8766265891016785,0.30498882748318135 -66,3.255383934383351,0.433833444693727,2.855615448765288,0.365125591561657 -67,-2.349452217608828,0.35348390125369494,-2.1242551883876617,0.33462301235640335 -68,0.08635216328876531,0.3060311413013919,0.08033703308218636,0.30598835150279863 -69,0.5253621638017819,0.30802536170361117,0.47631376638693923,0.3079328848781848 -70,-0.34270254007884676,0.30332366523178717,-0.3158938955107988,0.30341827595200466 -71,0.4158903588247219,0.30777278331456837,0.37695944269411846,0.3076422026080383 -72,0.17462608601823512,0.3066253857825623,0.16102902169378971,0.3065386349137881 -73,1.2931528125808889,0.30489079889841314,1.1763748896122685,0.30592309559680153 -74,2.565120359207661,0.33363880519712397,2.337008240715823,0.31760460791941403 -75,-2.4707796188002784,0.3660035716363793,-2.222362941932229,0.3421394529510895 -76,2.805641455275647,0.35871345205316124,2.538734620173131,0.33142580621877027 -77,0.8925772017668997,0.3076557993721729,0.8141542674566465,0.3079001263997979 -78,1.8063494724755713,0.3024893135545689,1.6561583639126574,0.3022579208233519 -79,-0.35581234886062013,0.3032836026723006,-0.3281615432056812,0.3033729061590717 -80,-0.39291884490434087,0.3031933011359565,-0.36027796376180354,0.3032709123436851 -81,0.08045602042036563,0.30598920019395487,0.07504233310174668,0.3059504976270422 -82,-0.11355876495060353,0.30456912050454193,-0.10528170020141399,0.30462764852904184 -83,-1.8298772892425408,0.3184736924919598,-1.6717876919162233,0.31343303392160454 -84,-1.66802066461559,0.31333924483049386,-1.5220264878437402,0.31047444749856723 -85,-1.2078321591090857,0.30740128170625863,-1.1065007320311044,0.3066883538436063 -86,1.352931858476105,0.3043365066781005,1.2369291205562367,0.305401773967447 -87,2.721067578993394,0.3488407585079084,2.461268210527924,0.3254884436239299 -88,-2.0862992990883606,0.3320055586536638,-1.9017932097602628,0.32155909791752146 -89,3.848642111447811,0.6050144446041955,3.1711609446492712,0.4165921521127591 -90,1.576080131035285,0.30257443009947393,1.43949403397396,0.3035590304024674 -91,0.8301278681791631,0.30785825928622745,0.7521658335152794,0.3080259826740249 -92,-1.9231103774218077,0.32257689554003927,-1.7566789546542119,0.3158590750956975 -93,0.8285698495556992,0.3078625172370411,0.7468579795485826,0.30803412273865616 -94,1.0409396723283348,0.3069141952297097,0.9516888067141759,0.3074050988064524 -95,0.2716636901106792,0.30718131215676325,0.25030204307437304,0.3070691364719882 -96,0.14808850669698664,0.3064542958460234,0.1354655314312086,0.3063704457325791 -97,-0.5403842090069795,0.3031922275374423,-0.4933906174917755,0.30312950820130935 -98,-2.026539343312127,0.32821154691000565,-1.8457920883135064,0.319111101321839 -99,-1.4962704612247058,0.3101019513591121,-1.3699644052156956,0.3086762118715515 -100,1.6777556916578245,0.3022182303282904,1.5317431404702406,0.3028448245418307 +1,1.85889983,0.3028939,1.7005322,0.30220043 +2,0.63126803,0.30811308,0.57057888,0.3080814 +3,0.46636579,0.30791006,0.42316178,0.30779478 +4,1.08722392,0.30660699,0.98699347,0.30722714 +5,-1.62953039,0.31244318,-1.49136605,0.31003473 +6,0.59905342,0.30810246,0.54182426,0.30804899 +7,-0.21893815,0.30388777,-0.20167272,0.30398935 +8,0.52284985,0.30802143,0.47628569,0.30793282 +9,-3.30441494,0.50475801,-2.81370935,0.41070482 +10,0.79578856,0.30794342,0.72328116,0.30806539 +11,0.44651756,0.30786037,0.40440071,0.30773649 +12,-0.68805389,0.30374277,-0.63387535,0.30348359 +13,0.97863103,0.30727122,0.89051037,0.30766351 +14,0.68400148,0.30810004,0.6192395,0.30811076 +15,2.59080841,0.33588968,2.3576789,0.31878278 +16,-2.0513133,0.329736,-1.86786459,0.32003743 +17,0.53470488,0.30803922,0.4911219,0.30796431 +18,4.08156351,0.69996402,3.25841371,0.43448324 +19,1.04853978,0.30686616,0.95238329,0.3074018 +20,1.06488921,0.30675959,0.97048904,0.30731298 +21,-0.19732762,0.30401563,-0.17982711,0.30412426 +22,-2.56626081,0.37702904,-2.30713989,0.34950592 +23,-2.49332034,0.36851234,-2.24667107,0.34416916 +24,-1.10163599,0.30665381,-1.00428824,0.30594621 +25,1.37471919,0.3041358,1.25258585,0.30526146 +26,-0.99511242,0.30587793,-0.9145489,0.30527261 +27,-2.55072111,0.37516313,-2.29820627,0.34869166 +28,-0.51996893,0.3031578,-0.47967283,0.30312229 +29,-1.09081143,0.30657669,-1.00072576,0.30591973 +30,3.18769913,0.41985398,2.81430294,0.35979333 +31,0.35512085,0.30755948,0.32116794,0.3074174 +32,0.76792088,0.30799941,0.7057693,0.3080835 +33,-2.80345133,0.40915204,-2.46225668,0.36506998 +34,-3.12963471,0.46634607,-2.72103532,0.39718752 +35,1.65472424,0.30226131,1.51538914,0.30295829 +36,0.85231549,0.30779348,0.76924672,0.307997 +37,-1.83324978,0.31860665,-1.67186024,0.31343485 +38,2.2902719,0.31512793,2.10228645,0.30760952 +39,-6,2.39855996,-3.26665029,0.49593229 +40,-0.28323332,0.30355538,-0.2599286,0.30366686 +41,0.74195664,0.30804128,0.67057498,0.30810697 +42,-0.96119967,0.30562393,-0.88154031,0.30502537 +43,1.07646625,0.30668149,0.97755259,0.30727681 +44,2.32157123,0.31675814,2.12429528,0.30829953 +45,-0.96056314,0.30561914,-0.88408672,0.30504434 +46,-1.70025507,0.31417862,-1.55957472,0.31108109 +47,4.36191349,0.83929425,3.33930759,0.45261407 +48,0.93504058,0.30748165,0.85462606,0.30778629 +49,-0.83977986,0.30471835,-0.76905839,0.30422773 +50,1.53388162,0.30283049,1.40016977,0.30390466 +51,0.76190428,0.30800998,0.69192616,0.30809478 +52,-1.19805566,0.30733221,-1.09953129,0.30663884 +53,1.30021166,0.30482556,1.18686562,0.3058356 +54,0.02377179,0.30557666,0.02401236,0.30557844 +55,0.62073132,0.30811115,0.56342381,0.30807438 +56,-0.02939105,0.30518191,-0.02653375,0.30520313 +57,0.25709811,0.30710548,0.23530632,0.30698681 +58,0.6840762,0.30809999,0.62613233,0.30811233 +59,0.91857071,0.30755278,0.82882933,0.30786181 +60,0.13020644,0.30633507,0.12012196,0.30626655 +61,-1.23536538,0.3075979,-1.13305791,0.30687592 +62,1.11655524,0.30639466,1.01443427,0.30707412 +63,1.89099158,0.30323723,1.72435906,0.30221002 +64,-0.13218048,0.30443953,-0.12221581,0.30450849 +65,-0.96196711,0.3056297,-0.87662659,0.30498883 +66,3.25538393,0.43383344,2.85561545,0.36512559 +67,-2.34945222,0.3534839,-2.12425519,0.33462301 +68,0.08635216,0.30603114,0.08033703,0.30598835 +69,0.52536216,0.30802536,0.47631377,0.30793288 +70,-0.34270254,0.30332367,-0.3158939,0.30341828 +71,0.41589036,0.30777278,0.37695944,0.3076422 +72,0.17462609,0.30662539,0.16102902,0.30653863 +73,1.29315281,0.3048908,1.17637489,0.3059231 +74,2.56512036,0.33363881,2.33700824,0.31760461 +75,-2.47077962,0.36600357,-2.22236294,0.34213945 +76,2.80564146,0.35871345,2.53873462,0.33142581 +77,0.8925772,0.3076558,0.81415427,0.30790013 +78,1.80634947,0.30248931,1.65615836,0.30225792 +79,-0.35581235,0.3032836,-0.32816154,0.30337291 +80,-0.39291884,0.3031933,-0.36027796,0.30327091 +81,0.08045602,0.3059892,0.07504233,0.3059505 +82,-0.11355876,0.30456912,-0.1052817,0.30462765 +83,-1.82987729,0.31847369,-1.67178769,0.31343303 +84,-1.66802066,0.31333924,-1.52202649,0.31047445 +85,-1.20783216,0.30740128,-1.10650073,0.30668835 +86,1.35293186,0.30433651,1.23692912,0.30540177 +87,2.72106758,0.34884076,2.46126821,0.32548844 +88,-2.0862993,0.33200556,-1.90179321,0.3215591 +89,3.84864211,0.60501444,3.17116094,0.41659215 +90,1.57608013,0.30257443,1.43949403,0.30355903 +91,0.83012787,0.30785826,0.75216583,0.30802598 +92,-1.92311038,0.3225769,-1.75667895,0.31585908 +93,0.82856985,0.30786252,0.74685798,0.30803412 +94,1.04093967,0.3069142,0.95168881,0.3074051 +95,0.27166369,0.30718131,0.25030204,0.30706914 +96,0.14808851,0.3064543,0.13546553,0.30637045 +97,-0.54038421,0.30319223,-0.49339062,0.30312951 +98,-2.02653934,0.32821155,-1.84579209,0.3191111 +99,-1.49627046,0.31010195,-1.36996441,0.30867621 +100,1.67775569,0.30221823,1.53174314,0.30284482 diff --git a/src/__tests__/__fixtures__/golden/provenance.json b/src/__tests__/__fixtures__/golden/provenance.json index cf7b291..fb34ae0 100644 --- a/src/__tests__/__fixtures__/golden/provenance.json +++ b/src/__tests__/__fixtures__/golden/provenance.json @@ -7,7 +7,7 @@ 6 ], "generator": "scripts/generate-golden-fixtures.js", - "generatedAt": "2026-06-11T21:37:20.071Z", + "generatedAt": "2026-06-11T22:17:51.454Z", "jscatVersion": "5.3.2", "note": "expected-jscat.csv is a characterization baseline generated by jsCAT itself. expected-r-reference.csv (if present) is the independent reference generated by validation/generate-r-reference.R using the catR package." } From e8221ec47b866c8764f40e13c25f0c175f8fb395 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 22:18:18 +0000 Subject: [PATCH 08/11] Tighten wording in contribution docs --- .github/CONTRIBUTING.md | 6 +++--- README.md | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4c401ee..5a90799 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to jsCAT -Thank you for contributing! jsCAT is a research software project: it implements psychometric algorithms whose correctness matters for real assessments of real students. This guide explains how to set up a development environment, how the codebase is organized, and — most importantly — what we require before merging a new algorithm. +Thank you for contributing! jsCAT is a research software project: it implements psychometric algorithms whose correctness matters for real assessments of real students. This guide explains how to set up a development environment, how the codebase is organized, and, most importantly, what we require before merging a new algorithm. AI-assisted contributions are welcome (many of our best PRs are AI-generated). See [AI-assisted contributions](#ai-assisted-contributions) for the disclosure policy, and note that the requirements below are designed so that a reviewer can verify correctness without trusting the author or the tool that produced the code. @@ -103,7 +103,7 @@ Every PR that adds or changes a psychometric algorithm must include: 1. **A literature reference** for the method, cited in the class JSDoc (author, year, journal). 2. **Documented scope**: which IRT models (1PL/2PL/3PL/4PL) the implementation is exact for, and what happens outside that scope. 3. **Analytic unit tests** for closed-form cases, with the derivation in a comment. -4. **Golden test coverage**: committed fixtures showing agreement with an independent reference implementation (catR or mirt), passing in CI. A validation plot alone is not sufficient — plots rot, tests don't. +4. **Golden test coverage**: committed fixtures showing agreement with an independent reference implementation (catR or mirt), passing in CI. A validation plot alone is not sufficient. 5. **No new dispatch logic** outside the registries. PRs that change existing numerical behavior must regenerate the characterization baseline (`npm run fixtures:generate`) and justify the change in the PR description. @@ -128,4 +128,4 @@ If you are using an AI agent in this repo, point it at `CLAUDE.md` / `AGENTS.md` ## Questions -Open a [GitHub issue](https://github.com/yeatmanlab/jsCAT/issues) — including for questions about whether an algorithmic idea fits the library before you build it. We'd rather discuss the design first. +Open a [GitHub issue](https://github.com/yeatmanlab/jsCAT/issues) for questions about whether an algorithmic idea fits the library before you build it. We'd rather discuss the design first. diff --git a/README.md b/README.md index df78cda..26dfe2a 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,6 @@ const stimuli = [{ discrimination: 1, difficulty: -2, guessing: 0, slipping: 1, const nextItem = cat1.findNextItem(stimuli, 'MFI'); ``` - ## Validation ### Validation of theta estimate and theta standard error @@ -65,7 +64,7 @@ const nextItem = cat1.findNextItem(stimuli, 'MFI'); Reference software: mirt (Chalmers, 2012) ![img.png](validation/plots/jsCAT_validation_1.png) -### Validation of MFI algorithm +### Validation of MFI algorithm Reference software: catR (Magis et al., 2017) ![img_1.png](validation/plots/jsCAT_validation_2.png) @@ -92,12 +91,14 @@ The `Clowder` class is a powerful tool for managing multiple `Cat` instances and ### 1. Replacing Single `Cat` Usage #### Single `Cat` Example: + ```typescript const cat = new Cat({ method: 'MLE', theta: 0.5 }); const nextItem = cat.findNextItem(stimuli); ``` #### Clowder Equivalent: + ```typescript const clowder = new Clowder({ cats: { cat1: { method: 'MLE', theta: 0.5 } }, @@ -141,17 +142,20 @@ export const defaultZeta = (desiredFormat: 'symbolic' | 'semantic' = 'symbolic') }; ``` + - If desiredFormat is not specified, it defaults to 'symbolic'. - This ensures consistency across different stimuli and prevents errors from missing Zeta parameters. - You can pass 'semantic' as an argument to convert the default Zeta values into a different representation. #### Validate the Corpus: + ```typescript import { checkNoDuplicateCatNames } from './corpus'; checkNoDuplicateCatNames(corpus); ``` #### Filter Stimuli for a Specific Cat: + ```typescript import { filterItemsByCatParameterAvailability } from './corpus'; const { available, missing } = filterItemsByCatParameterAvailability(corpus, 'cat1'); @@ -164,6 +168,7 @@ const { available, missing } = filterItemsByCatParameterAvailability(corpus, 'ca Integrate early stopping mechanisms to optimize the testing process. #### Example: Stop After N Items + ```typescript import { StopAfterNItems } from './stopping'; From af4884618a1f923045f7c26f4b8c2242a3acc307 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 22:21:01 +0000 Subject: [PATCH 09/11] Fix thetaEst arguments in R reference script lower/upper/nqp are arguments of catR's standalone eapEst(); thetaEst() takes the EAP integration grid as parInt = c(lower, upper, nqp). --- validation/generate-r-reference.R | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/validation/generate-r-reference.R b/validation/generate-r-reference.R index d63938c..08fcb58 100644 --- a/validation/generate-r-reference.R +++ b/validation/generate-r-reference.R @@ -29,7 +29,8 @@ resp_matrix <- as.matrix(responses[, grepl("^i", names(responses))]) # Match jsCAT's settings: theta bounds [-6, 6]; EAP with a standard normal # prior. jsCAT quantizes the EAP prior on a 0.1-step grid over [-6, 6] -# (121 points), so we use 121 quadrature points here as well. +# (121 points). In thetaEst, the EAP integration grid is specified via +# parInt = c(lower, upper, nqp), so we use parInt = c(-6, 6, 121). theta_mle <- apply(resp_matrix, 1, function(x) { thetaEst(it, x, method = "ML", range = c(-6, 6)) }) @@ -40,9 +41,7 @@ theta_eap <- apply(resp_matrix, 1, function(x) { method = "EAP", priorDist = "norm", priorPar = c(0, 1), - lower = -6, - upper = 6, - nqp = 121 + parInt = c(-6, 6, 121) ) }) From b9e3bb7fdfb81a8a31121080a2929807070a6149 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 22:24:31 +0000 Subject: [PATCH 10/11] Activate catR reference comparison in golden tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commits the catR reference estimates (generated locally with validation/generate-r-reference.R) and fixes the test's CSV parser to strip the surrounding quotes that R's write.csv puts on headers — unquoted lookup made every reference value parse as NaN. With the comparison live, tightens the provisional tolerances based on observed agreement (MLE max |diff| = 0.0036 from optimizer differences; EAP max |diff| = 2.4e-6 from shared rectangular quadrature): max 0.02, mean 0.002. All estimators now validate against catR on every test run. --- .../golden/expected-r-reference.csv | 101 ++++++++++++++++++ src/__tests__/golden.test.ts | 22 ++-- 2 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 src/__tests__/__fixtures__/golden/expected-r-reference.csv diff --git a/src/__tests__/__fixtures__/golden/expected-r-reference.csv b/src/__tests__/__fixtures__/golden/expected-r-reference.csv new file mode 100644 index 0000000..2b31ed0 --- /dev/null +++ b/src/__tests__/__fixtures__/golden/expected-r-reference.csv @@ -0,0 +1,101 @@ +"pid","theta_mle","theta_eap" +1,1.85916317916527,1.70053220383385 +2,0.631578293167174,0.570578883165489 +3,0.466598929574921,0.423161780996526 +4,1.08748517211986,0.986993469420148 +5,-1.62967599298914,-1.49136604993946 +6,0.59945591625603,0.541824263851743 +7,-0.219231789607587,-0.201672717946501 +8,0.523262147434129,0.476285685169899 +9,-3.30517397567978,-2.81370935180981 +10,0.79593299739603,0.723281156973511 +11,0.446833660741805,0.404400706996326 +12,-0.68828700586777,-0.63387535485683 +13,0.978796122667714,0.890510369898711 +14,0.684227906979882,0.619239497818164 +15,2.59106960192018,2.3576788952256 +16,-2.05152519276174,-1.86786458764088 +17,0.53507325259398,0.491121904049265 +18,4.08388750435082,3.25841353567917 +19,1.04881637873646,0.95238328607777 +20,1.06504298258401,0.970489042836658 +21,-0.19766738341084,-0.179827105046569 +22,-2.56681057256445,-2.30713989053538 +23,-2.49372884961898,-2.24667106982266 +24,-1.10190419727505,-1.00428824239164 +25,1.37495033790402,1.25258584837766 +26,-0.99525456538794,-0.914548904205581 +27,-2.55114725741679,-2.29820626726147 +28,-0.520251403092408,-0.479672825365567 +29,-1.091037831285,-1.00072575793938 +30,3.188060086848,2.81430293635856 +31,0.355364550892202,0.321167942252227 +32,0.768094825226093,0.705769297425718 +33,-2.80393327326807,-2.46225668320169 +34,-3.13013947196292,-2.72103531663809 +35,1.65491023829243,1.5153891406231 +36,0.852672024607699,0.769246718556089 +37,-1.83361532793187,-1.67186024052406 +38,2.29043364919976,2.10228644515904 +39,-6,-3.26664788916646 +40,-0.283492430981973,-0.25992860198798 +41,0.742208734171354,0.670574980879013 +42,-0.961349766849677,-0.881540310474357 +43,1.07670797589427,0.977552589127713 +44,2.32177952918066,2.12429528082884 +45,-0.960854675428289,-0.884086715365007 +46,-1.70059200259974,-1.5595747182634 +47,4.36551035854026,3.33930697845971 +48,0.935354362383423,0.854626060822695 +49,-0.839959484157002,-0.769058387998021 +50,1.53412831990851,1.40016976612547 +51,0.762052328439374,0.691926155301484 +52,-1.19823283264508,-1.09953129334207 +53,1.3004012551248,1.18686561708751 +54,0.0243679549347296,0.0240123644526721 +55,0.621113426903609,0.563423814270911 +56,-0.029905215379342,-0.0265337480953012 +57,0.257348120028631,0.23530632364101 +58,0.684335317291641,0.626132332971347 +59,0.918952960737102,0.82882932728843 +60,0.130504469457929,0.120121961877045 +61,-1.23554139286045,-1.13305790928481 +62,1.11686405091534,1.01443426978555 +63,1.89111335537333,1.72435906035801 +64,-0.13259517794669,-0.122215809754499 +65,-0.962248713661804,-0.876626589101678 +66,3.25611601950961,2.85561544872435 +67,-2.34968412781554,-2.12425518838766 +68,0.0868205022409714,0.0803370330821858 +69,0.525687851601873,0.476313766386939 +70,-0.342926609282379,-0.315893895510799 +71,0.416098513960949,0.376959442694118 +72,0.174879445273176,0.161029021693789 +73,1.29342099131203,1.17637488961227 +74,2.5653246019871,2.33700824071582 +75,-2.47129393018551,-2.22236294193223 +76,2.80588616394103,2.53873462017313 +77,0.892756605097103,0.814154267456646 +78,1.80655051297656,1.65615836391266 +79,-0.356043783412972,-0.328161543205681 +80,-0.393246211064582,-0.360277963761804 +81,0.0808064843677143,0.0750423331017462 +82,-0.113917599213574,-0.105281700201414 +83,-1.83016817868104,-1.67178769191622 +84,-1.66825455347795,-1.52202648784374 +85,-1.20808465512531,-1.10650073203111 +86,1.35308194999454,1.23692912055624 +87,2.72128945206324,2.46126821052792 +88,-2.0864680390566,-1.90179320976026 +89,3.84981568086712,3.17116090640699 +90,1.57629453883453,1.43949403397396 +91,0.830385533022691,0.752165833515279 +92,-1.92342535361085,-1.75667895465421 +93,0.828940481193918,0.746857979548582 +94,1.04128231557542,0.951688806714176 +95,0.271911359078885,0.250302043074372 +96,0.148403293060996,0.135465531431208 +97,-0.540666373045144,-0.493390617491776 +98,-2.02672705242644,-1.84579208831351 +99,-1.49667168671506,-1.3699644052157 +100,1.67800353051831,1.53174314047024 diff --git a/src/__tests__/golden.test.ts b/src/__tests__/golden.test.ts index a8a6904..6848fc4 100644 --- a/src/__tests__/golden.test.ts +++ b/src/__tests__/golden.test.ts @@ -26,11 +26,14 @@ const FIXTURE_DIR = path.join(__dirname, '__fixtures__', 'golden'); const R_REFERENCE_FILE = path.join(FIXTURE_DIR, 'expected-r-reference.csv'); // Tolerances for agreement with the external (catR) reference. jsCAT and catR -// use different optimizers and quadrature schemes, so exact equality is not -// expected. These bounds are intentionally provisional and should be tightened -// after the first committed regeneration if observed agreement allows. -const R_REFERENCE_MAX_ABS_DIFF = 0.05; -const R_REFERENCE_MEAN_ABS_DIFF = 0.01; +// use different optimizers, so exact equality is not expected. Observed +// agreement on the initial fixture set: MLE max |diff| = 0.0036 (Powell vs. +// catR's optimizer, same objective), EAP max |diff| = 2.4e-6 (same rectangular +// quadrature). The bounds below give ~5x headroom over the observed MLE +// differences; if a new estimator legitimately needs looser bounds, document +// why in the PR rather than silently widening these. +const R_REFERENCE_MAX_ABS_DIFF = 0.02; +const R_REFERENCE_MEAN_ABS_DIFF = 0.002; // The characterization baseline is jsCAT's own output, so agreement should be // exact up to floating point noise across platforms. @@ -40,11 +43,16 @@ interface CsvRow { [key: string]: string; } +/** Strip surrounding double quotes (R's write.csv quotes character fields and headers). */ +function unquote(value: string): string { + return value.trim().replace(/^"|"$/g, ''); +} + function parseCsv(filepath: string): CsvRow[] { const lines = fs.readFileSync(filepath, 'utf8').trim().split('\n'); - const headers = lines[0].split(',').map((h) => h.trim()); + const headers = lines[0].split(',').map(unquote); return lines.slice(1).map((line) => { - const values = line.split(',').map((v) => v.trim()); + const values = line.split(',').map(unquote); return headers.reduce((row, header, i) => { row[header] = values[i] ?? ''; return row; From 47a0b7430c61b403363bfbb039aa9d735b6ced15 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Thu, 11 Jun 2026 23:32:43 +0000 Subject: [PATCH 11/11] Document the WLE penalized-form scope trap in the estimator tutorial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tutorial's penalized objective (max ln L + 0.5 ln I) equals Warm's WLE only for 1PL/2PL, where Warm's J(theta) coincides with I'(theta). Under 3PL they diverge (verified against catR source: method = 'WL' solves S + J/2I = 0 via Ji(), while BM + Jeffreys prior uses I'/2I), producing theta differences up to ~0.2 on an all-3PL bank and ~0.04 on the golden fixture bank — beyond the golden test tolerance. The warning spells out the two valid resolutions so contributors hit this as a documented decision, not a surprise test failure. --- .github/CONTRIBUTING.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5a90799..72fab26 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -51,6 +51,8 @@ import { fisherInformation } from '../utils'; * * @remarks * Document the math, the IRT models it is exact for, and any approximations. + * See the scope warning below this example — this simple penalized form is + * NOT Warm's estimator under 3PL/4PL. * * Reference: Warm, T. A. (1989). Weighted likelihood estimation of ability in * item response theory. Psychometrika, 54(3), 427-450. @@ -67,6 +69,26 @@ export class WLEEstimator implements AbilityEstimator { } ``` +> [!WARNING] +> **Worked example of a scope trap — read before implementing WLE.** +> The penalized objective above (maximize ln L + ½ ln I, first-order condition +> S(θ) + I′/(2I) = 0) equals Warm's weighted likelihood estimator **only for +> 1PL/2PL models**, where Warm's correction term J(θ) = Σ P′ᵢP″ᵢ/(PᵢQᵢ) +> coincides with I′(θ). Under 3PL/4PL they diverge: with c = 0.2 items, J and +> I′ at the same theta can differ in sign, and theta estimates differ by up to +> ≈0.2 on an all-3PL bank (≈0.04 on this repo's golden fixture bank, which +> exceeds the golden test tolerance — the penalized form **will fail step 4** +> against catR's `method = "WL"`, which solves the exact equation +> S(θ) + J/(2I) = 0; see catR's `Ji()`). +> +> Your options, in order of preference: (a) implement the exact correction — +> solve S(θ) + J/(2I) = 0, computing J from the first and second derivatives +> of the IRF; or (b) ship the penalized form but document it as Jeffreys-prior +> bias reduction (Firth, 1993) exact for 1PL/2PL only, and validate against +> catR's `method = "BM", priorDist = "Jeffreys"` (which uses I′/(2I)) instead +> of `"WL"`. Do not silently widen the golden test tolerances to make the +> penalized form pass as "WLE". + **2. Register it** in `src/estimators/registry.ts`: ```typescript @@ -88,7 +110,7 @@ The `EstimationMethod` type, runtime validation, `Cat` dispatch, and error messa **4. Wire it into the golden tests** (see `validation/README.md`): - Add the estimator to `scripts/generate-golden-fixtures.js` and run `npm run fixtures:generate`. -- Add the corresponding reference call to `validation/generate-r-reference.R` (catR's `thetaEst` supports `method = "WL"`) and run it with `Rscript`. +- Add the corresponding reference call to `validation/generate-r-reference.R` and run it with `Rscript`. Pick the catR method that matches what you actually implemented (see the warning in step 1: exact Warm WLE → `method = "WL"`; penalized/Jeffreys form → `method = "BM", priorDist = "Jeffreys"`). - Add the new columns to `src/__tests__/golden.test.ts`. - Commit the regenerated CSVs. The PR must show the golden test passing against the independent R reference.