diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..74fd87b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.github +.venv +__pycache__ +*.py[cod] +.pytest_cache +.ruff_cache +data +results +content +docs +*.ipynb +.env* +kaggle.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ecfd749 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: [main, "portfolio-upgrade/**"] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11.15" + cache: pip + - name: Install + run: python -m pip install ".[dev]" + - name: Validate dependencies + run: python -m pip check + - name: Audit dependencies + run: pip-audit + - name: Lint + run: ruff check . + - name: Format + run: ruff format --check . + - name: Test + run: pytest -q + + docker-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build test image + run: docker build --tag ios-risk-eval:test . + - name: Run container tests + run: docker run --rm ios-risk-eval:test diff --git a/.gitignore b/.gitignore index ee2f3fa..1c1bd6a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,16 +14,21 @@ __pycache__/ .Python venv/ env/ +.venv/ .env *.egg-info/ dist/ build/ +.pytest_cache/ +.ruff_cache/ +.coverage # Data *.csv data/raw/ data/processed/ data/versioned/ +data/**/*.zip # ML experiment tracking wandb/ @@ -36,6 +41,12 @@ mlruns/ kaggle.json *.key *_secret* +*.pem +*.p12 + +# Generated benchmark outputs (the reviewed reference artifact is explicitly allowed) +results/* +!results/benchmark.json # OS .DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..58f2fc2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11.15-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +COPY tests ./tests + +RUN python -m pip install --no-cache-dir ".[dev]" \ + && python -m pip check + +CMD ["python", "-m", "pytest", "-q"] diff --git a/README.md b/README.md index 5e5aba5..c575ef7 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,131 @@ -# IntelligenceOS β€” Eval Harness +# IntelligenceOS Risk Evaluation Harness -> Building a domain-specific AI platform for Finance Ops & Risk β€” in public, one project at a time. - -This repository contains the open-source evaluation and measurement infrastructure for **IntelligenceOS** β€” a 12-project build toward a production AI platform. - ---- - -## Projects - -| Project | Layer | Status | Description | -|---------|-------|--------|-------------| -| [IOS Risk Eval Harness](./ios-risk-eval-harness) | Evaluation Layer | βœ… Complete | ML evaluation framework for fraud detection. Benchmarks three model architectures. XGBoost: avg_precision 0.869. | -| IOS Risk Data Foundry | Data Layer | πŸ”œ In Progress | Domain data pipeline β€” ingest, clean, label, version, export. | -| IOS Domain Intelligence Core | Model Families | ⏳ Upcoming | Fine-tuned Llama/Mistral on finance & risk domain data. | -| IOS Verification System | Governance Layer | ⏳ Upcoming | Structured, auditable, schema-validated model outputs. | -| IOS Agent Runtime v1 | Orchestration | πŸš€ First Deploy | Closed-loop agent: Perceive β†’ Plan β†’ Act β†’ Verify β†’ Learn. | - ---- - -## What Is IntelligenceOS - -IntelligenceOS is a domain-specific AI platform being built from the ground up for Finance Ops & Risk. - -The architecture: **Perceive β†’ Understand β†’ Plan β†’ Act β†’ Verify β†’ Learn** - -Built as a closed-loop system β€” every output is measurable, auditable, and feeds back into improvement. Not a wrapper around a general model. A platform. - ---- - -## Build in Public - -Every project is documented publicly as it's built: - -- **GitHub** β€” all code, open source, Apache 2.0 -- **Substack** β€” deep dives on every project -- **X / LinkedIn** β€” progress updates - ---- +> Leakage-safe, reproducible evaluation of imbalanced financial-risk classifiers. + +**Evidence status:** Benchmarked Research Prototype +**Domain:** Financial Risk Β· Fraud Detection Β· Model Evaluation + +This repository compares three classical model strategies on the public ULB/Kaggle credit-card fraud +dataset. It is an engineering and benchmark artifactβ€”not a deployed fraud service, client result, or +production-performance claim. + +## Verified benchmark + +The execution pass independently reran every model with Python 3.11.15 and the pinned dependencies in +[`pyproject.toml`](./pyproject.toml). Unlike the earlier result, the corrected workflow selects each F1 +threshold on validation labels and evaluates the chosen operating point once on a held-out test split. + +| Model | Average Precision | ROC AUC | Precision | Recall | F1 | FP | FN | Validation-selected threshold | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Logistic Regression | 0.7192 | 0.9720 | 0.5526 | 0.8571 | 0.6720 | 68 | 14 | 0.99 | +| SMOTE + Random Forest | 0.7920 | 0.9768 | 0.7500 | 0.8265 | 0.7864 | 27 | 17 | 0.71 | +| XGBoost | **0.8756** | 0.9749 | **0.9011** | 0.8367 | **0.8677** | **9** | 16 | 0.54 | + +**Evidence label:** public-dataset benchmark. The machine-readable evidence is +[`results/benchmark.json`](./results/benchmark.json), including the dataset SHA-256, split details, +configuration, environment, model parameters, thresholds, and full metrics. + +The prior `0.8693` XGBoost result used final test labels to select its reporting threshold. Average +precision itself is threshold-independent, but the old threshold-dependent precision/recall/F1/FP/FN +figures were not defensible held-out estimates and have been superseded. + +## Leakage controls + +```text +creditcard.csv + ↓ validate schema, labels, numeric/finite values, and fingerprint +stratified train / validation / test split (60% / 20% / 20%) + β”œβ”€β”€ train: fit Amount scaler; fit model; apply SMOTE only here for RF + β”œβ”€β”€ validation: select operating threshold by F1 + └── test: score the selected threshold once + ↓ +JSON evidence artifact + console comparison +``` + +These boundaries are independently testable. Changing final test labels cannot change the selected +threshold, scaler metadata records `train_only`, and SMOTE provenance records its train-only scope. + +## Quick start + +```bash +python3.11 -m venv .venv +source .venv/bin/activate +python -m pip install ".[dev]" +python -m pip check +ruff check . +ruff format --check . +pytest -q +``` + +Download the public dataset and place it at `data/raw/creditcard.csv`: + +```bash +python -m pip install kaggle +kaggle datasets download -d mlg-ulb/creditcardfraud -p data/raw --unzip +``` + +The verified file has 284,807 rows, 31 columns, and SHA-256: + +```text +76274b691b16a6c49d3f159c883398e03ccd6d1ee12d9d8ee38f4b4b98551a89 +``` + +Run all three models: + +```bash +ios-eval \ + --data data/raw/creditcard.csv \ + --config configs/benchmark.json \ + --output results/benchmark.json +``` + +The macOS XGBoost wheel requires the OpenMP runtime (`brew install libomp`). Linux wheels used by CI and +the Docker image include their supported runtime path. + +## Reproducible checks + +```bash +docker build -t ios-risk-eval:test . +docker run --rm ios-risk-eval:test +``` + +GitHub Actions runs Python 3.11 dependency validation, Ruff lint/format checks, 32 automated tests, and +the same test suite inside Docker. The public dataset is intentionally not required for CI and is never +committed. + +## Repository structure + +```text +src/ios_eval/ +β”œβ”€β”€ data_loader.py # validation, deterministic splits, train-fitted scaling +β”œβ”€β”€ metrics.py # metrics, threshold sweeps, deterministic selection +β”œβ”€β”€ evaluation.py # model fitting, train-only SMOTE, provenance orchestration +β”œβ”€β”€ report.py # atomic JSON serialization and console comparison +└── cli.py # ios-eval command +configs/benchmark.json +tests/ +results/benchmark.json +docs/ +pyproject.toml +Dockerfile +``` + +The historical nested project wrapper, empty modules/notebooks/config, duplicate leaky scripts, oversized +environment dump, stale single-model result, and obsolete marketing drafts were removed. + +## Evidence boundaries and limitations + +- One historical public dataset does not establish performance on another processor, geography, product, + fraud regime, or live client stream. +- The selected threshold optimizes validation F1, not a real institution's fraud-loss/review-cost function. +- This is a single deterministic split, not temporal or out-of-time validation. +- No calibration, drift monitoring, serving latency, SLA, production deployment, or client outcome is claimed. +- Average precision measures ranking quality; threshold-dependent metrics describe only this selected operating + point and held-out split. + +See [architecture](./docs/architecture.md), [evidence policy](./docs/evidence.md), +[failure modes](./docs/failure-modes.md), and [benchmark reproduction](./docs/benchmark-reproduction.md). ## License -Copyright 2026 Ugo Chukwu / Etherlabs - -Licensed under the Apache License, Version 2.0. See [LICENSE](./LICENSE) for full terms. - ---- - -*IntelligenceOS v0.1 β€” built with purpose.* +Copyright 2026 Ugo Chukwu / Etherlabs. Licensed under the Apache License, Version 2.0. diff --git a/configs/benchmark.json b/configs/benchmark.json new file mode 100644 index 0000000..2be9d8e --- /dev/null +++ b/configs/benchmark.json @@ -0,0 +1,45 @@ +{ + "random_state": 42, + "validation_size": 0.2, + "test_size": 0.2, + "threshold_metric": "f1", + "threshold_step": 0.01, + "models": { + "Logistic Regression": { + "type": "logistic_regression", + "estimator": { + "class_weight": "balanced", + "max_iter": 1000, + "random_state": 42 + } + }, + "SMOTE + Random Forest": { + "type": "smote_random_forest", + "smote": { + "random_state": 42, + "k_neighbors": 5 + }, + "estimator": { + "n_estimators": 100, + "max_depth": 10, + "class_weight": "balanced", + "random_state": 42, + "n_jobs": 1 + } + }, + "XGBoost": { + "type": "xgboost", + "estimator": { + "n_estimators": 200, + "max_depth": 6, + "learning_rate": 0.1, + "scale_pos_weight": "train_class_ratio", + "random_state": 42, + "eval_metric": "aucpr", + "tree_method": "hist", + "n_jobs": 1, + "verbosity": 0 + } + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..b1b295e --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,62 @@ +# Architecture + +## Purpose + +The package compares imbalanced binary classifiers while keeping training, operating-point selection, and +final evaluation in distinct data boundaries. + +## Runtime flow + +```text +CLI + benchmark JSON config + ↓ +schema and value validation + ↓ +deterministic stratified split + β”œβ”€β”€ train (60%) + β”‚ β”œβ”€β”€ fit Amount scaler + β”‚ β”œβ”€β”€ optional SMOTE for Random Forest + β”‚ └── fit estimator + β”œβ”€β”€ validation (20%) + β”‚ └── select F1 threshold + └── test (20%) + └── report selected threshold once + ↓ +atomic JSON evidence artifact + console table +``` + +`Time` is dropped because it is an elapsed sequence field rather than an anchored event timestamp. `Amount` +is scaled using training statistics only. V1-V28 are already PCA-transformed source features. + +## Module boundaries + +| Module | Responsibility | +|---|---| +| `data_loader.py` | File fingerprinting, schema/value validation, deterministic splitting, train-fitted scaling | +| `metrics.py` | Model-agnostic confusion/ranking metrics, threshold sweep and deterministic tie-breaking | +| `evaluation.py` | Estimator construction, train-only SMOTE, validation threshold selection, provenance assembly | +| `report.py` | Atomic JSON serialization and dependency-light text rendering | +| `cli.py` | Configurable executable entry point | + +The deterministic logic lives in Python rather than notebooks or orchestration tools. No n8n workflow exists in +this repository; adding one would not improve this local benchmark. A future integration may call `ios-eval` from +n8n for schedules or notifications without moving metric or leakage controls into nodes. + +## Reproducibility contract + +The JSON artifact records: + +- evidence label and claim boundary; +- dataset filename, source, row/column/class counts, and SHA-256; +- random seed and exact train/validation/test composition; +- preprocessing scope and fitted scaler statistics; +- model types, resolved parameters, imbalance handling, and SMOTE scope; +- threshold candidate count, selection split/metric, and validation metrics; +- default and selected-threshold held-out test metrics; +- Python, platform, and dependency versions; +- configuration fingerprint and source Git revision. + +## Scope boundary + +The repository intentionally has no web API, database, queue, authentication layer, live network dependency, +model registry, or production service. Those would add speculative surface without improving benchmark validity. diff --git a/docs/benchmark-reproduction.md b/docs/benchmark-reproduction.md new file mode 100644 index 0000000..dfe8aa9 --- /dev/null +++ b/docs/benchmark-reproduction.md @@ -0,0 +1,43 @@ +# Benchmark Reproduction + +## Dataset + +- Source: Kaggle `mlg-ulb/creditcardfraud` +- File: `creditcard.csv` +- Rows/columns: 284,807 / 31 +- Class counts: 284,315 legitimate / 492 fraud +- SHA-256: `76274b691b16a6c49d3f159c883398e03ccd6d1ee12d9d8ee38f4b4b98551a89` + +The dataset is public but is not committed. Download it directly from Kaggle and verify the hash before comparing +results. + +## Environment and commands + +```bash +python3.11 -m venv .venv +source .venv/bin/activate +python -m pip install ".[dev]" +python -m pip check +pytest -q +ruff check . +ruff format --check . +ios-eval --data data/raw/creditcard.csv --config configs/benchmark.json --output results/benchmark.json +``` + +Verified environment: CPython 3.11.15, NumPy 2.4.3, pandas 3.0.1, scikit-learn 1.8.0, +imbalanced-learn 0.14.1, and XGBoost 3.2.0. + +## Method + +1. Validate exact columns, numeric/finite values, and both classes. +2. Stratify into train 60%, validation 20%, and final test 20% with seed 42. +3. Fit the `Amount` scaler on train and transform validation/test. +4. Fit each model on train; SMOTE applies only to Random Forest training data. +5. Select F1 threshold from candidates 0.00 through 1.00 on validation. +6. Evaluate that selected threshold once on final test. +7. Serialize full provenance and metrics to `results/benchmark.json`. + +## Interpretation + +The artifact is evidence for this public dataset and procedure only. It is not evidence of production readiness, +fraud-loss prevention, client impact, or generalization to another transaction distribution. diff --git a/docs/evidence.md b/docs/evidence.md new file mode 100644 index 0000000..9cdfe32 --- /dev/null +++ b/docs/evidence.md @@ -0,0 +1,39 @@ +# Evidence and Claim Policy + +## Evidence classes + +| Label | Meaning | Required proof | +|---|---|---| +| Implemented | Capability exists | Reachable source path | +| Tested | Deterministic behavior is asserted | Passing automated test and CI | +| Benchmarked | Metrics were reproduced | Named dataset, config, environment, method, and result artifact | +| Synthetic | Inputs or outcomes are simulated | Generator, seed, and assumptions | +| Production | Observed in a deployed system | Deployment and runtime evidence | +| Client outcome | Real engagement result | Permission and attributable/anonymized evidence | + +## Current evidence + +**Benchmarked Research Prototype** + +- Implemented: reusable data, metrics, evaluation, reporting, and CLI package. +- Tested: 32 deterministic tests cover metrics, threshold sweeps, malformed data/config, deterministic splits, + train-only preprocessing/SMOTE, test-label isolation, and report serialization. +- Benchmarked: all three models reran on the named public dataset; results are in + `results/benchmark.json`. +- Docker/CI: test execution is configured and must be reported from actual GitHub Actions runs. +- Production/client: no evidence and no claim. + +## Corrected evidence + +The earlier three-model table selected thresholds using the same final test labels it reported. This execution +pass replaced that method with train/validation/test separation. The old threshold-dependent operating metrics +and related marketing drafts are superseded rather than presented as comparable production improvements. + +The reproduced XGBoost average precision is **0.8756**. Average precision is threshold-independent; precision, +recall, F1, FP, and FN in the current report use a threshold selected on validation data only. + +## Prohibited claims + +Do not infer fraud dollars prevented, generalization to client traffic, production readiness, SLA performance, +or business KPI impact from this benchmark. Any later website or case study should link to the JSON artifact and +retain the label **public-dataset benchmark**. diff --git a/docs/failure-modes.md b/docs/failure-modes.md new file mode 100644 index 0000000..ac2d29f --- /dev/null +++ b/docs/failure-modes.md @@ -0,0 +1,28 @@ +# Failure Modes and Controls + +| Failure mode | Control implemented | Verification | +|---|---|---| +| Threshold optimized on final test labels | Select on validation; test once | Test-label mutation cannot change threshold | +| Scaling fitted before split | Split first; fit `Amount` scaler on train | Fitted mean equals raw train mean | +| SMOTE contaminates evaluation | Resample training partition only | Training-only provenance and tests | +| Non-deterministic splits/models | Explicit seed 42 and single-thread model configs | Repeated split tests and pinned config | +| Missing/malformed data | Required schema, numeric/finite values, both labels | Missing, empty, extra, nonnumeric, NaN/Inf tests | +| Metric crashes or silent nonsense | Shape/range/binary validation; explicit single-class ranking result | Known-array and edge-case tests | +| Ambiguous threshold ties | Highest threshold wins equal target score | Deterministic tie test | +| Partial result artifact | Atomic temporary write then replace | Serialization test | +| Dataset drift | SHA-256 plus row/column/class provenance | Machine-readable report | +| Dependency drift/supply-chain findings | Exact minimal pins, `pip check`, Ruff, `pip-audit` | Execution report and CI | +| Secret or dataset leakage | Git ignore rules and committed-file scan | Pre-publication audit | + +## Remaining limitations + +- A random stratified split does not test temporal drift or out-of-time generalization. +- Validation F1 is a generic operating objective; a real deployment needs review cost, fraud loss, capacity, and + policy constraints. +- One validation split can still produce threshold variance. Repeated or nested validation would provide stronger + uncertainty estimates. +- The public source lacks institution/account context and cannot establish live performance. +- Probability calibration is not evaluated. +- Docker is validated in Linux CI because Docker is not installed in the local macOS execution environment. + +These are technical limitations, not production incidents or client outcomes. diff --git a/ios-risk-eval-harness/.gitignore b/ios-risk-eval-harness/.gitignore deleted file mode 100644 index f5a1d94..0000000 --- a/ios-risk-eval-harness/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -data/raw/ -venv/ -*.csv -__pycache__/ -*.pyc -wandb/ -.history/ -.claude/ \ No newline at end of file diff --git a/ios-risk-eval-harness/README.md b/ios-risk-eval-harness/README.md deleted file mode 100644 index adb5143..0000000 --- a/ios-risk-eval-harness/README.md +++ /dev/null @@ -1,267 +0,0 @@ -# IOS Risk β€” Evaluation Harness - -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![Python](https://img.shields.io/badge/Python-3.9%2B-blue)](https://www.python.org/) -[![Project](https://img.shields.io/badge/IntelligenceOS-Project%2001-green)](https://github.com/Etherlabs-dev/eval-harness) - -### Project 01 of IntelligenceOS Β· Phase 1 Β· Evaluation Layer - -> "You cannot improve what you cannot measure. Every AI lab builds this first internally." - -A production-ready ML evaluation harness for credit card fraud detection. Trains and benchmarks three model architectures against a real-world imbalanced dataset, with reusable evaluation infrastructure designed to score every future model in the IntelligenceOS platform. - ---- - -## Results - -``` -================================================================= -IOS RISK EVAL HARNESS β€” FINAL MODEL COMPARISON REPORT -================================================================= - avg_precision precision recall f1 fp fn best_threshold -model -Logistic Regression 0.7156 0.8070 0.4694 0.5935 11 52 1.00 -SMOTE + Random Forest 0.8293 0.8041 0.7959 0.8000 19 20 0.85 -XGBoost 0.8693 0.8842 0.8571 0.8705 11 14 0.35 -================================================================= - -Best model by avg_precision: XGBoost -Best model by f1: XGBoost -Fewest missed fraud (fn): XGBoost -``` - -**XGBoost catches 84 of 98 fraud cases in the test set with only 11 false alarms across 56,962 transactions β€” a 21% improvement in average precision over the baseline.** - ---- - -## Project Structure - -``` -ios-risk-eval-harness/ -β”‚ -β”œβ”€β”€ ios_eval/ # Core evaluation library -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ metrics.py # Evaluation functions (compute, sweep, optimise) -β”‚ └── data_loader.py # Data pipeline (load, scale, split) -β”‚ -β”œβ”€β”€ scripts/ # Training and reporting scripts -β”‚ β”œβ”€β”€ run_eval.py # Baseline: Logistic Regression -β”‚ β”œβ”€β”€ train_smote_rf.py # SMOTE + Random Forest -β”‚ β”œβ”€β”€ train_xgboost.py # XGBoost -β”‚ └── generate_report.py # Full model comparison report -β”‚ -β”œβ”€β”€ data/ -β”‚ └── raw/ # creditcard.csv (not committed β€” see setup) -β”‚ -β”œβ”€β”€ results/ # JSON outputs from evaluation runs -β”œβ”€β”€ content/ # Case study, Substack posts, video script -β”œβ”€β”€ requirements.txt -└── README.md -``` - ---- - -## Quickstart - -### 1. Clone and set up environment - -```bash -git clone https://github.com/Etherlabs-dev/eval-harness.git -cd eval-harness/ios-risk-eval-harness -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -``` - -### 2. Get the dataset - -You need a Kaggle account and API token. - -```bash -# Install Kaggle CLI and set your token -pip install kaggle -export KAGGLE_API_TOKEN=your_token_here - -# Download and extract -kaggle datasets download -d mlg-ulb/creditcardfraud -unzip creditcardfraud.zip -d data/raw/ -rm creditcardfraud.zip -``` - -Verify the data: -```bash -python3 -c " -import pandas as pd -df = pd.read_csv('data/raw/creditcard.csv') -print(df.shape) # (284807, 31) -print(df.Class.value_counts()) # 0: 284315, 1: 492 -" -``` - -### 3. Run the full report - -```bash -PYTHONPATH=. python3 scripts/generate_report.py -``` - -This trains all three models and prints the comparison table. Takes approximately 3-4 minutes. - ---- - -## Core Library - -### `ios_eval/metrics.py` - -Three reusable functions that every model is evaluated through. - -**`compute_core_metrics(y_true, y_pred, y_prob=None, threshold=0.5)`** - -Calculates all key metrics from the confusion matrix up. - -```python -from ios_eval.metrics import compute_core_metrics -import numpy as np - -metrics = compute_core_metrics( - y_true=np.array([0, 1, 0, 1, 0]), - y_pred=np.array([0, 1, 0, 0, 0]), - y_prob=np.array([0.1, 0.9, 0.2, 0.4, 0.05]) -) -# Returns: precision, recall, f1, fpr, tn, fp, fn, tp, roc_auc, avg_precision -``` - -**`evaluate_threshold_range(y_true, y_prob, thresholds=None)`** - -Sweeps thresholds from 0.0 to 1.0 and returns a DataFrame showing metrics at each point. Used to understand the precision/recall tradeoff before committing to a threshold. - -```python -from ios_eval.metrics import evaluate_threshold_range - -df = evaluate_threshold_range(y_true, y_prob) -print(df[['precision', 'recall', 'f1', 'fpr']].to_string()) -``` - -**`find_optimal_threshold(y_true, y_prob, metric='f1')`** - -Automatically selects the threshold that maximises a given metric. Parameterise with `metric='recall'` to optimise for fraud capture, or `metric='f1'` for balance. - -```python -from ios_eval.metrics import find_optimal_threshold - -best_threshold, best_metrics = find_optimal_threshold(y_true, y_prob) -``` - ---- - -### `ios_eval/data_loader.py` - -**`load_and_prepare_data(data_path, test_size=0.2, random_state=42, scale_amount=True)`** - -Handles the full data pipeline: -- Drops `Time` column (recording artifact, not a transaction property) -- Applies `StandardScaler` to `Amount` to match the scale of PCA features V1–V28 -- Stratified train/test split β€” preserves the 0.17% fraud ratio in both sets -- Returns `X_train, X_test, y_train, y_test` as numpy arrays - -```python -from ios_eval.data_loader import load_and_prepare_data - -X_train, X_test, y_train, y_test = load_and_prepare_data() -# X_train: (227845, 29) X_test: (56962, 29) -# Train fraud %: 0.173 Test fraud %: 0.172 -``` - ---- - -## Run Individual Models - -```bash -# Baseline β€” Logistic Regression -PYTHONPATH=. python3 scripts/run_eval.py - -# SMOTE + Random Forest -PYTHONPATH=. python3 scripts/train_smote_rf.py - -# XGBoost -PYTHONPATH=. python3 scripts/train_xgboost.py - -# Full comparison report -PYTHONPATH=. python3 scripts/generate_report.py -``` - ---- - -## Dataset - -**Credit Card Fraud Detection** β€” ULB Machine Learning Group -- Source: [Kaggle](https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud) -- 284,807 transactions over two days in September 2013 -- 492 fraud cases (0.17%) -- Features: `Time`, `V1`–`V28` (PCA-transformed), `Amount`, `Class` -- `data/raw/` is excluded from version control β€” download instructions above - ---- - -## The Core Problem - -On a 0.17% fraud rate, a model that predicts "not fraud" on every transaction achieves 99.83% accuracy. This is why accuracy is the wrong metric for imbalanced classification. - -The metrics that matter here: - -| Metric | What it answers | -|--------|----------------| -| **Precision** | Of all fraud flags raised, how many were real fraud? | -| **Recall** | Of all real fraud, how much did we catch? | -| **F1** | Balanced score across both | -| **Avg Precision** | Overall quality on imbalanced data β€” more honest than ROC AUC | -| **FPR** | How often do we wrongly flag innocent customers? | - -Each model in this project attacks the imbalance problem differently: -- **Logistic Regression** β€” `class_weight='balanced'` as a simple weight adjustment -- **Random Forest** β€” SMOTE generates synthetic fraud examples to balance training data -- **XGBoost** β€” `scale_pos_weight=577` handles imbalance natively without synthetic data - ---- - -## Stack - -| Library | Use | -|---------|-----| -| `scikit-learn` | Logistic Regression, Random Forest, preprocessing, metrics | -| `xgboost` | XGBoost classifier | -| `imbalanced-learn` | SMOTE oversampling | -| `pandas` | Data manipulation, results tables | -| `numpy` | Array operations | - ---- - -## Context - -This is **Project 01** of IntelligenceOS β€” a 12-project build toward a domain-specific AI platform for finance and risk. - -The evaluation harness built here is the measurement infrastructure that every future model, agent, and system in IntelligenceOS gets scored against. It was built first because consistent measurement is the precondition for consistent improvement. - -**Roadmap:** -| Project | Layer | Status | -|---------|-------|--------| -| 01 β€” IOS Risk Eval Harness | Evaluation Layer | βœ… Complete | -| 02 β€” Domain Data Foundry | Data Layer | πŸ”œ Next | -| 03 β€” Domain Intelligence Core | Model Families | Upcoming | -| 04 β€” Verification System | Governance Layer | Upcoming | -| 05 β€” Agent Runtime v1 | Orchestration | πŸš€ First Deploy | - ---- - ---- - -## License - -Copyright 2026 Ugo Chukwu / Etherlabs - -Licensed under the **Apache License, Version 2.0** β€” you may use, modify, and distribute this code freely, including for commercial purposes, provided attribution is maintained and the license notice is included. - -See the [LICENSE](../LICENSE) file for full terms, or visit [apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). - ---- - -*Built by [Ugo Chukwu](https://github.com/Etherlabs-dev) Β· IntelligenceOS v0.1* diff --git a/ios-risk-eval-harness/configs/eval_config.yaml b/ios-risk-eval-harness/configs/eval_config.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/content/build-in-public-linkedin.md b/ios-risk-eval-harness/content/build-in-public-linkedin.md deleted file mode 100644 index 9b13da3..0000000 --- a/ios-risk-eval-harness/content/build-in-public-linkedin.md +++ /dev/null @@ -1,76 +0,0 @@ -# LinkedIn Post β€” Build in Public - ---- - -**Option A (Professional / Technical audience)** - -I just completed the first project in a 12-month AI platform build. - -Here's what I built, what I learned, and why the order of operations matters more than the model. - -**The project:** A fraud detection evaluation harness. 284,807 real credit card transactions. 0.17% fraud rate. - -**The approach most people take:** Pick a model. Train it. Check accuracy. Ship it. - -**The problem with that approach:** A model that predicts "not fraud" on every transaction is 99.83% accurate. And completely useless. Accuracy is the wrong metric when your data is this imbalanced. - -**What I did instead:** Built the measurement infrastructure first. - -Before training a single model, I built a reusable evaluation module with three functions: -- compute_core_metrics() β€” precision, recall, F1, FPR, ROC AUC, average precision -- evaluate_threshold_range() β€” shows the full precision/recall tradeoff across every threshold -- find_optimal_threshold() β€” automatically selects the best threshold for a given business priority - -Only then did I train models. - -**The results across three models:** - -Logistic Regression (baseline) β†’ avg_precision: 0.716, false alarms: 1,441 -SMOTE + Random Forest β†’ avg_precision: 0.829, false alarms: 112 (-92%) -XGBoost β†’ avg_precision: 0.869, missed fraud: 14, false alarms: 11 - -XGBoost caught 84 of 98 fraud cases in the test set while raising only 11 false alarms across 56,962 transactions. - -**The lesson that will stick:** - -Evaluation infrastructure outlasts any individual model. The functions I built in week 1 will evaluate every model I train for the next 12 months. Building the measurement layer first wasn't overhead β€” it was the most leveraged work I did. - -This is Project 01 of IntelligenceOS β€” a 12-project build from eval harness to full AI platform. - -Full case study and code in comments. - ---- - -**Option B (Shorter / Story-driven)** - -7 days ago I started building an AI platform from scratch. - -Not a tutorial project. The real thing. - -Project 01: a fraud detection evaluation harness. - -The dataset has 284,807 transactions and 492 fraud cases. That's 0.17%. Before I trained a single model, I had to answer one question: how do I even know if a model is working? - -Because accuracy is useless here. A model that never flags fraud is 99.83% accurate. - -So I built the measurement infrastructure first. Precision, recall, threshold analysis, average precision β€” a full evaluation suite that every model runs through. - -Then I trained three models: -β†’ Logistic Regression: missed 52 fraud cases -β†’ Random Forest + SMOTE: missed 20 -β†’ XGBoost: missed 14 - -The same evaluation code ran on all three. The comparison is fair because the measurement is consistent. - -21% improvement from baseline to best model. 38 additional fraud cases caught. - -Building in public. 11 more projects to go. - -Code: [GitHub link] -Deep dive: [Substack link] - -#BuildInPublic #MachineLearning #FraudDetection #AI #Python - ---- - -*Use Option A for a technical/ML audience. Use Option B for a broader professional audience. Add real links before posting.* diff --git a/ios-risk-eval-harness/content/build-in-public-reddit.md b/ios-risk-eval-harness/content/build-in-public-reddit.md deleted file mode 100644 index 2fac8cd..0000000 --- a/ios-risk-eval-harness/content/build-in-public-reddit.md +++ /dev/null @@ -1,86 +0,0 @@ -# Reddit Posts β€” Build in Public - ---- - -## Post 1: r/MachineLearning or r/learnmachinelearning - -**Title:** Built a fraud detection eval harness from scratch β€” here's what the numbers actually look like on real imbalanced data - -**Body:** - -I just finished a week-long project building an evaluation harness for credit card fraud detection. Sharing the full results because I think the numbers tell an interesting story about imbalanced classification that tutorials often skip over. - -**The dataset:** ULB Credit Card Fraud (Kaggle) β€” 284,807 transactions, 492 fraud. 0.17% positive class. - -**What I built before training any model:** - -A reusable metrics module with three functions: - -1. `compute_core_metrics(y_true, y_pred, y_prob)` β€” calculates precision, recall, F1, FPR, ROC AUC, average precision from scratch using confusion matrix components. Zero-division guards throughout. - -2. `evaluate_threshold_range(y_true, y_prob)` β€” sweeps thresholds 0.0 to 1.0 in 0.05 steps, returns a DataFrame. Useful for visualising the precision/recall tradeoff before committing to a threshold. - -3. `find_optimal_threshold(y_true, y_prob, metric='f1')` β€” finds the threshold that maximises a given metric. Parameterised so you can optimise for recall if you're prioritising fraud capture, or F1 for balance. - -**Results across three models (at optimal threshold for each):** - -``` -Model avg_prec precision recall f1 fp fn threshold -Logistic Regression 0.7156 0.8070 0.4694 0.5935 11 52 1.00 -SMOTE + Random Forest 0.8293 0.8041 0.7959 0.8000 19 20 0.85 -XGBoost 0.8693 0.8842 0.8571 0.8705 11 14 0.35 -``` - -**A few things worth noting:** - -- ROC AUC was 0.97 for all three models. Average precision ranged from 0.716 to 0.869. On imbalanced data, avg_precision is the more honest metric β€” ROC AUC inflates because of the massive true negative count. - -- The optimal threshold varied wildly: 1.0 for Logistic Regression, 0.85 for Random Forest, 0.35 for XGBoost. Default 0.5 would have given significantly worse results for all three. - -- SMOTE dropped false positives from 1,441 to 112 at default threshold (0.5). That's the biggest single-step improvement in the whole project. - -- XGBoost with `scale_pos_weight=577` performed better than SMOTE + RF without generating any synthetic data. Faster training too. - -**What I'm not sure about:** - -Using `find_optimal_threshold` on the test set to pick the threshold feels slightly leaky β€” ideally you'd pick the threshold on a validation set and evaluate on test. But for a single-dataset project without a three-way split, I used the full test set for both. Happy to hear thoughts on this. - -Code on GitHub: [link] - ---- - -## Post 2: r/buildinpublic or r/startups - -**Title:** Month 1 of building an AI platform in public β€” first project complete, here's what I shipped - -**Body:** - -I'm building IntelligenceOS β€” a domain-specific AI platform for finance and risk β€” from scratch over 12 months. Sharing every project publicly as I go. - -**Project 01 is done: IOS Risk Eval Harness** - -The first project isn't a product. It's the measurement infrastructure that every future product gets evaluated against. This is intentional β€” you cannot improve what you cannot measure, and you cannot trust improvement claims without consistent measurement. - -**What I shipped:** - -- `ios_eval/metrics.py` β€” reusable evaluation module, three functions, works on any binary classifier -- `ios_eval/data_loader.py` β€” data pipeline with stratified splitting and scaling -- Three model scripts: Logistic Regression, SMOTE + Random Forest, XGBoost -- `scripts/generate_report.py` β€” runs all three models and outputs a comparison table in one command - -**The headline result:** - -21% improvement in average precision from baseline (0.716) to best model (0.869). XGBoost catches 84 of 98 fraud cases in the test set with only 11 false alarms across 56,962 transactions. - -**Why I'm building in public:** - -Accountability. I have a 12-project roadmap. Each project builds infrastructure the next one requires. The sequence is deliberate β€” by Month 4 I have something deployable. By Month 12 I have a platform. - -If you're building something similar or have feedback on the approach, I'd love to hear it. - -GitHub: [link] -Full writeup: [Substack link] - ---- - -*Choose the post based on subreddit. r/MachineLearning and r/learnmachinelearning for Post 1. r/buildinpublic, r/startups, or r/SideProject for Post 2. Add links before posting.* diff --git a/ios-risk-eval-harness/content/build-in-public-twitter.md b/ios-risk-eval-harness/content/build-in-public-twitter.md deleted file mode 100644 index 0144536..0000000 --- a/ios-risk-eval-harness/content/build-in-public-twitter.md +++ /dev/null @@ -1,145 +0,0 @@ -# Build in Public β€” X (Twitter) Thread - ---- - -**Tweet 1 (Hook)** -Built a fraud detection system from scratch this week. - -99.83% accuracy. Sounds great. - -It caught zero fraud. - -Here's why accuracy is the most dangerous metric in ML 🧡 - ---- - -**Tweet 2** -The dataset: 284,807 credit card transactions. - -492 are fraud. That's 0.17%. - -A model that says "not fraud" on every single transaction gets 99.83% accuracy. - -That model is useless. And most teams would ship it. - ---- - -**Tweet 3** -The problem is called class imbalance. - -When your positive class (fraud) is 577x rarer than your negative class (legit), standard ML breaks down in 3 ways: - -β€’ Accuracy becomes meaningless -β€’ Default thresholds fail -β€’ Models learn the wrong thing - ---- - -**Tweet 4** -So what metrics actually matter? - -β€’ Precision β€” of all fraud flags, how many were real? -β€’ Recall β€” of all real fraud, how much did we catch? -β€’ Avg Precision β€” overall quality on imbalanced data -β€’ FPR β€” how often do we wrongly flag innocent customers? - -Before I trained a single model, I built the measurement system. - ---- - -**Tweet 5** -Baseline: Logistic Regression with class_weight='balanced' - -At default threshold 0.5: -β€’ Precision: 0.059 -β€’ Recall: 0.918 -β€’ False positives: 1,441 - -It caught 92% of fraud β€” but raised 1,441 false alarms to do it. - ---- - -**Tweet 6** -The threshold insight that changed everything: - -The 0.5 cutoff is arbitrary on imbalanced data. - -I built a function that sweeps every threshold from 0.0 to 1.0 and shows you the precision/recall tradeoff at each point. - -The real optimal threshold was 0.35 β€” not 0.5. - ---- - -**Tweet 7** -Model 2: SMOTE + Random Forest - -SMOTE generates synthetic fraud examples to balance training data. -394 fraud cases β†’ 227,451 synthetic examples. - -Result: -β€’ Avg precision: 0.829 (+16% over baseline) -β€’ False positives: 112 (down from 1,441) - -One technique. 92% fewer false alarms. - ---- - -**Tweet 8** -Model 3: XGBoost - -Instead of SMOTE, XGBoost handles imbalance natively. -scale_pos_weight = 577 (legit/fraud ratio). - -Builds trees sequentially β€” each one corrects the last one's mistakes. - -Result: -β€’ Avg precision: 0.869 (+21% over baseline) -β€’ Caught 84 of 98 fraud cases -β€’ Only 11 false alarms across 56,962 transactions - ---- - -**Tweet 9** -Final comparison: - -``` -Model avg_precision recall fp fn -Logistic Regression 0.716 0.469 11 52 -SMOTE + Random Forest 0.829 0.796 19 20 -XGBoost 0.869 0.857 11 14 -``` - -Same data. Same eval system. Three different approaches to the same problem. - ---- - -**Tweet 10** -The lesson: - -Build your measurement system first. -Before the model. Before the training loop. Before anything. - -You can't improve what you can't measure. - -This eval harness is Project 01 of IntelligenceOS β€” the measurement infrastructure every future model gets scored against. - ---- - -**Tweet 11** -What's next: - -Project 02 β€” Domain Data Foundry -Building the pipeline that takes raw data β†’ cleans β†’ engineers features β†’ versions β†’ exports training-ready datasets with one command. - -Building in public. Every project. Every step. - -GitHub: https://github.com/Etherlabs-dev/eval-harness - ---- - -**Tweet 12 (CTA)** -If you're building in fintech or fraud detection and want to talk shop β€” my DMs are open. - -And if this was useful, repost the first tweet. Someone building in ML right now needs to hear that accuracy is a lie. - -πŸ” diff --git a/ios-risk-eval-harness/content/build-in-public-x-thread.md b/ios-risk-eval-harness/content/build-in-public-x-thread.md deleted file mode 100644 index 01bea5b..0000000 --- a/ios-risk-eval-harness/content/build-in-public-x-thread.md +++ /dev/null @@ -1,165 +0,0 @@ -# X / Twitter Thread β€” Build in Public - ---- - -**Tweet 1 (Hook)** -I spent 7 days building a fraud detection system from scratch. - -Not a tutorial. Not a Kaggle notebook. - -A real evaluation harness that proved a 21% performance improvement across 3 model architectures. - -Here's everything I built and what I learned 🧡 - ---- - -**Tweet 2 (The Problem)** -The dataset: 284,807 credit card transactions. - -492 are fraud. That's 0.17%. - -If a model predicts "not fraud" on every single transaction β€” it's right 99.83% of the time. - -And it's completely useless. - -This is why accuracy is a lie in fraud detection. - ---- - -**Tweet 3 (The Right Metrics)** -What actually matters: - -β€’ Precision β€” of every fraud flag raised, how many were real? -β€’ Recall β€” of all real fraud, how much did you catch? -β€’ Avg Precision β€” honest performance on imbalanced data -β€’ FPR β€” how often did you wrongly accuse an innocent person? - -These 4 numbers tell the real story. - ---- - -**Tweet 4 (Build Order)** -Most people build the model first. - -I built the measurement infrastructure first. - -metrics.py β†’ 3 functions: -- compute_core_metrics() -- evaluate_threshold_range() -- find_optimal_threshold() - -Every model I ever train runs through these same functions. Results are always comparable. - ---- - -**Tweet 5 (The Threshold Insight)** -Here's something most ML tutorials skip: - -The 0.5 threshold is arbitrary. - -This table shows precision and recall at every threshold from 0.0 to 1.0: - -At 0.0 β†’ catch everything, annoy everyone -At 0.95 β†’ catch nothing, bother no one -At 0.35 β†’ the sweet spot for XGBoost - -Threshold is a business decision, not a technical default. - ---- - -**Tweet 6 (Baseline Model)** -Model 1: Logistic Regression - -Results at threshold 0.5: -β€’ Caught 90/98 fraud cases βœ“ -β€’ Raised 1,441 false alarms βœ— -β€’ avg_precision: 0.716 - -It catches fraud but at the cost of flagging 1,441 innocent customers. - -This is the baseline. Everything has to beat this. - ---- - -**Tweet 7 (SMOTE)** -The problem: 492 fraud cases vs 227,451 legit. - -The model sees so few fraud examples it barely learns what fraud looks like. - -Fix: SMOTE β€” generates synthetic fraud examples by interpolating between real ones. - -Before: 394 fraud in training set -After: 227,451 fraud in training set - -Same test set. No fake data in evaluation. - ---- - -**Tweet 8 (Random Forest Results)** -Model 2: SMOTE + Random Forest - -β€’ Fraud missed: 52 β†’ 20 (-62%) -β€’ False alarms: 1,441 β†’ 112 (-92%) -β€’ avg_precision: 0.716 β†’ 0.829 (+16%) - -The model finally learned what fraud looks like. - -But we're not done. - ---- - -**Tweet 9 (XGBoost)** -Model 3: XGBoost - -Different approach to imbalance β€” no SMOTE. - -Instead: scale_pos_weight = 577 (legit/fraud ratio) -Each tree corrects the mistakes of the previous one. - -Result: -β€’ Fraud missed: 14 (down from 52) -β€’ False alarms: 11 -β€’ avg_precision: 0.869 - ---- - -**Tweet 10 (Final Report)** -Final comparison across 56,962 test transactions: - -``` -Model avg_prec recall f1 fp fn -Logistic Regression 0.716 0.469 0.594 11 52 -Random Forest 0.829 0.796 0.800 19 20 -XGBoost 0.869 0.857 0.870 11 14 -``` - -XGBoost wins everything. -38 additional fraud cases caught vs baseline. -Same false alarm count as Logistic Regression. - ---- - -**Tweet 11 (What I Learned)** -The 5 things this project taught me: - -1. Measure before you model -2. Accuracy is useless on imbalanced data -3. Threshold is a business decision -4. SMOTE before training, never before testing -5. Evaluation infrastructure outlasts any individual model - ---- - -**Tweet 12 (What's Next)** -This is Project 01 of a 12-project AI platform I'm building over 12 months. - -Next: Domain Data Foundry β€” building the pipeline that feeds real financial data into everything I just built. - -Following the full build in public. Every project ships real code. - -Repo: [GitHub link] -Full writeup: [Substack link] - ---- - -*Note: Add actual GitHub and Substack links before posting. Post tweets 1-3 together for the hook, then space remaining tweets 30-60 mins apart for engagement.* diff --git a/ios-risk-eval-harness/content/case-study-contra-upwork.md b/ios-risk-eval-harness/content/case-study-contra-upwork.md deleted file mode 100644 index 850ddf6..0000000 --- a/ios-risk-eval-harness/content/case-study-contra-upwork.md +++ /dev/null @@ -1,136 +0,0 @@ -# Case Study: IOS Risk Eval Harness -## ML Evaluation Infrastructure for Fraud Detection - -**Domain:** Fintech / Risk Intelligence -**Stack:** Python, scikit-learn, XGBoost, imbalanced-learn, Pandas, NumPy -**Duration:** 7 days -**Result:** 21% improvement in fraud detection performance across three model architectures - ---- - -## The Problem - -Credit card fraud detection sounds simple until you look at the data. - -In a real-world dataset of 284,807 transactions, only 492 are fraud. That's 0.17%. A model that predicts "not fraud" for every single transaction would be correct 99.83% of the time β€” and completely useless. - -This is the class imbalance problem, and it breaks every standard machine learning metric. Accuracy tells you nothing. You need precision, recall, F1, average precision, and threshold analysis to understand whether your model is actually working. - -Most fraud detection projects jump straight to building models. This project started differently: **build the measurement infrastructure first.** - ---- - -## The Approach - -### Phase 1 β€” Build the Evaluation Harness - -Before training a single model, I built `metrics.py` β€” a reusable evaluation module with three functions: - -**`compute_core_metrics(y_true, y_pred, y_prob)`** -Calculates every relevant metric from the confusion matrix up: -- Precision, Recall, F1, FPR from raw TP/TN/FP/FN counts -- ROC AUC and Average Precision from probability scores -- Zero-division guards throughout β€” production-safe - -**`evaluate_threshold_range(y_true, y_prob)`** -Sweeps thresholds from 0.0 to 1.0 in 0.05 steps and returns a full DataFrame showing how precision and recall trade off at every cut point. This is how you make an informed threshold decision instead of defaulting to 0.5. - -**`find_optimal_threshold(y_true, y_prob, metric='f1')`** -Automatically selects the threshold that maximises a given metric. Parameterised β€” can optimise for F1, recall, or any metric in the output dictionary. - -**Why build this first:** Every model trained in this project, and every model trained in future projects, runs through the same evaluation functions. Results are comparable by design. - -### Phase 2 β€” Data Pipeline - -Built `data_loader.py` with a single function `load_and_prepare_data()` that handles: -- Dropping the `Time` column (recording artifact, not a transaction property) -- StandardScaler on the `Amount` column to match the scale of PCA-transformed features V1–V28 -- Stratified train/test split β€” preserves the 0.17% fraud ratio in both sets -- Returns clean numpy arrays ready for any scikit-learn compatible model - -Critical detail: scaling happens after splitting. Fitting the scaler on the full dataset before splitting would leak test set information into the training process. - -### Phase 3 β€” Three Models, Increasing Sophistication - -**Model 1 β€” Logistic Regression (Baseline)** -Simple linear model with `class_weight='balanced'`. Establishes the performance floor. Every subsequent model must beat this. - -**Model 2 β€” SMOTE + Random Forest** -SMOTE (Synthetic Minority Oversampling Technique) generates synthetic fraud examples by interpolating between real fraud cases β€” balancing the training set from 394 fraud vs 227,451 legit to 227,451 vs 227,451. Random Forest trained on this balanced dataset with 100 trees, max_depth 10. - -**Model 3 β€” XGBoost** -Sequential boosting β€” each tree corrects the errors of the previous one. Uses `scale_pos_weight=577` (the legit/fraud ratio) instead of SMOTE, handling imbalance natively. 200 trees, max_depth 6, learning_rate 0.1. - ---- - -## The Results - -``` -================================================================= -IOS RISK EVAL HARNESS β€” FINAL MODEL COMPARISON REPORT -================================================================= - avg_precision precision recall f1 fp fn best_threshold -model -Logistic Regression 0.7156 0.8070 0.4694 0.5935 11 52 1.00 -SMOTE + Random Forest 0.8293 0.8041 0.7959 0.8000 19 20 0.85 -XGBoost 0.8693 0.8842 0.8571 0.8705 11 14 0.35 -================================================================= - -Best model by avg_precision: XGBoost -Best model by f1: XGBoost -Fewest missed fraud (fn): XGBoost -``` - -### What These Numbers Mean in Practice - -Across 56,962 test transactions (98 actual fraud cases): - -| | Logistic Regression | Random Forest | XGBoost | -|--|--|--|--| -| Fraud caught | 46/98 | 78/98 | 84/98 | -| Fraud missed | 52 | 20 | **14** | -| False alarms | 11 | 19 | **11** | -| Avg Precision | 0.716 | 0.829 | **0.869** | - -XGBoost catches 84 out of 98 fraud cases while raising only 11 false alarms. That's 85.7% of all fraud detected, with a false positive rate of 0.019% β€” less than 2 in every 10,000 legitimate transactions flagged incorrectly. - -The baseline Logistic Regression missed 52 fraud cases. XGBoost misses 14. **That's 38 additional fraud cases caught** β€” each one representing real financial loss prevented. - ---- - -## Key Engineering Decisions - -**Evaluation-first architecture:** The eval harness was built before any model. This is the correct order β€” you cannot improve what you cannot measure, and you cannot trust improvement claims without consistent measurement. - -**Threshold as a business decision:** The optimal threshold varies by model (1.0 for LR, 0.85 for RF, 0.35 for XGBoost) and by business priority. A bank prioritising fraud prevention optimises for recall. A bank prioritising customer experience optimises for precision. The `find_optimal_threshold()` function supports both by parameterising the target metric. - -**Reproducibility by design:** Every random operation uses `random_state=42`. The stratified split preserves class ratios. The same data pipeline runs identically every time. Results are reproducible without notebooks or manual steps. - -**Production-safe code patterns:** Zero-division guards on every metric calculation. Python native float/int casting before logging (avoids numpy serialisation bugs). Modular functions with explicit type signatures. - ---- - -## What This Enables - -This evaluation harness is domain-agnostic. The same `metrics.py` and `data_loader.py` pattern applies to: -- Insurance claim fraud -- Anti-money laundering transaction monitoring -- Credit underwriting risk scoring -- Any imbalanced binary classification problem in fintech - -The infrastructure built here is the measurement layer that every subsequent model, agent, or system gets evaluated against. - ---- - -## Services Available - -- **ML Evaluation Framework Design** β€” build measurement infrastructure before models -- **Fraud Detection System Development** β€” end-to-end pipeline from raw data to evaluated model -- **Imbalanced Classification** β€” SMOTE, threshold tuning, cost-sensitive evaluation -- **Model Comparison & Selection** β€” systematic benchmarking across model families -- **Production-Ready ML Code** β€” clean, documented, reproducible Python - ---- - -*Built as Project 01 of IntelligenceOS β€” a 12-project AI platform build across 12 months.* -*GitHub: [link] | Substack: [link]* diff --git a/ios-risk-eval-harness/content/substack-post-1-accuracy-is-worthless.md b/ios-risk-eval-harness/content/substack-post-1-accuracy-is-worthless.md deleted file mode 100644 index f991db7..0000000 --- a/ios-risk-eval-harness/content/substack-post-1-accuracy-is-worthless.md +++ /dev/null @@ -1,142 +0,0 @@ -# Substack Post 1 of 3 -## "Why 99.8% Accuracy is Worthless β€” and What to Use Instead" - -*Part 1 of a 3-part series on building a fraud detection system from scratch* - ---- - -There's a number that looks great on a resume, sounds impressive in a demo, and means almost nothing in real fraud detection. - -That number is accuracy. - -Here's why. - ---- - -### The Dataset - -Imagine you're handed a dataset of 284,807 credit card transactions. Your job is to build a model that identifies which ones are fraudulent. - -You open the data. You find 492 fraud cases. - -That's 0.17% of all transactions. - -Before you write a single line of code, I want you to think about what happens if your model just predicts "not fraud" on every single transaction, no matter what. - -Let's count: -- 284,315 legitimate transactions β†’ predicted "not fraud" β†’ all correct -- 492 fraud cases β†’ predicted "not fraud" β†’ all wrong - -Total correct: 284,315 out of 284,807. - -**Accuracy: 99.83%.** - -Your model caught zero fraud. It's completely useless. And it scores 99.83% accuracy. - -This is the class imbalance problem. It breaks accuracy as a metric completely. - ---- - -### What Actually Matters - -When your model makes a prediction, every transaction falls into one of four buckets: - -``` - Predicted: Legit Predicted: Fraud -Actual: Legit TN FP -Actual: Fraud FN TP -``` - -- **TP (True Positive):** Fraud correctly caught. This is the goal. -- **TN (True Negative):** Legitimate transaction correctly cleared. Good. -- **FP (False Positive):** Legitimate transaction wrongly flagged as fraud. Costs customer trust. -- **FN (False Negative):** Fraud missed entirely. Costs real money. - -From these four numbers, the metrics that actually tell you something: - -**Precision = TP / (TP + FP)** -*Of all the fraud flags my model raised, what percentage were real fraud?* - -If your model flags 100 transactions as fraud and only 6 are actually fraud β€” your precision is 6%. You're crying wolf constantly. - -**Recall = TP / (TP + FN)** -*Of all the real fraud in the dataset, what percentage did my model catch?* - -If there are 98 fraud cases and your model catches 84 β€” your recall is 85.7%. You're catching most of the fraud. - -**F1 = 2 Γ— (Precision Γ— Recall) / (Precision + Recall)** -*A single number that balances both.* - -Useful when you care about both precision and recall equally. In practice, fraud teams usually weight recall higher β€” missing fraud is more expensive than a false alarm. - -**Average Precision** -*The overall quality of your probability scores across all possible thresholds.* - -This is the metric I trust most on imbalanced data. It answers: regardless of where you set your decision threshold, how good is this model at separating fraud from legit? Score of 1.0 is perfect. Score of 0.5 is barely better than random. - ---- - -### The Threshold Nobody Talks About - -Here's something most tutorials skip entirely. - -Your model doesn't output a 0 or a 1. It outputs a probability. Something like: "I'm 73% confident this transaction is fraud." - -You need a rule to convert that probability into a decision. That rule is the threshold. - -The default threshold is 0.5 β€” if the model is more than 50% confident it's fraud, flag it as fraud. - -But 0.5 is completely arbitrary. - -Look at what happens when you sweep the threshold from 0.0 to 1.0 on a real model: - -``` -threshold precision recall fpr -0.00 0.400 1.000 1.000 ← flags everything as fraud -0.15 0.571 1.000 0.500 ← still too many false alarms -0.35 0.884 0.857 0.000 ← sweet spot -0.70 1.000 0.500 0.000 ← misses half the fraud -0.95 0.000 0.000 0.000 ← flags nothing -``` - -At threshold 0.0: the model flags every transaction as fraud. Perfect recall (catches 100% of fraud) but terrible precision (flags innocent customers constantly). - -At threshold 0.95: the model is so conservative it barely flags anything. Almost no false alarms, but it misses most of the fraud. - -At threshold 0.35: the sweet spot for this model β€” 88% precision, 86% recall, zero false positive rate. - -**The threshold is a business decision, not a technical default.** A bank that prioritises catching fraud sets a low threshold. A bank that prioritises customer experience sets a higher one. The data scientist's job is to show the tradeoff table, not to pick 0.5 by default. - ---- - -### What My Baseline Model Actually Did - -When I trained the simplest possible model (Logistic Regression with balanced class weights) on this dataset and evaluated it at the default 0.5 threshold: - -- Caught 90 out of 98 fraud cases β€” recall: 91.8% βœ“ -- Raised 1,441 false alarms on innocent transactions βœ— -- Precision: 5.9% - -Every 16 fraud flags, only 1 was real fraud. The other 15 were innocent customers being incorrectly flagged. - -That's not a usable fraud detection system. It's a false alarm machine that occasionally catches fraud. - -**Average precision: 0.716.** - -That number is the baseline. Everything I build from here has to beat it. - ---- - -### Coming Up - -In Part 2, I'll show exactly how I fixed the false alarm problem β€” using SMOTE to balance the training data and Random Forest to learn more complex fraud patterns. False alarms dropped from 1,441 to 112. - -In Part 3, I'll show the final model comparison: three models, one dataset, one winner β€” and why XGBoost outperformed both without any synthetic data generation. - -The code for everything in this series is on GitHub: [link] - ---- - -*This is part of my build-in-public series documenting IntelligenceOS β€” a 12-project AI platform build across 12 months. Project 01 is the evaluation harness. Everything ships as real, working code.* - -*Subscribe to follow the full build.* diff --git a/ios-risk-eval-harness/content/substack-post-2-how-to-measure-fraud.md b/ios-risk-eval-harness/content/substack-post-2-how-to-measure-fraud.md deleted file mode 100644 index a1f0284..0000000 --- a/ios-risk-eval-harness/content/substack-post-2-how-to-measure-fraud.md +++ /dev/null @@ -1,220 +0,0 @@ -# Substack Post 2 of 3 -## "How to Actually Measure a Fraud Detection Model" - -*Part 2 of a 3-part series on building a fraud detection system from scratch* - ---- - -In Part 1, I showed why accuracy is useless on imbalanced data and introduced the metrics that actually matter: precision, recall, F1, and average precision. - -In this post I want to show you the code β€” specifically, the evaluation infrastructure I built before training a single model. - -This is the piece most tutorials skip. They jump straight to the model. I think that's backwards. - ---- - -### Build the Measurement Layer First - -Here's the principle I used to design this project: - -**You cannot improve what you cannot measure. And you cannot trust improvement claims without consistent measurement.** - -If you build a model, evaluate it with some ad hoc code, then build a better model and evaluate it differently β€” your comparison is meaningless. You're not measuring the models against each other. You're measuring the models against two different rulers. - -The solution is to build the evaluation infrastructure first and run every model through the same functions. The code I wrote in week 1 will evaluate every model I build for the next 12 months. That's the leverage. - ---- - -### The Three Functions - -All of this lives in `ios_eval/metrics.py`. - -**Function 1: `compute_core_metrics()`** - -```python -def compute_core_metrics( - y_true: np.ndarray, - y_pred: np.ndarray, - y_prob: Optional[np.ndarray] = None, - threshold: float = 0.5 -) -> Dict[str, float]: -``` - -Takes three inputs: -- `y_true` β€” the real labels from the dataset (0 = legit, 1 = fraud) -- `y_pred` β€” what the model predicted (0 or 1) -- `y_prob` β€” the model's confidence score (0.0 to 1.0), optional - -Returns a dictionary with every relevant metric: - -```python -{ - 'precision': float, - 'recall': float, - 'f1': float, - 'fpr': float, - 'tn': int, - 'fp': int, - 'fn': int, - 'tp': int, - 'roc_auc': float, # None if y_prob not provided - 'avg_precision': float, # None if y_prob not provided -} -``` - -A few design decisions worth explaining: - -**Why a dictionary?** If you return raw numbers (`return precision, recall, f1...`) the caller has to remember the exact order to unpack them. A dictionary means any code that calls this function can do `metrics['recall']` and get exactly what it asked for β€” unambiguous regardless of how long ago you wrote the calling code. - -**Why `float()` and `int()` casting?** NumPy has its own number types (`np.float64`, `np.int64`) that look like Python numbers but cause silent bugs when you try to log them to JSON or experiment trackers. Casting to plain Python types prevents that. - -**Why zero-division guards on every metric?** - -```python -precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 -``` - -If a model never predicts fraud (which can happen at high thresholds), `tp + fp = 0`. Without the guard, Python crashes with a ZeroDivisionError. With it, you get 0.0 and execution continues. This matters in production β€” you don't want your evaluation pipeline crashing because one model was too conservative. - ---- - -**Function 2: `evaluate_threshold_range()`** - -```python -def evaluate_threshold_range( - y_true: np.ndarray, - y_prob: np.ndarray, - thresholds: Optional[np.ndarray] = None -) -> pd.DataFrame: -``` - -Sweeps thresholds from 0.0 to 1.0 in 0.05 steps and returns a full DataFrame. - -The key line inside the function: - -```python -y_pred = (y_prob >= thresh).astype(int) -``` - -This is doing the threshold conversion automatically. Take the probability scores, compare each one to the threshold, convert True/False to 1/0. One line. - -The output on a real model looks like this: - -``` -threshold precision recall f1 fpr -0.00 0.400 1.000 0.571 1.000 -0.05 0.400 1.000 0.571 1.000 -0.10 0.444 1.000 0.615 0.833 -0.15 0.571 1.000 0.727 0.500 -0.30 0.800 1.000 0.889 0.167 -0.45 1.000 1.000 1.000 0.000 -0.60 1.000 0.750 0.857 0.000 -0.70 1.000 0.500 0.667 0.000 -0.85 1.000 0.250 0.400 0.000 -``` - -Read this table top to bottom: - -As threshold increases, recall decreases and precision increases. You're trading off catching more fraud (lower threshold) against raising fewer false alarms (higher threshold). - -This table is what you show a fraud team when they ask "how do we tune the system?" You don't pick a threshold. You show them the tradeoffs and let the business priority drive the decision. - ---- - -**Function 3: `find_optimal_threshold()`** - -```python -def find_optimal_threshold( - y_true: np.ndarray, - y_prob: np.ndarray, - metric: str = 'f1' -) -> Tuple[float, Dict[str, float]]: -``` - -Takes the full threshold table and returns the threshold where the target metric is highest. - -Default is F1 β€” balanced. But if a client says "we must catch at least 90% of fraud, no matter what" β€” you call it with `metric='recall'` and it finds the lowest threshold that maximises recall instead. - -```python -best_threshold = float(df[metric].idxmax()) -best_metrics = df.loc[best_threshold].to_dict() -``` - -Two lines. Find the index (threshold value) where the metric column is maximum. Pull that entire row as a dictionary. Return both. - ---- - -### The Data Pipeline - -Before the model sees data, it goes through `data_loader.py`: - -```python -X_train, X_test, y_train, y_test = load_and_prepare_data( - data_path='data/raw/creditcard.csv', - test_size=0.2, - random_state=42 -) -``` - -Three things happen inside: - -**1. Drop `Time`** -The `Time` column is seconds elapsed since the first transaction in the dataset. It's a recording artifact β€” not a property of the transaction itself. A model that learns "fraud happens more at certain times in this recording" won't generalise to real data. Drop it. - -**2. Scale `Amount`** -The dataset has 28 PCA-transformed features (V1–V28) that are already scaled to roughly -5 to 5. But `Amount` ranges from 0 to 25,000+. If you feed unscaled data to a model, the large `Amount` values distort the training. - -```python -scaler = StandardScaler() -df['Amount'] = scaler.fit_transform(df[['Amount']]) -``` - -StandardScaler transforms `Amount` to mean=0, standard deviation=1. Now everything is on the same scale. - -**3. Stratified split** -```python -X_train, X_test, y_train, y_test = train_test_split( - X, y, - test_size=0.2, - random_state=42, - stratify=y # ← this line -) -``` - -Without `stratify=y`, the random split might put most of the 492 fraud cases in training and almost none in test β€” or vice versa. `stratify=y` guarantees the 0.17% fraud ratio is preserved in both splits. - -The result: -``` -X_train shape: (227845, 29) -X_test shape: (56962, 29) -Train fraud %: 0.173 -Test fraud %: 0.172 -``` - -0.17% in both. Confirmed. - ---- - -### One Critical Order - -Scale after split, not before. - -If you StandardScaler the full dataset before splitting, the scaler learns the mean and standard deviation from test set data. Information from your test set leaks into the training process. Your model has technically "seen" the test set during preprocessing. - -This is called data leakage. It makes your evaluation results look better than they'll be in production. Always split first, then scale using only the training set's statistics. - ---- - -### Coming Up - -In Part 3, I'll show the three models β€” Logistic Regression, SMOTE + Random Forest, and XGBoost β€” and the final comparison table. You'll see exactly why each model improved and which one wins. - -Part 3 also has the final report output β€” one command, all three models compared. - -GitHub: [link] -Part 1: [link] -Part 3: [link] - ---- - -*Part of my build-in-public series: IntelligenceOS β€” a 12-project AI platform built in 12 months. Real code, real results, shipped publicly.* diff --git a/ios-risk-eval-harness/content/substack-post-3-three-models-one-winner.md b/ios-risk-eval-harness/content/substack-post-3-three-models-one-winner.md deleted file mode 100644 index ee015d8..0000000 --- a/ios-risk-eval-harness/content/substack-post-3-three-models-one-winner.md +++ /dev/null @@ -1,217 +0,0 @@ -# Substack Post 3 of 3 -## "Three Models, One Dataset, One Winner" - -*Part 3 of a 3-part series on building a fraud detection system from scratch* - ---- - -In Part 1: why accuracy is useless on imbalanced data. -In Part 2: the evaluation infrastructure β€” how I measure models consistently. - -This is the payoff post. Three models, increasing sophistication, one final comparison table. - ---- - -### The Setup - -Same dataset throughout: 284,807 credit card transactions, 492 fraud (0.17%). -Same data pipeline: stratified 80/20 split, Amount scaled, Time dropped. -Same evaluation: every model runs through the same `compute_core_metrics()` and `find_optimal_threshold()` functions. - -The only thing that changes between models is the training approach. Everything else is identical. That's what makes the comparison fair. - ---- - -### Model 1: Logistic Regression - -The simplest possible baseline. - -Logistic Regression draws a straight line (technically a hyperplane) through your feature space and puts fraud on one side, legit on the other. It can't learn complex patterns β€” it can only learn linear relationships. - -One modification: `class_weight='balanced'`. This tells the model to weight fraud examples higher during training, proportional to how rare they are. Without this, the model sees so little fraud that it barely learns to recognise it. - -**Training: 2 seconds.** - -Results at default threshold (0.5): - -``` -precision: 0.059 ← only 6% of fraud flags were real fraud -recall: 0.918 ← caught 92% of actual fraud -fp: 1,441 ← 1,441 innocent customers wrongly flagged -fn: 8 ← only missed 8 actual fraud cases -avg_precision: 0.716 -``` - -The model is extremely trigger-happy. It catches almost all fraud β€” but to do it, it raises 1,441 false alarms. For every real fraud it flags, it also accuses 16 innocent customers. - -At the optimal threshold (1.0 β€” only flag when 100% confident): -``` -precision: 0.807 recall: 0.469 f1: 0.594 -fp: 11 fn: 52 -``` - -Better precision, but now it misses 52 out of 98 fraud cases. Not good enough. - -**Baseline avg_precision: 0.716.** This is the floor. Every subsequent model has to beat this. - ---- - -### Model 2: SMOTE + Random Forest - -Two changes: a smarter training dataset and a more powerful model. - -**The SMOTE problem:** - -The Logistic Regression model sees 394 fraud cases during training (80% of 492). That's 394 examples vs 227,451 legit examples. Even with `class_weight='balanced'`, the model has very few real fraud examples to learn from. - -**The SMOTE solution:** - -SMOTE (Synthetic Minority Oversampling Technique) generates synthetic fraud examples. - -Here's how it works: -1. Take a real fraud case -2. Find its nearest neighbours among other fraud cases -3. Draw a new synthetic example somewhere between them - -The result: instead of 394 fraud cases, the training set has 227,451 fraud cases β€” one for every legit transaction. Perfectly balanced. - -**Critical rule:** SMOTE goes on the training set only. Never the test set. The test set has to reflect reality β€” 0.17% fraud. If you balance the test set, you're evaluating on a fantasy dataset. - -**Random Forest:** - -Instead of one straight line (Logistic Regression), Random Forest builds 100 decision trees independently. Each tree learns slightly different patterns. When making a prediction, all 100 trees vote β€” the majority wins. - -100 trees, max_depth 10, `class_weight='balanced'` as a second layer of imbalance handling. - -**Training: ~90 seconds** (SMOTE generates 450k+ rows, then trains 100 trees on them). - -Before SMOTE: -``` -Train fraud count: 394 -Train size: (227,845, 29) -``` - -After SMOTE: -``` -Train fraud count: 227,451 -Train size: (454,902, 29) -``` - -Results at optimal threshold (0.85): -``` -precision: 0.804 recall: 0.796 f1: 0.800 -fp: 19 fn: 20 -avg_precision: 0.829 -``` - -**The improvement:** -- False alarms: 1,441 β†’ 19 at the same precision level. A 99% reduction. -- Missed fraud: 52 β†’ 20. 32 additional fraud cases caught. -- avg_precision: 0.716 β†’ 0.829. +16%. - -This is what SMOTE actually does β€” not magic, just giving the model enough examples to learn what fraud looks like. - ---- - -### Model 3: XGBoost - -Different approach. Same goal. - -**The key difference from Random Forest:** - -Random Forest builds 100 trees independently and averages them. Each tree knows nothing about the others. - -XGBoost builds trees sequentially. After each tree, it looks at where the model made mistakes and builds the next tree specifically to fix those errors. This is called **boosting** β€” you're iteratively correcting your own weaknesses. - -In practice, this sequential correction almost always outperforms independent averaging on structured tabular data. - -**No SMOTE needed:** - -Instead of generating synthetic data, XGBoost has a built-in parameter: `scale_pos_weight`. - -```python -neg = (y_train == 0).sum() # 227,451 legit -pos = (y_train == 1).sum() # 394 fraud -scale_pos_weight = neg / pos # 577.3 -``` - -This tells the model: "when you misclassify a fraud case, treat it as 577x more costly than misclassifying a legit case." Mathematically equivalent to SMOTE's balancing effect, but faster β€” no data generation. - -200 trees, max_depth 6, learning_rate 0.1. - -**Training: ~25 seconds** β€” faster than Random Forest despite 2x the trees, because no SMOTE preprocessing and shallower trees. - -Results at optimal threshold (0.35): -``` -precision: 0.884 recall: 0.857 f1: 0.870 -fp: 11 fn: 14 -avg_precision: 0.869 -``` - ---- - -### The Final Report - -``` -================================================================= -IOS RISK EVAL HARNESS β€” FINAL MODEL COMPARISON REPORT -================================================================= - avg_precision precision recall f1 fp fn best_threshold -model -Logistic Regression 0.7156 0.8070 0.4694 0.5935 11 52 1.00 -SMOTE + Random Forest 0.8293 0.8041 0.7959 0.8000 19 20 0.85 -XGBoost 0.8693 0.8842 0.8571 0.8705 11 14 0.35 -================================================================= - -Best model by avg_precision: XGBoost -Best model by f1: XGBoost -Fewest missed fraud (fn): XGBoost -``` - -**XGBoost wins everything.** - -Across 56,962 test transactions with 98 actual fraud cases: -- Catches 84 fraud cases (85.7%) -- Raises only 11 false alarms -- avg_precision 0.869 vs baseline 0.716 β€” a 21% improvement - -The 38 additional fraud cases caught vs the baseline aren't a statistic. Each one represents a real fraudulent transaction that would have gone undetected. - ---- - -### What the Progression Teaches - -Each step improved performance because it attacked a specific problem: - -**Logistic Regression β†’ Random Forest:** The problem was that one linear boundary can't capture complex fraud patterns. Random Forest's 100 trees can. But the model was still learning from imbalanced data. - -**Random Forest β†’ XGBoost:** Two improvements at once. Sequential boosting means each tree specifically fixes previous errors. And `scale_pos_weight` handles imbalance more efficiently than SMOTE. - -The jump from baseline (0.716) to XGBoost (0.869) happened in three steps, each one solving a specific identified problem. This is how systematic ML engineering works β€” not trying random things and hoping for improvement. - ---- - -### One Thing I'd Do Differently - -The optimal threshold was selected using the test set. In a rigorous setup, you'd use a three-way split: train / validation / test. Select the threshold on validation, report final results on test. Using the test set for threshold selection is a minor form of leakage β€” the threshold is overfit to your specific test set. - -For a single-dataset project this tradeoff is acceptable. For production, use a validation set. - ---- - -### What's Next - -Project 01 is the evaluation layer β€” the measurement infrastructure everything else gets scored against. - -Project 02 is the data layer β€” building a pipeline that ingests, cleans, labels, and versions domain-specific financial data. The same `data_loader.py` pattern from this project becomes the template. - -The sequence is deliberate. By Month 4, these pieces combine into a deployable system. - -Full code on GitHub: [link] -Part 1: [link] -Part 2: [link] - ---- - -*IntelligenceOS β€” 12 projects, 12 months, full platform. Built and documented publicly.* -*Subscribe to follow the build.* diff --git a/ios-risk-eval-harness/content/video-script.md b/ios-risk-eval-harness/content/video-script.md deleted file mode 100644 index c94380d..0000000 --- a/ios-risk-eval-harness/content/video-script.md +++ /dev/null @@ -1,224 +0,0 @@ -# Video Script -## "I Built a Fraud Detection System From Scratch β€” Here's Everything" - -**Format:** Screen recording + voiceover -**Target length:** 10-12 minutes -**Tone:** Direct, technical but accessible. No filler. No hype. - ---- - -### SECTION 1 β€” HOOK (0:00 – 1:00) -*[Screen: Show the final report output in terminal]* - -"This table is the result of 7 days of work. - -Three models. One dataset. 284,807 real credit card transactions. - -What you're looking at is a comparison of three fraud detection models β€” and by the end of this video, you'll understand exactly what every number in that table means, how I built the system that generated it, and why I built it in this specific order. - -This isn't a tutorial. There's no dataset.head() and then a sklearn import. This is the way you'd actually approach a real fraud detection problem β€” measurement first, models second. - -Let's go." - ---- - -### SECTION 2 β€” THE PROBLEM (1:00 – 2:30) -*[Screen: Open creditcard.csv, show shape, show Class value_counts()]* - -"Here's the dataset. 284,807 transactions. 31 columns. - -One column matters most: Class. Zero means legitimate. One means fraud. - -Let me show you the distribution." - -*[Run: df.Class.value_counts()]* - -"492 fraud. 284,315 legitimate. That's 0.17%. - -Now here's the question I want you to sit with: if I built a model that predicted 'not fraud' on every single transaction β€” no matter what β€” what would its accuracy be? - -[pause] - -99.83%. It would be right 99.83% of the time. And it would catch zero fraud. - -This is why accuracy is completely useless as a metric in fraud detection. The class imbalance breaks it. A model can look great on paper while doing nothing useful. - -So the first thing I had to build wasn't a model. It was the right way to measure one." - ---- - -### SECTION 3 β€” THE METRICS (2:30 – 4:30) -*[Screen: Open metrics.py, walk through compute_core_metrics()]* - -"Before I touched a model, I wrote this file β€” metrics.py. - -This is the evaluation infrastructure. Every model I train β€” today, next month, next year β€” runs through the same functions. That's the point. Consistent measurement makes comparison meaningful. - -The foundation is the confusion matrix. When a model makes a prediction, every transaction falls into one of four buckets." - -*[Draw or show confusion matrix diagram]* - -"True Positive β€” fraud correctly caught. This is the goal. -True Negative β€” legit correctly cleared. Good. -False Positive β€” legit flagged as fraud. Costs customer trust. -False Negative β€” fraud missed. Costs real money. - -From those four numbers, every metric I care about is derived." - -*[Scroll through the function, point to each metric]* - -"Precision β€” of all the fraud flags my model raises, how many are real? -Recall β€” of all real fraud, how much am I catching? -Average precision β€” the honest overall quality score on imbalanced data. - -Notice these zero-division guards on every line. If the model never predicts fraud, the denominator is zero. Without the guard β€” crash. With it β€” 0.0 and the code continues. This is the difference between research code and production code. - -Now let me show you something more interesting β€” the threshold." - ---- - -### SECTION 4 β€” THE THRESHOLD (4:30 – 6:00) -*[Screen: Run evaluate_threshold_range() demo, show output table]* - -"Your model doesn't output a 0 or a 1. It outputs a probability. '73% confident this is fraud.' - -You need a rule to convert that into a decision. That rule is the threshold. Default is 0.5. But 0.5 is completely arbitrary. - -This function sweeps every threshold from 0.0 to 1.0 and shows you what happens to your metrics at each point. - -Watch what happens." - -*[Run the function, let the table print]* - -"At threshold 0.0 β€” the model flags everything as fraud. Perfect recall, terrible precision. Every innocent customer gets flagged. - -As you increase the threshold β€” precision goes up, recall goes down. You're catching less fraud but raising fewer false alarms. - -The threshold is a business decision. A bank that's most worried about fraud losses sets it low. A bank that prioritises customer experience sets it higher. My job as the engineer is to show this table, not to pick 0.5 by default." - ---- - -### SECTION 5 β€” MODEL 1: LOGISTIC REGRESSION (6:00 – 7:30) -*[Screen: Open run_eval.py, run it]* - -"Model 1. The baseline. Logistic Regression. - -I always start with the simplest possible model. Not because I think it'll win. Because I need a floor β€” a number that every subsequent model has to beat." - -*[Wait for output to print]* - -"At default threshold: precision 6%, recall 92%, 1,441 false alarms. - -It catches almost all the fraud β€” but it flags 1,441 innocent customers to do it. For every real fraud flag, there are 16 false alarms. - -At the optimal threshold β€” 1.0, only flagging when 100% confident β€” precision jumps to 81% but recall drops to 47%. It's now missing more than half the fraud. - -Baseline average precision: 0.716. That's the number to beat." - ---- - -### SECTION 6 β€” MODEL 2: SMOTE + RANDOM FOREST (7:30 – 9:00) -*[Screen: Open train_smote_rf.py, run it]* - -"Model 2. Two changes: better training data and a more powerful model. - -The core problem with the baseline is that the model only sees 394 fraud cases during training. That's not enough to learn what fraud looks like. - -SMOTE fixes that. It generates synthetic fraud examples by interpolating between real ones. Watch the before and after." - -*[Let it run, point to the SMOTE output lines]* - -"Before SMOTE: 394 fraud cases. After: 227,451. One for every legitimate transaction. - -Critical rule: SMOTE goes on the training data only. The test set stays untouched β€” it has to reflect reality. - -Random Forest: 100 independent decision trees, each learning slightly different patterns. The majority vote wins." - -*[Point to results]* - -"False alarms dropped from 1,441 to 19. That's a 99% reduction. -Missed fraud: 52 down to 20. -Average precision: 0.829. Up from 0.716." - ---- - -### SECTION 7 β€” MODEL 3: XGBOOST (9:00 – 10:30) -*[Screen: Open train_xgboost.py, run it]* - -"Model 3. XGBoost. - -Random Forest builds 100 trees independently and averages them. XGBoost builds trees sequentially β€” each tree specifically corrects the mistakes of the previous one. That's the key difference. - -Also, no SMOTE. Instead: scale_pos_weight equals the legit-to-fraud ratio. 577. This tells the model that misclassifying a fraud case is 577 times more costly than misclassifying a legitimate one. Same effect as SMOTE β€” faster, no synthetic data." - -*[Let it run, point to results]* - -"Average precision: 0.869. The best yet. -14 missed fraud cases. 11 false alarms. -F1: 0.870. - -And the optimal threshold is 0.35 β€” much lower than the other models. XGBoost's probability scores are better calibrated, so a lower threshold catches more fraud without flooding you with false alarms." - ---- - -### SECTION 8 β€” FINAL REPORT (10:30 – 11:30) -*[Screen: Run generate_report.py]* - -"Now the final report. One command. All three models trained and compared." - -*[Let it run β€” takes 2-3 minutes, fast-forward or cut to result]* - -"There it is. - -XGBoost wins every metric that matters. Best average precision. Best F1. Fewest missed fraud cases. - -The progression tells the story: -Logistic Regression: 0.716 β€” the floor -Random Forest: 0.829 β€” solved the data imbalance problem -XGBoost: 0.869 β€” solved it more elegantly and learned more complex patterns - -38 additional fraud cases caught vs the baseline. Each one a real transaction that would have been missed." - ---- - -### SECTION 9 β€” CLOSING (11:30 – 12:00) -*[Screen: Show GitHub repo]* - -"Everything in this video is on GitHub. The metrics module, the data pipeline, all three training scripts, the report generator. Clean code, no notebooks. - -This is Project 01 of a 12-project AI platform I'm building over 12 months. The eval harness I built here will score every model I train for the rest of the build. - -Next project: Domain Data Foundry β€” building the data pipeline that feeds real financial data into everything I just built. - -I'm documenting everything publicly. Links in the description. - -If you have questions about anything I covered β€” leave a comment." - ---- - -### Production Notes - -**Recording setup:** -- Use a dark terminal theme β€” easier to read on video -- Increase font size to at least 16pt before recording -- Close all notifications -- Record at 1080p minimum - -**Editing cuts to make:** -- Cut the SMOTE training wait time (it takes ~90 seconds) β€” jump cut to results -- Cut the generate_report.py model training (2-3 mins total) β€” jump cut to final table -- Add simple text overlays on the confusion matrix explanation - -**Thumbnail:** -- Show the final comparison table -- Text overlay: "3 models. 1 dataset. 21% improvement." -- Keep it clean β€” no stock photos of credit cards - -**Posting:** -- YouTube as primary -- Clip Section 2 (the 99.83% accuracy problem) as a standalone 60-second short -- LinkedIn video post using the Section 8 final report reveal - ---- - -*Total recording time estimate: 45-60 mins with pauses and retakes. Edit down to 10-12 mins.* diff --git a/ios-risk-eval-harness/ios_eval/__init__.py b/ios-risk-eval-harness/ios_eval/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/ios_eval/data_loader.py b/ios-risk-eval-harness/ios_eval/data_loader.py deleted file mode 100644 index 7e3104a..0000000 --- a/ios-risk-eval-harness/ios_eval/data_loader.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -IOS Risk β€” Evaluation Harness -Data loader module. Handles loading, cleaning, scaling, and splitting the credit card fraud dataset. -""" -import numpy as np -import pandas as pd -from pathlib import Path -from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import train_test_split -from typing import Tuple, Optional -import warnings -warnings.filterwarnings('ignore') - - -def load_and_prepare_data( - data_path: str = 'data/raw/creditcard.csv', - test_size: float = 0.2, - random_state: int = 42, - scale_amount: bool = True -) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - df = pd.read_csv(data_path) - df = df.drop(columns=['Time']) - - if scale_amount: - scaler = StandardScaler() - df['Amount'] = scaler.fit_transform(df[['Amount']]) - - X = df.drop(columns=['Class']).values - y = df['Class'].values - - X_train, X_test, y_train, y_test = train_test_split( - X, y, - test_size=test_size, - random_state=random_state, - stratify=y - ) - - return X_train, X_test, y_train, y_test diff --git a/ios-risk-eval-harness/ios_eval/metrics.py b/ios-risk-eval-harness/ios_eval/metrics.py deleted file mode 100644 index 6cecab7..0000000 --- a/ios-risk-eval-harness/ios_eval/metrics.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -IOS Risk β€” Evaluation Harness -Core metrics module. Every model trained for IOS Risk is evaluated through these functions. -""" -import numpy as np -import pandas as pd -from sklearn.metrics import ( - precision_score, recall_score, f1_score, - roc_auc_score, average_precision_score, - confusion_matrix, classification_report, - precision_recall_curve, roc_curve -) -from typing import Dict, Tuple, Optional -import warnings -warnings.filterwarnings('ignore') - - -def compute_core_metrics( - y_true: np.ndarray, - y_pred: np.ndarray, - y_prob: Optional[np.ndarray] = None, - threshold: float = 0.5 -) -> Dict[str, float]: - if len(y_true) == 0: - raise ValueError('y_true cannot be empty') - tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel() - precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 - recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 - f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0 - fpr = fp / (fp + tn) if (fp + tn) > 0 else 0.0 - roc_auc = roc_auc_score(y_true, y_prob) if y_prob is not None else None - avg_precision = average_precision_score(y_true, y_prob) if y_prob is not None else None - return { - 'precision': float(precision), - 'recall': float(recall), - 'f1': float(f1), - 'fpr': float(fpr), - 'tn': int(tn), - 'fp': int(fp), - 'fn': int(fn), - 'tp': int(tp), - 'roc_auc': float(roc_auc) if roc_auc is not None else None, - 'avg_precision': float(avg_precision) if avg_precision is not None else None, - } - -def evaluate_threshold_range( - y_true: np.ndarray, - y_prob: np.ndarray, - thresholds: Optional[np.ndarray] = None -) -> pd.DataFrame: - - if thresholds is None: - thresholds = np.arange(0.0, 1.01, 0.05) - - rows = [] - for thresh in thresholds: - y_pred = (y_prob >= thresh).astype(int) - metrics = compute_core_metrics(y_true, y_pred, y_prob) - metrics['threshold'] = round(float(thresh), 2) - rows.append(metrics) - - return pd.DataFrame(rows).set_index('threshold') - - - -def find_optimal_threshold( - y_true: np.ndarray, - y_prob: np.ndarray, - metric: str = 'f1' -) -> Tuple[float, Dict[str, float]]: - - df = evaluate_threshold_range(y_true, y_prob) - - best_threshold = float(df[metric].idxmax()) - best_metrics = df.loc[best_threshold].to_dict() - best_metrics['threshold'] = best_threshold - - return best_threshold, best_metrics diff --git a/ios-risk-eval-harness/ios_eval/report.py b/ios-risk-eval-harness/ios_eval/report.py deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/ios_eval/visualize.py b/ios-risk-eval-harness/ios_eval/visualize.py deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/ios_eval/wandb_logger.py b/ios-risk-eval-harness/ios_eval/wandb_logger.py deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/notebooks/01_data_exploration.ipynb b/ios-risk-eval-harness/notebooks/01_data_exploration.ipynb deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/notebooks/02_baseline_eval.ipynb b/ios-risk-eval-harness/notebooks/02_baseline_eval.ipynb deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/notebooks/03_eval_harness_demo.ipynb b/ios-risk-eval-harness/notebooks/03_eval_harness_demo.ipynb deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/requirements.txt b/ios-risk-eval-harness/requirements.txt deleted file mode 100644 index 1b2d705..0000000 --- a/ios-risk-eval-harness/requirements.txt +++ /dev/null @@ -1,136 +0,0 @@ -annotated-types==0.7.0 -anyio==4.12.1 -appdirs==1.4.4 -appnope==0.1.4 -argon2-cffi==25.1.0 -argon2-cffi-bindings==25.1.0 -arrow==1.4.0 -asttokens==3.0.1 -async-lru==2.3.0 -attrs==26.1.0 -babel==2.18.0 -beautifulsoup4==4.14.3 -bleach==6.3.0 -certifi==2026.2.25 -cffi==2.0.0 -charset-normalizer==3.4.6 -click==8.3.1 -comm==0.2.3 -contourpy==1.3.3 -cycler==0.12.1 -debugpy==1.8.20 -decorator==5.2.1 -defusedxml==0.7.1 -docker-pycreds==0.4.0 -executing==2.2.1 -fastjsonschema==2.21.2 -fonttools==4.62.1 -fqdn==1.5.1 -gitdb==4.0.12 -GitPython==3.1.46 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.11 -imbalanced-learn==0.14.1 -iniconfig==2.3.0 -ipykernel==7.2.0 -ipython==9.11.0 -ipython_pygments_lexers==1.1.1 -ipywidgets==8.1.8 -isoduration==20.11.0 -jedi==0.19.2 -Jinja2==3.1.6 -joblib==1.5.3 -json5==0.13.0 -jsonpointer==3.1.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 -jupyter==1.1.1 -jupyter-console==6.6.3 -jupyter-events==0.12.0 -jupyter-lsp==2.3.0 -jupyter_client==8.8.0 -jupyter_core==5.9.1 -jupyter_server==2.17.0 -jupyter_server_terminals==0.5.4 -jupyterlab==4.5.6 -jupyterlab_pygments==0.3.0 -jupyterlab_server==2.28.0 -jupyterlab_widgets==3.0.16 -kaggle==2.0.0 -kagglesdk==0.1.16 -kiwisolver==1.5.0 -lark==1.3.1 -MarkupSafe==3.0.3 -matplotlib==3.10.8 -matplotlib-inline==0.2.1 -mistune==3.2.0 -nbclient==0.10.4 -nbconvert==7.17.0 -nbformat==5.10.4 -nest-asyncio==1.6.0 -notebook==7.5.5 -notebook_shim==0.2.4 -numpy==2.4.3 -packaging==26.0 -pandas==3.0.1 -pandocfilters==1.5.1 -parso==0.8.6 -pexpect==4.9.0 -pillow==12.1.1 -platformdirs==4.9.4 -pluggy==1.6.0 -prometheus_client==0.24.1 -prompt_toolkit==3.0.52 -protobuf==4.25.8 -psutil==7.2.2 -ptyprocess==0.7.0 -pure_eval==0.2.3 -pycparser==3.0 -pydantic==2.12.5 -pydantic_core==2.41.5 -Pygments==2.19.2 -pyparsing==3.3.2 -pytest==9.0.2 -python-dateutil==2.9.0.post0 -python-json-logger==4.0.0 -python-slugify==8.0.4 -PyYAML==6.0.3 -pyzmq==27.1.0 -referencing==0.37.0 -requests==2.32.5 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3987-syntax==1.1.0 -rpds-py==0.30.0 -scikit-learn==1.8.0 -scipy==1.17.1 -seaborn==0.13.2 -Send2Trash==2.1.0 -sentry-sdk==2.55.0 -setproctitle==1.3.7 -setuptools==82.0.1 -six==1.17.0 -sklearn-compat==0.1.5 -smmap==5.0.3 -soupsieve==2.8.3 -stack-data==0.6.3 -terminado==0.18.1 -text-unidecode==1.3 -threadpoolctl==3.6.0 -tinycss2==1.4.0 -tornado==6.5.5 -tqdm==4.67.3 -traitlets==5.14.3 -typing-inspection==0.4.2 -typing_extensions==4.15.0 -tzdata==2025.3 -uri-template==1.3.0 -urllib3==2.6.3 -wcwidth==0.6.0 -webcolors==25.10.0 -webencodings==0.5.1 -websocket-client==1.9.0 -widgetsnbextension==4.0.15 -xgboost==3.2.0 diff --git a/ios-risk-eval-harness/results/baseline-logistic-regression_20260324_171615.json b/ios-risk-eval-harness/results/baseline-logistic-regression_20260324_171615.json deleted file mode 100644 index 88f7565..0000000 --- a/ios-risk-eval-harness/results/baseline-logistic-regression_20260324_171615.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "timestamp": "2026-03-24T17:16:15.641261", - "config": { - "name": "baseline-logistic-regression", - "model": "LogisticRegression", - "test_size": 0.2, - "random_state": 42, - "class_weight": "balanced", - "max_iter": 1000 - }, - "metrics_at_default_threshold": { - "precision": 0.05878510777269758, - "recall": 0.9183673469387755, - "f1": 0.11049723756906077, - "fpr": 0.02534116488463703, - "tn": 55423, - "fp": 1441, - "fn": 8, - "tp": 90, - "roc_auc": 0.9714465520310545, - "avg_precision": 0.7155864892965794 - }, - "best_threshold": 1.0, - "metrics_at_best_threshold": { - "precision": 0.8070175438596491, - "recall": 0.46938775510204084, - "f1": 0.5935483870967743, - "fpr": 0.00019344400675295442, - "tn": 56853.0, - "fp": 11.0, - "fn": 52.0, - "tp": 46.0, - "roc_auc": 0.9714465520310545, - "avg_precision": 0.7155864892965794, - "threshold": 1.0 - } -} \ No newline at end of file diff --git a/ios-risk-eval-harness/scripts/generate_report.py b/ios-risk-eval-harness/scripts/generate_report.py deleted file mode 100644 index f1bf46a..0000000 --- a/ios-risk-eval-harness/scripts/generate_report.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -IOS Risk β€” Evaluation Harness -Final evaluation report. Runs all models and produces a comparison table. -""" -import numpy as np -import pandas as pd -from sklearn.linear_model import LogisticRegression -from sklearn.ensemble import RandomForestClassifier -from xgboost import XGBClassifier -from imblearn.over_sampling import SMOTE -from ios_eval.data_loader import load_and_prepare_data -from ios_eval.metrics import compute_core_metrics, find_optimal_threshold - - -def generate_report(): - print('Loading data...') - X_train, X_test, y_train, y_test = load_and_prepare_data( - test_size=0.2, - random_state=42 - ) - - neg = (y_train == 0).sum() - pos = (y_train == 1).sum() - scale_pos_weight = neg / pos - - print('Data loaded. Training all models...\n') - results = [] - - # --- Model 1: Logistic Regression --- - print('Training Logistic Regression...') - lr = LogisticRegression( - class_weight='balanced', - max_iter=1000, - random_state=42 - ) - lr.fit(X_train, y_train) - y_pred = lr.predict(X_test) - y_prob = lr.predict_proba(X_test)[:, 1] - best_thresh, best_metrics = find_optimal_threshold(y_test, y_prob) - results.append({ - 'model': 'Logistic Regression', - 'avg_precision': round(best_metrics['avg_precision'], 4), - 'precision': round(best_metrics['precision'], 4), - 'recall': round(best_metrics['recall'], 4), - 'f1': round(best_metrics['f1'], 4), - 'fp': int(best_metrics['fp']), - 'fn': int(best_metrics['fn']), - 'best_threshold': best_thresh - }) - print(' Done.') - - # --- Model 2: SMOTE + Random Forest --- - print('Training SMOTE + Random Forest...') - smote = SMOTE(random_state=42) - X_train_balanced, y_train_balanced = smote.fit_resample(X_train, y_train) - rf = RandomForestClassifier( - n_estimators=100, - max_depth=10, - class_weight='balanced', - random_state=42, - n_jobs=-1 - ) - rf.fit(X_train_balanced, y_train_balanced) - y_pred = rf.predict(X_test) - y_prob = rf.predict_proba(X_test)[:, 1] - best_thresh, best_metrics = find_optimal_threshold(y_test, y_prob) - results.append({ - 'model': 'SMOTE + Random Forest', - 'avg_precision': round(best_metrics['avg_precision'], 4), - 'precision': round(best_metrics['precision'], 4), - 'recall': round(best_metrics['recall'], 4), - 'f1': round(best_metrics['f1'], 4), - 'fp': int(best_metrics['fp']), - 'fn': int(best_metrics['fn']), - 'best_threshold': best_thresh - }) - print(' Done.') - - # --- Model 3: XGBoost --- - print('Training XGBoost...') - xgb = XGBClassifier( - n_estimators=200, - max_depth=6, - learning_rate=0.1, - scale_pos_weight=scale_pos_weight, - random_state=42, - eval_metric='aucpr', - verbosity=0 - ) - xgb.fit(X_train, y_train) - y_pred = xgb.predict(X_test) - y_prob = xgb.predict_proba(X_test)[:, 1] - best_thresh, best_metrics = find_optimal_threshold(y_test, y_prob) - results.append({ - 'model': 'XGBoost', - 'avg_precision': round(best_metrics['avg_precision'], 4), - 'precision': round(best_metrics['precision'], 4), - 'recall': round(best_metrics['recall'], 4), - 'f1': round(best_metrics['f1'], 4), - 'fp': int(best_metrics['fp']), - 'fn': int(best_metrics['fn']), - 'best_threshold': best_thresh - }) - print(' Done.\n') - - # --- Final Report --- - df = pd.DataFrame(results).set_index('model') - print('=' * 65) - print('IOS RISK EVAL HARNESS β€” FINAL MODEL COMPARISON REPORT') - print('=' * 65) - print(df.to_string()) - print('=' * 65) - print(f'\nBest model by avg_precision: {df["avg_precision"].idxmax()}') - print(f'Best model by f1: {df["f1"].idxmax()}') - print(f'Fewest missed fraud (fn): {df["fn"].idxmin()}') - - -if __name__ == '__main__': - generate_report() diff --git a/ios-risk-eval-harness/scripts/run_eval.py b/ios-risk-eval-harness/scripts/run_eval.py deleted file mode 100644 index 5ace7c7..0000000 --- a/ios-risk-eval-harness/scripts/run_eval.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -IOS Risk β€” Evaluation Harness -Baseline model training and evaluation script. -""" -import json -import numpy as np -from datetime import datetime -from pathlib import Path -from sklearn.linear_model import LogisticRegression -from ios_eval.data_loader import load_and_prepare_data -from ios_eval.metrics import compute_core_metrics, find_optimal_threshold - - -def log_run(config: dict, metrics: dict, best_threshold: float, best_metrics: dict): - """Save run results to a JSON file in results/ folder.""" - Path('results').mkdir(exist_ok=True) - run = { - 'timestamp': datetime.now().isoformat(), - 'config': config, - 'metrics_at_default_threshold': metrics, - 'best_threshold': best_threshold, - 'metrics_at_best_threshold': best_metrics, - } - filename = f"results/{config['name']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - with open(filename, 'w') as f: - json.dump(run, f, indent=2) - print(f'\nRun saved to {filename}') - - -def run_baseline_eval(): - config = { - 'name': 'baseline-logistic-regression', - 'model': 'LogisticRegression', - 'test_size': 0.2, - 'random_state': 42, - 'class_weight': 'balanced', - 'max_iter': 1000 - } - - print('Loading data...') - X_train, X_test, y_train, y_test = load_and_prepare_data( - test_size=config['test_size'], - random_state=config['random_state'] - ) - print(f'Train size: {X_train.shape}, Test size: {X_test.shape}') - - print('Training model...') - model = LogisticRegression( - class_weight=config['class_weight'], - max_iter=config['max_iter'], - random_state=config['random_state'] - ) - model.fit(X_train, y_train) - print('Training complete.') - - print('Evaluating...') - y_pred = model.predict(X_test) - y_prob = model.predict_proba(X_test)[:, 1] - - metrics = compute_core_metrics(y_true=y_test, y_pred=y_pred, y_prob=y_prob) - best_threshold, best_metrics = find_optimal_threshold(y_test, y_prob) - - print('\n--- Results at default threshold (0.5) ---') - for k, v in metrics.items(): - print(f' {k}: {v}') - - print(f'\n--- Best threshold: {best_threshold} ---') - for k, v in best_metrics.items(): - print(f' {k}: {v}') - - log_run(config, metrics, best_threshold, best_metrics) - - -if __name__ == '__main__': - run_baseline_eval() diff --git a/ios-risk-eval-harness/scripts/train_smote_rf.py b/ios-risk-eval-harness/scripts/train_smote_rf.py deleted file mode 100644 index a7a0c58..0000000 --- a/ios-risk-eval-harness/scripts/train_smote_rf.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -IOS Risk β€” Evaluation Harness -SMOTE + Random Forest training script. -""" -import numpy as np -from imblearn.over_sampling import SMOTE -from sklearn.ensemble import RandomForestClassifier -from ios_eval.data_loader import load_and_prepare_data -from ios_eval.metrics import compute_core_metrics, find_optimal_threshold - - -def run_smote_rf_eval(): - print('Loading data...') - X_train, X_test, y_train, y_test = load_and_prepare_data( - test_size=0.2, - random_state=42 - ) - print(f'Train size before SMOTE: {X_train.shape}') - print(f'Train fraud count before SMOTE: {y_train.sum()}') - - print('Applying SMOTE...') - smote = SMOTE(random_state=42) - X_train_balanced, y_train_balanced = smote.fit_resample(X_train, y_train) - print(f'Train size after SMOTE: {X_train_balanced.shape}') - print(f'Train fraud count after SMOTE: {y_train_balanced.sum()}') - - print('Training Random Forest...') - model = RandomForestClassifier( - n_estimators=100, - max_depth=10, - class_weight='balanced', - random_state=42, - n_jobs=-1 - ) - model.fit(X_train_balanced, y_train_balanced) - print('Training complete.') - - print('Evaluating...') - y_pred = model.predict(X_test) - y_prob = model.predict_proba(X_test)[:, 1] - - metrics = compute_core_metrics(y_true=y_test, y_pred=y_pred, y_prob=y_prob) - best_threshold, best_metrics = find_optimal_threshold(y_test, y_prob) - - print('\n--- Results at default threshold (0.5) ---') - for k, v in metrics.items(): - print(f' {k}: {v}') - - print(f'\n--- Best threshold: {best_threshold} ---') - for k, v in best_metrics.items(): - print(f' {k}: {v}') - - -if __name__ == '__main__': - run_smote_rf_eval() diff --git a/ios-risk-eval-harness/scripts/train_xgboost.py b/ios-risk-eval-harness/scripts/train_xgboost.py deleted file mode 100644 index 4aa4af5..0000000 --- a/ios-risk-eval-harness/scripts/train_xgboost.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -IOS Risk β€” Evaluation Harness -XGBoost training script. -""" -import numpy as np -from xgboost import XGBClassifier -from ios_eval.data_loader import load_and_prepare_data -from ios_eval.metrics import compute_core_metrics, find_optimal_threshold - - -def run_xgboost_eval(): - print('Loading data...') - X_train, X_test, y_train, y_test = load_and_prepare_data( - test_size=0.2, - random_state=42 - ) - print(f'Train size: {X_train.shape}') - - neg = (y_train == 0).sum() - pos = (y_train == 1).sum() - scale_pos_weight = neg / pos - print(f'Legit: {neg}, Fraud: {pos}, scale_pos_weight: {scale_pos_weight:.1f}') - - print('Training XGBoost...') - model = XGBClassifier( - n_estimators=200, - max_depth=6, - learning_rate=0.1, - scale_pos_weight=scale_pos_weight, - random_state=42, - eval_metric='aucpr', - verbosity=0 - ) - model.fit(X_train, y_train) - print('Training complete.') - - print('Evaluating...') - y_pred = model.predict(X_test) - y_prob = model.predict_proba(X_test)[:, 1] - - metrics = compute_core_metrics(y_true=y_test, y_pred=y_pred, y_prob=y_prob) - best_threshold, best_metrics = find_optimal_threshold(y_test, y_prob) - - print('\n--- Results at default threshold (0.5) ---') - for k, v in metrics.items(): - print(f' {k}: {v}') - - print(f'\n--- Best threshold: {best_threshold} ---') - for k, v in best_metrics.items(): - print(f' {k}: {v}') - - -if __name__ == '__main__': - run_xgboost_eval() diff --git a/ios-risk-eval-harness/tests/test_edge_cases.py b/ios-risk-eval-harness/tests/test_edge_cases.py deleted file mode 100644 index e69de29..0000000 diff --git a/ios-risk-eval-harness/tests/test_metrics.py b/ios-risk-eval-harness/tests/test_metrics.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..523517e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,55 @@ +[build-system] +requires = ["setuptools==83.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "ios-risk-eval" +version = "0.2.0" +description = "Leakage-safe evaluation harness for imbalanced financial-risk classification" +readme = "README.md" +requires-python = ">=3.11,<3.12" +license = "Apache-2.0" +authors = [{name = "Ugo Chukwu / Etherlabs"}] +dependencies = [ + "imbalanced-learn==0.14.1", + "numpy==2.4.3", + "pandas==3.0.1", + "scikit-learn==1.8.0", + "xgboost==3.2.0", +] + +[project.optional-dependencies] +dev = [ + "pip-audit==2.10.0", + "pytest==9.0.3", + "ruff==0.15.1", + "setuptools==83.0.0", +] +notebooks = [ + "ipykernel==7.2.0", + "jupyterlab==4.5.6", + "matplotlib==3.10.8", + "seaborn==0.13.2", +] + +[project.scripts] +ios-eval = "ios_eval.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +addopts = "-ra --strict-config --strict-markers" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py311" +line-length = 110 +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "SIM", "RUF"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" diff --git a/results/benchmark.json b/results/benchmark.json new file mode 100644 index 0000000..ee8f3a3 --- /dev/null +++ b/results/benchmark.json @@ -0,0 +1,394 @@ +{ + "claim_boundary": "Not a production or client outcome.", + "config": { + "models": { + "Logistic Regression": { + "estimator": { + "class_weight": "balanced", + "max_iter": 1000, + "random_state": 42 + }, + "type": "logistic_regression" + }, + "SMOTE + Random Forest": { + "estimator": { + "class_weight": "balanced", + "max_depth": 10, + "n_estimators": 100, + "n_jobs": 1, + "random_state": 42 + }, + "smote": { + "k_neighbors": 5, + "random_state": 42 + }, + "type": "smote_random_forest" + }, + "XGBoost": { + "estimator": { + "eval_metric": "aucpr", + "learning_rate": 0.1, + "max_depth": 6, + "n_estimators": 200, + "n_jobs": 1, + "random_state": 42, + "scale_pos_weight": "train_class_ratio", + "tree_method": "hist", + "verbosity": 0 + }, + "type": "xgboost" + } + }, + "random_state": 42, + "test_size": 0.2, + "threshold_metric": "f1", + "threshold_step": 0.01, + "validation_size": 0.2 + }, + "config_sha256": "d481e6e5cc1f3b4ff8e95014632be1e78e2471986b04f516b7d5c059c820af55", + "dataset": { + "class_counts": { + "0": 284315, + "1": 492 + }, + "columns": [ + "Time", + "V1", + "V2", + "V3", + "V4", + "V5", + "V6", + "V7", + "V8", + "V9", + "V10", + "V11", + "V12", + "V13", + "V14", + "V15", + "V16", + "V17", + "V18", + "V19", + "V20", + "V21", + "V22", + "V23", + "V24", + "V25", + "V26", + "V27", + "V28", + "Amount", + "Class" + ], + "dataset": { + "filename": "creditcard.csv", + "sha256": "76274b691b16a6c49d3f159c883398e03ccd6d1ee12d9d8ee38f4b4b98551a89", + "source": "https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud" + }, + "features": [ + "V1", + "V2", + "V3", + "V4", + "V5", + "V6", + "V7", + "V8", + "V9", + "V10", + "V11", + "V12", + "V13", + "V14", + "V15", + "V16", + "V17", + "V18", + "V19", + "V20", + "V21", + "V22", + "V23", + "V24", + "V25", + "V26", + "V27", + "V28", + "Amount" + ], + "preprocessing": { + "dropped_columns": [ + "Time" + ], + "scaler": { + "columns": [ + "Amount" + ], + "fit_scope": "train_only", + "mean": [ + 87.63173996243044 + ], + "scale": [ + 243.8004485588741 + ] + } + }, + "rows": 284807, + "split": { + "class_counts": { + "test": { + "0": 56864, + "1": 98 + }, + "train": { + "0": 170588, + "1": 295 + }, + "validation": { + "0": 56863, + "1": 99 + } + }, + "random_state": 42, + "rows": { + "test": 56962, + "train": 170883, + "validation": 56962 + }, + "strategy": "stratified_train_validation_test", + "test_size": 0.2, + "validation_size": 0.2 + } + }, + "environment": { + "dependencies": { + "imbalanced-learn": "0.14.1", + "numpy": "2.4.3", + "pandas": "3.0.1", + "scikit-learn": "1.8.0", + "xgboost": "3.2.0" + }, + "implementation": "CPython", + "platform": "macOS-27.0-arm64-arm-64bit", + "python": "3.11.15" + }, + "evidence_label": "public_dataset_benchmark", + "generated_at_utc": "2026-08-09T03:52:23.002972+00:00", + "git_revision": "6be18ab2474361e0267e1af16a9588de1c73d4d5", + "models": [ + { + "imbalance_handling": { + "smote": null, + "smote_fit_scope": null, + "training_class_counts_after": { + "0": 170588, + "1": 295 + }, + "training_class_counts_before": { + "0": 170588, + "1": 295 + }, + "training_rows_after": 170883, + "training_rows_before": 170883 + }, + "name": "Logistic Regression", + "parameters": { + "class_weight": "balanced", + "max_iter": 1000, + "random_state": 42 + }, + "test_metrics_at_default_threshold": { + "average_precision": 0.7192267752904014, + "f1": 0.11076540136901057, + "fn": 9, + "fp": 1420, + "fpr": 0.024971862689926842, + "precision": 0.058979456593770706, + "recall": 0.9081632653061225, + "roc_auc": 0.9719772669197111, + "tn": 55444, + "tp": 89 + }, + "test_metrics_at_selected_threshold": { + "average_precision": 0.7192267752904014, + "f1": 0.672, + "fn": 14, + "fp": 68, + "fpr": 0.0011958356781091727, + "precision": 0.5526315789473685, + "recall": 0.8571428571428571, + "roc_auc": 0.9719772669197111, + "tn": 56796, + "tp": 84 + }, + "threshold_selection": { + "candidate_count": 101, + "metric": "f1", + "selected_threshold": 0.99, + "split": "validation", + "validation_metrics": { + "average_precision": 0.6750977924324548, + "f1": 0.6584362139917695, + "fn": 19.0, + "fp": 64.0, + "fpr": 0.0011255121959798111, + "precision": 0.5555555555555556, + "recall": 0.8080808080808081, + "roc_auc": 0.9757402027947022, + "threshold": 0.99, + "tn": 56799.0, + "tp": 80.0 + } + }, + "type": "logistic_regression" + }, + { + "imbalance_handling": { + "smote": { + "k_neighbors": 5, + "random_state": 42 + }, + "smote_fit_scope": "train_only", + "training_class_counts_after": { + "0": 170588, + "1": 170588 + }, + "training_class_counts_before": { + "0": 170588, + "1": 295 + }, + "training_rows_after": 341176, + "training_rows_before": 170883 + }, + "name": "SMOTE + Random Forest", + "parameters": { + "class_weight": "balanced", + "max_depth": 10, + "n_estimators": 100, + "n_jobs": 1, + "random_state": 42 + }, + "test_metrics_at_default_threshold": { + "average_precision": 0.792029901383661, + "f1": 0.5754385964912281, + "fn": 16, + "fp": 105, + "fpr": 0.0018465109735509285, + "precision": 0.4385026737967914, + "recall": 0.8367346938775511, + "roc_auc": 0.9767641088511938, + "tn": 56759, + "tp": 82 + }, + "test_metrics_at_selected_threshold": { + "average_precision": 0.792029901383661, + "f1": 0.7864077669902914, + "fn": 17, + "fp": 27, + "fpr": 0.00047481710748452447, + "precision": 0.75, + "recall": 0.826530612244898, + "roc_auc": 0.9767641088511938, + "tn": 56837, + "tp": 81 + }, + "threshold_selection": { + "candidate_count": 101, + "metric": "f1", + "selected_threshold": 0.71, + "split": "validation", + "validation_metrics": { + "average_precision": 0.7533800259328808, + "f1": 0.8, + "fn": 23.0, + "fp": 15.0, + "fpr": 0.00026379192093276823, + "precision": 0.8351648351648352, + "recall": 0.7676767676767676, + "roc_auc": 0.9707407685706404, + "threshold": 0.71, + "tn": 56848.0, + "tp": 76.0 + } + }, + "type": "smote_random_forest" + }, + { + "imbalance_handling": { + "smote": null, + "smote_fit_scope": null, + "training_class_counts_after": { + "0": 170588, + "1": 295 + }, + "training_class_counts_before": { + "0": 170588, + "1": 295 + }, + "training_rows_after": 170883, + "training_rows_before": 170883 + }, + "name": "XGBoost", + "parameters": { + "eval_metric": "aucpr", + "learning_rate": 0.1, + "max_depth": 6, + "n_estimators": 200, + "n_jobs": 1, + "random_state": 42, + "scale_pos_weight": 578.264406779661, + "tree_method": "hist", + "verbosity": 0 + }, + "test_metrics_at_default_threshold": { + "average_precision": 0.8756165035034519, + "f1": 0.8677248677248677, + "fn": 16, + "fp": 9, + "fpr": 0.00015827236916150816, + "precision": 0.9010989010989011, + "recall": 0.8367346938775511, + "roc_auc": 0.9748590981130776, + "tn": 56855, + "tp": 82 + }, + "test_metrics_at_selected_threshold": { + "average_precision": 0.8756165035034519, + "f1": 0.8677248677248677, + "fn": 16, + "fp": 9, + "fpr": 0.00015827236916150816, + "precision": 0.9010989010989011, + "recall": 0.8367346938775511, + "roc_auc": 0.9748590981130776, + "tn": 56855, + "tp": 82 + }, + "threshold_selection": { + "candidate_count": 101, + "metric": "f1", + "selected_threshold": 0.54, + "split": "validation", + "validation_metrics": { + "average_precision": 0.8043451590721952, + "f1": 0.8432432432432431, + "fn": 21.0, + "fp": 8.0, + "fpr": 0.0001406890244974764, + "precision": 0.9069767441860465, + "recall": 0.7878787878787878, + "roc_auc": 0.9631822862570449, + "threshold": 0.54, + "tn": 56855.0, + "tp": 78.0 + } + }, + "type": "xgboost" + } + ], + "schema_version": 1 +} diff --git a/scripts/generate_report.py b/scripts/generate_report.py new file mode 100644 index 0000000..f6130aa --- /dev/null +++ b/scripts/generate_report.py @@ -0,0 +1,6 @@ +"""Compatibility wrapper for the package command-line entry point.""" + +from ios_eval.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ios_eval/__init__.py b/src/ios_eval/__init__.py new file mode 100644 index 0000000..e864059 --- /dev/null +++ b/src/ios_eval/__init__.py @@ -0,0 +1,14 @@ +"""Reusable evaluation infrastructure for imbalanced financial-risk models.""" + +from ios_eval.data_loader import DatasetSplits, load_and_prepare_data, load_dataset, prepare_data +from ios_eval.metrics import compute_core_metrics, evaluate_threshold_range, find_optimal_threshold + +__all__ = [ + "DatasetSplits", + "compute_core_metrics", + "evaluate_threshold_range", + "find_optimal_threshold", + "load_and_prepare_data", + "load_dataset", + "prepare_data", +] diff --git a/src/ios_eval/cli.py b/src/ios_eval/cli.py new file mode 100644 index 0000000..6f8dd1a --- /dev/null +++ b/src/ios_eval/cli.py @@ -0,0 +1,31 @@ +"""Command-line entry point for reproducible benchmark execution.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence + +from ios_eval.evaluation import load_config, run_benchmark +from ios_eval.report import format_comparison, write_json_report + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the IOS public fraud-model benchmark") + parser.add_argument("--data", default="data/raw/creditcard.csv", help="Path to creditcard.csv") + parser.add_argument("--config", default="configs/benchmark.json", help="Benchmark JSON configuration") + parser.add_argument("--output", default="results/benchmark.json", help="Machine-readable result path") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + report = run_benchmark(data_path=args.data, config=load_config(args.config)) + output_path = write_json_report(report, args.output) + print(format_comparison(report)) + print(f"\nEvidence artifact: {output_path}") + print("Evidence boundary: public dataset benchmark; not a production or client outcome.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ios_eval/data_loader.py b/src/ios_eval/data_loader.py new file mode 100644 index 0000000..28a870a --- /dev/null +++ b/src/ios_eval/data_loader.py @@ -0,0 +1,194 @@ +"""Validated, leakage-safe loading and splitting for the ULB fraud dataset.""" + +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler + +PCA_COLUMNS = tuple(f"V{number}" for number in range(1, 29)) +REQUIRED_COLUMNS = ("Time", *PCA_COLUMNS, "Amount", "Class") + + +@dataclass(slots=True) +class DatasetSplits: + """Train, validation, and final-test partitions plus preparation provenance.""" + + X_train: pd.DataFrame + X_validation: pd.DataFrame + X_test: pd.DataFrame + y_train: pd.Series + y_validation: pd.Series + y_test: pd.Series + metadata: dict[str, Any] + + +def file_sha256(path: str | Path) -> str: + """Return a streaming SHA-256 fingerprint for a dataset file.""" + + digest = sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_dataset(data_path: str | Path) -> pd.DataFrame: + """Load and validate the expected public credit-card fraud schema.""" + + path = Path(data_path) + if not path.is_file(): + raise FileNotFoundError(f"Dataset not found: {path}") + + try: + frame = pd.read_csv(path) + except pd.errors.EmptyDataError as exc: + raise ValueError(f"Dataset is empty: {path}") from exc + + return _validate_frame(frame) + + +def _validate_frame(frame: pd.DataFrame) -> pd.DataFrame: + """Validate a frame from either CSV or an in-memory caller.""" + + missing = sorted(set(REQUIRED_COLUMNS) - set(frame.columns)) + if missing: + raise ValueError(f"Dataset is missing required columns: {', '.join(missing)}") + + unexpected = sorted(set(frame.columns) - set(REQUIRED_COLUMNS)) + if unexpected: + raise ValueError(f"Dataset contains unexpected columns: {', '.join(unexpected)}") + + non_numeric = [column for column in REQUIRED_COLUMNS if not pd.api.types.is_numeric_dtype(frame[column])] + if non_numeric: + raise ValueError(f"Dataset columns must be numeric: {', '.join(non_numeric)}") + + values = frame.loc[:, REQUIRED_COLUMNS].to_numpy(dtype=float) + if not np.isfinite(values).all(): + raise ValueError("Dataset contains missing or non-finite values") + + labels = set(frame["Class"].unique().tolist()) + if not labels.issubset({0, 1}) or labels != {0, 1}: + raise ValueError("Class must contain both binary labels 0 and 1") + + return frame.loc[:, REQUIRED_COLUMNS].copy() + + +def _class_counts(labels: pd.Series) -> dict[str, int]: + counts = labels.value_counts().reindex([0, 1], fill_value=0) + return {"0": int(counts.loc[0]), "1": int(counts.loc[1])} + + +def prepare_data( + frame: pd.DataFrame, + *, + validation_size: float = 0.2, + test_size: float = 0.2, + random_state: int = 42, + scale_amount: bool = True, +) -> DatasetSplits: + """Create deterministic 60/20/20-style splits without preprocessing leakage.""" + + if validation_size <= 0 or test_size <= 0 or validation_size + test_size >= 1: + raise ValueError("validation_size and test_size must be positive and sum to less than 1") + + frame = _validate_frame(frame) + labels = frame["Class"] + if int(labels.value_counts().min()) < 3: + raise ValueError("Each class needs at least three rows for stratified train/validation/test splits") + + features = frame.drop(columns=["Class", "Time"]).copy() + train_validation_X, test_X, train_validation_y, test_y = train_test_split( + features, + labels, + test_size=test_size, + random_state=random_state, + stratify=labels, + ) + relative_validation_size = validation_size / (1.0 - test_size) + train_X, validation_X, train_y, validation_y = train_test_split( + train_validation_X, + train_validation_y, + test_size=relative_validation_size, + random_state=random_state, + stratify=train_validation_y, + ) + + scaler_metadata: dict[str, Any] | None = None + if scale_amount: + scaler = StandardScaler() + train_X.loc[:, "Amount"] = scaler.fit_transform(train_X[["Amount"]]).ravel() + validation_X.loc[:, "Amount"] = scaler.transform(validation_X[["Amount"]]).ravel() + test_X.loc[:, "Amount"] = scaler.transform(test_X[["Amount"]]).ravel() + scaler_metadata = { + "columns": ["Amount"], + "fit_scope": "train_only", + "mean": [float(value) for value in scaler.mean_], + "scale": [float(value) for value in scaler.scale_], + } + + metadata = { + "rows": len(frame), + "columns": list(frame.columns), + "features": list(features.columns), + "class_counts": _class_counts(labels), + "split": { + "strategy": "stratified_train_validation_test", + "random_state": random_state, + "validation_size": validation_size, + "test_size": test_size, + "rows": { + "train": len(train_X), + "validation": len(validation_X), + "test": len(test_X), + }, + "class_counts": { + "train": _class_counts(train_y), + "validation": _class_counts(validation_y), + "test": _class_counts(test_y), + }, + }, + "preprocessing": {"dropped_columns": ["Time"], "scaler": scaler_metadata}, + } + return DatasetSplits( + X_train=train_X, + X_validation=validation_X, + X_test=test_X, + y_train=train_y, + y_validation=validation_y, + y_test=test_y, + metadata=metadata, + ) + + +def load_and_prepare_data( + data_path: str | Path = "data/raw/creditcard.csv", + *, + validation_size: float = 0.2, + test_size: float = 0.2, + random_state: int = 42, + scale_amount: bool = True, +) -> DatasetSplits: + """Load the dataset, split first, then fit preprocessing on training data only.""" + + path = Path(data_path) + frame = load_dataset(path) + splits = prepare_data( + frame, + validation_size=validation_size, + test_size=test_size, + random_state=random_state, + scale_amount=scale_amount, + ) + splits.metadata["dataset"] = { + "filename": path.name, + "sha256": file_sha256(path), + "source": "https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud", + } + return splits diff --git a/src/ios_eval/evaluation.py b/src/ios_eval/evaluation.py new file mode 100644 index 0000000..f074eca --- /dev/null +++ b/src/ios_eval/evaluation.py @@ -0,0 +1,216 @@ +"""Leakage-safe orchestration for the three-model public benchmark.""" + +from __future__ import annotations + +import importlib.metadata +import json +import platform +import subprocess +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from hashlib import sha256 +from pathlib import Path +from typing import Any + +import numpy as np +from imblearn.over_sampling import SMOTE +from sklearn.base import ClassifierMixin +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import LogisticRegression + +from ios_eval.data_loader import DatasetSplits, load_and_prepare_data +from ios_eval.metrics import compute_core_metrics, find_optimal_threshold + + +def load_config(path: str | Path) -> dict[str, Any]: + """Load a JSON benchmark configuration with basic structural validation.""" + + config_path = Path(path) + if not config_path.is_file(): + raise FileNotFoundError(f"Configuration not found: {config_path}") + try: + config = json.loads(config_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Configuration is not valid JSON: {config_path}") from exc + if not isinstance(config, dict) or not isinstance(config.get("models"), dict): + raise ValueError("Configuration must contain a models object") + if not config["models"]: + raise ValueError("Configuration must enable at least one model") + return config + + +def _resolved_model( + name: str, + model_config: Mapping[str, Any], + y_train: Sequence[int], +) -> tuple[ClassifierMixin, dict[str, Any] | None, dict[str, Any]]: + estimator_config = dict(model_config.get("estimator", {})) + model_type = model_config.get("type") + smote_config: dict[str, Any] | None = None + + if model_type == "logistic_regression": + estimator: ClassifierMixin = LogisticRegression(**estimator_config) + elif model_type == "smote_random_forest": + smote_config = dict(model_config.get("smote", {})) + estimator = RandomForestClassifier(**estimator_config) + elif model_type == "xgboost": + from xgboost import XGBClassifier + + if estimator_config.pop("scale_pos_weight", None) != "train_class_ratio": + raise ValueError(f"{name}: xgboost scale_pos_weight must be train_class_ratio") + labels = np.asarray(y_train) + positives = int(np.sum(labels == 1)) + if positives == 0: + raise ValueError(f"{name}: training data has no positive examples") + estimator_config["scale_pos_weight"] = float(np.sum(labels == 0) / positives) + estimator = XGBClassifier(**estimator_config) + else: + raise ValueError(f"{name}: unsupported model type {model_type!r}") + + return estimator, smote_config, estimator_config + + +def evaluate_model( + name: str, + model_config: Mapping[str, Any], + splits: DatasetSplits, + *, + threshold_metric: str, + thresholds: Sequence[float], +) -> dict[str, Any]: + """Fit one model on train, select threshold on validation, report test once.""" + + estimator, smote_config, resolved_parameters = _resolved_model(name, model_config, splits.y_train) + train_X = splits.X_train + train_y = splits.y_train + training_rows_before = len(train_X) + training_counts_before = splits.metadata["split"]["class_counts"]["train"] + + if smote_config is not None: + sampler = SMOTE(**smote_config) + train_X, train_y = sampler.fit_resample(train_X, train_y) + + estimator.fit(train_X, train_y) + validation_probabilities = estimator.predict_proba(splits.X_validation)[:, 1] + selected_threshold, validation_metrics = find_optimal_threshold( + splits.y_validation, + validation_probabilities, + metric=threshold_metric, + thresholds=thresholds, + ) + + test_probabilities = estimator.predict_proba(splits.X_test)[:, 1] + selected_predictions = (test_probabilities >= selected_threshold).astype(int) + default_predictions = (test_probabilities >= 0.5).astype(int) + + training_counts_after = { + "0": int(np.sum(np.asarray(train_y) == 0)), + "1": int(np.sum(np.asarray(train_y) == 1)), + } + return { + "name": name, + "type": model_config["type"], + "parameters": resolved_parameters, + "imbalance_handling": { + "smote": smote_config, + "smote_fit_scope": "train_only" if smote_config is not None else None, + "training_rows_before": training_rows_before, + "training_rows_after": len(train_X), + "training_class_counts_before": training_counts_before, + "training_class_counts_after": training_counts_after, + }, + "threshold_selection": { + "split": "validation", + "metric": threshold_metric, + "candidate_count": len(thresholds), + "selected_threshold": selected_threshold, + "validation_metrics": validation_metrics, + }, + "test_metrics_at_selected_threshold": compute_core_metrics( + splits.y_test, + selected_predictions, + test_probabilities, + ), + "test_metrics_at_default_threshold": compute_core_metrics( + splits.y_test, + default_predictions, + test_probabilities, + ), + } + + +def _git_revision() -> str | None: + try: + completed = subprocess.run( + ["git", "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + except (FileNotFoundError, subprocess.SubprocessError): + return None + return completed.stdout.strip() or None + + +def _environment() -> dict[str, Any]: + dependencies = {} + for distribution in ("numpy", "pandas", "scikit-learn", "imbalanced-learn"): + dependencies[distribution] = importlib.metadata.version(distribution) + dependencies["xgboost"] = importlib.metadata.version("xgboost") + return { + "python": platform.python_version(), + "implementation": platform.python_implementation(), + "platform": platform.platform(), + "dependencies": dependencies, + } + + +def run_benchmark( + *, + data_path: str | Path, + config: Mapping[str, Any], +) -> dict[str, Any]: + """Run configured models and return a machine-readable evidence artifact.""" + + random_state = int(config.get("random_state", 42)) + validation_size = float(config.get("validation_size", 0.2)) + test_size = float(config.get("test_size", 0.2)) + threshold_metric = str(config.get("threshold_metric", "f1")) + threshold_step = float(config.get("threshold_step", 0.01)) + if threshold_step <= 0 or threshold_step > 1: + raise ValueError("threshold_step must be greater than 0 and no greater than 1") + threshold_intervals = round(1.0 / threshold_step) + if not np.isclose(threshold_intervals * threshold_step, 1.0): + raise ValueError("threshold_step must divide the 0-to-1 range evenly") + thresholds = np.linspace(0.0, 1.0, threshold_intervals + 1).tolist() + + splits = load_and_prepare_data( + data_path, + validation_size=validation_size, + test_size=test_size, + random_state=random_state, + ) + results = [ + evaluate_model( + name, + model_config, + splits, + threshold_metric=threshold_metric, + thresholds=thresholds, + ) + for name, model_config in config["models"].items() + ] + canonical_config = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() + return { + "schema_version": 1, + "evidence_label": "public_dataset_benchmark", + "generated_at_utc": datetime.now(UTC).isoformat(), + "git_revision": _git_revision(), + "config_sha256": sha256(canonical_config).hexdigest(), + "config": dict(config), + "dataset": splits.metadata, + "environment": _environment(), + "models": results, + "claim_boundary": "Not a production or client outcome.", + } diff --git a/src/ios_eval/metrics.py b/src/ios_eval/metrics.py new file mode 100644 index 0000000..466c5a0 --- /dev/null +++ b/src/ios_eval/metrics.py @@ -0,0 +1,129 @@ +"""Model-agnostic binary classification metrics and threshold selection.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import TypeAlias + +import numpy as np +import pandas as pd +from sklearn.metrics import average_precision_score, confusion_matrix, roc_auc_score + +MetricValue: TypeAlias = float | int | None +MetricResult: TypeAlias = dict[str, MetricValue] + + +def _one_dimensional(name: str, values: Iterable[float]) -> np.ndarray: + array = np.asarray(values) + if array.ndim != 1: + raise ValueError(f"{name} must be one-dimensional") + if array.size == 0: + raise ValueError(f"{name} cannot be empty") + return array + + +def _binary_labels(name: str, values: Iterable[int]) -> np.ndarray: + array = _one_dimensional(name, values) + if not set(np.unique(array).tolist()).issubset({0, 1}): + raise ValueError(f"{name} must contain only binary labels 0 and 1") + return array.astype(int) + + +def _probabilities(values: Iterable[float]) -> np.ndarray: + array = _one_dimensional("y_prob", values).astype(float) + if not np.isfinite(array).all() or np.any((array < 0) | (array > 1)): + raise ValueError("y_prob must contain finite probabilities between 0 and 1") + return array + + +def compute_core_metrics( + y_true: Iterable[int], + y_pred: Iterable[int], + y_prob: Iterable[float] | None = None, +) -> MetricResult: + """Compute operational and ranking metrics with explicit edge-case behavior.""" + + true = _binary_labels("y_true", y_true) + predicted = _binary_labels("y_pred", y_pred) + if true.size != predicted.size: + raise ValueError("y_true and y_pred must have the same length") + + probabilities = None if y_prob is None else _probabilities(y_prob) + if probabilities is not None and true.size != probabilities.size: + raise ValueError("y_true and y_prob must have the same length") + + tn, fp, fn, tp = confusion_matrix(true, predicted, labels=[0, 1]).ravel() + precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0 + fpr = fp / (fp + tn) if (fp + tn) > 0 else 0.0 + + has_both_classes = np.unique(true).size == 2 + roc_auc = roc_auc_score(true, probabilities) if probabilities is not None and has_both_classes else None + average_precision = ( + average_precision_score(true, probabilities) + if probabilities is not None and has_both_classes + else None + ) + return { + "precision": float(precision), + "recall": float(recall), + "f1": float(f1), + "fpr": float(fpr), + "tn": int(tn), + "fp": int(fp), + "fn": int(fn), + "tp": int(tp), + "roc_auc": float(roc_auc) if roc_auc is not None else None, + "average_precision": float(average_precision) if average_precision is not None else None, + } + + +def evaluate_threshold_range( + y_true: Iterable[int], + y_prob: Iterable[float], + thresholds: Iterable[float] | None = None, +) -> pd.DataFrame: + """Evaluate candidate operating thresholds against one labeled split.""" + + true = _binary_labels("y_true", y_true) + probabilities = _probabilities(y_prob) + if true.size != probabilities.size: + raise ValueError("y_true and y_prob must have the same length") + if thresholds is None: + threshold_values = np.linspace(0.0, 1.0, 101) + else: + threshold_values = _one_dimensional("thresholds", thresholds).astype(float) + if not np.isfinite(threshold_values).all() or np.any((threshold_values < 0) | (threshold_values > 1)): + raise ValueError("thresholds must be finite values between 0 and 1") + if np.unique(threshold_values).size != threshold_values.size: + raise ValueError("thresholds must not contain duplicates") + + rows = [] + for threshold in threshold_values: + predicted = (probabilities >= threshold).astype(int) + metrics = compute_core_metrics(true, predicted, probabilities) + metrics["threshold"] = float(threshold) + rows.append(metrics) + + return pd.DataFrame(rows).set_index("threshold").sort_index() + + +def find_optimal_threshold( + y_true: Iterable[int], + y_prob: Iterable[float], + metric: str = "f1", + thresholds: Iterable[float] | None = None, +) -> tuple[float, MetricResult]: + """Select a threshold deterministically; ties prefer the higher threshold.""" + + if metric not in {"f1", "precision", "recall"}: + raise ValueError("metric must be one of: f1, precision, recall") + + frame = evaluate_threshold_range(y_true, y_prob, thresholds) + best_score = frame[metric].max() + best_threshold = float(frame.index[frame[metric] == best_score].max()) + best_metrics: MetricResult = frame.loc[best_threshold].to_dict() + best_metrics["threshold"] = best_threshold + + return best_threshold, best_metrics diff --git a/src/ios_eval/report.py b/src/ios_eval/report.py new file mode 100644 index 0000000..afae023 --- /dev/null +++ b/src/ios_eval/report.py @@ -0,0 +1,48 @@ +"""Serialization and human-readable rendering for benchmark evidence.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def write_json_report(report: dict[str, Any], output_path: str | Path) -> Path: + """Write a report atomically so interrupted runs do not leave partial JSON.""" + + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_suffix(f"{path.suffix}.tmp") + temporary_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary_path.replace(path) + return path + + +def format_comparison(report: dict[str, Any]) -> str: + """Render the final held-out test metrics without requiring notebook tooling.""" + + headers = ("Model", "AP", "ROC AUC", "Precision", "Recall", "F1", "FP", "FN", "Threshold") + rows = [] + for model in report["models"]: + metrics = model["test_metrics_at_selected_threshold"] + rows.append( + ( + model["name"], + f"{metrics['average_precision']:.4f}", + f"{metrics['roc_auc']:.4f}", + f"{metrics['precision']:.4f}", + f"{metrics['recall']:.4f}", + f"{metrics['f1']:.4f}", + str(metrics["fp"]), + str(metrics["fn"]), + f"{model['threshold_selection']['selected_threshold']:.2f}", + ) + ) + + widths = [max(len(headers[index]), *(len(row[index]) for row in rows)) for index in range(len(headers))] + + def format_row(row: tuple[str, ...]) -> str: + return " | ".join(value.ljust(widths[index]) for index, value in enumerate(row)) + + separator = "-+-".join("-" * width for width in widths) + return "\n".join((format_row(headers), separator, *(format_row(row) for row in rows))) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..35d82e0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from ios_eval.data_loader import PCA_COLUMNS + + +@pytest.fixture +def fraud_frame() -> pd.DataFrame: + rng = np.random.default_rng(42) + rows = 150 + frame = pd.DataFrame({"Time": np.arange(rows, dtype=float)}) + for column in PCA_COLUMNS: + frame[column] = rng.normal(size=rows) + frame["Amount"] = rng.lognormal(mean=3.0, sigma=1.0, size=rows) + labels = np.zeros(rows, dtype=int) + labels[rng.choice(rows, size=30, replace=False)] = 1 + frame["Class"] = labels + return frame diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py new file mode 100644 index 0000000..5511fd7 --- /dev/null +++ b/tests/test_edge_cases.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import pytest + +from ios_eval.data_loader import load_dataset, prepare_data + + +def test_missing_dataset_has_clear_error(tmp_path) -> None: + with pytest.raises(FileNotFoundError, match="Dataset not found"): + load_dataset(tmp_path / "missing.csv") + + +def test_empty_dataset_has_clear_error(tmp_path) -> None: + path = tmp_path / "empty.csv" + path.write_text("", encoding="utf-8") + + with pytest.raises(ValueError, match="Dataset is empty"): + load_dataset(path) + + +def test_missing_required_column_is_rejected(tmp_path, fraud_frame) -> None: + path = tmp_path / "missing.csv" + fraud_frame.drop(columns=["Amount"]).to_csv(path, index=False) + + with pytest.raises(ValueError, match="Amount"): + load_dataset(path) + + +def test_unexpected_column_is_rejected(tmp_path, fraud_frame) -> None: + path = tmp_path / "unexpected.csv" + fraud_frame.assign(CustomerId=1).to_csv(path, index=False) + + with pytest.raises(ValueError, match="unexpected columns: CustomerId"): + load_dataset(path) + + +@pytest.mark.parametrize("bad_value", [float("nan"), float("inf")]) +def test_non_finite_values_are_rejected(tmp_path, fraud_frame, bad_value) -> None: + path = tmp_path / "invalid.csv" + frame = fraud_frame.copy() + frame.loc[0, "Amount"] = bad_value + frame.to_csv(path, index=False) + + with pytest.raises(ValueError, match="missing or non-finite"): + load_dataset(path) + + +def test_non_numeric_feature_is_rejected(tmp_path, fraud_frame) -> None: + path = tmp_path / "non_numeric.csv" + frame = fraud_frame.copy() + frame["V1"] = frame["V1"].astype(object) + frame.loc[0, "V1"] = "not-a-number" + frame.to_csv(path, index=False) + + with pytest.raises(ValueError, match="must be numeric: V1"): + load_dataset(path) + + +def test_non_binary_or_single_class_labels_are_rejected(fraud_frame) -> None: + invalid = fraud_frame.copy() + invalid.loc[invalid.index[0], "Class"] = 2 + with pytest.raises(ValueError, match="both binary labels"): + prepare_data(invalid) + + one_class = fraud_frame.assign(Class=0) + with pytest.raises(ValueError, match="both binary labels"): + prepare_data(one_class) + + +def test_split_sizes_must_be_defensible(fraud_frame) -> None: + with pytest.raises(ValueError, match="sum to less than 1"): + prepare_data(fraud_frame, validation_size=0.5, test_size=0.5) + with pytest.raises(ValueError, match="positive"): + prepare_data(fraud_frame, validation_size=0.0, test_size=0.2) + + +def test_minority_class_must_support_three_stratified_splits(fraud_frame) -> None: + frame = fraud_frame.copy() + frame["Class"] = 0 + frame.loc[frame.index[:2], "Class"] = 1 + + with pytest.raises(ValueError, match="at least three rows"): + prepare_data(frame) + + +def test_prepare_data_is_deterministic_and_partitions_do_not_overlap(fraud_frame) -> None: + first = prepare_data(fraud_frame, random_state=7) + second = prepare_data(fraud_frame, random_state=7) + + assert first.X_train.index.tolist() == second.X_train.index.tolist() + assert first.X_validation.index.tolist() == second.X_validation.index.tolist() + assert first.X_test.index.tolist() == second.X_test.index.tolist() + assert set(first.X_train.index).isdisjoint(first.X_validation.index) + assert set(first.X_train.index).isdisjoint(first.X_test.index) + assert set(first.X_validation.index).isdisjoint(first.X_test.index) + assert len(first.X_train) + len(first.X_validation) + len(first.X_test) == len(fraud_frame) + + +def test_scaler_is_fitted_on_training_rows_only(fraud_frame) -> None: + splits = prepare_data(fraud_frame, random_state=13) + train_amounts = fraud_frame.loc[splits.X_train.index, "Amount"] + all_amounts = fraud_frame["Amount"] + scaler = splits.metadata["preprocessing"]["scaler"] + + assert scaler["fit_scope"] == "train_only" + assert scaler["mean"][0] == pytest.approx(train_amounts.mean()) + assert scaler["mean"][0] != pytest.approx(all_amounts.mean()) + assert splits.X_train["Amount"].mean() == pytest.approx(0.0, abs=1e-12) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000..f12eb64 --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import json +from dataclasses import replace + +import pytest + +from ios_eval.data_loader import prepare_data +from ios_eval.evaluation import evaluate_model, load_config, run_benchmark + + +def test_load_config_rejects_missing_malformed_and_empty_configs(tmp_path) -> None: + with pytest.raises(FileNotFoundError, match="Configuration not found"): + load_config(tmp_path / "missing.json") + + malformed = tmp_path / "malformed.json" + malformed.write_text("{", encoding="utf-8") + with pytest.raises(ValueError, match="not valid JSON"): + load_config(malformed) + + empty = tmp_path / "empty.json" + empty.write_text('{"models": {}}', encoding="utf-8") + with pytest.raises(ValueError, match="at least one model"): + load_config(empty) + + +def test_threshold_is_selected_on_validation_and_test_is_reported_separately(fraud_frame) -> None: + splits = prepare_data(fraud_frame, random_state=42) + result = evaluate_model( + "Logistic Regression", + { + "type": "logistic_regression", + "estimator": {"class_weight": "balanced", "max_iter": 300, "random_state": 42}, + }, + splits, + threshold_metric="f1", + thresholds=[0.25, 0.5, 0.75], + ) + + assert result["threshold_selection"]["split"] == "validation" + assert result["threshold_selection"]["selected_threshold"] in {0.25, 0.5, 0.75} + assert result["threshold_selection"]["validation_metrics"] + assert result["test_metrics_at_selected_threshold"]["tn"] + result["test_metrics_at_selected_threshold"][ + "fp" + ] == int((splits.y_test == 0).sum()) + + +def test_final_test_labels_cannot_influence_selected_threshold(fraud_frame) -> None: + splits = prepare_data(fraud_frame, random_state=42) + model_config = { + "type": "logistic_regression", + "estimator": {"class_weight": "balanced", "max_iter": 300, "random_state": 42}, + } + original = evaluate_model( + "Logistic Regression", + model_config, + splits, + threshold_metric="f1", + thresholds=[0.25, 0.5, 0.75], + ) + altered = evaluate_model( + "Logistic Regression", + model_config, + replace(splits, y_test=1 - splits.y_test), + threshold_metric="f1", + thresholds=[0.25, 0.5, 0.75], + ) + + assert original["threshold_selection"] == altered["threshold_selection"] + assert original["test_metrics_at_selected_threshold"] != altered["test_metrics_at_selected_threshold"] + + +def test_smote_resamples_only_the_training_partition(fraud_frame) -> None: + splits = prepare_data(fraud_frame, random_state=42) + result = evaluate_model( + "SMOTE + Random Forest", + { + "type": "smote_random_forest", + "smote": {"random_state": 42, "k_neighbors": 2}, + "estimator": {"n_estimators": 5, "max_depth": 3, "random_state": 42, "n_jobs": 1}, + }, + splits, + threshold_metric="f1", + thresholds=[0.25, 0.5, 0.75], + ) + handling = result["imbalance_handling"] + + assert handling["smote_fit_scope"] == "train_only" + assert handling["training_rows_before"] == len(splits.X_train) + assert handling["training_rows_after"] > handling["training_rows_before"] + assert handling["training_class_counts_after"]["0"] == handling["training_class_counts_after"]["1"] + assert len(splits.X_validation) == splits.metadata["split"]["rows"]["validation"] + assert len(splits.X_test) == splits.metadata["split"]["rows"]["test"] + + +def test_run_benchmark_records_dataset_environment_config_and_claim_boundary(tmp_path, fraud_frame) -> None: + data_path = tmp_path / "creditcard.csv" + fraud_frame.to_csv(data_path, index=False) + config = { + "random_state": 42, + "validation_size": 0.2, + "test_size": 0.2, + "threshold_metric": "f1", + "threshold_step": 0.5, + "models": { + "Logistic Regression": { + "type": "logistic_regression", + "estimator": {"class_weight": "balanced", "max_iter": 300, "random_state": 42}, + } + }, + } + + report = run_benchmark(data_path=data_path, config=config) + + assert report["schema_version"] == 1 + assert report["evidence_label"] == "public_dataset_benchmark" + assert report["claim_boundary"] == "Not a production or client outcome." + assert len(report["dataset"]["dataset"]["sha256"]) == 64 + assert report["dataset"]["preprocessing"]["scaler"]["fit_scope"] == "train_only" + assert report["models"][0]["threshold_selection"]["split"] == "validation" + assert report["environment"]["python"].startswith("3.11") + json.dumps(report) + + +def test_run_benchmark_rejects_threshold_step_that_cannot_cover_range(tmp_path, fraud_frame) -> None: + data_path = tmp_path / "creditcard.csv" + fraud_frame.to_csv(data_path, index=False) + config = { + "threshold_step": 0.3, + "models": { + "Logistic Regression": { + "type": "logistic_regression", + "estimator": {"max_iter": 100, "random_state": 42}, + } + }, + } + + with pytest.raises(ValueError, match="divide the 0-to-1 range evenly"): + run_benchmark(data_path=data_path, config=config) + + +def test_unsupported_model_type_fails_clearly(fraud_frame) -> None: + splits = prepare_data(fraud_frame) + with pytest.raises(ValueError, match="unsupported model type"): + evaluate_model( + "Unknown", + {"type": "mystery", "estimator": {}}, + splits, + threshold_metric="f1", + thresholds=[0.5], + ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..5b8b7ef --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from ios_eval.metrics import compute_core_metrics, evaluate_threshold_range, find_optimal_threshold + + +def test_compute_core_metrics_matches_known_confusion_matrix() -> None: + metrics = compute_core_metrics( + y_true=[0, 0, 0, 1, 1, 1], + y_pred=[0, 1, 0, 1, 0, 1], + y_prob=[0.1, 0.7, 0.2, 0.8, 0.4, 0.9], + ) + + assert metrics["tn"] == 2 + assert metrics["fp"] == 1 + assert metrics["fn"] == 1 + assert metrics["tp"] == 2 + assert metrics["precision"] == pytest.approx(2 / 3) + assert metrics["recall"] == pytest.approx(2 / 3) + assert metrics["f1"] == pytest.approx(2 / 3) + assert metrics["fpr"] == pytest.approx(1 / 3) + assert metrics["roc_auc"] == pytest.approx(8 / 9) + assert metrics["average_precision"] == pytest.approx(0.9166666667) + + +def test_threshold_sweep_returns_requested_candidates() -> None: + frame = evaluate_threshold_range( + [0, 1, 0, 1], + [0.1, 0.8, 0.4, 0.6], + thresholds=[0.25, 0.5, 0.75], + ) + + assert frame.index.tolist() == [0.25, 0.5, 0.75] + assert frame.loc[0.5, "f1"] == pytest.approx(1.0) + assert frame.loc[0.75, "recall"] == pytest.approx(0.5) + + +def test_threshold_selection_prefers_higher_threshold_on_tie() -> None: + threshold, metrics = find_optimal_threshold( + [0, 1], + [0.1, 0.9], + thresholds=[0.2, 0.5, 0.8], + ) + + assert threshold == 0.8 + assert metrics["f1"] == pytest.approx(1.0) + + +@pytest.mark.parametrize( + ("y_true", "y_pred", "y_prob", "message"), + [ + ([], [], None, "y_true cannot be empty"), + ([0, 2], [0, 1], None, "binary labels"), + ([0, 1], [0], None, "same length"), + ([0, 1], [0, 1], [0.2], "same length"), + ([0, 1], [0, 1], [0.2, np.nan], "finite probabilities"), + ([0, 1], [0, 1], [0.2, 1.1], "between 0 and 1"), + ], +) +def test_compute_core_metrics_rejects_malformed_inputs(y_true, y_pred, y_prob, message) -> None: + with pytest.raises(ValueError, match=message): + compute_core_metrics(y_true, y_pred, y_prob) + + +def test_single_class_metrics_are_explicitly_undefined_for_ranking() -> None: + metrics = compute_core_metrics([0, 0], [0, 0], [0.1, 0.2]) + + assert metrics["roc_auc"] is None + assert metrics["average_precision"] is None + assert metrics["fpr"] == 0.0 + + +def test_threshold_helpers_reject_invalid_candidates_and_metric() -> None: + with pytest.raises(ValueError, match="duplicates"): + evaluate_threshold_range([0, 1], [0.1, 0.9], thresholds=[0.5, 0.5]) + with pytest.raises(ValueError, match="between 0 and 1"): + evaluate_threshold_range([0, 1], [0.1, 0.9], thresholds=[-0.1]) + with pytest.raises(ValueError, match="metric must be"): + find_optimal_threshold([0, 1], [0.1, 0.9], metric="accuracy") diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..258fde2 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json + +from ios_eval.report import format_comparison, write_json_report + + +def sample_report() -> dict: + return { + "models": [ + { + "name": "Example", + "threshold_selection": {"selected_threshold": 0.4}, + "test_metrics_at_selected_threshold": { + "average_precision": 0.9, + "roc_auc": 0.95, + "precision": 0.8, + "recall": 0.75, + "f1": 0.7742, + "fp": 2, + "fn": 3, + }, + } + ] + } + + +def test_json_report_is_machine_readable_and_replaces_existing_file(tmp_path) -> None: + output = tmp_path / "nested" / "benchmark.json" + output.parent.mkdir() + output.write_text("stale", encoding="utf-8") + + written = write_json_report(sample_report(), output) + + assert written == output + assert json.loads(output.read_text(encoding="utf-8")) == sample_report() + assert not output.with_suffix(".json.tmp").exists() + + +def test_comparison_format_exposes_held_out_metrics() -> None: + rendered = format_comparison(sample_report()) + + assert "Model" in rendered + assert "Example" in rendered + assert "0.9000" in rendered + assert "0.40" in rendered