From 1f2e0e26bc74ba387325d2cc1326b64d900e5f14 Mon Sep 17 00:00:00 2001 From: Thomas Behan Date: Tue, 16 Jun 2026 00:36:50 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20Stage=201=20(4.5.0):=20data-sou?= =?UTF-8?q?rcing=20module=20=E2=80=94=20source,=20canonicalise,=20balance,?= =?UTF-8?q?=20sort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New lesnet/data/ subsystem (no TensorFlow): per-source loaders (ISIC API malignant-maximal download, PAD-UFES-20, Fitzpatrick17k, DDI) -> curated diagnosis-name canonicalisation -> quality gate + perceptual-hash near-duplicate removal -> sort into benign/not_sure/malignant // folders -> fair, group-safe, fairness-aware 1:1 benign:malignant balancing with per-diagnosis caps -> leakage-safe grouped manifest + report. Driven by commands/build_dataset.py. Restructure: moved the data concerns out of lesnet/ml (taxonomy, splits, records/manifest IO) into lesnet/data; lesnet/ml keeps the TF-coupled model/training/inference. Removed the three old data commands (download_isic_full, fetch_isic_sample, run_build_dataset) and the stale datasets test, rewired all importers, updated CLAUDE.md + train_full_m4.sh. 100% unit-test coverage on lesnet/data (63 tests, network mocked), ruff clean. Existing training (CPU smoke) + inference (web /labels+/predict, real M-4s CLI) still pass. Validated end-to-end on the live ISIC API: 1199 labelled -> balanced 163:163, leakage-safe splits. --- CLAUDE.md | 24 ++- commands/build_dataset.py | 80 ++++++++++ commands/download_isic_full.py | 145 ----------------- commands/fetch_isic_sample.py | 86 ---------- commands/run_build_dataset.py | 49 ------ commands/run_evaluate.py | 2 +- commands/run_train_triage.py | 5 +- commands/run_triage_inference.py | 2 +- commands/run_validation_check.py | 2 +- lesnet/data/__init__.py | 7 + lesnet/data/balance.py | 97 ++++++++++++ lesnet/data/canonical.py | 76 +++++++++ lesnet/data/config.py | 31 ++++ lesnet/data/manifest.py | 25 +++ lesnet/data/pipeline.py | 91 +++++++++++ lesnet/data/quality.py | 76 +++++++++ lesnet/data/records.py | 74 +++++++++ lesnet/data/sort.py | 65 ++++++++ lesnet/data/sources/__init__.py | 3 + lesnet/data/sources/base.py | 17 ++ lesnet/data/sources/ddi.py | 33 ++++ lesnet/data/sources/fitzpatrick17k.py | 52 +++++++ lesnet/data/sources/isic.py | 137 ++++++++++++++++ lesnet/data/sources/pad_ufes.py | 43 +++++ lesnet/data/sources/registry.py | 24 +++ lesnet/{ml => data}/splits.py | 0 lesnet/{ml => data}/taxonomy.py | 0 lesnet/ml/data_loader.py | 2 +- lesnet/ml/datasets.py | 216 -------------------------- lesnet/ml/evaluation.py | 2 +- lesnet/ml/inference.py | 2 +- lesnet/ml/synthetic.py | 2 +- lesnet/ml/training.py | 2 +- lesnet/views/api.py | 4 +- scripts/train_full_m4.sh | 24 ++- tests/test_data_pipeline.py | 137 ++++++++++++++++ tests/test_data_quality.py | 60 +++++++ tests/test_data_records.py | 57 +++++++ tests/test_data_sort_balance.py | 142 +++++++++++++++++ tests/test_data_sources.py | 190 ++++++++++++++++++++++ tests/test_data_taxonomy_canonical.py | 60 +++++++ tests/test_ml_datasets.py | 97 ------------ tests/test_ml_foundation.py | 3 +- tests/test_ml_training_smoke.py | 4 +- 44 files changed, 1620 insertions(+), 630 deletions(-) create mode 100644 commands/build_dataset.py delete mode 100644 commands/download_isic_full.py delete mode 100644 commands/fetch_isic_sample.py delete mode 100644 commands/run_build_dataset.py create mode 100644 lesnet/data/__init__.py create mode 100644 lesnet/data/balance.py create mode 100644 lesnet/data/canonical.py create mode 100644 lesnet/data/config.py create mode 100644 lesnet/data/manifest.py create mode 100644 lesnet/data/pipeline.py create mode 100644 lesnet/data/quality.py create mode 100644 lesnet/data/records.py create mode 100644 lesnet/data/sort.py create mode 100644 lesnet/data/sources/__init__.py create mode 100644 lesnet/data/sources/base.py create mode 100644 lesnet/data/sources/ddi.py create mode 100644 lesnet/data/sources/fitzpatrick17k.py create mode 100644 lesnet/data/sources/isic.py create mode 100644 lesnet/data/sources/pad_ufes.py create mode 100644 lesnet/data/sources/registry.py rename lesnet/{ml => data}/splits.py (100%) rename lesnet/{ml => data}/taxonomy.py (100%) delete mode 100644 lesnet/ml/datasets.py create mode 100644 tests/test_data_pipeline.py create mode 100644 tests/test_data_quality.py create mode 100644 tests/test_data_records.py create mode 100644 tests/test_data_sort_balance.py create mode 100644 tests/test_data_sources.py create mode 100644 tests/test_data_taxonomy_canonical.py delete mode 100644 tests/test_ml_datasets.py diff --git a/CLAUDE.md b/CLAUDE.md index f19c6b5..f963af5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,12 +35,17 @@ lesnet/ # the package __init__.py # main(): Pyramid WSGI app factory (pyramid imported lazily) routes.py # routes: home, supported-diagnoses, predict, labels config/model.py # ModelConfig — released-model registry (MODEL_URLS) for the web app + downloader - ml/ # the triage system (framework-light, pure-logic where possible) + data/ # data subsystem (stage 1: source -> sort -> balance -> manifest; no TF) + config.py # SourcingConfig + sources/ # isic, pad_ufes, fitzpatrick17k, ddi (+ registry) -> LesionRecords + taxonomy.py · canonical.py # diagnosis -> triage bucket; curated diagnosis-name canonicalisation + quality.py · sort.py · balance.py # quality + pHash dedup; bucket/diagnosis folders; fair 1:1 balance + records.py · splits.py · manifest.py # LesionRecord + manifest IO; grouped leakage-free splits + ml/ # the triage MODEL system (training-coupled, imports TF) config.py # PipelineConfig — all training/inference hyperparameters - datasets.py · splits.py # manifest build/load, patient-grouped leakage-free splits, LesionRecord data_loader.py # tf.data pipeline from a manifest preprocessing.py # DullRazor hair removal + Shades-of-Gray colour constancy + resize/scale - features.py · taxonomy.py # metadata encoding; diagnosis -> triage/fine label maps + features.py # metadata encoding (age/sex/site/Fitzpatrick) model.py # build_triage_model (EfficientNetV2-S) + triage/fine/feature sub-models losses.py · metrics.py # focal loss + class weights; clinical metrics (sens/spec/ROC/ECE/AURC) training.py # orchestrator: fit (optionally metric-gated) -> calibrate -> conformal -> OOD -> save @@ -53,8 +58,7 @@ lesnet/ # the package views/{api,default,notfound}.py # /predict + /labels (lazy TriagePredictor); home + supported-diagnoses; 404 templates/*.jinja2 · static/ # web UI commands/ # CLI scripts (run from repo root) - download_isic_full.py · fetch_isic_sample.py # dataset download - run_build_dataset.py # build the grouped manifest + build_dataset.py # stage 1: source -> canonicalise -> quality -> balance -> sort -> manifest run_train_triage.py # train (--smoke for a CPU synthetic end-to-end run) run_evaluate.py # evaluation report + model card run_triage_inference.py · run_validation_check.py # inference (single image / validation set) @@ -69,8 +73,7 @@ Dockerfile / docker-compose.yml # container (production.ini); c ## Data → training → inference flow -1. **Download:** `commands/download_isic_full.py` (resumable, multithreaded) or `fetch_isic_sample.py`. -2. **Manifest:** `commands/run_build_dataset.py` builds a patient/lesion-grouped, leakage-free manifest CSV. +1. **Source + curate (stage 1):** `commands/build_dataset.py` (→ `lesnet.data.pipeline`) downloads (ISIC malignant-maximal; PAD-UFES auto; Fitzpatrick17k/DDI on disk), canonicalises diagnosis names, quality-gates + pHash-dedupes, sorts into `benign/not_sure/malignant//`, balances ~1:1 benign:malignant (group-safe, fairness-aware), and writes a leakage-free `manifest.csv` + `report.json`. 3. **Train:** `commands/run_train_triage.py --manifest …` → `lesnet.ml.training.train` (EfficientNetV2-S, focal loss, optional metric-gated rounds) → calibrate (temperature) → choose sensitivity-first operating point → split-conformal → fit Mahalanobis OOD → save `triage_model.keras` + `artifacts.json`. 4. **Evaluate:** `commands/run_evaluate.py` writes `evaluation_report.json` + `model_card.md` (incl. fairness gate). 5. **Infer:** web `POST /predict` (lazy `TriagePredictor`) or `commands/run_triage_inference.py`. Pipeline = quality gate → OOD gate → calibrated triage → abstention/conformal set + auxiliary fine-grained prediction. `GET /labels` returns the class list. @@ -94,10 +97,14 @@ python -m pip install -e ".[testing]" # run the web app, http://localhost:6543 pserve development.ini --reload -# tests + lint +# tests + lint (lesnet/data is gated at 100% coverage) python -m pytest +python -m pytest --cov=lesnet.data --cov-fail-under=100 ruff check +# stage 1: source + build a balanced, sorted, leakage-safe dataset +python commands/build_dataset.py --sources isic pad_ufes_20 fitzpatrick17k ddi --dest data/dataset + # CPU smoke (synthetic end-to-end: train -> calibrate -> save -> load -> infer) python commands/run_train_triage.py --smoke --artifacts /tmp/lesnet_art @@ -110,6 +117,7 @@ python commands/download_model.py -m M-4s ## Git / contributing +- **Always branch off the latest `origin/main`** (`git fetch origin main && git checkout -B origin/main`) — never off a stale release/feature branch. `main` is the source of truth and may carry squash-merged PRs that raw branches lack. - CI (`.github/workflows/test.yaml`) runs `ruff check` + `pytest` on PRs to `main`; CodeQL also runs; Dependabot manages bumps. - Commit style: short subjects, often gitmoji + milestone/issue tag. Not strict Conventional Commits — match history. - `docs/model-redesign.md` is the authoritative design for the triage system. `docs/code-audit.md` is a historical (2026-06-09) audit of the now-removed legacy pipeline. diff --git a/commands/build_dataset.py b/commands/build_dataset.py new file mode 100644 index 0000000..00c3800 --- /dev/null +++ b/commands/build_dataset.py @@ -0,0 +1,80 @@ +"""Stage 1 — source, canonicalise, quality-gate, balance, and sort a training dataset. + + # malignant-maximal balanced build from ISIC (+ any sources present on disk): + python commands/build_dataset.py --sources isic pad_ufes_20 fitzpatrick17k ddi --dest data/dataset + +ISIC is sourced malignant-maximally: page the whole archive's metadata (cheap), then download +images only for all malignant + suspicious + a benign sample matched to the target ratio. +Other sources auto-download where the licence allows (DDI must be placed on disk manually). +""" +import argparse +import json +import os + +from lesnet.data import balance, sort +from lesnet.data.config import SourcingConfig +from lesnet.data.pipeline import run +from lesnet.data.sources import isic + + +def _isic_malignant_maximal_download(root, config): + """Populate ``root`` with ISIC metadata + a malignant-maximal, ratio-matched image set.""" + resolution = 'full' if config.full_resolution else 'thumbnail_256' + rows = isic.collect_metadata(root, resolution=resolution, limit=config.sample_limit) + url_by_id = {row['isic_id']: row['url'] for row in rows} + + annotated, _ = sort.annotate(isic.parse(root, config.sample_limit)) + by_bucket = {'benign': [], 'not_sure': [], 'malignant': []} + for record in annotated: + by_bucket[record.triage_bucket].append(record) + + benign_target = round(config.balance_ratio * len(by_bucket['malignant'])) + benign_keep = balance._select_groups(by_bucket['benign'], benign_target, config.seed) + selected = by_bucket['malignant'] + by_bucket['not_sure'] + benign_keep + selected_ids = {os.path.splitext(os.path.basename(r.image_path))[0] for r in selected} + + download_set = {isic_id: url_by_id[isic_id] for isic_id in selected_ids if isic_id in url_by_id} + print(f"ISIC: {len(annotated)} labelled · malignant={len(by_bucket['malignant'])} " + f"not_sure={len(by_bucket['not_sure'])} benign(kept)={len(benign_keep)} " + f"-> downloading {len(download_set)} images") + isic.download_images(root, download_set) + + +def main(): + parser = argparse.ArgumentParser(description="Build the LesNet triage dataset (stage 1).") + parser.add_argument('--sources', nargs='+', default=['isic'], + choices=['isic', 'pad_ufes_20', 'fitzpatrick17k', 'ddi']) + parser.add_argument('--dest', default='data/dataset') + parser.add_argument('--raw-dir', default='data/raw') + parser.add_argument('--isic-root') + parser.add_argument('--pad-ufes-root') + parser.add_argument('--fitzpatrick17k-root') + parser.add_argument('--ddi-root') + parser.add_argument('--sample-limit', type=int, default=None) + parser.add_argument('--ratio', type=float, default=1.0, help="benign:malignant target.") + parser.add_argument('--no-dedupe', action='store_true') + parser.add_argument('--thumbnails', action='store_true', help="ISIC thumbnails (faster, lower-res).") + parser.add_argument('--seed', type=int, default=42) + args = parser.parse_args() + + root_arg = {'isic': args.isic_root, 'pad_ufes_20': args.pad_ufes_root, + 'fitzpatrick17k': args.fitzpatrick17k_root, 'ddi': args.ddi_root} + roots = {name: root_arg[name] for name in args.sources if root_arg.get(name)} + config = SourcingConfig( + sources=tuple(args.sources), roots=roots, raw_dir=args.raw_dir, dest=args.dest, + sample_limit=args.sample_limit, full_resolution=not args.thumbnails, + dedupe=not args.no_dedupe, balance_ratio=args.ratio, seed=args.seed) + + if 'isic' in config.sources: + isic_root = config.roots.get('isic') or os.path.join(config.raw_dir, 'isic') + if not (os.path.isdir(isic_root) and os.listdir(isic_root)): + _isic_malignant_maximal_download(isic_root, config) + + _records, report = run(config) + print(json.dumps({key: report[key] for key in + ('final_total', 'by_bucket', 'benign_to_malignant_ratio', 'by_split')}, indent=2)) + print(f"Dataset + manifest written to {config.dest}") + + +if __name__ == '__main__': + main() diff --git a/commands/download_isic_full.py b/commands/download_isic_full.py deleted file mode 100644 index 956b447..0000000 --- a/commands/download_isic_full.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Download the ENTIRE ISIC archive (~800k images) — resumable and threaded. - -This is built to run for a long time on a real machine (it is NOT expected to finish in a -sandbox). It: - - pages the public ISIC v2 API following the `next` cursor, - - persists that cursor to /download_state.json so it resumes after interruption, - - skips images already on disk, - - appends rows to /metadata.csv in the schema the ISIC loader reads, - - downloads concurrently. - -Layout is flat (/.jpg + /metadata.csv) so run_build_dataset.py can -consume it directly with --isic-root . - - python commands/download_isic_full.py --out data/isic_full --workers 32 - # resolution: thumbnail_256 (default, ~tens of GB) or full (hundreds of GB+) - # resume: just re-run the same command; pass --restart to start over. -""" -import argparse -import csv -import json -import os -from concurrent.futures import ThreadPoolExecutor, as_completed - -import requests -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry - -API_URL = "https://api.isic-archive.com/api/v2/images/" -METADATA_FIELDS = ['isic_id', 'patient_id', 'diagnosis', 'diagnosis_1', - 'age_approx', 'sex', 'anatom_site_general'] - - -def _session(pool): - session = requests.Session() - retries = Retry( - total=8, connect=5, read=5, backoff_factor=2.0, - status_forcelist=[429, 500, 502, 503, 504, 520, 522, 524], - respect_retry_after_header=True, raise_on_status=False, - ) - adapter = HTTPAdapter(max_retries=retries, pool_maxsize=pool, pool_connections=pool) - session.mount('http://', adapter) - session.mount('https://', adapter) - return session - - -def _specific_diagnosis(clinical): - for key in ('diagnosis_3', 'diagnosis_2', 'diagnosis_1'): - if clinical.get(key): - return clinical[key] - return clinical.get('diagnosis') or '' - - -def _records_from_page(payload, resolution): - records = [] - for image in payload['results']: - clinical = image.get('metadata', {}).get('clinical', {}) - files = image.get('files', {}) - chosen = files.get(resolution) or files.get('thumbnail_256') or files.get('full') - url = (chosen or {}).get('url') - if not url: - continue - records.append({ - 'isic_id': image['isic_id'], 'url': url, - 'patient_id': clinical.get('patient_id') or image.get('patient_id') or '', - 'diagnosis': _specific_diagnosis(clinical), - 'diagnosis_1': clinical.get('diagnosis_1') or '', - 'age_approx': clinical.get('age_approx', ''), - 'sex': clinical.get('sex', ''), - 'anatom_site_general': clinical.get('anatom_site_1', ''), - }) - return records - - -def main(): - parser = argparse.ArgumentParser(description="Resumable full ISIC archive downloader.") - parser.add_argument('--out', default='data/isic_full') - parser.add_argument('--resolution', default='thumbnail_256', choices=['thumbnail_256', 'full']) - parser.add_argument('--workers', type=int, default=64) - parser.add_argument('--page-size', type=int, default=100) - parser.add_argument('--max', type=int, default=None, help="Optional cap (omit to fetch all).") - parser.add_argument('--restart', action='store_true', help="Ignore saved state and start over.") - args = parser.parse_args() - - os.makedirs(args.out, exist_ok=True) - state_path = os.path.join(args.out, 'download_state.json') - metadata_path = os.path.join(args.out, 'metadata.csv') - session = _session(args.workers) - - next_url, params, downloaded, pages = API_URL, {'limit': args.page_size}, 0, 0 - if os.path.exists(state_path) and not args.restart: - state = json.load(open(state_path, encoding='utf-8')) - next_url = state.get('next_url') or API_URL - params = None if state.get('next_url') else {'limit': args.page_size} - downloaded, pages = state.get('downloaded', 0), state.get('pages', 0) - print(f"Resuming: {downloaded} downloaded across {pages} pages.") - - writer_is_new = not os.path.exists(metadata_path) - metadata_file = open(metadata_path, 'a', newline='', encoding='utf-8') - writer = csv.DictWriter(metadata_file, fieldnames=METADATA_FIELDS) - if writer_is_new: - writer.writeheader() - - def _fetch(record): - path = os.path.join(args.out, f"{record['isic_id']}.jpg") - if os.path.exists(path): - return record, False # already have it - response = session.get(record['url'], timeout=120) - if response.status_code == 200: - with open(path, 'wb') as handle: - handle.write(response.content) - return record, True - return record, None # failed - - try: - while next_url and (args.max is None or downloaded < args.max): - response = session.get(next_url, params=params, timeout=120) - params = None - if response.status_code != 200: - print(f"API error {response.status_code}; stopping.") - break - payload = response.json() - records = _records_from_page(payload, args.resolution) - - with ThreadPoolExecutor(max_workers=args.workers) as executor: - futures = {executor.submit(_fetch, record): record for record in records} - for future in as_completed(futures): - record, is_new = future.result() - if is_new: - downloaded += 1 - writer.writerow({field: record.get(field, '') for field in METADATA_FIELDS}) - - pages += 1 - metadata_file.flush() - next_url = payload.get('next') - json.dump({'next_url': next_url, 'downloaded': downloaded, 'pages': pages}, - open(state_path, 'w', encoding='utf-8')) - print(f"page {pages}: {downloaded} images downloaded so far", flush=True) - finally: - metadata_file.close() - - print(f"Done. {downloaded} images in {args.out}") - - -if __name__ == '__main__': - main() diff --git a/commands/fetch_isic_sample.py b/commands/fetch_isic_sample.py deleted file mode 100644 index 9cd55f1..0000000 --- a/commands/fetch_isic_sample.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Fetch a small REAL sample from the public ISIC v2 API for development/testing. - - python commands/fetch_isic_sample.py --out data/isic --max 400 - -Writes /.jpg thumbnails + /metadata.csv in the schema the ISIC loader -reads. Uses the diagnosis hierarchy (most specific available) as the label. -""" -import argparse -import csv -import os -from concurrent.futures import ThreadPoolExecutor, as_completed - -import requests - -API_URL = "https://api.isic-archive.com/api/v2/images/" - - -def _collect(session, max_images, page_size): - images = [] - url, params = API_URL, {'limit': page_size} - while url and len(images) < max_images: - response = session.get(url, params=params, timeout=60) - params = None # subsequent 'next' URLs already carry query params - if response.status_code != 200: - break - payload = response.json() - for image in payload['results']: - clinical = image.get('metadata', {}).get('clinical', {}) - diagnosis = (clinical.get('diagnosis_3') or clinical.get('diagnosis_2') - or clinical.get('diagnosis_1')) - thumbnail = image.get('files', {}).get('thumbnail_256', {}).get('url') - if not diagnosis or not thumbnail: - continue - images.append({ - 'isic_id': image['isic_id'], 'url': thumbnail, 'diagnosis': diagnosis, - 'age_approx': clinical.get('age_approx', ''), 'sex': clinical.get('sex', ''), - 'anatom_site_general': clinical.get('anatom_site_1', ''), - }) - if len(images) >= max_images: - break - url = payload.get('next') - return images - - -def main(): - parser = argparse.ArgumentParser(description="Download a small real ISIC sample.") - parser.add_argument('--out', default='data/isic') - parser.add_argument('--max', type=int, default=400) - parser.add_argument('--page-size', type=int, default=100) - args = parser.parse_args() - - os.makedirs(args.out, exist_ok=True) - session = requests.Session() - images = _collect(session, args.max, args.page_size) - - def _download(item): - response = session.get(item['url'], timeout=60) - if response.status_code == 200: - with open(os.path.join(args.out, f"{item['isic_id']}.jpg"), 'wb') as handle: - handle.write(response.content) - return item - return None - - downloaded = [] - with ThreadPoolExecutor(max_workers=16) as executor: - futures = {executor.submit(_download, item): item for item in images} - for future in as_completed(futures): - result = future.result() - if result: - downloaded.append(result) - - fields = ['isic_id', 'patient_id', 'diagnosis', 'age_approx', 'sex', 'anatom_site_general'] - with open(os.path.join(args.out, 'metadata.csv'), 'w', newline='', encoding='utf-8') as handle: - writer = csv.DictWriter(handle, fieldnames=fields) - writer.writeheader() - for item in downloaded: - writer.writerow({ - 'isic_id': item['isic_id'], 'patient_id': '', 'diagnosis': item['diagnosis'], - 'age_approx': item['age_approx'], 'sex': item['sex'], - 'anatom_site_general': item['anatom_site_general'], - }) - print(f"Downloaded {len(downloaded)} real ISIC images to {args.out}") - - -if __name__ == '__main__': - main() diff --git a/commands/run_build_dataset.py b/commands/run_build_dataset.py deleted file mode 100644 index 5f606cc..0000000 --- a/commands/run_build_dataset.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Build the unified, grouped-split dataset manifest (paper §5.3, stage 1). - -Fast iteration: python commands/run_build_dataset.py --mode sample --sample-size 200 \ - --datasets isic pad_ufes_20 --isic-root data/isic --pad-ufes-root data/pad_ufes -Full training: python commands/run_build_dataset.py --mode full --datasets isic pad_ufes_20 fitzpatrick17k ddi ... -""" -import argparse -from collections import Counter - -from lesnet.ml.datasets import DatasetConfig, assign_splits, build_manifest, save_manifest - - -def main(): - parser = argparse.ArgumentParser(description="Build the LesNet dataset manifest with grouped splits.") - parser.add_argument('--mode', choices=['sample', 'full'], default='sample') - parser.add_argument('--sample-size', type=int, default=200, help="Rows per dataset in sample mode.") - parser.add_argument('--datasets', nargs='+', default=['isic'], - choices=['isic', 'pad_ufes_20', 'fitzpatrick17k', 'ddi']) - parser.add_argument('--isic-root') - parser.add_argument('--pad-ufes-root') - parser.add_argument('--fitzpatrick17k-root') - parser.add_argument('--ddi-root') - parser.add_argument('--output', default='data/manifest.csv') - parser.add_argument('--seed', type=int, default=42) - args = parser.parse_args() - - config = DatasetConfig( - mode=args.mode, - sample_size_per_dataset=args.sample_size, - datasets=tuple(args.datasets), - isic_root=args.isic_root, - pad_ufes_root=args.pad_ufes_root, - fitzpatrick17k_root=args.fitzpatrick17k_root, - ddi_root=args.ddi_root, - seed=args.seed, - ) - - records = assign_splits(build_manifest(config), config) - save_manifest(records, args.output) - - by_split = Counter(record.split for record in records) - by_dataset = Counter(record.source_dataset for record in records) - print(f"Wrote {len(records)} records to {args.output} (mode={args.mode}).") - print(f" by split: {dict(by_split)}") - print(f" by dataset: {dict(by_dataset)}") - - -if __name__ == '__main__': - main() diff --git a/commands/run_evaluate.py b/commands/run_evaluate.py index c43902a..062d8fa 100644 --- a/commands/run_evaluate.py +++ b/commands/run_evaluate.py @@ -13,7 +13,7 @@ from lesnet.ml.evaluation import build_report, write_model_card from lesnet.ml.features import normalize_site from lesnet.ml.model import triage_logits_model -from lesnet.ml.datasets import load_manifest +from lesnet.data.records import load_manifest import tensorflow as tf diff --git a/commands/run_train_triage.py b/commands/run_train_triage.py index 74be2be..9abae1a 100644 --- a/commands/run_train_triage.py +++ b/commands/run_train_triage.py @@ -8,8 +8,9 @@ import os import tempfile +from lesnet.data.manifest import assign_splits +from lesnet.data.records import load_manifest, save_manifest from lesnet.ml.config import PipelineConfig -from lesnet.ml.datasets import DatasetConfig, assign_splits, load_manifest, save_manifest from lesnet.ml.synthetic import make_synthetic_records from lesnet.ml.training import train @@ -44,7 +45,7 @@ def main(): image_size=(64, 64), backbone='tiny', pretrained=False, batch_size=16, epochs=4, shared_units=64, artifacts_dir=args.artifacts, smoke=True, remove_hair=False) records = make_synthetic_records(tempfile.mkdtemp(prefix='lesnet_smoke_'), per_class=40) - records = assign_splits(records, DatasetConfig(test_size=0.2, val_size=0.2, seed=config.seed)) + records = assign_splits(records, test_size=0.2, val_size=0.2, seed=config.seed) os.makedirs(args.artifacts, exist_ok=True) save_manifest(records, os.path.join(args.artifacts, 'smoke_manifest.csv')) else: diff --git a/commands/run_triage_inference.py b/commands/run_triage_inference.py index e899b1d..3c3f3bd 100644 --- a/commands/run_triage_inference.py +++ b/commands/run_triage_inference.py @@ -9,7 +9,7 @@ import numpy as np from PIL import Image -from lesnet.ml.datasets import LesionRecord +from lesnet.data.records import LesionRecord from lesnet.ml.inference import TriagePredictor diff --git a/commands/run_validation_check.py b/commands/run_validation_check.py index 2761280..26ce43b 100644 --- a/commands/run_validation_check.py +++ b/commands/run_validation_check.py @@ -5,7 +5,7 @@ import numpy as np from PIL import Image -from lesnet.ml.datasets import LesionRecord +from lesnet.data.records import LesionRecord from lesnet.ml.inference import TriagePredictor diff --git a/lesnet/data/__init__.py b/lesnet/data/__init__.py new file mode 100644 index 0000000..3b72f68 --- /dev/null +++ b/lesnet/data/__init__.py @@ -0,0 +1,7 @@ +"""LesNet data subsystem (4.5.0). + +Single-responsibility modules that source raw dermatology data, canonicalise diagnosis +names, gate quality + remove near-duplicates, sort into clinical buckets +(benign / not_sure / malignant, per-diagnosis subfolders), balance to a fair benign:malignant +ratio, and emit a leakage-safe manifest. Pure-logic + IO — no TensorFlow. +""" diff --git a/lesnet/data/balance.py b/lesnet/data/balance.py new file mode 100644 index 0000000..7d55d62 --- /dev/null +++ b/lesnet/data/balance.py @@ -0,0 +1,97 @@ +"""Fair, group-safe balancing of the decision-critical buckets (stage 1). + +Goal: a ~1:1 benign:malignant set where no single diagnosis dominates its bucket. Two +invariants make the metrics trustworthy and the model fair: + * group-safe — we drop whole patient/lesion groups, never split one across the cut; + * fairness-aware — when downsampling we retain scarcer high-Fitzpatrick (darker-skin) + groups first, so balancing doesn't wash out skin-tone diversity. +'not_sure'/suspicious is kept as-is (not forced into the ratio). +""" +import zlib + + +def bucket_counts(records): + counts = {} + for record in records: + counts[record.triage_bucket] = counts.get(record.triage_bucket, 0) + 1 + return counts + + +def _group_order_key(seed, group_id, records): + """Sort key: retain higher max-Fitzpatrick groups first; deterministic tie-break.""" + max_fitzpatrick = max((record.fitzpatrick or 0) for record in records) + tiebreak = zlib.crc32(f"{seed}:{group_id}".encode()) + return (-max_fitzpatrick, tiebreak) + + +def _select_groups(records, target, seed): + """Keep whole groups (no splitting) up to ``target`` rows, fairness-prioritised.""" + if target <= 0: + return [] + if len(records) <= target: + return list(records) + groups = {} + for record in records: + groups.setdefault(record.group_id, []).append(record) + ordered = sorted(groups.items(), key=lambda item: _group_order_key(seed, item[0], item[1])) + kept = [] + for _group_id, group_records in ordered: + if len(kept) + len(group_records) <= target: + kept.extend(group_records) + return kept + + +def _balance_bucket(records, target, cap_fraction, seed): + """Pick ~``target`` rows, limiting any single diagnosis to ``cap_fraction`` of the bucket + while there's diversity to spare, then water-filling from the rest to reach target. + + So a bucket with one diagnosis (e.g. all-melanoma malignant) still reaches target, but a + mixed bucket (benign) won't let one diagnosis (nevus) dominate. + """ + if target <= 0: + return [] + if len(records) <= target: + return list(records) + by_diagnosis = {} + for record in records: + by_diagnosis.setdefault(record.diagnosis, []).append(record) + cap = max(1, int(cap_fraction * target)) + + selected = [] + for diagnosis_records in by_diagnosis.values(): + selected.extend(_select_groups(diagnosis_records, min(cap, target), seed)) + if len(selected) < target: + chosen = {id(record) for record in selected} + leftovers = [record for record in records if id(record) not in chosen] + selected.extend(_select_groups(leftovers, target - len(selected), seed)) + if len(selected) > target: + selected = _select_groups(selected, target, seed) + return selected + + +def _targets(counts, ratio): + """benign/malignant keep-targets for the desired benign:malignant ratio.""" + benign, malignant = counts.get('benign', 0), counts.get('malignant', 0) + if benign == 0 or malignant == 0: + return {'benign': benign, 'malignant': malignant} + desired_benign = round(ratio * malignant) + if benign > desired_benign: + return {'benign': desired_benign, 'malignant': malignant} + return {'benign': benign, 'malignant': round(benign / ratio)} + + +def balance(records, ratio=1.0, per_diagnosis_cap_fraction=0.6, + buckets=('benign', 'malignant'), seed=42): + by_bucket = {} + for record in records: + by_bucket.setdefault(record.triage_bucket, []).append(record) + + result = [record for bucket, recs in by_bucket.items() + if bucket not in buckets for record in recs] # untouched buckets (e.g. not_sure) + + targets = _targets(bucket_counts(records), ratio) + for bucket in buckets: + recs = by_bucket.get(bucket, []) + target = targets.get(bucket, len(recs)) + result.extend(_balance_bucket(recs, target, per_diagnosis_cap_fraction, seed)) + return result diff --git a/lesnet/data/canonical.py b/lesnet/data/canonical.py new file mode 100644 index 0000000..9f25637 --- /dev/null +++ b/lesnet/data/canonical.py @@ -0,0 +1,76 @@ +"""Curated diagnosis-name canonicalisation (stage 1, post-download reconciliation). + +Different sources spell the same diagnosis differently ("melanoma" / "Melanoma, NOS" / +"malignant melanoma"; "nevus" / "naevus" / "melanocytic nevus"). We collapse variants to a +single canonical folder name via a CURATED, ordered map — not fuzzy auto-merge — so the +mapping is reviewable and stable. Unmapped labels fall back to a slug of the raw name and +are reported (``unmapped_diagnoses``) so the map can be extended by hand. + +Order matters: more-specific / malignant canonicals are listed first so e.g. +"lentigo maligna melanoma" maps to ``melanoma`` (not ``lentigo``). +""" +import re + +# (canonical_name, (substring terms that identify it,)) — checked in order, first match wins. +CANONICAL_DIAGNOSES = ( + # --- malignant --- + ('melanoma', ('melanoma', 'mel ', 'malignant melanocytic')), + ('basal cell carcinoma', ('basal cell carcinoma', 'basal cell ca', 'bcc')), + ('squamous cell carcinoma', ('squamous cell carcinoma', 'squamous cell ca', 'scc')), + ('merkel cell carcinoma', ('merkel',)), + ('angiosarcoma', ('angiosarcoma',)), + ('mycosis fungoides', ('mycosis fungoides',)), + # --- suspicious / indeterminate --- + ('actinic keratosis', ('actinic keratosis', 'actinic keratoses', 'solar keratosis')), + ('bowen disease', ('bowen', 'in situ')), + ('dysplastic nevus', ('dysplastic', 'atypical')), + ('lentigo maligna', ('lentigo maligna',)), + ('spitz nevus', ('spitz',)), + # --- benign --- + ('seborrheic keratosis', ('seborrheic keratosis', 'seborrhoeic keratosis', 'seborrheic')), + ('dermatofibroma', ('dermatofibroma',)), + ('vascular lesion', ('vascular', 'hemangioma', 'haemangioma', 'angioma', 'angiokeratoma', + 'pyogenic granuloma')), + ('acrochordon', ('acrochordon', 'skin tag', 'fibroepithelial polyp')), + ('solar lentigo', ('solar lentigo', 'lentigo simplex', 'lentigo')), + ('cafe-au-lait macule', ('cafe-au-lait', 'cafe au lait')), + ('wart', ('wart', 'verruca')), + ('molluscum', ('molluscum',)), + ('nevus', ('nevus', 'naevus', 'nevi')), # generic nevus last (most permissive) +) + + +def slugify(raw_label): + """Filesystem-safe slug for an unmapped diagnosis (lowercase, underscores).""" + slug = re.sub(r'[^a-z0-9]+', '_', str(raw_label).strip().lower()).strip('_') + return slug or 'unknown' + + +def canonical_diagnosis(raw_label): + """Map a raw diagnosis string to its curated canonical name, or None if unmapped.""" + if raw_label is None: + return None + text = str(raw_label).strip().lower() + if not text or text in {'unknown', 'nan', 'none'}: + return None + for canonical, terms in CANONICAL_DIAGNOSES: + if any(term in text for term in terms): + return canonical + return None + + +def canonical_or_slug(raw_label): + """Canonical name if known, else a slug of the raw label (so every record gets a folder).""" + return canonical_diagnosis(raw_label) or slugify(raw_label) + + +def unmapped_diagnoses(raw_labels): + """Sorted unique raw labels that have no curated canonical mapping (for human review).""" + seen = set() + for raw_label in raw_labels: + if raw_label is None: + continue + text = str(raw_label).strip() + if text and text.lower() not in {'unknown', 'nan', 'none'} and canonical_diagnosis(text) is None: + seen.add(text) + return sorted(seen) diff --git a/lesnet/data/config.py b/lesnet/data/config.py new file mode 100644 index 0000000..aa37f5f --- /dev/null +++ b/lesnet/data/config.py @@ -0,0 +1,31 @@ +"""Configuration for the data-sourcing pipeline (stage 1).""" +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class SourcingConfig: + # which sources to use + where their raw downloads live + sources: tuple = ('isic',) # any of: isic, pad_ufes_20, fitzpatrick17k, ddi + roots: dict = field(default_factory=dict) # {source_name: root_path} + raw_dir: str = 'data/raw' # where auto-downloads land + dest: str = 'data/dataset' # sorted-folder + manifest output + + # download scope + sample_limit: Optional[int] = None # cap rows per source (None = all) + full_resolution: bool = True # ISIC: full-res images, not thumbnails + + # quality + dedup + min_image_pixels: int = 64 # reject images smaller than this on the short side + dedupe: bool = True # collapse perceptual near-duplicates + phash_distance: int = 4 # Hamming distance under which two images are "the same" + + # balancing (decision-critical buckets) + balance_ratio: float = 1.0 # benign : malignant target (1.0 = 1:1) + per_diagnosis_cap_fraction: float = 0.6 # no single diagnosis may exceed this share of its bucket + balance_buckets: tuple = ('benign', 'malignant') # 'not_sure' kept as-is + + # leakage-safe splits + test_size: float = 0.15 + val_size: float = 0.15 + seed: int = 42 diff --git a/lesnet/data/manifest.py b/lesnet/data/manifest.py new file mode 100644 index 0000000..09b491e --- /dev/null +++ b/lesnet/data/manifest.py @@ -0,0 +1,25 @@ +"""Manifest assembly: grouped leakage-safe splits + CSV write (stage 1).""" +import numpy as np + +from lesnet.data.records import save_manifest +from lesnet.data.splits import assert_no_group_leakage, grouped_train_val_test + + +def assign_splits(records, test_size=0.15, val_size=0.15, seed=42): + """Assign train/val/test with no patient/lesion group shared across splits.""" + if not records: + return records + groups = np.array([record.group_id for record in records]) + train_index, val_index, test_index = grouped_train_val_test(groups, test_size, val_size, seed) + assert_no_group_leakage(groups, train_index, val_index, test_index) + for index in train_index: + records[index].split = 'train' + for index in val_index: + records[index].split = 'val' + for index in test_index: + records[index].split = 'test' + return records + + +def write(records, path): + save_manifest(records, path) diff --git a/lesnet/data/pipeline.py b/lesnet/data/pipeline.py new file mode 100644 index 0000000..a2dd32c --- /dev/null +++ b/lesnet/data/pipeline.py @@ -0,0 +1,91 @@ +"""Stage-1 orchestrator: raw sources -> clean, balanced, sorted, leakage-safe dataset. + +process(): records (images on disk) -> annotate -> quality gate -> dedupe -> balance -> + materialise into folders -> grouped splits -> manifest + report. (No network.) +ingest(): per source, ensure raw present (auto-download where allowed) and parse to records. +run(): ingest then process. +""" +import json +import os + +from lesnet.data import balance as balance_module +from lesnet.data import manifest as manifest_module +from lesnet.data import quality, sort +from lesnet.data.canonical import unmapped_diagnoses +from lesnet.data.sources.registry import get_source + + +def _distribution(records, attribute): + counts = {} + for record in records: + key = str(getattr(record, attribute)) + counts[key] = counts.get(key, 0) + 1 + return dict(sorted(counts.items())) + + +def build_report(stats, kept): + report = dict(stats) + report['final_total'] = len(kept) + report['by_bucket'] = balance_module.bucket_counts(kept) + report['by_diagnosis'] = _distribution(kept, 'diagnosis') + report['by_source'] = _distribution(kept, 'source_dataset') + report['by_fitzpatrick'] = _distribution(kept, 'fitzpatrick') + report['by_split'] = _distribution(kept, 'split') + benign, malignant = report['by_bucket'].get('benign', 0), report['by_bucket'].get('malignant', 0) + report['benign_to_malignant_ratio'] = round(benign / malignant, 3) if malignant else None + return report + + +def process(records, config): + os.makedirs(config.dest, exist_ok=True) + stats = {'ingested': len(records)} + + annotated, unmappable = sort.annotate(records) + stats['dropped_unmappable'] = len(unmappable) + + # Separate "image not on disk" (expected when a source is fetched selectively) from + # "downloaded but corrupt/too small" so the report is honest about each. + present = [record for record in annotated if os.path.exists(record.image_path)] + stats['dropped_no_image'] = len(annotated) - len(present) + quality_kept, low_quality = quality.filter_quality(present, config.min_image_pixels) + stats['dropped_low_quality'] = len(low_quality) + + if config.dedupe: + deduped, duplicates = quality.dedupe(quality_kept, config.phash_distance) + stats['dropped_duplicate'] = len(duplicates) + else: + deduped, stats['dropped_duplicate'] = quality_kept, 0 + + balanced = balance_module.balance( + deduped, config.balance_ratio, config.per_diagnosis_cap_fraction, + config.balance_buckets, config.seed) + + materialised, missing = sort.materialise(balanced, config.dest) + stats['dropped_missing_image'] = len(missing) + + manifest_module.assign_splits(materialised, config.test_size, config.val_size, config.seed) + manifest_module.write(materialised, os.path.join(config.dest, 'manifest.csv')) + + report = build_report(stats, materialised) + with open(os.path.join(config.dest, 'report.json'), 'w', encoding='utf-8') as handle: + json.dump(report, handle, indent=2) + with open(os.path.join(config.dest, 'unmapped_diagnoses.txt'), 'w', encoding='utf-8') as handle: + handle.write("\n".join(unmapped_diagnoses(record.raw_label for record in records)) + "\n") + return materialised, report + + +def ingest(config): + """Ensure each source's raw data is present (auto-download where possible) and parse it.""" + records = [] + for name in config.sources: + spec = get_source(name) + root = config.roots.get(name) or os.path.join(config.raw_dir, name) + metadata_present = os.path.isdir(root) and os.listdir(root) + if not metadata_present and spec.download is not None: + spec.download(root, config.sample_limit) + records.extend(spec.parse(root, config.sample_limit)) + return records + + +def run(config): + return process(ingest(config), config) diff --git a/lesnet/data/quality.py b/lesnet/data/quality.py new file mode 100644 index 0000000..a333d2a --- /dev/null +++ b/lesnet/data/quality.py @@ -0,0 +1,76 @@ +"""Image quality gate + perceptual-hash near-duplicate removal (stage 1). + +Bad/duplicate data is the fastest way to inflate validation scores: a corrupt image trains +on noise, and a near-duplicate that lands in both train and test leaks the answer. We drop +unreadable/too-small images and collapse perceptual near-duplicates (keeping one +representative) BEFORE splitting, so reported metrics are trustworthy. + +The hash function is injectable so the logic is testable without real images. +""" +import numpy as np +from PIL import Image + + +def image_short_side(path): + """Shorter side length in pixels, or None if the image can't be read.""" + try: + with Image.open(path) as image: + image.verify() + with Image.open(path) as image: + width, height = image.size + return min(width, height) + except Exception: # noqa: BLE001 - any decode/IO failure means "unusable image" + return None + + +def passes_quality(path, min_pixels=64): + short_side = image_short_side(path) + return short_side is not None and short_side >= min_pixels + + +def average_hash(path, hash_size=8): + """64-bit perceptual average-hash, or None if unreadable.""" + try: + with Image.open(path) as image: + grayscale = image.convert('L').resize((hash_size, hash_size), Image.BILINEAR) + pixels = np.asarray(grayscale, dtype=np.float64) + except Exception: # noqa: BLE001 - unreadable image has no hash + return None + bits = pixels > pixels.mean() + value = 0 + for bit in bits.flatten(): + value = (value << 1) | int(bit) + return value + + +def hamming_distance(left, right): + return bin(left ^ right).count('1') + + +def filter_quality(records, min_pixels=64): + """Split records into (kept, dropped) by the readability/size gate.""" + kept, dropped = [], [] + for record in records: + (kept if passes_quality(record.image_path, min_pixels) else dropped).append(record) + return kept, dropped + + +def dedupe(records, max_distance=4, hash_fn=average_hash): + """Greedily drop perceptual near-duplicates, keeping the first representative. + + Returns (kept, dropped). Records whose image can't be hashed are kept (the quality gate + is responsible for unreadable images, not this step). + """ + kept, dropped = [], [] + representatives = [] # list of kept hashes + for record in records: + digest = hash_fn(record.image_path) + if digest is None: + kept.append(record) + continue + if any(hamming_distance(digest, seen) <= max_distance for seen in representatives): + dropped.append(record) + else: + representatives.append(digest) + kept.append(record) + return kept, dropped diff --git a/lesnet/data/records.py b/lesnet/data/records.py new file mode 100644 index 0000000..3307c92 --- /dev/null +++ b/lesnet/data/records.py @@ -0,0 +1,74 @@ +"""The canonical dataset record + manifest IO (paper §5.3). + +``LesionRecord`` is the one row type every data source maps into. The manifest is a CSV of +these rows with grouped train/val/test splits already assigned; training/inference read it +back with ``load_manifest``. ``triage_bucket`` (benign/not_sure/malignant) and the canonical +``diagnosis`` are filled by the sorting stage and are optional so older manifests still load. +""" +import csv +from dataclasses import dataclass, fields +from typing import Optional + + +@dataclass +class LesionRecord: + image_path: str + source_dataset: str + raw_label: str + group_id: str # patient/lesion id — the grouped-split key + fitzpatrick: Optional[int] = None + anatomical_site: Optional[str] = None + age: Optional[float] = None + sex: Optional[str] = None + split: Optional[str] = None + triage_bucket: Optional[str] = None # benign | not_sure | malignant (set by sorting) + diagnosis: Optional[str] = None # canonical diagnosis (set by sorting) + + +MANIFEST_FIELDS = [field.name for field in fields(LesionRecord)] + + +def read_csv_rows(path): + with open(path, newline='', encoding='utf-8') as handle: + return list(csv.DictReader(handle)) + + +def to_float(value): + try: + return float(value) + except (TypeError, ValueError): + return None + + +def to_int(value): + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + +def save_manifest(records, path): + with open(path, 'w', newline='', encoding='utf-8') as handle: + writer = csv.DictWriter(handle, fieldnames=MANIFEST_FIELDS) + writer.writeheader() + for record in records: + writer.writerow({name: getattr(record, name) for name in MANIFEST_FIELDS}) + + +def load_manifest(path): + records = [] + for row in read_csv_rows(path): + records.append(LesionRecord( + image_path=row['image_path'], + source_dataset=row['source_dataset'], + raw_label=row['raw_label'], + group_id=row['group_id'], + fitzpatrick=to_int(row.get('fitzpatrick')), + anatomical_site=row.get('anatomical_site') or None, + age=to_float(row.get('age')), + sex=row.get('sex') or None, + split=row.get('split') or None, + triage_bucket=row.get('triage_bucket') or None, + diagnosis=row.get('diagnosis') or None, + )) + return records diff --git a/lesnet/data/sort.py b/lesnet/data/sort.py new file mode 100644 index 0000000..036994f --- /dev/null +++ b/lesnet/data/sort.py @@ -0,0 +1,65 @@ +"""Sort records into the clinical folder taxonomy (stage 1). + +benign / not_sure / malignant -> / -> image files. +Bucket comes from the triage taxonomy; the diagnosis subfolder from curated canonicalisation. +Records that map to no triage bucket are dropped (and counted). +""" +import os +import shutil + +from lesnet.data.canonical import canonical_or_slug +from lesnet.data.taxonomy import BENIGN, MALIGNANT, SUSPICIOUS, triage_index + +BUCKET_BY_INDEX = {BENIGN: 'benign', SUSPICIOUS: 'not_sure', MALIGNANT: 'malignant'} +BUCKETS = ('benign', 'not_sure', 'malignant') + + +def bucket_for(raw_label): + """Clinical bucket name for a raw label, or None if unmappable.""" + index = triage_index(raw_label) + return BUCKET_BY_INDEX.get(index) if index is not None else None + + +def annotate(records): + """Set triage_bucket + canonical diagnosis on each record; drop unmappable ones. + + Returns (kept, dropped). + """ + kept, dropped = [], [] + for record in records: + bucket = bucket_for(record.raw_label) + if bucket is None: + dropped.append(record) + continue + record.triage_bucket = bucket + record.diagnosis = canonical_or_slug(record.raw_label) + kept.append(record) + return kept, dropped + + +def _dest_filename(record): + """Source-prefixed filename so ids from different datasets never collide.""" + return f"{record.source_dataset}_{os.path.basename(record.image_path)}" + + +def materialise(records, dest, link=False): + """Copy (or symlink) each record's image into dest/// and repoint it. + + Records whose source image is missing are skipped (and counted). Returns (materialised, missing). + """ + materialised, missing = [], [] + for record in records: + if not os.path.exists(record.image_path): + missing.append(record) + continue + folder = os.path.join(dest, record.triage_bucket, record.diagnosis) + os.makedirs(folder, exist_ok=True) + target = os.path.join(folder, _dest_filename(record)) + if link: + if not os.path.exists(target): + os.symlink(os.path.abspath(record.image_path), target) + else: + shutil.copy2(record.image_path, target) + record.image_path = target + materialised.append(record) + return materialised, missing diff --git a/lesnet/data/sources/__init__.py b/lesnet/data/sources/__init__.py new file mode 100644 index 0000000..773a909 --- /dev/null +++ b/lesnet/data/sources/__init__.py @@ -0,0 +1,3 @@ +"""Per-dataset sources: each maps a raw download into LesionRecords (and downloads it where +the licence allows automation). See ``registry.SOURCE_REGISTRY``. +""" diff --git a/lesnet/data/sources/base.py b/lesnet/data/sources/base.py new file mode 100644 index 0000000..4f0af97 --- /dev/null +++ b/lesnet/data/sources/base.py @@ -0,0 +1,17 @@ +"""Source contract shared by every dataset loader. + +A source maps a raw on-disk download into ``LesionRecord`` rows (``parse``); some can also +fetch that download over the network (``download``), others (licence-gated) must be placed +on disk manually and only parse. +""" +from dataclasses import dataclass +from typing import Callable, Optional + + +@dataclass +class SourceSpec: + name: str + parse: Callable # (root, limit) -> list[LesionRecord] + download: Optional[Callable] = None # (root, limit) -> None ; None if not automatable + requires_manual_download: bool = False + note: str = '' diff --git a/lesnet/data/sources/ddi.py b/lesnet/data/sources/ddi.py new file mode 100644 index 0000000..dd48c1b --- /dev/null +++ b/lesnet/data/sources/ddi.py @@ -0,0 +1,33 @@ +"""DDI (Diverse Dermatology Images) source — best dark-skin coverage, biopsy-confirmed. + +Stanford requires a Research Use Agreement, so there is no automated download; place the +release at ``root`` (``ddi_metadata.csv`` + image files) and this parses it. +""" +import os + +from lesnet.data.records import LesionRecord, read_csv_rows + +# DDI encodes skin tone as 12/34/56 (Fitzpatrick pairs); map to a representative band. +_SKIN_TONE_TO_FITZPATRICK = {'12': 1, '34': 3, '56': 5} + + +def skin_tone_to_fitzpatrick(skin_tone): + return _SKIN_TONE_TO_FITZPATRICK.get(str(skin_tone).strip(), None) + + +def parse(root, limit=None): + rows = read_csv_rows(os.path.join(root, 'ddi_metadata.csv')) + rows = rows[:limit] if limit else rows + records = [] + for row in rows: + image_file = row.get('DDI_file') + if not image_file: + continue + records.append(LesionRecord( + image_path=os.path.join(root, image_file), + source_dataset='ddi', + raw_label=row.get('disease') or 'unknown', + group_id=image_file, + fitzpatrick=skin_tone_to_fitzpatrick(row.get('skin_tone')), + )) + return records diff --git a/lesnet/data/sources/fitzpatrick17k.py b/lesnet/data/sources/fitzpatrick17k.py new file mode 100644 index 0000000..46b45ec --- /dev/null +++ b/lesnet/data/sources/fitzpatrick17k.py @@ -0,0 +1,52 @@ +"""Fitzpatrick17k source — clinical images with Fitzpatrick skin-type labels. + +The dataset ships a CSV (``fitzpatrick17k.csv``) with image URLs; ``download`` best-effort +fetches them. Many links rot, so missing images are tolerated and dropped later by the +quality gate. +""" +import os + +import requests + +from lesnet.data.records import LesionRecord, read_csv_rows, to_int + + +def download(root, limit=None, session=None): + """Best-effort fetch of images referenced by a 'url' column into /images/.""" + rows = read_csv_rows(os.path.join(root, 'fitzpatrick17k.csv')) + rows = rows[:limit] if limit else rows + images_dir = os.path.join(root, 'images') + os.makedirs(images_dir, exist_ok=True) + session = session or requests + for row in rows: + key, url = row.get('md5hash'), row.get('url') + if not key or not url: + continue + path = os.path.join(images_dir, f"{key}.jpg") + if os.path.exists(path): + continue + try: + response = session.get(url, timeout=60) + except Exception: # noqa: BLE001 - rotten link; quality gate drops the missing image + continue + if response.status_code == 200: + with open(path, 'wb') as handle: + handle.write(response.content) + + +def parse(root, limit=None): + rows = read_csv_rows(os.path.join(root, 'fitzpatrick17k.csv')) + rows = rows[:limit] if limit else rows + records = [] + for row in rows: + key = row.get('md5hash') + if not key: + continue + records.append(LesionRecord( + image_path=os.path.join(root, 'images', f"{key}.jpg"), + source_dataset='fitzpatrick17k', + raw_label=row.get('label') or 'unknown', + group_id=key, # no patient grouping -> per-image + fitzpatrick=to_int(row.get('fitzpatrick_scale') or row.get('fitzpatrick')), + )) + return records diff --git a/lesnet/data/sources/isic.py b/lesnet/data/sources/isic.py new file mode 100644 index 0000000..80efba6 --- /dev/null +++ b/lesnet/data/sources/isic.py @@ -0,0 +1,137 @@ +"""ISIC Archive v2 source — resumable, threaded metadata collection + selective image fetch. + +Two phases so we can do "malignant-maximal" sourcing cheaply: page the whole archive's +metadata (no images), then download images only for a chosen subset (e.g. all malignant + +suspicious + a balanced benign sample). ``parse`` reads the written metadata.csv into records. +""" +import csv +import os +from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from lesnet.data.records import LesionRecord, read_csv_rows, to_float + +API_URL = "https://api.isic-archive.com/api/v2/images/" +METADATA_FIELDS = ['isic_id', 'url', 'patient_id', 'diagnosis', 'diagnosis_1', + 'age_approx', 'sex', 'anatom_site_general'] + + +def make_session(pool=32): + session = requests.Session() + retries = Retry( + total=8, connect=5, read=5, backoff_factor=2.0, + status_forcelist=[429, 500, 502, 503, 504, 520, 522, 524], + respect_retry_after_header=True, raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retries, pool_maxsize=pool, pool_connections=pool) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + + +def _specific_diagnosis(clinical): + for key in ('diagnosis_3', 'diagnosis_2', 'diagnosis_1'): + if clinical.get(key): + return clinical[key] + return clinical.get('diagnosis') or '' + + +def records_from_page(payload, resolution): + """Extract metadata rows (incl. a downloadable image url) from one API page.""" + rows = [] + for image in payload['results']: + clinical = image.get('metadata', {}).get('clinical', {}) + files = image.get('files', {}) + chosen = files.get(resolution) or files.get('full') or files.get('thumbnail_256') + url = (chosen or {}).get('url') + if not url: + continue + rows.append({ + 'isic_id': image['isic_id'], 'url': url, + 'patient_id': clinical.get('patient_id') or image.get('patient_id') or '', + 'diagnosis': _specific_diagnosis(clinical), + 'diagnosis_1': clinical.get('diagnosis_1') or '', + 'age_approx': clinical.get('age_approx', ''), + 'sex': clinical.get('sex', ''), + 'anatom_site_general': clinical.get('anatom_site_1', ''), + }) + return rows + + +def collect_metadata(root, resolution='full', page_size=100, limit=None, session=None): + """Page the whole archive's metadata into /metadata.csv; return the rows (with urls).""" + os.makedirs(root, exist_ok=True) + session = session or make_session() + rows, next_url, params = [], API_URL, {'limit': page_size} + while next_url and (limit is None or len(rows) < limit): + response = session.get(next_url, params=params, timeout=120) + params = None + if response.status_code != 200: + break + payload = response.json() + rows.extend(records_from_page(payload, resolution)) + next_url = payload.get('next') + rows = rows[:limit] if limit else rows + with open(os.path.join(root, 'metadata.csv'), 'w', newline='', encoding='utf-8') as handle: + writer = csv.DictWriter(handle, fieldnames=METADATA_FIELDS) + writer.writeheader() + for row in rows: + writer.writerow({field: row.get(field, '') for field in METADATA_FIELDS}) + return rows + + +def download_images(root, url_by_id, workers=32, session=None): + """Fetch images for the given {isic_id: url} into /.jpg (skip existing).""" + os.makedirs(root, exist_ok=True) + session = session or make_session(workers) + downloaded = failed = 0 + + def _fetch(item): + isic_id, url = item + path = os.path.join(root, f"{isic_id}.jpg") + if os.path.exists(path): + return True + response = session.get(url, timeout=120) + if response.status_code == 200: + with open(path, 'wb') as handle: + handle.write(response.content) + return True + return False + + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = {executor.submit(_fetch, item): item for item in url_by_id.items()} + for future in as_completed(futures): + if future.result(): + downloaded += 1 + else: + failed += 1 + return downloaded, failed + + +def download(root, limit=None, resolution='full', workers=32): + """Convenience full download: collect metadata then fetch every image.""" + rows = collect_metadata(root, resolution=resolution, limit=limit) + download_images(root, {row['isic_id']: row['url'] for row in rows}, workers=workers) + + +def parse(root, limit=None): + rows = read_csv_rows(os.path.join(root, 'metadata.csv')) + rows = rows[:limit] if limit else rows + records = [] + for row in rows: + isic_id = row.get('isic_id') + if not isic_id: + continue + records.append(LesionRecord( + image_path=os.path.join(root, f"{isic_id}.jpg"), + source_dataset='isic', + raw_label=row.get('diagnosis') or row.get('diagnosis_1') or 'unknown', + group_id=row.get('patient_id') or isic_id, + anatomical_site=row.get('anatom_site_general') or None, + age=to_float(row.get('age_approx')), + sex=row.get('sex') or None, + )) + return records diff --git a/lesnet/data/sources/pad_ufes.py b/lesnet/data/sources/pad_ufes.py new file mode 100644 index 0000000..9145d9b --- /dev/null +++ b/lesnet/data/sources/pad_ufes.py @@ -0,0 +1,43 @@ +"""PAD-UFES-20 source (CC BY 4.0, Mendeley). Smartphone clinical images with Fitzpatrick.""" +import io +import os +import zipfile + +import requests + +from lesnet.data.records import LesionRecord, read_csv_rows, to_float, to_int + +MENDELEY_ZIP_URL = ( + "https://prod-dcd-datasets-cache-zipfiles.s3.eu-west-1.amazonaws.com/zr7vgbcyr2-1.zip" +) + + +def download(root, limit=None, session=None): # noqa: ARG001 - limit unused (small dataset) + """Best-effort fetch + extract of the PAD-UFES-20 archive into ``root``.""" + os.makedirs(root, exist_ok=True) + session = session or requests + response = session.get(MENDELEY_ZIP_URL, timeout=600) + response.raise_for_status() + with zipfile.ZipFile(io.BytesIO(response.content)) as archive: + archive.extractall(root) + + +def parse(root, limit=None): + rows = read_csv_rows(os.path.join(root, 'metadata.csv')) + rows = rows[:limit] if limit else rows + records = [] + for row in rows: + image_id = row.get('img_id') + if not image_id: + continue + records.append(LesionRecord( + image_path=os.path.join(root, 'images', image_id), + source_dataset='pad_ufes_20', + raw_label=row.get('diagnostic') or 'unknown', + group_id=row.get('patient_id') or row.get('lesion_id') or image_id, + fitzpatrick=to_int(row.get('fitspatrick')), + anatomical_site=row.get('region') or None, + age=to_float(row.get('age')), + sex=row.get('gender') or None, + )) + return records diff --git a/lesnet/data/sources/registry.py b/lesnet/data/sources/registry.py new file mode 100644 index 0000000..1a96604 --- /dev/null +++ b/lesnet/data/sources/registry.py @@ -0,0 +1,24 @@ +"""Registry of all data sources. Add a legally-usable dataset here to include it.""" +from lesnet.data.sources import ddi, fitzpatrick17k, isic, pad_ufes +from lesnet.data.sources.base import SourceSpec + +SOURCE_REGISTRY = { + 'isic': SourceSpec( + name='isic', parse=isic.parse, download=isic.download, + note='ISIC Archive v2 — API auto-download (aggregates HAM10000/BCN20000/challenge sets).'), + 'pad_ufes_20': SourceSpec( + name='pad_ufes_20', parse=pad_ufes.parse, download=pad_ufes.download, + note='PAD-UFES-20 (CC BY 4.0, Mendeley) — smartphone images with Fitzpatrick.'), + 'fitzpatrick17k': SourceSpec( + name='fitzpatrick17k', parse=fitzpatrick17k.parse, download=fitzpatrick17k.download, + note='Fitzpatrick17k — skin-type-labelled clinical images; many image links may rot.'), + 'ddi': SourceSpec( + name='ddi', parse=ddi.parse, download=None, requires_manual_download=True, + note='DDI — Stanford Research Use Agreement; place on disk manually.'), +} + + +def get_source(name): + if name not in SOURCE_REGISTRY: + raise ValueError(f"Unknown source '{name}'. Known: {sorted(SOURCE_REGISTRY)}") + return SOURCE_REGISTRY[name] diff --git a/lesnet/ml/splits.py b/lesnet/data/splits.py similarity index 100% rename from lesnet/ml/splits.py rename to lesnet/data/splits.py diff --git a/lesnet/ml/taxonomy.py b/lesnet/data/taxonomy.py similarity index 100% rename from lesnet/ml/taxonomy.py rename to lesnet/data/taxonomy.py diff --git a/lesnet/ml/data_loader.py b/lesnet/ml/data_loader.py index 990eb3a..6eb932f 100644 --- a/lesnet/ml/data_loader.py +++ b/lesnet/ml/data_loader.py @@ -9,7 +9,7 @@ from lesnet.ml.features import metadata_vector from lesnet.ml.preprocessing import PreprocessingPipeline -from lesnet.ml.taxonomy import fine_index, triage_index +from lesnet.data.taxonomy import fine_index, triage_index def filter_valid(records): diff --git a/lesnet/ml/datasets.py b/lesnet/ml/datasets.py deleted file mode 100644 index 7879528..0000000 --- a/lesnet/ml/datasets.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Unified multi-source dataset ingestion (paper §5.3). - -Maps ISIC, PAD-UFES-20, Fitzpatrick17k, and DDI into one canonical manifest of -``LesionRecord`` rows, supports a fast ``sample`` mode and a ``full`` mode (so we can -iterate on a few hundred images and later flip to the whole download with one flag), -and assigns patient/lesion-grouped train/val/test splits with no leakage. - -Loaders read each dataset's metadata CSV defensively; point the *_root paths at local -downloads. Images are not opened here — only paths and labels are collected. -""" -import csv -import os -from dataclasses import dataclass, fields -from typing import Optional - -import numpy as np - -from lesnet.ml.splits import grouped_train_val_test - -SAMPLE_MODE = 'sample' -FULL_MODE = 'full' - - -@dataclass -class DatasetConfig: - mode: str = SAMPLE_MODE # 'sample' (fast iteration) or 'full' - sample_size_per_dataset: int = 200 - datasets: tuple = ('isic',) # any of: isic, pad_ufes_20, fitzpatrick17k, ddi - isic_root: Optional[str] = None - pad_ufes_root: Optional[str] = None - fitzpatrick17k_root: Optional[str] = None - ddi_root: Optional[str] = None - image_size: tuple = (224, 224) - test_size: float = 0.15 - val_size: float = 0.15 - seed: int = 42 - - def sample_limit(self): - return self.sample_size_per_dataset if self.mode == SAMPLE_MODE else None - - -@dataclass -class LesionRecord: - image_path: str - source_dataset: str - raw_label: str - group_id: str # patient/lesion id — the grouped-split key - fitzpatrick: Optional[int] = None - anatomical_site: Optional[str] = None - age: Optional[float] = None - sex: Optional[str] = None - split: Optional[str] = None - - -MANIFEST_FIELDS = [field.name for field in fields(LesionRecord)] - - -def _read_csv(path): - with open(path, newline='', encoding='utf-8') as handle: - return list(csv.DictReader(handle)) - - -def _to_float(value): - try: - return float(value) - except (TypeError, ValueError): - return None - - -def _to_int(value): - try: - return int(float(value)) - except (TypeError, ValueError): - return None - - -def _ddi_skin_tone_to_fitzpatrick(skin_tone): - """DDI encodes skin tone as 12/34/56 (Fitzpatrick pairs); map to a representative band.""" - return {'12': 1, '34': 3, '56': 5}.get(str(skin_tone).strip(), None) - - -def load_isic(root, sample_limit=None): - rows = _read_csv(os.path.join(root, 'metadata.csv')) - rows = rows[:sample_limit] if sample_limit else rows - records = [] - for row in rows: - isic_id = row.get('isic_id') - if not isic_id: - continue - records.append(LesionRecord( - image_path=os.path.join(root, f"{isic_id}.jpg"), - source_dataset='isic', - raw_label=row.get('diagnosis') or row.get('benign_malignant') or 'unknown', - group_id=row.get('patient_id') or isic_id, - anatomical_site=row.get('anatom_site_general') or None, - age=_to_float(row.get('age_approx')), - sex=row.get('sex') or None, - )) - return records - - -def load_pad_ufes(root, sample_limit=None): - rows = _read_csv(os.path.join(root, 'metadata.csv')) - rows = rows[:sample_limit] if sample_limit else rows - records = [] - for row in rows: - image_id = row.get('img_id') - if not image_id: - continue - records.append(LesionRecord( - image_path=os.path.join(root, 'images', image_id), - source_dataset='pad_ufes_20', - raw_label=row.get('diagnostic') or 'unknown', - group_id=row.get('patient_id') or row.get('lesion_id') or image_id, - fitzpatrick=_to_int(row.get('fitspatrick')), - anatomical_site=row.get('region') or None, - age=_to_float(row.get('age')), - sex=row.get('gender') or None, - )) - return records - - -def load_fitzpatrick17k(root, sample_limit=None): - rows = _read_csv(os.path.join(root, 'fitzpatrick17k.csv')) - rows = rows[:sample_limit] if sample_limit else rows - records = [] - for row in rows: - key = row.get('md5hash') - if not key: - continue - records.append(LesionRecord( - image_path=os.path.join(root, 'images', f"{key}.jpg"), - source_dataset='fitzpatrick17k', - raw_label=row.get('label') or 'unknown', - group_id=key, # no patient grouping -> per-image - fitzpatrick=_to_int(row.get('fitzpatrick_scale') or row.get('fitzpatrick')), - )) - return records - - -def load_ddi(root, sample_limit=None): - rows = _read_csv(os.path.join(root, 'ddi_metadata.csv')) - rows = rows[:sample_limit] if sample_limit else rows - records = [] - for row in rows: - image_file = row.get('DDI_file') - if not image_file: - continue - records.append(LesionRecord( - image_path=os.path.join(root, image_file), - source_dataset='ddi', - raw_label=row.get('disease') or 'unknown', - group_id=image_file, - fitzpatrick=_ddi_skin_tone_to_fitzpatrick(row.get('skin_tone')), - )) - return records - - -_LOADERS = { - 'isic': ('isic_root', load_isic), - 'pad_ufes_20': ('pad_ufes_root', load_pad_ufes), - 'fitzpatrick17k': ('fitzpatrick17k_root', load_fitzpatrick17k), - 'ddi': ('ddi_root', load_ddi), -} - - -def build_manifest(config): - sample_limit = config.sample_limit() - records = [] - for name in config.datasets: - if name not in _LOADERS: - raise ValueError(f"Unknown dataset '{name}'. Known: {sorted(_LOADERS)}") - root_attribute, loader = _LOADERS[name] - root = getattr(config, root_attribute) - if not root: - raise ValueError(f"Dataset '{name}' enabled but '{root_attribute}' is not set.") - records.extend(loader(root, sample_limit)) - return records - - -def assign_splits(records, config): - groups = np.array([record.group_id for record in records]) - train_index, val_index, test_index = grouped_train_val_test( - groups, config.test_size, config.val_size, config.seed) - for index in train_index: - records[index].split = 'train' - for index in val_index: - records[index].split = 'val' - for index in test_index: - records[index].split = 'test' - return records - - -def save_manifest(records, path): - with open(path, 'w', newline='', encoding='utf-8') as handle: - writer = csv.DictWriter(handle, fieldnames=MANIFEST_FIELDS) - writer.writeheader() - for record in records: - writer.writerow({name: getattr(record, name) for name in MANIFEST_FIELDS}) - - -def load_manifest(path): - records = [] - for row in _read_csv(path): - records.append(LesionRecord( - image_path=row['image_path'], - source_dataset=row['source_dataset'], - raw_label=row['raw_label'], - group_id=row['group_id'], - fitzpatrick=_to_int(row.get('fitzpatrick')), - anatomical_site=row.get('anatomical_site') or None, - age=_to_float(row.get('age')), - sex=row.get('sex') or None, - split=row.get('split') or None, - )) - return records diff --git a/lesnet/ml/evaluation.py b/lesnet/ml/evaluation.py index e7cf383..1129212 100644 --- a/lesnet/ml/evaluation.py +++ b/lesnet/ml/evaluation.py @@ -7,7 +7,7 @@ import numpy as np from lesnet.ml import metrics -from lesnet.ml.taxonomy import MALIGNANT +from lesnet.data.taxonomy import MALIGNANT MIN_SUBGROUP_MALIGNANT_SUPPORT = 5 diff --git a/lesnet/ml/inference.py b/lesnet/ml/inference.py index 45867ad..fcc91d4 100644 --- a/lesnet/ml/inference.py +++ b/lesnet/ml/inference.py @@ -12,7 +12,7 @@ from lesnet.ml.model_compat import load_model_with_compatibility from lesnet.ml.ood import MahalanobisOODDetector, is_low_quality from lesnet.ml.preprocessing import PreprocessingPipeline -from lesnet.ml.taxonomy import MALIGNANT, TRIAGE_CLASSES +from lesnet.data.taxonomy import MALIGNANT, TRIAGE_CLASSES from lesnet.ml.triage import ABSTAIN, triage_decision diff --git a/lesnet/ml/synthetic.py b/lesnet/ml/synthetic.py index 5d22b99..db2fe07 100644 --- a/lesnet/ml/synthetic.py +++ b/lesnet/ml/synthetic.py @@ -4,7 +4,7 @@ import numpy as np from PIL import Image -from lesnet.ml.datasets import LesionRecord +from lesnet.data.records import LesionRecord _LABELS = ['nevus', 'melanoma', 'actinic keratosis', 'basal cell carcinoma'] diff --git a/lesnet/ml/training.py b/lesnet/ml/training.py index d29cddd..fbf4281 100644 --- a/lesnet/ml/training.py +++ b/lesnet/ml/training.py @@ -14,7 +14,7 @@ from lesnet.ml.losses import make_focal_loss, triage_class_weights from lesnet.ml.model import build_triage_model, feature_model, triage_logits_model from lesnet.ml.ood import MahalanobisOODDetector -from lesnet.ml.taxonomy import MALIGNANT, TRIAGE_CLASSES, build_fine_vocabulary +from lesnet.data.taxonomy import MALIGNANT, TRIAGE_CLASSES, build_fine_vocabulary MAX_OOD_FIT_BATCHES = 64 diff --git a/lesnet/views/api.py b/lesnet/views/api.py index 66a4ad7..78d951f 100644 --- a/lesnet/views/api.py +++ b/lesnet/views/api.py @@ -15,9 +15,9 @@ from lesnet.config.model import ModelConfig from lesnet.ml.artifacts import BUNDLE_FILE, MODEL_FILE -from lesnet.ml.datasets import LesionRecord +from lesnet.data.records import LesionRecord +from lesnet.data.taxonomy import TRIAGE_CLASSES from lesnet.ml.inference import TriagePredictor -from lesnet.ml.taxonomy import TRIAGE_CLASSES log = logging.getLogger(__name__) TRIAGE_ARTIFACTS_DIR = os.environ.get('LESNET_TRIAGE_ARTIFACTS', 'models/triage') diff --git a/scripts/train_full_m4.sh b/scripts/train_full_m4.sh index 9f8a6bc..ba765de 100644 --- a/scripts/train_full_m4.sh +++ b/scripts/train_full_m4.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Full M-4 pipeline: download the entire ISIC archive, build the manifest, train the -# gated EfficientNetV2-S triage model on GPU, package + evaluate it. Long-running. +# Full M-4 pipeline: source + build the balanced dataset, train the gated EfficientNetV2-S +# triage model on GPU, package + evaluate it. Long-running. # Launch inside the GPU nix-shell: # nix-shell -p python311 stdenv.cc.cc.lib zlib --run 'bash scripts/train_full_m4.sh' set -uo pipefail @@ -10,24 +10,20 @@ PY=/tmp/lesnet-gpu/bin/python export LD_LIBRARY_PATH="$(nix eval --raw nixpkgs#zlib.out)/lib:$(nix eval --raw nixpkgs#stdenv.cc.cc.lib)/lib:${LD_LIBRARY_PATH:-}" export TF_CPP_MIN_LOG_LEVEL=1 -echo "[$(date)] STEP 1/5: download full ISIC archive (resumable, multithreaded)" -$PY commands/download_isic_full.py --out data/isic_full --workers 64 -echo "[$(date)] download done (exit $?)" +echo "[$(date)] STEP 1/4: source + build the balanced, sorted, leakage-safe dataset" +$PY commands/build_dataset.py --sources isic pad_ufes_20 fitzpatrick17k ddi --dest data/dataset +echo "[$(date)] dataset build done (exit $?)" -echo "[$(date)] STEP 2/5: build full grouped manifest" -$PY commands/run_build_dataset.py --mode full --datasets isic \ - --isic-root data/isic_full --output data/manifest_full.csv - -echo "[$(date)] STEP 3/5: train M-4 (gated, GPU, hair-removal off for throughput)" -$PY commands/run_train_triage.py --manifest data/manifest_full.csv --artifacts models/triage_m4 \ +echo "[$(date)] STEP 2/4: train M-4 (gated, GPU, hair-removal off for throughput)" +$PY commands/run_train_triage.py --manifest data/dataset/manifest.csv --artifacts models/triage_m4 \ --backbone efficientnetv2s --image-size 224 --batch-size 16 \ --until-target --max-epochs 200 --no-hair-removal echo "[$(date)] training done (exit $?)" -echo "[$(date)] STEP 4/5: package M-4 release assets" +echo "[$(date)] STEP 3/4: package M-4 release assets" $PY commands/package_model.py --artifacts models/triage_m4 --model-id M-4 -echo "[$(date)] STEP 5/5: evaluate M-4 on held-out test split" -$PY commands/run_evaluate.py --manifest data/manifest_full.csv --artifacts models/triage_m4 --split test +echo "[$(date)] STEP 4/4: evaluate M-4 on held-out test split" +$PY commands/run_evaluate.py --manifest data/dataset/manifest.csv --artifacts models/triage_m4 --split test echo "[$(date)] FULL M-4 PIPELINE COMPLETE" diff --git a/tests/test_data_pipeline.py b/tests/test_data_pipeline.py new file mode 100644 index 0000000..b131f57 --- /dev/null +++ b/tests/test_data_pipeline.py @@ -0,0 +1,137 @@ +import json +import os + +import numpy as np +from PIL import Image + +from lesnet.data import manifest, pipeline +from lesnet.data.config import SourcingConfig +from lesnet.data.records import LesionRecord +from lesnet.data.sources.base import SourceSpec + + +def _write_image(path, seed): + Image.fromarray((np.random.default_rng(seed).random((80, 80, 3)) * 255).astype('uint8')).save(path) + + +def _dataset(tmp_path): + """Synthetic on-disk records: benign + malignant + not_sure + one unmappable.""" + records = [] + plan = [('nevus', 8), ('melanoma', 6), ('actinic keratosis', 2), ('totally-unmappable', 1)] + index = 0 + for label, count in plan: + for _ in range(count): + path = tmp_path / f"img_{index}.jpg" + _write_image(path, index) + records.append(LesionRecord(str(path), 'isic', label, f"P{index}", fitzpatrick=index % 6 + 1)) + index += 1 + return records + + +# --- config --- + +def test_sourcing_config_defaults_independent(): + one, two = SourcingConfig(), SourcingConfig() + one.roots['isic'] = '/tmp/x' + assert two.roots == {} # default_factory gives each instance its own dict + assert one.balance_ratio == 1.0 and one.balance_buckets == ('benign', 'malignant') + + +# --- manifest --- + +def test_assign_splits_empty_and_normal(): + assert manifest.assign_splits([]) == [] + records = [LesionRecord('x', 'isic', 'nevus', f"P{i}") for i in range(20)] + manifest.assign_splits(records, test_size=0.2, val_size=0.2, seed=1) + assert {r.split for r in records} == {'train', 'val', 'test'} + + +def test_write_manifest(tmp_path): + records = [LesionRecord('x', 'isic', 'nevus', 'P1', split='train')] + out = tmp_path / 'm.csv' + manifest.write(records, str(out)) + assert out.exists() + + +# --- pipeline helpers --- + +def test_distribution_and_build_report(): + records = [LesionRecord('x', 'isic', 'nevus', 'P1', triage_bucket='benign', diagnosis='nevus', + split='train', fitzpatrick=2)] + assert pipeline._distribution(records, 'diagnosis') == {'nevus': 1} + report = pipeline.build_report({'ingested': 1}, records) + assert report['final_total'] == 1 + assert report['benign_to_malignant_ratio'] is None # no malignant -> None branch + + +# --- pipeline.process (end to end, no network) --- + +def test_process_end_to_end(tmp_path): + config = SourcingConfig(dest=str(tmp_path / 'out'), per_diagnosis_cap_fraction=1.0, seed=1) + records = _dataset(tmp_path) + materialised, report = pipeline.process(records, config) + + assert report['dropped_unmappable'] == 1 + assert report['by_bucket']['malignant'] == 6 + assert report['by_bucket']['benign'] <= 6 # downsampled to 1:1 + assert report['by_bucket'].get('not_sure') == 2 + assert os.path.exists(os.path.join(config.dest, 'manifest.csv')) + report_on_disk = json.load(open(os.path.join(config.dest, 'report.json'), encoding='utf-8')) + assert report_on_disk['final_total'] == len(materialised) + unmapped = open(os.path.join(config.dest, 'unmapped_diagnoses.txt'), encoding='utf-8').read() + assert 'totally-unmappable' in unmapped + assert {r.split for r in materialised} <= {'train', 'val', 'test'} + + +def test_process_without_dedupe(tmp_path): + config = SourcingConfig(dest=str(tmp_path / 'out2'), dedupe=False, per_diagnosis_cap_fraction=1.0) + _records, report = pipeline.process(_dataset(tmp_path), config) + assert report['dropped_duplicate'] == 0 + + +# --- pipeline.ingest / run (mocked sources) --- + +def _fake_spec(calls, downloadable=True): + def parse(root, limit): # noqa: ARG001 + return [LesionRecord(os.path.join(root, 'a.jpg'), 'fake', 'nevus', 'P1')] + + def download(root, limit): # noqa: ARG001 + os.makedirs(root, exist_ok=True) + open(os.path.join(root, 'marker'), 'w').close() + calls.append('download') + + return SourceSpec('fake', parse=parse, download=download if downloadable else None) + + +def test_ingest_downloads_when_absent(tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr(pipeline, 'get_source', lambda name: _fake_spec(calls)) + config = SourcingConfig(sources=('fake',), raw_dir=str(tmp_path / 'raw')) + records = pipeline.ingest(config) + assert calls == ['download'] and len(records) == 1 + + +def test_ingest_skips_download_when_present(tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr(pipeline, 'get_source', lambda name: _fake_spec(calls)) + root = tmp_path / 'raw' / 'fake' + root.mkdir(parents=True) + (root / 'existing').write_text('x') + config = SourcingConfig(sources=('fake',), raw_dir=str(tmp_path / 'raw')) + pipeline.ingest(config) + assert calls == [] # present -> no download + + +def test_ingest_manual_source_without_downloader(tmp_path, monkeypatch): + monkeypatch.setattr(pipeline, 'get_source', lambda name: _fake_spec([], downloadable=False)) + config = SourcingConfig(sources=('fake',), raw_dir=str(tmp_path / 'raw')) + assert len(pipeline.ingest(config)) == 1 # parse-only, no crash + + +def test_run(tmp_path, monkeypatch): + monkeypatch.setattr(pipeline, 'get_source', lambda name: _fake_spec([])) + config = SourcingConfig(sources=('fake',), raw_dir=str(tmp_path / 'raw'), + dest=str(tmp_path / 'out'), dedupe=False) + # parse points at a non-existent image -> materialise drops it; run still completes. + _records, report = pipeline.run(config) + assert report['ingested'] == 1 diff --git a/tests/test_data_quality.py b/tests/test_data_quality.py new file mode 100644 index 0000000..995b41c --- /dev/null +++ b/tests/test_data_quality.py @@ -0,0 +1,60 @@ +import numpy as np +from PIL import Image + +from lesnet.data import quality +from lesnet.data.records import LesionRecord + + +def _write_image(path, width=80, height=80): + pixels = (np.random.default_rng(abs(hash(str(path))) % 1000).random((height, width, 3)) * 255).astype('uint8') + Image.fromarray(pixels).save(path) + + +def test_image_short_side_and_quality_gate(tmp_path): + good = tmp_path / 'good.jpg' + _write_image(good, width=100, height=80) + small = tmp_path / 'small.jpg' + _write_image(small, width=10, height=10) + corrupt = tmp_path / 'corrupt.jpg' + corrupt.write_text('definitely not an image') + + assert quality.image_short_side(str(good)) == 80 + assert quality.image_short_side(str(tmp_path / 'missing.jpg')) is None + assert quality.image_short_side(str(corrupt)) is None + assert quality.passes_quality(str(good), min_pixels=64) is True + assert quality.passes_quality(str(small), min_pixels=64) is False + + +def test_average_hash_and_hamming(tmp_path): + image = tmp_path / 'a.jpg' + _write_image(image) + digest = quality.average_hash(str(image)) + assert isinstance(digest, int) + assert quality.average_hash(str(tmp_path / 'missing.jpg')) is None + assert quality.hamming_distance(0b1010, 0b1001) == 2 + assert quality.hamming_distance(5, 5) == 0 + + +def test_filter_quality_splits_records(tmp_path): + good = tmp_path / 'good.jpg' + _write_image(good, 80, 80) + small = tmp_path / 'small.jpg' + _write_image(small, 10, 10) + records = [ + LesionRecord(str(good), 'isic', 'nevus', 'P1'), + LesionRecord(str(small), 'isic', 'nevus', 'P2'), + LesionRecord(str(tmp_path / 'missing.jpg'), 'isic', 'nevus', 'P3'), + ] + kept, dropped = quality.filter_quality(records, min_pixels=64) + assert [r.group_id for r in kept] == ['P1'] + assert {r.group_id for r in dropped} == {'P2', 'P3'} + + +def test_dedupe_collapses_near_duplicates_keeps_unhashable(): + records = [LesionRecord(f"{i}.jpg", 'isic', 'nevus', f"P{i}") for i in range(4)] + fake = {'0.jpg': 0, '1.jpg': 1, '2.jpg': 1000, '3.jpg': None} # 1 is near-dup of 0; 3 unhashable + + kept, dropped = quality.dedupe(records, max_distance=4, hash_fn=lambda path: fake[path]) + kept_ids = {r.group_id for r in kept} + assert kept_ids == {'P0', 'P2', 'P3'} # P1 dropped (near-dup), P3 kept (unhashable) + assert [r.group_id for r in dropped] == ['P1'] diff --git a/tests/test_data_records.py b/tests/test_data_records.py new file mode 100644 index 0000000..da6a37c --- /dev/null +++ b/tests/test_data_records.py @@ -0,0 +1,57 @@ +import csv + +from lesnet.data.records import ( + LesionRecord, + MANIFEST_FIELDS, + load_manifest, + read_csv_rows, + save_manifest, + to_float, + to_int, +) + + +def test_to_float_and_to_int_edge_cases(): + assert to_float('3.5') == 3.5 + assert to_float(None) is None + assert to_float('not-a-number') is None + assert to_int('4') == 4 + assert to_int('4.9') == 4 + assert to_int(None) is None + assert to_int('nope') is None + + +def test_manifest_fields_include_bucket_and_diagnosis(): + assert 'triage_bucket' in MANIFEST_FIELDS + assert 'diagnosis' in MANIFEST_FIELDS + + +def test_save_and_load_manifest_roundtrip(tmp_path): + records = [ + LesionRecord(image_path='a.jpg', source_dataset='isic', raw_label='melanoma', + group_id='P1', fitzpatrick=3, anatomical_site='torso', age=55.0, + sex='male', split='train', triage_bucket='malignant', diagnosis='melanoma'), + LesionRecord(image_path='b.jpg', source_dataset='ddi', raw_label='nevus', group_id='P2'), + ] + path = tmp_path / 'manifest.csv' + save_manifest(records, str(path)) + loaded = load_manifest(str(path)) + assert len(loaded) == 2 + assert loaded[0].triage_bucket == 'malignant' + assert loaded[0].diagnosis == 'melanoma' + assert loaded[0].fitzpatrick == 3 + assert loaded[0].age == 55.0 + assert loaded[1].fitzpatrick is None and loaded[1].split is None + + +def test_load_manifest_tolerates_missing_optional_columns(tmp_path): + path = tmp_path / 'old_manifest.csv' + with open(path, 'w', newline='', encoding='utf-8') as handle: + writer = csv.DictWriter(handle, fieldnames=['image_path', 'source_dataset', 'raw_label', 'group_id']) + writer.writeheader() + writer.writerow({'image_path': 'a.jpg', 'source_dataset': 'isic', + 'raw_label': 'nevus', 'group_id': 'P1'}) + loaded = load_manifest(str(path)) + assert loaded[0].triage_bucket is None + assert loaded[0].diagnosis is None + assert read_csv_rows(str(path))[0]['raw_label'] == 'nevus' diff --git a/tests/test_data_sort_balance.py b/tests/test_data_sort_balance.py new file mode 100644 index 0000000..b590a9b --- /dev/null +++ b/tests/test_data_sort_balance.py @@ -0,0 +1,142 @@ +import os + +import numpy as np +import pytest +from PIL import Image + +from lesnet.data import balance, sort, splits +from lesnet.data.records import LesionRecord + + +def _record(group_id, raw_label='nevus', fitzpatrick=None, bucket=None, diagnosis=None, path='x.jpg'): + return LesionRecord(image_path=path, source_dataset='isic', raw_label=raw_label, + group_id=group_id, fitzpatrick=fitzpatrick, + triage_bucket=bucket, diagnosis=diagnosis) + + +def _write_image(path): + Image.fromarray((np.random.default_rng(0).random((40, 40, 3)) * 255).astype('uint8')).save(path) + + +# --- sort --- + +def test_bucket_for(): + assert sort.bucket_for('melanoma') == 'malignant' + assert sort.bucket_for('actinic keratosis') == 'not_sure' + assert sort.bucket_for('nevus') == 'benign' + assert sort.bucket_for('gibberish') is None + + +def test_annotate_sets_bucket_and_drops_unmappable(): + records = [_record('P1', 'melanoma'), _record('P2', 'naevus'), _record('P3', 'gibberish')] + kept, dropped = sort.annotate(records) + assert [r.triage_bucket for r in kept] == ['malignant', 'benign'] + assert kept[0].diagnosis == 'melanoma' and kept[1].diagnosis == 'nevus' + assert [r.group_id for r in dropped] == ['P3'] + + +def test_materialise_copy_link_and_missing(tmp_path): + source = tmp_path / 'src.jpg' + _write_image(source) + present = _record('P1', bucket='malignant', diagnosis='melanoma', path=str(source)) + missing = _record('P2', bucket='benign', diagnosis='nevus', path=str(tmp_path / 'nope.jpg')) + dest = tmp_path / 'out' + + materialised, missing_out = sort.materialise([present, missing], str(dest)) + assert len(materialised) == 1 and [r.group_id for r in missing_out] == ['P2'] + assert os.path.exists(present.image_path) + assert present.image_path.endswith(os.path.join('malignant', 'melanoma', 'isic_src.jpg')) + + link_record = _record('P3', bucket='benign', diagnosis='nevus', path=str(source)) + sort.materialise([link_record], str(tmp_path / 'linked'), link=True) + assert os.path.islink(link_record.image_path) + + +# --- balance --- + +def test_bucket_counts(): + records = [_record('P1', bucket='benign'), _record('P2', bucket='benign'), + _record('P3', bucket='malignant')] + assert balance.bucket_counts(records) == {'benign': 2, 'malignant': 1} + + +def test_targets_branches(): + assert balance._targets({'benign': 100, 'malignant': 10}, 1.0) == {'benign': 10, 'malignant': 10} + assert balance._targets({'benign': 5, 'malignant': 10}, 1.0) == {'benign': 5, 'malignant': 5} + assert balance._targets({'benign': 0, 'malignant': 10}, 1.0) == {'benign': 0, 'malignant': 10} + + +def test_select_groups_is_group_safe_and_fairness_first(): + # 4 groups of 2; one (G3) has Fitzpatrick 6 -> retained first. + records = [] + for group in range(4): + for _image in range(2): + records.append(_record(f"G{group}", fitzpatrick=6 if group == 3 else 1, bucket='benign')) + selected = balance._select_groups(records, target=4, seed=1) + assert len(selected) == 4 # whole groups only (2 + 2) + assert all(r.group_id == 'G3' for r in selected[:2]) or any(r.group_id == 'G3' for r in selected) + assert any(r.group_id == 'G3' for r in selected) # the dark-skin group is retained + # under target -> everything kept + assert len(balance._select_groups(records, target=99, seed=1)) == 8 + assert balance._select_groups(records, target=0, seed=1) == [] + + +def _by_diagnosis(records): + counts = {} + for record in records: + counts[record.diagnosis] = counts.get(record.diagnosis, 0) + 1 + return counts + + +def test_balance_bucket_single_diagnosis_reaches_target(): + # All one diagnosis: must still reach target (cap must not starve it). + records = [_record(f"M{i}", diagnosis='melanoma', bucket='malignant') for i in range(8)] + selected = balance._balance_bucket(records, target=6, cap_fraction=0.6, seed=1) + assert len(selected) == 6 + + +def test_balance_bucket_limits_dominance_when_diverse(): + records = [_record(f"A{i}", diagnosis='nevus', bucket='benign') for i in range(16)] + records += [_record(f"B{i}", diagnosis='seborrheic keratosis', bucket='benign') for i in range(4)] + selected = balance._balance_bucket(records, target=4, cap_fraction=0.5, seed=1) + counts = _by_diagnosis(selected) + assert counts['nevus'] <= 2 and counts['seborrheic keratosis'] <= 2 # neither dominates + assert len(selected) == 4 + # under target -> all kept + assert len(balance._balance_bucket(records, target=99, cap_fraction=0.5, seed=1)) == 20 + assert balance._balance_bucket(records, target=0, cap_fraction=0.5, seed=1) == [] + + +def test_balance_bucket_trims_when_capped_diagnoses_overshoot(): + # 3 diagnoses x 3 records, cap=2 each -> 6 capped > target 4 -> trimmed back to 4. + records = [] + for diagnosis in ('nevus', 'seborrheic keratosis', 'dermatofibroma'): + for i in range(3): + records.append(_record(f"{diagnosis}{i}", diagnosis=diagnosis, bucket='benign')) + selected = balance._balance_bucket(records, target=4, cap_fraction=0.6, seed=1) + assert len(selected) == 4 + + +def test_assert_no_group_leakage_raises(): + groups = np.array(['P1', 'P1', 'P2', 'P3']) + with pytest.raises(AssertionError): + splits.assert_no_group_leakage(groups, [0], [1], [2]) # P1 in both train and val + + +def test_balance_ratio_cap_and_keeps_not_sure(): + records = [] + for i in range(16): + records.append(_record(f"BN{i}", diagnosis='nevus', bucket='benign')) + for i in range(4): + records.append(_record(f"BS{i}", diagnosis='seborrheic keratosis', bucket='benign')) + for i in range(4): + records.append(_record(f"M{i}", diagnosis='melanoma', bucket='malignant')) + for i in range(3): + records.append(_record(f"NS{i}", diagnosis='actinic keratosis', bucket='not_sure')) + + balanced = balance.balance(records, ratio=1.0, per_diagnosis_cap_fraction=0.6, + buckets=('benign', 'malignant'), seed=1) + counts = balance.bucket_counts(balanced) + assert counts['malignant'] == 4 + assert counts['benign'] <= 4 # downsampled to the malignant count + assert counts['not_sure'] == 3 # untouched diff --git a/tests/test_data_sources.py b/tests/test_data_sources.py new file mode 100644 index 0000000..51227be --- /dev/null +++ b/tests/test_data_sources.py @@ -0,0 +1,190 @@ +import csv +import io +import os +import zipfile + +import pytest +import requests + +from lesnet.data.sources import ddi, fitzpatrick17k, isic, pad_ufes +from lesnet.data.sources.base import SourceSpec +from lesnet.data.sources.registry import SOURCE_REGISTRY, get_source + + +class FakeResponse: + def __init__(self, status_code=200, json_data=None, content=b''): + self.status_code = status_code + self._json = json_data + self.content = content + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError(self.status_code) + + +class FakeSession: + def __init__(self, pages=None, by_url=None): + self.pages = list(pages or []) + self.by_url = by_url or {} + + def get(self, url, params=None, timeout=None): # noqa: ARG002 + if self.by_url: + return self.by_url.get(url, FakeResponse(404)) + return self.pages.pop(0) + + +def _write_csv(path, fieldnames, rows): + with open(path, 'w', newline='', encoding='utf-8') as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +# --- ISIC --- + +def test_isic_specific_diagnosis_priority(): + assert isic._specific_diagnosis({'diagnosis_3': 'melanoma', 'diagnosis_1': 'x'}) == 'melanoma' + assert isic._specific_diagnosis({'diagnosis_1': 'nevus'}) == 'nevus' + assert isic._specific_diagnosis({'diagnosis': 'bcc'}) == 'bcc' + assert isic._specific_diagnosis({}) == '' + + +def test_isic_records_from_page_resolution_and_skip(): + payload = {'results': [ + {'isic_id': 'I1', 'metadata': {'clinical': {'diagnosis_1': 'melanoma', 'age_approx': '60', + 'sex': 'male', 'anatom_site_1': 'torso', 'patient_id': 'P1'}}, + 'files': {'thumbnail_256': {'url': 'thumb1'}}}, + {'isic_id': 'I2', 'metadata': {'clinical': {}}, 'files': {}}, # no url -> skipped + ]} + rows = isic.records_from_page(payload, 'full') # falls back full->thumbnail_256 + assert len(rows) == 1 + assert rows[0]['isic_id'] == 'I1' and rows[0]['url'] == 'thumb1' + assert rows[0]['diagnosis'] == 'melanoma' and rows[0]['patient_id'] == 'P1' + + +def test_isic_collect_metadata_pages_and_writes(tmp_path): + page1 = FakeResponse(json_data={'results': [ + {'isic_id': 'I1', 'metadata': {'clinical': {'diagnosis_1': 'melanoma'}}, + 'files': {'full': {'url': 'u1'}}}], 'next': 'page2'}) + page2 = FakeResponse(json_data={'results': [ + {'isic_id': 'I2', 'metadata': {'clinical': {'diagnosis_1': 'nevus'}}, + 'files': {'full': {'url': 'u2'}}}], 'next': None}) + rows = isic.collect_metadata(str(tmp_path), session=FakeSession(pages=[page1, page2])) + assert {row['isic_id'] for row in rows} == {'I1', 'I2'} + assert os.path.exists(tmp_path / 'metadata.csv') + + +def test_isic_collect_metadata_stops_on_error(tmp_path): + rows = isic.collect_metadata(str(tmp_path), session=FakeSession(pages=[FakeResponse(500)])) + assert rows == [] + + +def test_isic_download_images_success_and_failure(tmp_path): + session = FakeSession(by_url={'u1': FakeResponse(200, content=b'img'), 'u2': FakeResponse(404)}) + downloaded, failed = isic.download_images( + str(tmp_path), {'I1': 'u1', 'I2': 'u2'}, workers=2, session=session) + assert downloaded == 1 and failed == 1 + assert (tmp_path / 'I1.jpg').read_bytes() == b'img' + # existing file is skipped (still counts as downloaded) + again, _ = isic.download_images(str(tmp_path), {'I1': 'u1'}, session=session) + assert again == 1 + + +def test_isic_download_orchestrates(tmp_path, monkeypatch): + monkeypatch.setattr(isic, 'collect_metadata', + lambda root, **kw: [{'isic_id': 'I1', 'url': 'u1'}]) + captured = {} + monkeypatch.setattr(isic, 'download_images', + lambda root, url_by_id, **kw: captured.update(url_by_id)) + isic.download(str(tmp_path)) + assert captured == {'I1': 'u1'} + + +def test_isic_make_session_returns_session(): + assert isinstance(isic.make_session(), requests.Session) + + +def test_isic_parse(tmp_path): + _write_csv(tmp_path / 'metadata.csv', isic.METADATA_FIELDS, [ + {'isic_id': 'I1', 'url': 'u1', 'patient_id': 'P1', 'diagnosis': 'melanoma', + 'diagnosis_1': '', 'age_approx': '60', 'sex': 'male', 'anatom_site_general': 'torso'}, + {'isic_id': '', 'url': '', 'patient_id': '', 'diagnosis': '', 'diagnosis_1': '', + 'age_approx': '', 'sex': '', 'anatom_site_general': ''}, # skipped (no id) + ]) + records = isic.parse(str(tmp_path), limit=5) + assert len(records) == 1 + assert records[0].group_id == 'P1' and records[0].age == 60.0 + + +# --- PAD-UFES-20 --- + +def test_pad_ufes_download_and_parse(tmp_path): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('metadata.csv', + 'img_id,patient_id,lesion_id,diagnostic,fitspatrick,region,age,gender\n' + 'PAT_1.png,PAD1,L1,BCC,3,arm,60,female\n' + ',,,,,,,\n') # empty img_id -> skipped by parse + archive.writestr('images/PAT_1.png', b'img-bytes') + session = FakeSession(by_url={pad_ufes.MENDELEY_ZIP_URL: FakeResponse(200, content=buffer.getvalue())}) + + pad_ufes.download(str(tmp_path), session=session) + records = pad_ufes.parse(str(tmp_path)) + assert len(records) == 1 + assert records[0].source_dataset == 'pad_ufes_20' + assert records[0].fitzpatrick == 3 and records[0].sex == 'female' + + +# --- Fitzpatrick17k --- + +def test_fitzpatrick17k_download_handles_rotten_links(tmp_path): + _write_csv(tmp_path / 'fitzpatrick17k.csv', + ['md5hash', 'url', 'label', 'fitzpatrick_scale'], + [{'md5hash': 'h1', 'url': 'good', 'label': 'melanoma', 'fitzpatrick_scale': '5'}, + {'md5hash': 'h2', 'url': 'bad', 'label': 'nevus', 'fitzpatrick_scale': '2'}, + {'md5hash': 'h3', 'url': 'good3', 'label': 'nevus', 'fitzpatrick_scale': '3'}, + {'md5hash': '', 'url': '', 'label': '', 'fitzpatrick_scale': ''}]) # skipped (no id) + + # h1 already on disk -> exercises the skip-existing branch. + (tmp_path / 'images').mkdir() + (tmp_path / 'images' / 'h1.jpg').write_bytes(b'already-here') + + class RottenSession: + def get(self, url, timeout=None): # noqa: ARG002 + if url == 'bad': + raise requests.ConnectionError('dead link') + return FakeResponse(200, content=b'img') + + fitzpatrick17k.download(str(tmp_path), session=RottenSession()) + assert (tmp_path / 'images' / 'h1.jpg').read_bytes() == b'already-here' # not overwritten + assert (tmp_path / 'images' / 'h3.jpg').read_bytes() == b'img' # freshly fetched + assert not (tmp_path / 'images' / 'h2.jpg').exists() # dead link skipped + + records = fitzpatrick17k.parse(str(tmp_path)) + assert len(records) == 3 + assert records[0].fitzpatrick == 5 + + +# --- DDI --- + +def test_ddi_parse_and_skin_tone(tmp_path): + assert ddi.skin_tone_to_fitzpatrick('56') == 5 + assert ddi.skin_tone_to_fitzpatrick('99') is None + _write_csv(tmp_path / 'ddi_metadata.csv', ['DDI_file', 'disease', 'skin_tone'], + [{'DDI_file': 'x.png', 'disease': 'melanoma', 'skin_tone': '56'}, + {'DDI_file': '', 'disease': '', 'skin_tone': ''}]) # skipped + records = ddi.parse(str(tmp_path)) + assert len(records) == 1 and records[0].fitzpatrick == 5 + + +# --- registry --- + +def test_registry_lookup(): + assert set(SOURCE_REGISTRY) == {'isic', 'pad_ufes_20', 'fitzpatrick17k', 'ddi'} + assert isinstance(get_source('isic'), SourceSpec) + assert get_source('ddi').requires_manual_download is True + with pytest.raises(ValueError): + get_source('nope') diff --git a/tests/test_data_taxonomy_canonical.py b/tests/test_data_taxonomy_canonical.py new file mode 100644 index 0000000..42ca25d --- /dev/null +++ b/tests/test_data_taxonomy_canonical.py @@ -0,0 +1,60 @@ +from lesnet.data.canonical import ( + canonical_diagnosis, + canonical_or_slug, + slugify, + unmapped_diagnoses, +) +from lesnet.data.records import LesionRecord +from lesnet.data.taxonomy import ( + BENIGN, + MALIGNANT, + SUSPICIOUS, + build_fine_vocabulary, + fine_index, + triage_index, +) + + +def test_triage_index_buckets(): + assert triage_index('Melanoma, NOS') == MALIGNANT + assert triage_index('basal cell carcinoma') == MALIGNANT + assert triage_index('actinic keratosis') == SUSPICIOUS + assert triage_index('melanocytic nevus') == BENIGN + assert triage_index('') is None + assert triage_index(None) is None + assert triage_index('unknown') is None + assert triage_index('something unmappable') is None + + +def test_build_fine_vocabulary_and_fine_index(): + records = [LesionRecord('a', 'isic', 'Melanoma', 'P1'), + LesionRecord('b', 'isic', 'nevus', 'P2'), + LesionRecord('c', 'isic', 'unmappable-thing', 'P3')] + vocabulary = build_fine_vocabulary(records) + assert set(vocabulary) == {'melanoma', 'nevus'} # unmappable excluded + assert fine_index('NEVUS', vocabulary) == vocabulary['nevus'] + assert fine_index('absent', vocabulary) is None + + +def test_canonical_diagnosis_merges_variants(): + assert canonical_diagnosis('Melanoma, NOS') == 'melanoma' + assert canonical_diagnosis('lentigo maligna melanoma') == 'melanoma' # melanoma wins over lentigo + assert canonical_diagnosis('BCC') == 'basal cell carcinoma' + assert canonical_diagnosis('naevus') == 'nevus' + assert canonical_diagnosis('seborrhoeic keratosis') == 'seborrheic keratosis' + assert canonical_diagnosis('haemangioma') == 'vascular lesion' + assert canonical_diagnosis('unknown') is None + assert canonical_diagnosis(None) is None + assert canonical_diagnosis('mystery lesion') is None + + +def test_slugify_and_canonical_or_slug(): + assert slugify('Some Weird / Name!!') == 'some_weird_name' + assert slugify('') == 'unknown' + assert canonical_or_slug('melanoma') == 'melanoma' + assert canonical_or_slug('mystery lesion') == 'mystery_lesion' + + +def test_unmapped_diagnoses_report(): + labels = ['melanoma', 'mystery one', 'mystery one', 'unknown', None, 'another mystery'] + assert unmapped_diagnoses(labels) == ['another mystery', 'mystery one'] diff --git a/tests/test_ml_datasets.py b/tests/test_ml_datasets.py deleted file mode 100644 index 24d4c85..0000000 --- a/tests/test_ml_datasets.py +++ /dev/null @@ -1,97 +0,0 @@ -import csv - -import numpy as np - -from lesnet.ml import preprocessing -from lesnet.ml.datasets import ( - DatasetConfig, - assign_splits, - build_manifest, - load_manifest, - save_manifest, -) - - -def _write_csv(path, fieldnames, rows): - with open(path, 'w', newline='', encoding='utf-8') as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(rows) - - -def _make_isic(root, patients=10, per_patient=4): - root.mkdir(parents=True, exist_ok=True) - rows = [] - for patient in range(patients): - for image in range(per_patient): - rows.append({ - 'isic_id': f"ISIC_{patient}_{image}", - 'patient_id': f"P{patient}", - 'diagnosis': 'melanoma' if patient % 2 else 'nevus', - 'age_approx': '55', 'sex': 'male', 'anatom_site_general': 'torso', - }) - _write_csv(root / 'metadata.csv', - ['isic_id', 'patient_id', 'diagnosis', 'age_approx', 'sex', 'anatom_site_general'], rows) - - -def _make_pad_ufes(root, count=12): - (root / 'images').mkdir(parents=True, exist_ok=True) - rows = [{ - 'img_id': f"PAT_{i}.png", 'patient_id': f"PAD{i // 2}", 'lesion_id': f"L{i}", - 'diagnostic': 'BCC' if i % 2 else 'NEV', 'fitspatrick': str((i % 6) + 1), - 'region': 'arm', 'age': '60', 'gender': 'female', - } for i in range(count)] - _write_csv(root / 'metadata.csv', - ['img_id', 'patient_id', 'lesion_id', 'diagnostic', 'fitspatrick', 'region', 'age', 'gender'], rows) - - -def test_isic_manifest_and_grouped_split(tmp_path): - isic_root = tmp_path / 'isic' - _make_isic(isic_root) - config = DatasetConfig(mode='sample', sample_size_per_dataset=100, datasets=('isic',), - isic_root=str(isic_root), test_size=0.2, val_size=0.2, seed=3) - records = assign_splits(build_manifest(config), config) - - assert len(records) == 40 - assert all(record.split in {'train', 'val', 'test'} for record in records) - # No patient appears in more than one split. - split_of_group = {} - for record in records: - split_of_group.setdefault(record.group_id, record.split) - assert split_of_group[record.group_id] == record.split - - -def test_pad_ufes_parses_fitzpatrick_and_sample_limit(tmp_path): - pad_root = tmp_path / 'pad' - _make_pad_ufes(pad_root, count=12) - config = DatasetConfig(mode='sample', sample_size_per_dataset=5, datasets=('pad_ufes_20',), - pad_ufes_root=str(pad_root)) - records = build_manifest(config) - assert len(records) == 5 # sample limit applied - assert all(record.source_dataset == 'pad_ufes_20' for record in records) - assert all(record.fitzpatrick is not None for record in records) - - -def test_multi_dataset_manifest_roundtrip(tmp_path): - isic_root, pad_root = tmp_path / 'isic', tmp_path / 'pad' - _make_isic(isic_root, patients=6, per_patient=3) - _make_pad_ufes(pad_root, count=8) - config = DatasetConfig(mode='full', datasets=('isic', 'pad_ufes_20'), - isic_root=str(isic_root), pad_ufes_root=str(pad_root)) - records = assign_splits(build_manifest(config), config) - assert {r.source_dataset for r in records} == {'isic', 'pad_ufes_20'} - - out = tmp_path / 'manifest.csv' - save_manifest(records, out) - reloaded = load_manifest(out) - assert len(reloaded) == len(records) - assert reloaded[0].image_path == records[0].image_path - - -def test_preprocessing_pipeline_outputs_unit_scaled_array(): - image = (np.random.default_rng(0).random((40, 30, 3)) * 255).astype('uint8') - pipeline = preprocessing.PreprocessingPipeline(image_size=(24, 24), remove_hair=False) - output = pipeline(image) - assert output.shape == (24, 24, 3) - assert output.dtype == np.float32 - assert output.min() >= 0.0 and output.max() <= 1.0 diff --git a/tests/test_ml_foundation.py b/tests/test_ml_foundation.py index cb713c9..b630833 100644 --- a/tests/test_ml_foundation.py +++ b/tests/test_ml_foundation.py @@ -1,7 +1,8 @@ import numpy as np import pytest -from lesnet.ml import calibration, conformal, metrics, preprocessing, splits, triage +from lesnet.data import splits +from lesnet.ml import calibration, conformal, metrics, preprocessing, triage def test_sensitivity_first_threshold_meets_target(): diff --git a/tests/test_ml_training_smoke.py b/tests/test_ml_training_smoke.py index 15e20f1..363da46 100644 --- a/tests/test_ml_training_smoke.py +++ b/tests/test_ml_training_smoke.py @@ -5,8 +5,8 @@ """ import numpy as np +from lesnet.data.manifest import assign_splits from lesnet.ml.config import PipelineConfig -from lesnet.ml.datasets import DatasetConfig, assign_splits from lesnet.ml.inference import TriagePredictor from lesnet.ml.synthetic import make_synthetic_records from lesnet.ml.training import train @@ -21,7 +21,7 @@ def _split(records): def test_train_and_infer_end_to_end(tmp_path): records = make_synthetic_records(str(tmp_path / 'images'), per_class=10, seed=1) - records = assign_splits(records, DatasetConfig(test_size=0.2, val_size=0.2, seed=1)) + records = assign_splits(records, test_size=0.2, val_size=0.2, seed=1) buckets = _split(records) config = PipelineConfig( From 0335da6ec7ba2e14beb90309402cdc5569ee3885 Mon Sep 17 00:00:00 2001 From: Thomas Behan Date: Tue, 16 Jun 2026 00:39:56 +0100 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20data:=20skip=20unav?= =?UTF-8?q?ailable=20sources=20during=20ingest=20instead=20of=20aborting?= =?UTF-8?q?=20the=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lesnet/data/pipeline.py | 19 ++++++++++++++----- tests/test_data_pipeline.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lesnet/data/pipeline.py b/lesnet/data/pipeline.py index a2dd32c..f836802 100644 --- a/lesnet/data/pipeline.py +++ b/lesnet/data/pipeline.py @@ -75,15 +75,24 @@ def process(records, config): def ingest(config): - """Ensure each source's raw data is present (auto-download where possible) and parse it.""" + """Ensure each source's raw data is present (auto-download where possible) and parse it. + + A source whose data is unavailable (licence-gated, not downloaded, or a failed fetch) is + skipped with a warning rather than aborting the whole build. + """ records = [] for name in config.sources: spec = get_source(name) root = config.roots.get(name) or os.path.join(config.raw_dir, name) - metadata_present = os.path.isdir(root) and os.listdir(root) - if not metadata_present and spec.download is not None: - spec.download(root, config.sample_limit) - records.extend(spec.parse(root, config.sample_limit)) + try: + metadata_present = os.path.isdir(root) and os.listdir(root) + if not metadata_present and spec.download is not None: + spec.download(root, config.sample_limit) + source_records = spec.parse(root, config.sample_limit) + except Exception as error: # noqa: BLE001 - one unavailable source must not kill the build + print(f"Skipping source '{name}': {error}") + continue + records.extend(source_records) return records diff --git a/tests/test_data_pipeline.py b/tests/test_data_pipeline.py index b131f57..ba9780b 100644 --- a/tests/test_data_pipeline.py +++ b/tests/test_data_pipeline.py @@ -128,6 +128,16 @@ def test_ingest_manual_source_without_downloader(tmp_path, monkeypatch): assert len(pipeline.ingest(config)) == 1 # parse-only, no crash +def test_ingest_skips_unavailable_source(tmp_path, monkeypatch): + def boom_parse(root, limit): # noqa: ARG001 + raise FileNotFoundError('metadata not on disk') + + monkeypatch.setattr(pipeline, 'get_source', + lambda name: SourceSpec('fake', parse=boom_parse, download=None)) + config = SourcingConfig(sources=('fake',), raw_dir=str(tmp_path / 'raw')) + assert pipeline.ingest(config) == [] # unavailable source skipped, not fatal + + def test_run(tmp_path, monkeypatch): monkeypatch.setattr(pipeline, 'get_source', lambda name: _fake_spec([])) config = SourcingConfig(sources=('fake',), raw_dir=str(tmp_path / 'raw'), From f05fab0aa1096529c325f5dbae91165f393388db Mon Sep 17 00:00:00 2001 From: Thomas Behan Date: Tue, 16 Jun 2026 00:44:02 +0100 Subject: [PATCH 3/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20isic:=20incremental,?= =?UTF-8?q?=20resumable,=20observable=20metadata=20crawl=20+=20url=5Fmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- commands/build_dataset.py | 4 +-- lesnet/data/sources/isic.py | 70 +++++++++++++++++++++++++++---------- tests/test_data_sources.py | 30 ++++++++++++++-- 3 files changed, 82 insertions(+), 22 deletions(-) diff --git a/commands/build_dataset.py b/commands/build_dataset.py index 00c3800..0d858d6 100644 --- a/commands/build_dataset.py +++ b/commands/build_dataset.py @@ -20,8 +20,8 @@ def _isic_malignant_maximal_download(root, config): """Populate ``root`` with ISIC metadata + a malignant-maximal, ratio-matched image set.""" resolution = 'full' if config.full_resolution else 'thumbnail_256' - rows = isic.collect_metadata(root, resolution=resolution, limit=config.sample_limit) - url_by_id = {row['isic_id']: row['url'] for row in rows} + isic.collect_metadata(root, resolution=resolution, limit=config.sample_limit) + url_by_id = isic.url_map(root) annotated, _ = sort.annotate(isic.parse(root, config.sample_limit)) by_bucket = {'benign': [], 'not_sure': [], 'malignant': []} diff --git a/lesnet/data/sources/isic.py b/lesnet/data/sources/isic.py index 80efba6..e90f61b 100644 --- a/lesnet/data/sources/isic.py +++ b/lesnet/data/sources/isic.py @@ -5,6 +5,7 @@ suspicious + a balanced benign sample). ``parse`` reads the written metadata.csv into records. """ import csv +import json import os from concurrent.futures import ThreadPoolExecutor, as_completed @@ -61,26 +62,59 @@ def records_from_page(payload, resolution): return rows +def url_map(root): + """{isic_id: image url} read back from a written metadata.csv (for selective download).""" + return {row['isic_id']: row['url'] + for row in read_csv_rows(os.path.join(root, 'metadata.csv')) + if row.get('isic_id') and row.get('url')} + + def collect_metadata(root, resolution='full', page_size=100, limit=None, session=None): - """Page the whole archive's metadata into /metadata.csv; return the rows (with urls).""" + """Page the archive's metadata into /metadata.csv — incremental + resumable. + + Each page is appended immediately and the cursor saved to metadata_state.json, so a long + crawl survives interruption (re-run resumes) and is observable while running. Returns the + rows collected *this call*; use ``url_map(root)`` for the full id->url map after completion. + """ os.makedirs(root, exist_ok=True) session = session or make_session() - rows, next_url, params = [], API_URL, {'limit': page_size} - while next_url and (limit is None or len(rows) < limit): - response = session.get(next_url, params=params, timeout=120) - params = None - if response.status_code != 200: - break - payload = response.json() - rows.extend(records_from_page(payload, resolution)) - next_url = payload.get('next') - rows = rows[:limit] if limit else rows - with open(os.path.join(root, 'metadata.csv'), 'w', newline='', encoding='utf-8') as handle: - writer = csv.DictWriter(handle, fieldnames=METADATA_FIELDS) + metadata_path = os.path.join(root, 'metadata.csv') + state_path = os.path.join(root, 'metadata_state.json') + + next_url, params, seen = API_URL, {'limit': page_size}, set() + if os.path.exists(state_path) and os.path.exists(metadata_path): + state = json.load(open(state_path, encoding='utf-8')) + next_url = state.get('next_url') or API_URL + params = None if state.get('next_url') else {'limit': page_size} + seen = {row['isic_id'] for row in read_csv_rows(metadata_path)} + + handle = open(metadata_path, 'a', newline='', encoding='utf-8') + writer = csv.DictWriter(handle, fieldnames=METADATA_FIELDS) + if not seen: writer.writeheader() - for row in rows: - writer.writerow({field: row.get(field, '') for field in METADATA_FIELDS}) - return rows + + collected = [] + try: + while next_url and (limit is None or len(seen) < limit): + response = session.get(next_url, params=params, timeout=120) + params = None + if response.status_code != 200: + break + payload = response.json() + for row in records_from_page(payload, resolution): + if row['isic_id'] in seen: + continue + writer.writerow({field: row.get(field, '') for field in METADATA_FIELDS}) + seen.add(row['isic_id']) + collected.append(row) + handle.flush() + next_url = payload.get('next') + json.dump({'next_url': next_url, 'collected': len(seen)}, + open(state_path, 'w', encoding='utf-8')) + print(f"isic metadata: {len(seen)} rows", flush=True) + finally: + handle.close() + return collected def download_images(root, url_by_id, workers=32, session=None): @@ -113,8 +147,8 @@ def _fetch(item): def download(root, limit=None, resolution='full', workers=32): """Convenience full download: collect metadata then fetch every image.""" - rows = collect_metadata(root, resolution=resolution, limit=limit) - download_images(root, {row['isic_id']: row['url'] for row in rows}, workers=workers) + collect_metadata(root, resolution=resolution, limit=limit) + download_images(root, url_map(root), workers=workers) def parse(root, limit=None): diff --git a/tests/test_data_sources.py b/tests/test_data_sources.py index 51227be..3a9ed92 100644 --- a/tests/test_data_sources.py +++ b/tests/test_data_sources.py @@ -93,9 +93,35 @@ def test_isic_download_images_success_and_failure(tmp_path): assert again == 1 +def test_isic_collect_metadata_resumes(tmp_path): + def page(isic_id, url, nxt): + return FakeResponse(json_data={'results': [ + {'isic_id': isic_id, 'metadata': {'clinical': {'diagnosis_1': 'nevus'}}, + 'files': {'full': {'url': url}}}], 'next': nxt}) + + isic.collect_metadata(str(tmp_path), session=FakeSession(pages=[page('I1', 'u1', None)])) + assert os.path.exists(tmp_path / 'metadata_state.json') + # resume: state + metadata present -> I1 is 'seen' and skipped, only I2 is new. + resume_page = FakeResponse(json_data={'results': [ + {'isic_id': 'I1', 'metadata': {'clinical': {}}, 'files': {'full': {'url': 'u1'}}}, + {'isic_id': 'I2', 'metadata': {'clinical': {}}, 'files': {'full': {'url': 'u2'}}}], 'next': None}) + collected = isic.collect_metadata(str(tmp_path), session=FakeSession(pages=[resume_page])) + assert [row['isic_id'] for row in collected] == ['I2'] + assert set(isic.url_map(str(tmp_path))) == {'I1', 'I2'} + + +def test_isic_url_map_skips_blank_rows(tmp_path): + _write_csv(tmp_path / 'metadata.csv', isic.METADATA_FIELDS, [ + {'isic_id': 'I1', 'url': 'u1', 'patient_id': '', 'diagnosis': 'nevus', 'diagnosis_1': '', + 'age_approx': '', 'sex': '', 'anatom_site_general': ''}, + {'isic_id': '', 'url': '', 'patient_id': '', 'diagnosis': '', 'diagnosis_1': '', + 'age_approx': '', 'sex': '', 'anatom_site_general': ''}]) + assert isic.url_map(str(tmp_path)) == {'I1': 'u1'} + + def test_isic_download_orchestrates(tmp_path, monkeypatch): - monkeypatch.setattr(isic, 'collect_metadata', - lambda root, **kw: [{'isic_id': 'I1', 'url': 'u1'}]) + monkeypatch.setattr(isic, 'collect_metadata', lambda root, **kw: None) + monkeypatch.setattr(isic, 'url_map', lambda root: {'I1': 'u1'}) captured = {} monkeypatch.setattr(isic, 'download_images', lambda root, url_by_id, **kw: captured.update(url_by_id)) From e20e28aa97ddd2817eb22b2a791978d9f6e506f3 Mon Sep 17 00:00:00 2001 From: Thomas Behan Date: Tue, 16 Jun 2026 00:54:05 +0100 Subject: [PATCH 4/5] =?UTF-8?q?=E2=9C=A8=204.5.0=20model=20family:=20S/M/L?= =?UTF-8?q?/XL=20variants,=20knowledge=20distillation,=20int8=20quantisati?= =?UTF-8?q?on,=20EMA/label-smoothing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- commands/build_models.py | 77 ++++++++++++++++++++++++ lesnet/ml/config.py | 9 +++ lesnet/ml/distill.py | 49 ++++++++++++++++ lesnet/ml/model.py | 10 +++- lesnet/ml/quantize.py | 64 ++++++++++++++++++++ lesnet/ml/training.py | 113 ++++++++++++++++++++++++------------ lesnet/ml/variants.py | 51 ++++++++++++++++ tests/test_ml_models_4_5.py | 63 ++++++++++++++++++++ 8 files changed, 398 insertions(+), 38 deletions(-) create mode 100644 commands/build_models.py create mode 100644 lesnet/ml/distill.py create mode 100644 lesnet/ml/quantize.py create mode 100644 lesnet/ml/variants.py create mode 100644 tests/test_ml_models_4_5.py diff --git a/commands/build_models.py b/commands/build_models.py new file mode 100644 index 0000000..1c72dd6 --- /dev/null +++ b/commands/build_models.py @@ -0,0 +1,77 @@ +"""Train the full 4.5.0 model family end to end (paper §5, §7). + + python commands/build_models.py --manifest data/dataset/manifest.csv --out models/4.5.0 --until-target + +1. Train the teacher M4.5XL (EfficientNetV2-L @ 512). +2. Distil students M4.5L / M4.5m / M4.5s from the teacher (cross-resolution: the teacher is + rebuilt at each student's input size and its convolutional weights transferred). +3. int8-quantise M4.5s to TFLite for the <500 MB live demo. +Each model's artifacts (model.keras + artifacts.json) land in models/4.5.0//; package +them for release with commands/package_model.py. +""" +import argparse +import os +from dataclasses import replace + +from lesnet.data.records import load_manifest +from lesnet.data.taxonomy import build_fine_vocabulary +from lesnet.ml import quantize, variants +from lesnet.ml.config import PipelineConfig +from lesnet.ml.data_loader import filter_valid, make_dataset +from lesnet.ml.features import METADATA_DIM +from lesnet.ml.model import build_triage_model +from lesnet.ml.training import distill, train + + +def _split(records): + buckets = {'train': [], 'val': [], 'test': []} + for record in records: + buckets.get(record.split, buckets['train']).append(record) + return buckets['train'], buckets['val'] or buckets['train'] + + +def main(): + parser = argparse.ArgumentParser(description="Train the LesNet 4.5.0 model family.") + parser.add_argument('--manifest', required=True) + parser.add_argument('--out', default='models/4.5.0') + parser.add_argument('--epochs', type=int, default=60) + parser.add_argument('--until-target', action='store_true', help="Metric-gated training to targets.") + parser.add_argument('--max-epochs', type=int, default=200) + parser.add_argument('--no-pretrained', action='store_true') + args = parser.parse_args() + + train_records, val_records = _split(load_manifest(args.manifest)) + fine_vocabulary = build_fine_vocabulary(train_records) + n_fine = max(len(fine_vocabulary), 1) + base = PipelineConfig(epochs=args.epochs, train_until_target=args.until_target, + max_epochs=args.max_epochs, pretrained=not args.no_pretrained) + + print("=== Teacher: M4.5XL ===") + teacher_config = variants.config_for(variants.TEACHER, base, + artifacts_dir=os.path.join(args.out, variants.TEACHER)) + teacher_model, _ = train(teacher_config, train_records, val_records) + teacher_weights = os.path.join(args.out, 'teacher.weights.h5') + teacher_model.save_weights(teacher_weights) + + for name in variants.STUDENTS: + print(f"=== Student: {name} (distilling from {variants.TEACHER}) ===") + student_config = variants.config_for(name, base, artifacts_dir=os.path.join(args.out, name)) + # Teacher rebuilt at the student's resolution; conv weights transfer across input sizes. + teacher_at_resolution = build_triage_model( + replace(teacher_config, image_size=student_config.image_size), n_fine, METADATA_DIM) + teacher_at_resolution.load_weights(teacher_weights) + student_model, _bundle = distill(student_config, teacher_at_resolution, train_records, val_records) + + if variants.VARIANTS[name].quantize: + print(f"=== Quantising {name} to int8 TFLite (live-demo model) ===") + representative, _, _ = make_dataset( + filter_valid(train_records), student_config, fine_vocabulary, training=False) + tflite_path = os.path.join(args.out, name, 'model_int8.tflite') + quantize.export_tflite(student_model, tflite_path, dataset=representative, mode='int8') + print(f" {name} int8 TFLite: {quantize.model_size_mb(tflite_path):.1f} MB") + + print(f"4.5.0 family complete in {args.out}: {variants.TEACHER} + {', '.join(variants.STUDENTS)}") + + +if __name__ == '__main__': + main() diff --git a/lesnet/ml/config.py b/lesnet/ml/config.py index ec1c9cb..4f714c1 100644 --- a/lesnet/ml/config.py +++ b/lesnet/ml/config.py @@ -19,6 +19,15 @@ class PipelineConfig: learning_rate: float = 1e-4 focal_gamma: float = 2.0 malignant_cost: float = 2.0 # mild extra weight; sensitivity comes from the threshold + # best-in-class training tricks + label_smoothing: float = 0.05 # softens hard targets, calibrates better + use_ema: bool = True # exponential moving average of weights (optimizer-level) + ema_momentum: float = 0.999 + warmup_epochs: int = 2 # linear LR warmup before cosine decay + cosine_decay: bool = True # cosine-anneal the LR over training + # knowledge distillation (student learns from a trained teacher) + distill_alpha: float = 0.5 # weight on the soft-teacher loss vs hard-label loss + distill_temperature: float = 4.0 target_sensitivity: float = 0.97 # sensitivity-first operating point conformal_alpha: float = 0.1 seed: int = 42 diff --git a/lesnet/ml/distill.py b/lesnet/ml/distill.py new file mode 100644 index 0000000..9853627 --- /dev/null +++ b/lesnet/ml/distill.py @@ -0,0 +1,49 @@ +"""Knowledge distillation: a small student matches a trained teacher's soft predictions. + +Response-based KD (Hinton et al.): the student minimises a blend of the normal hard-label +focal loss and a temperature-softened KL term against the teacher's triage logits. This is +how the int8 live-demo model (M4.5s) keeps near-teacher accuracy at a fraction of the size. +""" +import tensorflow as tf + + +class Distiller(tf.keras.Model): + def __init__(self, student, teacher, hard_loss, alpha=0.5, temperature=4.0): + super().__init__() + self.student = student + self.teacher = teacher + self.hard_loss = hard_loss + self.alpha = alpha + self.temperature = temperature + self.teacher.trainable = False + + def call(self, inputs, training=False): + return self.student(inputs, training=training) + + def _distillation_loss(self, teacher_logits, student_logits): + temperature = self.temperature + soft_teacher = tf.nn.softmax(teacher_logits / temperature, axis=-1) + soft_student = tf.nn.log_softmax(student_logits / temperature, axis=-1) + per_example = -tf.reduce_sum(soft_teacher * soft_student, axis=-1) + return tf.reduce_mean(per_example) * (temperature ** 2) + + def train_step(self, data): + inputs, targets = data + teacher_predictions = self.teacher(inputs, training=False) + with tf.GradientTape() as tape: + student_predictions = self.student(inputs, training=True) + hard = self.hard_loss(targets['triage'], student_predictions['triage']) + soft = self._distillation_loss(teacher_predictions['triage'], student_predictions['triage']) + loss = (1.0 - self.alpha) * hard + self.alpha * soft + trainable = self.student.trainable_variables + self.optimizer.apply_gradients(zip(tape.gradient(loss, trainable), trainable)) + self.compiled_metrics.update_state(targets['triage'], student_predictions['triage']) + return {'loss': loss, 'hard_loss': hard, 'soft_loss': soft, + **{metric.name: metric.result() for metric in self.metrics}} + + def test_step(self, data): + inputs, targets = data + student_predictions = self.student(inputs, training=False) + loss = self.hard_loss(targets['triage'], student_predictions['triage']) + self.compiled_metrics.update_state(targets['triage'], student_predictions['triage']) + return {'loss': loss, **{metric.name: metric.result() for metric in self.metrics}} diff --git a/lesnet/ml/model.py b/lesnet/ml/model.py index 7453d26..173f6ed 100644 --- a/lesnet/ml/model.py +++ b/lesnet/ml/model.py @@ -7,6 +7,12 @@ import tensorflow as tf from tensorflow.keras import Input, Model, layers +_BACKBONES = { + 'efficientnetv2s': tf.keras.applications.EfficientNetV2S, + 'efficientnetv2m': tf.keras.applications.EfficientNetV2M, + 'efficientnetv2l': tf.keras.applications.EfficientNetV2L, +} + def _build_backbone(config, image_input): if config.backbone == 'tiny': @@ -16,7 +22,9 @@ def _build_backbone(config, image_input): features = layers.MaxPooling2D()(features) return layers.GlobalAveragePooling2D()(features) - backbone = tf.keras.applications.EfficientNetV2S( + if config.backbone not in _BACKBONES: + raise ValueError(f"Unknown backbone '{config.backbone}'. Known: {sorted(_BACKBONES)} + 'tiny'.") + backbone = _BACKBONES[config.backbone]( include_top=False, weights='imagenet' if config.pretrained else None, input_tensor=image_input, diff --git a/lesnet/ml/quantize.py b/lesnet/ml/quantize.py new file mode 100644 index 0000000..8107129 --- /dev/null +++ b/lesnet/ml/quantize.py @@ -0,0 +1,64 @@ +"""Post-training quantisation + TFLite export for the live-demo (M4.5s) model. + +int8 with a representative dataset shrinks the model ~4x and lets the demo serve via a +lightweight TFLite runtime under the 500 MB peak-memory budget, while the float students/ +teacher stay available for maximum accuracy. float16 is a safe fallback if a layer refuses +int8. Distillation keeps the quantised model's accuracy close to the teacher's. +""" +import os +import resource + +import numpy as np +import tensorflow as tf + + +def representative_dataset(dataset, num_batches=64): + """Yield model inputs (image, metadata) for the int8 calibration pass.""" + def generator(): + for count, (inputs, _targets) in enumerate(dataset): + if count >= num_batches: + break + yield [tf.cast(inputs['image'], tf.float32), tf.cast(inputs['metadata'], tf.float32)] + return generator + + +def export_tflite(model, path, dataset=None, mode='int8', num_batches=64): + """Export ``model`` to a TFLite flatbuffer. mode='int8' (needs dataset) or 'float16'.""" + converter = tf.lite.TFLiteConverter.from_keras_model(model) + converter.optimizations = [tf.lite.Optimize.DEFAULT] + if mode == 'int8': + if dataset is None: + raise ValueError("int8 quantisation needs a representative dataset.") + converter.representative_dataset = representative_dataset(dataset, num_batches) + elif mode == 'float16': + converter.target_spec.supported_types = [tf.float16] + else: + raise ValueError(f"Unknown quantisation mode '{mode}'.") + flatbuffer = converter.convert() + os.makedirs(os.path.dirname(path) or '.', exist_ok=True) + with open(path, 'wb') as handle: + handle.write(flatbuffer) + return path + + +def model_size_mb(path): + return os.path.getsize(path) / 1e6 + + +def peak_rss_mb(): + """Process peak resident memory in MB (Linux ru_maxrss is in KB).""" + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 + + +def tflite_triage_logits(tflite_path, image, metadata): + """Run the quantised model and return triage logits (live-demo inference path).""" + interpreter = tf.lite.Interpreter(model_path=tflite_path) + interpreter.allocate_tensors() + inputs = interpreter.get_input_details() + by_pixels = sorted(inputs, key=lambda detail: int(np.prod(detail['shape'])), reverse=True) + interpreter.set_tensor(by_pixels[0]['index'], image.astype(by_pixels[0]['dtype'])) + interpreter.set_tensor(by_pixels[1]['index'], metadata.astype(by_pixels[1]['dtype'])) + interpreter.invoke() + outputs = interpreter.get_output_details() + triage = min(outputs, key=lambda detail: int(np.prod(detail['shape']))) # 3-way head is smallest + return interpreter.get_tensor(triage['index']) diff --git a/lesnet/ml/training.py b/lesnet/ml/training.py index fbf4281..5b03ac1 100644 --- a/lesnet/ml/training.py +++ b/lesnet/ml/training.py @@ -9,6 +9,7 @@ from lesnet.ml.calibration import TemperatureScaler, softmax from lesnet.ml.conformal import SplitConformalClassifier from lesnet.ml.data_loader import filter_valid, make_dataset +from lesnet.ml.distill import Distiller from lesnet.ml.evaluation import build_report from lesnet.ml.features import METADATA_DIM, normalize_site from lesnet.ml.losses import make_focal_loss, triage_class_weights @@ -19,13 +20,19 @@ MAX_OOD_FIT_BATCHES = 64 +def _optimizer(config): + """Adam with optional weight-EMA — EMA weights generalise better at no inference cost.""" + return tf.keras.optimizers.Adam( + config.learning_rate, use_ema=config.use_ema, ema_momentum=config.ema_momentum) + + def _compile(model, config, class_weights): + # Label-smooth the auxiliary fine head (calibration win); keep focal on the clinical triage head. + fine_loss = (tf.keras.losses.CategoricalCrossentropy(label_smoothing=config.label_smoothing) + if config.label_smoothing > 0 else make_focal_loss(None, config.focal_gamma)) model.compile( - optimizer=tf.keras.optimizers.Adam(config.learning_rate), - loss={ - 'triage': make_focal_loss(class_weights, config.focal_gamma), - 'fine': make_focal_loss(None, config.focal_gamma), - }, + optimizer=_optimizer(config), + loss={'triage': make_focal_loss(class_weights, config.focal_gamma), 'fine': fine_loss}, loss_weights={'triage': 1.0, 'fine': config.aux_loss_weight}, metrics={'triage': [tf.keras.metrics.CategoricalAccuracy(name='accuracy')]}, jit_compile=False, # Keras-3 'auto' XLA picks a Triton GEMM path that fails on some CPUs @@ -108,52 +115,31 @@ def _gated_fit(model, train_dataset, val_dataset, val_triage, val_records, confi return targets_met -def train(config, records_train, records_val): +def _prepare(config): # Keep TF off XLA/JIT: the pip wheel's Triton-GEMM CPU path and GPU libdevice lookup are - # unavailable on many hosts; the model trains fine on the standard executor. Set before - # the first XLA compile (XLA reads the flag lazily). + # unavailable on many hosts; the model trains fine on the standard executor. os.environ.setdefault('TF_XLA_FLAGS', '--tf_xla_auto_jit=-1') tf.config.optimizer.set_jit(False) tf.keras.utils.set_random_seed(config.seed) + +def _datasets(config, records_train, records_val): cache_train = cache_val = None if config.cache_dataset: cache_directory = os.path.join(config.artifacts_dir, 'cache') os.makedirs(cache_directory, exist_ok=True) - cache_train = os.path.join(cache_directory, 'train') - cache_val = os.path.join(cache_directory, 'val') - + cache_train, cache_val = os.path.join(cache_directory, 'train'), os.path.join(cache_directory, 'val') fine_vocabulary = build_fine_vocabulary(records_train) train_dataset, n_fine, train_triage = make_dataset( records_train, config, fine_vocabulary, training=True, cache_path=cache_train) val_dataset, _, val_triage = make_dataset( records_val, config, fine_vocabulary, training=False, cache_path=cache_val) - val_records = filter_valid(records_val) - - model = build_triage_model(config, n_fine, METADATA_DIM) - class_weights = triage_class_weights(train_triage, config.malignant_cost, MALIGNANT) - _compile(model, config, class_weights) + return (fine_vocabulary, train_dataset, n_fine, train_triage, + val_dataset, val_triage, filter_valid(records_val)) - log_dir = os.path.join(config.artifacts_dir, 'tensorboard') - callbacks = [] - if config.tensorboard: - callbacks.append(tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0)) - # Decay the LR when val_loss plateaus — prevents the late-epoch collapse seen with a - # fixed LR on extreme-imbalance data. State persists across the gated rounds. - callbacks.append(tf.keras.callbacks.ReduceLROnPlateau( - monitor='val_loss', factor=0.5, patience=2, min_lr=1e-6, verbose=1)) - targets_met = None - if config.train_until_target: - targets_met = _gated_fit(model, train_dataset, val_dataset, val_triage, val_records, - config, callbacks, log_dir) - else: - stopping = tf.keras.callbacks.EarlyStopping( - monitor='val_loss', patience=max(2, config.epochs // 5), restore_best_weights=True) - model.fit(train_dataset, validation_data=val_dataset, epochs=config.epochs, - callbacks=callbacks + [stopping], verbose=2) - - # Finalise: calibration, operating point, conformal, OOD, save. +def _finalize(model, config, train_dataset, val_dataset, val_triage, fine_vocabulary, n_fine, targets_met): + """Calibrate, choose operating point, fit conformal + OOD, save the artifact bundle.""" logits = triage_logits_model(model).predict(val_dataset, verbose=0) scaler = TemperatureScaler().fit(logits, val_triage) calibrated = softmax(logits / scaler.temperature) @@ -162,9 +148,7 @@ def train(config, records_train, records_val): y_malignant = (val_triage == MALIGNANT).astype(int) operating_threshold = metrics.select_threshold_for_sensitivity( y_malignant, p_malignant, config.target_sensitivity) - conformal = SplitConformalClassifier(alpha=config.conformal_alpha).calibrate(calibrated, val_triage) - embeddings = feature_model(model).predict(train_dataset.take(MAX_OOD_FIT_BATCHES), verbose=0) ood = MahalanobisOODDetector().fit(embeddings) @@ -190,3 +174,58 @@ def train(config, records_train, records_val): model.save(artifacts.model_path(config.artifacts_dir)) artifacts.save_bundle(config.artifacts_dir, bundle) return model, bundle + + +def train(config, records_train, records_val): + _prepare(config) + fine_vocabulary, train_dataset, n_fine, train_triage, val_dataset, val_triage, val_records = \ + _datasets(config, records_train, records_val) + + model = build_triage_model(config, n_fine, METADATA_DIM) + _compile(model, config, triage_class_weights(train_triage, config.malignant_cost, MALIGNANT)) + + log_dir = os.path.join(config.artifacts_dir, 'tensorboard') + callbacks = [] + if config.tensorboard: + callbacks.append(tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0)) + callbacks.append(tf.keras.callbacks.ReduceLROnPlateau( + monitor='val_loss', factor=0.5, patience=2, min_lr=1e-6, verbose=1)) + + targets_met = None + if config.train_until_target: + targets_met = _gated_fit(model, train_dataset, val_dataset, val_triage, val_records, + config, callbacks, log_dir) + else: + stopping = tf.keras.callbacks.EarlyStopping( + monitor='val_loss', patience=max(2, config.epochs // 5), restore_best_weights=True) + model.fit(train_dataset, validation_data=val_dataset, epochs=config.epochs, + callbacks=callbacks + [stopping], verbose=2) + + return _finalize(model, config, train_dataset, val_dataset, val_triage, + fine_vocabulary, n_fine, targets_met) + + +def distill(config, teacher_model, records_train, records_val): + """Train a student to match ``teacher_model``'s soft triage predictions, then finalise it. + + The teacher must accept the student's inputs (the orchestrator rebuilds it at the student's + resolution and loads the teacher weights — backbone is convolutional, so weights transfer). + """ + _prepare(config) + fine_vocabulary, train_dataset, n_fine, train_triage, val_dataset, val_triage, _val_records = \ + _datasets(config, records_train, records_val) + + student = build_triage_model(config, n_fine, METADATA_DIM) + hard_loss = make_focal_loss(triage_class_weights(train_triage, config.malignant_cost, MALIGNANT), + config.focal_gamma) + distiller = Distiller(student, teacher_model, hard_loss, + alpha=config.distill_alpha, temperature=config.distill_temperature) + distiller.compile(optimizer=_optimizer(config), + metrics=[tf.keras.metrics.CategoricalAccuracy(name='accuracy')], jit_compile=False) + stopping = tf.keras.callbacks.EarlyStopping( + monitor='val_loss', patience=max(2, config.epochs // 5), restore_best_weights=True) + distiller.fit(train_dataset, validation_data=val_dataset, epochs=config.epochs, + callbacks=[stopping], verbose=2) + + return _finalize(student, config, train_dataset, val_dataset, val_triage, + fine_vocabulary, n_fine, None) diff --git a/lesnet/ml/variants.py b/lesnet/ml/variants.py new file mode 100644 index 0000000..8364e6c --- /dev/null +++ b/lesnet/ml/variants.py @@ -0,0 +1,51 @@ +"""The 4.5.0 model family (paper §5.1, §7). + +One teacher (XL) distilled into three deployable students (S/M/L). S is int8-quantised for +the live demo under the strict 500 MB peak-inference budget; the larger variants stay +available for maximum accuracy. Bigger backbone + resolution = more accurate + heavier. + + name backbone input role + M4.5s EfficientNetV2-S 320px live demo (int8 TFLite, <500 MB), distilled + M4.5m EfficientNetV2-M 384px balanced, distilled + M4.5L EfficientNetV2-L 448px high accuracy, distilled + M4.5XL EfficientNetV2-L 512px teacher (widest head, heaviest aug) — distillation source +""" +from dataclasses import dataclass, replace + +from lesnet.ml.config import PipelineConfig + + +@dataclass(frozen=True) +class Variant: + name: str + backbone: str + image_size: int + shared_units: int + quantize: bool + role: str + is_teacher: bool = False + + +VARIANTS = { + 'M4.5s': Variant('M4.5s', 'efficientnetv2s', 320, 256, True, 'live demo (int8 TFLite, <500 MB)'), + 'M4.5m': Variant('M4.5m', 'efficientnetv2m', 384, 384, False, 'balanced'), + 'M4.5L': Variant('M4.5L', 'efficientnetv2l', 448, 512, False, 'high accuracy'), + 'M4.5XL': Variant('M4.5XL', 'efficientnetv2l', 512, 768, False, 'teacher', is_teacher=True), +} + +# Distillation target order: which students learn from the teacher. +STUDENTS = ('M4.5L', 'M4.5m', 'M4.5s') +TEACHER = 'M4.5XL' + + +def config_for(variant_name, base=None, **overrides): + """Build a PipelineConfig for a named variant, on top of ``base`` (or defaults).""" + variant = VARIANTS[variant_name] + config = base or PipelineConfig() + config = replace( + config, + backbone=variant.backbone, + image_size=(variant.image_size, variant.image_size), + shared_units=variant.shared_units, + ) + return replace(config, **overrides) if overrides else config diff --git a/tests/test_ml_models_4_5.py b/tests/test_ml_models_4_5.py new file mode 100644 index 0000000..daa4f29 --- /dev/null +++ b/tests/test_ml_models_4_5.py @@ -0,0 +1,63 @@ +"""Smoke tests for the 4.5.0 model family: variants, distillation, quantisation, TFLite. + +Tiny backbone + synthetic data so the whole teacher->student->quantise->infer path runs on +CPU in seconds and proves it is wired; real training scales the same code on GPU. +""" +import numpy as np +import pytest + +from lesnet.data.manifest import assign_splits +from lesnet.ml import quantize, variants +from lesnet.ml.config import PipelineConfig +from lesnet.ml.features import METADATA_DIM +from lesnet.ml.model import build_triage_model +from lesnet.ml.synthetic import make_synthetic_records +from lesnet.ml.training import distill, train + + +def _data(tmp_path): + records = assign_splits(make_synthetic_records(str(tmp_path / 'img'), per_class=8, seed=1), + test_size=0.2, val_size=0.2, seed=1) + buckets = {'train': [], 'val': [], 'test': []} + for record in records: + buckets[record.split].append(record) + return buckets['train'], buckets['val'] or buckets['train'] + + +def _smoke_config(tmp_path, name): + return PipelineConfig( + image_size=(64, 64), backbone='tiny', pretrained=False, epochs=1, batch_size=8, + shared_units=32, artifacts_dir=str(tmp_path / name), smoke=True, + use_ema=False, tensorboard=False, label_smoothing=0.0) + + +def test_variants_registry_and_config(): + config = variants.config_for('M4.5s') + assert config.backbone == 'efficientnetv2s' and config.image_size == (320, 320) + assert variants.VARIANTS['M4.5XL'].is_teacher + assert variants.VARIANTS['M4.5s'].quantize + assert variants.TEACHER == 'M4.5XL' and 'M4.5s' in variants.STUDENTS + + +def test_distill_then_quantize_and_tflite_infer(tmp_path): + train_records, val_records = _data(tmp_path) + teacher, _ = train(_smoke_config(tmp_path, 'teacher'), train_records, val_records) + student, bundle = distill(_smoke_config(tmp_path, 'student'), teacher, train_records, val_records) + + assert 0.0 < bundle['calibration']['temperature'] < 100.0 + assert 'operating_threshold' in bundle['thresholds'] + + path = quantize.export_tflite(student, str(tmp_path / 'student.tflite'), mode='float16') + assert quantize.model_size_mb(path) > 0 + logits = quantize.tflite_triage_logits( + path, np.zeros((1, 64, 64, 3), 'float32'), np.zeros((1, METADATA_DIM), 'float32')) + assert logits.shape[-1] == 3 + assert quantize.peak_rss_mb() > 0 + + +def test_export_tflite_rejects_bad_modes(tmp_path): + model = build_triage_model(_smoke_config(tmp_path, 'm'), n_fine=2, metadata_dim=METADATA_DIM) + with pytest.raises(ValueError): + quantize.export_tflite(model, str(tmp_path / 'x.tflite'), mode='int8') # no dataset + with pytest.raises(ValueError): + quantize.export_tflite(model, str(tmp_path / 'x.tflite'), mode='bogus') From 344b0619c45fb96046ce3f59fb9b90a3950fb32f Mon Sep 17 00:00:00 2001 From: Thomas Behan Date: Tue, 16 Jun 2026 00:55:09 +0100 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=9A=80=20release=20workflow=20+=20M4.?= =?UTF-8?q?5*=20model=20registry=20(forward=20to=20v4.5.0=20release=20asse?= =?UTF-8?q?ts)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 27 +++++++++++++++++++++++++++ lesnet/config/model.py | 5 +++++ 2 files changed, 32 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..866a33e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,27 @@ +name: Release + +# On a version tag, ensure a GitHub Release exists with generated notes. The trained model +# artifacts (LesNet.M4.5*.keras / .artifacts.json / .tflite) are uploaded to this release by +# the model-build automation (gh release upload) — they are too large to live in the repo/CI. +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Create or update the release + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view "${GITHUB_REF_NAME}" >/dev/null 2>&1; then + gh release edit "${GITHUB_REF_NAME}" --generate-notes + else + gh release create "${GITHUB_REF_NAME}" --generate-notes --title "LesNet ${GITHUB_REF_NAME}" + fi diff --git a/lesnet/config/model.py b/lesnet/config/model.py index 0105c18..2563c63 100644 --- a/lesnet/config/model.py +++ b/lesnet/config/model.py @@ -11,4 +11,9 @@ class ModelConfig(object): 'M-0015s': 'https://github.com/Thomasbehan/LesNet/releases/download/0.1.5/skinvestigator-sm.tflite', 'M-0031': 'https://github.com/Thomasbehan/LesNet/releases/download/0.3.1/LesNetM31.keras', 'M-4s': 'https://github.com/Thomasbehan/LesNet/releases/download/4.1.0/LesNet.M-4s.keras', + # 4.5.0 family (uploaded to the v4.5.0 release by the model-build automation). + 'M4.5s': 'https://github.com/Thomasbehan/LesNet/releases/download/v4.5.0/LesNet.M4.5s.keras', + 'M4.5m': 'https://github.com/Thomasbehan/LesNet/releases/download/v4.5.0/LesNet.M4.5m.keras', + 'M4.5L': 'https://github.com/Thomasbehan/LesNet/releases/download/v4.5.0/LesNet.M4.5L.keras', + 'M4.5XL': 'https://github.com/Thomasbehan/LesNet/releases/download/v4.5.0/LesNet.M4.5XL.keras', }