diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 151a3e5..592de20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,3 +32,7 @@ jobs: - name: Tests run: | pytest + + - name: Quick import smoke test + run: | + python -c "import epydem; print('epydem imported')" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 75e471c..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Tests - -on: - push: - branches: [ main, develop ] - pull_request: - branches: [ main, develop ] - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - pip install -e . - - - name: Run tests with pytest - run: | - pytest tests/ -v --tb=short --cov=epydem --cov-report=xml --cov-report=term-missing - - - name: Test package functionality - run: | - python -c " - import epydem - result = epydem.calculate('2024-01-01') - print(f'2024-01-01 is epidemiological week {result}') - assert result > 0, 'Expected positive week number' - print('Package functionality test passed') - " - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - file: ./coverage.xml - flags: unittests - name: codecov-umbrella - fail_ci_if_error: false - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 epydem --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 epydem --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Check code formatting with black - run: | - black --check epydem - - - name: Check import sorting with isort - run: | - isort --check-only epydem - - - name: Check tests formatting - run: | - black --check tests/ \ No newline at end of file diff --git a/PR_ACTIONS_FIX.md b/PR_ACTIONS_FIX.md new file mode 100644 index 0000000..c34ea74 --- /dev/null +++ b/PR_ACTIONS_FIX.md @@ -0,0 +1,29 @@ +GitHub Actions fix: CI was failing on Python 3.9 because epydem now requires Python >= 3.10 (pyproject.toml). + +Changes +- Consolidate CI by removing redundant `.github/workflows/test.yml` (keep `ci.yml` as the single CI). +- Ensure CI only tests supported Python versions (>=3.10). +- Install via `pip install -e '.[dev]'` (single source of truth). +- Add a quick import smoke test to CI. + +Why this implementation +- Running CI on unsupported Python versions creates noisy failures and slows iteration. +- Keeping workflows consistent reduces maintenance and confusion. + +Multi-role debate (differences, not consensus) + +Role A β€” pragmatic developer +- πŸ‘ Likes: CI goes green and matches supported versions; simpler workflow. +- ⚠️ Concern: removes older-Python signal; but we explicitly don’t support <3.10. + +Role B β€” architecture +- πŸ‘ Likes: single tooling stack (ruff) and consistent install path. +- ⚠️ Concern: having both `ci.yml` and `test.yml` is redundant; consider consolidating later. + +Role C β€” developer user (DX) +- πŸ‘ Likes: less CI noise; clearer support policy. +- ⚠️ Concern: if users want older Python, they’ll need a documented support decision. + +Points of divergence to revisit later +1) Consolidate workflows (keep one CI file). +2) Add `python-version: 3.13` when ready. diff --git a/PR_PHASE1.md b/PR_PHASE1.md new file mode 100644 index 0000000..54b993c --- /dev/null +++ b/PR_PHASE1.md @@ -0,0 +1,11 @@ +Implements CDC/MMWR epidemiological weeks as the default epiweek logic. + +Key points +- Add `mmwr_week(date)` returning `(mmwr_year, mmwr_week)`. +- Add `mmwr_week1_start(year)` (Sunday on/before Jan 4). +- Add `parse_ymd()` to accept `YYYY-MM-DD` strings, `date`, or `datetime`. +- Keep `calculate()` as backward-compatible wrapper returning week number only. +- Replace prior week-0-based tests with focused MMWR boundary tests. + +Notes +- This removes the old ISO-like "first Thursday" implementation and eliminates week 0 for MMWR logic. diff --git a/PR_PHASE1_DEBATE.md b/PR_PHASE1_DEBATE.md new file mode 100644 index 0000000..07f89a7 --- /dev/null +++ b/PR_PHASE1_DEBATE.md @@ -0,0 +1,33 @@ +Implements CDC/MMWR epidemiological weeks as the default epiweek logic. + +Key points +- Add `mmwr_week(date)` returning `(mmwr_year, mmwr_week)`. +- Add `mmwr_week1_start(year)` (Sunday on/before Jan 4). +- Add `parse_ymd()` to accept `YYYY-MM-DD` strings, `date`, or `datetime`. +- Keep `calculate()` as backward-compatible wrapper returning week number only. +- Replace prior week-0-based tests with focused MMWR boundary tests. + +Why this implementation +- CDC/MMWR weeks start on Sunday and Week 1 is defined as the week containing Jan 4. That implies Week 1 starts on the Sunday on/before Jan 4. +- Computing `week1_start(year)` from Jan 4 is a simple, O(1) pure function that avoids tricky edge cases. +- Returning `(mmwr_year, mmwr_week)` is essential because early January dates can belong to the previous MMWR year. +- `parse_ymd()` centralizes date parsing/validation so later features (incidence curves, summaries) can reuse it. + +Multi-role debate (differences, not consensus) + +Role A β€” pragmatic developer +- πŸ‘ Likes: pure function, O(1), easy to test; `parse_ymd()` makes downstream code simpler. +- ⚠️ Concern: backward-compat behavior changes for early-January dates (previously could be week 0; now becomes week 52/53). Correct for MMWR but can surprise users. + +Role B β€” architecture +- πŸ‘ Likes: explicit MMWR semantics; returning `(year, week)` avoids ambiguity. +- ⚠️ Concern: module/API organization. Might prefer a `time/` namespace (e.g., `epydem.time.mmwr_week`) if we later add ISO/WHO week systems. + +Role C β€” developer user (DX) +- πŸ‘ Likes: no week 0; tuple output makes grouping across years correct. +- ⚠️ Concern: naming. Users may want `epiweek()` as the default MMWR function and clear docs with boundary examples (e.g., 2022-01-01 -> (2021, 52)). + +Points of divergence to revisit later +1) Naming: `mmwr_week()` vs providing a default `epiweek()` alias. +2) Compatibility: keep `calculate()` returning week-only vs switching `calculate()` to tuple (breaking change). +3) Structure: keep in `epiweek.py` vs split into `epydem/time.py` or `epydem/time/` module. diff --git a/PR_PHASE2_1.md b/PR_PHASE2_1.md new file mode 100644 index 0000000..b54aceb --- /dev/null +++ b/PR_PHASE2_1.md @@ -0,0 +1,29 @@ +Phase 2.1: incidence DX improvements (docs + rolling/cumulative) + +Key points +- Add optional `cumulative=True` for wide output. +- Add optional `rolling=` for wide output with `rolling_kind` (sum/mean). +- Expand README examples for incidence. + +Why this implementation +- These options are common in epi workflows (cumulative incidence, smoothed curves) and improve usability without changing the core incidence semantics. +- Implemented at the wide-output layer so behavior is consistent across strata columns. + +Multi-role debate (differences, not consensus) + +Role A β€” pragmatic developer +- πŸ‘ Likes: small API surface change, easy to test, minimal code. +- ⚠️ Concern: rolling/cumulative after pivot means we assume the index ordering fully represents time (true for date and (epi_year, epi_week) sorted). + +Role B β€” architecture +- πŸ‘ Likes: keeps incidence core as counts; transformations are optional flags. +- ⚠️ Concern: feature creep: might prefer a separate `transform_incidence()` pipeline later. + +Role C β€” developer user (DX) +- πŸ‘ Likes: fewer steps in notebooks; one-liners for rolling/cumulative. +- ⚠️ Concern: wants more control (e.g., min_periods, centered rolling, cumulative per-calendar-year vs epi-year). + +Points of divergence to revisit later +1) Whether these transforms belong inside `incidence()` long-term. +2) Add `min_periods`, `center`, and `cumulative_by` options. +3) Apply transforms to `output="long"` as well. diff --git a/PR_PHASE2_DEBATE.md b/PR_PHASE2_DEBATE.md new file mode 100644 index 0000000..22c3862 --- /dev/null +++ b/PR_PHASE2_DEBATE.md @@ -0,0 +1,33 @@ +Adds basic incidence computation from a line list (daily and weekly CDC/MMWR). + +Key points +- Add `epydem.incidence(df, date_col=..., freq=...)`. +- Supports: + - `freq="D"` -> daily counts by calendar date + - `freq="W-MMWR"` -> weekly counts by CDC/MMWR epiweek (adds epi_year + epi_week) +- Optional stratification via `by=[...]`. + +Why this implementation +- Incidence is the most common first step for line-list analysis; we want a minimal, composable primitive. +- Using `epi_year` + `epi_week` avoids ambiguity at year boundaries. +- Returning a tidy DataFrame (group columns + `cases`) makes plotting and further transforms straightforward in pandas. +- We intentionally do not implement missing-week filling, rolling averages, or cumulative sums yet to keep the first API small. + +Multi-role debate (differences, not consensus) + +Role A β€” pragmatic developer +- πŸ‘ Likes: minimal API, returns tidy DF, works with `by` strata. +- ⚠️ Concern: performance if we iterate row-by-row for epiweek on large datasets; may need vectorization/caching later. + +Role B β€” architecture +- πŸ‘ Likes: keeps time semantics centralized in `epydem.time` and reuses `epiweek()`. +- ⚠️ Concern: frequency naming: `W-MMWR` is clear but may expand to ISO/WHO; may want a more general `freq="W"` + `system=`. + +Role C β€” developer user (DX) +- πŸ‘ Likes: one-liner to get epicurve-like table. +- ⚠️ Concern: expects convenience features soon (fill missing weeks, ordering, start/end bounds, cumulative) and clear examples in docs. + +Points of divergence to revisit later +1) Performance: pure-Python loop vs vectorized epiweek mapping. +2) Output schema: keep tidy long DF vs allow pivot/wide convenience. +3) Feature growth: add `fill_missing`, `cumulative`, `rolling`, and `freq` generalization. diff --git a/README.md b/README.md index 4484f5a..f266aba 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,58 @@ pip install -e '.[dev]' ```python import epydem -# Current API (will change): -week_number = epydem.calculate("2024-01-01") -print(week_number) +year, week = epydem.epiweek("2024-01-01") +print(year, week) + +week_only = epydem.epiweek_number("2024-01-01") +print(week_only) + +# Incidence (line list -> counts) +# df = pandas.DataFrame({"onset_date": [...], "sex": [...]}) +# weekly = epydem.incidence(df, date_col="onset_date", freq="W-MMWR", by=["sex"], fill_missing=True) +# weekly_rolling2 = epydem.transform_incidence(weekly, rolling=2) +# weekly_cum = epydem.transform_incidence(weekly, cumulative=True) +``` + +### `summary()` β€” Descriptive statistics + +```python +import pandas as pd +import epydem + +df = pd.DataFrame({ + "region": ["N", "N", "S", "S", "S"], + "onset": ["2024-01-01", "2024-02-15", "2024-03-10", "NOT_A_DATE", None], + "age": [25, 40, 33, 55, 19], + "dx": ["flu", "flu", "covid", "covid", "flu"], +}) + +# Default: returns only row count +epydem.summary(df) + +# With column specs (long format, the default) +result = epydem.summary( + df, + by="region", + date_cols=["onset"], + numeric_cols=["age"], + categorical_cols=["dx"], +) +print(result) +# region column metric value +# 0 N _n n 2 +# 1 N onset missing_n 0 +# 2 N onset missing_pct 0.0 +# 3 N onset min 2024-01-01 ... +# ... + +# Wide format +wide = epydem.summary(df, numeric_cols=["age"], output="wide") +print(wide) ``` ## Roadmap (high level) -- CDC/MMWR epiweek (week starts Sunday; week 1 contains Jan 4) -- Incidence / epicurves from line lists (pandas) -- Summary statistics (descriptive epi) +- CDC/MMWR epiweek (week starts Sunday; week 1 contains Jan 4) βœ… (in progress) +- Incidence / epicurves from line lists (pandas) βœ… (in progress) +- Summary statistics (descriptive epi) βœ… diff --git a/epydem/__init__.py b/epydem/__init__.py index 37bc386..42d49f9 100644 --- a/epydem/__init__.py +++ b/epydem/__init__.py @@ -1 +1,19 @@ +"""epydem public API.""" + from .epiweek import calculate +from .incidence import incidence +from .summary import summary +from .time import epiweek, epiweek_number, mmwr_week, mmwr_week1_start, parse_ymd +from .transform import transform_incidence + +__all__ = [ + "calculate", + "epiweek", + "epiweek_number", + "mmwr_week", + "mmwr_week1_start", + "parse_ymd", + "incidence", + "transform_incidence", + "summary", +] diff --git a/epydem/epiweek.py b/epydem/epiweek.py index 5dc8d3e..4df3c32 100644 --- a/epydem/epiweek.py +++ b/epydem/epiweek.py @@ -1,60 +1,19 @@ -import re -from datetime import datetime, timedelta +"""Backward-compat module. +Historically this project exposed `epydem.calculate()` from `epydem.epiweek`. +We are moving time-related utilities to `epydem.time`. -def calculate(date_str): - """ - Calculate the epidemiological week number for a given date string. +Prefer using: +- `epydem.epiweek()` -> (year, week) +- `epydem.epiweek_number()` -> week +""" - This function determines the week number of the year for a given date - based on epidemiological weeks, where the first epidemiological week - starts on a Sunday and includes the first Thursday of the year. +from __future__ import annotations - Args: - date_str (str): The date in 'YYYY-MM-DD' format. +from .time import epiweek_number - Returns: - int: The epidemiological week number (1-53). Returns 0 if the date is - before the start of the epidemiological year. - Raises: - ValueError: If the provided date string does not match the 'YYYY-MM-DD' format. - """ - if _verify_date_str(date_str): - date = datetime.strptime(date_str, "%Y-%m-%d") - year = date.year +def calculate(date_str: str) -> int: + """Compatibility alias for week number only.""" - # Find the first Thursday of the year - jan_1 = datetime(year, 1, 1) - days_to_thursday = (3 - jan_1.weekday()) % 7 - first_thursday = jan_1 + timedelta(days=days_to_thursday) - - # The epidemiological week 1 starts on the Sunday before the first Thursday - epi_week_1_start = first_thursday - timedelta(days=3) - - # If the date is before the start of the epidemiological year, it's week 0 - if date < epi_week_1_start: - return 0 - else: - return ((date - epi_week_1_start).days // 7) + 1 - - -def _verify_date_str(date_str): - """ - Verify that the date string is in the 'YYYY-MM-DD' format. - - Args: - date_str (str): The date string to verify. - - Returns: - bool: True if the date string is in the correct format. - - Raises: - ValueError: If the date string does not match the 'YYYY-MM-DD' format. - """ - date_pattern = r"^\d{4}-\d{2}-\d{2}$" # Pattern for YYYY-MM-DD - - if not re.match(date_pattern, date_str): - raise ValueError("Invalid date format. Please use YYYY-MM-DD") - - return True + return epiweek_number(date_str) diff --git a/epydem/incidence.py b/epydem/incidence.py new file mode 100644 index 0000000..da37631 --- /dev/null +++ b/epydem/incidence.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import Literal + +import pandas as pd + +from .time import epiweek, mmwr_week1_start + +EpiFreq = Literal["D", "W-MMWR"] +OutputFormat = Literal["long", "wide"] + + +@dataclass(frozen=True) +class IncidenceSpec: + date_col: str = "date" + freq: EpiFreq = "W-MMWR" + by: tuple[str, ...] = () + + +def incidence( + df: pd.DataFrame, + *, + date_col: str, + freq: EpiFreq = "W-MMWR", + by: Sequence[str] | None = None, + count_col: str = "cases", + output: OutputFormat = "wide", + fill_missing: bool = True, +) -> pd.DataFrame: + """Compute incidence counts from a line list. + + Args: + df: Line list dataframe. + date_col: Column containing dates. Supports values acceptable to `epydem.time.epiweek`. + freq: + - "D": daily counts by calendar date + - "W-MMWR": weekly counts by CDC/MMWR epiweek (returns epi_year + epi_week) + by: Optional stratification columns. + count_col: Name of count column in the output. + output: + - "wide" (default): pivot table style (DX-friendly) + - "long": tidy long-form table + fill_missing: If True, fill missing dates/weeks with 0 counts. + + Returns: + DataFrame in the requested output format. + + Notes: + - Performance: for weekly counts we compute epiweek for *unique* dates and map back, + avoiding a pure Python loop per row. + """ + + if by is None: + by_cols: list[str] = [] + else: + by_cols = list(by) + + if date_col not in df.columns: + raise KeyError(f"date_col not found: {date_col}") + + work = df.copy() + + if freq == "D": + work["date"] = pd.to_datetime(work[date_col]).dt.date + group_cols = by_cols + ["date"] + long = work.groupby(group_cols, dropna=False).size().rename(count_col).reset_index() + + if fill_missing: + all_dates = pd.date_range(long["date"].min(), long["date"].max(), freq="D").date + + if not by_cols: + long = ( + long.set_index("date") + .reindex(all_dates, fill_value=0) + .rename_axis("date") + .reset_index() + ) + else: + # Fill missing dates per stratum combination. + strata = long[by_cols].drop_duplicates() + strata["_k"] = 1 + full_dates = pd.DataFrame({"date": list(all_dates)}) + full_dates["_k"] = 1 + full = strata.merge(full_dates, on="_k", how="outer").drop(columns=["_k"]) + + long = full.merge(long, on=by_cols + ["date"], how="left") + long[count_col] = long[count_col].fillna(0).astype(int) + + if output == "long": + return long + + # wide + if by_cols: + wide = long.pivot_table( + index="date", + columns=by_cols, + values=count_col, + aggfunc="sum", + fill_value=0, + ) + else: + wide = long.set_index("date")[[count_col]] + + return wide.sort_index() + + if freq == "W-MMWR": + # Compute (epi_year, epi_week) for unique dates, then map back for performance. + uniq = pd.unique(work[date_col]) + mapping = {v: epiweek(v) for v in uniq} + epi_pairs = work[date_col].map(mapping) + + work["epi_year"] = epi_pairs.map(lambda t: t[0]) + work["epi_week"] = epi_pairs.map(lambda t: t[1]) + + group_cols = by_cols + ["epi_year", "epi_week"] + long = work.groupby(group_cols, dropna=False).size().rename(count_col).reset_index() + + if fill_missing: + # Fill missing epiweeks between min and max observed. + long = long.sort_values(["epi_year", "epi_week"]).reset_index(drop=True) + start_y, start_w = int(long.iloc[0]["epi_year"]), int(long.iloc[0]["epi_week"]) + end_y, end_w = int(long.iloc[-1]["epi_year"]), int(long.iloc[-1]["epi_week"]) + + # Build the full sequence of (y,w) by stepping Sundays. + start_date = mmwr_week1_start(start_y) + timedelta(days=(start_w - 1) * 7) + end_date = mmwr_week1_start(end_y) + timedelta(days=(end_w - 1) * 7) + + full_pairs = [] + d = start_date + while d <= end_date: + full_pairs.append(epiweek(d)) + d += timedelta(days=7) + + idx = pd.MultiIndex.from_tuples(full_pairs, names=["epi_year", "epi_week"]) + + if not by_cols: + long = ( + long.set_index(["epi_year", "epi_week"]) + .reindex(idx, fill_value=0) + .reset_index() + ) + else: + # Fill missing epiweeks per stratum combination. + strata = long[by_cols].drop_duplicates() + strata["_k"] = 1 + full_time = idx.to_frame(index=False) + full_time["_k"] = 1 + full = strata.merge(full_time, on="_k", how="outer").drop(columns=["_k"]) + + long = full.merge(long, on=by_cols + ["epi_year", "epi_week"], how="left") + long[count_col] = long[count_col].fillna(0).astype(int) + + if output == "long": + return long + + # wide + if by_cols: + wide = long.pivot_table( + index=["epi_year", "epi_week"], + columns=by_cols, + values=count_col, + aggfunc="sum", + fill_value=0, + ) + else: + wide = long.set_index(["epi_year", "epi_week"])[[count_col]] + + return wide.sort_index() + + raise ValueError(f"Unknown freq: {freq}") diff --git a/epydem/summary.py b/epydem/summary.py new file mode 100644 index 0000000..7b39632 --- /dev/null +++ b/epydem/summary.py @@ -0,0 +1,191 @@ +"""Descriptive summary statistics for epidemiological DataFrames.""" + +from __future__ import annotations + +from typing import Optional, Union + +import pandas as pd + + +def summary( + df: pd.DataFrame, + by: Optional[Union[str, list[str]]] = None, + date_cols: Optional[list[str]] = None, + numeric_cols: Optional[list[str]] = None, + categorical_cols: Optional[list[str]] = None, + top_k: int = 3, + output: str = "long", +) -> pd.DataFrame: + """Compute descriptive summary statistics for an epidemiological DataFrame. + + Args: + df: Input DataFrame. + by: Column name(s) to group by. If None, summarise the whole frame. + date_cols: Date columns to summarise (min, max after coercion). + numeric_cols: Numeric columns to summarise (count, mean, std, quartiles). + categorical_cols: Categorical columns to summarise (top-k values). + top_k: Number of top categories to report (default 3). + output: ``"long"`` (default) or ``"wide"``. + + Returns: + A DataFrame with summary statistics in the requested format. + + Long format columns: ``by…``, ``column``, ``metric``, ``value``. + Wide format: pivoted so metrics become columns, indexed by ``by…`` + ``column``. + """ + date_cols = date_cols or [] + numeric_cols = numeric_cols or [] + categorical_cols = categorical_cols or [] + + if by is None: + by_cols: list[str] = [] + elif isinstance(by, str): + by_cols = [by] + else: + by_cols = list(by) + + groups = df.groupby(by_cols, sort=True) if by_cols else [(None, df)] + + all_rows: list[dict] = [] + + for key, group in groups: + by_dict = _by_dict(by_cols, key) + n = len(group) + all_rows.append({**by_dict, "column": "_n", "metric": "n", "value": str(n)}) + + for c in date_cols: + all_rows.extend(_date_metrics(group, c, by_dict)) + + for c in numeric_cols: + all_rows.extend(_numeric_metrics(group, c, by_dict)) + + for c in categorical_cols: + all_rows.extend(_categorical_metrics(group, c, by_dict, top_k)) + + result = pd.DataFrame(all_rows) + + if output == "wide": + index_cols = by_cols + ["column"] + result = result.pivot_table( + index=index_cols, columns="metric", values="value", aggfunc="first", + ).reset_index() + result.columns.name = None + + return result + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _by_dict(by_cols: list[str], key) -> dict: + """Build a dict mapping by-column names to their group key values.""" + if not by_cols: + return {} + if len(by_cols) == 1: + # groupby with a list always yields tuple keys; unpack single-element. + val = key[0] if isinstance(key, tuple) else key + return {by_cols[0]: val} + return dict(zip(by_cols, key)) + + +def _missingness(series: pd.Series) -> tuple[int, float]: + """Return (missing_n, missing_pct) for a series.""" + missing_n = int(series.isna().sum()) + missing_pct = round(missing_n / len(series) * 100, 2) if len(series) > 0 else 0.0 + return missing_n, missing_pct + + +def _date_metrics(group: pd.DataFrame, col: str, by_dict: dict) -> list[dict]: + """Compute date-column metrics: missingness, min, max (post-coercion).""" + coerced = pd.to_datetime(group[col], errors="coerce") + missing_n, missing_pct = _missingness(coerced) + + rows = [ + {**by_dict, "column": col, "metric": "missing_n", "value": str(missing_n)}, + {**by_dict, "column": col, "metric": "missing_pct", "value": str(missing_pct)}, + ] + valid = coerced.dropna() + if len(valid) > 0: + rows.append( + {**by_dict, "column": col, "metric": "min", "value": str(valid.min())} + ) + rows.append( + {**by_dict, "column": col, "metric": "max", "value": str(valid.max())} + ) + else: + rows.append({**by_dict, "column": col, "metric": "min", "value": ""}) + rows.append({**by_dict, "column": col, "metric": "max", "value": ""}) + return rows + + +def _numeric_metrics(group: pd.DataFrame, col: str, by_dict: dict) -> list[dict]: + """Compute numeric-column metrics: missingness and descriptive stats.""" + series = pd.to_numeric(group[col], errors="coerce").astype(float) + missing_n, missing_pct = _missingness(series) + + rows = [ + {**by_dict, "column": col, "metric": "missing_n", "value": str(missing_n)}, + {**by_dict, "column": col, "metric": "missing_pct", "value": str(missing_pct)}, + ] + valid = series.dropna() + if len(valid) > 0: + std_val = valid.std() + std_str = "" if pd.isna(std_val) else str(round(std_val, 4)) + rows.extend([ + {**by_dict, "column": col, "metric": "count", "value": str(len(valid))}, + {**by_dict, "column": col, "metric": "mean", "value": str(round(valid.mean(), 4))}, + {**by_dict, "column": col, "metric": "std", "value": std_str}, + {**by_dict, "column": col, "metric": "min", "value": str(valid.min())}, + {**by_dict, "column": col, "metric": "p25", "value": str(valid.quantile(0.25))}, + {**by_dict, "column": col, "metric": "median", "value": str(valid.median())}, + {**by_dict, "column": col, "metric": "p75", "value": str(valid.quantile(0.75))}, + {**by_dict, "column": col, "metric": "max", "value": str(valid.max())}, + ]) + else: + for m in ("count", "mean", "std", "min", "p25", "median", "p75", "max"): + rows.append({**by_dict, "column": col, "metric": m, "value": ""}) + return rows + + +def _categorical_metrics( + group: pd.DataFrame, col: str, by_dict: dict, top_k: int +) -> list[dict]: + """Compute categorical-column metrics: missingness and top-k values. + + Deterministic tie-break: count descending, then string(value) ascending. + Missing values are represented as ````. + """ + series = group[col].copy() + missing_n, missing_pct = _missingness(series) + rows = [ + {**by_dict, "column": col, "metric": "missing_n", "value": str(missing_n)}, + {**by_dict, "column": col, "metric": "missing_pct", "value": str(missing_pct)}, + ] + + filled = series.fillna("") + counts = filled.value_counts() + + # Deterministic tie-break: count desc, then string(value) asc + sorted_items = sorted( + counts.items(), key=lambda x: (-x[1], str(x[0])) + ) + + for rank in range(1, top_k + 1): + if rank <= len(sorted_items): + val, cnt = sorted_items[rank - 1] + rows.append( + {**by_dict, "column": col, "metric": f"top_{rank}", "value": str(val)} + ) + rows.append( + {**by_dict, "column": col, "metric": f"top_{rank}_n", "value": str(cnt)} + ) + else: + rows.append( + {**by_dict, "column": col, "metric": f"top_{rank}", "value": ""} + ) + rows.append( + {**by_dict, "column": col, "metric": f"top_{rank}_n", "value": ""} + ) + return rows diff --git a/epydem/time.py b/epydem/time.py new file mode 100644 index 0000000..3aa8904 --- /dev/null +++ b/epydem/time.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import re +from datetime import date, datetime, timedelta +from typing import Literal + +DateLike = str | date | datetime +EpiWeekSystem = Literal["mmwr"] + +_DATE_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +def parse_ymd(value: DateLike) -> date: + """Parse a date-like value into a `datetime.date`. + + Supported inputs: + - `datetime.date` + - `datetime.datetime` (date portion is used) + - `str` in YYYY-MM-DD format + """ + + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + if isinstance(value, str): + if not _DATE_PATTERN.match(value): + raise ValueError("Invalid date format. Please use YYYY-MM-DD") + return datetime.strptime(value, "%Y-%m-%d").date() + + raise TypeError(f"Unsupported type for date: {type(value)!r}") + + +def mmwr_week1_start(year: int) -> date: + """Start date (Sunday) of CDC/MMWR week 1 for the given calendar year. + + CDC/MMWR definition: + - Weeks start on Sunday. + - Week 1 is the week that contains January 4. + + Therefore: week 1 starts on the Sunday on or before Jan 4. + """ + + jan4 = date(year, 1, 4) + # Python weekday: Monday=0 .. Sunday=6. Offset back to Sunday. + days_since_sunday = (jan4.weekday() + 1) % 7 + return jan4 - timedelta(days=days_since_sunday) + + +def mmwr_week(value: DateLike) -> tuple[int, int]: + """Compute CDC/MMWR epidemiological week for a date. + + Returns: + (mmwr_year, mmwr_week) + + Notes: + - Weeks start Sunday. + - Week 1 is the week containing Jan 4. + - The MMWR year is *not* always the calendar year of the date. + Example: 2023-12-31 is the start of 2024 week 1. + + Implementation rule: + - Find the unique `mmwr_year` such that: + week1_start(mmwr_year) <= d < week1_start(mmwr_year + 1) + """ + + d = parse_ymd(value) + + year = d.year + start = mmwr_week1_start(year) + start_next = mmwr_week1_start(year + 1) + + if d < start: + year -= 1 + start = mmwr_week1_start(year) + elif d >= start_next: + year += 1 + start = start_next + + week = ((d - start).days // 7) + 1 + return year, week + + +def epiweek(value: DateLike, system: EpiWeekSystem = "mmwr") -> tuple[int, int]: + """Compute epidemiological week. + + Currently supported systems: + - "mmwr" (CDC/MMWR): week starts Sunday, week 1 contains Jan 4. + + Returns: + (epi_year, epi_week) + + Rationale for returning a tuple: + - Epi week numbering crosses calendar-year boundaries. + - A week number alone is ambiguous without its epidemiological year. + """ + + if system == "mmwr": + return mmwr_week(value) + raise ValueError(f"Unknown epiweek system: {system}") + + +def epiweek_number(value: DateLike, system: EpiWeekSystem = "mmwr") -> int: + """Convenience wrapper returning week number only.""" + + _y, w = epiweek(value, system=system) + return w diff --git a/epydem/transform.py b/epydem/transform.py new file mode 100644 index 0000000..325c763 --- /dev/null +++ b/epydem/transform.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Literal + +import pandas as pd + +RollingKind = Literal["sum", "mean"] + + +@dataclass(frozen=True) +class TransformSpec: + rolling: int | None = None + rolling_kind: RollingKind = "sum" + min_periods: int = 1 + center: bool = False + cumulative: bool = False + + +def transform_incidence( + data: pd.DataFrame, + *, + rolling: int | None = None, + rolling_kind: RollingKind = "sum", + min_periods: int = 1, + center: bool = False, + cumulative: bool = False, + time_cols: Sequence[str] | None = None, +) -> pd.DataFrame: + """Apply time-series transforms to incidence outputs. + + This is intentionally separate from `incidence()` to keep `incidence()` as a + stable counts primitive. + + Supported inputs: + - Wide incidence output: index is time (e.g. date or (epi_year, epi_week)). + - Long incidence output: requires `time_cols` (or inferred), and transforms + are applied per non-time column group. + + Args: + data: Output from `epydem.incidence(...)`. + rolling: Rolling window size. + rolling_kind: "sum" or "mean". + min_periods: Passed to pandas rolling. + center: Passed to pandas rolling. + cumulative: If True, apply cumulative sum. + time_cols: For long-form data, the time columns. If None, we try to infer + ("date") or ("epi_year", "epi_week"). + + Returns: + Transformed DataFrame with the same shape/schema as input. + """ + + if rolling is None and not cumulative: + return data + + # Wide: DataFrameIndex is time. + if time_cols is None and ("date" not in data.columns) and ("epi_year" not in data.columns): + wide = data.sort_index() + if rolling is not None: + r = wide.rolling(window=rolling, min_periods=min_periods, center=center) + wide = r.sum() if rolling_kind == "sum" else r.mean() + if cumulative: + wide = wide.cumsum() + return wide + + # Long: operate on a value column per group. + df = data.copy() + + if time_cols is None: + if "date" in df.columns: + time_cols = ["date"] + elif ("epi_year" in df.columns) and ("epi_week" in df.columns): + time_cols = ["epi_year", "epi_week"] + else: + raise ValueError("Cannot infer time_cols for long-form incidence") + + value_cols = [c for c in df.columns if c not in set(time_cols)] + if len(value_cols) != 1: + raise ValueError( + "Long-form incidence must have exactly one value column (e.g., 'cases'). " + f"Got {value_cols}" + ) + value_col = value_cols[0] + + group_cols = [c for c in df.columns if c not in set(time_cols + [value_col])] + + df = df.sort_values(list(time_cols)) + + def _apply(group: pd.DataFrame) -> pd.DataFrame: + s = group[value_col] + if rolling is not None: + r = s.rolling(window=rolling, min_periods=min_periods, center=center) + s = r.sum() if rolling_kind == "sum" else r.mean() + if cumulative: + s = s.cumsum() + group[value_col] = s + return group + + if group_cols: + out = df.groupby(group_cols, dropna=False, sort=False, group_keys=False).apply(_apply) + else: + out = _apply(df) + + return out diff --git a/tests/test_epiweek.py b/tests/test_epiweek.py index f3443a1..425ccc9 100644 --- a/tests/test_epiweek.py +++ b/tests/test_epiweek.py @@ -1,283 +1,55 @@ -from datetime import datetime +from __future__ import annotations + +from datetime import date, datetime import pytest import epydem -from epydem.epiweek import _verify_date_str - - -class TestCalculate: - """Test cases for the calculate function.""" - - def test_basic_calculation(self): - """Test basic epidemiological week calculation.""" - # 2024: January 1st is Monday, first Thursday is January 4th - # Week 1 starts on Sunday December 31st, 2023 - assert epydem.calculate("2024-01-01") == 1 - assert epydem.calculate("2024-01-04") == 1 # First Thursday - assert epydem.calculate("2024-01-07") == 1 # First Sunday - assert epydem.calculate("2024-01-08") == 2 # Second Monday - - def test_week_boundaries(self): - """Test week boundary calculations.""" - # Test various days within the same week - assert epydem.calculate("2024-01-01") == 1 # Monday - assert epydem.calculate("2024-01-02") == 1 # Tuesday - assert epydem.calculate("2024-01-03") == 1 # Wednesday - assert epydem.calculate("2024-01-04") == 1 # Thursday - assert epydem.calculate("2024-01-05") == 1 # Friday - assert epydem.calculate("2024-01-06") == 1 # Saturday - assert epydem.calculate("2024-01-07") == 1 # Sunday - - def test_different_years(self): - """Test calculations for different years.""" - # Test different year scenarios - assert ( - epydem.calculate("2023-01-01") == 0 - ) # 2023: Jan 1st is Sunday, before epi week 1 - assert ( - epydem.calculate("2025-01-01") == 1 - ) # 2025: Jan 1st is Wednesday, in epi week 1 - assert ( - epydem.calculate("2022-01-01") == 0 - ) # 2022: Jan 1st is Saturday, before epi year - - def test_year_end_scenarios(self): - """Test end of year scenarios.""" - # December dates that might be in the next year's epi week 1 - assert epydem.calculate("2023-12-31") == 52 # Late in 2023's epi year - - def test_week_zero_cases(self): - """Test cases that should return week 0.""" - # Dates before the epidemiological year starts - assert epydem.calculate("2022-01-01") == 0 # Saturday, before epi year - assert epydem.calculate("2022-01-02") == 0 # Sunday, week 1 starts Jan 3rd - - def test_mid_year_calculations(self): - """Test calculations for various dates throughout the year.""" - # Test some mid-year dates - assert epydem.calculate("2024-06-01") > 20 # Should be in the 20s - assert epydem.calculate("2024-12-01") > 45 # Should be in the 40s or 50s - - def test_leap_year(self): - """Test leap year handling.""" - # 2024 is a leap year - assert epydem.calculate("2024-02-29") > 0 # Should handle leap day - assert epydem.calculate("2024-03-01") > 0 - - def test_consistent_week_progression(self): - """Test that week numbers progress consistently.""" - # Test a sequence of dates to ensure proper progression - week1 = epydem.calculate("2024-01-01") - week2 = epydem.calculate("2024-01-08") - week3 = epydem.calculate("2024-01-15") - - assert week2 == week1 + 1 - assert week3 == week2 + 1 - - -class TestVerifyDateStr: - """Test cases for date string validation.""" - - def test_valid_date_formats(self): - """Test valid date string formats.""" - # These should not raise exceptions - epydem.calculate("2024-01-01") - epydem.calculate("2024-12-31") - epydem.calculate("2023-02-28") - epydem.calculate("2024-02-29") # Leap year - - def test_invalid_date_formats(self): - """Test invalid date string formats.""" - with pytest.raises(ValueError, match="Invalid date format"): - epydem.calculate("24-01-01") # Wrong year format - - with pytest.raises(ValueError, match="Invalid date format"): - epydem.calculate("2024-1-1") # Wrong month/day format - - with pytest.raises(ValueError, match="Invalid date format"): - epydem.calculate("2024/01/01") # Wrong separator - - with pytest.raises(ValueError, match="Invalid date format"): - epydem.calculate("01-01-2024") # Wrong order - with pytest.raises(ValueError): - epydem.calculate("2024-01-32") # Invalid day - with pytest.raises(ValueError): - epydem.calculate("2024-13-01") # Invalid month +class TestParseYmd: + def test_accepts_date(self): + assert epydem.parse_ymd(date(2024, 1, 1)) == date(2024, 1, 1) - def test_invalid_types(self): - """Test invalid input types.""" - with pytest.raises((AttributeError, TypeError)): - epydem.calculate(20240101) # Integer instead of string + def test_accepts_datetime(self): + assert epydem.parse_ymd(datetime(2024, 1, 1, 12, 30)) == date(2024, 1, 1) - with pytest.raises((AttributeError, TypeError)): - epydem.calculate(None) # None instead of string + def test_accepts_ymd_string(self): + assert epydem.parse_ymd("2024-01-01") == date(2024, 1, 1) - def test_empty_and_malformed_strings(self): - """Test empty and malformed strings.""" - with pytest.raises(ValueError, match="Invalid date format"): - epydem.calculate("") # Empty string + def test_rejects_bad_string(self): + with pytest.raises(ValueError, match=r"Invalid date format"): + epydem.parse_ymd("2024/01/01") - with pytest.raises(ValueError, match="Invalid date format"): - epydem.calculate("not-a-date") # Random string + def test_rejects_bad_type(self): + with pytest.raises(TypeError): + epydem.parse_ymd(20240101) # type: ignore[arg-type] - with pytest.raises(ValueError, match="Invalid date format"): - epydem.calculate("2024-01-01 10:30:00") # Too much info +class TestEpiweekMmwr: + def test_week1_contains_jan4(self): + # MMWR week 1 starts on Sunday on/before Jan 4. + assert epydem.mmwr_week1_start(2024) == date(2023, 12, 31) + assert epydem.mmwr_week1_start(2023) == date(2023, 1, 1) + assert epydem.mmwr_week1_start(2022) == date(2022, 1, 2) -class TestEdgeCases: - """Test edge cases and boundary conditions.""" + def test_known_boundaries_tuple_output(self): + # Early January can belong to previous MMWR year. + assert epydem.epiweek("2022-01-01") == (2021, 52) + assert epydem.epiweek("2022-01-02") == (2022, 1) - def test_first_day_scenarios(self): - """Test first day of year scenarios for different weekdays.""" - # Test years where Jan 1st falls on different weekdays - test_cases = [ - ("2023-01-01", 0), # Sunday - before epi week 1 - ("2024-01-01", 1), # Monday - in epi week 1 - ("2025-01-01", 1), # Wednesday - in epi week 1 - ] + # New year transitions. + assert epydem.epiweek("2023-01-01") == (2023, 1) + # 2023-12-31 is the start of 2024 week 1 under CDC/MMWR. + assert epydem.epiweek("2023-12-31") == (2024, 1) - for date_str, expected_week in test_cases: - assert epydem.calculate(date_str) == expected_week + assert epydem.epiweek("2024-01-01") == (2024, 1) + assert epydem.epiweek("2024-01-07") == (2024, 2) + assert epydem.epiweek("2024-12-31") == (2025, 1) - def test_year_transition(self): - """Test the transition between epidemiological years.""" - # The end of 2023's epi year and start of 2024's - assert epydem.calculate("2023-12-30") >= 52 # Late in 2023's epi year - assert epydem.calculate("2023-12-31") == 52 # Late in 2023's epi year - assert epydem.calculate("2024-01-01") == 1 # Start of 2024's week 1 + assert epydem.epiweek("2026-01-01") == (2025, 53) - def test_maximum_week_numbers(self): - """Test that week numbers don't exceed expected maximums.""" - # Test various dates to ensure no week number is > 53 - test_dates = [ - "2024-12-31", - "2024-12-30", - "2024-12-29", - "2023-12-31", - "2023-12-30", - "2023-12-29", - "2025-12-31", - "2025-12-30", - "2025-12-29", - ] - - for date_str in test_dates: - week_num = epydem.calculate(date_str) - assert ( - 0 <= week_num <= 53 - ), f"Week number {week_num} out of range for {date_str}" - - -class TestPrivateVerifyFunction: - """Test cases for the private _verify_date_str function.""" - - def test_verify_valid_formats(self): - """Test that _verify_date_str accepts valid formats.""" - assert _verify_date_str("2024-01-01") == True - assert _verify_date_str("2023-12-31") == True - assert _verify_date_str("2000-02-29") == True # Leap year - assert _verify_date_str("1999-01-01") == True - - def test_verify_invalid_formats(self): - """Test that _verify_date_str rejects invalid formats.""" - invalid_formats = [ - "24-01-01", # Wrong year format - "2024-1-01", # Wrong month format - "2024-01-1", # Wrong day format - "2024/01/01", # Wrong separator - "2024-01", # Missing day - "01-01-2024", # Wrong order - "not-a-date", # Not a date - "", # Empty string - "2024-01-01T10:30", # With time - ] - - for invalid_date in invalid_formats: - with pytest.raises(ValueError, match="Invalid date format"): - _verify_date_str(invalid_date) - - -class TestSpecificYearScenarios: - """Test specific year scenarios to ensure accuracy.""" - - def test_2023_specific_dates(self): - """Test specific dates for 2023 (Jan 1 is Sunday).""" - # 2023: Jan 1 is Sunday, first Thursday is Jan 5 - # Week 1 starts on Jan 2 (Monday) - assert epydem.calculate("2023-01-01") == 0 # Sunday - before epi week 1 - assert epydem.calculate("2023-01-02") == 1 # Monday - week 1 start - assert epydem.calculate("2023-01-05") == 1 # Thursday of week 1 - assert epydem.calculate("2023-01-08") == 1 # Sunday of week 1 - - def test_2024_specific_dates(self): - """Test specific dates for 2024 (Jan 1 is Monday).""" - # 2024: Jan 1 is Monday, first Thursday is Jan 4 - # Week 1 starts on Dec 31, 2023 (Sunday) - assert epydem.calculate("2024-01-01") == 1 # Monday of week 1 - assert epydem.calculate("2024-01-04") == 1 # Thursday of week 1 - assert epydem.calculate("2024-01-07") == 1 # Sunday of week 1 - assert epydem.calculate("2024-01-08") == 2 # Monday - week 2 start - - def test_2022_specific_dates(self): - """Test specific dates for 2022 (Jan 1 is Saturday).""" - # 2022: Jan 1 is Saturday, first Thursday is Jan 6 - # Week 1 starts on Jan 3, 2022 (Monday) - assert epydem.calculate("2022-01-01") == 0 # Saturday before epi year - assert epydem.calculate("2022-01-02") == 0 # Sunday before epi year - assert epydem.calculate("2022-01-03") == 1 # Monday - week 1 start - assert epydem.calculate("2022-01-06") == 1 # Thursday of week 1 - assert epydem.calculate("2022-01-09") == 1 # Sunday of week 1 - - -class TestComprehensiveValidation: - """Comprehensive validation tests.""" - - def test_all_months_valid(self): - """Test that all months work correctly.""" - months = [ - "01", - "02", - "03", - "04", - "05", - "06", - "07", - "08", - "09", - "10", - "11", - "12", - ] - - for month in months: - date_str = f"2024-{month}-15" - result = epydem.calculate(date_str) - assert 1 <= result <= 53, f"Invalid week {result} for {date_str}" - - def test_sequential_days_progression(self): - """Test that sequential days show proper week progression.""" - # Test a full week transition - dates_and_expected = [ - ("2024-01-07", 1), # Sunday of week 1 - ("2024-01-08", 2), # Monday of week 2 - ("2024-01-14", 2), # Sunday of week 2 - ("2024-01-15", 3), # Monday of week 3 - ] - - for date_str, expected_week in dates_and_expected: - assert epydem.calculate(date_str) == expected_week - - def test_different_years_july_first(self): - """Test July 1st across different years for consistency.""" - years = ["2020", "2021", "2022", "2023", "2024", "2025"] - - for year in years: - date_str = f"{year}-07-01" - result = epydem.calculate(date_str) - # July 1st should always be in week 26-28 range - assert ( - 25 <= result <= 29 - ), f"Week {result} for {date_str} seems out of range" + def test_week_number_wrapper(self): + assert epydem.epiweek_number("2024-01-01") == 1 + assert epydem.calculate("2024-01-01") == 1 + assert epydem.calculate("2022-01-01") == 52 diff --git a/tests/test_incidence.py b/tests/test_incidence.py new file mode 100644 index 0000000..fb30e56 --- /dev/null +++ b/tests/test_incidence.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import pandas as pd + +import epydem + + +def test_incidence_daily_basic(): + df = pd.DataFrame( + { + "onset_date": [ + "2024-01-01", + "2024-01-01", + "2024-01-02", + ], + "sex": ["M", "F", "M"], + } + ) + + out = epydem.incidence(df, date_col="onset_date", freq="D") + # default output is wide + assert list(out.columns) == ["cases"] + + assert out.loc[pd.to_datetime("2024-01-01").date(), "cases"] == 2 + assert out.loc[pd.to_datetime("2024-01-02").date(), "cases"] == 1 + + +def test_incidence_weekly_mmwr_with_strata_fills_missing(): + # Create a gap: only week 1 has cases for stratum B. + df = pd.DataFrame( + { + "onset_date": [ + "2024-01-01", # (2024,1) + "2024-01-01", + "2024-01-08", # (2024,2) + ], + "province": ["A", "B", "A"], + } + ) + + out = epydem.incidence( + df, + date_col="onset_date", + freq="W-MMWR", + by=["province"], + fill_missing=True, + ) + + # Both strata should have rows for both weeks. + assert out.loc[(2024, 1), "A"] == 1 + assert out.loc[(2024, 2), "A"] == 1 + + assert out.loc[(2024, 1), "B"] == 1 + assert out.loc[(2024, 2), "B"] == 0 + + # Long format remains available + long = epydem.incidence( + df, + date_col="onset_date", + freq="W-MMWR", + by=["province"], + output="long", + fill_missing=True, + ) + assert set(long.columns) == {"province", "epi_year", "epi_week", "cases"} diff --git a/tests/test_incidence_phase2_1.py b/tests/test_incidence_phase2_1.py new file mode 100644 index 0000000..99ea2cd --- /dev/null +++ b/tests/test_incidence_phase2_1.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import pandas as pd + +import epydem + + +def test_incidence_weekly_rolling_and_cumulative(): + df = pd.DataFrame( + { + "onset_date": [ + "2024-01-01", # (2024,1) + "2024-01-08", # (2024,2) + "2024-01-08", + ], + "province": ["A", "A", "A"], + } + ) + + base = epydem.incidence( + df, + date_col="onset_date", + freq="W-MMWR", + by=["province"], + fill_missing=True, + ) + # week1=1, week2=2 + assert base.loc[(2024, 1), "A"] == 1 + assert base.loc[(2024, 2), "A"] == 2 + + roll2 = epydem.transform_incidence( + base, + rolling=2, + rolling_kind="sum", + ) + assert roll2.loc[(2024, 1), "A"] == 1 + assert roll2.loc[(2024, 2), "A"] == 3 + + cum = epydem.transform_incidence( + base, + cumulative=True, + ) + assert cum.loc[(2024, 1), "A"] == 1 + assert cum.loc[(2024, 2), "A"] == 3 diff --git a/tests/test_summary.py b/tests/test_summary.py new file mode 100644 index 0000000..ca4c013 --- /dev/null +++ b/tests/test_summary.py @@ -0,0 +1,258 @@ +"""Tests for epydem.summary().""" + +import pandas as pd +import pytest + +from epydem import summary + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _long_val(result: pd.DataFrame, column: str, metric: str) -> str: + """Extract a single value from long-format summary result.""" + mask = (result["column"] == column) & (result["metric"] == metric) + rows = result.loc[mask, "value"] + assert len(rows) == 1, f"Expected 1 row for ({column}, {metric}), got {len(rows)}" + return rows.iloc[0] + + +def _long_val_by(result: pd.DataFrame, by_val, column: str, metric: str) -> str: + """Extract a value from long-format with a single by-column.""" + by_col = [c for c in result.columns if c not in ("column", "metric", "value")][0] + mask = ( + (result[by_col] == by_val) + & (result["column"] == column) + & (result["metric"] == metric) + ) + rows = result.loc[mask, "value"] + assert len(rows) == 1 + return rows.iloc[0] + + +# --------------------------------------------------------------------------- +# 1. Default-only-n behavior +# --------------------------------------------------------------------------- + + +class TestDefaultOnlyN: + """When no column lists are specified, only n is returned.""" + + def test_no_cols_returns_only_n(self): + df = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) + result = summary(df) + assert len(result) == 1 + assert result.iloc[0]["column"] == "_n" + assert result.iloc[0]["metric"] == "n" + assert result.iloc[0]["value"] == "3" + + def test_no_cols_with_by_returns_n_per_group(self): + df = pd.DataFrame({"grp": ["A", "A", "B"], "x": [1, 2, 3]}) + result = summary(df, by="grp") + assert len(result) == 2 + assert set(result["grp"]) == {"A", "B"} + assert all(result["metric"] == "n") + assert _long_val_by(result, "A", "_n", "n") == "2" + assert _long_val_by(result, "B", "_n", "n") == "1" + + def test_empty_lists_same_as_none(self): + df = pd.DataFrame({"a": [1, 2]}) + result = summary(df, date_cols=[], numeric_cols=[], categorical_cols=[]) + assert len(result) == 1 + assert result.iloc[0]["metric"] == "n" + + +# --------------------------------------------------------------------------- +# 2. By vs no-by +# --------------------------------------------------------------------------- + + +class TestByVsNoBy: + """Verify grouping behavior.""" + + def test_no_by_single_group(self): + df = pd.DataFrame({"age": [10, 20, 30]}) + result = summary(df, numeric_cols=["age"]) + assert "column" in result.columns + # Should have n row + age metrics + n_row = result[result["column"] == "_n"] + assert len(n_row) == 1 + assert n_row.iloc[0]["value"] == "3" + + def test_by_single_column(self): + df = pd.DataFrame({ + "region": ["N", "N", "S", "S", "S"], + "cases": [1, 2, 3, 4, 5], + }) + result = summary(df, by="region", numeric_cols=["cases"]) + regions = result["region"].unique() + assert set(regions) == {"N", "S"} + + # n per group + assert _long_val_by(result, "N", "_n", "n") == "2" + assert _long_val_by(result, "S", "_n", "n") == "3" + + def test_by_multiple_columns(self): + df = pd.DataFrame({ + "region": ["N", "N", "S", "S"], + "year": [2020, 2020, 2020, 2021], + "val": [1, 2, 3, 4], + }) + result = summary(df, by=["region", "year"], numeric_cols=["val"]) + assert "region" in result.columns + assert "year" in result.columns + + +# --------------------------------------------------------------------------- +# 3. Date coercion +# --------------------------------------------------------------------------- + + +class TestDateCoercion: + """Date columns: coerce invalid -> NaT, count as missing.""" + + def test_valid_dates(self): + df = pd.DataFrame({"onset": ["2024-01-01", "2024-06-15", "2024-12-31"]}) + result = summary(df, date_cols=["onset"]) + assert _long_val(result, "onset", "missing_n") == "0" + assert _long_val(result, "onset", "missing_pct") == "0.0" + assert "2024-01-01" in _long_val(result, "onset", "min") + assert "2024-12-31" in _long_val(result, "onset", "max") + + def test_invalid_dates_become_missing(self): + df = pd.DataFrame({"onset": ["2024-01-01", "NOT_A_DATE", None]}) + result = summary(df, date_cols=["onset"]) + # Both None and "NOT_A_DATE" should be missing after coercion + assert _long_val(result, "onset", "missing_n") == "2" + assert _long_val(result, "onset", "missing_pct") == "66.67" + assert "2024-01-01" in _long_val(result, "onset", "min") + assert "2024-01-01" in _long_val(result, "onset", "max") + + def test_all_invalid_dates(self): + df = pd.DataFrame({"onset": ["bad", "worse", None]}) + result = summary(df, date_cols=["onset"]) + assert _long_val(result, "onset", "missing_n") == "3" + assert _long_val(result, "onset", "min") == "" + assert _long_val(result, "onset", "max") == "" + + +# --------------------------------------------------------------------------- +# 4. Numeric quartiles +# --------------------------------------------------------------------------- + + +class TestNumericQuartiles: + """Numeric columns: count, mean, std, min, p25, median, p75, max.""" + + def test_basic_numeric(self): + df = pd.DataFrame({"age": [10, 20, 30, 40, 50]}) + result = summary(df, numeric_cols=["age"]) + assert _long_val(result, "age", "count") == "5" + assert _long_val(result, "age", "mean") == "30.0" + assert _long_val(result, "age", "min") == "10.0" + assert _long_val(result, "age", "max") == "50.0" + assert _long_val(result, "age", "median") == "30.0" + assert float(_long_val(result, "age", "p25")) == 20.0 + assert float(_long_val(result, "age", "p75")) == 40.0 + + def test_numeric_with_missing(self): + df = pd.DataFrame({"age": [10, None, 30]}) + result = summary(df, numeric_cols=["age"]) + assert _long_val(result, "age", "missing_n") == "1" + assert _long_val(result, "age", "count") == "2" + assert _long_val(result, "age", "mean") == "20.0" + + def test_numeric_missingness_pct(self): + df = pd.DataFrame({"age": [None, None, 30, 40]}) + result = summary(df, numeric_cols=["age"]) + assert _long_val(result, "age", "missing_n") == "2" + assert _long_val(result, "age", "missing_pct") == "50.0" + + +# --------------------------------------------------------------------------- +# 5. Categorical tie-breaking +# --------------------------------------------------------------------------- + + +class TestCategoricalTieBreaking: + """Categorical: top-k with deterministic tie-break (count desc, str asc).""" + + def test_basic_top_k(self): + df = pd.DataFrame({"dx": ["flu", "flu", "flu", "covid", "covid", "rsv"]}) + result = summary(df, categorical_cols=["dx"]) + assert _long_val(result, "dx", "top_1") == "flu" + assert _long_val(result, "dx", "top_1_n") == "3" + assert _long_val(result, "dx", "top_2") == "covid" + assert _long_val(result, "dx", "top_2_n") == "2" + assert _long_val(result, "dx", "top_3") == "rsv" + assert _long_val(result, "dx", "top_3_n") == "1" + + def test_tie_break_alphabetical(self): + """When counts are equal, break ties by string value ascending.""" + df = pd.DataFrame({"dx": ["beta", "alpha", "beta", "alpha"]}) + result = summary(df, categorical_cols=["dx"]) + # Both have count=2; "alpha" < "beta" alphabetically + assert _long_val(result, "dx", "top_1") == "alpha" + assert _long_val(result, "dx", "top_2") == "beta" + + def test_missing_token_na(self): + """Missing values appear as in categorical top-k.""" + df = pd.DataFrame({"dx": [None, None, None, "flu", "flu"]}) + result = summary(df, categorical_cols=["dx"]) + assert _long_val(result, "dx", "missing_n") == "3" + assert _long_val(result, "dx", "top_1") == "" + assert _long_val(result, "dx", "top_1_n") == "3" + assert _long_val(result, "dx", "top_2") == "flu" + + def test_fewer_than_top_k(self): + """When there are fewer unique values than top_k, extra slots are empty.""" + df = pd.DataFrame({"dx": ["flu", "flu"]}) + result = summary(df, categorical_cols=["dx"], top_k=3) + assert _long_val(result, "dx", "top_1") == "flu" + assert _long_val(result, "dx", "top_2") == "" + assert _long_val(result, "dx", "top_3") == "" + + def test_custom_top_k(self): + df = pd.DataFrame({"dx": ["a", "a", "b", "b", "c", "d"]}) + result = summary(df, categorical_cols=["dx"], top_k=2) + # Only top_1 and top_2 should appear + metrics = result[result["column"] == "dx"]["metric"].tolist() + assert "top_1" in metrics + assert "top_2" in metrics + assert "top_3" not in metrics + + +# --------------------------------------------------------------------------- +# 6. Long and wide output +# --------------------------------------------------------------------------- + + +class TestOutputFormats: + """Verify long (default) and wide output formats.""" + + def test_long_schema(self): + df = pd.DataFrame({"age": [10, 20]}) + result = summary(df, numeric_cols=["age"]) + assert list(result.columns) == ["column", "metric", "value"] + + def test_long_with_by_schema(self): + df = pd.DataFrame({"grp": ["A", "B"], "age": [10, 20]}) + result = summary(df, by="grp", numeric_cols=["age"]) + assert result.columns[0] == "grp" + assert list(result.columns[-3:]) == ["column", "metric", "value"] + + def test_wide_output(self): + df = pd.DataFrame({"age": [10, 20, 30]}) + result = summary(df, numeric_cols=["age"], output="wide") + # Wide should have metric names as columns + assert "column" in result.columns + # Check that at least some metric columns exist + assert "mean" in result.columns or "count" in result.columns + + def test_wide_with_by(self): + df = pd.DataFrame({"grp": ["A", "A", "B"], "age": [10, 20, 30]}) + result = summary(df, by="grp", numeric_cols=["age"], output="wide") + assert "grp" in result.columns + assert "column" in result.columns