diff --git a/PR_PHASE3.md b/PR_PHASE3.md new file mode 100644 index 0000000..f644a2b --- /dev/null +++ b/PR_PHASE3.md @@ -0,0 +1,37 @@ +Phase 3: summary() (descriptive epidemiology) + +Key points +- Add `epydem.summary(df, by=..., date_cols=..., numeric_cols=..., categorical_cols=...)`. +- Supports stratified summaries via `by=[...]`. +- Long (tidy) output by default; optional `output="wide"`. + +WHY +- After incidence/epicurve, the next most common need is quick descriptive EDA: missingness, date ranges, numeric distributions, and top categories. +- A standardized summary makes notebooks and reports faster and more consistent. + +WHAT +- Implemented metrics: + - group size `n` + - missingness: `missing_n`, `missing_pct` + - date columns: `min`, `max` + - numeric columns: `count`, `mean`, `std`, `min`, `p25`, `median`, `p75`, `max` + - categorical columns: `top_1..top_k` + `top_#_n` + +Multi-role debate (differences, not consensus) + +Role A โ€” pragmatic developer +- ๐Ÿ‘ Likes: covers 80% EDA quickly; configurable columns; works stratified. +- โš ๏ธ Concern: output schema can grow; must keep naming stable. + +Role B โ€” architecture +- ๐Ÿ‘ Likes: explicit outputs; can be extended with typed spec objects later. +- โš ๏ธ Concern: inference heuristics (auto-detect columns) might be surprising; keep it minimal. + +Role C โ€” developer user (DX) +- ๐Ÿ‘ Likes: one function for a full โ€œfirst lookโ€ summary. +- โš ๏ธ Concern: wants better defaults and nicer presentation (markdown tables, formatting), but those can be separate utilities. + +Points of divergence to revisit later +1) Whether to provide a pretty-printed report (markdown) vs raw DataFrame only. +2) How aggressive to make column inference defaults. +3) Support for weighted summaries / population-standardized metrics. diff --git a/README.md b/README.md index 2732ba6..edc5fa2 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ pip install -e '.[dev]' ## Usage ```python +import pandas as pd import epydem year, week = epydem.epiweek("2024-01-01") @@ -32,10 +33,47 @@ week_only = epydem.epiweek_number("2024-01-01") print(week_only) # Incidence (line list -> counts) -# df = pandas.DataFrame({"onset_date": [...], "sex": [...]}) -# weekly = epydem.incidence(df, date_col="onset_date", freq="W-MMWR", by=["sex"], fill_missing=True) -# weekly_rolling2 = epydem.transform_incidence(weekly, rolling=2) -# weekly_cum = epydem.transform_incidence(weekly, cumulative=True) +df = pd.DataFrame({"onset_date": ["2024-01-01", "2024-01-02"], "sex": ["M", "F"]}) +weekly = epydem.incidence(df, date_col="onset_date", freq="W-MMWR", by=["sex"], fill_missing=True) +weekly_rolling2 = epydem.transform_incidence(weekly, rolling=2) +weekly_cum = epydem.transform_incidence(weekly, cumulative=True) +``` + +## Descriptive summary + +`summary()` returns a DataFrame (raw stats). `summary_markdown()` returns a pretty Markdown table +string for quick sharing. + +```python +import pandas as pd +import epydem + +df = pd.DataFrame( + { + "onset_date": ["2024-01-01", "2024-01-02", None], + "age": [10, 20, None], + "sex": ["M", "F", "F"], + } +) + +# Raw DataFrame summary (long/tidy by default) +out = epydem.summary( + df, + by=["sex"], + date_cols=["onset_date"], + numeric_cols=["age"], + categorical_cols=["sex"], +) + +# Pretty report (Markdown table) +md = epydem.summary_markdown( + df, + by=["sex"], + date_cols=["onset_date"], + numeric_cols=["age"], + categorical_cols=["sex"], +) +print(md) ``` ## Roadmap (high level) diff --git a/epydem/__init__.py b/epydem/__init__.py index 09ebc34..85212c4 100644 --- a/epydem/__init__.py +++ b/epydem/__init__.py @@ -2,6 +2,8 @@ from .epiweek import calculate from .incidence import incidence +from .summary import summary +from .summary_report import summary_markdown from .time import epiweek, epiweek_number, mmwr_week, mmwr_week1_start, parse_ymd from .transform import transform_incidence @@ -14,4 +16,6 @@ "parse_ymd", "incidence", "transform_incidence", + "summary", + "summary_markdown", ] diff --git a/epydem/formatting.py b/epydem/formatting.py new file mode 100644 index 0000000..2f7b544 --- /dev/null +++ b/epydem/formatting.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from collections.abc import Sequence + + +def to_markdown_table(headers: Sequence[str], rows: Sequence[Sequence[str]]) -> str: + """Render a simple GitHub-flavored Markdown table. + + Avoids optional deps like `tabulate`. + """ + + headers = list(headers) + row_strs = [[str(x) for x in r] for r in rows] + + # Escape pipes minimally. + def esc(s: str) -> str: + return s.replace("|", "\\|") + + header_line = "| " + " | ".join(esc(h) for h in headers) + " |" + sep_line = "| " + " | ".join(["---"] * len(headers)) + " |" + body_lines = ["| " + " | ".join(esc(c) for c in r) + " |" for r in row_strs] + + return "\n".join([header_line, sep_line, *body_lines]) diff --git a/epydem/summary.py b/epydem/summary.py new file mode 100644 index 0000000..3c33a26 --- /dev/null +++ b/epydem/summary.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Literal + +import pandas as pd + +OutputFormat = Literal["long", "wide"] + + +@dataclass(frozen=True) +class SummarySpec: + """Configuration for `summary()`. + + This will likely evolve; keep it minimal. + """ + + by: tuple[str, ...] = () + date_cols: tuple[str, ...] = () + numeric_cols: tuple[str, ...] = () + categorical_cols: tuple[str, ...] = () + + +def summary( + df: pd.DataFrame, + *, + by: Sequence[str] | None = None, + date_cols: Sequence[str] | None = None, + numeric_cols: Sequence[str] | None = None, + categorical_cols: Sequence[str] | None = None, + top_k: int = 5, + output: OutputFormat = "long", +) -> pd.DataFrame: + """Descriptive summary statistics for an epidemiological line list. + + The goal is a practical, opinionated summary useful for quick EDA. + + Args: + df: Input dataframe. + by: Stratification columns. + date_cols: Date-like columns to summarize (min/max). + numeric_cols: Numeric columns to summarize. + categorical_cols: Categorical columns to summarize (top-k frequency table). + top_k: Number of top categories to include per categorical column. + output: + - "long" (default): tidy rows: group + metric/value + - "wide": pivoted table with metrics as columns + + Returns: + DataFrame. + + Notes: + - Missingness is reported as missing_n and missing_pct. + - Numeric summaries: count, mean, std, min, p25, median, p75, max. + """ + + by_cols = list(by) if by is not None else [] + + date_cols = list(date_cols) if date_cols is not None else [] + numeric_cols = list(numeric_cols) if numeric_cols is not None else [] + categorical_cols = list(categorical_cols) if categorical_cols is not None else [] + + # Default heuristic: if user doesn't specify, infer a little. + if not date_cols: + for c in df.columns: + if c.endswith("_date") or c in {"date", "onset_date", "report_date"}: + date_cols.append(c) + + if not numeric_cols: + numeric_cols = [c for c in df.select_dtypes(include="number").columns] + + # treat remaining non-numeric, non-by, non-date as categorical if user didn't specify + if not categorical_cols: + candidates = [c for c in df.columns if c not in set(by_cols + date_cols + numeric_cols)] + # avoid very wide free-text columns by default + categorical_cols = candidates + + work = df.copy() + + if by_cols: + grouped = work.groupby(by_cols, dropna=False) + else: + # single group + grouped = [((), work)] + + rows: list[dict] = [] + + def _add(group_key: tuple, metric: str, value, col: str | None = None): + row: dict = {} + for i, b in enumerate(by_cols): + row[b] = group_key[i] if by_cols else None + row["column"] = col + row["metric"] = metric + row["value"] = value + rows.append(row) + + for key, g in grouped: + if not isinstance(key, tuple): + key = (key,) + + n = len(g) + _add(key, "n", n, col=None) + + # Missingness per column + for c in date_cols + numeric_cols + categorical_cols: + if c not in g.columns: + continue + miss_n = int(g[c].isna().sum()) + miss_pct = (miss_n / n * 100.0) if n else 0.0 + _add(key, "missing_n", miss_n, col=c) + _add(key, "missing_pct", round(miss_pct, 3), col=c) + + # Date min/max + for c in date_cols: + if c not in g.columns: + continue + s = pd.to_datetime(g[c], errors="coerce") + if s.notna().any(): + _add(key, "min", s.min().date().isoformat(), col=c) + _add(key, "max", s.max().date().isoformat(), col=c) + else: + _add(key, "min", None, col=c) + _add(key, "max", None, col=c) + + # Numeric summaries + for c in numeric_cols: + if c not in g.columns: + continue + s = pd.to_numeric(g[c], errors="coerce") + s_non = s.dropna() + if s_non.empty: + for m in ["count", "mean", "std", "min", "p25", "median", "p75", "max"]: + _add(key, m, None, col=c) + continue + + desc = s_non.describe(percentiles=[0.25, 0.5, 0.75]) + _add(key, "count", int(desc["count"]), col=c) + _add(key, "mean", float(desc["mean"]), col=c) + _add(key, "std", float(desc["std"]) if "std" in desc else None, col=c) + _add(key, "min", float(desc["min"]), col=c) + _add(key, "p25", float(desc["25%"]), col=c) + _add(key, "median", float(desc["50%"]), col=c) + _add(key, "p75", float(desc["75%"]), col=c) + _add(key, "max", float(desc["max"]), col=c) + + # Categorical top-k + for c in categorical_cols: + if c not in g.columns: + continue + s = g[c].astype("object") + vc = s.value_counts(dropna=False) + for rank, (val, cnt) in enumerate(vc.head(top_k).items(), start=1): + label = "" if pd.isna(val) else str(val) + _add(key, f"top_{rank}", label, col=c) + _add(key, f"top_{rank}_n", int(cnt), col=c) + + out = pd.DataFrame(rows) + + # If no by, we inserted None; remove those columns. + if not by_cols and "column" in out.columns: + out = out.drop(columns=[]) + + if output == "long": + return out + + # wide output: index=by, columns=metric names joined with column. + idx_cols = by_cols if by_cols else [] + out_wide = out.copy() + out_wide["metric_key"] = out_wide.apply( + lambda r: (f"{r['column']}.{r['metric']}" if r["column"] is not None else str(r["metric"])), + axis=1, + ) + + if idx_cols: + wide = out_wide.pivot_table( + index=idx_cols, + columns="metric_key", + values="value", + aggfunc="first", + ) + else: + wide = out_wide.set_index("metric_key")[["value"]].T + + return wide diff --git a/epydem/summary_report.py b/epydem/summary_report.py new file mode 100644 index 0000000..e47b57d --- /dev/null +++ b/epydem/summary_report.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from collections.abc import Sequence + +import pandas as pd + +from .formatting import to_markdown_table +from .summary import summary + + +def summary_markdown( + df: pd.DataFrame, + *, + by: Sequence[str] | None = None, + date_cols: Sequence[str] | None = None, + numeric_cols: Sequence[str] | None = None, + categorical_cols: Sequence[str] | None = None, + top_k: int = 5, + max_rows: int = 40, +) -> str: + """Return a human-readable Markdown report for `summary()`. + + This keeps `summary()` as the raw-data primitive (DataFrame) while providing a + pretty format for quick sharing. + + The report uses wide output internally and prints the first `max_rows` rows. + """ + + wide = summary( + df, + by=by, + date_cols=date_cols, + numeric_cols=numeric_cols, + categorical_cols=categorical_cols, + top_k=top_k, + output="wide", + ) + + # Normalize to a 2D table. + table = wide.reset_index() + if len(table) > max_rows: + table = table.head(max_rows) + truncated_note = f"\n\n_(truncated to first {max_rows} rows)_" + else: + truncated_note = "" + + headers = [str(c) for c in table.columns] + rows = table.astype("object").fillna("").values.tolist() + + return to_markdown_table(headers, rows) + truncated_note diff --git a/tests/test_summary.py b/tests/test_summary.py new file mode 100644 index 0000000..0c79eb2 --- /dev/null +++ b/tests/test_summary.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import pandas as pd + +import epydem + + +def test_summary_basic_long(): + df = pd.DataFrame( + { + "onset_date": ["2024-01-01", "2024-01-02", None], + "age": [10, 20, None], + "sex": ["M", "F", "F"], + } + ) + + out = epydem.summary( + df, + by=["sex"], + date_cols=["onset_date"], + numeric_cols=["age"], + categorical_cols=["sex"], + top_k=2, + ) + + # Expect rows for both strata + assert set(out["sex"].dropna().unique()) == {"M", "F"} + + # Missingness should be present for onset_date + assert ((out["column"] == "onset_date") & (out["metric"] == "missing_n")).any() + + +def test_summary_wide_shape(): + df = pd.DataFrame( + { + "onset_date": ["2024-01-01", "2024-01-02"], + "age": [10, 20], + "sex": ["M", "F"], + } + ) + + wide = epydem.summary( + df, + by=["sex"], + date_cols=["onset_date"], + numeric_cols=["age"], + categorical_cols=["sex"], + output="wide", + ) + + # Index should be sex + assert set(wide.index.tolist()) == {"M", "F"} + + # Should have some expected metric columns + assert any(c.startswith("age.mean") for c in wide.columns) + assert any(c.startswith("onset_date.min") for c in wide.columns) diff --git a/tests/test_summary_markdown.py b/tests/test_summary_markdown.py new file mode 100644 index 0000000..7c97587 --- /dev/null +++ b/tests/test_summary_markdown.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import pandas as pd + +import epydem + + +def test_summary_markdown_smoke(): + df = pd.DataFrame( + { + "onset_date": ["2024-01-01", "2024-01-02", None], + "age": [10, 20, None], + "sex": ["M", "F", "F"], + } + ) + + md = epydem.summary_markdown( + df, + by=["sex"], + date_cols=["onset_date"], + numeric_cols=["age"], + categorical_cols=["sex"], + top_k=2, + ) + + assert "|" in md + assert "onset_date.min" in md + assert "age.mean" in md