Phase 3: summary() descriptive stats

PR_PHASE3.md

@@ -0,0 +1,37 @@
+Phase 3: summary() (descriptive epidemiology)
+
+Key points
+- Add `epydem.summary(df, by=..., date_cols=..., numeric_cols=..., categorical_cols=...)`.
+- Supports stratified summaries via `by=[...]`.
+- Long (tidy) output by default; optional `output="wide"`.
+
+WHY
+- After incidence/epicurve, the next most common need is quick descriptive EDA: missingness, date ranges, numeric distributions, and top categories.
+- A standardized summary makes notebooks and reports faster and more consistent.
+
+WHAT
+- Implemented metrics:
+  - group size `n`
+  - missingness: `missing_n`, `missing_pct`
+  - date columns: `min`, `max`
+  - numeric columns: `count`, `mean`, `std`, `min`, `p25`, `median`, `p75`, `max`
+  - categorical columns: `top_1..top_k` + `top_#_n`
+
+Multi-role debate (differences, not consensus)
+
+Role A — pragmatic developer
+- 👍 Likes: covers 80% EDA quickly; configurable columns; works stratified.
+- ⚠️ Concern: output schema can grow; must keep naming stable.
+
+Role B — architecture
+- 👍 Likes: explicit outputs; can be extended with typed spec objects later.
+- ⚠️ Concern: inference heuristics (auto-detect columns) might be surprising; keep it minimal.
+
+Role C — developer user (DX)
+- 👍 Likes: one function for a full “first look” summary.
+- ⚠️ Concern: wants better defaults and nicer presentation (markdown tables, formatting), but those can be separate utilities.
+
+Points of divergence to revisit later
+1) Whether to provide a pretty-printed report (markdown) vs raw DataFrame only.
+2) How aggressive to make column inference defaults.
+3) Support for weighted summaries / population-standardized metrics.

README.md

@@ -23,6 +23,7 @@ pip install -e '.[dev]'
 ## Usage
 
 ```python
+import pandas as pd
 import epydem
 
 year, week = epydem.epiweek("2024-01-01")
@@ -32,10 +33,47 @@ week_only = epydem.epiweek_number("2024-01-01")
 print(week_only)
 
 # Incidence (line list -> counts)
-# df = pandas.DataFrame({"onset_date": [...], "sex": [...]})
-# weekly = epydem.incidence(df, date_col="onset_date", freq="W-MMWR", by=["sex"], fill_missing=True)
-# weekly_rolling2 = epydem.transform_incidence(weekly, rolling=2)
-# weekly_cum = epydem.transform_incidence(weekly, cumulative=True)
+df = pd.DataFrame({"onset_date": ["2024-01-01", "2024-01-02"], "sex": ["M", "F"]})
+weekly = epydem.incidence(df, date_col="onset_date", freq="W-MMWR", by=["sex"], fill_missing=True)
+weekly_rolling2 = epydem.transform_incidence(weekly, rolling=2)
+weekly_cum = epydem.transform_incidence(weekly, cumulative=True)
+```
+
+## Descriptive summary
+
+`summary()` returns a DataFrame (raw stats). `summary_markdown()` returns a pretty Markdown table
+string for quick sharing.
+
+```python
+import pandas as pd
+import epydem
+
+df = pd.DataFrame(
+    {
+        "onset_date": ["2024-01-01", "2024-01-02", None],
+        "age": [10, 20, None],
+        "sex": ["M", "F", "F"],
+    }
+)
+
+# Raw DataFrame summary (long/tidy by default)
+out = epydem.summary(
+    df,
+    by=["sex"],
+    date_cols=["onset_date"],
+    numeric_cols=["age"],
+    categorical_cols=["sex"],
+)
+
+# Pretty report (Markdown table)
+md = epydem.summary_markdown(
+    df,
+    by=["sex"],
+    date_cols=["onset_date"],
+    numeric_cols=["age"],
+    categorical_cols=["sex"],
+)
+print(md)
 ```
 
 ## Roadmap (high level)

epydem/__init__.py

@@ -2,6 +2,8 @@
 
 from .epiweek import calculate
 from .incidence import incidence
+from .summary import summary
+from .summary_report import summary_markdown
 from .time import epiweek, epiweek_number, mmwr_week, mmwr_week1_start, parse_ymd
 from .transform import transform_incidence
 
@@ -14,4 +16,6 @@
     "parse_ymd",
     "incidence",
     "transform_incidence",
+    "summary",
+    "summary_markdown",
 ]

epydem/formatting.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+
+def to_markdown_table(headers: Sequence[str], rows: Sequence[Sequence[str]]) -> str:
+    """Render a simple GitHub-flavored Markdown table.
+
+    Avoids optional deps like `tabulate`.
+    """
+
+    headers = list(headers)
+    row_strs = [[str(x) for x in r] for r in rows]
+
+    # Escape pipes minimally.
+    def esc(s: str) -> str:
+        return s.replace("|", "\\|")
+
+    header_line = "| " + " | ".join(esc(h) for h in headers) + " |"
+    sep_line = "| " + " | ".join(["---"] * len(headers)) + " |"
+    body_lines = ["| " + " | ".join(esc(c) for c in r) + " |" for r in row_strs]
+
+    return "\n".join([header_line, sep_line, *body_lines])

epydem/summary.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Literal
+
+import pandas as pd
+
+OutputFormat = Literal["long", "wide"]
+
+
+@dataclass(frozen=True)
+class SummarySpec:
+    """Configuration for `summary()`.
+
+    This will likely evolve; keep it minimal.
+    """
+
+    by: tuple[str, ...] = ()
+    date_cols: tuple[str, ...] = ()
+    numeric_cols: tuple[str, ...] = ()
+    categorical_cols: tuple[str, ...] = ()
+
+
+def summary(
+    df: pd.DataFrame,
+    *,
+    by: Sequence[str] | None = None,
+    date_cols: Sequence[str] | None = None,
+    numeric_cols: Sequence[str] | None = None,
+    categorical_cols: Sequence[str] | None = None,
+    top_k: int = 5,
+    output: OutputFormat = "long",
+) -> pd.DataFrame:
+    """Descriptive summary statistics for an epidemiological line list.
+
+    The goal is a practical, opinionated summary useful for quick EDA.
+
+    Args:
+        df: Input dataframe.
+        by: Stratification columns.
+        date_cols: Date-like columns to summarize (min/max).
+        numeric_cols: Numeric columns to summarize.
+        categorical_cols: Categorical columns to summarize (top-k frequency table).
+        top_k: Number of top categories to include per categorical column.
+        output:
+          - "long" (default): tidy rows: group + metric/value
+          - "wide": pivoted table with metrics as columns
+
+    Returns:
+        DataFrame.
+
+    Notes:
+    - Missingness is reported as missing_n and missing_pct.
+    - Numeric summaries: count, mean, std, min, p25, median, p75, max.
+    """
+
+    by_cols = list(by) if by is not None else []
+
+    date_cols = list(date_cols) if date_cols is not None else []
+    numeric_cols = list(numeric_cols) if numeric_cols is not None else []
+    categorical_cols = list(categorical_cols) if categorical_cols is not None else []
+
+    # Default heuristic: if user doesn't specify, infer a little.
+    if not date_cols:
+        for c in df.columns:
+            if c.endswith("_date") or c in {"date", "onset_date", "report_date"}:
+                date_cols.append(c)
+
+    if not numeric_cols:
+        numeric_cols = [c for c in df.select_dtypes(include="number").columns]
+
+    # treat remaining non-numeric, non-by, non-date as categorical if user didn't specify
+    if not categorical_cols:
+        candidates = [c for c in df.columns if c not in set(by_cols + date_cols + numeric_cols)]
+        # avoid very wide free-text columns by default
+        categorical_cols = candidates
+
+    work = df.copy()
+
+    if by_cols:
+        grouped = work.groupby(by_cols, dropna=False)
+    else:
+        # single group
+        grouped = [((), work)]
+
+    rows: list[dict] = []
+
+    def _add(group_key: tuple, metric: str, value, col: str | None = None):
+        row: dict = {}
+        for i, b in enumerate(by_cols):
+            row[b] = group_key[i] if by_cols else None
+        row["column"] = col
+        row["metric"] = metric
+        row["value"] = value
+        rows.append(row)
+
+    for key, g in grouped:
+        if not isinstance(key, tuple):
+            key = (key,)
+
+        n = len(g)
+        _add(key, "n", n, col=None)
+
+        # Missingness per column
+        for c in date_cols + numeric_cols + categorical_cols:
+            if c not in g.columns:
+                continue
+            miss_n = int(g[c].isna().sum())
+            miss_pct = (miss_n / n * 100.0) if n else 0.0
+            _add(key, "missing_n", miss_n, col=c)
+            _add(key, "missing_pct", round(miss_pct, 3), col=c)
+
+        # Date min/max
+        for c in date_cols:
+            if c not in g.columns:
+                continue
+            s = pd.to_datetime(g[c], errors="coerce")
+            if s.notna().any():
+                _add(key, "min", s.min().date().isoformat(), col=c)
+                _add(key, "max", s.max().date().isoformat(), col=c)
+            else:
+                _add(key, "min", None, col=c)
+                _add(key, "max", None, col=c)
+
+        # Numeric summaries
+        for c in numeric_cols:
+            if c not in g.columns:
+                continue
+            s = pd.to_numeric(g[c], errors="coerce")
+            s_non = s.dropna()
+            if s_non.empty:
+                for m in ["count", "mean", "std", "min", "p25", "median", "p75", "max"]:
+                    _add(key, m, None, col=c)
+                continue
+
+            desc = s_non.describe(percentiles=[0.25, 0.5, 0.75])
+            _add(key, "count", int(desc["count"]), col=c)
+            _add(key, "mean", float(desc["mean"]), col=c)
+            _add(key, "std", float(desc["std"]) if "std" in desc else None, col=c)
+            _add(key, "min", float(desc["min"]), col=c)
+            _add(key, "p25", float(desc["25%"]), col=c)
+            _add(key, "median", float(desc["50%"]), col=c)
+            _add(key, "p75", float(desc["75%"]), col=c)
+            _add(key, "max", float(desc["max"]), col=c)
+
+        # Categorical top-k
+        for c in categorical_cols:
+            if c not in g.columns:
+                continue
+            s = g[c].astype("object")
+            vc = s.value_counts(dropna=False)
+            for rank, (val, cnt) in enumerate(vc.head(top_k).items(), start=1):
+                label = "<NA>" if pd.isna(val) else str(val)
+                _add(key, f"top_{rank}", label, col=c)
+                _add(key, f"top_{rank}_n", int(cnt), col=c)
+
+    out = pd.DataFrame(rows)
+
+    # If no by, we inserted None; remove those columns.
+    if not by_cols and "column" in out.columns:
+        out = out.drop(columns=[])
+
+    if output == "long":
+        return out
+
+    # wide output: index=by, columns=metric names joined with column.
+    idx_cols = by_cols if by_cols else []
+    out_wide = out.copy()
+    out_wide["metric_key"] = out_wide.apply(
+        lambda r: (f"{r['column']}.{r['metric']}" if r["column"] is not None else str(r["metric"])),
+        axis=1,
+    )
+
+    if idx_cols:
+        wide = out_wide.pivot_table(
+            index=idx_cols,
+            columns="metric_key",
+            values="value",
+            aggfunc="first",
+        )
+    else:
+        wide = out_wide.set_index("metric_key")[["value"]].T
+
+    return wide

epydem/summary_report.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import pandas as pd
+
+from .formatting import to_markdown_table
+from .summary import summary
+
+
+def summary_markdown(
+    df: pd.DataFrame,
+    *,
+    by: Sequence[str] | None = None,
+    date_cols: Sequence[str] | None = None,
+    numeric_cols: Sequence[str] | None = None,
+    categorical_cols: Sequence[str] | None = None,
+    top_k: int = 5,
+    max_rows: int = 40,
+) -> str:
+    """Return a human-readable Markdown report for `summary()`.
+
+    This keeps `summary()` as the raw-data primitive (DataFrame) while providing a
+    pretty format for quick sharing.
+
+    The report uses wide output internally and prints the first `max_rows` rows.
+    """
+
+    wide = summary(
+        df,
+        by=by,
+        date_cols=date_cols,
+        numeric_cols=numeric_cols,
+        categorical_cols=categorical_cols,
+        top_k=top_k,
+        output="wide",
+    )
+
+    # Normalize to a 2D table.
+    table = wide.reset_index()
+    if len(table) > max_rows:
+        table = table.head(max_rows)
+        truncated_note = f"\n\n_(truncated to first {max_rows} rows)_"
+    else:
+        truncated_note = ""
+
+    headers = [str(c) for c in table.columns]
+    rows = table.astype("object").fillna("").values.tolist()
+
+    return to_markdown_table(headers, rows) + truncated_note