eda.py

"""EDA del dataset: dimensiones, nulos, balance, estadisticas y anomalias.

Guarda las figuras en figures/ y escribe el resumen por consola.
    python eda.py
"""
from __future__ import annotations

import numpy as np
import pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

import config as cfg


def section(title: str) -> None:
    print("\n" + "=" * 62)
    print(title)
    print("=" * 62)


def main() -> None:
    cfg.FIGURES_DIR.mkdir(exist_ok=True)
    print(f"Datos: {cfg.DATA_DIR}")

    train_df = pd.read_csv(cfg.TRAIN_CSV)
    test_df = pd.read_csv(cfg.TEST_CSV)

    # ------------------------------------------------------------------ #
    section("1. DIMENSIONES, TIPOS Y NULOS")
    print(f"train.csv : {train_df.shape}  (1 label + {train_df.shape[1] - 1} pixeles)")
    print(f"test.csv  : {test_df.shape}  (solo pixeles)")
    print(f"dtypes    : train {train_df.dtypes.unique()} | test {test_df.dtypes.unique()}")
    print(f"nulos     : train {int(train_df.isnull().sum().sum())} | "
          f"test {int(test_df.isnull().sum().sum())}")
    print(f"labels    : {train_df['label'].min()}-{train_df['label'].max()}, "
          f"{train_df['label'].nunique()} clases distintas")

    y = train_df["label"].to_numpy()
    flat = train_df.drop(columns="label").to_numpy(dtype=np.uint8)
    flat_test = test_df.to_numpy(dtype=np.uint8)
    X = flat.reshape(-1, cfg.IMG_SIZE, cfg.IMG_SIZE)

    # ------------------------------------------------------------------ #
    section("2. BALANCE DE CLASES")
    counts = pd.Series(y).value_counts().sort_index()
    pct = (counts / len(y) * 100).round(2)
    print(pd.DataFrame({"n": counts, "%": pct}).to_string())
    print(f"\nmin={counts.min()} (digito {counts.idxmin()})  "
          f"max={counts.max()} (digito {counts.idxmax()})  "
          f"ratio={counts.max() / counts.min():.3f}")
    print("Balanceado: no hacen falta pesos de clase, solo split estratificado.")

    fig, ax = plt.subplots(figsize=(7, 4))
    ax.bar(counts.index, counts.values, color="#4C72B0")
    ax.axhline(len(y) / 10, color="crimson", ls="--", lw=1.2, label="uniforme")
    ax.set_xticks(range(10))
    ax.set_xlabel("digito"); ax.set_ylabel("n imagenes")
    ax.set_title("Distribucion de clases"); ax.legend()
    fig.tight_layout(); fig.savefig(cfg.FIGURES_DIR / "class_balance.png", dpi=110)
    plt.close(fig)

    # ------------------------------------------------------------------ #
    section("3. MUESTRAS RECONSTRUIDAS")
    rng = np.random.default_rng(cfg.SEED)
    idx = rng.choice(len(X), 15, replace=False)
    fig, axes = plt.subplots(3, 5, figsize=(11, 7))
    for ax, i in zip(axes.ravel(), idx):
        ax.imshow(X[i], cmap="gray")
        ax.set_title(f"idx {i} | label {y[i]}", fontsize=9)
        ax.axis("off")
    fig.suptitle("15 imagenes aleatorias de train")
    fig.tight_layout(); fig.savefig(cfg.FIGURES_DIR / "samples.png", dpi=110)
    plt.close(fig)
    print("figures/samples.png -> 15 muestras aleatorias")
    print("Reconstruccion reshape(28,28) en orden C: correcta, no hay que transponer.")

    # ------------------------------------------------------------------ #
    section("4. ESTADISTICAS DE PIXELES")
    print(f"TRAIN  min={flat.min()} max={flat.max()} "
          f"mean={flat.mean():.4f} std={flat.std():.4f}")
    print(f"TEST   min={flat_test.min()} max={flat_test.max()} "
          f"mean={flat_test.mean():.4f} std={flat_test.std():.4f}")
    print(f"pixeles en 0 (fondo): {(flat == 0).mean() * 100:.2f} %")
    print(f"pixeles en 255      : {(flat == 255).mean() * 100:.2f} %")
    mean01, std01 = flat.mean() / 255.0, flat.std() / 255.0
    print(f"\nTras escalar a [0,1]: mean={mean01:.4f} std={std01:.4f}")
    print(f"config.py usa PIXEL_MEAN={cfg.PIXEL_MEAN} PIXEL_STD={cfg.PIXEL_STD}")
    print("Hay que normalizar: 0-255 crudo descuadra la escala que esperan He/Glorot.")

    fig, ax = plt.subplots(1, 3, figsize=(15, 4))
    ax[0].hist(flat.ravel(), bins=64, color="#4C72B0", log=True)
    ax[0].set_title("Intensidades (log)"); ax[0].set_xlabel("valor de pixel")
    ax[1].hist(flat[flat > 0], bins=64, color="#55A868")
    ax[1].set_title("Solo pixeles > 0 (el trazo)")
    im = ax[2].imshow(flat.mean(axis=0).reshape(28, 28), cmap="viridis")
    ax[2].set_title("Imagen promedio"); ax[2].axis("off")
    fig.colorbar(im, ax=ax[2], fraction=0.046)
    fig.tight_layout(); fig.savefig(cfg.FIGURES_DIR / "pixel_stats.png", dpi=110)
    plt.close(fig)

    # ------------------------------------------------------------------ #
    section("5. ANOMALIAS")
    row_max = flat.max(axis=1)
    ink = (flat > 0).sum(axis=1)
    print(f"imagenes en blanco (train/test) : "
          f"{int((row_max == 0).sum())} / {int((flat_test.max(axis=1) == 0).sum())}")
    print(f"imagenes casi vacias (max < 20) : {int((row_max < 20).sum())}")
    print(f"filas duplicadas exactas        : {int(train_df.duplicated().sum())}")
    pix_only = train_df.drop(columns="label")
    print(f"imagenes duplicadas (pixeles)   : {int(pix_only.duplicated().sum())}")
    print(f"duplicados dentro de test       : {int(test_df.duplicated().sum())}")

    h_train = set(pd.util.hash_pandas_object(pix_only, index=False))
    h_test = set(pd.util.hash_pandas_object(test_df, index=False))
    print(f"fuga train<->test               : {len(h_train & h_test)}")

    dead_tr = flat.max(axis=0) == 0
    dead_te = flat_test.max(axis=0) == 0
    print(f"pixeles siempre 0 (train/test/ambos): "
          f"{dead_tr.sum()} / {dead_te.sum()} / {(dead_tr & dead_te).sum()}")
    print(f"tinta por imagen: min={ink.min()} mediana={int(np.median(ink))} max={ink.max()}")
    print("\nLos pixeles muertos NO se podan: romperian la estructura 28x28 de la CNN,")
    print("y ademas los de train y los de test no son el mismo conjunto.")

    low = np.argsort(ink)[:10]
    fig, axes = plt.subplots(2, 5, figsize=(11, 5))
    for ax, i in zip(axes.ravel(), low):
        ax.imshow(X[i], cmap="gray")
        ax.set_title(f"idx {i} | y={y[i]} | tinta={ink[i]}", fontsize=8)
        ax.axis("off")
    fig.suptitle("Las 10 imagenes con menos tinta")
    fig.tight_layout(); fig.savefig(cfg.FIGURES_DIR / "low_ink.png", dpi=110)
    plt.close(fig)
    print("figures/low_ink.png -> son '1' de trazo fino legitimos, no basura")

    print(f"\nFiguras guardadas en {cfg.FIGURES_DIR}")


if __name__ == "__main__":
    main()