diff --git a/README.md b/README.md index 1aa8e0b..cc6ec85 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,10 @@ python -m src.sample_data python -m src.make_features ` --input data\sample_aep_hourly.csv ` --output data\sample_features_aep.csv +python -m src.baseline_eval ` + --features data\sample_features_aep.csv ` + --metrics reports\sample_baseline_metrics.csv ` + --plot reports\figures\sample_baseline.png python -m src.forecast_24h ` --input data\sample_aep_hourly.csv ` --features data\sample_features_aep.csv ` diff --git a/src/baseline_eval.py b/src/baseline_eval.py index 193da3e..0beb441 100644 --- a/src/baseline_eval.py +++ b/src/baseline_eval.py @@ -1,67 +1,230 @@ +"""Evaluate naive load baselines and persist their artifacts.""" + +from __future__ import annotations + +import argparse from pathlib import Path -import matplotlib.pyplot as plt +import numpy as np import pandas as pd +from matplotlib.figure import Figure from src.evaluation import HOURS_PER_DAY, trailing_window -from src.reporting import evaluate_predictions, format_result, save_results - -FEATURES_PATH = r"data\features_aep.csv" -EVALUATION_DAYS = 30 -METRICS_PATH = Path("reports") / "baseline_metrics.csv" -PLOT_PATH = Path("reports") / "figures" / "baseline_evaluation.png" - -df = pd.read_csv(FEATURES_PATH) -df["Datetime"] = pd.to_datetime(df["Datetime"]) -df = df.set_index("Datetime").sort_index() - -# Testzeitraum: exakt die letzten 30 Tage (stündlich) -end = df.index.max() -test = trailing_window( - df, - hours=EVALUATION_DAYS * HOURS_PER_DAY, +from src.reporting import ( + EvaluationResult, + evaluate_predictions, + format_result, + save_results, ) -y_test = test["y"] - -# Baselines: -# 1) "Gestern gleiche Stunde" -pred_yesterday = test["lag_24"] - -# 2) "Letzte Woche gleiche Stunde" -pred_lastweek = test["lag_168"] - -# 3) Mischung (oft überraschend gut) -pred_blend = 0.5 * pred_yesterday + 0.5 * pred_lastweek - - -print("Baseline-Auswertung (Test: letzte 30 Tage):") -results = [ - evaluate_predictions("Yesterday (lag_24)", y_test, pred_yesterday), - evaluate_predictions("Last week (lag_168)", y_test, pred_lastweek), - evaluate_predictions("Blend 50/50", y_test, pred_blend), -] -for result in results: - print(format_result(result)) -saved_metrics = save_results(results, METRICS_PATH) -print(f"Metriken gespeichert: {saved_metrics}") - -# Plot: letzte 7 Tage im Test -plot_start = end - pd.Timedelta(days=7) -plot_df = test.loc[plot_start:end, ["y"]].copy() -plot_df["Yesterday"] = pred_yesterday.loc[plot_start:end] -plot_df["LastWeek"] = pred_lastweek.loc[plot_start:end] -plot_df["Blend"] = pred_blend.loc[plot_start:end] - -figure, axis = plt.subplots() -axis.plot(plot_df.index, plot_df["y"], label="Actual") -axis.plot(plot_df.index, plot_df["Blend"], label="Blend 50/50") -axis.set_title("Baseline Forecast (last 7 days of test)") -axis.set_xlabel("Time") -axis.set_ylabel("MW") -axis.legend() -figure.tight_layout() -PLOT_PATH.parent.mkdir(parents=True, exist_ok=True) -figure.savefig(PLOT_PATH, dpi=150) -plt.close(figure) -print(f"Diagramm gespeichert: {PLOT_PATH}") +DEFAULT_FEATURES = Path("data") / "features_aep.csv" +DEFAULT_METRICS = Path("reports") / "baseline_metrics.csv" +DEFAULT_PLOT = Path("reports") / "figures" / "baseline_evaluation.png" +DEFAULT_EVALUATION_DAYS = 30 +DEFAULT_PLOT_DAYS = 7 +REQUIRED_COLUMNS = ("y", "lag_24", "lag_168") +HOURLY_STEP = pd.Timedelta(hours=1) + + +def load_baseline_features(csv_path: str | Path) -> pd.DataFrame: + """Load and validate the columns required for baseline evaluation.""" + + path = Path(csv_path) + if not path.is_file(): + raise FileNotFoundError(f"Feature CSV not found: {path}") + + frame = pd.read_csv(path) + required = {"Datetime", *REQUIRED_COLUMNS} + missing_columns = sorted(required.difference(frame.columns)) + if missing_columns: + raise ValueError( + "Feature CSV is missing required columns: " + + ", ".join(missing_columns) + ) + if frame.empty: + raise ValueError("Feature CSV contains no rows.") + + timestamps = pd.to_datetime(frame["Datetime"], errors="coerce") + invalid_timestamps = int(timestamps.isna().sum()) + if invalid_timestamps: + raise ValueError( + f"Feature CSV contains {invalid_timestamps} invalid timestamp(s)." + ) + if timestamps.duplicated().any(): + raise ValueError("Feature CSV contains duplicate timestamps.") + + numeric = frame.loc[:, REQUIRED_COLUMNS].apply( + pd.to_numeric, + errors="coerce", + ) + invalid_values = int(numeric.isna().sum().sum()) + if invalid_values: + raise ValueError( + "Feature CSV contains " + f"{invalid_values} non-numeric or missing value(s)." + ) + if not np.isfinite(numeric.to_numpy(dtype=float)).all(): + raise ValueError("Feature CSV contains non-finite values.") + + numeric.index = pd.DatetimeIndex(timestamps, name="Datetime") + numeric = numeric.sort_index() + steps = numeric.index.to_series().diff().dropna() + if not steps.eq(HOURLY_STEP).all(): + raise ValueError( + "Feature CSV must contain consecutive hourly timestamps." + ) + return numeric.astype(float) + + +def baseline_predictions(features: pd.DataFrame) -> pd.DataFrame: + """Build yesterday, last-week, and blended baseline predictions.""" + + missing_columns = sorted(set(REQUIRED_COLUMNS).difference(features.columns)) + if missing_columns: + raise ValueError( + "Evaluation data is missing required columns: " + + ", ".join(missing_columns) + ) + + predictions = pd.DataFrame(index=features.index) + predictions["Yesterday"] = features["lag_24"].astype(float) + predictions["Last week"] = features["lag_168"].astype(float) + predictions["Blend 50/50"] = ( + 0.5 * predictions["Yesterday"] + 0.5 * predictions["Last week"] + ) + return predictions + + +def evaluate_baselines( + features: pd.DataFrame, + *, + evaluation_hours: int, +) -> tuple[pd.DataFrame, pd.DataFrame, list[EvaluationResult]]: + """Evaluate all baselines on an exact trailing time window.""" + + test = trailing_window(features, hours=evaluation_hours) + predictions = baseline_predictions(test) + results = [ + evaluate_predictions( + "Yesterday (lag_24)", + test["y"], + predictions["Yesterday"], + ), + evaluate_predictions( + "Last week (lag_168)", + test["y"], + predictions["Last week"], + ), + evaluate_predictions( + "Blend 50/50", + test["y"], + predictions["Blend 50/50"], + ), + ] + return test, predictions, results + + +def save_baseline_plot( + test: pd.DataFrame, + predictions: pd.DataFrame, + output_path: str | Path, + *, + plot_hours: int, +) -> Path: + """Plot actual load and the blended baseline over a trailing window.""" + + if not test.index.equals(predictions.index): + raise ValueError("Test data and predictions must use the same index.") + + plot_test = trailing_window(test, hours=plot_hours) + plot_predictions = predictions.loc[plot_test.index] + path = Path(output_path) + + figure = Figure() + axis = figure.subplots() + axis.plot(plot_test.index, plot_test["y"], label="Actual") + axis.plot( + plot_predictions.index, + plot_predictions["Blend 50/50"], + label="Blend 50/50", + ) + axis.set_title("Baseline Forecast") + axis.set_xlabel("Time") + axis.set_ylabel("MW") + axis.legend() + figure.tight_layout() + + path.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(path, dpi=150) + return path + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate naive load baselines on a trailing window." + ) + parser.add_argument( + "--features", + type=Path, + default=DEFAULT_FEATURES, + help=f"Feature CSV path (default: {DEFAULT_FEATURES.as_posix()})", + ) + parser.add_argument( + "--metrics", + type=Path, + default=DEFAULT_METRICS, + help=f"Metrics CSV path (default: {DEFAULT_METRICS.as_posix()})", + ) + parser.add_argument( + "--plot", + type=Path, + default=DEFAULT_PLOT, + help=f"Plot path (default: {DEFAULT_PLOT.as_posix()})", + ) + parser.add_argument( + "--days", + type=int, + default=DEFAULT_EVALUATION_DAYS, + help=( + "Number of trailing evaluation days " + f"(default: {DEFAULT_EVALUATION_DAYS})" + ), + ) + parser.add_argument( + "--plot-days", + type=int, + default=DEFAULT_PLOT_DAYS, + help=( + "Number of trailing evaluation days to plot " + f"(default: {DEFAULT_PLOT_DAYS})" + ), + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + features = load_baseline_features(args.features) + test, predictions, results = evaluate_baselines( + features, + evaluation_hours=args.days * HOURS_PER_DAY, + ) + + print(f"Baseline evaluation (last {args.days} days):") + for result in results: + print(format_result(result)) + + saved_metrics = save_results(results, args.metrics) + print(f"Metrics saved: {saved_metrics}") + saved_plot = save_baseline_plot( + test, + predictions, + args.plot, + plot_hours=args.plot_days * HOURS_PER_DAY, + ) + print(f"Plot saved: {saved_plot}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_baseline_eval.py b/tests/test_baseline_eval.py new file mode 100644 index 0000000..7a53ed2 --- /dev/null +++ b/tests/test_baseline_eval.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from src.baseline_eval import ( + evaluate_baselines, + load_baseline_features, + main, +) + + +def baseline_frame(periods: int = 200) -> pd.DataFrame: + index = pd.date_range("2025-01-01", periods=periods, freq="h") + target = np.arange(periods, dtype=float) + return pd.DataFrame( + { + "y": target, + "lag_24": target - 24.0, + "lag_168": target - 168.0, + }, + index=index, + ) + + +def test_evaluate_baselines_uses_exact_trailing_window() -> None: + test, predictions, results = evaluate_baselines( + baseline_frame(), + evaluation_hours=48, + ) + + assert len(test) == 48 + assert test.index.equals(predictions.index) + assert [result.model for result in results] == [ + "Yesterday (lag_24)", + "Last week (lag_168)", + "Blend 50/50", + ] + assert [result.mae_mw for result in results] == [24.0, 168.0, 96.0] + assert [result.rmse_mw for result in results] == [24.0, 168.0, 96.0] + + +@pytest.mark.parametrize( + ("contents", "message"), + [ + ( + "Datetime,y,lag_24\n2025-01-01,1,1\n", + "missing required columns", + ), + ( + ( + "Datetime,y,lag_24,lag_168\n" + "not-a-date,1,1,1\n" + ), + "invalid timestamp", + ), + ( + ( + "Datetime,y,lag_24,lag_168\n" + "2025-01-01 00:00:00,1,bad,1\n" + ), + "non-numeric", + ), + ( + ( + "Datetime,y,lag_24,lag_168\n" + "2025-01-01 00:00:00,1,1,1\n" + "2025-01-01 02:00:00,2,2,2\n" + ), + "consecutive hourly", + ), + ], +) +def test_load_baseline_features_rejects_invalid_csv( + tmp_path, + contents: str, + message: str, +) -> None: + path = tmp_path / "features.csv" + path.write_text(contents, encoding="utf-8") + + with pytest.raises(ValueError, match=message): + load_baseline_features(path) + + +def test_command_writes_metrics_and_plot(tmp_path) -> None: + features_path = tmp_path / "features.csv" + metrics_path = tmp_path / "nested" / "metrics.csv" + plot_path = tmp_path / "figures" / "baseline.png" + frame = baseline_frame() + frame.index.name = "Datetime" + frame.to_csv(features_path) + + result = main( + [ + "--features", + str(features_path), + "--metrics", + str(metrics_path), + "--plot", + str(plot_path), + "--days", + "2", + "--plot-days", + "1", + ] + ) + + assert result == 0 + assert metrics_path.is_file() + assert plot_path.is_file() + metrics = pd.read_csv(metrics_path) + assert metrics["model"].tolist() == [ + "Yesterday (lag_24)", + "Last week (lag_168)", + "Blend 50/50", + ]