validate_forecasts.py

"""
Forecast Validation Script
Trains models on 2022-2023 data, predicts 2024-2025, and compares to actual results
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime
import warnings
warnings.filterwarnings('ignore')

from src.main import PortfolioOptimizer
from stock_database import STOCK_DATABASE

# Set style
sns.set_style("whitegrid")
plt.rcParams['figure.figsize'] = (14, 8)

# Preset portfolios to validate
PRESETS = {
    "Tech Giants": ["AAPL", "GOOGL", "MSFT", "AMZN", "META"],
    "Mag 7": ["AAPL", "GOOGL", "MSFT", "AMZN", "NVDA", "TSLA", "META"],
    "Financial": ["JPM", "BAC", "WFC", "GS", "MS"],
    "Healthcare": ["JNJ", "UNH", "PFE", "ABBV", "TMO"],
    "Diversified": ["SPY", "TLT", "GLD", "VNQ", "IEF"]
}

def validate_forecasts():
    """Run forecast validation for all preset portfolios"""
    
    print("\n" + "="*80)
    print("FORECAST VALIDATION: Model Predictions vs Actual Returns (2022-2025)")
    print("="*80 + "\n")
    
    results = {}
    
    for preset_name, tickers in PRESETS.items():
        print(f"\n{'='*80}")
        print(f"Validating: {preset_name} ({', '.join(tickers)})")
        print(f"{'='*80}\n")
        
        try:
            # Get full data (2022-2025)
            optimizer = PortfolioOptimizer(
                tickers=tickers,
                start_date='2020-01-01',
                end_date='2025-12-31'
            )
            
            print("📥 Collecting data...")
            optimizer.collect_data()
            
            if optimizer.prices is None or len(optimizer.prices) == 0:
                print(f"⚠️  No data available for {preset_name}")
                continue
            
            # Split data: Train on 2022-2023, Test on 2024-2025
            train_end = '2023-12-31'
            test_start = '2024-01-01'
            
            train_prices = optimizer.prices[optimizer.prices.index <= train_end]
            test_prices = optimizer.prices[optimizer.prices.index >= test_start]
            
            if len(train_prices) == 0 or len(test_prices) == 0:
                print(f"⚠️  Insufficient data split for {preset_name}")
                continue
            
            print(f"✅ Training period: {train_prices.index[0].date()} to {train_prices.index[-1].date()} ({len(train_prices)} days)")
            print(f"✅ Test period: {test_prices.index[0].date()} to {test_prices.index[-1].date()} ({len(test_prices)} days)")
            
            # Train models on training data
            print("\n🤖 Training ML models on training data...")
            train_optimizer = PortfolioOptimizer(
                tickers=tickers,
                start_date='2020-01-01',
                end_date='2023-12-31'
            )
            train_optimizer.collect_data()
            train_optimizer.engineer_features()
            train_optimizer.train_models(use_lstm=False, use_rf=True, use_ensemble=True)
            
            print("✅ Models trained successfully")
            
            # Get actual returns for test period
            test_returns = test_prices.pct_change().dropna()
            actual_returns = test_returns.mean() * 252  # Annualized
            
            # Make predictions for test period
            print("\n🔮 Making predictions for test period...")
            
            # Get features for test period (need to engineer features on full data)
            optimizer.engineer_features()
            
            # Predict returns for each asset
            predicted_returns = {}
            prediction_errors = {}
            
            for ticker in tickers:
                try:
                    # Get the best available model
                    model = None
                    if f'{ticker}_ensemble' in train_optimizer.models:
                        model = train_optimizer.models[f'{ticker}_ensemble']
                    elif f'{ticker}_rf' in train_optimizer.models:
                        model = train_optimizer.models[f'{ticker}_rf']
                    
                    if model is None:
                        # Use historical mean as fallback
                        train_returns = train_prices[ticker].pct_change().dropna()
                        predicted_returns[ticker] = train_returns.mean() * 252
                        prediction_errors[ticker] = abs(predicted_returns[ticker] - actual_returns[ticker])
                        continue
                    
                    # Get features for test period
                    ticker_features = optimizer.features[[col for col in optimizer.features.columns 
                                                         if col.startswith(ticker)]]
                    
                    # Filter to test period
                    test_features = ticker_features[ticker_features.index >= test_start]
                    
                    if len(test_features) == 0:
                        # Fallback to historical mean
                        train_returns = train_prices[ticker].pct_change().dropna()
                        predicted_returns[ticker] = train_returns.mean() * 252
                    else:
                        # Make predictions
                        predictions = model.predict(test_features)
                        predicted_returns[ticker] = np.mean(predictions) * 252  # Annualized
                    
                    # Calculate error
                    if ticker in actual_returns.index:
                        prediction_errors[ticker] = abs(predicted_returns[ticker] - actual_returns[ticker])
                    
                except Exception as e:
                    print(f"  ⚠️  Error predicting {ticker}: {str(e)}")
                    # Fallback to historical mean
                    train_returns = train_prices[ticker].pct_change().dropna()
                    predicted_returns[ticker] = train_returns.mean() * 252
                    if ticker in actual_returns.index:
                        prediction_errors[ticker] = abs(predicted_returns[ticker] - actual_returns[ticker])
            
            predicted_returns_series = pd.Series(predicted_returns)
            
            # Calculate portfolio-level metrics
            # Use equal weights for comparison
            equal_weights = pd.Series(1.0 / len(tickers), index=tickers)
            
            predicted_portfolio_return = (predicted_returns_series * equal_weights).sum()
            actual_portfolio_return = (actual_returns * equal_weights).sum()
            
            portfolio_error = abs(predicted_portfolio_return - actual_portfolio_return)
            portfolio_error_pct = (portfolio_error / abs(actual_portfolio_return) * 100) if actual_portfolio_return != 0 else 0
            
            # Calculate individual asset errors
            mae = np.mean(list(prediction_errors.values()))
            mape = np.mean([abs(err / actual_returns[t]) * 100 
                           for t, err in prediction_errors.items() 
                           if actual_returns[t] != 0])
            
            # Store results
            results[preset_name] = {
                'tickers': tickers,
                'predicted_returns': predicted_returns_series,
                'actual_returns': actual_returns,
                'prediction_errors': prediction_errors,
                'predicted_portfolio_return': predicted_portfolio_return,
                'actual_portfolio_return': actual_portfolio_return,
                'portfolio_error': portfolio_error,
                'portfolio_error_pct': portfolio_error_pct,
                'mae': mae,
                'mape': mape,
                'train_dates': (train_prices.index[0], train_prices.index[-1]),
                'test_dates': (test_prices.index[0], test_prices.index[-1])
            }
            
            # Print summary
            print(f"\n📊 Results for {preset_name}:")
            print(f"  Predicted Portfolio Return: {predicted_portfolio_return:.2%}")
            print(f"  Actual Portfolio Return:     {actual_portfolio_return:.2%}")
            print(f"  Error:                       {portfolio_error:.2%} ({portfolio_error_pct:.1f}%)")
            print(f"  Mean Absolute Error:         {mae:.2%}")
            print(f"  Mean Absolute % Error:       {mape:.1f}%")
            
            # Individual asset comparison
            print(f"\n  Individual Asset Predictions:")
            for ticker in tickers:
                pred = predicted_returns.get(ticker, 0)
                actual = actual_returns.get(ticker, 0)
                error = prediction_errors.get(ticker, 0)
                print(f"    {ticker:6s} | Predicted: {pred:7.2%} | Actual: {actual:7.2%} | Error: {error:6.2%}")
            
        except Exception as e:
            print(f"❌ Error validating {preset_name}: {str(e)}")
            import traceback
            traceback.print_exc()
            continue
    
    # Create comprehensive visualization
    print("\n\n" + "="*80)
    print("Creating comparison visualizations...")
    print("="*80 + "\n")
    
    create_comparison_plots(results)
    
    # Create summary report
    print_summary_report(results)
    
    return results


def create_comparison_plots(results):
    """Create visualization comparing predictions vs actual"""
    
    n_portfolios = len(results)
    if n_portfolios == 0:
        print("No results to plot")
        return
    
    # Create figure with subplots
    fig = plt.figure(figsize=(16, 10))
    gs = fig.add_gridspec(3, 2, hspace=0.3, wspace=0.3)
    
    # 1. Portfolio-level comparison
    ax1 = fig.add_subplot(gs[0, :])
    portfolios = list(results.keys())
    predicted = [results[p]['predicted_portfolio_return'] for p in portfolios]
    actual = [results[p]['actual_portfolio_return'] for p in portfolios]
    
    x = np.arange(len(portfolios))
    width = 0.35
    
    ax1.bar(x - width/2, [p*100 for p in predicted], width, label='Predicted', 
            color='#667eea', alpha=0.8)
    ax1.bar(x + width/2, [a*100 for a in actual], width, label='Actual', 
            color='#f5576c', alpha=0.8)
    
    ax1.set_xlabel('Portfolio', fontsize=12, fontweight='bold')
    ax1.set_ylabel('Annual Return (%)', fontsize=12, fontweight='bold')
    ax1.set_title('Portfolio-Level Return Predictions vs Actual (2024-2025)', 
                  fontsize=14, fontweight='bold', pad=20)
    ax1.set_xticks(x)
    ax1.set_xticklabels(portfolios, rotation=45, ha='right')
    ax1.legend(fontsize=11)
    ax1.grid(True, alpha=0.3, axis='y')
    
    # Add error bars
    for i, p in enumerate(portfolios):
        error = results[p]['portfolio_error'] * 100
        ax1.plot([i, i], [predicted[i]*100, actual[i]*100], 'k-', linewidth=2, alpha=0.5)
        ax1.text(i, (predicted[i] + actual[i])/2 * 100 + 2, 
                f'{error:.1f}%', ha='center', fontsize=9, fontweight='bold')
    
    # 2. Error metrics comparison
    ax2 = fig.add_subplot(gs[1, 0])
    mae_values = [results[p]['mae'] * 100 for p in portfolios]
    mape_values = [results[p]['mape'] for p in portfolios]
    
    x2 = np.arange(len(portfolios))
    ax2_twin = ax2.twinx()
    
    bars1 = ax2.bar(x2 - 0.2, mae_values, 0.4, label='MAE (%)', color='#4facfe', alpha=0.8)
    bars2 = ax2_twin.bar(x2 + 0.2, mape_values, 0.4, label='MAPE (%)', color='#43e97b', alpha=0.8)
    
    ax2.set_xlabel('Portfolio', fontsize=11, fontweight='bold')
    ax2.set_ylabel('Mean Absolute Error (%)', fontsize=11, fontweight='bold', color='#4facfe')
    ax2_twin.set_ylabel('Mean Absolute % Error (%)', fontsize=11, fontweight='bold', color='#43e97b')
    ax2.set_title('Prediction Error Metrics', fontsize=12, fontweight='bold')
    ax2.set_xticks(x2)
    ax2.set_xticklabels(portfolios, rotation=45, ha='right')
    ax2.tick_params(axis='y', labelcolor='#4facfe')
    ax2_twin.tick_params(axis='y', labelcolor='#43e97b')
    ax2.grid(True, alpha=0.3, axis='y')
    
    # 3. Individual asset predictions (for first portfolio)
    if portfolios:
        first_portfolio = portfolios[0]
        ax3 = fig.add_subplot(gs[1, 1])
        
        tickers = results[first_portfolio]['tickers']
        pred_vals = [results[first_portfolio]['predicted_returns'].get(t, 0) * 100 for t in tickers]
        actual_vals = [results[first_portfolio]['actual_returns'].get(t, 0) * 100 for t in tickers]
        
        x3 = np.arange(len(tickers))
        ax3.bar(x3 - 0.2, pred_vals, 0.4, label='Predicted', color='#667eea', alpha=0.8)
        ax3.bar(x3 + 0.2, actual_vals, 0.4, label='Actual', color='#f5576c', alpha=0.8)
        
        ax3.set_xlabel('Asset', fontsize=11, fontweight='bold')
        ax3.set_ylabel('Annual Return (%)', fontsize=11, fontweight='bold')
        ax3.set_title(f'Individual Asset Predictions: {first_portfolio}', fontsize=12, fontweight='bold')
        ax3.set_xticks(x3)
        ax3.set_xticklabels(tickers)
        ax3.legend()
        ax3.grid(True, alpha=0.3, axis='y')
    
    # 4. Scatter plot: Predicted vs Actual
    ax4 = fig.add_subplot(gs[2, :])
    
    all_predicted = []
    all_actual = []
    all_labels = []
    
    for p_name, p_data in results.items():
        for ticker in p_data['tickers']:
            pred = p_data['predicted_returns'].get(ticker, 0)
            actual = p_data['actual_returns'].get(ticker, 0)
            if not np.isnan(pred) and not np.isnan(actual):
                all_predicted.append(pred * 100)
                all_actual.append(actual * 100)
                all_labels.append(f"{p_name}\n{ticker}")
    
    if all_predicted:
        ax4.scatter(all_predicted, all_actual, alpha=0.6, s=100, c='#667eea', edgecolors='black', linewidth=1.5)
        
        # Add diagonal line (perfect prediction)
        min_val = min(min(all_predicted), min(all_actual))
        max_val = max(max(all_predicted), max(all_actual))
        ax4.plot([min_val, max_val], [min_val, max_val], 'r--', linewidth=2, label='Perfect Prediction', alpha=0.7)
        
        # Calculate R²
        from sklearn.metrics import r2_score
        r2 = r2_score(all_actual, all_predicted)
        
        ax4.set_xlabel('Predicted Return (%)', fontsize=12, fontweight='bold')
        ax4.set_ylabel('Actual Return (%)', fontsize=12, fontweight='bold')
        ax4.set_title(f'Predicted vs Actual Returns (All Assets)\nR² = {r2:.3f}', 
                     fontsize=13, fontweight='bold')
        ax4.legend()
        ax4.grid(True, alpha=0.3)
        
        # Add R² annotation
        ax4.text(0.05, 0.95, f'R² = {r2:.3f}', transform=ax4.transAxes,
                fontsize=12, fontweight='bold', verticalalignment='top',
                bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))
    
    plt.suptitle('ML Model Forecast Validation: 2022-2023 Training → 2024-2025 Testing', 
                 fontsize=16, fontweight='bold', y=0.98)
    
    # Save figure
    output_path = 'outputs/forecast_validation.png'
    import os
    os.makedirs('outputs', exist_ok=True)
    plt.savefig(output_path, dpi=300, bbox_inches='tight')
    print(f"✅ Visualization saved to: {output_path}")
    
    plt.show()


def print_summary_report(results):
    """Print comprehensive summary report"""
    
    print("\n" + "="*80)
    print("FORECAST VALIDATION SUMMARY REPORT")
    print("="*80 + "\n")
    
    print("Training Period: 2022-01-01 to 2023-12-31")
    print("Test Period: 2024-01-01 to 2025-12-31")
    print("\n" + "-"*80 + "\n")
    
    # Overall statistics
    all_mae = [results[p]['mae'] for p in results.keys()]
    all_mape = [results[p]['mape'] for p in results.keys()]
    all_portfolio_errors = [results[p]['portfolio_error_pct'] for p in results.keys()]
    
    print("OVERALL STATISTICS:")
    print(f"  Average MAE:              {np.mean(all_mae):.2%}")
    print(f"  Average MAPE:             {np.mean(all_mape):.1f}%")
    print(f"  Average Portfolio Error:  {np.mean(all_portfolio_errors):.1f}%")
    print(f"  Best Portfolio (lowest error): {min(results.keys(), key=lambda x: results[x]['portfolio_error_pct'])}")
    print(f"  Worst Portfolio (highest error): {max(results.keys(), key=lambda x: results[x]['portfolio_error_pct'])}")
    
    print("\n" + "-"*80 + "\n")
    
    # Detailed portfolio results
    print("PORTFOLIO-BY-PORTFOLIO RESULTS:\n")
    
    for p_name, p_data in results.items():
        print(f"{p_name}:")
        print(f"  Predicted Return: {p_data['predicted_portfolio_return']:7.2%}")
        print(f"  Actual Return:    {p_data['actual_portfolio_return']:7.2%}")
        print(f"  Error:            {p_data['portfolio_error']:7.2%} ({p_data['portfolio_error_pct']:.1f}%)")
        print(f"  MAE:              {p_data['mae']:7.2%}")
        print(f"  MAPE:             {p_data['mape']:7.1f}%")
        print()
    
    print("="*80 + "\n")


if __name__ == '__main__':
    results = validate_forecasts()
    print("\n✅ Forecast validation completed!")