evaluate_models.py

"""
Model Evaluation Module
Compares all hyperparameter tuning approaches and generates summary reports.
"""

import numpy as np
import os


def load_results():
    """Load results from all tuning methods."""
    results = {}
    
    files = {
        'baseline': 'output/baseline_results.npy',
        'grid_search': 'output/grid_search_results.npy',
        'random_search': 'output/random_search_results.npy',
        'bayesian': 'output/bayesian_results.npy'
    }
    
    for name, filepath in files.items():
        if os.path.exists(filepath):
            results[name] = np.load(filepath, allow_pickle=True).item()
            print(f"Loaded: {filepath}")
        else:
            results[name] = None
            print(f"Warning: {filepath} not found")
    
    return results


def compare_methods(baseline_acc, baseline_time, grid_acc, grid_time, 
                   random_acc, random_time, bayes_acc, bayes_time):
    """Compare all tuning methods and create a summary."""
    results = {}
    results['Baseline'] = {'accuracy': baseline_acc, 'time': baseline_time}
    results['Grid Search'] = {'accuracy': grid_acc, 'time': grid_time}
    results['Random Search'] = {'accuracy': random_acc, 'time': random_time}
    results['Bayesian'] = {'accuracy': bayes_acc, 'time': bayes_time}
    return results


def print_comparison_table(results):
    """Print a formatted comparison table of all methods."""
    if results is None:
        print("No results to display. Complete Task 8 first.")
        return
    
    print("\n" + "=" * 70)
    print("HYPERPARAMETER TUNING COMPARISON")
    print("=" * 70)
    print(f"{'Method':<20} {'Accuracy':<15} {'Time (s)':<15} {'Improvement':<15}")
    print("-" * 70)
    
    baseline_acc = results.get('Baseline', {}).get('accuracy', 0)
    
    for method, data in results.items():
        if data:
            acc = data['accuracy']
            time_s = data['time']
            improvement = ((acc - baseline_acc) / baseline_acc * 100) if baseline_acc > 0 else 0
            
            if method == 'Baseline':
                print(f"{method:<20} {acc:.4f}         {time_s:.2f}           -")
            else:
                print(f"{method:<20} {acc:.4f}         {time_s:.2f}           {improvement:+.2f}%")
    
    print("=" * 70)


def analyze_tradeoffs(results):
    """Analyze and print trade-offs between different methods."""
    if results is None:
        return
    
    print("\n" + "=" * 70)
    print("TRADE-OFF ANALYSIS")
    print("=" * 70)
    
    # Find best accuracy
    best_method = max(results.items(), key=lambda x: x[1]['accuracy'] if x[1] else 0)
    print(f"Highest Accuracy: {best_method[0]} ({best_method[1]['accuracy']:.4f})")
    
    # Find fastest tuning method (excluding baseline)
    tuning_methods = {k: v for k, v in results.items() if k != 'Baseline' and v}
    if tuning_methods:
        fastest = min(tuning_methods.items(), key=lambda x: x[1]['time'])
        print(f"Fastest Tuning: {fastest[0]} ({fastest[1]['time']:.2f}s)")
    
    # Efficiency analysis
    baseline_acc = results.get('Baseline', {}).get('accuracy', 0)
    print("\nEfficiency (accuracy gain per second of tuning):")
    for method, data in results.items():
        if method != 'Baseline' and data:
            improvement = data['accuracy'] - baseline_acc
            efficiency = improvement / data['time'] if data['time'] > 0 else 0
            print(f"  {method}: {efficiency:.6f} accuracy/second")
    
    print("=" * 70)


def save_comparison(results):
    """Save the comparison results."""
    if results:
        np.save('output/comparison_results.npy', results)
        print("\nComparison results saved to output/comparison_results.npy")


if __name__ == "__main__":
    os.makedirs('output', exist_ok=True)
    
    print("Loading results from all tuning methods...")
    print("-" * 50)
    saved_results = load_results()
    
    missing = [k for k, v in saved_results.items() if v is None]
    if missing:
        print(f"\nMissing results for: {', '.join(missing)}")
        print("Run the corresponding scripts first to generate results.")
    
    if saved_results['baseline'] is not None:
        baseline = saved_results['baseline']
        grid = saved_results.get('grid_search') or {'test_accuracy': 0, 'search_time': 0}
        random = saved_results.get('random_search') or {'test_accuracy': 0, 'search_time': 0}
        bayesian = saved_results.get('bayesian') or {'test_accuracy': 0, 'search_time': 0}
        
        results = compare_methods(
            baseline_acc=baseline.get('accuracy', 0),
            baseline_time=baseline.get('train_time', 0),
            grid_acc=grid.get('test_accuracy', 0),
            grid_time=grid.get('search_time', 0),
            random_acc=random.get('test_accuracy', 0),
            random_time=random.get('search_time', 0),
            bayes_acc=bayesian.get('test_accuracy', 0),
            bayes_time=bayesian.get('search_time', 0)
        )
        
        print_comparison_table(results)
        analyze_tradeoffs(results)
        save_comparison(results)
    else:
        print("\nRun baseline_model.py first to generate baseline results.")