generate_examples.py

"""
Generate example plots from the EMA regression strategy for documentation.
This script runs the full analysis and saves all plots to the examples/ folder.
"""

import os
import sys
import pandas as pd
import numpy as np
import matplotlib
matplotlib.use('Agg')  # Non-interactive backend
import matplotlib.pyplot as plt

from signal_analyzer import analyze, AnalysisConfig

# Configuration
DATA_PATH = r'Y:\1m_candle_dukascopy\EURGBP_GMT+0_NO-DST.csv'
EXAMPLES_DIR = 'examples'
USE_REAL_DATA = os.path.exists(DATA_PATH)


def load_data():
    """Load EURGBP data or generate synthetic data."""
    if USE_REAL_DATA:
        print(f"Loading real data from: {DATA_PATH}")
        eu = pd.read_csv(DATA_PATH, index_col=0)
        eu.index = pd.to_datetime(eu.index, errors='coerce')
        eu = eu.sort_index()
        eu = eu.resample('1h', closed='right', label='right').ohlc()
        
        df = pd.DataFrame()
        df['Open'] = eu['open']['open']
        df['High'] = eu['high']['high']
        df['Low'] = eu['low']['low']
        df['Close'] = eu['close']['close']
        
        print(f"Loaded {len(df)} bars of real EURGBP data")
    else:
        print("Generating synthetic data...")
        np.random.seed(42)
        n_bars = 5000
        
        start_price = 0.8500
        returns = np.random.randn(n_bars) * 0.0003 + 0.00001
        prices = start_price * np.exp(np.cumsum(returns))
        
        high_noise = np.abs(np.random.randn(n_bars) * 0.0002)
        low_noise = np.abs(np.random.randn(n_bars) * 0.0002)
        open_noise = np.random.randn(n_bars) * 0.0001
        
        df = pd.DataFrame({
            'Close': prices,
            'Open': prices + open_noise,
            'High': prices + high_noise,
            'Low': prices - low_noise,
        })
        
        df['High'] = df[['Open', 'High', 'Close']].max(axis=1)
        df['Low'] = df[['Open', 'Low', 'Close']].min(axis=1)
        df.index = pd.date_range(start='2024-01-01', periods=n_bars, freq='1H')
        
        print(f"Generated {len(df)} bars of synthetic data")
    
    return df


def calculate_strategy(df):
    """Calculate EMA regression mean-reversion strategy."""
    print("Calculating strategy signals...")
    
    # EMA(10)
    df['ema_10'] = df.Close.ewm(alpha=2/(4+1), adjust=False).mean()
    
    # ATR(10)
    def atr(high, low, close, length):
        tr = pd.Series(0.0, index=high.index)
        for i in range(1, len(high)):
            terms = [
                high.iloc[i] - low.iloc[i],
                abs(high.iloc[i] - close.iloc[i - 1]),
                abs(close.iloc[i - 1] - low.iloc[i])
            ]
            tr.iloc[i] = max(terms)
        return tr.rolling(window=length, min_periods=1).mean()
    
    df['atr_10'] = atr(df.High, df.Low, df.Close, 10)
    
    # EMA prediction using linear regression
    REG_LB = 100
    ema = df["ema_10"].to_numpy(dtype=float)
    pred = np.full_like(ema, np.nan, dtype=float)
    
    x = np.arange(REG_LB, dtype=float)
    x_mean = x.mean()
    x_demean = x - x_mean
    den = np.sum(x_demean**2)
    
    for t in range(REG_LB, len(ema)):
        y = ema[t-REG_LB:t]
        y_mean = y.mean()
        slope = np.sum(x_demean * (y - y_mean)) / den
        intercept = y_mean - slope * x_mean
        pred[t] = slope * REG_LB + intercept
    
    df["ema_pred"] = pred
    df["ema_resid"] = df["ema_10"] - df["ema_pred"]
    df["ema_resid_norm"] = df["ema_resid"] / df["atr_10"]
    
    # Normal CDF approximation
    def normal_cdf(z):
        return 0.5 * (1 + np.sign(z) * np.sqrt(1 - np.exp(-2 * (z**2) / np.pi)))
    
    df["ema_resid_cdf"] = (normal_cdf(df["ema_resid_norm"]) - 0.5) * 2
    
    # Generate signals
    df['sig'] = 0.0
    df.loc[df.ema_resid_cdf > 0, 'sig'] = 1
    df.loc[df.ema_resid_cdf < 0, 'sig'] = -1
    df['sig'] = df.sig.shift(1).fillna(0)
    
    return df


def save_plots(result, config):
    """Save all plots to the examples folder."""
    print(f"\nSaving plots to {EXAMPLES_DIR}/...")
    
    saved_plots = []
    
    for plot_name, fig in result.plots.items():
        filename = f"{plot_name}.png"
        filepath = os.path.join(EXAMPLES_DIR, filename)
        
        fig.savefig(filepath, dpi=150, bbox_inches='tight')
        saved_plots.append(filename)
        print(f"  Saved: {filename}")
        plt.close(fig)
    
    return saved_plots


def generate_plot_descriptions():
    """Generate descriptions for each plot type."""
    descriptions = {
        'scatter': {
            'title': 'Scatter Plot: MFE vs MAE',
            'description': 'Shows the 2D distribution of Max Favorable Excursion (profit) vs Max Adverse Excursion (drawdown). Each point represents one trade. The shape reveals the trade geometry - wedge shapes indicate good asymmetry, tight clusters show consistent behavior.'
        },
        'marginals': {
            'title': 'Marginal Distributions',
            'description': 'Histograms and KDE curves showing the distribution of MFE and MAE separately. Reveals whether profits come from many small wins or few large winners, and shows the tail risk in drawdowns.'
        },
        'frontiers': {
            'title': 'Risk/Reward Frontiers',
            'description': 'Top: Risk-constrained frontier showing maximum achievable MFE for given drawdown limits. Bottom: Opportunity-constrained frontier showing required drawdown to achieve profit targets. Red stars mark knee points (optimal stop sizes).'
        },
        'ordering': {
            'title': 'Time Sequencing Analysis',
            'description': 'Scatter plot colored by ordering: Green = MFE-first (profit before pain), Red = MAE-first (pain before profit). Reveals whether the strategy needs room to breathe or if trailing stops are suitable.'
        },
        'heatmap_prob': {
            'title': 'TP/SL Probability Heatmap',
            'description': 'Path-dependent probability of hitting Take Profit before Stop Loss for various TP/SL combinations. Warmer colors indicate higher success probability. Black contour lines show 30%, 50%, 70% probability levels.'
        },
        'heatmap_ev': {
            'title': 'Expected Value Heatmap',
            'description': 'Expected value (EV) for each TP/SL combination accounting for probabilities and costs. Green zones are profitable, red zones are unprofitable. Black contour shows break-even line (EV=0).'
        },
        'volnorm_long': {
            'title': 'Vol-Normalized Comparison (Long)',
            'description': 'Side-by-side comparison of trade geometry in percentage space (left) vs volatility-adjusted space (right). Shows whether the edge is regime-dependent or stable across volatility conditions.'
        },
        'regime_long': {
            'title': 'Volatility Regime Analysis (Long)',
            'description': 'Trade geometry split by volatility regimes (low/mid/high). Reveals whether different exit rules are needed for different market conditions.'
        },
        'clusters_long': {
            'title': 'Trade Clusters (Long)',
            'description': 'Unsupervised clustering reveals distinct trade archetypes. Different colors represent different patterns (fast winners, needs room, noise, etc.). Helps identify which trades carry the edge.'
        },
        'cluster_stats_long': {
            'title': 'Cluster Statistics (Long)',
            'description': 'Detailed statistics for each trade cluster including size, median MFE/MAE, win rate, and ordering proportions. Bottom panel suggests appropriate exit strategies per cluster type.'
        },
    }
    return descriptions


def update_readme(saved_plots, data_type):
    """Update README with example images."""
    descriptions = generate_plot_descriptions()
    
    # Build the examples section
    examples_md = f"""
## Example Output

This analysis was run on a **{data_type}** using an EMA regression mean-reversion strategy on EURGBP hourly data.

### Strategy Description
The strategy:
- Predicts EMA using linear regression on historical EMA values
- Trades residuals (actual - predicted) normalized by ATR
- Goes long when residual > 0, short when residual < 0
- This is a **mean-reversion** strategy that captures small, frequent moves

### Generated Visualizations

"""
    
    # Add images with descriptions
    for plot_name in saved_plots:
        base_name = plot_name.replace('.png', '')
        if base_name in descriptions:
            desc = descriptions[base_name]
            examples_md += f"""
#### {desc['title']}

{desc['description']}

![{desc['title']}](examples/{plot_name})

---

"""
    
    return examples_md


def main():
    """Main execution."""
    print("="*80)
    print("GENERATING EXAMPLE PLOTS FOR DOCUMENTATION")
    print("="*80)
    
    # 1. Load data
    df = load_data()
    
    # 2. Calculate strategy
    df = calculate_strategy(df)
    df = df.dropna()
    
    n_long = (df.sig > 0).sum()
    n_short = (df.sig < 0).sum()
    print(f"Signal distribution: Long={n_long}, Short={n_short}")
    
    # 3. Run analysis
    print("\nRunning full analysis (all sections)...")
    config = AnalysisConfig(
        H=20,
        sections=['A', 'B', 'C', 'D', 'E', 'F'],
        trim_method='iqr',
        trim_k=3.0,
        store_paths=True,
        vol_col='atr_10',
        n_regimes=3,
    )
    
    result = analyze(df, sig_col='sig', config=config)
    
    print(f"Analysis complete:")
    print(f"  Long trades:  {result.long_trades.n_trades if result.long_trades else 0}")
    print(f"  Short trades: {result.short_trades.n_trades if result.short_trades else 0}")
    print(f"  Plots generated: {len(result.plots)}")
    
    # 4. Save plots
    saved_plots = save_plots(result, config)
    
    # 5. Generate README section
    data_type = "real EURGBP hourly data" if USE_REAL_DATA else "synthetic EURGBP-like data"
    examples_section = update_readme(saved_plots, data_type)
    
    # Save examples section to a separate file
    with open('EXAMPLES.md', 'w') as f:
        f.write(examples_section)
    
    print(f"\n{'='*80}")
    print("COMPLETE!")
    print(f"{'='*80}")
    print(f"\nGenerated {len(saved_plots)} plots in {EXAMPLES_DIR}/")
    print("Example documentation saved to EXAMPLES.md")
    print("\nTo add to README.md, insert the contents of EXAMPLES.md")
    print("before the 'Contributing' section.")


if __name__ == "__main__":
    main()