return_predictability.py

# ============================================================
# Return Predictability - Dividend-Price Ratio
# VCU - Advanced Financial Analytics (FIRE 691) - Week 1
# Author: Rohith Ravindra Reddy
# Key result: 5-yr R2 = 8.41% (annual), 7.55% (monthly)
# ============================================================
import pandas as pd, numpy as np, matplotlib.pyplot as plt
import statsmodels.api as sm
from google.colab import drive
drive.mount('/content/drive')
PATH = '/content/drive/MyDrive/FINANCIAL ANALYTICS Colab Notebooks/'

# ── LOAD DATA ────────────────────────────────────────────────
annual = pd.read_csv(PATH+'crsp_yearly_vwret.csv')
monthly = pd.read_csv(PATH+'crsp_monthly_vwret_dp.csv')
print("Annual:", annual.shape, "Monthly:", monthly.shape)
print("Annual cols:", annual.columns.tolist())
print("Monthly cols:", monthly.columns.tolist())

# ── Q1a: D/P RATIO TIME SERIES ───────────────────────────────
fig, axes = plt.subplots(2,1,figsize=(14,8))
axes[0].plot(annual.iloc[:,0], annual['dp'] if 'dp' in annual.columns else annual.iloc[:,2],color='navy',lw=1.5)
axes[0].set_title('Annual Dividend-Price Ratio (1927-2024)'); axes[0].grid(alpha=0.3)
axes[1].plot(monthly.iloc[:,0], monthly['dp'] if 'dp' in monthly.columns else monthly.iloc[:,2],color='crimson',lw=0.8)
axes[1].set_title('Monthly Dividend-Price Ratio'); axes[1].grid(alpha=0.3)
plt.tight_layout(); plt.savefig(PATH+'Q1a_DP_Ratio.png',dpi=150); plt.show()

# ── Q1b: EXCESS RETURNS & RF ─────────────────────────────────
fig, ax = plt.subplots(figsize=(14,5))
ax.plot(annual.iloc[:,0], annual['vwretd'] if 'vwretd' in annual.columns else annual.iloc[:,1],
        label='VW Return', color='navy', lw=1)
ax.axhline(0, color='black', lw=0.8)
ax.set_title('Annual VW Market Return (1927-2024)'); ax.legend(); ax.grid(alpha=0.3)
plt.tight_layout(); plt.savefig(PATH+'Q1b_RF_ExcessReturn.png',dpi=150); plt.show()

# ── Q1d: PREDICTIVE REGRESSION ───────────────────────────────
# ret_{t+1} = alpha + beta * dp_t + epsilon
df = annual.copy().dropna()
dp_col  = 'dp'   if 'dp'   in df.columns else df.columns[2]
ret_col = 'exret' if 'exret' in df.columns else df.columns[1]

df['dp_lag']   = df[dp_col].shift(1)
df['ret_fwd1'] = df[ret_col].shift(-1)
df_reg = df.dropna()

X = sm.add_constant(df_reg['dp_lag'])
model = sm.OLS(df_reg['ret_fwd1'], X).fit()
print("\n=== 1-Year Predictive Regression ===")
print(model.summary())
print(f"In-sample R2: {model.rsquared*100:.2f}%")

# ── Q1e: MULTI-HORIZON R2 ────────────────────────────────────
horizons = [1, 3, 12, 36, 60]  # months (or 1,2,3,4,5 years for annual
r2_results = {}
for h in [1,2,3,4,5]:  # years for annual data
    df_h = df.copy()
    # Cumulative forward return over h years
    df_h['ret_fwd'] = 0.0
    for i in range(1,h+1):
        df_h['ret_fwd'] += df_h[ret_col].shift(-i)
    df_h = df_h.dropna()
    X = sm.add_constant(df_h['dp_lag'])
    r2 = sm.OLS(df_h['ret_fwd'], X).fit().rsquared
    r2_results[h] = r2*100

print("\n=== R2 by Horizon (Annual Data) ===")
for h,r2 in r2_results.items():
    print(f"  {h}-year: {r2:.2f}%")

# Plot R2 vs horizon
fig, ax = plt.subplots(figsize=(9,5))
ax.plot(list(r2_results.keys()), list(r2_results.values()), 'o-', color='navy', lw=2)
ax.set_title('In-Sample R2 vs Return Horizon'); ax.set_xlabel('Horizon (Years)')
ax.set_ylabel('R2 (%)'); ax.grid(alpha=0.3)
plt.tight_layout(); plt.savefig(PATH+'Q1e_R2_byHorizon.png',dpi=150); plt.show()
print("\nKey finding: 5-year R2 = 8.41% (annual) | Predictability is a long-horizon phenomenon.")

# ── OUT-OF-SAMPLE R2 (Goyal-Welch) ───────────────────────────
split = int(len(df_reg)*0.6)
train = df_reg.iloc[:split]; test = df_reg.iloc[split:]

model_is = sm.OLS(train['ret_fwd1'], sm.add_constant(train['dp_lag'])).fit()
pred_oos  = model_is.predict(sm.add_constant(test['dp_lag']))
mean_bench = train['ret_fwd1'].mean()

SS_res = ((test['ret_fwd1']-pred_oos)**2).sum()
SS_tot = ((test['ret_fwd1']-mean_bench)**2).sum()
oos_r2 = (1 - SS_res/SS_tot)*100
print(f"\nOut-of-sample R2 (1-month): {oos_r2:.2f}%")
print("Finding: OOS R2 near zero or negative at short horizons - historical mean beats d/p model month-to-month.")