dashboard.py

"""
FactorDB Dashboard - Professional Factor Analytics Platform

A clean, professional interface for quantitative factor research.

Usage:
    uv run streamlit run dashboard.py
    .venv\\Scripts\\python -m streamlit run dashboard.py
"""

import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'src'))

import streamlit as st
import pandas as pd
import numpy as np
from datetime import datetime
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots

from factordb import FactorDBConfig, AssetType, FactorType
from factordb.storage import ClickHouseStorage
import networkx as nx


# =============================================================================
# Professional Light Theme & Styling
# =============================================================================

# Color palette - Professional quant style (light theme)
COLORS = {
    'primary': '#1E3A5F',       # Deep blue - primary
    'secondary': '#2E86AB',     # Medium blue
    'accent': '#3498DB',        # Light blue accent
    'positive': '#27AE60',      # Green - positive
    'negative': '#E74C3C',      # Red - negative
    'neutral': '#95A5A6',       # Gray - neutral
    'background': '#FFFFFF',    # White background
    'surface': '#F8F9FA',       # Light gray surface
    'border': '#E9ECEF',        # Border color
    'text': '#2C3E50',          # Dark text
    'text_light': '#495057',    # Light text (darker for readability)
}


def apply_professional_theme():
    """Apply professional light theme styling."""
    st.markdown("""
    <style>
    /* Import professional font */
    @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap');

    /* Global Theme - Clean White Professional Style */
    .stApp {
        background: #FFFFFF;
        font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
    }

    /* Main container */
    .main .block-container {
        padding-top: 2rem;
        padding-bottom: 2rem;
        max-width: 100%;
    }

    /* Header styling */
    .main-header {
        background: linear-gradient(135deg, #1E3A5F 0%, #2E86AB 100%);
        border-radius: 12px;
        padding: 1.8rem 2.5rem;
        margin-bottom: 2rem;
        box-shadow: 0 4px 20px rgba(30, 58, 95, 0.15);
    }

    .main-header h1 {
        color: #FFFFFF;
        font-size: 1.9rem;
        font-weight: 700;
        margin: 0;
        letter-spacing: -0.02em;
    }

    .main-header .subtitle {
        color: rgba(255, 255, 255, 0.85);
        font-size: 0.95rem;
        margin-top: 0.4rem;
        font-weight: 400;
    }

    /* Metric cards styling */
    .metric-card {
        background: #FFFFFF;
        border: 1px solid #E9ECEF;
        border-radius: 12px;
        padding: 1.2rem 1.5rem;
        text-align: center;
        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.04);
        transition: all 0.2s ease;
    }

    .metric-card:hover {
        transform: translateY(-2px);
        box-shadow: 0 4px 16px rgba(0, 0, 0, 0.08);
        border-color: #3498DB;
    }

    .metric-card .label {
        color: #495057;
        font-size: 0.8rem;
        text-transform: uppercase;
        letter-spacing: 0.08em;
        margin-bottom: 0.5rem;
        font-weight: 600;
    }

    .metric-card .value {
        color: #2C3E50;
        font-size: 1.25rem;
        font-weight: 700;
        font-family: 'JetBrains Mono', monospace;
        white-space: nowrap;
        overflow: hidden;
        text-overflow: ellipsis;
    }

    .metric-card .value.positive { color: #27AE60; }
    .metric-card .value.negative { color: #E74C3C; }
    .metric-card .value.accent { color: #2E86AB; }

    /* Section headers */
    .section-header {
        color: #1E3A5F;
        font-size: 1.15rem;
        font-weight: 700;
        margin: 2rem 0 1.2rem 0;
        padding-bottom: 0.6rem;
        border-bottom: 3px solid #3498DB;
        display: flex;
        align-items: center;
        gap: 0.5rem;
    }

    /* Info cards */
    .info-card {
        background: #F8F9FA;
        border: 1px solid #E9ECEF;
        border-radius: 10px;
        padding: 1.2rem;
        margin-bottom: 0.8rem;
    }

    .info-card .label {
        color: #495057;
        font-size: 0.75rem;
        text-transform: uppercase;
        letter-spacing: 0.05em;
        font-weight: 600;
    }

    .info-card .value {
        color: #2C3E50;
        font-size: 1.1rem;
        font-weight: 600;
        margin-top: 0.4rem;
        font-family: 'JetBrains Mono', monospace;
    }

    /* Expression display */
    .expression-box {
        background: #F8F9FA;
        border: 1px solid #E9ECEF;
        border-left: 4px solid #3498DB;
        border-radius: 8px;
        padding: 1.2rem 1.5rem;
        font-family: 'JetBrains Mono', monospace;
        font-size: 1rem;
        color: #1E3A5F;
        overflow-x: auto;
        white-space: pre-wrap;
        word-break: break-all;
    }

    /* Performance table styling */
    .perf-table {
        width: 100%;
        border-collapse: collapse;
        font-size: 1rem;
        margin: 1rem 0;
    }

    .perf-table th {
        background: #1E3A5F;
        color: #FFFFFF;
        font-weight: 600;
        text-transform: uppercase;
        font-size: 0.8rem;
        letter-spacing: 0.05em;
        padding: 1rem 1.2rem;
        text-align: center;
    }

    .perf-table td {
        background: #FFFFFF;
        color: #2C3E50;
        padding: 0.9rem 1.2rem;
        text-align: center;
        border-bottom: 1px solid #E9ECEF;
        font-family: 'JetBrains Mono', monospace;
        font-size: 0.95rem;
    }

    .perf-table tr:hover td {
        background: #F8F9FA;
    }

    .perf-table td.positive { color: #27AE60; font-weight: 600; }
    .perf-table td.negative { color: #E74C3C; font-weight: 600; }
    .perf-table td.period-label {
        background: #F8F9FA;
        font-weight: 600;
        color: #1E3A5F;
        text-align: left;
        padding-left: 1.5rem;
    }

    /* Status badges */
    .badge {
        display: inline-block;
        padding: 0.35rem 0.8rem;
        border-radius: 6px;
        font-size: 0.8rem;
        font-weight: 600;
        text-transform: uppercase;
        letter-spacing: 0.03em;
    }

    .badge-long { background: rgba(39, 174, 96, 0.12); color: #27AE60; }
    .badge-short { background: rgba(231, 76, 60, 0.12); color: #E74C3C; }
    .badge-pass { background: rgba(39, 174, 96, 0.12); color: #27AE60; }
    .badge-fail { background: rgba(231, 76, 60, 0.12); color: #E74C3C; }

    /* Sidebar styling */
    section[data-testid="stSidebar"] {
        background: #F8F9FA;
        border-right: 1px solid #E9ECEF;
    }

    section[data-testid="stSidebar"] .stSelectbox label,
    section[data-testid="stSidebar"] .stNumberInput label,
    section[data-testid="stSidebar"] .stCheckbox label {
        color: #2C3E50 !important;
        font-weight: 600;
    }

    /* Tab styling */
    .stTabs [data-baseweb="tab-list"] {
        background: #F8F9FA;
        border-radius: 10px;
        padding: 0.4rem;
        gap: 0.5rem;
        border: 1px solid #E9ECEF;
    }

    .stTabs [data-baseweb="tab"] {
        background: transparent;
        color: #495057;
        border-radius: 8px;
        padding: 0.6rem 1.8rem;
        font-weight: 600;
        font-size: 0.95rem;
    }

    .stTabs [aria-selected="true"] {
        background: #1E3A5F !important;
        color: white !important;
    }

    /* Dataframe styling - LARGER FONTS */
    .stDataFrame {
        border-radius: 10px;
        overflow: hidden;
        border: 1px solid #E9ECEF;
    }

    .stDataFrame table {
        font-size: 1rem !important;
    }

    .stDataFrame th {
        font-size: 0.9rem !important;
        font-weight: 600 !important;
        background: #1E3A5F !important;
        color: white !important;
        padding: 1rem !important;
    }

    .stDataFrame td {
        font-size: 1rem !important;
        padding: 0.9rem !important;
        font-family: 'JetBrains Mono', monospace !important;
    }

    /* Button styling */
    .stButton > button {
        background: linear-gradient(135deg, #1E3A5F 0%, #2E86AB 100%);
        color: white;
        border: none;
        border-radius: 8px;
        padding: 0.6rem 1.8rem;
        font-weight: 600;
        font-size: 0.95rem;
        transition: all 0.2s;
    }

    .stButton > button:hover {
        background: linear-gradient(135deg, #2E86AB 0%, #3498DB 100%);
        box-shadow: 0 4px 12px rgba(30, 58, 95, 0.25);
        transform: translateY(-1px);
    }

    /* Hide Streamlit branding */
    #MainMenu {visibility: hidden;}
    footer {visibility: hidden;}
    header {visibility: hidden;}

    /* Chart container */
    .chart-container {
        background: #FFFFFF;
        border: 1px solid #E9ECEF;
        border-radius: 12px;
        padding: 1.5rem;
        margin: 1rem 0;
        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.04);
    }

    /* Scrollbar styling */
    ::-webkit-scrollbar {
        width: 8px;
        height: 8px;
    }

    ::-webkit-scrollbar-track {
        background: #F8F9FA;
    }

    ::-webkit-scrollbar-thumb {
        background: #CED4DA;
        border-radius: 4px;
    }

    ::-webkit-scrollbar-thumb:hover {
        background: #ADB5BD;
    }
    </style>
    """, unsafe_allow_html=True)


# =============================================================================
# Data Loading
# =============================================================================

@st.cache_data(ttl=300)
def load_factor_data(asset_type_str: str, factor_type_str: str) -> pd.DataFrame:
    """Load factor data from database with caching."""
    asset_type = AssetType[asset_type_str]
    factor_type = FactorType[factor_type_str]
    config = FactorDBConfig()
    db_name = config.get_factor_db_name(asset_type, factor_type)

    # All metrics from evaluation (matching actual schema)
    metric_columns = """
        e.ic_mean,
        e.rank_ic_mean,
        e.ic_ir,
        e.rank_ic_ir,
        e.ic_t_stat,
        e.direction,
        e.ic_mean_5y,
        e.rank_ic_5y,
        e.ic_ir_5y,
        e.rank_ic_ir_5y,
        e.ic_mean_1y,
        e.rank_ic_1y,
        e.ic_ir_1y,
        e.rank_ic_ir_1y,
        e.is_mono_5,
        e.is_mono_10,
        e.is_mono_15,
        e.is_mono_5_5y,
        e.is_mono_10_5y,
        e.is_mono_15_5y,
        e.is_mono_5_1y,
        e.is_mono_10_1y,
        e.is_mono_15_1y,
        e.top_bottom_return_abs_5,
        e.top_bottom_return_abs_10,
        e.top_bottom_return_abs_15,
        e.top_bottom_sr_5,
        e.top_bottom_sr_10,
        e.top_bottom_sr_15,
        e.factor_std,
        e.factor_skewness,
        e.factor_kurtosis,
        e.upgrade_datetime as eval_datetime
    """

    sql = f"""
        SELECT
            b.factor_id as factor_id,
            b.name,
            b.expression,
            b.explanation,
            b.register_datetime,
            u.latest_date,
            u.upgrade_datetime as upgrade_datetime,
            {metric_columns}
        FROM {db_name}.A1_factor_basic b
        LEFT JOIN {db_name}.A3_factor_upgrade u ON b.factor_id = u.factor_id
        LEFT JOIN {db_name}.A2_factor_evaluate e ON b.factor_id = e.factor_id
        ORDER BY b.factor_id
    """

    with ClickHouseStorage(config) as storage:
        df = storage._fetch(sql, asset_type, factor_type)

    return df


def format_metric(value, decimals=4, percentage=False, show_sign=False):
    """Format metric value for display."""
    if pd.isna(value):
        return "-"
    if percentage:
        return f"{value * 100:+.2f}%" if show_sign else f"{value * 100:.2f}%"
    if show_sign:
        return f"{value:+.{decimals}f}"
    return f"{value:.{decimals}f}"


@st.cache_data(ttl=300)
def load_factor_values(
    factor_id: str,
    asset_type_str: str,
    factor_type_str: str
) -> pd.Series:
    """Load past 3 years of factor values for distribution display."""
    asset_type = AssetType[asset_type_str]
    factor_type = FactorType[factor_type_str]
    config = FactorDBConfig()
    db_name = config.get_factor_db_name(asset_type, factor_type)

    sql = f"""
        SELECT factor_value
        FROM {db_name}.{factor_id}
        WHERE factor_value IS NOT NULL
                    AND date >= addDays(today(), -365 * 3)
        ORDER BY date DESC
    """

    try:
        with ClickHouseStorage(config) as storage:
            df = storage._fetch(sql, asset_type, factor_type)
            if 'factor_value' in df.columns:
                return pd.to_numeric(df['factor_value'], errors='coerce').dropna()
            return pd.Series(dtype=float)
    except Exception:
        return pd.Series(dtype=float)


# =============================================================================
# Orthogonality Analysis Data Loading
# =============================================================================

@st.cache_data(ttl=300)
def load_orthogonality_analysis_list(asset_type_str: str, factor_type_str: str) -> pd.DataFrame:
    """Load list of orthogonality analyses."""
    asset_type = AssetType[asset_type_str]
    factor_type = FactorType[factor_type_str]
    config = FactorDBConfig()
    db_name = config.get_factor_db_name(asset_type, factor_type)

    sql = f"""
        SELECT
            analysis_id,
            analysis_datetime,
            total_factors,
            data_start_date,
            data_end_date,
            effective_n_90,
            effective_n_95,
            mean_correlation,
            max_correlation,
            redundancy_ratio,
            high_corr_pairs_count,
            n_clusters,
            n_selected,
            n_removed
        FROM {db_name}.B1_orthogonality_analysis
        ORDER BY analysis_datetime DESC
        LIMIT 20
    """

    try:
        with ClickHouseStorage(config) as storage:
            df = storage._fetch(sql, asset_type, factor_type)
        return df
    except Exception:
        return pd.DataFrame()


@st.cache_data(ttl=300)
def load_orthogonality_detail(analysis_id: str, asset_type_str: str, factor_type_str: str) -> dict:
    """Load detailed orthogonality analysis results."""
    asset_type = AssetType[asset_type_str]
    factor_type = FactorType[factor_type_str]
    config = FactorDBConfig()
    db_name = config.get_factor_db_name(asset_type, factor_type)

    result = {}

    # Load summary
    sql_summary = f"""
        SELECT * FROM {db_name}.B1_orthogonality_analysis
        WHERE analysis_id = '{analysis_id}'
    """

    # Load high correlation pairs
    sql_pairs = f"""
        SELECT factor_id_1, factor_id_2, correlation, same_cluster
        FROM {db_name}.B2_high_correlation_pairs
        WHERE analysis_id = '{analysis_id}'
        ORDER BY abs(correlation) DESC
        LIMIT 100
    """

    # Load clusters
    sql_clusters = f"""
        SELECT factor_id, cluster_id, is_representative, centrality_score
        FROM {db_name}.B3_factor_clusters
        WHERE analysis_id = '{analysis_id}'
        ORDER BY cluster_id, centrality_score DESC
    """

    # Load MST edges
    sql_mst = f"""
        SELECT source_factor, target_factor, distance, correlation
        FROM {db_name}.B4_mst_edges
        WHERE analysis_id = '{analysis_id}'
    """

    try:
        with ClickHouseStorage(config) as storage:
            result['summary'] = storage._fetch(sql_summary, asset_type, factor_type)
            result['high_corr_pairs'] = storage._fetch(sql_pairs, asset_type, factor_type)
            result['clusters'] = storage._fetch(sql_clusters, asset_type, factor_type)
            result['mst_edges'] = storage._fetch(sql_mst, asset_type, factor_type)
    except Exception as e:
        st.error(f"Failed to load orthogonality details: {e}")
        return {}

    return result


@st.cache_data(ttl=300)
def load_correlation_matrix(analysis_id: str, asset_type_str: str, factor_type_str: str) -> pd.DataFrame:
    """Load correlation matrix from parquet file."""
    asset_type = AssetType[asset_type_str]
    factor_type = FactorType[factor_type_str]
    config = FactorDBConfig()
    db_name = config.get_factor_db_name(asset_type, factor_type)

    # Get file path from database
    sql = f"""
        SELECT correlation_matrix_path
        FROM {db_name}.B1_orthogonality_analysis
        WHERE analysis_id = '{analysis_id}'
    """

    def _sparse_corr_to_matrix(df: pd.DataFrame) -> pd.DataFrame:
        """Convert upper-triangular long format (factor_i, factor_j, correlation) to square matrix."""
        factors = sorted(set(df['factor_i'].tolist()) | set(df['factor_j'].tolist()))
        if not factors:
            return pd.DataFrame()
        idx = {f: i for i, f in enumerate(factors)}
        mat = np.zeros((len(factors), len(factors)), dtype=float)
        for _, row in df.iterrows():
            i = idx[row['factor_i']]
            j = idx[row['factor_j']]
            val = row['correlation']
            mat[i, j] = val
            mat[j, i] = val
        return pd.DataFrame(mat, index=factors, columns=factors)

    try:
        with ClickHouseStorage(config) as storage:
            result = storage._fetch(sql, asset_type, factor_type)
            if len(result) > 0 and result.iloc[0]['correlation_matrix_path']:
                path = result.iloc[0]['correlation_matrix_path']
                if os.path.exists(path):
                    df = pd.read_parquet(path)
                    # Handle sparse upper-triangular format used by orthogonality_storage
                    if {'factor_i', 'factor_j', 'correlation'}.issubset(df.columns):
                        return _sparse_corr_to_matrix(df)
                    return df
    except Exception:
        pass

    return pd.DataFrame()


# =============================================================================
# Orthogonality Visualization Components
# =============================================================================

def create_correlation_heatmap(corr_matrix: pd.DataFrame, max_factors: int = 50) -> go.Figure:
    """Create correlation matrix heatmap."""
    # Limit size for visualization
    if len(corr_matrix) > max_factors:
        corr_matrix = corr_matrix.iloc[:max_factors, :max_factors]

    fig = go.Figure(data=go.Heatmap(
        z=corr_matrix.values,
        x=corr_matrix.columns,
        y=corr_matrix.index,
        colorscale='RdBu_r',
        zmid=0,
        zmin=-1,
        zmax=1,
        colorbar=dict(title='Correlation', tickfont=dict(size=11)),
        hovertemplate='%{x}<br>%{y}<br>Corr: %{z:.3f}<extra></extra>'
    ))

    fig.update_layout(
        title=dict(text=f'Factor Correlation Matrix ({len(corr_matrix)} factors)',
                   font=dict(size=18, color=COLORS['text'])),
        plot_bgcolor='white',
        paper_bgcolor='white',
        font=dict(family='Inter, sans-serif', size=12, color=COLORS['text']),
        height=600,
        margin=dict(t=60, b=100, l=100, r=40),
        xaxis=dict(tickangle=45, tickfont=dict(size=9)),
        yaxis=dict(tickfont=dict(size=9), automargin=True)
    )

    return fig


def build_corr_matrix_from_pairs(pairs_df: pd.DataFrame, factor_ids: list[str]) -> pd.DataFrame:
    """Reconstruct a symmetric correlation matrix from pair list for a given factor subset."""
    if pairs_df.empty or not factor_ids:
        return pd.DataFrame()

    subset = pairs_df[
        pairs_df['factor_id_1'].isin(factor_ids) & pairs_df['factor_id_2'].isin(factor_ids)
    ]
    if subset.empty:
        return pd.DataFrame()

    matrix = pd.DataFrame(index=factor_ids, columns=factor_ids, dtype=float)
    for _, row in subset.iterrows():
        f1, f2, corr = row['factor_id_1'], row['factor_id_2'], row['correlation']
        matrix.loc[f1, f2] = corr
        matrix.loc[f2, f1] = corr
    np.fill_diagonal(matrix.values, 1.0)

    # Fill any remaining gaps with 0 to avoid NaNs in the heatmap while keeping symmetry.
    matrix = matrix.fillna(0.0)
    return matrix


def sanitize_corr_matrix(corr_matrix: pd.DataFrame) -> pd.DataFrame:
    """Ensure the correlation matrix is square, numeric, and NaN-free for plotting."""
    if corr_matrix.empty:
        return corr_matrix

    corr_matrix = corr_matrix.copy()

    # If factor ids are stored as a column, promote it to index first.
    idx_candidates = ['factor_id', 'index', '__index_level_0__']
    for cand in idx_candidates:
        if cand in corr_matrix.columns and corr_matrix[cand].nunique() == len(corr_matrix):
            corr_matrix = corr_matrix.set_index(cand)
            break

    # Heuristic: parquet sometimes saves factor ids as the first column rather than index.
    if isinstance(corr_matrix.index, pd.RangeIndex) and corr_matrix.shape[1] == len(corr_matrix) + 1:
        first_col = corr_matrix.columns[0]
        if corr_matrix[first_col].nunique() == len(corr_matrix):
            corr_matrix = corr_matrix.set_index(first_col)

    # Normalize labels to strings to improve intersection matching
    corr_matrix.index = corr_matrix.index.map(str)
    corr_matrix.columns = corr_matrix.columns.map(str)

    # Align rows/cols to shared labels to guarantee squareness
    common = [c for c in corr_matrix.columns if c in corr_matrix.index]
    if not common:
        return pd.DataFrame()
    corr_matrix = corr_matrix.loc[common, common]

    # Coerce to numeric and fill diagonals + gaps
    corr_matrix = corr_matrix.apply(pd.to_numeric, errors='coerce')
    if len(corr_matrix) > 0:
        np.fill_diagonal(corr_matrix.values, 1.0)
    corr_matrix = corr_matrix.fillna(0.0)

    return corr_matrix


def create_effective_n_chart(summary: pd.Series) -> go.Figure:
    """Create effective N visualization."""
    total = summary.get('total_factors', 0)
    eff_90 = summary.get('effective_n_90', 0)
    eff_95 = summary.get('effective_n_95', 0)

    categories = ['Total Factors', 'Effective N (90%)', 'Effective N (95%)']
    values = [total, eff_90, eff_95]
    colors = [COLORS['primary'], COLORS['positive'], COLORS['accent']]

    fig = go.Figure(data=[
        go.Bar(
            x=categories,
            y=values,
            marker_color=colors,
            text=values,
            textposition='auto',
            textfont=dict(size=16, color='white')
        )
    ])

    fig.update_layout(
        title=dict(text='Factor Dimensionality Analysis',
                   font=dict(size=18, color=COLORS['text'])),
        plot_bgcolor='white',
        paper_bgcolor='white',
        font=dict(family='Inter, sans-serif', size=14, color=COLORS['text']),
        height=400,
        margin=dict(t=60, b=50, l=70, r=40),
        yaxis=dict(title='Number of Factors', gridcolor=COLORS['border']),
        xaxis=dict(tickfont=dict(size=13))
    )

    # Add redundancy ratio annotation
    redundancy = summary.get('redundancy_ratio', 0)
    fig.add_annotation(
        x=0.5, y=1.05, xref='paper', yref='paper',
        text=f'Redundancy Ratio: {redundancy:.1%}',
        showarrow=False,
        font=dict(size=14, color=COLORS['negative'])
    )

    return fig


def create_cluster_distribution_chart(clusters_df: pd.DataFrame) -> go.Figure:
    """Create cluster size distribution chart."""
    if clusters_df.empty:
        return go.Figure()

    cluster_sizes = clusters_df.groupby('cluster_id').size().reset_index(name='size')
    cluster_sizes = cluster_sizes.sort_values('size', ascending=False)

    fig = go.Figure(data=[
        go.Bar(
            x=[f'Cluster {i}' for i in cluster_sizes['cluster_id']],
            y=cluster_sizes['size'],
            marker_color=COLORS['accent'],
            text=cluster_sizes['size'],
            textposition='auto',
            textfont=dict(size=12, color='white')
        )
    ])

    fig.update_layout(
        title=dict(text='Cluster Size Distribution',
                   font=dict(size=18, color=COLORS['text'])),
        plot_bgcolor='white',
        paper_bgcolor='white',
        font=dict(family='Inter, sans-serif', size=14, color=COLORS['text']),
        height=400,
        margin=dict(t=60, b=80, l=70, r=40),
        yaxis=dict(title='Number of Factors', gridcolor=COLORS['border']),
        xaxis=dict(tickangle=45, tickfont=dict(size=11))
    )

    return fig


def create_mst_network_graph(mst_edges: pd.DataFrame, clusters_df: pd.DataFrame) -> go.Figure:
    """Create MST network visualization using Plotly."""
    if mst_edges.empty:
        return go.Figure()

    # Build networkx graph
    G = nx.Graph()

    for _, row in mst_edges.iterrows():
        G.add_edge(row['source_factor'], row['target_factor'], weight=row.get('distance', 1))

    # Get layout
    pos = nx.spring_layout(G, k=2, iterations=50, seed=42)

    # Create cluster color mapping
    cluster_colors = {}
    if not clusters_df.empty:
        color_palette = px.colors.qualitative.Set3
        for _, row in clusters_df.iterrows():
            cluster_id = row['cluster_id']
            cluster_colors[row['factor_id']] = color_palette[cluster_id % len(color_palette)]

    # Edge trace
    edge_x, edge_y = [], []
    for edge in G.edges():
        x0, y0 = pos[edge[0]]
        x1, y1 = pos[edge[1]]
        edge_x.extend([x0, x1, None])
        edge_y.extend([y0, y1, None])

    edge_trace = go.Scatter(
        x=edge_x, y=edge_y,
        line=dict(width=1, color=COLORS['neutral']),
        hoverinfo='none',
        mode='lines'
    )

    # Node trace
    node_x = [pos[node][0] for node in G.nodes()]
    node_y = [pos[node][1] for node in G.nodes()]
    node_colors = [cluster_colors.get(node, COLORS['accent']) for node in G.nodes()]
    node_text = list(G.nodes())

    node_trace = go.Scatter(
        x=node_x, y=node_y,
        mode='markers+text',
        hoverinfo='text',
        text=node_text,
        textposition='top center',
        textfont=dict(size=8),
        marker=dict(
            size=12,
            color=node_colors,
            line=dict(width=1, color='white')
        )
    )

    fig = go.Figure(data=[edge_trace, node_trace])

    fig.update_layout(
        title=dict(text='Factor MST Network (colored by cluster)',
                   font=dict(size=18, color=COLORS['text'])),
        showlegend=False,
        plot_bgcolor='white',
        paper_bgcolor='white',
        height=500,
        margin=dict(t=60, b=20, l=20, r=20),
        xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
        yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
    )

    return fig


def render_orthogonality_tab(asset_type: str, factor_type: str, factor_df: pd.DataFrame):
    """Render the orthogonality analysis tab."""
    st.markdown('<div class="section-header">Orthogonality Analysis Results</div>',
                unsafe_allow_html=True)

    # Load analysis list
    analyses_df = load_orthogonality_analysis_list(asset_type, factor_type)

    if analyses_df.empty:
        st.info("No orthogonality analysis results found. Run orthogonality analysis first using: `python main.py orthogonality`")
        return

    # Analysis selector
    col1, col2 = st.columns([2, 1])

    with col1:
        analysis_options = []
        for _, row in analyses_df.iterrows():
            dt = str(row['analysis_datetime'])[:19]
            n_factors = row['total_factors']
            analysis_options.append(f"{row['analysis_id']} ({dt}, {n_factors} factors)")

        selected_idx = st.selectbox(
            "Select Analysis",
            range(len(analysis_options)),
            format_func=lambda x: analysis_options[x],
            key="ortho_analysis_select"
        )

    selected_analysis_id = analyses_df.iloc[selected_idx]['analysis_id']

    # Load detailed results
    detail = load_orthogonality_detail(selected_analysis_id, asset_type, factor_type)

    if not detail or detail.get('summary') is None or detail['summary'].empty:
        st.error("Failed to load analysis details.")
        return

    summary = detail['summary'].iloc[0]

    # Summary metrics row
    st.markdown("<br>", unsafe_allow_html=True)

    cols = st.columns(6)
    metrics = [
        ("Total Factors", summary.get('total_factors', 0), "accent"),
        ("Effective N (90%)", summary.get('effective_n_90', 0), "positive"),
        ("Redundancy", f"{summary.get('redundancy_ratio', 0):.1%}", "negative"),
        ("Clusters", summary.get('n_clusters', 0), "accent"),
        ("High Corr Pairs", summary.get('high_corr_pairs_count', 0), "negative"),
        ("Mean Corr", f"{summary.get('mean_correlation', 0):.3f}", "accent"),
    ]

    for col, (label, value, color) in zip(cols, metrics):
        with col:
            st.markdown(f"""
            <div class="metric-card">
                <div class="label">{label}</div>
                <div class="value {color}">{value}</div>
            </div>
            """, unsafe_allow_html=True)

    st.markdown("<br>", unsafe_allow_html=True)

    # Tabs for different views
    ortho_tab1, ortho_tab2, ortho_tab3, ortho_tab4 = st.tabs([
        "Overview", "Correlation Matrix", "Clusters & MST", "High Correlation Pairs"
    ])

    with ortho_tab1:
        col1, col2 = st.columns(2)

        with col1:
            # Effective N chart
            fig_eff = create_effective_n_chart(summary)
            st.plotly_chart(fig_eff, width='stretch')

        with col2:
            # Cluster distribution
            if 'clusters' in detail and not detail['clusters'].empty:
                fig_cluster = create_cluster_distribution_chart(detail['clusters'])
                st.plotly_chart(fig_cluster, width='stretch')
            else:
                st.info("No cluster data available.")

        # Analysis info
        st.markdown('<div class="section-header">Analysis Details</div>', unsafe_allow_html=True)

        info_cols = st.columns(4)
        with info_cols[0]:
            st.markdown(f"""
            <div class="info-card">
                <div class="label">Data Period</div>
                <div class="value">{str(summary.get('data_start_date', ''))[:10]} ~ {str(summary.get('data_end_date', ''))[:10]}</div>
            </div>
            """, unsafe_allow_html=True)

        with info_cols[1]:
            st.markdown(f"""
            <div class="info-card">
                <div class="label">Max Correlation</div>
                <div class="value">{summary.get('max_correlation', 0):.3f}</div>
            </div>
            """, unsafe_allow_html=True)

        with info_cols[2]:
            st.markdown(f"""
            <div class="info-card">
                <div class="label">Effective N (95%)</div>
                <div class="value">{summary.get('effective_n_95', 0)}</div>
            </div>
            """, unsafe_allow_html=True)

        with info_cols[3]:
            st.markdown(f"""
            <div class="info-card">
                <div class="label">Analysis Time</div>
                <div class="value">{str(summary.get('analysis_datetime', ''))[:19]}</div>
            </div>
            """, unsafe_allow_html=True)

    with ortho_tab2:
        # Load base correlation matrix from parquet
        corr_matrix_raw = load_correlation_matrix(selected_analysis_id, asset_type, factor_type)
        corr_matrix = sanitize_corr_matrix(corr_matrix_raw)

        pairs_df = detail.get('high_corr_pairs', pd.DataFrame())

        # Derive factor sets
        top_factors = []
        if factor_df is not None and 'rank_ic_mean' in factor_df.columns:
            # Sort by absolute RankIC to capture both long/short skill
            top_factors = (
                factor_df[['factor_id', 'rank_ic_mean']]
                .dropna(subset=['rank_ic_mean'])
                .assign(rank_ic_abs=lambda x: x['rank_ic_mean'].abs())
                .sort_values('rank_ic_abs', ascending=False)
                .head(50)['factor_id']
                .tolist()
            )
        pair_factor_ids = []
        if not pairs_df.empty:
            pair_factor_ids = pd.unique(pairs_df[['factor_id_1', 'factor_id_2']].values.ravel()).tolist()

        # Fallback: rebuild overall matrix from pairs if parquet missing
        if corr_matrix.empty and not pairs_df.empty:
            corr_matrix = sanitize_corr_matrix(build_corr_matrix_from_pairs(pairs_df, pair_factor_ids))

        # Build RankIC Top 50 matrix from the best available source
        corr_rankic = pd.DataFrame()
        if not corr_matrix.empty and top_factors:
            usable = [f for f in top_factors if f in corr_matrix.index and f in corr_matrix.columns]
            if usable:
                corr_rankic = sanitize_corr_matrix(corr_matrix.loc[usable, usable])
        if corr_rankic.empty and top_factors and not pairs_df.empty:
            corr_rankic = sanitize_corr_matrix(build_corr_matrix_from_pairs(pairs_df, top_factors))

        corr_tab_all, corr_tab_top = st.tabs([
            "Overall Correlation", "Top |RankIC| 50"
        ])

        with corr_tab_all:
            if not corr_matrix.empty:
                # Allow full matrix size while keeping UI reasonable
                max_default = min(50, max(10, corr_matrix.shape[0]))
                max_display = st.slider(
                    "Max factors to display (overall)",
                    min_value=10,
                    max_value=min(200, corr_matrix.shape[0]),
                    value=max_default,
                    key="corr_max_display_all"
                )
                fig_corr = create_correlation_heatmap(corr_matrix, max_display)
                st.plotly_chart(fig_corr, width='stretch')
            else:
                st.info("Correlation matrix not available. Ensure analysis exports the parquet file or high_corr_pairs contains data.")

        with corr_tab_top:
            if not corr_rankic.empty:
                max_default_top = min(50, max(10, corr_rankic.shape[0]))
                max_display_top = st.slider(
                    "Max factors to display (|RankIC| top 50)",
                    min_value=10,
                    max_value=min(200, corr_rankic.shape[0]),
                    value=min(max_default_top, corr_rankic.shape[0]),
                    key="corr_max_display_top"
                )
                fig_corr_top = create_correlation_heatmap(corr_rankic, max_display_top)
                st.plotly_chart(fig_corr_top, width='stretch')
            else:
                st.info("Top RankIC correlation matrix not available. Make sure parquet is exported or high_corr_pairs covers these factors.")

    with ortho_tab3:
        col1, col2 = st.columns([1, 1])

        with col1:
            # MST Network
            if 'mst_edges' in detail and not detail['mst_edges'].empty:
                fig_mst = create_mst_network_graph(
                    detail['mst_edges'],
                    detail.get('clusters', pd.DataFrame())
                )
                st.plotly_chart(fig_mst, width='stretch')
            else:
                st.info("No MST data available.")

        with col2:
            # Cluster table
            if 'clusters' in detail and not detail['clusters'].empty:
                st.markdown("**Cluster Assignments**")
                clusters_display = detail['clusters'][['factor_id', 'cluster_id', 'is_representative', 'centrality_score']].copy()
                clusters_display['is_representative'] = clusters_display['is_representative'].apply(
                    lambda x: '★' if x else '')
                clusters_display = clusters_display.rename(columns={
                    'factor_id': 'Factor',
                    'cluster_id': 'Cluster',
                    'is_representative': 'Rep',
                    'centrality_score': 'Centrality'
                })
                st.dataframe(clusters_display, height=450, hide_index=True)
            else:
                st.info("No cluster data available.")

    with ortho_tab4:
        if 'high_corr_pairs' in detail and not detail['high_corr_pairs'].empty:
            pairs_df = detail['high_corr_pairs'].copy()
            pairs_df['correlation'] = pairs_df['correlation'].round(4)
            pairs_df['same_cluster'] = pairs_df['same_cluster'].apply(
                lambda x: 'Yes' if x else 'No')
            pairs_df = pairs_df.rename(columns={
                'factor_id_1': 'Factor 1',
                'factor_id_2': 'Factor 2',
                'correlation': 'Correlation',
                'same_cluster': 'Same Cluster'
            })

            st.markdown(f"**High Correlation Pairs** (>{0.7:.0%} correlation)")
            st.dataframe(pairs_df, height=500, hide_index=True, width='stretch')
        else:
            st.info("No high correlation pairs found.")

def render_header(asset_type: str, factor_count: int):
    """Render professional header."""
    current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    st.markdown(f"""
    <div class="main-header">
        <h1>FactorDB Analytics Platform</h1>
        <div class="subtitle">
            {asset_type} Daily Factors  |  {factor_count} Factors  |  {current_time}
        </div>
    </div>
    """, unsafe_allow_html=True)


def render_sidebar():
    """Render sidebar with filters and settings."""
    with st.sidebar:
        st.markdown("### Settings")

        asset_type = st.selectbox(
            "Asset Type",
            ["STOCK", "ETF"],
            index=0,
            help="Select factor asset class"
        )

        factor_type = "DAILY"

        st.markdown("---")
        st.markdown("### Filters")

        min_ic = st.number_input(
            "Min |IC Mean|",
            min_value=0.0,
            max_value=1.0,
            value=0.0,
            step=0.005,
            format="%.3f"
        )

        min_icir = st.number_input(
            "Min |IC IR|",
            min_value=0.0,
            max_value=10.0,
            value=0.0,
            step=0.1,
            format="%.2f"
        )

        only_mono = st.checkbox("Monotonic Only (10g)", value=False)

        st.markdown("---")

        if st.button("Refresh Data"):
            st.cache_data.clear()
            st.rerun()

        return {
            "asset_type": asset_type,
            "factor_type": factor_type,
            "min_ic": min_ic,
            "min_icir": min_icir,
            "only_mono": only_mono,
        }


def render_summary_metrics(df: pd.DataFrame):
    """Render summary statistics with professional styling."""
    total = len(df)
    with_values = df['latest_date'].notna().sum() if 'latest_date' in df.columns else 0
    with_eval = df['ic_mean'].notna().sum() if 'ic_mean' in df.columns else 0

    latest_date = "-"
    if 'latest_date' in df.columns and df['latest_date'].notna().any():
        latest_date = str(df['latest_date'].max())[:10]

    avg_ic = df['rank_ic_mean'].abs().mean() if 'rank_ic_mean' in df.columns else 0
    avg_icir = df['rank_ic_ir'].abs().mean() if 'rank_ic_ir' in df.columns else 0

    cols = st.columns(6)

    metrics = [
        ("Total Factors", f"{total:,}", "accent"),
        ("With Values", f"{with_values:,}", "accent"),
        ("With Eval", f"{with_eval:,}", "accent"),
        ("Avg |RankIC|", f"{avg_ic:.4f}" if avg_ic else "-", "positive" if avg_ic > 0.02 else "accent"),
        ("Avg |RankICIR|", f"{avg_icir:.3f}" if avg_icir else "-", "positive" if avg_icir > 0.5 else "accent"),
        ("Latest Date", latest_date, "accent"),
    ]

    for col, (label, value, color) in zip(cols, metrics):
        with col:
            st.markdown(f"""
            <div class="metric-card">
                <div class="label">{label}</div>
                <div class="value {color}">{value}</div>
            </div>
            """, unsafe_allow_html=True)


def render_factor_table(df: pd.DataFrame, filters: dict):
    """Render the main factor table with professional styling."""
    filtered_df = df.copy()

    if filters["min_ic"] > 0 and 'ic_mean' in filtered_df.columns:
        filtered_df = filtered_df[filtered_df['ic_mean'].abs() >= filters["min_ic"]]

    if filters["min_icir"] > 0 and 'ic_ir' in filtered_df.columns:
        filtered_df = filtered_df[filtered_df['ic_ir'].abs() >= filters["min_icir"]]

    if filters["only_mono"] and 'is_mono_10' in filtered_df.columns:
        filtered_df = filtered_df[filtered_df['is_mono_10'] == 1]

    st.markdown(f'<div class="section-header">Factor Performance ({len(filtered_df)} factors)</div>',
                unsafe_allow_html=True)

    if len(filtered_df) == 0:
        st.info("No factors match the current filters.")
        return None

    display_cols = ['factor_id', 'latest_date', 'direction',
                    'ic_mean', 'rank_ic_mean', 'ic_ir', 'rank_ic_ir',
                    'ic_mean_5y', 'rank_ic_5y', 'ic_ir_5y', 'rank_ic_ir_5y',
                    'ic_mean_1y', 'rank_ic_1y', 'ic_ir_1y', 'rank_ic_ir_1y',
                    'is_mono_10', 'top_bottom_return_abs_10', 'top_bottom_sr_10']

    available_cols = [c for c in display_cols if c in filtered_df.columns]
    display_df = filtered_df[available_cols].copy()

    rename_map = {
        'factor_id': 'Factor ID',
        'latest_date': 'Data Date',
        'direction': 'Direction',
        'ic_mean': 'IC',
        'rank_ic_mean': 'RankIC',
        'ic_ir': 'ICIR',
        'rank_ic_ir': 'RankICIR',
        'ic_mean_5y': 'IC(5Y)',
        'rank_ic_5y': 'RankIC(5Y)',
        'ic_ir_5y': 'ICIR(5Y)',
        'rank_ic_ir_5y': 'RankICIR(5Y)',
        'ic_mean_1y': 'IC(1Y)',
        'rank_ic_1y': 'RankIC(1Y)',
        'ic_ir_1y': 'ICIR(1Y)',
        'rank_ic_ir_1y': 'RankICIR(1Y)',
        'is_mono_10': 'Mono',
        'top_bottom_return_abs_10': 'L/S Ret',
        'top_bottom_sr_10': 'L/S SR'
    }
    display_df = display_df.rename(columns=rename_map)

    # L/S returns are stored in decimal form (e.g., 0.05 for 5%).
    # Scale to percentage for display to avoid showing values near 0.00%.
    if 'L/S Ret' in display_df.columns:
        display_df['L/S Ret'] = display_df['L/S Ret'] * 100

    if 'Data Date' in display_df.columns:
        display_df['Data Date'] = display_df['Data Date'].apply(
            lambda x: str(x)[:10] if pd.notna(x) else "-")

    if 'Direction' in display_df.columns:
        display_df['Direction'] = display_df['Direction'].apply(
            lambda x: "LONG" if x == 1 else ("SHORT" if x == -1 else "-"))

    if 'Mono' in display_df.columns:
        display_df['Mono'] = display_df['Mono'].apply(
            lambda x: "PASS" if x == 1 else ("FAIL" if x == 0 else "-"))

    col1, col2 = st.columns([2, 1])

    with col1:
        sort_options = ['Factor ID', 'IC', 'RankIC', 'ICIR', 'RankICIR',
                        'IC(5Y)', 'RankIC(5Y)', 'IC(1Y)', 'RankIC(1Y)', 'L/S SR', 'Data Date']
        sort_options = [s for s in sort_options if s in display_df.columns]
        default_idx = sort_options.index('ICIR') if 'ICIR' in sort_options else 0
        sort_col = st.selectbox("Sort by", sort_options, index=default_idx, key="sort_col")

    with col2:
        sort_asc = st.checkbox("Ascending", value=False, key="sort_asc")

    numeric_cols = ['IC', 'RankIC', 'ICIR', 'RankICIR', 'IC(5Y)', 'RankIC(5Y)', 'ICIR(5Y)', 'RankICIR(5Y)',
                    'IC(1Y)', 'RankIC(1Y)', 'ICIR(1Y)', 'RankICIR(1Y)', 'L/S Ret', 'L/S SR']

    if sort_col in numeric_cols:
        display_df['_sort'] = display_df[sort_col].abs()
        display_df = display_df.sort_values('_sort', ascending=sort_asc, na_position='last')
        display_df = display_df.drop('_sort', axis=1)
    else:
        display_df = display_df.sort_values(sort_col, ascending=sort_asc, na_position='last')

    column_config = {
        "Factor ID": st.column_config.TextColumn(width="medium"),
        "Data Date": st.column_config.TextColumn(width="small"),
        "Direction": st.column_config.TextColumn(width="small"),
        "IC": st.column_config.NumberColumn(format="%.4f", width="small"),
        "RankIC": st.column_config.NumberColumn(format="%.4f", width="small"),
        "ICIR": st.column_config.NumberColumn(format="%.3f", width="small"),
        "RankICIR": st.column_config.NumberColumn(format="%.3f", width="small"),
        "IC(5Y)": st.column_config.NumberColumn(format="%.4f", width="small"),
        "RankIC(5Y)": st.column_config.NumberColumn(format="%.4f", width="small"),
        "ICIR(5Y)": st.column_config.NumberColumn(format="%.3f", width="small"),
        "RankICIR(5Y)": st.column_config.NumberColumn(format="%.3f", width="small"),
        "IC(1Y)": st.column_config.NumberColumn(format="%.4f", width="small"),
        "RankIC(1Y)": st.column_config.NumberColumn(format="%.4f", width="small"),
        "ICIR(1Y)": st.column_config.NumberColumn(format="%.3f", width="small"),
        "RankICIR(1Y)": st.column_config.NumberColumn(format="%.3f", width="small"),
        "Mono": st.column_config.TextColumn(width="small"),
        "L/S Ret": st.column_config.NumberColumn(format="%.2f%%", width="small"),
        "L/S SR": st.column_config.NumberColumn(format="%.3f", width="small"),
    }

    st.dataframe(
        display_df,
        width='stretch',
        height=500,
        column_config=column_config,
        hide_index=True,
    )

    return filtered_df


def render_basic_info_table(df: pd.DataFrame):
    """Render factor basic information table."""
    st.markdown('<div class="section-header">Factor Basic Information</div>',
                unsafe_allow_html=True)

    if len(df) == 0:
        st.info("No factors available.")
        return

    basic_cols = ['factor_id', 'name', 'expression', 'register_datetime',
                  'latest_date', 'upgrade_datetime']
    available_cols = [c for c in basic_cols if c in df.columns]
    basic_df = df[available_cols].copy()

    rename_map = {
        'factor_id': 'Factor ID',
        'name': 'Name',
        'expression': 'Expression',
        'register_datetime': 'Registered',
        'latest_date': 'Latest Date',
        'upgrade_datetime': 'Updated'
    }
    basic_df = basic_df.rename(columns=rename_map)

    if 'Registered' in basic_df.columns:
        basic_df['Registered'] = basic_df['Registered'].apply(
            lambda x: str(x)[:16] if pd.notna(x) else "-")
    if 'Latest Date' in basic_df.columns:
        basic_df['Latest Date'] = basic_df['Latest Date'].apply(
            lambda x: str(x)[:10] if pd.notna(x) else "-")
    if 'Updated' in basic_df.columns:
        basic_df['Updated'] = basic_df['Updated'].apply(
            lambda x: str(x)[:16] if pd.notna(x) else "-")

    basic_df = basic_df.sort_values('Factor ID')

    column_config = {
        "Factor ID": st.column_config.TextColumn(width="medium"),
        "Name": st.column_config.TextColumn(width="medium"),
        "Expression": st.column_config.TextColumn(width="large"),
        "Registered": st.column_config.TextColumn(width="medium"),
        "Latest Date": st.column_config.TextColumn(width="small"),
        "Updated": st.column_config.TextColumn(width="medium"),
    }

    st.dataframe(
        basic_df,
        width='stretch',
        height=400,
        column_config=column_config,
        hide_index=True,
    )


def render_upgrade_status_table(df: pd.DataFrame):
    """Render factor data update status from A3_factor_upgrade."""
    st.markdown('<div class="section-header">Factor Data Update Status (A3_factor_upgrade)</div>',
                unsafe_allow_html=True)

    if len(df) == 0:
        st.info("No factors available.")
        return

    # Calculate summary statistics
    total_factors = len(df)
    with_data = df['latest_date'].notna().sum() if 'latest_date' in df.columns else 0

    # Get date statistics
    latest_date_max = None
    latest_date_min = None
    if 'latest_date' in df.columns and df['latest_date'].notna().any():
        latest_date_max = df['latest_date'].max()
        latest_date_min = df['latest_date'].min()

    # Summary metrics
    cols = st.columns(4)

    with cols[0]:
        st.markdown(f"""
        <div class="metric-card">
            <div class="label">Total Factors</div>
            <div class="value accent">{total_factors:,}</div>
        </div>
        """, unsafe_allow_html=True)

    with cols[1]:
        st.markdown(f"""
        <div class="metric-card">
            <div class="label">With Data</div>
            <div class="value positive">{with_data:,}</div>
        </div>
        """, unsafe_allow_html=True)

    with cols[2]:
        st.markdown(f"""
        <div class="metric-card">
            <div class="label">Newest Data</div>
            <div class="value accent">{str(latest_date_max)[:10] if latest_date_max else '-'}</div>
        </div>
        """, unsafe_allow_html=True)

    with cols[3]:
        st.markdown(f"""
        <div class="metric-card">
            <div class="label">Oldest Data</div>
            <div class="value accent">{str(latest_date_min)[:10] if latest_date_min else '-'}</div>
        </div>
        """, unsafe_allow_html=True)

    st.markdown("<br>", unsafe_allow_html=True)

    # Data table
    upgrade_cols = ['factor_id', 'latest_date', 'upgrade_datetime']
    available_cols = [c for c in upgrade_cols if c in df.columns]

    if not available_cols:
        st.warning("No upgrade status data available.")
        return

    upgrade_df = df[available_cols].copy()

    rename_map = {
        'factor_id': 'Factor ID',
        'latest_date': 'Latest Data Date',
        'upgrade_datetime': 'Last Update Time'
    }
    upgrade_df = upgrade_df.rename(columns=rename_map)

    # Format datetime columns
    if 'Latest Data Date' in upgrade_df.columns:
        upgrade_df['Latest Data Date'] = upgrade_df['Latest Data Date'].apply(
            lambda x: str(x)[:10] if pd.notna(x) else "-")

    if 'Last Update Time' in upgrade_df.columns:
        upgrade_df['Last Update Time'] = upgrade_df['Last Update Time'].apply(
            lambda x: str(x)[:19] if pd.notna(x) else "-")

    # Sorting options
    col1, col2 = st.columns([2, 1])

    with col1:
        sort_options = ['Factor ID', 'Latest Data Date', 'Last Update Time']
        sort_options = [s for s in sort_options if s in upgrade_df.columns]
        sort_col = st.selectbox("Sort by", sort_options, index=1 if len(sort_options) > 1 else 0, key="upgrade_sort_col")

    with col2:
        sort_asc = st.checkbox("Ascending", value=False, key="upgrade_sort_asc")

    upgrade_df = upgrade_df.sort_values(sort_col, ascending=sort_asc, na_position='last')

    column_config = {
        "Factor ID": st.column_config.TextColumn(width="medium"),
        "Latest Data Date": st.column_config.TextColumn(width="medium"),
        "Last Update Time": st.column_config.TextColumn(width="medium"),
    }

    st.dataframe(
        upgrade_df,
        width='stretch',
        height=450,
        column_config=column_config,
        hide_index=True,
    )


def create_ic_comparison_chart(factor_data: pd.Series):
    """Create IC metrics comparison chart across periods."""
    def get_val(col):
        if col in factor_data.index:
            val = factor_data[col]
            return val if pd.notna(val) else 0
        return 0

    # Data for IC comparison
    periods = ['Full', '5 Year', '1 Year']
    ic_values = [get_val('ic_mean'), get_val('ic_mean_5y'), get_val('ic_mean_1y')]
    rank_ic_values = [get_val('rank_ic_mean'), get_val('rank_ic_5y'), get_val('rank_ic_1y')]

    fig = go.Figure()

    fig.add_trace(go.Bar(
        name='IC Mean',
        x=periods,
        y=ic_values,
        marker_color=COLORS['primary'],
        text=[f'{v:.4f}' for v in ic_values],
        textposition='auto',
        insidetextfont=dict(size=12, color='white'),
        outsidetextfont=dict(size=12, color=COLORS['text'])
    ))

    fig.add_trace(go.Bar(
        name='Rank IC',
        x=periods,
        y=rank_ic_values,
        marker_color=COLORS['accent'],
        text=[f'{v:.4f}' for v in rank_ic_values],
        textposition='auto',
        insidetextfont=dict(size=12, color='white'),
        outsidetextfont=dict(size=12, color=COLORS['text'])
    ))

    fig.update_layout(
        title=dict(text='IC Metrics by Period', font=dict(size=18, color=COLORS['text'])),
        barmode='group',
        plot_bgcolor='white',
        paper_bgcolor='white',
        font=dict(family='Inter, sans-serif', size=14, color=COLORS['text']),
        legend=dict(
            orientation='h', yanchor='bottom', y=1.02, xanchor='right', x=1,
            font=dict(size=13, color=COLORS['text'])
        ),
        yaxis=dict(
            title=dict(text='IC Value', font=dict(size=13, color=COLORS['text'])),
            gridcolor=COLORS['border'],
            zerolinecolor=COLORS['border'],
            tickfont=dict(size=12, color=COLORS['text']),
            automargin=True
        ),
        xaxis=dict(tickfont=dict(size=13, color=COLORS['text'])),
        height=380,
        margin=dict(t=80, b=50, l=70, r=40)
    )

    # Add zero line
    fig.add_hline(y=0, line_dash="solid", line_color=COLORS['neutral'], line_width=1)

    return fig


def create_icir_comparison_chart(factor_data: pd.Series):
    """Create ICIR comparison chart across periods."""
    def get_val(col):
        if col in factor_data.index:
            val = factor_data[col]
            return val if pd.notna(val) else 0
        return 0

    periods = ['Full', '5 Year', '1 Year']
    icir_values = [get_val('ic_ir'), get_val('ic_ir_5y'), get_val('ic_ir_1y')]
    rank_icir_values = [get_val('rank_ic_ir'), get_val('rank_ic_ir_5y'), get_val('rank_ic_ir_1y')]

    fig = go.Figure()

    fig.add_trace(go.Bar(
        name='ICIR',
        x=periods,
        y=icir_values,
        marker_color=COLORS['primary'],
        text=[f'{v:.3f}' for v in icir_values],
        textposition='auto',
        insidetextfont=dict(size=12, color='white'),
        outsidetextfont=dict(size=12, color=COLORS['text'])
    ))

    fig.add_trace(go.Bar(
        name='Rank ICIR',
        x=periods,
        y=rank_icir_values,
        marker_color=COLORS['accent'],
        text=[f'{v:.3f}' for v in rank_icir_values],
        textposition='auto',
        insidetextfont=dict(size=12, color='white'),
        outsidetextfont=dict(size=12, color=COLORS['text'])
    ))

    fig.update_layout(
        title=dict(text='ICIR Metrics by Period', font=dict(size=18, color=COLORS['text'])),
        barmode='group',
        plot_bgcolor='white',
        paper_bgcolor='white',
        font=dict(family='Inter, sans-serif', size=14, color=COLORS['text']),
        legend=dict(
            orientation='h', yanchor='bottom', y=1.02, xanchor='right', x=1,
            font=dict(size=13, color=COLORS['text'])
        ),
        yaxis=dict(
            title=dict(text='ICIR Value', font=dict(size=13, color=COLORS['text'])),
            gridcolor=COLORS['border'],
            zerolinecolor=COLORS['border'],
            tickfont=dict(size=12, color=COLORS['text']),
            automargin=True
        ),
        xaxis=dict(tickfont=dict(size=13, color=COLORS['text'])),
        height=380,
        margin=dict(t=80, b=50, l=70, r=40)
    )

    # Add threshold lines
    fig.add_hline(y=0.5, line_dash="dash", line_color=COLORS['positive'],
                  annotation_text="Good (0.5)", annotation_position="right",
                  annotation_font_size=12)
    fig.add_hline(y=0, line_dash="solid", line_color=COLORS['neutral'], line_width=1)

    return fig


def create_long_short_returns_chart(factor_data: pd.Series):
    """Create long-short returns comparison chart."""
    def get_val(col):
        if col in factor_data.index:
            val = factor_data[col]
            return val if pd.notna(val) else 0
        return 0

    groups = ['5 Groups', '10 Groups', '15 Groups']
    returns = [
        get_val('top_bottom_return_abs_5') * 100,
        get_val('top_bottom_return_abs_10') * 100,
        get_val('top_bottom_return_abs_15') * 100
    ]
    sharpes = [
        get_val('top_bottom_sr_5'),
        get_val('top_bottom_sr_10'),
        get_val('top_bottom_sr_15')
    ]

    fig = make_subplots(
        rows=1, cols=2,
        subplot_titles=('Long-Short Returns (%)', 'Long-Short Sharpe Ratio'),
        horizontal_spacing=0.15
    )

    # Returns bar chart
    colors = [COLORS['positive'] if r > 0 else COLORS['negative'] for r in returns]
    fig.add_trace(
        go.Bar(
            x=groups,
            y=returns,
            marker_color=colors,
            text=[f'{v:.2f}%' for v in returns],
            textposition='auto',
            insidetextfont=dict(size=12, color='white'),
            outsidetextfont=dict(size=12, color=COLORS['text']),
            showlegend=False
        ),
        row=1, col=1
    )

    # Sharpe ratio bar chart
    colors_sr = [COLORS['positive'] if s > 0 else COLORS['negative'] for s in sharpes]
    fig.add_trace(
        go.Bar(
            x=groups,
            y=sharpes,
            marker_color=colors_sr,
            text=[f'{v:.3f}' for v in sharpes],
            textposition='auto',
            insidetextfont=dict(size=12, color='white'),
            outsidetextfont=dict(size=12, color=COLORS['text']),
            showlegend=False
        ),
        row=1, col=2
    )

    fig.update_layout(
        plot_bgcolor='white',
        paper_bgcolor='white',
        font=dict(family='Inter, sans-serif', size=14, color=COLORS['text']),
        height=380,
        margin=dict(t=60, b=50, l=70, r=40),
        title=dict(text='Long-Short Portfolio Performance', font=dict(size=18, color=COLORS['text']))
    )

    fig.update_xaxes(tickfont=dict(size=12, color=COLORS['text']))
    fig.update_yaxes(gridcolor=COLORS['border'], zerolinecolor=COLORS['border'], tickfont=dict(size=12, color=COLORS['text']), automargin=True)

    return fig


def create_monotonicity_chart(factor_data: pd.Series):
    """Create monotonicity heatmap across periods and group sizes."""
    def get_val(col):
        if col in factor_data.index:
            val = factor_data[col]
            return 1 if val == 1 else (0 if val == 0 else None)
        return None

    # Build data matrix
    periods = ['Full', '5 Year', '1 Year']
    groups = ['5 Groups', '10 Groups', '15 Groups']

    data = [
        [get_val('is_mono_5'), get_val('is_mono_10'), get_val('is_mono_15')],
        [get_val('is_mono_5_5y'), get_val('is_mono_10_5y'), get_val('is_mono_15_5y')],
        [get_val('is_mono_5_1y'), get_val('is_mono_10_1y'), get_val('is_mono_15_1y')]
    ]

    # Convert to display text
    text = []
    for row in data:
        text_row = []
        for val in row:
            if val == 1:
                text_row.append('PASS')
            elif val == 0:
                text_row.append('FAIL')
            else:
                text_row.append('-')
        text.append(text_row)

    # Color scale: None=gray, 0=red, 1=green
    z_data = [[0.5 if v is None else v for v in row] for row in data]

    fig = go.Figure(data=go.Heatmap(
        z=z_data,
        x=groups,
        y=periods,
        text=text,
        texttemplate="%{text}",
        textfont={"size": 16, "color": "white"},
        colorscale=[
            [0, COLORS['negative']],
            [0.5, COLORS['neutral']],
            [1, COLORS['positive']]
        ],
        showscale=False,
        hoverinfo='text'
    ))

    fig.update_layout(
        title=dict(text='Monotonicity Status', font=dict(size=18, color=COLORS['text'])),
        plot_bgcolor='white',
        paper_bgcolor='white',
        font=dict(family='Inter, sans-serif', size=14, color=COLORS['text']),
        height=320,
        margin=dict(t=60, b=50, l=100, r=40),
        xaxis=dict(tickfont=dict(size=13, color=COLORS['text'])),
        yaxis=dict(tickfont=dict(size=13, color=COLORS['text']), automargin=True)
    )

    return fig


def render_factor_detail(df: pd.DataFrame, asset_type: str, factor_type: str):
    """Render detailed view for a selected factor with visualizations."""
    if 'factor_id' not in df.columns or len(df) == 0:
        st.info("No factors available.")
        return

    factor_ids = sorted(df['factor_id'].tolist())

    # Factor selection
    selected_id = st.selectbox("Select Factor", factor_ids, key="detail_factor")

    if not selected_id:
        return

    factor = df[df['factor_id'] == selected_id].iloc[0]

    def get_val(col, default=None):
        if col in factor.index:
            val = factor[col]
            return val if pd.notna(val) else default
        return default

    # Basic Information Panel
    st.markdown('<div class="section-header">Basic Information</div>', unsafe_allow_html=True)

    col1, col2, col3, col4 = st.columns(4)

    with col1:
        st.markdown(f"""
        <div class="info-card">
            <div class="label">Factor ID</div>
            <div class="value">{get_val('factor_id', '-')}</div>
        </div>
        """, unsafe_allow_html=True)

    with col2:
        st.markdown(f"""
        <div class="info-card">
            <div class="label">Name</div>
            <div class="value">{get_val('name') or '-'}</div>
        </div>
        """, unsafe_allow_html=True)

    with col3:
        reg_time = get_val('register_datetime')
        st.markdown(f"""
        <div class="info-card">
            <div class="label">Registered</div>
            <div class="value">{str(reg_time)[:16] if reg_time else '-'}</div>
        </div>
        """, unsafe_allow_html=True)

    with col4:
        direction = get_val('direction')
        dir_class = "badge-long" if direction == 1 else ("badge-short" if direction == -1 else "")
        dir_str = "LONG" if direction == 1 else ("SHORT" if direction == -1 else "-")
        st.markdown(f"""
        <div class="info-card">
            <div class="label">Direction</div>
            <div class="value"><span class="badge {dir_class}">{dir_str}</span></div>
        </div>
        """, unsafe_allow_html=True)

    # Data Status Row
    col1, col2, col3, col4 = st.columns(4)

    with col1:
        latest = get_val('latest_date')
        st.markdown(f"""
        <div class="info-card">
            <div class="label">Latest Data</div>
            <div class="value">{str(latest)[:10] if latest else '-'}</div>
        </div>
        """, unsafe_allow_html=True)

    with col2:
        upg_time = get_val('upgrade_datetime')
        st.markdown(f"""
        <div class="info-card">
            <div class="label">Data Updated</div>
            <div class="value">{str(upg_time)[:16] if upg_time else '-'}</div>
        </div>
        """, unsafe_allow_html=True)

    with col3:
        eval_time = get_val('eval_datetime')
        st.markdown(f"""
        <div class="info-card">
            <div class="label">Evaluated</div>
            <div class="value">{str(eval_time)[:16] if eval_time else '-'}</div>
        </div>
        """, unsafe_allow_html=True)

    with col4:
        ic_tstat = get_val('ic_t_stat')
        tstat_color = "positive" if ic_tstat and abs(ic_tstat) > 2 else ""
        st.markdown(f"""
        <div class="info-card">
            <div class="label">IC t-stat</div>
            <div class="value {tstat_color}">{format_metric(ic_tstat, 2)}</div>
        </div>
        """, unsafe_allow_html=True)

    # Expression
    st.markdown('<div class="section-header">Expression</div>', unsafe_allow_html=True)
    expr = get_val('expression') or "N/A"
    st.markdown(f'<div class="expression-box">{expr}</div>', unsafe_allow_html=True)

    explanation = get_val('explanation')
    if explanation:
        st.markdown(f"**Explanation:** {explanation}")

    # Factor value distribution
    st.markdown('<div class="section-header">Factor Value Distribution</div>', unsafe_allow_html=True)
    with st.spinner("Loading factor values..."):
        values = load_factor_values(
            factor_id=get_val('factor_id', ''),
            asset_type_str=asset_type,
            factor_type_str=factor_type
        )

    if values is not None and len(values) > 0:
        q01 = values.quantile(0.01)
        q99 = values.quantile(0.99)
        clipped = values.clip(lower=q01, upper=q99)
        mean_value = clipped.mean()
        fig_hist = go.Figure()
        fig_hist.add_trace(go.Histogram(
            x=clipped,
            nbinsx=50,
            marker_color=COLORS['accent'],
            opacity=0.8
        ))
        fig_hist.add_vline(
            x=mean_value,
            line_color=COLORS['primary'],
            line_dash="dash",
            annotation_text=f"Mean: {mean_value:.4f}",
            annotation_position="top right"
        )
        fig_hist.update_layout(
            title=dict(text='Factor Value Distribution (past 3 years, 1% winsorized)', font=dict(size=16, color=COLORS['text'])),
            xaxis_title='Factor Value',
            yaxis_title='Count',
            bargap=0.05,
            plot_bgcolor='white',
            paper_bgcolor='white',
            font=dict(family='Inter, sans-serif', size=12, color=COLORS['text']),
            height=320,
            margin=dict(t=60, b=60, l=60, r=40)
        )
        st.plotly_chart(fig_hist, width='stretch')
        st.caption(f"Histogram applies 1% winsorization (clip to 1st/99th percentiles: {q01:.4f}, {q99:.4f}) to reduce extreme outlier impact.")
    else:
        st.info("No factor values available for distribution plot.")

    # Performance Visualizations
    st.markdown('<div class="section-header">Performance Analysis</div>', unsafe_allow_html=True)

    # Row 1: IC and ICIR comparison
    col1, col2 = st.columns(2)
    with col1:
        fig_ic = create_ic_comparison_chart(factor)
        st.plotly_chart(fig_ic, width='stretch')

    with col2:
        fig_icir = create_icir_comparison_chart(factor)
        st.plotly_chart(fig_icir, width='stretch')

    # Row 2: Long-Short Returns and Monotonicity
    col1, col2 = st.columns(2)
    with col1:
        fig_ls = create_long_short_returns_chart(factor)
        st.plotly_chart(fig_ls, width='stretch')

    with col2:
        fig_mono = create_monotonicity_chart(factor)
        st.plotly_chart(fig_mono, width='stretch')

    # Factor Statistics
    st.markdown('<div class="section-header">Factor Statistics</div>', unsafe_allow_html=True)

    cols = st.columns(4)
    stats = [
        ("IC t-stat", get_val('ic_t_stat'), 2),
        ("Factor Std", get_val('factor_std'), 4),
        ("Factor Skewness", get_val('factor_skewness'), 3),
        ("Factor Kurtosis", get_val('factor_kurtosis'), 3),
    ]

    def fmt(val, dec=4):
        if pd.isna(val):
            return "-"
        return f"{val:.{dec}f}"

    for col, (label, val, dec) in zip(cols, stats):
        with col:
            st.markdown(f"""
            <div class="metric-card">
                <div class="label">{label}</div>
                <div class="value accent">{fmt(val, dec)}</div>
            </div>
            """, unsafe_allow_html=True)


# =============================================================================
# Main App
# =============================================================================

def main():
    st.set_page_config(
        page_title="FactorDB Analytics",
        page_icon="",
        layout="wide",
        initial_sidebar_state="expanded"
    )

    # Apply professional theme
    apply_professional_theme()

    # Sidebar
    filters = render_sidebar()

    # Load data
    try:
        with st.spinner("Loading factor data..."):
            df = load_factor_data(filters['asset_type'], filters['factor_type'])
    except Exception as e:
        st.error(f"Failed to load data: {e}")
        st.info("Please check database connection and configuration.")
        return

    # Header
    render_header(filters['asset_type'], len(df))

    if len(df) == 0:
        st.warning("No factors found in the database.")
        return

    # Summary metrics
    render_summary_metrics(df)

    st.markdown("<br>", unsafe_allow_html=True)

    # Main content tabs
    tab1, tab2, tab3, tab4, tab5 = st.tabs([
        "Performance Overview",
        "Basic Information",
        "Data Update Status",
        "Factor Detail",
        "Orthogonality Analysis"
    ])

    with tab1:
        render_factor_table(df, filters)

    with tab2:
        render_basic_info_table(df)

    with tab3:
        render_upgrade_status_table(df)

    with tab4:
        render_factor_detail(df, filters['asset_type'], filters['factor_type'])

    with tab5:
        render_orthogonality_tab(filters['asset_type'], filters['factor_type'], df)


if __name__ == "__main__":
    main()