server.js

require('dotenv').config();
const express = require('express');
const http = require('http');
const { Server } = require('socket.io');
const cors = require('cors');
const path = require('path');

const db = require('./db');
const entities = require('./entities');
const manager = require('./collectors/manager');

// ─── Collectors ────────────────────────────────────────────────────
const rssCollector = require('./collectors/rss-collector');
let gdeltCollector, redditCollector, wikiCollector, hnCollector;
try { gdeltCollector = require('./collectors/gdelt-collector'); } catch (e) { console.log('[Server] GDELT collector not available yet'); }
try { redditCollector = require('./collectors/reddit-collector'); } catch (e) { console.log('[Server] Reddit collector not available yet'); }
try { wikiCollector = require('./collectors/wiki-collector'); } catch (e) { console.log('[Server] Wikipedia collector not available yet'); }
try { hnCollector = require('./collectors/hackernews-collector'); } catch (e) { console.log('[Server] HackerNews collector not available yet'); }

// ─── Finance (standalone lifecycle — not registered with manager) ──
const tickers = require('./entities/tickers');
let financeCollector, marketAnalyzer;
try { financeCollector = require('./collectors/finance-collector'); } catch (e) { console.log('[Server] Finance collector not available yet'); }
try { marketAnalyzer = require('./collectors/market-analyzer'); } catch (e) { console.log('[Server] Market analyzer not available yet'); }

// ─── Calendar (standalone lifecycle — Phase 6A) ──────────────────
let calendarCollector;
try { calendarCollector = require('./collectors/calendar-collector'); } catch (e) { console.log('[Server] Calendar collector not available yet'); }

// ─── AI Analysis (Phase 2B — via ModelManager) ─────────────────────
let aiAnalyzer, aiQueue;
let ModelManager, ModelRouter;
let modelManagerInstance = null;
try {
  aiAnalyzer = require('./analysis/ai-analyzer');
  ModelManager = require('./analysis/model-manager');
  const { ModelRouter: MR } = require('./analysis/model-router');
  ModelRouter = MR;
  aiQueue = require('./analysis/multi-tier-queue');
} catch (e) { console.log('[Server] AI analysis module not available yet:', e.message); }

// ─── Pattern Detector (Oracle Engine — optional) ─────────────────────
let patternDetector;
try {
  patternDetector = require('./analysis/pattern-detector');
} catch (e) { console.log('[Server] Pattern detector not available yet'); }

// ─── Prediction Generator (Oracle Engine Phase 3 — optional) ─────────
let predictionGenerator;
try {
  predictionGenerator = require('./oracle/prediction-generator');
} catch (e) { console.log('[Server] Prediction generator not available yet'); }

// ─── Prediction Resolver (Oracle Engine Phase 4A — optional) ─────────
let predictionResolver;
try {
  predictionResolver = require('./oracle/prediction-resolver');
} catch (e) { console.log('[Server] Prediction resolver not available yet'); }

// ─── Scoring (Oracle Engine Phase 4B — optional) ─────────────────────
let scoringModule;
try {
  scoringModule = require('./oracle/scoring');
} catch (e) { console.log('[Server] Scoring module not available yet'); }

// ─── Error Analyzer (Oracle Engine Phase 7A — optional) ──────────────
let errorAnalyzer;
try {
  errorAnalyzer = require('./oracle/error-analyzer');
} catch (e) { console.log('[Server] Error analyzer not available yet'); }

// ─── Auto-Tuner (Oracle Engine Phase 7B — optional) ──────────────────
let autoTuner;
try {
  autoTuner = require('./oracle/auto-tuner');
} catch (e) { console.log('[Server] Auto-tuner not available yet'); }

// ─── Report Generator (Oracle Engine Phase 7C — optional) ────────────
let reportGenerator;
try {
  reportGenerator = require('./oracle/report-generator');
} catch (e) { console.log('[Server] Report generator not available yet'); }

// ─── Polymarket (standalone lifecycle — Phase 8A) ────────────────
let polymarketCollector;
try { polymarketCollector = require('./collectors/polymarket-collector'); } catch (e) { console.log('[Server] Polymarket collector not available yet'); }

// ─── Google Trends (standalone lifecycle — Phase 8B) ──────────────
let trendsCollector;
try { trendsCollector = require('./collectors/trends-collector'); } catch (e) { console.log('[Server] Trends collector not available yet'); }

// ─── FRED (standalone lifecycle — Phase 8C) ───────────────────────
let fredCollector;
try { fredCollector = require('./collectors/fred-collector'); } catch (e) { console.log('[Server] FRED collector not available yet'); }

// ─── USGS Earthquake (standalone lifecycle) ───────────────────────
let usgsCollector;
try { usgsCollector = require('./collectors/usgs-collector'); } catch (e) { console.log('[Server] USGS collector not available yet'); }

// ─── NASA FIRMS (standalone lifecycle) ────────────────────────────
let firmsCollector;
try { firmsCollector = require('./collectors/firms-collector'); } catch (e) { console.log('[Server] FIRMS collector not available yet'); }

// ─── Finnhub (standalone lifecycle) ───────────────────────────────
let finnhubCollector;
try { finnhubCollector = require('./collectors/finnhub-collector'); } catch (e) { console.log('[Server] Finnhub collector not available yet'); }

// ─── Polymarket Trader (Phase 9B — optional) ─────────────────────
let polymarketTrader;
try { polymarketTrader = require('./oracle/polymarket-trader'); } catch (e) { console.log('[Server] Polymarket trader not available yet'); }

// ─── Polymarket Learner (Phase 9C — optional) ─────────────────────
let polymarketLearner;
try { polymarketLearner = require('./oracle/polymarket-learner'); } catch (e) { console.log('[Server] Polymarket learner not available yet'); }

// ─── Express + Socket.IO ──────────────────────────────────────────
const app = express();
const server = http.createServer(app);
const io = new Server(server, {
  cors: { origin: '*', methods: ['GET', 'POST'] }
});

app.use(cors());
app.use(express.json());
app.use(express.static(path.join(__dirname, 'public')));

const PORT = process.env.PORT || 3001;

// ─── Initialization Status Tracker ────────────────────────────────
const initStatus = {
  started: Date.now(),
  database: 'ready',         // Already loaded synchronously via require('./db')
  collectors: 'waiting',     // waiting → loading → ready
  ai: 'waiting',             // waiting → loading → ready | unavailable
  oracle: 'waiting',         // waiting → loading → ready
  ready: false
};

function updateInitStatus(key, status) {
  if (key) initStatus[key] = status;
  // Check if core systems are ready (don't wait for AI)
  if (initStatus.database === 'ready' && initStatus.collectors === 'ready') {
    initStatus.ready = true;
  }
  io.emit('init_progress', initStatus);
}

// Lightweight endpoint — NO DB queries, available immediately
app.get('/api/init-status', (req, res) => {
  res.json({
    ...initStatus,
    uptime: Date.now() - initStatus.started
  });
});

// ─── API Routes ────────────────────────────────────────────────────

// Events (with filters + pagination)
app.get('/api/events', (req, res) => {
  try {
    const { entity, category, region, source, search, sort, sentiment, page = 1, limit = 50 } = req.query;
    // Exclude NASA_FIRMS from default feed unless explicitly requested
    const excludeSources = (!source || source === '') ? ['firms', 'NASA_FIRMS'] : [];
    const result = db.getEvents({
      entity,
      category,
      region,
      source,
      search,
      sort,
      sentiment,
      excludeSources,
      page: parseInt(page),
      limit: Math.min(parseInt(limit) || 50, 100)
    });

    // Get duplicate sources for these events ("+N sources" badges)
    const eventIds = result.data.map(evt => evt.id);
    const dupSources = db.getDuplicateSourcesForEvents(eventIds);

    // Get correlations for these events (market badges)
    const eventCorrelations = db.getCorrelationsForEvents(eventIds);

    // Get AI analysis for these events (Phase 2B)
    const aiAnalysisMap = db.getAiAnalysisForEvents(eventIds);

    // Parse JSON fields for client + attach other_sources + correlations + ai_analysis
    result.data = result.data.map(evt => ({
      ...evt,
      entity_ids: safeParseJSON(evt.entity_ids, []),
      categories: safeParseJSON(evt.categories, []),
      regions: safeParseJSON(evt.regions, []),
      other_sources: dupSources[evt.id] || [],
      correlations: eventCorrelations[evt.id] || [],
      ai_analysis: aiAnalysisMap[evt.id] || null
    }));

    // Filter by AI impact score if requested
    const minScore = parseInt(req.query.min_score);
    if (minScore > 0) {
      result.data = result.data.filter(evt =>
        evt.ai_analysis && evt.ai_analysis.impact_score >= minScore
      );
    }

    res.json(result);
  } catch (error) {
    console.error('[API] /api/events error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Entity detail
app.get('/api/entity/:id', (req, res) => {
  try {
    const entity = entities.getEntityById(req.params.id);
    if (!entity) return res.status(404).json({ error: 'Entity not found' });

    const stats = db.getEntityStats(req.params.id);
    const wikiActivity = db.getWikiActivity(req.params.id, 14);

    // Market data (tickers + latest prices + correlations)
    let marketData = null;
    const entityTickers = tickers.getTickersForEntity(req.params.id);
    if (entityTickers) {
      const allTickerSymbols = [entityTickers.primary, ...entityTickers.secondary, ...entityTickers.indices].filter(Boolean);
      const prices = allTickerSymbols.length > 0 ? db.getLatestPrices(allTickerSymbols) : [];
      const correlations = db.getCorrelationsByEntity(req.params.id);
      marketData = { tickers: entityTickers, prices, correlations };
    }

    res.json({ entity, stats, wikiActivity, marketData });
  } catch (error) {
    console.error('[API] /api/entity/:id error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Entity events
app.get('/api/entity/:id/events', (req, res) => {
  try {
    const { page = 1, limit = 20 } = req.query;
    const result = db.getEventsByEntity(req.params.id, parseInt(page), Math.min(parseInt(limit) || 20, 100));

    const eventIds = result.data.map(evt => evt.id);
    const dupSources = db.getDuplicateSourcesForEvents(eventIds);
    const aiAnalysisMap = db.getAiAnalysisForEvents(eventIds);

    result.data = result.data.map(evt => ({
      ...evt,
      entity_ids: safeParseJSON(evt.entity_ids, []),
      categories: safeParseJSON(evt.categories, []),
      regions: safeParseJSON(evt.regions, []),
      other_sources: dupSources[evt.id] || [],
      ai_analysis: aiAnalysisMap[evt.id] || null
    }));

    res.json(result);
  } catch (error) {
    console.error('[API] /api/entity/:id/events error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// All entities
app.get('/api/entities', (req, res) => {
  try {
    const { category, region, influence, search } = req.query;
    let result = entities.getAllEntities();

    if (category) result = result.filter(e => e.category === category);
    if (region) result = result.filter(e => e.region === region);
    if (influence) result = result.filter(e => e.influence === influence);
    if (search) result = entities.searchEntities(search);

    res.json({ data: result, total: result.length });
  } catch (error) {
    console.error('[API] /api/entities error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Top entities (most mentioned in last 24h)
app.get('/api/entities/top', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const limit = Math.min(parseInt(req.query.limit) || 20, 50);
    const topEntities = db.getTopEntities(hours, limit);

    // Enrich with entity metadata
    const enriched = topEntities.map(item => {
      const entity = entities.getEntityById(item.entity_id);
      return {
        ...item,
        entity: entity || { id: item.entity_id, name: item.entity_id }
      };
    });

    res.json({ data: enriched, hours });
  } catch (error) {
    console.error('[API] /api/entities/top error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Global stats
app.get('/api/stats', (req, res) => {
  try {
    const stats = db.getGlobalStats();
    stats.trackedEntities = entities.getEntityCount();
    stats.collectorStates = manager.getCollectorStates();
    res.json(stats);
  } catch (error) {
    console.error('[API] /api/stats error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Source stats
app.get('/api/stats/sources', (req, res) => {
  try {
    res.json(db.getSourceStats());
  } catch (error) {
    console.error('[API] /api/stats/sources error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Events by region
app.get('/api/regions', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    res.json({ data: db.getEventsByRegion(hours), hours });
  } catch (error) {
    console.error('[API] /api/regions error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Events by category
app.get('/api/categories', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    res.json({ data: db.getEventsByCategory(hours), hours });
  } catch (error) {
    console.error('[API] /api/categories error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Market API Routes ──────────────────────────────────────────────

app.get('/api/markets/overview', (req, res) => {
  try {
    const prices = db.getMarketOverview();
    const fearGreed = db.getLatestFearGreed();
    const crypto = db.getLatestCryptoPrices();
    res.json({ prices, fearGreed, crypto });
  } catch (error) {
    console.error('[API] /api/markets/overview error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/markets/ticker/:ticker', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const history = db.getTickerHistory(req.params.ticker, hours);
    const tickerName = tickers.getTickerName(req.params.ticker);
    res.json({ ticker: req.params.ticker, name: tickerName, history, hours });
  } catch (error) {
    console.error('[API] /api/markets/ticker error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/markets/anomalies', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const anomalies = db.getRecentAnomalies(hours);
    res.json({ data: anomalies, hours });
  } catch (error) {
    console.error('[API] /api/markets/anomalies error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/markets/correlations', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const grouped = req.query.grouped === 'true';

    if (grouped) {
      const correlations = db.getCorrelationsGrouped(hours);
      res.json({ data: correlations, hours, grouped: true });
    } else {
      const correlations = db.getCorrelations(hours);
      res.json({ data: correlations, hours });
    }
  } catch (error) {
    console.error('[API] /api/markets/correlations error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/markets/correlations/:entityId', (req, res) => {
  try {
    const correlations = db.getCorrelationsByEntity(req.params.entityId);
    res.json({ data: correlations, entityId: req.params.entityId });
  } catch (error) {
    console.error('[API] /api/markets/correlations/:entityId error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/markets/fear-greed', (req, res) => {
  try {
    const current = db.getLatestFearGreed();
    const history = db.getFearGreedHistory(30);
    res.json({ current, history });
  } catch (error) {
    console.error('[API] /api/markets/fear-greed error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── AI Analysis API Routes (Phase 2B) ─────────────────────────────

app.get('/api/ai/status', (req, res) => {
  try {
    const queueStatus = aiQueue ? aiQueue.getQueueStatus() : { enabled: false };
    const totalAnalyzed = db.getAiAnalysisCount();
    res.json({ ...queueStatus, totalAnalyzed });
  } catch (error) {
    console.error('[API] /api/ai/status error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/ai/stats', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const stats = db.getAiStats(hours);
    res.json({ data: stats, hours });
  } catch (error) {
    console.error('[API] /api/ai/stats error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/events/top-impact', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const limit = Math.min(parseInt(req.query.limit) || 20, 50);
    const events = db.getTopImpactEvents(hours, limit);

    const data = events.map(evt => ({
      ...evt,
      entity_ids: safeParseJSON(evt.entity_ids, []),
      categories: safeParseJSON(evt.categories, []),
      regions: safeParseJSON(evt.regions, [])
    }));

    res.json({ data, hours });
  } catch (error) {
    console.error('[API] /api/events/top-impact error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.post('/api/ai/test', (req, res) => {
  try {
    if (!aiQueue) return res.status(503).json({ error: 'AI module not loaded' });
    if (!modelManagerInstance || !modelManagerInstance.isAvailable()) return res.status(503).json({ error: 'Ollama not available' });
    const eventId = parseInt(req.body.event_id);
    if (!eventId) return res.status(400).json({ error: 'event_id required' });
    const result = aiQueue.enqueueManual(eventId);
    res.json(result);
  } catch (error) {
    console.error('[API] /api/ai/test error:', error.message);
    res.status(500).json({ error: error.message });
  }
});

app.get('/api/ai/dead-letters', (req, res) => {
  try {
    if (!aiQueue) return res.status(503).json({ error: 'AI module not loaded' });
    const items = aiQueue.getDeadLetters();
    res.json({ data: items, count: items.length });
  } catch (error) {
    console.error('[API] /api/ai/dead-letters error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/nav-badges', (req, res) => {
  try {
    var dashboard = 0;
    var markets = 0;
    var mapCount = 0;
    var categories = 0;

    // Dashboard: count high-impact events (score >= 60) in last 24h
    try {
      var highImpact = db.getTopImpactEvents(24, 100);
      dashboard = highImpact.filter(function(e) { return e.impact_score >= 60; }).length;
    } catch (e) { /* */ }

    // Markets: count anomalies in last 24h
    try {
      markets = db.getRecentAnomalies(24).length;
    } catch (e) { /* */ }

    // Map: count active entities (with events in 24h)
    try {
      mapCount = db.getTopEntities(24, 999).length;
    } catch (e) { /* */ }

    // Categories: today's event count
    try {
      var stats = db.getGlobalStats();
      categories = stats.todayEvents || 0;
    } catch (e) { /* */ }

    // Summaries: count hourly summaries in last 24h
    var summaries = 0;
    try {
      summaries = db.getHourlySummaries(24).length;
    } catch (e) { /* */ }

    // Oracle: count pending predictions
    var oracle = 0;
    try {
      oracle = db.getPendingPredictions().length;
    } catch (e) { /* */ }

    res.json({ dashboard, markets, map: mapCount, categories, summaries, oracle });
  } catch (error) {
    console.error('[API] /api/nav-badges error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/ai/world-summary', async (req, res) => {
  try {
    if (!aiAnalyzer) return res.status(503).json({ error: 'AI module not loaded' });
    const summary = await aiAnalyzer.generateWorldSummary();
    res.json(summary);
  } catch (error) {
    res.status(503).json({ error: error.message });
  }
});

app.get('/api/ai/hourly-summaries', (req, res) => {
  try {
    const limit = Math.min(parseInt(req.query.limit) || 24, 72);
    const summaries = db.getHourlySummaries(limit);
    const data = summaries.map(s => ({
      ...s,
      top_event_ids: safeParseJSON(s.top_event_ids, [])
    }));
    res.json({ data, limit });
  } catch (error) {
    console.error('[API] /api/ai/hourly-summaries error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Impact Summary & Tension ─────────────────────────────────────

app.get('/api/ai/impact-summary', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const stats = db.getAverageImpactScore(hours);
    res.json({ current: stats, hours });
  } catch (error) {
    console.error('[API] /api/ai/impact-summary error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/tension', (req, res) => {
  try {
    const data = db.getTensionData();

    // Compute component scores (each 0-100)
    const firmsScore = Math.min((data.firmsConflict || 0) * 10, 100);

    const militaryCount = data.military?.count || 0;
    const militaryAvg = data.military?.avg_impact || 0;
    const militaryScore = Math.min((militaryCount * 3) + (militaryAvg * 0.5), 100);

    const highImpactScore = Math.min((data.highImpact || 0) * 5, 100);

    const sentMap = {};
    for (const s of (data.sentiments || [])) sentMap[s.sentiment] = s.count;
    const totalSent = data.totalAnalyzed || 1;
    const negativeRatio = ((sentMap['very_negative'] || 0) + (sentMap['negative'] || 0)) / totalSent;
    const sentimentScore = Math.min(Math.round(negativeRatio * 150), 100);

    const vixPrice = data.vix ? data.vix.price : 15;
    const vixScore = Math.min(Math.max(Math.round((vixPrice - 12) * 3), 0), 100);

    const gdeltProxy = militaryCount > 0 ? Math.min(militaryCount * 5, 100) : 0;

    const tension = Math.min(Math.round(
      firmsScore * 0.25 +
      militaryScore * 0.20 +
      highImpactScore * 0.20 +
      sentimentScore * 0.15 +
      vixScore * 0.10 +
      gdeltProxy * 0.10
    ), 100);

    let level, color;
    if (tension >= 80) { level = 'CRITIQUE'; color = '#ef4444'; }
    else if (tension >= 60) { level = '\u00c9LEV\u00c9'; color = '#f59e0b'; }
    else if (tension >= 40) { level = 'MOD\u00c9R\u00c9'; color = '#eab308'; }
    else if (tension >= 20) { level = 'NORMAL'; color = '#06b6d4'; }
    else { level = 'CALME'; color = '#10b981'; }

    res.json({
      score: tension,
      level,
      color,
      components: {
        firmsScore: Math.round(firmsScore),
        militaryScore: Math.round(militaryScore),
        highImpactScore: Math.round(highImpactScore),
        sentimentScore: Math.round(negativeRatio * 100),
        vixScore: Math.round(vixScore)
      },
      updatedAt: new Date().toISOString()
    });
  } catch (error) {
    console.error('[API] /api/tension error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/firms/events', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const since = new Date(Date.now() - hours * 60 * 60 * 1000).toISOString();
    const events = db.db.prepare(`
      SELECT id, title, summary, published_at, collected_at, country
      FROM events
      WHERE source IN ('firms', 'NASA_FIRMS') AND duplicate_of IS NULL AND published_at >= ?
      ORDER BY published_at DESC
    `).all(since);

    const parsed = events.map(e => {
      // Parse lat/lon from summary: "near LAT, LON"
      const match = (e.summary || '').match(/near\s+([-\d.]+),\s*([-\d.]+)/);
      const lat = match ? parseFloat(match[1]) : null;
      const lon = match ? parseFloat(match[2]) : null;
      const isConflict = (e.title || '').includes('Strike') || (e.title || '').includes('Explosion');
      return { ...e, lat, lon, isConflict };
    }).filter(e => e.lat !== null && e.lon !== null);

    res.json({ data: parsed, hours });
  } catch (error) {
    console.error('[API] /api/firms/events error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Oracle Engine API Routes ──────────────────────────────────────

app.get('/api/predictions', (req, res) => {
  try {
    const { status, type, depth, page = 1, limit = 50 } = req.query;
    const result = db.getPredictions({
      status,
      type,
      depth: depth !== undefined ? parseInt(depth) : undefined,
      page: parseInt(page),
      limit: Math.min(parseInt(limit) || 50, 100)
    });

    result.data = result.data.map(pred => ({
      ...pred,
      entity_ids: safeParseJSON(pred.entity_ids, []),
      categories: safeParseJSON(pred.categories, []),
      regions: safeParseJSON(pred.regions, []),
      tickers: safeParseJSON(pred.tickers, []),
      resolution_event_ids: safeParseJSON(pred.resolution_event_ids, [])
    }));

    res.json(result);
  } catch (error) {
    console.error('[API] /api/predictions error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/predictions/tree', (req, res) => {
  try {
    const { status, type } = req.query;
    const roots = db.getPredictionsTree({ status: status || undefined, type: type || undefined });
    const parseNode = (node) => ({
      ...node,
      entity_ids: safeParseJSON(node.entity_ids, []),
      categories: safeParseJSON(node.categories, []),
      regions: safeParseJSON(node.regions, []),
      tickers: safeParseJSON(node.tickers, []),
      resolution_event_ids: safeParseJSON(node.resolution_event_ids, []),
      children: (node.children || []).map(parseNode)
    });
    res.json({ data: roots.map(parseNode) });
  } catch (error) {
    console.error('[API] /api/predictions/tree error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/oracle/stats', (req, res) => {
  try {
    const period = req.query.period || 'daily';
    const limit = Math.min(parseInt(req.query.limit) || 30, 90);
    const stats = db.getOracleStats(period, limit);
    const latest = db.getOracleStatsLatest();
    res.json({ data: stats, latest, period });
  } catch (error) {
    console.error('[API] /api/oracle/stats error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/oracle/active', (req, res) => {
  try {
    const predictions = db.getPendingPredictions();
    const data = predictions.map(pred => ({
      ...pred,
      entity_ids: safeParseJSON(pred.entity_ids, []),
      categories: safeParseJSON(pred.categories, []),
      regions: safeParseJSON(pred.regions, []),
      tickers: safeParseJSON(pred.tickers, [])
    }));
    res.json({ data, count: data.length });
  } catch (error) {
    console.error('[API] /api/oracle/active error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/predictions/:id', (req, res) => {
  try {
    const prediction = db.getPrediction(parseInt(req.params.id));
    if (!prediction) return res.status(404).json({ error: 'Prediction not found' });

    const signals = db.getSignalsForPrediction(prediction.id);
    const updates = db.getUpdatesForPrediction(prediction.id);
    const children = db.getChildPredictions(prediction.id);

    res.json({
      prediction: {
        ...prediction,
        entity_ids: safeParseJSON(prediction.entity_ids, []),
        categories: safeParseJSON(prediction.categories, []),
        regions: safeParseJSON(prediction.regions, []),
        tickers: safeParseJSON(prediction.tickers, []),
        resolution_event_ids: safeParseJSON(prediction.resolution_event_ids, [])
      },
      signals,
      updates,
      children
    });
  } catch (error) {
    console.error('[API] /api/predictions/:id error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.post('/api/predictions', (req, res) => {
  try {
    const now = new Date().toISOString();
    const data = {
      title: req.body.title,
      description: req.body.description || null,
      prediction_type: req.body.prediction_type,
      probability: parseFloat(req.body.probability),
      confidence: req.body.confidence || 'medium',
      depth: parseInt(req.body.depth) || 0,
      window_start: req.body.window_start,
      window_end: req.body.window_end,
      entity_ids: req.body.entity_ids ? JSON.stringify(req.body.entity_ids) : null,
      categories: req.body.categories ? JSON.stringify(req.body.categories) : null,
      regions: req.body.regions ? JSON.stringify(req.body.regions) : null,
      tickers: req.body.tickers ? JSON.stringify(req.body.tickers) : null,
      source: req.body.source || 'human',
      model_used: req.body.model_used || null,
      status: 'pending',
      parent_id: req.body.parent_id ? parseInt(req.body.parent_id) : null,
      created_at: now,
      updated_at: now
    };

    if (!data.title || !data.prediction_type || isNaN(data.probability) || !data.window_start || !data.window_end) {
      return res.status(400).json({ error: 'title, prediction_type, probability, window_start, window_end required' });
    }

    const result = db.insertPrediction(data);
    res.json(result);
  } catch (error) {
    console.error('[API] POST /api/predictions error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.patch('/api/predictions/:id', (req, res) => {
  try {
    const existing = db.getPrediction(parseInt(req.params.id));
    if (!existing) return res.status(404).json({ error: 'Prediction not found' });

    const now = new Date().toISOString();
    const oldProb = existing.probability;
    const newProb = req.body.probability !== undefined ? parseFloat(req.body.probability) : existing.probability;

    const data = {
      id: existing.id,
      probability: newProb,
      confidence: req.body.confidence || existing.confidence,
      status: req.body.status || existing.status,
      resolved_at: req.body.status && req.body.status !== 'pending' ? now : existing.resolved_at,
      resolution_notes: req.body.resolution_notes || existing.resolution_notes,
      resolution_event_ids: req.body.resolution_event_ids ? JSON.stringify(req.body.resolution_event_ids) : existing.resolution_event_ids,
      brier_score: req.body.brier_score !== undefined ? parseFloat(req.body.brier_score) : existing.brier_score,
      updated_at: now
    };

    db.updatePrediction(data);

    // Record probability change if different
    if (oldProb !== newProb) {
      db.insertPredictionUpdate({
        prediction_id: existing.id,
        probability_before: oldProb,
        probability_after: newProb,
        reason: req.body.reason || null,
        trigger_event_id: req.body.trigger_event_id ? parseInt(req.body.trigger_event_id) : null,
        updated_at: now
      });
    }

    res.json(data);
  } catch (error) {
    console.error('[API] PATCH /api/predictions/:id error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Oracle Signal & Entity Search Routes ─────────────────────────────

app.get('/api/oracle/signals', (req, res) => {
  try {
    const limit = Math.min(parseInt(req.query.limit) || 20, 100);
    const consumed = req.query.consumed !== undefined
      ? (req.query.consumed === 'true' ? true : req.query.consumed === 'false' ? false : null)
      : null;
    const data = db.getOracleSignalsRecent(limit, consumed);
    res.json({ data, limit });
  } catch (error) {
    console.error('[API] /api/oracle/signals error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/oracle/signals/stats', (req, res) => {
  try {
    const stats = db.getOracleSignalsStats();
    res.json({ data: stats });
  } catch (error) {
    console.error('[API] /api/oracle/signals/stats error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/entities/search', (req, res) => {
  try {
    const q = (req.query.q || '').trim().toLowerCase();
    if (q.length < 1) return res.json({ data: [] });
    const all = entities.getAllEntities();
    const matches = all.filter(e =>
      e.name.toLowerCase().includes(q) || e.id.toLowerCase().includes(q)
    ).slice(0, 20);
    res.json({ data: matches });
  } catch (error) {
    console.error('[API] /api/entities/search error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Oracle Resolver & Scoring Routes ─────────────────────────────

app.get('/api/oracle/calibration', (req, res) => {
  try {
    const data = db.getCalibrationStats();
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/oracle/calibration error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/oracle/performance', (req, res) => {
  try {
    const byType = db.getPerformanceByType();
    const byDepth = db.getPerformanceByDepth();
    const latest = db.getOracleStatsLatest();
    res.json({ byType, byDepth, latest });
  } catch (error) {
    console.error('[API] /api/oracle/performance error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/oracle/stats/history', (req, res) => {
  try {
    const days = Math.min(parseInt(req.query.days) || 30, 90);
    const data = db.getOracleStats('daily', days);
    res.json({ data, days });
  } catch (error) {
    console.error('[API] /api/oracle/stats/history error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/oracle/entity/:entityId/stats', (req, res) => {
  try {
    const entityId = req.params.entityId;
    // Predictions store entity names, not slugs — resolve the name
    const entity = entities.getEntityById(entityId);
    const searchKey = entity ? entity.name : entityId;
    const stats = db.getOracleEntityStats(searchKey);
    const predictions = db.getPredictionsByEntity(searchKey).map(pred => ({
      ...pred,
      entity_ids: safeParseJSON(pred.entity_ids, []),
      categories: safeParseJSON(pred.categories, []),
      regions: safeParseJSON(pred.regions, []),
      tickers: safeParseJSON(pred.tickers, [])
    }));
    res.json({ stats, predictions });
  } catch (error) {
    console.error('[API] /api/oracle/entity/:entityId/stats error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.post('/api/oracle/resolve', async (req, res) => {
  try {
    if (!predictionResolver) return res.status(503).json({ error: 'Resolver not loaded' });
    const result = await predictionResolver.runResolution();
    res.json(result);
  } catch (error) {
    console.error('[API] /api/oracle/resolve error:', error.message);
    res.status(500).json({ error: error.message });
  }
});

app.post('/api/oracle/score', async (req, res) => {
  try {
    if (!scoringModule) return res.status(503).json({ error: 'Scoring not loaded' });
    const result = await scoringModule.runScoring();
    res.json(result);
  } catch (error) {
    console.error('[API] /api/oracle/score error:', error.message);
    res.status(500).json({ error: error.message });
  }
});

// ─── Phase 7 API Routes (Auto-Improvement) ──────────────────────

app.get('/api/oracle/errors', (req, res) => {
  try {
    const days = parseInt(req.query.days) || 30;
    const limit = parseInt(req.query.limit) || 50;
    const errorType = req.query.type || null;
    const errors = db.getPredictionErrors(limit, errorType);
    const stats = db.getErrorStats(days);
    res.json({ errors, stats });
  } catch (error) {
    console.error('[API] /api/oracle/errors error:', error.message);
    res.status(500).json({ error: error.message });
  }
});

app.post('/api/oracle/analyze-errors', async (req, res) => {
  try {
    if (!errorAnalyzer) return res.status(503).json({ error: 'Error analyzer not loaded' });
    const result = await errorAnalyzer.runAnalysis();
    res.json(result);
  } catch (error) {
    console.error('[API] /api/oracle/analyze-errors error:', error.message);
    res.status(500).json({ error: error.message });
  }
});

app.get('/api/oracle/tuner/history', (req, res) => {
  try {
    const limit = parseInt(req.query.limit) || 50;
    const history = db.getTunerHistory(limit);
    res.json({ history });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.post('/api/oracle/tune', async (req, res) => {
  try {
    if (!autoTuner) return res.status(503).json({ error: 'Auto-tuner not loaded' });
    const result = await autoTuner.runTuning();
    res.json(result);
  } catch (error) {
    console.error('[API] /api/oracle/tune error:', error.message);
    res.status(500).json({ error: error.message });
  }
});

app.get('/api/oracle/config', (req, res) => {
  try {
    const config = db.getAllOracleConfig();
    res.json({ config });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.post('/api/oracle/config/reset', (req, res) => {
  try {
    const { key } = req.body;
    if (!key) return res.status(400).json({ error: 'key required' });
    const row = db.db.prepare('SELECT default_value FROM oracle_config WHERE config_key = ?').get(key);
    if (!row) return res.status(404).json({ error: 'key not found' });
    db.setOracleConfig(key, row.default_value, 'manual_reset');
    res.json({ success: true, key, reset_to: row.default_value });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.get('/api/oracle/reports', (req, res) => {
  try {
    const limit = parseInt(req.query.limit) || 10;
    const reports = db.getOracleReports(limit);
    res.json({ reports });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.get('/api/oracle/reports/:weekKey', (req, res) => {
  try {
    const report = db.getOracleReport(req.params.weekKey);
    if (!report) return res.status(404).json({ error: 'Report not found' });
    res.json({ report });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.post('/api/oracle/reports/generate', async (req, res) => {
  try {
    if (!reportGenerator) return res.status(503).json({ error: 'Report generator not loaded' });
    const force = req.query.force === 'true';
    const result = await reportGenerator.generateWeeklyReport(force);
    if (result && io) io.emit('oracle:report', { weekKey: result.weekKey });
    res.json(result);
  } catch (error) {
    console.error('[API] /api/oracle/reports/generate error:', error.message);
    res.status(500).json({ error: error.message });
  }
});

// ─── Calendar API Routes (Phase 6A) ──────────────────────────────

app.get('/api/calendar', (req, res) => {
  try {
    const startDate = req.query.start || new Date().toISOString().substring(0, 10);
    const endDate = req.query.end || new Date(Date.now() + 14 * 86400000).toISOString().substring(0, 10);
    const data = db.getScheduledEventsRange(startDate, endDate);
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/calendar error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/calendar/upcoming', (req, res) => {
  try {
    const hours = Math.min(parseInt(req.query.hours) || 48, 168);
    const data = db.getUpcomingScheduledEvents(hours);
    res.json({ data, hours });
  } catch (error) {
    console.error('[API] /api/calendar/upcoming error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Polymarket API Routes (Phase 8A) ────────────────────────────

app.get('/api/polymarket', (req, res) => {
  try {
    const activeOnly = req.query.active !== 'false';
    const data = db.getPolymarketMarkets(activeOnly);
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/polymarket error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/polymarket/entity/:entityId', (req, res) => {
  try {
    const data = db.getPolymarketMarketsByEntity(req.params.entityId);
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/polymarket/entity error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/polymarket/:marketId/history', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 48;
    const history = db.getPolymarketPriceHistory(req.params.marketId, hours);
    res.json({ data: history, hours });
  } catch (error) {
    console.error('[API] /api/polymarket/:marketId/history error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Polymarket Simulator API Routes (Phase 9) ──────────────────

app.get('/api/simulator/bets/open', (req, res) => {
  try {
    const data = db.getOpenBets();
    res.json({ data, count: data.length });
  } catch (error) {
    console.error('[API] /api/simulator/bets/open error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/bets', (req, res) => {
  try {
    const limit = Math.min(parseInt(req.query.limit) || 100, 200);
    const offset = parseInt(req.query.offset) || 0;
    const status = req.query.status || null;
    const data = db.getBetHistory(limit, offset, status);
    res.json({ data, count: data.length, limit, offset });
  } catch (error) {
    console.error('[API] /api/simulator/bets error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/bets/:id', (req, res) => {
  try {
    const bet = db.getBetById(parseInt(req.params.id));
    if (!bet) return res.status(404).json({ error: 'Bet not found' });
    const lessons = db.getLessonsForBet(bet.id);
    res.json({ bet, lessons });
  } catch (error) {
    console.error('[API] /api/simulator/bets/:id error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/portfolio/history', (req, res) => {
  try {
    const days = Math.min(parseInt(req.query.days) || 90, 365);
    const data = db.getPortfolioHistory(days);
    res.json({ data, days });
  } catch (error) {
    console.error('[API] /api/simulator/portfolio/history error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/portfolio/current', (req, res) => {
  try {
    const portfolio = db.getLatestPortfolio();
    const stats = db.getBetStats();
    res.json({ portfolio, stats });
  } catch (error) {
    console.error('[API] /api/simulator/portfolio/current error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/performance', (req, res) => {
  try {
    const stats = db.getBetStats();
    const byCategory = db.getBetStatsByCategory();
    res.json({ stats, byCategory });
  } catch (error) {
    console.error('[API] /api/simulator/performance error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/performance/category', (req, res) => {
  try {
    const data = db.getBetStatsByCategory();
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/simulator/performance/category error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/lessons', (req, res) => {
  try {
    const limit = Math.min(parseInt(req.query.limit) || 50, 200);
    const type = req.query.type || null;
    const data = db.getLessons(limit, type);
    res.json({ data, count: data.length });
  } catch (error) {
    console.error('[API] /api/simulator/lessons error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/simulator/config', (req, res) => {
  try {
    const data = db.getAllSimConfig();
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/simulator/config error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.post('/api/simulator/config', (req, res) => {
  try {
    const { key, value } = req.body;
    if (!key) return res.status(400).json({ error: 'key required' });
    db.setSimConfig(key, value);
    res.json({ success: true, key, value });
  } catch (error) {
    console.error('[API] POST /api/simulator/config error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.post('/api/simulator/config/reset', (req, res) => {
  try {
    const { key } = req.body;
    if (key) {
      db.resetSimConfig(key);
    } else {
      db.resetAllSimConfig();
    }
    res.json({ success: true, reset: key || 'all' });
  } catch (error) {
    console.error('[API] POST /api/simulator/config/reset error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.post('/api/simulator/analyze', async (req, res) => {
  try {
    if (!polymarketTrader) return res.status(503).json({ error: 'Polymarket trader not available' });
    const result = await polymarketTrader.runTrader();
    if (polymarketLearner && result.resolvedBets && result.resolvedBets.length > 0) {
      try { await polymarketLearner.runLearner(result.resolvedBets); } catch (e) {
        console.error('[API] Learner error:', e.message);
      }
    }
    res.json({ success: true, betsPlaced: result.betsPlaced, resolved: result.resolvedBets.length, openBets: result.openBets });
  } catch (error) {
    console.error('[API] POST /api/simulator/analyze error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── Google Trends API Routes (Phase 8B) ─────────────────────────

app.get('/api/trends/spikes', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const data = db.getRecentTrendSpikes(hours);
    res.json({ data, hours });
  } catch (error) {
    console.error('[API] /api/trends/spikes error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/trends/:keyword/history', (req, res) => {
  try {
    const limit = parseInt(req.query.limit) || 30;
    const data = db.getRecentTrends(req.params.keyword, limit);
    res.json({ data, keyword: req.params.keyword });
  } catch (error) {
    console.error('[API] /api/trends/:keyword/history error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/trends', (req, res) => {
  try {
    const data = db.getLatestTrends();
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/trends error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── FRED API Routes (Phase 8C) ──────────────────────────────────

app.get('/api/fred/alerts', (req, res) => {
  try {
    const hours = parseInt(req.query.hours) || 24;
    const data = db.getRecentFredAlerts(hours);
    res.json({ data, hours });
  } catch (error) {
    console.error('[API] /api/fred/alerts error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/fred/:seriesId/history', (req, res) => {
  try {
    const limit = parseInt(req.query.limit) || 50;
    const data = db.getFredHistory(req.params.seriesId, limit);
    const latest = db.getFredLatest(req.params.seriesId);
    res.json({ data, latest, seriesId: req.params.seriesId });
  } catch (error) {
    console.error('[API] /api/fred/:seriesId/history error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

app.get('/api/fred', (req, res) => {
  try {
    const data = db.getFredOverview();
    res.json({ data });
  } catch (error) {
    console.error('[API] /api/fred error:', error.message);
    res.status(500).json({ error: 'Internal server error' });
  }
});

// ─── HTML Page Routes ──────────────────────────────────────────────
app.get('/entity', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'entity.html'));
});

app.get('/map', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'map.html'));
});

app.get('/markets', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'markets.html'));
});

app.get('/categories', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'categories.html'));
});

app.get('/summaries', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'summaries.html'));
});

app.get('/oracle', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'oracle.html'));
});

// ─── Socket.IO ─────────────────────────────────────────────────────
let cachedInitialState = null;
let cachedInitialStateTime = 0;

io.on('connection', (socket) => {
  console.log(`[Socket.IO] Client connected: ${socket.id}`);

  // Send init progress immediately
  socket.emit('init_progress', initStatus);

  // Send initial state (cached 5s to avoid hammering DB on reconnects)
  try {
    const now = Date.now();
    if (!cachedInitialState || (now - cachedInitialStateTime) > 5000) {
      const stats = db.getGlobalStats();
      stats.trackedEntities = entities.getEntityCount();
      cachedInitialState = {
        stats,
        recentEvents: [],  // Frontend loads events via API — no need to send here
        sourceStats: db.getSourceStats()
      };
      cachedInitialStateTime = now;
    }
    socket.emit('initial_state', cachedInitialState);

    // Send initial market data (may not exist yet on first startup)
    try {
      const marketPrices = db.getMarketOverview();
      const fearGreed = db.getLatestFearGreed();
      const crypto = db.getLatestCryptoPrices();
      if (marketPrices.length > 0 || crypto.length > 0) {
        socket.emit('initial_market_state', { prices: marketPrices, fearGreed, crypto });
      }
    } catch (e) { /* market data may not be available yet */ }

    // Send AI status to new connections (Phase 2B)
    try {
      if (aiQueue) {
        const queueStatus = aiQueue.getQueueStatus();
        socket.emit('ai_status', { ...queueStatus, totalAnalyzed: db.getAiAnalysisCount() });
      }
    } catch (e) { /* AI not available */ }
  } catch (error) {
    console.error('[Socket.IO] Error sending initial state:', error.message);
  }

  socket.on('disconnect', () => {
    console.log(`[Socket.IO] Client disconnected: ${socket.id}`);
  });
});

// ─── Helpers ───────────────────────────────────────────────────────
function safeParseJSON(str, fallback) {
  try {
    return JSON.parse(str);
  } catch (e) {
    return fallback;
  }
}

// ─── Start Server ──────────────────────────────────────────────────
server.listen(PORT, () => {
  console.log(`\n╔══════════════════════════════════════════╗`);
  console.log(`║         PULSE WORLD v1.0                 ║`);
  console.log(`║   Real-time Global Monitoring            ║`);
  console.log(`╠══════════════════════════════════════════╣`);
  console.log(`║  Port     : ${PORT}                          ║`);
  console.log(`║  Entities : ${entities.getEntityCount()}                          ║`);
  console.log(`║  Dashboard: http://localhost:${PORT}          ║`);
  console.log(`╚══════════════════════════════════════════╝\n`);

  // DB is already ready (loaded synchronously via require('./db'))
  updateInitStatus('database', 'ready');
  console.log('[Server] Database ready');

  // Defer heavy cleanup — runs in batches to avoid blocking event loop (3GB DB)
  setTimeout(() => {
    try {
      console.log('[Server] Starting incremental duplicate cleanup...');
      db.cleanupExistingDuplicates(200, (cleaned) => {
        if (cleaned > 0) console.log(`[Server] Cleaned up ${cleaned} duplicate events in database`);
        else console.log('[Server] No duplicates found');
      });
    } catch (e) { console.error('[Server] Duplicate cleanup error:', e.message); }
  }, 15000);

  // Initialize manager and register collectors
  updateInitStatus('collectors', 'loading');
  manager.setIO(io);
  manager.registerCollector(rssCollector);
  if (gdeltCollector) manager.registerCollector(gdeltCollector);
  // Reddit désactivé — 100% d'erreurs, pas de clé API configurée
  // if (redditCollector) manager.registerCollector(redditCollector);
  if (wikiCollector) manager.registerCollector(wikiCollector);
  if (hnCollector) manager.registerCollector(hnCollector);

  // Finance collector (standalone lifecycle — not registered with manager)
  if (financeCollector) {
    financeCollector.init(io);
    if (marketAnalyzer) {
      marketAnalyzer.init(io);
      financeCollector.setAnalyzer(marketAnalyzer);
      manager.setOnNewEvents((events) => marketAnalyzer.analyzeNewEvents(events));
    }
    financeCollector.start();
    console.log('[Server] Finance collector + Market analyzer initialized');
  }

  // AI Analysis (Phase 2B — via ModelManager, multi-tier queue)
  // NON-BLOCKING: AI init runs in background, doesn't delay collectors/oracle
  let modelManager = null;
  let modelRouter = null;
  if (ModelManager && aiAnalyzer && aiQueue) {
    updateInitStatus('ai', 'loading');
    modelManager = new ModelManager();
    modelManagerInstance = modelManager; // hoist for graceful shutdown

    // Start AI warm-up in background — don't await
    modelManager.init(io).then(() => {
      updateInitStatus('ai', 'ready');
      console.log('[Server] AI models warmed up and ready');

      if (ModelRouter) {
        modelRouter = new ModelRouter(modelManager);
      }

      aiAnalyzer.init(io, modelManager);
      aiQueue.init({ io, modelManager, modelRouter, aiAnalyzer });

      // Register AI queue as a new-events subscriber (alongside market analyzer)
      manager.addOnNewEvents((events) => aiQueue.enqueueNewEvents(events));
      console.log('[Server] AI Analysis module initialized (ModelManager + multi-tier queue)');

      // Backfill: re-enqueue unanalyzed events from last 24h (deferred 30s after AI ready)
      setTimeout(() => {
        try {
          const unanalyzed = db.getUnanalyzedEvents(24, 500);
          if (unanalyzed.length > 0) {
            aiQueue.enqueueNewEvents(unanalyzed);
            console.log(`[Server] Backfill: ${unanalyzed.length} unanalyzed events re-enqueued`);
          } else {
            console.log('[Server] Backfill: all recent events already analyzed');
          }
        } catch (e) {
          console.error('[Server] Backfill error:', e.message);
        }
      }, 30000);

      // Hourly summary scheduler
      aiAnalyzer.startHourlySummaryScheduler();

      // Init modules that need modelManager (after AI is ready)
      if (errorAnalyzer) { errorAnalyzer.init(modelManager); console.log('[Server] Error analyzer initialized'); }
      if (reportGenerator) { reportGenerator.init(modelManager); console.log('[Server] Report generator initialized'); }
      if (polymarketLearner) { polymarketLearner.init(modelManager); console.log('[Server] Polymarket learner initialized'); }
      if (predictionGenerator) {
        predictionGenerator.init(io, modelManager);
        manager.setPredictionGenerator(predictionGenerator);
        console.log('[Server] Prediction generator initialized');
      }
      if (predictionResolver) {
        predictionResolver.init(io, modelManager);
        manager.setPredictionResolver(predictionResolver);
        console.log('[Server] Prediction resolver initialized');
      }
    }).catch(e => {
      updateInitStatus('ai', 'unavailable');
      console.error('[Server] AI Analysis init error:', e.message);
    });
  } else {
    updateInitStatus('ai', 'unavailable');
  }

  // Oracle (pattern detector + scoring — don't need AI to start)
  updateInitStatus('oracle', 'loading');
  if (patternDetector) {
    patternDetector.init(io);
    manager.setPatternDetector(patternDetector);
    console.log('[Server] Pattern detector initialized');
  }

  if (scoringModule) {
    scoringModule.init(io);
    manager.setScoring(scoringModule);
    console.log('[Server] Scoring module initialized');
  }
  updateInitStatus('oracle', 'ready');

  // Calendar collector (standalone lifecycle — Phase 6A, deferred 180s to avoid Yahoo rate-limit clash with Finance)
  if (calendarCollector) {
    calendarCollector.init(io);
    setTimeout(() => {
      calendarCollector.start();
      console.log('[Server] Calendar collector started (180s stagger)');
    }, 180000);
    console.log('[Server] Calendar collector initialized (will start in 180s)');
  }

  // Phase 8: New Data Source Collectors (standalone lifecycle)
  // Staggered starts to avoid network saturation (RSS runs at 0s, GDELT at 30s, etc.)
  if (polymarketCollector) {
    polymarketCollector.init(io);
    setTimeout(() => { polymarketCollector.start(); console.log('[Server] Polymarket collector started (60s stagger)'); }, 60000);
    console.log('[Server] Polymarket collector initialized (will start in 60s)');
  }
  if (trendsCollector) {
    trendsCollector.init(io);
    setTimeout(() => { trendsCollector.start(); console.log('[Server] Trends collector started (90s stagger)'); }, 90000);
    console.log('[Server] Trends collector initialized (will start in 90s)');
  }
  if (fredCollector) {
    fredCollector.init(io, aiQueue);
    setTimeout(() => { fredCollector.start(); console.log('[Server] FRED collector started (120s stagger)'); }, 120000);
    console.log('[Server] FRED collector initialized (will start in 120s)');
  }

  // New data source collectors (USGS, FIRMS, Finnhub) — further staggered
  if (usgsCollector) {
    usgsCollector.init(io, aiQueue);
    setTimeout(() => { usgsCollector.start(); console.log('[Server] USGS collector started (45s stagger)'); }, 45000);
    console.log('[Server] USGS earthquake collector initialized (will start in 45s)');
  }
  if (firmsCollector) {
    firmsCollector.init(io, aiQueue);
    setTimeout(() => { firmsCollector.start(); console.log('[Server] FIRMS collector started (150s stagger)'); }, 150000);
    console.log('[Server] NASA FIRMS collector initialized (will start in 150s)');
  }
  if (finnhubCollector) {
    finnhubCollector.init(io, aiQueue);
    setTimeout(() => { finnhubCollector.start(); console.log('[Server] Finnhub collector started (75s stagger)'); }, 75000);
    console.log('[Server] Finnhub collector initialized (will start in 75s)');
  }

  // Phase 9: Polymarket Trader + Learner
  if (polymarketTrader) {
    polymarketTrader.init(io, modelManager);

    const TRADER_INTERVAL = 4 * 60 * 60 * 1000; // 4h
    const PRICE_UPDATE_INTERVAL = 2 * 60 * 60 * 1000; // 2h

    // Run trader 180s after startup (let collectors + AI warm up first)
    setTimeout(async function traderRun() {
      try {
        console.log('[Server] Running Polymarket trader...');
        const result = await polymarketTrader.runTrader();
        console.log(`[Server] Trader: ${result.betsPlaced} placed, ${result.resolvedBets.length} resolved, ${result.openBets} open`);
        // Feed resolved bets to learner
        if (polymarketLearner && result.resolvedBets && result.resolvedBets.length > 0) {
          try { await polymarketLearner.runLearner(result.resolvedBets); } catch (e) {
            console.error('[Server] Learner error:', e.message);
          }
        }
      } catch (e) { console.error('[Server] Trader run error:', e.message); }
      setTimeout(traderRun, TRADER_INTERVAL);
    }, 180000);

    // Update open bet prices every 2h (aligned with collector)
    setTimeout(function priceUpdateLoop() {
      polymarketTrader.updateOpenBetPrices().catch(e => {
        console.error('[Server] Price update error:', e.message);
      });
      setTimeout(priceUpdateLoop, PRICE_UPDATE_INTERVAL);
    }, 20000);

    // Daily portfolio snapshot at midnight
    const DAY_MS = 24 * 60 * 60 * 1000;
    function msUntilMidnight() {
      const now = new Date();
      const next = new Date(now);
      next.setDate(now.getDate() + 1);
      next.setHours(0, 1, 0, 0);
      return next.getTime() - now.getTime();
    }
    setTimeout(async function dailySnapshot() {
      try {
        await polymarketTrader.computePortfolioSnapshot();
        console.log('[Server] Daily portfolio snapshot computed');
      } catch (e) { console.error('[Server] Snapshot error:', e.message); }
      setTimeout(dailySnapshot, DAY_MS);
    }, msUntilMidnight());

    console.log('[Server] Polymarket trader initialized (4h cycle, 45s initial delay)');
  }

  // Phase 7: Auto-Improvement weekly scheduling (Monday 00:05)
  if (autoTuner || reportGenerator) {
    const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
    function msUntilNextMonday() {
      const n = new Date();
      const day = n.getDay();
      const daysUntil = day === 0 ? 1 : (8 - day) % 7 || 7;
      const next = new Date(n);
      next.setDate(n.getDate() + daysUntil);
      next.setHours(0, 5, 0, 0);
      const delta = next.getTime() - n.getTime();
      return delta > 0 ? delta : delta + WEEK_MS;
    }
    const firstDelay = msUntilNextMonday();
    console.log(`[Server] Phase 7 weekly run scheduled in ${Math.round(firstDelay / 3600000)}h`);
    setTimeout(async function weeklyRun() {
      console.log('[Server] Running Phase 7 weekly auto-improvement...');
      if (autoTuner) {
        try {
          const tuneResult = await autoTuner.runTuning();
          console.log(`[Server] AutoTuner: ${tuneResult.adjusted ? tuneResult.adjusted.length : 0} adjustments`);
          if (io && tuneResult.adjusted && tuneResult.adjusted.length > 0) {
            io.emit('oracle:tuned', tuneResult);
          }
        } catch (e) { console.error('[Server] AutoTuner weekly error:', e.message); }
      }
      if (reportGenerator) {
        try {
          const repResult = await reportGenerator.generateWeeklyReport();
          if (repResult && !repResult.skipped && io) {
            io.emit('oracle:report', { weekKey: repResult.weekKey });
          }
        } catch (e) { console.error('[Server] ReportGenerator weekly error:', e.message); }
      }
      // Phase 9: Weekly trader threshold tuning
      if (polymarketLearner) {
        try {
          const tuneResult = await polymarketLearner.runWeeklyTuning();
          console.log(`[Server] Learner tuning: ${tuneResult.adjustments ? tuneResult.adjustments.length : 0} adjustment(s)`);
        } catch (e) { console.error('[Server] Learner weekly tuning error:', e.message); }
      }
      setTimeout(weeklyRun, WEEK_MS);
    }, firstDelay);
  }

  // Start collection (AFTER all modules are registered so intervals are set up)
  manager.startCollectors();
  updateInitStatus('collectors', 'ready');
  console.log(`[Server] All collectors started. Dashboard ready in ${((Date.now() - initStatus.started) / 1000).toFixed(1)}s`);
});

// ─── Graceful Shutdown ─────────────────────────────────────────────
function gracefulShutdown() {
  console.log('\n[Server] Shutting down...');
  manager.stopCollectors();
  if (financeCollector) financeCollector.stop();
  if (calendarCollector) calendarCollector.stop();
  if (polymarketCollector) polymarketCollector.stop();
  if (trendsCollector) trendsCollector.stop();
  if (fredCollector) fredCollector.stop();
  if (usgsCollector) usgsCollector.stop();
  if (firmsCollector) firmsCollector.stop();
  if (finnhubCollector) finnhubCollector.stop();
  if (aiAnalyzer && aiAnalyzer.stopHourlySummaryScheduler) aiAnalyzer.stopHourlySummaryScheduler();
  if (aiQueue && aiQueue.destroy) aiQueue.destroy();
  if (modelManagerInstance && modelManagerInstance.destroy) modelManagerInstance.destroy();
  db.close();
  server.close(() => {
    console.log('[Server] Goodbye!');
    process.exit(0);
  });
}

process.on('SIGINT', gracefulShutdown);
process.on('SIGTERM', gracefulShutdown);