config_manager.py

"""
Unified Config Manager for the Pelagic Fleet.

Bridges: fleet.yaml, agent.yaml, world/rooms.json, lighthouse configs,
environment variables, and runtime overrides into one coherent system.

Priority (highest first):
    1. CLI arguments
    2. Environment variables
    3. Runtime overrides (programmatic)
    4. Fleet config (fleet.yaml)
    5. Agent config (per-agent agent.yaml)
    6. Defaults (built-in)
"""

from __future__ import annotations

import copy
import json
import os
import re
import shutil
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from schema import ConfigSchema, ValidationResult
from templates import ConfigTemplates


# ---------------------------------------------------------------------------
# Utility helpers
# ---------------------------------------------------------------------------

def _deep_get(data: dict, keys: list[str], default: Any = None) -> Any:
    """Traverse a nested dict using a list of keys."""
    current = data
    for key in keys:
        if isinstance(current, dict) and key in current:
            current = current[key]
        else:
            return default
    return current


def _deep_set(data: dict, keys: list[str], value: Any) -> None:
    """Set a value in a nested dict, creating intermediate dicts as needed."""
    current = data
    for key in keys[:-1]:
        if key not in current or not isinstance(current[key], dict):
            current[key] = {}
        current = current[key]
    current[keys[-1]] = value


def _deep_merge(base: dict, override: dict) -> dict:
    """Deep-merge two dicts, returning a new dict."""
    result = copy.deepcopy(base)
    for key, val in override.items():
        if key in result and isinstance(result[key], dict) and isinstance(val, dict):
            result[key] = _deep_merge(result[key], val)
        else:
            result[key] = copy.deepcopy(val)
    return result


def _redact_secrets(data: Any, secret_keys: set[str] | None = None) -> Any:
    """Return a copy of *data* with secret values redacted."""
    if secret_keys is None:
        secret_keys = {
            "secrets", "secret", "password", "token", "api_key",
            "apikey", "private_key", "access_key",
        }
    if isinstance(data, dict):
        return {
            k: "*****" if k.lower() in secret_keys or "secret" in k.lower()
            else _redact_secrets(v, secret_keys)
            for k, v in data.items()
        }
    if isinstance(data, list):
        return [_redact_secrets(item, secret_keys) for item in data]
    return data


def _render_template_string(s: str, context: dict[str, Any]) -> str:
    """Replace ``{{placeholder}}`` tokens in a string."""
    def _replacer(match: re.Match) -> str:
        key = match.group(1).strip()
        return str(_deep_get(context, key.split("."), match.group(0)))
    return re.sub(r"\{\{(.+?)\}\}", _replacer, s)


def _render_template(data: Any, context: dict[str, Any]) -> Any:
    """Recursively render ``{{placeholders}}`` in dicts, lists, and strings."""
    if isinstance(data, str):
        return _render_template_string(data, context)
    if isinstance(data, dict):
        return {k: _render_template(v, context) for k, v in data.items()}
    if isinstance(data, list):
        return [_render_template(item, context) for item in data]
    return data


# ---------------------------------------------------------------------------
# FleetConfigManager
# ---------------------------------------------------------------------------

# Keys that should be treated as secret when redacting
_SECRET_FIELD_NAMES = {"secrets", "secret", "password", "token", "api_key",
                        "apikey", "private_key", "access_key"}


class FleetConfigManager:
    """Unified configuration management for the Pelagic fleet.

    Bridges: fleet.yaml, agent.yaml, world/rooms.json, lighthouse configs,
    environment variables, and runtime overrides into one system.
    """

    ENV_PREFIX = "FLEET_"

    def __init__(
        self,
        config_dir: str | Path | None = None,
        *,
        cli_args: dict[str, Any] | None = None,
    ) -> None:
        self.config_dir = Path(config_dir) if config_dir else Path.cwd()
        self.cli_args: dict[str, Any] = cli_args or {}
        self.runtime_overrides: dict[str, Any] = {}
        self._fleet_config: dict[str, Any] = {}
        self._agent_configs: dict[str, dict[str, Any]] = {}
        self._templates = ConfigTemplates()
        self._schema = ConfigSchema()
        self._snapshots_dir = self.config_dir / ".fleet-snapshots"

    # -- Loading / Saving -----------------------------------------------------

    def load_fleet_config(self, path: str | Path | None = None) -> dict[str, Any]:
        """Load fleet.yaml and apply layered overrides."""
        filepath = Path(path) if path else self.config_dir / "fleet.yaml"
        if not filepath.exists():
            self._fleet_config = self._templates.development()
            return self._fleet_config

        try:
            import yaml
            with open(filepath) as fh:
                self._fleet_config = yaml.safe_load(fh) or {}
        except Exception as exc:
            raise FileNotFoundError(
                f"Failed to load fleet config from {filepath}: {exc}"
            ) from exc

        # Apply layer 5 — agent configs (merge into fleet.agents)
        self._load_agent_configs()

        # Apply layer 3 — runtime overrides
        if self.runtime_overrides:
            self._fleet_config = _deep_merge(self._fleet_config, self.runtime_overrides)

        # Apply layer 2 — environment variables
        self._apply_env_overrides()

        # Apply layer 1 — CLI arguments
        if self.cli_args:
            self._fleet_config = _deep_merge(self._fleet_config, self.cli_args)

        return self._fleet_config

    def save_fleet_config(self, config: dict[str, Any] | None = None,
                          path: str | Path | None = None) -> Path:
        """Save current fleet config to fleet.yaml."""
        data = config or self._fleet_config
        filepath = Path(path) if path else self.config_dir / "fleet.yaml"
        filepath.parent.mkdir(parents=True, exist_ok=True)
        import yaml
        with open(filepath, "w") as fh:
            yaml.dump(data, fh, default_flow_style=False, sort_keys=False)
        return filepath

    def load_agent_config(self, agent_name: str,
                          path: str | Path | None = None) -> dict[str, Any]:
        """Load a per-agent agent.yaml."""
        filepath = Path(path) if path else self.config_dir / agent_name / "agent.yaml"
        if not filepath.exists():
            return {}
        import yaml
        with open(filepath) as fh:
            cfg = yaml.safe_load(fh) or {}
        self._agent_configs[agent_name] = cfg
        return cfg

    def save_agent_config(self, agent_name: str, config: dict[str, Any],
                          path: str | Path | None = None) -> Path:
        """Save a per-agent agent.yaml."""
        filepath = Path(path) if path else self.config_dir / agent_name / "agent.yaml"
        filepath.parent.mkdir(parents=True, exist_ok=True)
        import yaml
        with open(filepath, "w") as fh:
            yaml.dump(config, fh, default_flow_style=False, sort_keys=False)
        self._agent_configs[agent_name] = config
        return filepath

    # -- Agent config generation ----------------------------------------------

    def generate_agent_config(self, agent_name: str) -> dict[str, Any]:
        """Generate config for a specific agent from the fleet config."""
        fleet = self._fleet_config or self.load_fleet_config()
        network = fleet.get("network", {})
        agents = fleet.get("agents", {})
        base_port = network.get("base_port", 9000)
        default_host = network.get("default_host", "127.0.0.1")

        agent_names = list(agents.keys()) if isinstance(agents, dict) else []
        port_step = network.get("port_step", 100)

        if agent_name in agents:
            agent_spec = agents[agent_name]
            port = agent_spec.get("port", base_port + agent_names.index(agent_name) * port_step)
            host = agent_spec.get("host", default_host)
            atype = agent_spec.get("type", "generic")
        else:
            port = base_port + len(agent_names) * port_step
            host = default_host
            atype = "generic"

        return {
            "name": agent_name,
            "type": atype,
            "host": host,
            "port": port,
            "keeper_host": fleet.get("keeper", {}).get("host", default_host),
            "keeper_port": fleet.get("keeper", {}).get("port", 8000),
            "fleet_name": fleet.get("fleet_name", "pelagic-fleet"),
            "logging": copy.deepcopy(fleet.get("logging", {})),
            "environment": fleet.get("environment", "development"),
        }

    # -- Get / Set -----------------------------------------------------------

    def get(self, key: str, default: Any = None) -> Any:
        """Get a config value by dot-separated key, respecting layer priority."""
        keys = key.split(".")
        # Check CLI args first
        val = _deep_get(self.cli_args, keys)
        if val is not None:
            return val
        # Check env
        env_val = os.environ.get(self._key_to_env(key))
        if env_val is not None:
            return self._cast_env(env_val)
        # Check runtime overrides
        val = _deep_get(self.runtime_overrides, keys)
        if val is not None:
            return val
        # Check fleet config
        return _deep_get(self._fleet_config, keys, default)

    def set(self, key: str, value: Any) -> None:
        """Set a config value via runtime overrides."""
        _deep_set(self.runtime_overrides, key.split("."), value)
        if self._fleet_config:
            _deep_set(self._fleet_config, key.split("."), value)

    # -- Validation ----------------------------------------------------------

    def validate(self) -> ValidationResult:
        """Validate the current fleet config."""
        return self._schema.validate_fleet(self._fleet_config)

    # -- Diff / Merge --------------------------------------------------------

    @staticmethod
    def diff(config_a: dict, config_b: dict,
             redact: bool = True) -> dict[str, Any]:
        """Show structural differences between two configs."""
        a = _redact_secrets(config_a) if redact else config_a
        b = _redact_secrets(config_b) if redact else config_b
        return FleetConfigManager._dict_diff(a, b, path="")

    def diff_from_defaults(self) -> dict[str, Any]:
        """Show differences between current config and template defaults."""
        defaults = self._templates.development()
        return self.diff(defaults, self._fleet_config)

    @staticmethod
    def merge(base: dict, override: dict) -> dict:
        """Deep-merge two configs, returning a new dict."""
        return _deep_merge(base, override)

    # -- Template rendering ---------------------------------------------------

    def render_templates(self, config: dict[str, Any] | None = None) -> dict:
        """Replace ``{{placeholder}}`` tokens with config values."""
        ctx = config or self._fleet_config
        return _render_template(copy.deepcopy(ctx), ctx)

    # -- Snapshots -----------------------------------------------------------

    def snapshot(self, label: str | None = None) -> Path:
        """Save current config state for rollback."""
        self._snapshots_dir.mkdir(parents=True, exist_ok=True)
        ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
        name = f"{ts}-{label}" if label else ts
        snap_path = self._snapshots_dir / f"{name}.yaml"
        import yaml
        with open(snap_path, "w") as fh:
            yaml.dump(self._fleet_config, fh, default_flow_style=False, sort_keys=False)
        return snap_path

    def list_snapshots(self) -> list[dict[str, str]]:
        """List available config snapshots."""
        if not self._snapshots_dir.exists():
            return []
        snapshots = []
        for p in sorted(self._snapshots_dir.glob("*.yaml")):
            snapshots.append({
                "name": p.stem,
                "path": str(p),
                "modified": datetime.fromtimestamp(
                    p.stat().st_mtime, tz=timezone.utc
                ).isoformat(),
            })
        return snapshots

    def rollback(self, snapshot_name: str) -> dict[str, Any]:
        """Restore config from a snapshot."""
        snap_path = self._snapshots_dir / f"{snapshot_name}.yaml"
        if not snap_path.exists():
            # Try partial match
            matches = list(self._snapshots_dir.glob(f"*{snapshot_name}*.yaml"))
            if not matches:
                raise FileNotFoundError(f"Snapshot '{snapshot_name}' not found")
            snap_path = matches[-1]
        import yaml
        with open(snap_path) as fh:
            self._fleet_config = yaml.safe_load(fh) or {}
        # Clear runtime overrides so they don't shadow rolled-back config
        self.runtime_overrides.clear()
        return self._fleet_config

    # -- Export --------------------------------------------------------------

    def export(self, fmt: str = "yaml") -> str:
        """Export current config in the specified format."""
        data = _redact_secrets(self._fleet_config)
        if fmt == "json":
            return json.dumps(data, indent=2)
        if fmt == "env":
            return self._to_env(data)
        # default: yaml
        import yaml
        return yaml.dump(data, default_flow_style=False, sort_keys=False)

    # -- Doctor --------------------------------------------------------------

    def doctor(self) -> list[dict[str, str]]:
        """Diagnose common config issues."""
        issues: list[dict[str, str]] = []
        # 1. Validate
        result = self.validate()
        for err in result.errors:
            issues.append({"level": err.severity, "message": str(err)})
        # 2. Check fleet.yaml exists
        fleet_path = self.config_dir / "fleet.yaml"
        if not fleet_path.exists():
            issues.append({
                "level": "warning",
                "message": f"fleet.yaml not found at {fleet_path}",
            })
        # 3. Port conflicts
        self._schema._check_port_conflicts(self._fleet_config, result)
        # 4. Environment overrides
        for key, val in os.environ.items():
            if key.startswith(self.ENV_PREFIX):
                issues.append({
                    "level": "info",
                    "message": f"ENV override: {key}={val}",
                })
        return issues

    # -- Internal ------------------------------------------------------------

    def _load_agent_configs(self) -> None:
        """Load all per-agent agent.yaml files found in config_dir."""
        agents = self._fleet_config.get("agents", {})
        if not isinstance(agents, dict):
            return
        for agent_name in agents:
            agent_path = self.config_dir / agent_name / "agent.yaml"
            if agent_path.exists():
                try:
                    import yaml
                    with open(agent_path) as fh:
                        cfg = yaml.safe_load(fh) or {}
                    self._agent_configs[agent_name] = cfg
                    # Merge agent config into fleet.agents[name]
                    if isinstance(agents[agent_name], dict):
                        agents[agent_name] = _deep_merge(agents[agent_name], cfg)
                except Exception:
                    pass  # skip broken agent configs

    def _apply_env_overrides(self) -> None:
        """Apply ``FLEET_*`` environment variable overrides."""
        for key, val in os.environ.items():
            if not key.startswith(self.ENV_PREFIX):
                continue
            # FLEET_KEEPER_PORT=9000 -> keeper.port
            config_key = key[len(self.ENV_PREFIX):].lower()
            # Convert KEEPER_PORT -> keeper.port
            parts = config_key.split("_")
            # First part is section, rest are sub-keys
            if len(parts) >= 2:
                section = parts[0]
                sub_key = ".".join(parts[1:])
                full_key = f"{section}.{sub_key}"
            else:
                full_key = config_key
            _deep_set(self._fleet_config, full_key.split("."), self._cast_env(val))

    @staticmethod
    def _key_to_env(key: str) -> str:
        """Convert dot-key to env var name. ``keeper.port`` -> ``FLEET_KEEPER_PORT``."""
        return "FLEET_" + key.upper().replace(".", "_")

    @staticmethod
    def _cast_env(val: str) -> Any:
        """Attempt to cast env string to bool/int/float."""
        if val.lower() in ("true", "1", "yes"):
            return True
        if val.lower() in ("false", "0", "no"):
            return False
        try:
            return int(val)
        except ValueError:
            pass
        try:
            return float(val)
        except ValueError:
            pass
        return val

    @staticmethod
    def _dict_diff(a: dict, b: dict, path: str) -> dict:
        """Recursively diff two dicts, returning added/removed/changed keys."""
        result: dict[str, Any] = {}
        all_keys = set(list(a.keys()) + list(b.keys()))
        for key in sorted(all_keys):
            full = f"{path}.{key}" if path else key
            if key not in a:
                result[full] = {"status": "added", "value": b[key]}
            elif key not in b:
                result[full] = {"status": "removed", "was": a[key]}
            elif isinstance(a[key], dict) and isinstance(b[key], dict):
                nested = FleetConfigManager._dict_diff(a[key], b[key], full)
                if nested:
                    result.update(nested)
            elif a[key] != b[key]:
                result[full] = {"status": "changed", "from": a[key], "to": b[key]}
        return result

    @staticmethod
    def _to_env(data: dict, prefix: str = "FLEET_") -> str:
        """Flatten a nested dict into ENV-format key=value lines."""
        lines: list[str] = []
        for k, v in (data or {}).items():
            full = f"{prefix}{k.upper()}"
            if isinstance(v, dict):
                lines.append(FleetConfigManager._to_env(v, f"{full}_"))
            else:
                lines.append(f"{full}={v}")
        return "\n".join(lines)