feat: add llm cost controls

.env.example

@@ -6,6 +6,10 @@ OPENAI_API_KEY=
 OPENAI_BASE_URL=https://api.openai.com/v1
 MODEL_NAME=gpt-4o-mini
 
+# Cost controls for optional LLM enrichment. These bound evidence sent to the provider.
+LLM_MAX_LOG_ENTRIES=50
+LLM_MAX_PROMPT_CHARS=12000
+
 # Docker Compose mounts ./logs to /shared/logs in both app containers.
 DEMO_SERVICE_LOG_PATH=/shared/logs/demo-service.log
 

README.md

@@ -86,6 +86,8 @@ For security hardening basics, see `docs/07-security.md` and `SECURITY.md`.
 
 For secret handling and enforced assistant redaction rules, see `docs/15-secret-handling-and-redaction.md`.
 
+For practical cost optimization controls, see `docs/16-cost-optimization.md`.
+
 
 Ask the assistant directly:
 
@@ -126,6 +128,7 @@ ai-infra-starter-kit/
     13-kubernetes-incident-debugging.md
     14-kubernetes-production-next-steps.md
     15-secret-handling-and-redaction.md
+    16-cost-optimization.md
   infra/                 # Docker, Kubernetes, and Terraform starter notes
     k8s/                 # kind-first Kubernetes manifests and walkthrough
   scripts/               # Local traffic and log helper scripts

apps/ai-sre-assistant/README.md

@@ -60,11 +60,26 @@ LLM_PROVIDER=openai
 OPENAI_API_KEY=your_key
 OPENAI_BASE_URL=https://api.openai.com/v1
 MODEL_NAME=your_model
+LLM_MAX_LOG_ENTRIES=50
+LLM_MAX_PROMPT_CHARS=12000
 DEMO_SERVICE_METRICS_URL=http://localhost:8000/metrics
 ```
 
 For local Ollama experiments, set `LLM_PROVIDER=ollama` and use an OpenAI-compatible Ollama base URL.
 
+## Cost Controls
+
+The rule-based analyzer remains the lowest-cost path. Keep `LLM_PROVIDER=none` or set `use_llm=false` on requests when LLM enrichment is not needed.
+
+When an LLM provider is enabled, the assistant limits provider prompts with:
+
+- `LLM_MAX_LOG_ENTRIES`: how many recent log records can be included in the prompt.
+- `LLM_MAX_PROMPT_CHARS`: maximum assembled user prompt size before the provider request.
+
+API responses that attempt LLM enrichment include `llm_cost_controls` so callers can see which limits were active. If prompt evidence is truncated, the response includes an `llm_notice`.
+
+See `../../docs/16-cost-optimization.md` for the Day 3 cost optimization guide.
+
 ## Redaction
 
 The assistant replaces obvious sensitive values with `[REDACTED]` before analysis and before data is sent to an optional LLM provider.

apps/ai-sre-assistant/app/llm.py

@@ -9,12 +9,19 @@
 from app.redaction import redact_data, redact_text
 
 
+DEFAULT_LLM_MAX_LOG_ENTRIES = 50
+DEFAULT_LLM_MAX_PROMPT_CHARS = 12000
+TRUNCATION_MARKER = "\n...[truncated by LLM cost controls]\n"
+
+
 @dataclass(frozen=True)
 class LLMConfig:
     provider: str
     api_key: str
     base_url: str
     model: str
+    max_log_entries: int = DEFAULT_LLM_MAX_LOG_ENTRIES
+    max_prompt_chars: int = DEFAULT_LLM_MAX_PROMPT_CHARS
 
 
 def load_config() -> LLMConfig:
@@ -25,6 +32,8 @@ def load_config() -> LLMConfig:
         api_key=os.getenv("OPENAI_API_KEY", "").strip(),
         base_url=os.getenv("OPENAI_BASE_URL", default_base_url).strip().rstrip("/"),
         model=os.getenv("MODEL_NAME", "gpt-4o-mini").strip(),
+        max_log_entries=_read_positive_int("LLM_MAX_LOG_ENTRIES", DEFAULT_LLM_MAX_LOG_ENTRIES),
+        max_prompt_chars=_read_positive_int("LLM_MAX_PROMPT_CHARS", DEFAULT_LLM_MAX_PROMPT_CHARS),
     )
 
 
@@ -36,6 +45,13 @@ def is_configured(config: LLMConfig) -> bool:
     return bool(config.api_key and config.base_url and config.model)
 
 
+def cost_controls_summary(config: LLMConfig) -> dict[str, int]:
+    return {
+        "max_log_entries": config.max_log_entries,
+        "max_prompt_chars": config.max_prompt_chars,
+    }
+
+
 def analyze_with_llm(
     question: str,
     logs: list[dict[str, Any]],
@@ -46,22 +62,17 @@ def analyze_with_llm(
     if not is_configured(config):
         return None, "LLM provider is not configured; using rule-based analysis."
 
-    safe_question = redact_text(question)
-    safe_logs = redact_data(logs[-50:])
-    safe_analysis = redact_data(rule_based_analysis)
-    evidence = "\n".join(json.dumps(entry, default=str) for entry in safe_logs)
-    user_prompt = f"""
-Question:
-{safe_question}
-
-Rule-based pre-analysis:
-{json.dumps(safe_analysis, indent=2, default=str)}
-
-Recent log evidence:
-{evidence}
-
-{ANALYSIS_INSTRUCTIONS}
-"""
+    user_prompt, prompt_was_truncated = build_user_prompt(
+        question=question,
+        logs=logs,
+        rule_based_analysis=rule_based_analysis,
+        config=config,
+    )
+    cost_notice = (
+        f"LLM prompt was limited to {config.max_prompt_chars} characters by cost controls."
+        if prompt_was_truncated
+        else None
+    )
 
     headers = {"Content-Type": "application/json"}
     if config.api_key:
@@ -82,7 +93,64 @@ def analyze_with_llm(
             response.raise_for_status()
             data = response.json()
             content = data["choices"][0]["message"]["content"]
-            return redact_text(str(content).strip()), None
+            return redact_text(str(content).strip()), cost_notice
     except Exception as exc:
-        return None, f"LLM request failed; using rule-based analysis. Error: {redact_text(str(exc))}"
+        failure_notice = f"LLM request failed; using rule-based analysis. Error: {redact_text(str(exc))}"
+        if cost_notice:
+            failure_notice = f"{cost_notice} {failure_notice}"
+        return None, failure_notice
+
+
+def build_user_prompt(
+    question: str,
+    logs: list[dict[str, Any]],
+    rule_based_analysis: dict[str, Any],
+    config: LLMConfig,
+) -> tuple[str, bool]:
+    safe_question = redact_text(question)
+    safe_logs = redact_data(logs[-config.max_log_entries :])
+    safe_analysis = redact_data(rule_based_analysis)
+    evidence = "\n".join(json.dumps(entry, default=str) for entry in safe_logs)
+
+    prompt_header = f"""
+Question:
+{safe_question}
+
+Rule-based pre-analysis:
+{json.dumps(safe_analysis, indent=2, default=str)}
+
+Recent log evidence:
+"""
+    prompt_footer = f"""
+
+{ANALYSIS_INSTRUCTIONS}
+"""
+    evidence_budget = config.max_prompt_chars - len(prompt_header) - len(prompt_footer)
+    if evidence_budget < 0:
+        prompt, _ = _truncate_text(prompt_header + prompt_footer, config.max_prompt_chars)
+        return prompt, True
+
+    bounded_evidence, evidence_was_truncated = _truncate_text(evidence, evidence_budget)
+    return prompt_header + bounded_evidence + prompt_footer, evidence_was_truncated
+
+
+def _read_positive_int(name: str, default: int) -> int:
+    raw_value = os.getenv(name, "").strip()
+    if not raw_value:
+        return default
+    try:
+        value = int(raw_value)
+    except ValueError:
+        return default
+    return value if value > 0 else default
+
+
+def _truncate_text(value: str, max_chars: int) -> tuple[str, bool]:
+    if len(value) <= max_chars:
+        return value, False
+    if max_chars <= 0:
+        return "", True
+    if max_chars <= len(TRUNCATION_MARKER):
+        return value[:max_chars], True
+    return value[: max_chars - len(TRUNCATION_MARKER)].rstrip() + TRUNCATION_MARKER, True
 

apps/ai-sre-assistant/app/main.py

@@ -4,7 +4,7 @@
 from pydantic import BaseModel, Field
 
 from app.analyzer import analyze_logs
-from app.llm import analyze_with_llm
+from app.llm import analyze_with_llm, cost_controls_summary, load_config
 from app.log_reader import get_log_path, read_recent_logs
 from app.metrics_analyzer import analyze_metrics, combined_incident_analysis
 from app.metrics_reader import fetch_metrics_text, get_metrics_url, parse_prometheus_text
@@ -89,7 +89,14 @@ def summarize_incident(request: SummarizeIncidentRequest) -> dict[str, Any]:
     }
 
     if request.use_llm:
-        llm_analysis, llm_notice = analyze_with_llm(question=question, logs=[], rule_based_analysis=response)
+        llm_config = load_config()
+        response["llm_cost_controls"] = cost_controls_summary(llm_config)
+        llm_analysis, llm_notice = analyze_with_llm(
+            question=question,
+            logs=[],
+            rule_based_analysis=response,
+            config=llm_config,
+        )
         if llm_analysis:
             response["analysis_mode"] = "llm"
             response["llm_analysis"] = llm_analysis
@@ -114,7 +121,14 @@ def _analyze(question: str, max_lines: int, use_llm: bool) -> dict[str, Any]:
     }
 
     if use_llm:
-        llm_analysis, llm_notice = analyze_with_llm(question=question, logs=logs, rule_based_analysis=rule_based)
+        llm_config = load_config()
+        response["llm_cost_controls"] = cost_controls_summary(llm_config)
+        llm_analysis, llm_notice = analyze_with_llm(
+            question=question,
+            logs=logs,
+            rule_based_analysis=rule_based,
+            config=llm_config,
+        )
         if llm_analysis:
             response["analysis_mode"] = "llm"
             response["llm_analysis"] = llm_analysis

apps/ai-sre-assistant/tests/test_llm_cost_controls.py

@@ -0,0 +1,114 @@
+from fastapi.testclient import TestClient
+
+from app.llm import LLMConfig, analyze_with_llm, build_user_prompt, cost_controls_summary, load_config
+from app.main import app
+
+
+client = TestClient(app)
+
+
+def test_load_config_reads_llm_cost_controls(monkeypatch):
+    monkeypatch.setenv("LLM_MAX_LOG_ENTRIES", "7")
+    monkeypatch.setenv("LLM_MAX_PROMPT_CHARS", "2048")
+
+    config = load_config()
+
+    assert cost_controls_summary(config) == {
+        "max_log_entries": 7,
+        "max_prompt_chars": 2048,
+    }
+
+
+def test_llm_prompt_uses_recent_configured_log_window():
+    config = LLMConfig(
+        provider="openai",
+        api_key="provider-key",
+        base_url="https://example.invalid/v1",
+        model="test-model",
+        max_log_entries=2,
+        max_prompt_chars=12000,
+    )
+
+    prompt, truncated = build_user_prompt(
+        question="What happened?",
+        logs=[
+            {"message": "oldest-log"},
+            {"message": "older-log"},
+            {"message": "recent-log"},
+            {"message": "newest-log"},
+        ],
+        rule_based_analysis={"summary": "rule-based summary"},
+        config=config,
+    )
+
+    assert truncated is False
+    assert "recent-log" in prompt
+    assert "newest-log" in prompt
+    assert "oldest-log" not in prompt
+    assert "older-log" not in prompt
+
+
+def test_llm_prompt_respects_prompt_character_budget(monkeypatch):
+    captured = {}
+
+    class FakeResponse:
+        def raise_for_status(self):
+            return None
+
+        def json(self):
+            return {"choices": [{"message": {"content": "bounded analysis"}}]}
+
+    class FakeClient:
+        def __init__(self, timeout):
+            assert timeout == 30
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc, traceback):
+            return False
+
+        def post(self, url, headers, json):
+            captured["prompt"] = json["messages"][1]["content"]
+            return FakeResponse()
+
+    monkeypatch.setattr("app.llm.httpx.Client", FakeClient)
+    config = LLMConfig(
+        provider="openai",
+        api_key="provider-key",
+        base_url="https://example.invalid/v1",
+        model="test-model",
+        max_log_entries=5,
+        max_prompt_chars=350,
+    )
+
+    result, notice = analyze_with_llm(
+        question="Investigate the latest failure",
+        logs=[{"message": "x" * 2000}],
+        rule_based_analysis={"summary": "y" * 1000},
+        config=config,
+    )
+
+    assert result == "bounded analysis"
+    assert notice == "LLM prompt was limited to 350 characters by cost controls."
+    assert len(captured["prompt"]) <= 350
+
+
+def test_api_response_reports_llm_cost_controls(tmp_path, monkeypatch):
+    log_file = tmp_path / "demo-service.log"
+    log_file.write_text('{"level":"INFO","message":"ok","status_code":200}\n', encoding="utf-8")
+    monkeypatch.setenv("DEMO_SERVICE_LOG_PATH", str(log_file))
+    monkeypatch.setenv("LLM_PROVIDER", "none")
+    monkeypatch.setenv("LLM_MAX_LOG_ENTRIES", "3")
+    monkeypatch.setenv("LLM_MAX_PROMPT_CHARS", "4096")
+
+    response = client.post("/ask", json={"question": "What happened?", "use_llm": True})
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["analysis_mode"] == "rule-based"
+    assert body["llm_cost_controls"] == {
+        "max_log_entries": 3,
+        "max_prompt_chars": 4096,
+    }
+    assert "LLM provider is not configured" in body["llm_notice"]

docker-compose.yml

@@ -29,6 +29,8 @@ services:
       OPENAI_API_KEY: ${OPENAI_API_KEY:-}
       OPENAI_BASE_URL: ${OPENAI_BASE_URL:-https://api.openai.com/v1}
       MODEL_NAME: ${MODEL_NAME:-gpt-4o-mini}
+      LLM_MAX_LOG_ENTRIES: ${LLM_MAX_LOG_ENTRIES:-50}
+      LLM_MAX_PROMPT_CHARS: ${LLM_MAX_PROMPT_CHARS:-12000}
       DEMO_SERVICE_METRICS_URL: http://demo-service:8000/metrics
     volumes:
       - ./logs:/shared/logs

docs/07-security.md

@@ -109,6 +109,8 @@ When `LLM_PROVIDER=none`, the assistant uses deterministic rule-based analysis.
 
 When a provider is enabled, prompt inputs are redacted immediately before the request. The provider API key is used in the HTTP authorization header and is not included in the prompt.
 
+The assistant also applies cost controls before provider requests: only a bounded number of recent logs are included, and the assembled prompt is capped by `LLM_MAX_PROMPT_CHARS`. This reduces both spend and unnecessary data exposure.
+
 Before enabling an external LLM provider:
 
 - review what logs may be sent.

docs/11-kubernetes-config-and-secrets.md

@@ -34,14 +34,16 @@ Current values:
 | `LLM_PROVIDER` | Controls whether the assistant uses rule-based analysis, OpenAI-compatible APIs, or Ollama. |
 | `OPENAI_BASE_URL` | Stores the OpenAI-compatible API base URL. |
 | `MODEL_NAME` | Stores the model name used when an LLM provider is enabled. |
+| `LLM_MAX_LOG_ENTRIES` | Bounds how many recent log records can enter an optional LLM prompt. |
+| `LLM_MAX_PROMPT_CHARS` | Caps the assembled prompt size before an optional provider request. |
 
 Inspect the ConfigMap:
 
 ```bash
 kubectl describe configmap ai-infra-starter-kit-config -n ai-infra-starter-kit
 ```
 
-The default `LLM_PROVIDER` is `none` so the project works without an API key.
+The default `LLM_PROVIDER` is `none` so the project works without an API key. The default cost controls keep provider prompts bounded when a user enables LLM enrichment.
 
 ## What Belongs In A Secret