From af037a5cc283e4c793fe3c7fb1235dee38dc612c Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Wed, 27 May 2026 20:48:36 +0300 Subject: [PATCH] feat(core): Azure support via az CLI + skills (multi-cloud) Azure is CLI-first: the agent investigates Azure through the bash `az` CLI (+ kubectl for AKS) and runbook skills, rather than structured SDK tools. - bash_tool: allowlist read-only `az` verbs; inject the org's Azure service-principal creds into an isolated, cached AZURE_CONFIG_DIR (+ kubectl for AKS), failing closed. Cross-provider calls are denied (an Azure org can't run `aws`, and vice-versa). - providers/azure/credentials.py: per-org `az login` into an isolated config dir + verify_service_principal (azure-identity). Reuses the existing cloud-account contextvar. - aws/credentials.resolve_session() now fails closed for non-AWS accounts (was leaking the platform base creds to non-AWS orgs). - prompts: provider-neutral system prompt documenting the `az` surface + the AKS path. - 4 Azure skills (aks-debugging, app-service-errors, monitor-kql, vm-diagnostics). - azure-identity dependency. Tests: az allow/block, az injection, cross-provider deny. --- .../tests/test_tools/test_credentials.py | 95 +++++++++++++- apps/backend/uv.lock | 57 ++++++++ apps/core/pyproject.toml | 1 + .../core/src/opendevops_core/agent/prompts.py | 22 +++- .../providers/aws/credentials.py | 8 +- .../providers/azure/credentials.py | 123 ++++++++++++++++++ .../skills/azure-aks-debugging/SKILL.md | 57 ++++++++ .../skills/azure-app-service-errors/SKILL.md | 53 ++++++++ .../skills/azure-monitor-kql/SKILL.md | 59 +++++++++ .../skills/azure-vm-diagnostics/SKILL.md | 46 +++++++ .../src/opendevops_core/tools/bash_tool.py | 107 +++++++++++---- 11 files changed, 597 insertions(+), 31 deletions(-) create mode 100644 apps/core/src/opendevops_core/providers/azure/credentials.py create mode 100644 apps/core/src/opendevops_core/skills/azure-aks-debugging/SKILL.md create mode 100644 apps/core/src/opendevops_core/skills/azure-app-service-errors/SKILL.md create mode 100644 apps/core/src/opendevops_core/skills/azure-monitor-kql/SKILL.md create mode 100644 apps/core/src/opendevops_core/skills/azure-vm-diagnostics/SKILL.md diff --git a/apps/backend/tests/test_tools/test_credentials.py b/apps/backend/tests/test_tools/test_credentials.py index ff656cc..19ab517 100644 --- a/apps/backend/tests/test_tools/test_credentials.py +++ b/apps/backend/tests/test_tools/test_credentials.py @@ -1,10 +1,22 @@ -"""Per-org AWS credential resolution + cache isolation.""" +"""Per-org credential resolution, cache isolation, and multi-cloud fail-closed behavior.""" +import pytest from moto import mock_aws from opendevops_core.providers.aws import credentials as cred from opendevops_core.tools._cache import tool_cached +def _azure_account(client_id: str = "client-1") -> dict: + return { + "id": "az1", + "provider": "azure", + "auth_method": "service_principal", + "region": "eastus", + "config": {"tenant_id": "t", "client_id": client_id, "subscription_id": "sub-1"}, + "secret_enc": None, + } + + def _account(role_arn: str, acct_id: str = "acct") -> dict: return { "id": acct_id, @@ -181,3 +193,84 @@ def _boom(): assert res["success"] is False assert res.get("blocked") is True assert "credentials" in res["error"].lower() + + +# ── Multi-cloud (Azure) ────────────────────────────────────────────────────── + + +def test_resolve_session_fails_closed_for_non_aws_account(): + # An Azure org must NOT fall back to the platform's base AWS creds. + token = cred.set_current_cloud_account(_azure_account()) + try: + with pytest.raises(RuntimeError): + cred.resolve_session() + finally: + cred.reset_current_cloud_account(token) + + +def test_bash_az_allowed_and_blocked(): + from opendevops_core.tools import bash_tool + + assert bash_tool._allowed("az aks list", ["az", "aks", "list"]) + assert bash_tool._allowed( + "az aks get-credentials -g r -n n", ["az", "aks", "get-credentials", "-g", "r", "-n", "n"] + ) + assert bash_tool._allowed("az monitor metrics list", ["az", "monitor", "metrics", "list"]) + assert not bash_tool._allowed("az group create", ["az", "group", "create"]) + assert not bash_tool._allowed("az vm delete --name x", ["az", "vm", "delete", "--name", "x"]) + + +def test_bash_injects_azure_env_for_az(monkeypatch): + from opendevops_core.tools import bash_tool + + captured: dict = {} + monkeypatch.setattr( + bash_tool, + "azure_cli_env", + lambda account: {"AZURE_CONFIG_DIR": "/tmp/x", "AZURE_SUBSCRIPTION_ID": "sub-1"}, + ) + monkeypatch.setattr( + bash_tool.subprocess, + "run", + lambda tokens, **kw: (captured.__setitem__("env", kw.get("env")), _CompletedProc())[1], + ) + token = cred.set_current_cloud_account(_azure_account()) + try: + res = bash_tool.run_bash_command("az aks list") + finally: + cred.reset_current_cloud_account(token) + + assert res["success"] + assert captured["env"]["AZURE_SUBSCRIPTION_ID"] == "sub-1" + + +def test_bash_blocks_aws_for_azure_org(monkeypatch): + from opendevops_core.tools import bash_tool + + def _no_run(*a, **k): + raise AssertionError("subprocess must not run for a cross-provider command") + + monkeypatch.setattr(bash_tool.subprocess, "run", _no_run) + token = cred.set_current_cloud_account(_azure_account()) + try: + res = bash_tool.run_bash_command("aws s3api list-buckets") + finally: + cred.reset_current_cloud_account(token) + assert res["success"] is False and res.get("blocked") is True + assert "aws is not available" in res["error"].lower() + + +def test_bash_blocks_az_for_aws_org(monkeypatch): + from opendevops_core.tools import bash_tool + + def _no_run(*a, **k): + raise AssertionError("subprocess must not run for a cross-provider command") + + monkeypatch.setattr(bash_tool.subprocess, "run", _no_run) + token = cred.set_current_cloud_account(_account("arn:aws:iam::111111111111:role/x")) + try: + res = bash_tool.run_bash_command("az aks list") + finally: + cred.reset_current_cloud_account(token) + assert res["success"] is False and res.get("blocked") is True + assert "azure is not available" in res["error"].lower() diff --git a/apps/backend/uv.lock b/apps/backend/uv.lock index e574b67..a4436cf 100644 --- a/apps/backend/uv.lock +++ b/apps/backend/uv.lock @@ -223,6 +223,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/48/c954218b2a250e23f178f10167c4173fecb5a75d2c206f0a67ba58006c26/authlib-1.7.0-py2.py3-none-any.whl", hash = "sha256:e36817afb02f6f0b6bf55f150782499ddd6ddf44b402bb055d3263cc65ac9ae0", size = 258779, upload-time = "2026-04-18T11:00:26.64Z" }, ] +[[package]] +name = "azure-core" +version = "1.41.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, +] + +[[package]] +name = "azure-identity" +version = "1.25.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "msal" }, + { name = "msal-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/0e/3a63efb48aa4a5ae2cfca61ee152fbcb668092134d3eb8bfda472dd5c617/azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6", size = 286304, upload-time = "2026-03-13T01:12:20.892Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138, upload-time = "2026-03-13T01:12:22.951Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -1907,6 +1936,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/4f/8812a01e3e0bd6be3e13b90432fb5c696af9a720af3f00e6eba5ad748345/moto-5.1.22-py3-none-any.whl", hash = "sha256:d9f20ae3cf29c44f93c1f8f06c8f48d5560e5dc027816ef1d0d2059741ffcfbe", size = 6617400, upload-time = "2026-03-08T21:06:41.093Z" }, ] +[[package]] +name = "msal" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/cb/b02b0f748ac668922364ccb3c3bff5b71628a05f5adfec2ba2a5c3031483/msal-1.36.0.tar.gz", hash = "sha256:3f6a4af2b036b476a4215111c4297b4e6e236ed186cd804faefba23e4990978b", size = 174217, upload-time = "2026-04-09T10:20:33.525Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/d3/414d1f0a5f6f4fe5313c2b002c54e78a3332970feb3f5fed14237aa17064/msal-1.36.0-py3-none-any.whl", hash = "sha256:36ecac30e2ff4322d956029aabce3c82301c29f0acb1ad89b94edcabb0e58ec4", size = 121547, upload-time = "2026-04-09T10:20:32.336Z" }, +] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -2194,6 +2249,7 @@ version = "0.1.0" source = { editable = "../core" } dependencies = [ { name = "aiosqlite" }, + { name = "azure-identity" }, { name = "boto3" }, { name = "cachetools" }, { name = "deepagents" }, @@ -2215,6 +2271,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "aiosqlite", specifier = ">=0.20.0" }, + { name = "azure-identity", specifier = ">=1.17.0" }, { name = "boto3", specifier = ">=1.34.0" }, { name = "cachetools", specifier = ">=5.3.0" }, { name = "deepagents" }, diff --git a/apps/core/pyproject.toml b/apps/core/pyproject.toml index 415c699..fd62689 100644 --- a/apps/core/pyproject.toml +++ b/apps/core/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "langchain-openai>=0.1.0", "langgraph>=0.2.0", "boto3>=1.34.0", + "azure-identity>=1.17.0", "pydantic>=2.7.0", "pydantic-settings>=2.3.0", "python-dotenv>=1.0.0", diff --git a/apps/core/src/opendevops_core/agent/prompts.py b/apps/core/src/opendevops_core/agent/prompts.py index 64284f6..444381e 100644 --- a/apps/core/src/opendevops_core/agent/prompts.py +++ b/apps/core/src/opendevops_core/agent/prompts.py @@ -1,13 +1,22 @@ from __future__ import annotations -_BASE_PROMPT = """You are an expert AWS SRE investigating an incident. You have read-only access to AWS services via tools. +_BASE_PROMPT = """You are an expert cloud SRE investigating an incident. You have read-only access to the affected cloud (AWS or Azure) via structured tools and read-only CLIs. The active cloud for this run is given in the run context; if none is stated, assume AWS. ## Investigation Methodology +**On AWS** (structured tools available): 1. Start by checking CloudWatch alarms (`get_alarms` with state=ALARM) for the affected service. 2. Check CloudTrail for recent changes (deployments, config changes) in the last 2 hours. 3. Before calling `get_log_events`, always call `describe_log_groups` first with a relevant prefix to discover the real log group name — never guess it. 4. Pull relevant metrics and correlate spikes with log errors. + +**On Azure** (use the bash tool with the Azure CLI `az` — there are no structured Azure tools): +1. Check Azure Monitor metrics (`az monitor metrics list`) and the Activity Log for recent changes (`az monitor activity-log list`). +2. Inspect the affected resource (`az webapp show`, `az aks show`, `az vm list`, etc.) and its logs (`az webapp log tail`, Log Analytics queries). +3. For AKS/Kubernetes: run `az aks get-credentials -g -n `, then triage with `kubectl get/describe/logs`. +4. Call `use_skill` for the matching Azure runbook (e.g. `azure-aks-debugging`) early. + +In both clouds: 5. Form explicit hypotheses before calling tools to verify them. 6. Rank hypotheses by likelihood and confirm or rule out each one with evidence. @@ -43,11 +52,16 @@ `query*`, `batch-get-*` Examples: `aws s3api list-buckets`, `aws dynamodb describe-table --table-name X`, `aws sns list-topics`, `aws secretsmanager list-secrets`, `aws ssm describe-parameters` -- `kubectl get / describe / logs` — always use bash for kubectl, no boto3 equivalent -- `docker ps / logs / inspect` — always use bash for docker, no boto3 equivalent +- Any `az ` where the verb is read-only: `list`, `show`, `get-*`, + `check`, `describe`, `tail`, `query`, `version` + Examples: `az aks list`, `az aks show -g rg -n cluster`, `az monitor metrics list`, + `az monitor activity-log list`, `az webapp log tail`, `az aks get-credentials -g rg -n cluster` +- `kubectl get / describe / logs` — use bash for kubectl (works against AKS once you've run + `az aks get-credentials`) +- `docker ps / logs / inspect` **Rules:** -- For `docker` and `kubectl`: always go through this tool directly. +- For `docker`, `kubectl`, and `az`: always go through this tool directly (no structured equivalent). - For AWS: prefer the structured boto3 tools for CloudWatch, ECS, Lambda, EC2, RDS, CloudTrail, IAM since they return cleaner structured data. Use this tool for any other AWS service or when you need raw CLI output. diff --git a/apps/core/src/opendevops_core/providers/aws/credentials.py b/apps/core/src/opendevops_core/providers/aws/credentials.py index 82c05e8..150a0ff 100644 --- a/apps/core/src/opendevops_core/providers/aws/credentials.py +++ b/apps/core/src/opendevops_core/providers/aws/credentials.py @@ -163,8 +163,12 @@ def resolve_session() -> boto3.Session: provider = account.get("provider", "aws") if provider != "aws": - logger.warning("resolve_session for non-aws provider '{}'; using base creds", provider) - return _base_session() + # Fail closed: an org whose active account is non-AWS must NOT fall back to the + # platform's base AWS credentials — that would leak the platform account across + # tenants. AWS tools should error out instead. (Tools catch this and return an error.) + raise RuntimeError( + f"AWS is not available for this organization (active cloud provider: {provider})." + ) identity = current_credential_identity() cached = _session_cache.get(identity) diff --git a/apps/core/src/opendevops_core/providers/azure/credentials.py b/apps/core/src/opendevops_core/providers/azure/credentials.py new file mode 100644 index 0000000..62b6bbb --- /dev/null +++ b/apps/core/src/opendevops_core/providers/azure/credentials.py @@ -0,0 +1,123 @@ +"""Per-request Azure credential resolution for the bash `az` CLI path. + +Azure support is CLI-first: the agent investigates Azure through `az` (and `kubectl` for AKS) +via the bash tool, not structured SDK tools. Unlike AWS (which reads creds from env vars), +the Azure CLI is **stateful** — it needs `az login`, which writes tokens to AZURE_CONFIG_DIR. +So per-org `az` runs in an **isolated, cached config dir** with a one-time service-principal +login, never a shared ~/.azure. + +The active account comes from the same contextvar the AWS resolver uses +(`get_current_cloud_account`); the account dict's `provider` field disambiguates. Azure accounts +look like:: + + {"provider": "azure", "auth_method": "service_principal", + "config": {"tenant_id": "...", "client_id": "...", "subscription_id": "..."}, + "secret_enc": , "id": "..."} +""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +import tempfile +from pathlib import Path + +from cachetools import TTLCache + +from opendevops_core.providers.aws.credentials import _account_secrets, get_current_cloud_account + +# Identities that already have a logged-in AZURE_CONFIG_DIR (avoid re-login per tool call). +_login_cache: TTLCache = TTLCache(maxsize=64, ttl=1800) # 30 min +_BASE_DIR = Path(tempfile.gettempdir()) / "odo-azure" + + +def current_azure_account() -> dict | None: + acct = get_current_cloud_account() + if acct and acct.get("provider") == "azure": + return acct + return None + + +def azure_identity(account: dict) -> str: + """Stable identity for caching the login/config dir (and not bleeding across orgs).""" + cfg = account.get("config") or {} + return cfg.get("client_id") or str(account.get("id") or "azure") + + +def _sp_fields(account: dict) -> dict: + cfg = account.get("config") or {} + secrets = _account_secrets(account) + return { + "tenant_id": cfg.get("tenant_id"), + "client_id": cfg.get("client_id"), + "subscription_id": cfg.get("subscription_id"), + # secret lives encrypted in secret_enc; tolerate a plaintext config fallback (no key set). + "client_secret": secrets.get("client_secret") or cfg.get("client_secret"), + } + + +def _config_dir_for(identity: str) -> Path: + h = hashlib.sha256(identity.encode()).hexdigest()[:16] + d = _BASE_DIR / h + d.mkdir(parents=True, exist_ok=True) + return d + + +def _az_login(env: dict, sp: dict) -> None: + """Log the service principal into the isolated AZURE_CONFIG_DIR (our subprocess, not the + agent's — so it bypasses the bash-tool allowlist intentionally).""" + login = subprocess.run( + [ + "az", "login", "--service-principal", + "-u", sp["client_id"], + "-p", sp["client_secret"], + "--tenant", sp["tenant_id"], + "--only-show-errors", + ], + capture_output=True, text=True, timeout=60, env=env, + ) + if login.returncode != 0: + raise RuntimeError(f"az login failed: {login.stderr[:300]}") + if sp.get("subscription_id"): + subprocess.run( + ["az", "account", "set", "--subscription", sp["subscription_id"], "--only-show-errors"], + capture_output=True, text=True, timeout=30, env=env, + ) + + +def azure_cli_env(account: dict) -> dict: + """Env for running `az`/`kubectl` as the org's service principal in an isolated, logged-in + config dir. Raises on missing creds or login failure (caller fails closed).""" + sp = _sp_fields(account) + if not (sp["tenant_id"] and sp["client_id"] and sp["client_secret"]): + raise RuntimeError("Azure account is missing service principal credentials") + + identity = azure_identity(account) + cfg_dir = _config_dir_for(identity) + env = dict(os.environ) + env["AZURE_CONFIG_DIR"] = str(cfg_dir) + env["HOME"] = str(cfg_dir) # az + kubectl write under HOME by default; keep it per-org + env["KUBECONFIG"] = str(cfg_dir / "kubeconfig") # az aks get-credentials writes here + if sp["subscription_id"]: + env["AZURE_SUBSCRIPTION_ID"] = sp["subscription_id"] + + if _login_cache.get(identity) is None: + _az_login(env, sp) + _login_cache[identity] = True + return env + + +def verify_service_principal(account: dict) -> dict: + """Validate the SP by acquiring an ARM token (parallel to AWS sts:GetCallerIdentity).""" + sp = _sp_fields(account) + if not (sp["tenant_id"] and sp["client_id"] and sp["client_secret"]): + return {"ok": False, "error": "Missing tenant_id / client_id / client_secret"} + try: + from azure.identity import ClientSecretCredential + + cred = ClientSecretCredential(sp["tenant_id"], sp["client_id"], sp["client_secret"]) + token = cred.get_token("https://management.azure.com/.default") + return {"ok": bool(token and token.token), "subscription_id": sp.get("subscription_id")} + except Exception as e: # noqa: BLE001 - surfaced to the admin as a verify failure + return {"ok": False, "error": str(e)} diff --git a/apps/core/src/opendevops_core/skills/azure-aks-debugging/SKILL.md b/apps/core/src/opendevops_core/skills/azure-aks-debugging/SKILL.md new file mode 100644 index 0000000..c82cc14 --- /dev/null +++ b/apps/core/src/opendevops_core/skills/azure-aks-debugging/SKILL.md @@ -0,0 +1,57 @@ +--- +name: azure-aks-debugging +description: Debug Azure Kubernetes Service (AKS) incidents — pods crashing, not ready, image/pull errors, node pressure — via az + kubectl +--- + +## AKS Debugging Investigation + +All commands run through `run_bash_command`. Azure CLI (`az`) and `kubectl` are read-only here. + +### Step 1 — Find the cluster and get credentials + +- `az aks list -o json` — list clusters; note `name`, `resourceGroup`, `powerState`, `provisioningState`. +- `az aks show -g -n -o json` — check `powerState.code` (Running?), `provisioningState` + (Succeeded vs Failed), Kubernetes version, and `agentPoolProfiles` (node count, `provisioningState`). +- `az aks get-credentials -g -n ` — fetch the kubeconfig so `kubectl` works. + (This writes to the org's isolated kubeconfig automatically.) + +### Step 2 — Cluster-wide triage with kubectl + +- `kubectl get nodes` — any `NotReady`? Node problems cascade into pod failures. +- `kubectl get pods -A` — scan for `CrashLoopBackOff`, `ImagePullBackOff`, `Pending`, `Error`, + `OOMKilled`, high `RESTARTS`. +- `kubectl get events -A --sort-by=.lastTimestamp` — recent cluster events (scheduling failures, + evictions, failed mounts, image pull errors). + +### Step 3 — Drill into the failing workload + +For a specific failing pod/namespace: +- `kubectl describe pod -n ` — read `State`/`Last State` (look for `OOMKilled`, exit codes), + `Events` (FailedScheduling, Back-off, Failed to pull image, readiness/liveness probe failures), + and resource `Requests`/`Limits`. +- `kubectl logs -n --previous` — logs from the crashed container (use `--previous` for the + prior crash); without `--previous` for the current one. +- `kubectl describe node ` — if scheduling/pressure suspected: `Conditions` + (MemoryPressure, DiskPressure, PIDPressure), `Allocatable` vs allocated. + +### Common Root Causes + +| Symptom | Likely cause | Evidence | +|---|---|---| +| `CrashLoopBackOff` | App crashes on start / bad config / missing secret | `kubectl logs --previous`, describe Events | +| `OOMKilled` | Memory limit too low or leak | pod `Last State: OOMKilled`, node MemoryPressure | +| `ImagePullBackOff` | Bad image tag or registry auth (ACR) | describe Events "Failed to pull"; check ACR attach | +| `Pending` (unschedulable) | Insufficient CPU/mem, no matching node, taints | Events `FailedScheduling`; node Allocatable | +| Node `NotReady` | Kubelet/network issue, node pressure | `kubectl describe node`, `az aks show` agentPool state | +| Recent regression | New deploy/scale/upgrade | `az monitor activity-log list` (see azure-monitor-kql) + `kubectl rollout history` | + +### Step 4 — Correlate with recent changes + +- `az monitor activity-log list --resource-group --offset 3h -o json` — recent control-plane + changes (scale, upgrade, config) on the cluster/node pools. +- `kubectl rollout history deployment/ -n ` — recent workload rollouts. + +### Notes +- ACR image pull failures: confirm the cluster's managed identity has `AcrPull` on the registry. +- If `az aks get-credentials` fails with auth errors, the connected service principal may lack the + `Azure Kubernetes Service Cluster User Role` — surface that as the finding. diff --git a/apps/core/src/opendevops_core/skills/azure-app-service-errors/SKILL.md b/apps/core/src/opendevops_core/skills/azure-app-service-errors/SKILL.md new file mode 100644 index 0000000..442d696 --- /dev/null +++ b/apps/core/src/opendevops_core/skills/azure-app-service-errors/SKILL.md @@ -0,0 +1,53 @@ +--- +name: azure-app-service-errors +description: Investigate Azure App Service / Functions errors — 5xx spikes, restarts, slow responses, failed deploys — via az + Azure Monitor +--- + +## Azure App Service / Functions Investigation + +All commands run through `run_bash_command` with the Azure CLI (`az`). + +### Step 1 — Identify the app and its state + +- `az webapp list -o json` (or `az functionapp list -o json`) — find the app; note `resourceGroup`, + `state` (Running/Stopped), `defaultHostName`. +- `az webapp show -g -n -o json` — check `state`, `availabilityState`, `httpsOnly`, + runtime stack, and `siteConfig` (alwaysOn, healthCheckPath). + +### Step 2 — Pull the error signal from Azure Monitor + +- `az monitor metrics list --resource --metric "Http5xx" "Http2xx" "Requests" "ResponseTime" --interval PT1M -o json` + — correlate the 5xx spike with request volume and response time. +- High `Http5xx` with normal `Requests` → app-side failure. High `Http5xx` *with* a `Requests` spike → + capacity/throttling. Rising `ResponseTime` before 5xx → downstream/dependency slowness. + +### Step 3 — Read the application logs + +- `az webapp log tail -g -n ` — live stream (good for an active incident). +- For deeper history, query Log Analytics if the app sends logs there (see `azure-monitor-kql`): + `AppServiceHTTPLogs` and `AppServiceConsoleLogs` tables. + +### Step 4 — Check for recent changes / restarts + +- `az monitor activity-log list -g --offset 3h -o json` — recent deploys, config changes, + restarts, scale operations on the app. +- Look for `Microsoft.Web/sites/restart`, slot swaps, app-setting changes, or a new deployment just + before the incident window. + +### Step 5 — Dependencies and config + +- `az webapp config appsettings list -g -n -o json` — missing/changed connection strings or + settings (note: values may be masked; flag suspicious recent changes). +- If the app calls a DB / Key Vault / storage: check those resources' health and that the app's managed + identity still has access (`az role assignment list --assignee `). + +### Common Root Causes + +| Symptom | Likely cause | Evidence | +|---|---|---| +| 5xx after a deploy | Bad release / config | activity-log deploy event; console logs stack trace | +| 5xx with request spike | Plan undersized / throttling | Requests spike + high CPU/memory metrics | +| Slow then 5xx | Downstream (DB/API) slow | rising ResponseTime; dependency errors in logs | +| App `Stopped` | Manual stop / crash / billing | `state: Stopped`; activity-log stop event | +| Cold starts (Functions) | Consumption plan scale-to-zero | high first-request latency; consider Premium plan | +| Missing setting/secret | Removed/rotated app setting or Key Vault ref | appsettings diff; "secret not found" in logs | diff --git a/apps/core/src/opendevops_core/skills/azure-monitor-kql/SKILL.md b/apps/core/src/opendevops_core/skills/azure-monitor-kql/SKILL.md new file mode 100644 index 0000000..c729a59 --- /dev/null +++ b/apps/core/src/opendevops_core/skills/azure-monitor-kql/SKILL.md @@ -0,0 +1,59 @@ +--- +name: azure-monitor-kql +description: Query Azure Monitor metrics, the Activity Log, and Log Analytics (KQL) to find anomalies and recent changes across Azure resources +--- + +## Azure Monitor & Log Analytics (KQL) + +The cross-cutting Azure observability toolkit. Use it from any Azure investigation. All via +`run_bash_command` with `az`. + +### Recent changes — the Activity Log (always check early) + +The Activity Log is Azure's audit trail (the CloudTrail equivalent). Most incidents follow a change. + +- `az monitor activity-log list --offset 3h -o json` — subscription-wide recent operations. +- Scope it: `--resource-group ` or `--resource-id `. +- Look for write/action operations just before the incident: deploys, scale, restart, delete, + config/role changes. Field `operationName`, `status`, `caller`, `eventTimestamp`. + +### Metrics — Azure Monitor + +- List available metrics for a resource: `az monitor metrics list-definitions --resource -o table`. +- Pull a metric: `az monitor metrics list --resource --metric "" --interval PT1M --offset 1h -o json`. +- Common metrics: VMs `Percentage CPU`, App Service `Http5xx`/`ResponseTime`, AKS + `node_cpu_usage_percentage`/`kube_pod_status_ready`, SQL DB `cpu_percent`/`dtu_consumption_percent`, + Storage `Availability`/`Transactions`. + +### Logs — Log Analytics (KQL) + +If resources send logs to a Log Analytics workspace: + +- Find the workspace: `az monitor log-analytics workspace list -o json` (note `customerId` = workspace GUID). +- Run KQL: `az monitor log-analytics query -w --analytics-query "" -o json`. + +Useful KQL patterns (last hour): +```kusto +// App Service 5xx by operation +AppServiceHTTPLogs | where TimeGenerated > ago(1h) | where ScStatus >= 500 +| summarize count() by CsUriStem, ScStatus | order by count_ desc + +// AKS container logs containing errors +ContainerLogV2 | where TimeGenerated > ago(1h) | where LogMessage contains "error" +| summarize count() by PodName | order by count_ desc + +// VM heartbeat gaps (node down) +Heartbeat | where TimeGenerated > ago(15m) | summarize LastSeen=max(TimeGenerated) by Computer +``` + +### Method + +1. Activity Log first — did something change? +2. Metrics — quantify and time-box the anomaly (when did it start, how big?). +3. Logs (KQL) — find the concrete error and the affected component. +4. Correlate the metric spike / log errors with the change from step 1 to form the root cause. + +### Notes +- KQL queries need the workspace GUID (`customerId`), not the resource name. +- If `az monitor log-analytics query` fails with auth, the service principal likely lacks + `Log Analytics Reader` — surface that. diff --git a/apps/core/src/opendevops_core/skills/azure-vm-diagnostics/SKILL.md b/apps/core/src/opendevops_core/skills/azure-vm-diagnostics/SKILL.md new file mode 100644 index 0000000..81dd8a0 --- /dev/null +++ b/apps/core/src/opendevops_core/skills/azure-vm-diagnostics/SKILL.md @@ -0,0 +1,46 @@ +--- +name: azure-vm-diagnostics +description: Diagnose Azure VM and Virtual Machine Scale Set issues — unavailability, high CPU/memory, boot/health failures — via az + Azure Monitor +--- + +## Azure VM / Scale Set Diagnostics + +All commands run through `run_bash_command` with `az`. + +### Step 1 — Locate the VM and its power/provisioning state + +- `az vm list -d -o json` — list VMs with `powerState` (`-d` includes runtime view); note + `resourceGroup`, `powerState` (VM running?), `provisioningState`. +- `az vm get-instance-view -g -n -o json` — detailed `statuses`: power state, OS state, + and any provisioning/extension errors. +- Scale sets: `az vmss list -o json`, then `az vmss list-instances -g -n -o json` and + `az vmss get-instance-view -g -n --instance-id -o json`. + +### Step 2 — Resource health metrics + +- `az monitor metrics list --resource --metric "Percentage CPU" "Available Memory Bytes" "Network In Total" "Network Out Total" "OS Disk IOPS Consumed Percentage" --interval PT1M --offset 1h -o json` +- High `Percentage CPU` sustained near 100%, low `Available Memory Bytes`, or disk IOPS at 100% → + resource exhaustion. Correlate the onset time with recent changes (Step 4). + +### Step 3 — Boot / health diagnostics + +- `az vm boot-diagnostics get-boot-log -g -n ` — serial/boot log (kernel panics, fsck, cloud-init + failures) when a VM won't come up. +- Check extensions: in `get-instance-view`, look for failed extensions (custom script, monitoring agent). + +### Step 4 — Recent changes + +- `az monitor activity-log list -g --offset 3h -o json` — resize, restart, deallocate, + redeploy, NSG/disk changes, or autoscale actions on the VM/scale set just before the incident. + +### Common Root Causes + +| Symptom | Likely cause | Evidence | +|---|---|---| +| VM unreachable, `powerState` not running | Deallocated/stopped (manual or autoscale), host issue | instance-view statuses; activity-log stop/deallocate | +| Sustained 100% CPU | Undersized SKU or runaway process | `Percentage CPU` metric; scale up / investigate workload | +| Low available memory / swapping | Memory pressure | `Available Memory Bytes`; resize or fix leak | +| Disk at 100% IOPS | Disk SKU too slow for load | `OS Disk IOPS Consumed Percentage`; upgrade to Premium SSD | +| Won't boot | Bad image/cloud-init, corrupt disk | boot-diagnostics serial log | +| Scale set instances unhealthy | Failing health probe / bad model update | vmss instance-view; recent model/upgrade in activity log | +| Network unreachable | NSG/route change | activity-log NSG change; check effective rules | diff --git a/apps/core/src/opendevops_core/tools/bash_tool.py b/apps/core/src/opendevops_core/tools/bash_tool.py index 6add631..fcb7038 100644 --- a/apps/core/src/opendevops_core/tools/bash_tool.py +++ b/apps/core/src/opendevops_core/tools/bash_tool.py @@ -19,6 +19,7 @@ resolve_region, resolve_session, ) +from opendevops_core.providers.azure.credentials import azure_cli_env # AWS CLI read-only operation verbs (the word before the first dash in the operation name). # Every read-only AWS CLI command starts with one of these — across ALL services. @@ -80,6 +81,16 @@ _DOCKER_FLAGS_WITH_VALUE: frozenset[str] = frozenset({"-H", "--host", "--context", "--config"}) +# Azure CLI read-only verbs (the verb is the LAST positional token of the command path, +# e.g. "az aks show" → "show", "az aks get-credentials" → "get", "az monitor metrics list" → "list"). +_AZ_READONLY_VERBS: frozenset[str] = frozenset( + {"list", "show", "get", "check", "describe", "tail", "query", "version"} +) +_AZ_GLOBAL_FLAGS_WITH_VALUE: frozenset[str] = frozenset({"--subscription", "--output", "-o", "--query"}) +_AZ_GLOBAL_FLAGS_NO_VALUE: frozenset[str] = frozenset( + {"--debug", "--verbose", "--only-show-errors", "--help", "-h"} +) + _TIMEOUT = 30 _MAX_OUTPUT = 4000 _MAX_STDERR = 1000 @@ -156,6 +167,30 @@ def _allowed(command: str, tokens: list[str]) -> bool: return True return verb in _AWS_READONLY_VERBS + if binary == "az": + # Consume leading global flags, then the command path is the run of positional tokens + # up to the first flag; the verb is the last token of that path (az convention). + i = 1 + while i < len(tokens) and tokens[i].startswith("-"): + flag = tokens[i].lower() + if flag in _AZ_GLOBAL_FLAGS_WITH_VALUE: + if i + 1 >= len(tokens): + return False + i += 2 + continue + if flag in _AZ_GLOBAL_FLAGS_NO_VALUE: + i += 1 + continue + return False + path: list[str] = [] + while i < len(tokens) and not tokens[i].startswith("-"): + path.append(tokens[i]) + i += 1 + if not path: + return False + verb = path[-1].lower().split("-")[0] # "get-credentials" → "get" + return verb in _AZ_READONLY_VERBS + if binary == "kubectl": subcommand = _find_subcommand(tokens, 1, _KUBECTL_FLAGS_WITH_VALUE) return subcommand in {"get", "describe", "logs"} @@ -175,7 +210,10 @@ def run_bash_command(command: str) -> dict[str, Any]: verb: describe-*, list-*, get-*, lookup-*, filter-*, search-*, scan-*, query*, batch-get-*. Covers ALL AWS services (S3, DynamoDB, SNS, SQS, Route53, ACM, Secrets Manager, SSM, IAM, etc.) not just the structured boto3 tools. - - kubectl get / describe / logs + - ANY `az ` where the verb is read-only: list, show, get-*, + check, describe, tail, query, version (e.g. `az aks list`, `az monitor metrics list`, + `az webapp log tail`, `az aks get-credentials`). Covers the Azure surface. + - kubectl get / describe / logs (works against AKS after `az aks get-credentials`) - docker ps / logs / inspect Use for docker and kubectl always (no boto3 equivalent). For AWS, use when @@ -206,31 +244,52 @@ def run_bash_command(command: str) -> dict[str, Any]: logger.info("bash_tool RUN | cmd={!r}", command) - # When an org cloud account is active, `aws` CLI calls must use that org's assumed-role - # credentials — not the platform's ambient env creds. Inject temporary creds into the - # subprocess env, and fail closed (never fall back to platform creds for a scoped org). + # When an org cloud account is active, cloud CLI calls must use THAT org's credentials — + # never the platform's ambient creds — and cross-provider calls must fail closed (an Azure + # org's agent must not be able to run `aws`, and vice-versa). + account = get_current_cloud_account() + provider = account.get("provider") if account else None + binary = tokens[0] run_env = None - if tokens and tokens[0] == "aws" and get_current_cloud_account() is not None: + + def _blocked(msg: str) -> dict[str, Any]: + return {"success": False, "output": "", "error": msg, "command": command, "blocked": True} + + if binary == "aws": + if account is not None and provider != "aws": + return _blocked(f"AWS is not available for this organization (cloud: {provider}).") + if account is not None: + try: + frozen = resolve_session().get_credentials().get_frozen_credentials() + run_env = {k: v for k, v in os.environ.items() if k != "AWS_PROFILE"} + run_env["AWS_ACCESS_KEY_ID"] = frozen.access_key + run_env["AWS_SECRET_ACCESS_KEY"] = frozen.secret_key + region = resolve_region() + run_env["AWS_REGION"] = run_env["AWS_DEFAULT_REGION"] = region + if frozen.token: + run_env["AWS_SESSION_TOKEN"] = frozen.token + else: + run_env.pop("AWS_SESSION_TOKEN", None) + except Exception as e: # noqa: BLE001 - fail closed; do not leak platform creds + logger.error("bash_tool: could not resolve org AWS creds: {}", e) + return _blocked(f"Could not resolve organization AWS credentials: {e}") + elif binary == "az": + if account is not None and provider != "azure": + return _blocked(f"Azure is not available for this organization (cloud: {provider}).") + if account is not None: + try: + run_env = azure_cli_env(account) + except Exception as e: # noqa: BLE001 - fail closed + logger.error("bash_tool: could not resolve org Azure creds: {}", e) + return _blocked(f"Could not resolve organization Azure credentials: {e}") + elif binary == "kubectl" and provider == "azure": + # AKS: kubectl must read the per-org kubeconfig written by `az aks get-credentials` + # into the org's isolated config dir. try: - frozen = resolve_session().get_credentials().get_frozen_credentials() - run_env = {k: v for k, v in os.environ.items() if k != "AWS_PROFILE"} - run_env["AWS_ACCESS_KEY_ID"] = frozen.access_key - run_env["AWS_SECRET_ACCESS_KEY"] = frozen.secret_key - region = resolve_region() - run_env["AWS_REGION"] = run_env["AWS_DEFAULT_REGION"] = region - if frozen.token: - run_env["AWS_SESSION_TOKEN"] = frozen.token - else: - run_env.pop("AWS_SESSION_TOKEN", None) - except Exception as e: # noqa: BLE001 - fail closed; do not leak platform creds - logger.error("bash_tool: could not resolve org AWS creds: {}", e) - return { - "success": False, - "output": "", - "error": f"Could not resolve organization AWS credentials: {e}", - "command": command, - "blocked": True, - } + run_env = azure_cli_env(account) + except Exception as e: # noqa: BLE001 - fail closed + logger.error("bash_tool: could not resolve org Azure creds for kubectl: {}", e) + return _blocked(f"Could not resolve organization Azure credentials: {e}") try: proc = subprocess.run(