fix: reconcile persisted state with runtime

Author: Unicode01

API.md

@@ -699,6 +699,15 @@ Goja 控制脚本默认只能访问本插件资源。每个插件默认持有一
   "page_size": 20,
   "total": 5,
   "binary_hash": "abc123",
+  "dataplane": {
+    "status": "applied",
+    "desired_generation": 12,
+    "applied_generation": 12,
+    "pending_generations": 0
+  },
+  "plugins": {
+    "status": "applied"
+  },
   "workers": [
     {
       "kind": "kernel",
@@ -735,6 +744,8 @@ Goja 控制脚本默认只能访问本插件资源。每个插件默认持有一
 - `draining`
 - `error`
 
+`dataplane` 与 `plugins` 是鉴权后的 reconcile 诊断视图。失败时会额外返回 `retry_count`、`last_error`、`last_attempt_at` 和 `last_applied_at`；不要把这些错误详情写入匿名日志或公开状态页。
+
 ### KernelRuntimeResponse
 
 `GET /api/kernel/runtime` 是运行时调试视图，常见关键字段：
@@ -819,16 +830,25 @@ Goja 控制脚本默认只能访问本插件资源。每个插件默认持有一
 用途：
 
 - 判断本机 Veer 运行时是否 ready，而不只是进程是否完成启动
-- 持续检查承载有效规则和范围的 userspace worker、shared proxy，以及所有已启用 Egress NAT 是否已经进入预期的 kernel runtime
+- 持续检查控制面 generation 是否已应用、承载有效规则和范围的 userspace worker、shared proxy，以及所有已启用 Egress NAT 是否已经进入预期的 kernel runtime
 - 不探测转发后端服务，也不执行端到端转发链路健康检查
-- 不需要 Bearer Token
+- 不需要 Bearer Token；只返回状态和 generation 计数，不返回内部错误文本。完整错误位于鉴权后的 `GET /api/workers`
 
 启动中或必要运行时组件未就绪时返回 `503`：
 
 ```json
 {
   "status": "starting",
-  "ready": false
+  "ready": false,
+  "dataplane": {
+    "status": "pending",
+    "desired_generation": 12,
+    "applied_generation": 11,
+    "pending_generations": 1
+  },
+  "plugins": {
+    "status": "applied"
+  }
 }
 ```
 
@@ -837,10 +857,21 @@ Goja 控制脚本默认只能访问本插件资源。每个插件默认持有一
 ```json
 {
   "status": "ready",
-  "ready": true
+  "ready": true,
+  "dataplane": {
+    "status": "applied",
+    "desired_generation": 12,
+    "applied_generation": 12,
+    "pending_generations": 0
+  },
+  "plugins": {
+    "status": "applied"
+  }
 }
 ```
 
+`plugins.status=error` 表示已启用插件的期望状态尚未完整应用，会触发后台退避重试，并使 `/readyz` 保持 `503`，直到运行态收敛。
+
 `GET /metrics`
 
 用途与约束：
@@ -1491,6 +1522,7 @@ Goja 控制脚本默认只能访问本插件资源。每个插件默认持有一
 - `page_size` 最大 `1000`
 - 不传 `page_size` 时返回全部 worker
 - 该接口会把规则 worker、范围 worker、共享站点 worker、kernel worker、egress_nat worker 合并返回
+- 顶层 `dataplane` 与 `plugins` 返回完整 reconcile 状态和错误详情，用于确认数据库期望状态是否已经应用到运行态
 
 ### 9.2 获取内核运行时
 

internal/app/api.go

@@ -171,16 +171,27 @@ func buildAPIHandler(cfg *Config, db *sql.DB, pm *ProcessManager) http.Handler {
 			return
 		}
 		w.Header().Set("Cache-Control", "no-store")
-		ready := pm.isReady()
+		ready, reconcile, plugins := pm.readinessReconcileSnapshot()
 		statusCode := http.StatusServiceUnavailable
 		status := "starting"
 		if ready {
 			statusCode = http.StatusOK
 			status = "ready"
+		} else if reconcile.Status == "error" || plugins.Status == "error" {
+			status = "error"
+		} else if reconcile.Status == "pending" {
+			status = "reconciling"
 		}
 		writeJSON(w, statusCode, map[string]interface{}{
 			"status": status,
 			"ready":  ready,
+			"dataplane": map[string]interface{}{
+				"status":              reconcile.Status,
+				"desired_generation":  reconcile.DesiredGeneration,
+				"applied_generation":  reconcile.AppliedGeneration,
+				"pending_generations": reconcile.PendingGenerations,
+			},
+			"plugins": map[string]string{"status": plugins.Status},
 		})
 	})
 	mux.HandleFunc("/metrics", authMiddleware(cfg, func(w http.ResponseWriter, r *http.Request) {
@@ -2548,8 +2559,12 @@ func handleListWorkers(w http.ResponseWriter, r *http.Request, db *sql.DB, pm *P
 	kernelRuleIDs := make(map[int64]bool)
 	kernelRangeIDs := make(map[int64]bool)
 	kernelBinaryHash := ""
+	var dataplaneStatus DataplaneReconcileStatus
+	var pluginStatus PluginReconcileStatus
 	needsSharedSiteCount := false
 	pm.mu.Lock()
+	dataplaneStatus = pm.dataplaneReconcileStatusLocked()
+	pluginStatus = pm.pluginReconcileStatusLocked()
 	for id := range pm.kernelRules {
 		kernelRuleIDs[id] = true
 	}
@@ -2973,6 +2988,8 @@ func handleListWorkers(w http.ResponseWriter, r *http.Request, db *sql.DB, pm *P
 		PageSize:   pageSize,
 		Total:      total,
 		BinaryHash: kernelBinaryHash,
+		Dataplane:  dataplaneStatus,
+		Plugins:    pluginStatus,
 		Workers:    out,
 	}
 	writeJSON(w, http.StatusOK, resp)

internal/app/api_health_test.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"bytes"
 	"encoding/json"
 	"net"
 	"net/http"
@@ -83,6 +84,45 @@ func TestBuildAPIHandlerReadyzReflectsProcessManagerState(t *testing.T) {
 	}
 }
 
+func TestBuildAPIHandlerReadyzRedactsReconcileErrors(t *testing.T) {
+	db := openTestDB(t)
+	pm := &ProcessManager{
+		ready:                    true,
+		desiredGeneration:        2,
+		appliedGeneration:        1,
+		pluginReconcileLastError: "secret plugin path",
+		ruleWorkers: map[int]*WorkerInfo{
+			0: {errored: true, lastError: "secret dataplane path"},
+		},
+	}
+	handler := buildAPIHandler(&Config{WebBind: "127.0.0.1", WebPort: 8080, WebToken: "test-token"}, db, pm)
+
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil))
+
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("GET /readyz status = %d, want %d", rec.Code, http.StatusServiceUnavailable)
+	}
+	if bytes.Contains(rec.Body.Bytes(), []byte("secret")) || bytes.Contains(rec.Body.Bytes(), []byte("last_error")) {
+		t.Fatalf("/readyz leaked reconcile detail: %s", rec.Body.String())
+	}
+	var resp struct {
+		Status    string `json:"status"`
+		Dataplane struct {
+			Status string `json:"status"`
+		} `json:"dataplane"`
+		Plugins struct {
+			Status string `json:"status"`
+		} `json:"plugins"`
+	}
+	if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
+		t.Fatalf("decode /readyz response: %v", err)
+	}
+	if resp.Status != "error" || resp.Dataplane.Status != "error" || resp.Plugins.Status != "error" {
+		t.Fatalf("/readyz response = %+v", resp)
+	}
+}
+
 func TestBuildAPIHandlerReadyzRejectsUnavailableEgressNAT(t *testing.T) {
 	db := openTestDB(t)
 	pm := &ProcessManager{

internal/app/api_plugin_data.go

@@ -154,7 +154,15 @@ func handlePluginStateAPI(w http.ResponseWriter, r *http.Request, cfg *Config, d
 			return
 		}
 		if pm != nil {
-			pm.reconcilePluginsForRuntime()
+			if _, err := pm.reconcilePluginsForRuntimeWithError(); err != nil {
+				recordPluginAudit(db, pluginID, "plugin.state", "api", "error", map[string]any{"enabled": *req.Enabled, "error": err.Error()})
+				writeJSON(w, http.StatusServiceUnavailable, map[string]any{
+					"error":  err.Error(),
+					"state":  pluginStateResponseForID(cfg, db, pm, pluginID),
+					"stored": true,
+				})
+				return
+			}
 			pm.redistributeWorkers()
 		}
 		recordPluginAudit(db, pluginID, "plugin.state", "api", "success", map[string]any{"enabled": *req.Enabled})

internal/app/api_workers_test.go

@@ -4,9 +4,44 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http/httptest"
+	"strings"
 	"testing"
 )
 
+func TestHandleListWorkersIncludesDetailedReconcileStatus(t *testing.T) {
+	db := openTestDB(t)
+	pm := &ProcessManager{
+		binaryHash:                "deadbeefcafebabe",
+		ruleWorkers:               map[int]*WorkerInfo{},
+		rangeWorkers:              map[int]*WorkerInfo{},
+		kernelRules:               map[int64]bool{},
+		kernelRanges:              map[int64]bool{},
+		desiredGeneration:         4,
+		appliedGeneration:         3,
+		reconcileLastError:        "dataplane apply failed",
+		reconcileRetryCount:       2,
+		pluginReconcileLastError:  "plugin apply failed",
+		pluginReconcileRetryCount: 3,
+	}
+
+	w := httptest.NewRecorder()
+	handleListWorkers(w, httptest.NewRequest("GET", "/api/workers", nil), db, pm)
+	if w.Code != 200 {
+		t.Fatalf("unexpected status: %d body=%s", w.Code, w.Body.String())
+	}
+	var resp WorkerListResponse
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+	if resp.Dataplane.Status != "error" || resp.Dataplane.DesiredGeneration != 4 || resp.Dataplane.AppliedGeneration != 3 ||
+		resp.Dataplane.RetryCount != 2 || !strings.Contains(resp.Dataplane.LastError, "dataplane apply failed") {
+		t.Fatalf("dataplane reconcile status = %+v", resp.Dataplane)
+	}
+	if resp.Plugins.Status != "error" || resp.Plugins.RetryCount != 3 || !strings.Contains(resp.Plugins.LastError, "plugin apply failed") {
+		t.Fatalf("plugin reconcile status = %+v", resp.Plugins)
+	}
+}
+
 func TestHandleListWorkersIncludesEgressNATWorker(t *testing.T) {
 	db := openTestDB(t)